diff --git a/.github/banner_dark.png b/.github/banner_dark.png index 4b41216..d5ae64a 100644 Binary files a/.github/banner_dark.png and b/.github/banner_dark.png differ diff --git a/.github/banner_light.png b/.github/banner_light.png index aa04dc5..de494fb 100644 Binary files a/.github/banner_light.png and b/.github/banner_light.png differ diff --git a/.github/workflows/buildtest.yaml b/.github/workflows/buildtest.yaml index a012f3f..284ecf8 100644 --- a/.github/workflows/buildtest.yaml +++ b/.github/workflows/buildtest.yaml @@ -14,6 +14,9 @@ name: Test +permissions: + contents: read + on: workflow_dispatch: push: @@ -25,17 +28,17 @@ jobs: test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: shogo82148/actions-setup-redis@v1 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + - uses: shogo82148/actions-setup-redis@2f3253b148c73d7a0682eae73e862b777a4fa74e # v1 with: redis-version: "6.x" auto-start: true - run: redis-cli ping - name: Set up Go - uses: actions/setup-go@v5 + uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6 with: - go-version-file: "go.mod" + go-version: "^1.26" - name: Set up gotestfmt run: go install github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@v2.4.1 @@ -45,22 +48,21 @@ jobs: go get github.com/sasha-s/go-deadlock grep -rl sync.Mutex ./pkg | xargs sed -i 's/sync\.Mutex/deadlock\.Mutex/g' grep -rl sync.RWMutex ./pkg | xargs sed -i 's/sync\.RWMutex/deadlock\.RWMutex/g' - go install golang.org/x/tools/cmd/goimports + go install golang.org/x/tools/cmd/goimports@latest grep -rl deadlock.Mutex ./pkg | xargs goimports -w grep -rl deadlock.RWMutex ./pkg | xargs goimports -w go mod tidy - name: Mage Build - uses: magefile/mage-action@v3 + uses: magefile/mage-action@a662bd8c29d8106879588cfff83b2faf6e6f59db # v4 with: version: latest args: build - - name: Static Check - uses: amarpal/staticcheck-action@master + - name: Lint + uses: golangci/golangci-lint-action@1e7e51e771db61008b38414a730f564565cf7c20 # v9.2.0 with: - checks: '["all", "-ST1000", "-ST1003", "-ST1020", "-ST1021", "-ST1022", "-SA1019"]' - install-go: false + version: v2.11.4 - name: Test run: | @@ -69,7 +71,7 @@ jobs: # Upload the original go test log as an artifact for later review. - name: Upload test log - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 if: always() with: name: test-log diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index 8da0f17..1ee39fe 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -14,6 +14,9 @@ name: Release to Docker +permissions: + contents: read + # Controls when the action will run. on: workflow_dispatch: @@ -26,10 +29,10 @@ jobs: docker: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - name: Docker meta id: meta - uses: docker/metadata-action@v5 + uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6 with: # list of Docker images to use as base name for tags images: | @@ -41,7 +44,7 @@ jobs: type=semver,pattern=v{{major}}.{{minor}} - name: Set up Go - uses: actions/setup-go@v5 + uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6 with: go-version-file: "go.mod" @@ -49,24 +52,24 @@ jobs: run: go mod download - name: Generate code - uses: magefile/mage-action@v3 + uses: magefile/mage-action@a662bd8c29d8106879588cfff83b2faf6e6f59db # v4 with: version: latest args: generate - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4 - name: Login to DockerHub if: github.event_name != 'pull_request' - uses: docker/login-action@v3 + uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Build and push id: docker_build - uses: docker/build-push-action@v6 + uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7 with: context: . push: ${{ github.event_name != 'pull_request' }} diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index ce7e9a9..78b7ef7 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -28,17 +28,17 @@ jobs: release: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - name: Fetch all tags run: git fetch --force --tags - name: Set up Go - uses: actions/setup-go@v5 + uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6 with: go-version-file: "go.mod" - name: Run GoReleaser - uses: goreleaser/goreleaser-action@v6 + uses: goreleaser/goreleaser-action@ec59f474b9834571250b370d4735c50f8e2d1e29 # v7 with: distribution: goreleaser version: '~> v2' diff --git a/.github/workflows/slack-notifier.yaml b/.github/workflows/slack-notifier.yaml new file mode 100644 index 0000000..7770345 --- /dev/null +++ b/.github/workflows/slack-notifier.yaml @@ -0,0 +1,25 @@ +name: PR Slack Notifier + +on: + pull_request: + types: [review_requested, reopened, closed] + pull_request_review: + types: [submitted] + +permissions: + contents: read + pull-requests: write + issues: write + +concurrency: + group: pr-slack-${{ github.event.pull_request.number }}-${{ github.workflow }} + cancel-in-progress: false + +jobs: + notify-devs: + runs-on: ubuntu-latest + steps: + - uses: livekit/slack-notifier-action@main + with: + config_json: ${{ secrets.SLACK_NOTIFY_CONFIG_JSON }} + slack_token: ${{ secrets.SLACK_PR_NOTIFIER_TOKEN }} diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..fff5942 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,25 @@ +version: "2" +linters: + default: none + enable: + - depguard + - staticcheck + settings: + depguard: + rules: + protojson: + deny: + - pkg: google.golang.org/protobuf/encoding/protojson + desc: use github.com/livekit/protocol/utils/protojson instead — it sets DiscardUnknown by default to avoid schema-drift footguns + staticcheck: + checks: + - "all" + - "-ST1000" + - "-ST1003" + - "-ST1020" + - "-ST1021" + - "-ST1022" + - "-SA1019" + - "-QF1008" + exclusions: + generated: strict diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 44b9829..22f3d0e 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -53,7 +53,10 @@ changelog: - '^docs:' - '^test:' gomod: - proxy: true + # proxy must be disabled: it re-fetches modules from the Go proxy in a clean + # directory, which does not honor the replace directive in go.mod + # (replace github.com/pion/dtls/v3 => ... pinning v3.1.2 for gst whipsink compat) + proxy: false mod: mod checksum: name_template: 'checksums.txt' diff --git a/CHANGELOG.md b/CHANGELOG.md index cf60ce2..2331535 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,687 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.13.1] - 2026-06-08 + +### Fixed + +- ci: disable goreleaser gomod proxy, incompatible with go.mod replace directive + +## [1.13.0] - 2026-06-08 + +## ATTENTION: This release removes backwards compatibility for TURN authentication without TTL. Please refer to change log of [1.12.0](https://github.com/livekit/livekit/releases/tag/v1.12.0) for details on changes to TURN authentication and permissions handling. It was removed in https://github.com/livekit/livekit/pull/4539. + +### Added + +- rtc: report participant kind code and details (#4534) +- Add IsIntentionalDisconnect (#4537) +- rtc: emit per-data-track bytes via BytesTrackStats (#4540) +- Start tracking Twirp method request latency in prometheus too, not just in logs (#4545) +- Config documentation for advertise_internal_ip and skip_external_ip_validation (#4552, #4554, #4563) +- Metrics for participant active, i. e. fully established. (#4557) +- turn: allow for providing secret via file (#4564) + +### Changed + +- Always enable rtx codec (#4533) +- Don't require media sections when joining (#4535) +- Use NACKQuueInterface type. (#4538) +- Remove backwards compatibility support for TURN auth. (#4539) +- telemetry: split webhook-processed hook out of NewTelemetryService (#4548) +- rtc: prevent duration reporting for inactive participants (#4550) +- rtc: report participant session end time on room move (#4561) +- Update mediatransportutil to get ICE candidate timeout config (#4572) + +### Fixed + +- Check Less and LessEq in version compare. (#4532) +- fix sip error categorization (#4528) +- handle nil clientInfo (#4546, #4547) +- Prevent panic from nil(illegal) syncState.Subscriptions message (#4560) + +## [1.12.0] - 2026-05-16 + +## ATTENTION: This release introduces important changes to how TURN authentication and permissions are handled. These changes make the system more secure. This release maintains backwards compatibility. However, backwards compatibility will be removed in the next release. So, please plan accordingly. + +### TURN permission handling changes + +By default, TURN will not relay traffic to private IPs. If you need to relay traffic to private IPs, please use `allow_restricted_peer_cidrs` which is explained in config-sample.yaml and replicated below. + +``` +# # list of restricted peer CIDRs (loopback, link-local (unicast, multicast), multicast, private, unspecified) to allow access to. +# # By default (i. e. empty list), all restricted peer CIDRs are denied access. +# # When not empty, only the specified CIDRs are allowed access. +# # Note that this check is applied to restricted peer CIDRs only. +# allow_restricted_peer_cidrs: +# - 10.0.0.0/8 +# - 192.168.0.0/16 +``` + +That list can be paired with a deny list which takes precedence if there is an overlap. + +``` +# # list of peer CIDRs to deny access to. +# # This applies to all peer CIDRs, including restricted ones. +# # Deny list takes precedence over allow list. +# deny_peer_cidrs: +# - 10.0.0.0/8 +# - 192.168.0.0/16 +``` + +Relevant PRs: #4505 + +### TURN authentication handling changes + +TURN credentials now have a TTL (Time-To-Live) beyond which they are not valid to join a room. We recommend rotating your TURN authentication secrets to ensure older credentials are rendered stale and not usable. + +``` +# # TTL of the TURN credentials in seconds - defaults to 300 +# ttl_seconds: 300 +``` + +Please note that the TTL is always used in constructing the credentials. So, a value of 0 will create credentials that expire immediately. + +Relevant PRs: #4505, #4515, #4524, #4526 + +### Added + +- add support for client capabilities (#4461) +- Additional data tracks logging (#4489) +- Add CloseWithReason to agent SignalConn interface (#4492) +- add agent environment (#4498) +- add duration seconds reporting (#4500) +- add helper to check for agent worker endpoint (#4503) +- test: verify upstream and downstream connection stats end-to-end (#4508) +- Add TURN permission handler. (#4505) +- allow setting agent job assignment url (#4512) +- Add expiry to TURN password. (#4505, #4515) +- add AssignmentHook to AssignJob; propagate websocket write errors (#4516) +- Apply room tags from JWT grant room configuration (#4518) +- feat: auto create rooms for tokens with the RoomCreate grant (#4320) +- Add integration test for TURN auth failures (#4524) +- Support SIP auth realm for inbound. (#4522) +- Apply ttl check only when authenticate allocation creating (#4526) + +### Changed + +- feat(pion/ice): replace deprecated NAT1To1 with SetAddressRewriteRules (#4466) +- do not log data track stats if not started (#4468) +- Consolidate RTCP packets and do RTCP callback outside lock. (#4469) +- Keep a shadow copy of tracks for use by different stream allocator state (#4470) +- Avoid stream allocator event data cast to interface and back. (#4471) +- Convert sort.Slice -> slices.SortFunc (#4472) +- Turn off transceiver re-use on Safari. (#4474) +- reduce some heap use in packet path by (#4478) +- Close peer connection unconditionally to unblock set local/remote (#4485) +- Misc optimisations. (#4490) +- report all simulcast layers (#4491) +- Use mediatransportutil/codec package, no functional change (#4497) +- rename agent environment to deployment (#4506) +- Update protocol to support SIP media config. (#4509) +- update protocol for protojson (#4510) +- Populate data track loggers with context (#4514) +- Log large packets receive/send. (#4521) +- Log details of RTCP packets. (#4525) +- Create NACK tracker only once. (#4527) + +### Fixed + +- fix: wrap IPv6 addresses in brackets in UDP TURN URLs (RFC 3986) (#4476) +- Legacy TrackInfo.Simulcast flag. (#4493) +- Fix publish-only limitations being incorrectly applied to receivers (#4495) +- Include reception reoprts in receiver report callback. (#4496) +- Fix sense check in DeltaInfo gathering (#4507) +- Fix SIP media config upgrade. (#4511) + +## [1.11.0] - 2026-04-17 + +NOTE: Minor version bump that enables data tracks (https://docs.livekit.io/transport/data/data-tracks/) by default. + +### Added + +- Embedded turn test (#4412) +- chore: log API key during worker registration (#4428) +- Add some simple data track stats (#4431) +- Add `Close` method for UpDataTrackManager and call it on participant (#4432) +- Log join duration. (#4433) +- Add subscriber stream start event notification (#4449) + +### Changed + +- Cleaning up some logs and standardising log frequency. (#4420) +- Keep subscription synchronous when publisher is expected to resume. (#4424, #4425) +- Do not close publisher peer connection to aid migration. (#4426, #4427) +- Enable data tracks by default. (#4429) +- chore: pin GH commits and switch to golangci-lint (#4444) +- Switch to stdlib maps, slices (#4445) +- Store concrete ICE candidate for remote candidates. (#4458) + +### Fixed + +- clear track notifier observers on subscription teardown (#4413) +- Guard against timestamp inversion in RED -> Opus conversion. (#4414, #4415, #4418) +- ensure participant init is correctly serialized for logging (#4417) +- Clean up data track observers on unsubscribe. (#4421) +- compute agent dispatch affinity from target load (#4442) +- Apply IPFilter when get local ip (#4440) +- Unsubscribe from data track on close (#4443) +- Use Muted in TrackInfo to propagated published track muted. (#4453) +- fix: limit join request and WHIP request body to http.DefaultMaxHeaderBytes (#4450) +- fix publisher frame count reporting for simulcast streams (#4457) + +## [1.10.1] - 2026-03-30 + +### Added + +- add packet trailer stripping support (#4361) +- Path check helpers (#4392) +- add deadline to dtls connect context (#4395) +- feat(agent-dispatch): add job restart policy (#4401) + +### Changed + +- Close both peer connections to aid migration. (#4382) + +### Fixed + +- Fix TURN server URL (#4389) +- AV1 parser overflow fix. (#4405) +- Address malformed H264/H265 parsing issues (#4407) + +## [1.10.0] - 2026-03-23 + +## PLEASE NOTE: The logging key for participant session ID (a.k.a participant SID) has been changed from `pID` to `participantID` in this release for the sake of clarity. Hence the minor version bump. + +### Added + +- Add option to require media sections when participant joining (#4347, #4354) +- Support originating calls from custom domains (#4349) +- Add StopEgress function to the EgressLauncher interface (#4353) +- Add option to not re-use transceiver in e2ee. (#4356) +- Add API to restart lite stats. (#4366, #4368) + +* handle AGENT_ERROR disconnect reason (#4339) + +### Changed + +- Mark last run of grow bucket outside goroutine. (#4348) +- Refine ipv6 support (#4352) +- Sample data send error logging. (#4358) +- Switch data track extension to 1-byte ID/length. (#4362) +- Do not kick off migration of closed participant (#4363) +- Do not block all ext ID determination on stream allocator listener (#4364) +- Rename log field pID to participantID (#4365) +- Replace deprecated io/ioutil with io in whipservice (#4375) +- Update grpc to address CVE-2026-33186 (#4381) + +### Fixed + +- Fix repair stream ID reporting for RTX pairing. (#4369) + +## [1.9.12] - 2026-03-05 + +### Added + +- Add silent frame for pcmu/a (#4258) +- adds a test to ensure agent worker errors cause disconnection (#4273) +- Populate client_protocol field in ParticipantInfo (#4293) +- Read client protocol from query param (#4294) +- generate & log egressID for start egress request (#4303) +- ESP32 Client Info (#4267) +- feat: make INSTALL_PATH overridable in install script (#3954) + +### Changed + +- Defer setting clock rate in RTPStats module till codec is bound. (#4250) +- Wrapping SIP errors for invalid argument and not found (#4253) +- Ignore parse addr error when add remote candidate (#4264) +- Generate config flags (#4268) +- clear reference guard when resetting signal stats (#4279) +- refresh telemetry guard on participant move (#4280) +- use separate allocation for signal stats telemetry guard (#4281) +- Set up audio config in audio level module when config is updated. (#4290) +- Update self-hosting deployment documentation link (#4312) +- Key telemetry stats worker using combination of roomID, participantID (#4323) +- Two phase restart when doing external restart of receiver. (#4329) +- Send participant left event after track unpublished for moved (#4334) + +### Fixed + +- Fix receiver restart race (#4248) +- require participant broadcast when metadata/attributes are set in token (#4266) +- Create buffer if needed when a PLI is requested. (#4282) +- Do not increase max expected layer on track info update. (#4285) +- Publish is always on publisher peer connection.(#4307) +- Potential fix for code scanning alert no. 35: Workflow does not contain permissions (#4311) +- Potential fix for code scanning alert no. 36: Workflow does not contain permissions (#4310) +- Fix for some CodeQL reported issues (#4314) +- Protect against incorrect temporal layer. (#4327) +- do not discount packets lost on duplicate packets (#4333) +- Use ParticipantTelemetryListener of LocalParticipant. (#4342) +- Fix SIP client timeout. (#4345) + +## [1.9.11] - 2026-01-15 + +## PLEASE NOTE: The previous release tag v1.9.10 hit a panic under some conditions. Sincerely regret the inconvenience caused. Although we do test rigorously, it is not guaranteed to cover all scenarios. We request you to report any issues you encounter. Thank you. + +### Added + +- Support OpenTelemetry tracing. Add Jaeger support. (#4222) +- Add option to force simuclast codec. (#4226) +- Log timeout in API (#4231, #4232) +- Add participant option for data track auto-subscribe. (#4240) + +### Changed + +- Remove enable arrival time forwarding method. (#4217) +- sfu/receiver and sfu/buffer refactor (#4221, #4224, #4225) +- Change some logs to debugw (#4229) +- Changing field naming of data track packet (#4235) +- Update Pion transport package. (#4237) +- Wrapping the invalid request errors for CreateSipParticipant (#4239) + +### Fixed + +- Swap result sink atomically rather than closing and setting (#4216) +- Address crash in v1.9.10 (#4219, #4220) +- Return on SDP fragment read error. (#4228) + +## [1.9.10] - 2026-01-01 + +## WARNING: Please do not use this release. There is a run time issue which causes the server to panic. The issue has been addressed in #4219 and #4220. + +### Added + +- add explicit room exists servicestore op (#4175) +- Add support for TURN static auth secret credentials (#3796) +- Make new path for signalling v1.5 support. (#4180) +- report video size from media data for whip (#4211) +- Support preserving external supplied time. (#4212) + +### Changed + +- Use published track for model access in data down track. (#4176) +- Refactor receiver and buffer into Base and higher layer. (#4185, #4186, #4187, #4189, #4196, #4198, #4207) +- Update pion/webrtc to v4.2.1 (#4191) +- Receiver restart related changes. (#4192, #4200, #4202, #4208) +- Do not warn about track not bound if participant is not ready. (#4205, #4206) + +### Fixed + +- Flush ext packets on restart/close and release packets. (#4179) +- Resolve RTX pair via OnTrack also. (#4190) +- Handle repair SSRC of simulcast tracks during migration. (#4193) +- return iceservers for whip (#4210) + +## [1.9.9] - 2025-12-18 + +### Added + +- Add support for RTP stream restart. (#4161) + +### Changed + +- Avoid duplicate track add to room track manager. (#4152, #4153) +- Consistently undo update to sequence number and timestamp when the incoming packet cannot be sequenced. (#4156) +- deregister observability function when participant is closed (#4157) +- Ensure subscribe data track handles are unique (#4162) +- move delete to oss service store (#4164) +- clean up manual roomservice log redaction (#4165) +- skip lost sequence number ranges in getIntervalStats (#4166, #4169) + +### Fixed + +- chore: fix a large number of spelling issues (#4147) +- Handle case of sequence number jump just after start. (#4150) +- Drop run away receiver reports. (#4170) +- Publish/Unpublish counter match. (#4173) + +## [1.9.8] - 2025-12-10 + +### Added + +- Mark RTCP buffer Write as noinline. (for better heap attribution) (#4138) +- add debug metric for tracking references (#4134) + +### Changed + +- Use isEnding to indicate if down track could be resumed. (#4132) +- switch participant callbacks to room to listener interface (#4136) +- protocol deps to get inactive file adjusted memory usage. (#4137) +- update webrtc to 4.1.8 to pick up DTLS fingerprint check during handshake (#4140) + +### Fixed + +- Do not pause rid in SDP to prevent race with adaptive streaming (#4129) +- leak fixes (#4131, #4141, #4142, #4143, #4144) + +## [1.9.7] - 2025-12-05 + +### Added + +- Data tracks (experimental and not ready for use) (#4089) + +### Changed + +- log bucket growth (#4122) +- Update pion/ice to stop gather first on close (#4123) +- move utils.WrapAround to mediatransportutil (#4124) +- Let participant close remove the published tracks. (#4125) + +### Fixed + +- Fix concurrent map access for https://github.com/livekit/livekit/issues/4126. (#4127) + +## [1.9.6] - 2025-12-01 + +### Added + +- Control latency of lossy data channel (#4088) +- logger proto redaction. (#4090) +- Record join/publish/subscribe cancellations (#4102, #4104) + +### Fixed + +- Fix "address" typo in transport logs (addddress → address) (#4097) +- Clear stereo=1 if stereo is not enabled. (#4101) +- Participant session close deadlock fixes (#4107, #4113, #4116) + +### Changed + +- Switch forwarding latency log to Debugw (#4098) +- Update mediatransportutil to get OWD estimator relocation (#4115) + +## [1.9.5] - 2025-12-01 - scratched + +## [1.9.4] - 2025-11-15 + +### Added + +- Log reason for subscriber not being able to determine codec. (#4071) +- Kind details for connector (#4072) + +### Fixed + +- Prevent invalid track access while peer connection is shutting down. (#4054) + +### Changed + +- Update PsRPC to get redis pipeliner implementation (#4055) +- Forwarding latency measurement. (#4056. #4057, #4059, #4061, #4062, #4067, #4080) +- Update pion/transport to v3.1.1 (to get batch I/O ping-pong buffer) (#4070) +- Use sync.Pool for objects in packet path. (#4066) +- Bump protocol to pull sip validation changes and error mapping (#4081) + +## [1.9.3] - 2025-11-02 + +### Added + +- Opportunistic video layer allocation on setting max spatial layer. (#4003, #4030, #4031, #4033) +- use env var for GOARCH. (#4012) +- Use simulcast codec as default policy for audio track. (#4040) +- Enable AbsCaptureTimeURI in RTC configuration. (#4043) +- Add prom histogram for forwarding latency and jitter. (#4044, #4045) + +### Fixed + +- Correct direction for request/response for prom counters. (#4027) +- Do not bind buffer if codec is invalid. (#4028) +- Remove ~ from rid which indicates disabled layer to get the actual rid. (#4032) +- Prevent leakage of previous codec after codec regression. (#4035, #4037) +- fix: add missing Unlock() in AddReceiver. (#4036) + +### Changed + +- Some golang modernisation bits. (#4106) +- Use rtp converter from protocol/utils. (#4019, #4020) +- High forwarding latency. (#4034, #4038) +- if RingingTimeout is provided, deadline should be set to that timeout. (#4018) +- Don't warn 0 payload type for PCMU. (#4039) + +## [1.9.2] - 2025-10-17 + +### Added + +- Use gzip reader pool (#3903) +- Rpcs for ingress proxy WHIP (#3911) +- Include agent_name as a participant attribute (#3914) +- Clean code as there is no oss sweeper for ingress (#3918) +- Support simulcasting of audio (#3920) +- Subscrbed audio codecs - update from remote nodes. (#3921) +- Log some information around high forwarding latency. (#3944) +- feat: server rpc apis (#3904) +- short circuit participant broadcast filter in livestream mode (#3955) +- Adjust for hold time when fowarding RTCP report. (#3956) +- Add node_ip to config-sample.yaml (#3960) +- add idempotent reference count to telemetry stats worker (#3964) +- add config for user data recording (#3966) +- Provide the InputVideo/AudioState to Ingress in WHIPRTCConnectionNotify (#3982) +- Add encryption datapacket type (#3869) +- Allow passing inline trunk for outbound calls. (#3987) +- Log RPC details. (#3991) +- "Power of Two Random Choices" option for node selection (#3785) +- Adding ProviderInfo to GetSIPTrunkAuthenticationResponse (#3993) +- Use answer with mid -> trackID mapping when in single peer connection (#4005) +- Include mid -> trackID in both SDP offer and answer. (#4007) + +### Fixed + +- add incoming request id to request response message (#3912) +- Simulcast audio fixes (#3925) +- Fix dynacast subscriber node clearing on move participant. (#3926) +- mediatransportutil crash fix for logging local address (#3930) +- Do DD restart only if DD structure is present. (#3935) +- Avoid matching on empty track id. (#3937) +- fix stats worker closed condition (#3965) +- Update deps to fix redis issue when 1 cluster address is provided (#3969) +- Revert unintentional change to not handle transport fallback (#3970) +- Do not panic of redis is not configured (#3981) +- Sort codec layers when adding track (#3998) +- Resort to full search for requested quality is not available. (#4000) +- Do not try to read stats from peer connection after close. (#4002) +- Update pion/webrtc to prevent GetStats panic. (#4004) + +### Changed + +- update protocol for sip api change (#3902) +- Refactor subscribedTrack + mediaTrackSubscriptions. (#3908) +- Set publisher codec preferences after setting remote description (#3913) +- update protocol for psrpc (#3915) +- Wait for `SetRemoteDescription` before configuring senders. (#3924) +- Update mediatransportutil to log external IP found via STUN. (#3929) +- Add debugging from DD frame number wrap around. (#3933) +- More debugging of DD jump (#3934) +- Use difference in key frame counter to stop seeder. (#3936) +- Update protocol for SipCreateParticipant (#3939) +- mediatransportutil to log local address when validating external IP (#3942) +- Use microseconds for forwarding stats. (#3943) +- Tweaks tresholds for logging high forwarding latency/jitter. (#3945) +- Flush stats when there are no packets. (#3947) +- handle terminated job requests (#3948) +- Switch ops queue a singly linked list. (#3949) +- Revert "Switch ops queue a singly linked list." (#3950) +- Adjust stream allocator ping interval based on state. (#3951) +- avoid logging on small values (#3958) +- Update protocol for EventKey helper. (#3963) +- Do not force codec regression between opus and red. (#3980) +- Do not start forawarding on out-of-order packet. (#3985) +- Use padding only packets for dummy start of audio. (#3984) +- Support Opus mixed with RED when encrypted. (#3986) +- Limit check to red + opus when looking for primary codec match. (#3988) +- Increment RTP timestamp on padding when using dummy start. (#3989) +- Revert to using silence packets for audio dummy start. (#3999) +- Count request/response packets on both client and server side. (#4001) +- Do not call receiver methods under settings lock. (#4006) +- counterfeiter needs an older version of x/tools (#4009) + +## [1.9.1] - 2025-09-05 + +### Fixed + +- swap pub/sub track metrics (#3717) +- Fix bug with SDP rid, clear only overflow. (#3723) +- Don't check bindState on downtrack.Bind (#3726) +- Return highest available layer if requested quality is higher than max (#3729) +- Fix data packet ParticipantIdentity override logic in participant.go (#3735) +- Fix svc encoding for chrome mobile on iOS (#3751) +- offer could be nil when migrating. (#3752) +- fix(deps): update module github.com/livekit/protocol to v1.39.3 (#3733) +- bounds check layer index (#3768) +- Do not send leave if nil (to older clients) (#3817) +- Fix: RingingTimeout was being skipped for transferParticipant (#3831) +- Handle no codecs in track info (#3859) +- Fix missed unlock (#3861) +- Fix timeout handing in StopEgress (#3876) +- fix: ensure the participant kind is set on refresh tokens (#3881) +- Do not advertise NACK for RED. (#3889) +- Do not send both asb-send-time and twcc. (#3890) +- Prevent race in determining BWE type. (#3891) + +### Added + +- Adds Devin to readme so it auto updates DeepWiki weekly (#3699) +- Allow passing extra attributes to RTC endpoint. (#3693) +- warn about credentials when used in tokens (#3705) +- protocol dep for webhook stats buckets (#3706) +- for real, pick up protocol change for webhooks queue length buckets (#3707) +- implement observability for room metrics (#3712) +- e2e reliability for data channel (#3716) +- Add simulcast support for WHIP. (#3719) +- Add Id to SDP signalling messages. (#3722) +- Set and use rid/spatial layer in TrackInfo. (#3724) +- Add log for dropping out of order reliable message (#3728) +- chore: set workerid on job creation (#3737) +- return error when moving egress/agent participant (#3741) +- SVC with RID -> spatial layer mapping (#3754) +- feat(cli-flags): add option for cpu profiling (#3765) +- Enable H265 by default (#3773) +- Signalling V2 protocol implementation start (#3794) +- Signal v2: envelope and fragments as wire message format. (#3800) +- Grouping all signal messages into participant_signal. (#3801) +- starting signaller interface (#3802) +- Signal handling interfaces and participant specific HTTP PATCH. (#3804) +- Split signal segmenter and reassembler. (#3805) +- Filling out messages unlikely to change in v2. (#3806) +- Use signalling utils from protocol (#3807) +- Validation end point for v2 signalling. (#3811) +- More v2 signalling changes (#3814) +- Minor tweak to keep RPC type at service level. (#3815) +- Add country label to edge prom stats. (#3816) +- HTTP DELETE of participant session (#3819) +- Get to the point of establishing subscriber peer connection. (#3821) +- Get to the point of connecting publisher PC and using it for async signalling (#3822) +- Support join request as proto + base64 encoded query param (#3836) +- Use wrapped join request to be able to support compressed and uncompressed. (#3838) +- handle SyncState in join request (#3839) +- Support per simulcast codec layers. (#3840) +- Support video layer mode from client and make most of the code mime aware (#3843) +- Send `participant_connection_aborted` when participant session is closed (#3848) +- Support G.711 A-law and U-law (#3849) +- Extract video size from media stream (#3856) +- update mediatransport util for ice port 3478 (#3877) +- Single peer connection mode (#3873) +- handle frame number wrap back in svc (#3885) +- Use departure timeout from room preset. (#3888) +- Use `RequestResponse` to report protocol handling errors (#3895) + +### Changed + +- Add a trend check before declaring joint queuing region. (#3701) +- Small changes to add/use helper functions for length checks. (#3704) +- remove unused ws signal read loop (#3709) +- Flush stats on close (#3713) +- Do not require create permission for WHIP participant. (#3715) +- Create client config manager in room manager constructor. (#3718) +- Clear rids from default for layers not published. (#3721) +- Clear rids if not present in SDP. (#3731) +- Revert clearing RIDs. (#3732) +- Take ClientInfo from request. (#3738) +- remove unused code (#3740) +- reuse compiled client config scripts (#3743) +- feat(cli): update to urfave/cli/v3 (#3745) +- move egress roomID load to launcher (#3748) +- Log previous allocation to see changes. (#3759) +- Do not need to just clean up receivers. Remove that interface. (#3760) +- ClearAllReceivers interface is used to pause relay tracks. (#3761) +- Temporary change: use pre-defined rids (#3767) +- Revert "Temporary change: use pre-defined rids" (#3769) +- Log SDP rids to understand the mapping better. (#3770) +- Limit taking rids from SDP only in WHIP path. (#3771) +- Set rids for all codecs. (#3772) +- Return default layer for invalid rid + track info combination. (#3778) +- Normalize known rids. (#3779) +- forward agent id to job state (3786) +- Map ErrNoResponse to ErrRequestTimedOut in StopEgress to avoid returning 503 (#3788) +- Set participant active when peerconnection connected (#3790) +- Handle Metadata field from RoomConfig (#3798) +- [🤖 readme-manager] Update README (#3808) +- [🤖 readme-manager] Update README (#3809) +- Rename RTCRest -> WHIP (#3829) +- Delete v2 signalling (#3835) +- Clean up missed v2 pieces (#3837) +- Update go deps (#3849) +- Populate SDP cid in track info when available. (#3845) +- Log signal messages as debug. (#3851) +- Log signal messages on media node. (#3852) +- Log track settings more. (#3853) +- Update pion deps (#3863) +- Update golang Docker tag to v1.25 (#3864) +- Update module github.com/livekit/protocol to v1.40.0 (#3865) +- Remove unnecessary check (#3870) +- chunk room updates (#3880) +- Switch known rids from 012 -> 210, used by OBS. (#3882) +- init ua parser once (#3883) +- Revert to using answer for migration case. (#3884) +- Handle migration better in single peer connection case. (#3886) + +## [1.9.0] - 2025-06-02 + +### Added + +- Add pID and connID to log context to make it easier to search using pID. (#3518) +- add server agent load threshold config (#3520) +- Add a key frame seeder in up track. (#3524) +- Implement SIP update API. (#3141) +- Add option to use different pacer with send side bwe. (#3552) +- Allow specifying extra webhooks with egress requests (#3597) + +### Fixed + +- Fix missing RTCP sender report when forwarding RED as Opus. (#3480) +- Take RTT and jitter from receiver view while reporting track stats for (#3483) +- Fix receiver rtt/jitter. (#3487) +- fix: fix the wrong error return value (#3493) +- load mime type before calling writeBlankFrameRTP (#3502) +- Prevent bind lock deadlock on muted. (#3504) +- Handle subscribe race with track close better. (#3526) +- Do not instantiate 0 sized sequencer. (#3529) +- Fix: Return NotFoundErr instead of Unavailable when the participant does not exist in UpdateParticipant. (#3543) +- skip out of order participant state updates (#3583) +- Exclude RED from enabled codecs for Flutter + 2.4.2 + Android. (#3587) +- protocol update to fix IPv6 SDP fragment parsing (#3603) +- Forward transfer headers to internal request (#3615) +- Do not use Redis pipeline for SIP delete. Fixes Redis clustering support. (#3694) + +### Changed + +- Use a RED transformer to consolidate both RED -> Opus OR Opus -> RED (#3481) +- refactor: using slices.Contains to simplify the code (#3495) +- Do not bind lock across flush which could take time (#3501) +- Log packet drops/forward. (#3510) +- Clean up published track on participant removal. (#3527) +- Do not accept unsupported track type in AddTrack (#3530) +- Use cgroup for memstats. (#3573) +- Replace Promise with Fuse. (#3580) +- Do not drop audio codecs (#3590) +- map PEER_CONNECTION_DISCONNECTED -> CONNECTION_TIMEOUT (#3591) +- Update mediatransportutil for max sctp message (65K) (#3611) +- Disable vp9 for safari 18.4 due to compatibility (#3631) +- Avoid synthesising duplicate feature. (#3632) +- Take AudioFeatures from AddTrack. (#3635) +- Use unordered for lossy data channel. (#3653) +- Send self participant update immediately. (#3656) +- update mediatransportutil for sctp congestion control (#3673) + ## [1.8.4] - 2025-03-01 ### Added diff --git a/Dockerfile b/Dockerfile index 4d8e231..4fb0a92 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM golang:1.24-alpine AS builder +FROM golang:1.26-alpine AS builder ARG TARGETPLATFORM ARG TARGETARCH diff --git a/README.md b/README.md index 550deb4..c49da91 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ LiveKit's server is written in Go, using the awesome [Pion WebRTC](https://githu [![GitHub stars](https://img.shields.io/github/stars/livekit/livekit?style=social&label=Star&maxAge=2592000)](https://github.com/livekit/livekit/stargazers/) [![Slack community](https://img.shields.io/endpoint?url=https%3A%2F%2Flivekit.io%2Fbadges%2Fslack)](https://livekit.io/join-slack) [![Twitter Follow](https://img.shields.io/twitter/follow/livekit)](https://twitter.com/livekit) +[![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/livekit/livekit) [![GitHub release (latest SemVer)](https://img.shields.io/github/v/release/livekit/livekit)](https://github.com/livekit/livekit/releases/latest) [![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/livekit/livekit/buildtest.yaml?branch=master)](https://github.com/livekit/livekit/actions/workflows/buildtest.yaml) [![License](https://img.shields.io/github/license/livekit/livekit)](https://github.com/livekit/livekit/blob/master/LICENSE) @@ -276,7 +277,7 @@ Sign up for [LiveKit Cloud](https://cloud.livekit.io/). ### Self-host -Read our [deployment docs](https://docs.livekit.io/deploy/) for more information. +Read our [deployment docs](https://docs.livekit.io/transport/self-hosting/) for more information. ## Building from source @@ -307,12 +308,14 @@ LiveKit server is licensed under Apache License v2.0.
- + + + + - - - - + + +
LiveKit Ecosystem
LiveKit SDKsBrowser · iOS/macOS/visionOS · Android · Flutter · React Native · Rust · Node.js · Python · Unity · Unity (WebGL)
Agents SDKsPython · Node.js
LiveKit SDKsBrowser · Swift · Android · Flutter · React Native · Rust · Node.js · Python · Unity · Unity (WebGL) · ESP32 · C++
Starter AppsPython Agent · TypeScript Agent · React App · SwiftUI App · Android App · Flutter App · React Native App · Web Embed
UI ComponentsReact · Android Compose · SwiftUI · Flutter
Server APIsNode.js · Golang · Ruby · Java/Kotlin · Python · Rust · PHP (community) · .NET (community)
UI ComponentsReact · Android Compose · SwiftUI
Agents FrameworksPython · Node.js · Playground
ServicesLiveKit server · Egress · Ingress · SIP
ResourcesDocs · Example apps · Cloud · Self-hosting · CLI
ResourcesDocs · Docs MCP Server · CLI · LiveKit Cloud
LiveKit Server OSSLiveKit server · Egress · Ingress · SIP
CommunityDeveloper Community · Slack · X · YouTube
diff --git a/cmd/server/commands.go b/cmd/server/commands.go index 1473524..beba269 100644 --- a/cmd/server/commands.go +++ b/cmd/server/commands.go @@ -15,6 +15,7 @@ package main import ( + "context" "fmt" "os" "strconv" @@ -23,10 +24,12 @@ import ( "github.com/dustin/go-humanize" "github.com/olekukonko/tablewriter" - "github.com/urfave/cli/v2" + "github.com/olekukonko/tablewriter/tw" + "github.com/urfave/cli/v3" "gopkg.in/yaml.v3" "github.com/livekit/protocol/auth" + "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/utils" "github.com/livekit/protocol/utils/guid" @@ -35,7 +38,7 @@ import ( "github.com/livekit/livekit-server/pkg/service" ) -func generateKeys(_ *cli.Context) error { +func generateKeys(_ context.Context, _ *cli.Command) error { apiKey := guid.New(utils.APIKeyPrefix) secret := utils.RandomSecret() fmt.Println("API Key: ", apiKey) @@ -43,7 +46,7 @@ func generateKeys(_ *cli.Context) error { return nil } -func printPorts(c *cli.Context) error { +func printPorts(_ context.Context, c *cli.Command) error { conf, err := getConfig(c) if err != nil { return err @@ -84,17 +87,21 @@ func printPorts(c *cli.Context) error { return nil } -func helpVerbose(c *cli.Context) error { +func helpVerbose(_ context.Context, c *cli.Command) error { generatedFlags, err := config.GenerateCLIFlags(baseFlags, false) if err != nil { return err } - c.App.Flags = append(baseFlags, generatedFlags...) + flags := append([]cli.Flag{}, baseFlags...) + flags = append(flags, generatedFlags...) + root := c.Root() + root.Flags = flags + c.Flags = flags return cli.ShowAppHelp(c) } -func createToken(c *cli.Context) error { +func createToken(_ context.Context, c *cli.Command) error { room := c.String("room") identity := c.String("identity") @@ -160,7 +167,7 @@ func createToken(c *cli.Context) error { return nil } -func listNodes(c *cli.Context) error { +func listNodes(_ context.Context, c *cli.Command) error { conf, err := getConfig(c) if err != nil { return err @@ -181,10 +188,27 @@ func listNodes(c *cli.Context) error { return err } - table := tablewriter.NewWriter(os.Stdout) - table.SetRowLine(true) - table.SetAutoWrapText(false) - table.SetHeader([]string{ + table := tablewriter.NewTable(os.Stdout, + tablewriter.WithHeaderAutoFormat(tw.Off), + tablewriter.WithRowAutoWrap(0), + tablewriter.WithRendition(tw.Rendition{ + Settings: tw.Settings{ + Separators: tw.Separators{BetweenRows: tw.On}, + }, + }), + tablewriter.WithRowAlignmentConfig(tw.CellAlignment{ + PerColumn: []tw.Align{ + tw.AlignCenter, tw.AlignCenter, tw.AlignCenter, + tw.AlignRight, tw.AlignRight, + tw.AlignRight, + tw.AlignRight, tw.AlignRight, + tw.AlignRight, tw.AlignRight, tw.AlignRight, + tw.AlignRight, tw.AlignRight, + tw.AlignCenter, + }, + }), + ) + table.Header( "ID", "IP Address", "Region", "CPUs", "CPU Usage\nLoad Avg", "Memory Used/Total", @@ -192,19 +216,14 @@ func listNodes(c *cli.Context) error { "Bytes/s In/Out\nBytes Total", "Packets/s In/Out\nPackets Total", "System Dropped Pkts/s\nPkts/s Out/Dropped", "Nack/s\nNack Total", "Retrans/s\nRetrans Total", "Started At\nUpdated At", - }) - table.SetColumnAlignment([]int{ - tablewriter.ALIGN_CENTER, tablewriter.ALIGN_CENTER, tablewriter.ALIGN_CENTER, - tablewriter.ALIGN_RIGHT, tablewriter.ALIGN_RIGHT, - tablewriter.ALIGN_RIGHT, - tablewriter.ALIGN_RIGHT, tablewriter.ALIGN_RIGHT, - tablewriter.ALIGN_RIGHT, tablewriter.ALIGN_RIGHT, tablewriter.ALIGN_RIGHT, - tablewriter.ALIGN_RIGHT, tablewriter.ALIGN_RIGHT, - tablewriter.ALIGN_CENTER, - }) + ) for _, node := range nodes { stats := node.Stats + rate := &livekit.NodeStatsRate{} + if len(stats.Rates) > 0 { + rate = stats.Rates[0] + } // Id and state idAndState := fmt.Sprintf("%s\n(%s)", node.Id, node.State.Enum().String()) @@ -220,19 +239,23 @@ func listNodes(c *cli.Context) error { clientsAndTracks := fmt.Sprintf("%d\n%d / %d", stats.NumClients, stats.NumTracksIn, stats.NumTracksOut) // Packet stats - bytes := fmt.Sprintf("%sps / %sps\n%s / %s", humanize.Bytes(uint64(stats.BytesInPerSec)), humanize.Bytes(uint64(stats.BytesOutPerSec)), + bytes := fmt.Sprintf("%sps / %sps\n%s / %s", humanize.Bytes(uint64(rate.BytesIn)), humanize.Bytes(uint64(rate.BytesOut)), humanize.Bytes(stats.BytesIn), humanize.Bytes(stats.BytesOut)) - packets := fmt.Sprintf("%s / %s\n%s / %s", humanize.Comma(int64(stats.PacketsInPerSec)), humanize.Comma(int64(stats.PacketsOutPerSec)), + packets := fmt.Sprintf("%s / %s\n%s / %s", humanize.Comma(int64(rate.PacketsIn)), humanize.Comma(int64(rate.PacketsOut)), strings.TrimSpace(humanize.SIWithDigits(float64(stats.PacketsIn), 2, "")), strings.TrimSpace(humanize.SIWithDigits(float64(stats.PacketsOut), 2, ""))) - sysPackets := fmt.Sprintf("%.2f %%\n%v / %v", stats.SysPacketsDroppedPctPerSec*100, float64(stats.SysPacketsOutPerSec), float64(stats.SysPacketsDroppedPerSec)) - nacks := fmt.Sprintf("%.2f\n%s", stats.NackPerSec, strings.TrimSpace(humanize.SIWithDigits(float64(stats.NackTotal), 2, ""))) - retransmit := fmt.Sprintf("%.2f\n%s", stats.RetransmitPacketsOutPerSec, strings.TrimSpace(humanize.SIWithDigits(float64(stats.RetransmitPacketsOut), 2, ""))) + sysPacketsDroppedPct := float32(0) + if rate.SysPacketsOut+rate.SysPacketsDropped > 0 { + sysPacketsDroppedPct = float32(rate.SysPacketsDropped) / float32(rate.SysPacketsDropped+rate.SysPacketsOut) + } + sysPackets := fmt.Sprintf("%.2f %%\n%v / %v", sysPacketsDroppedPct*100, float64(rate.SysPacketsOut), float64(rate.SysPacketsDropped)) + nacks := fmt.Sprintf("%.2f\n%s", rate.NackTotal, strings.TrimSpace(humanize.SIWithDigits(float64(stats.NackTotal), 2, ""))) + retransmit := fmt.Sprintf("%.2f\n%s", rate.RetransmitPacketsOut, strings.TrimSpace(humanize.SIWithDigits(float64(stats.RetransmitPacketsOut), 2, ""))) // Date startedAndUpdated := fmt.Sprintf("%s\n%s", time.Unix(stats.StartedAt, 0).UTC().UTC().Format("2006-01-02 15:04:05"), time.Unix(stats.UpdatedAt, 0).UTC().Format("2006-01-02 15:04:05")) - table.Append([]string{ + table.Append( idAndState, node.Ip, node.Region, cpus, cpuUsageAndLoadAvg, memUsage, @@ -240,9 +263,9 @@ func listNodes(c *cli.Context) error { bytes, packets, sysPackets, nacks, retransmit, startedAndUpdated, - }) + ) } - table.Render() + table.Render() //nolint:errcheck return nil } diff --git a/cmd/server/main.go b/cmd/server/main.go index 6cdfba6..dde5b0a 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -15,6 +15,7 @@ package main import ( + "context" "fmt" "math/rand" "net" @@ -25,15 +26,16 @@ import ( "syscall" "time" - "github.com/urfave/cli/v2" + "github.com/urfave/cli/v3" - "github.com/livekit/livekit-server/pkg/rtc" - "github.com/livekit/livekit-server/pkg/telemetry/prometheus" "github.com/livekit/protocol/logger" + "github.com/livekit/protocol/tracer/jaeger" "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/livekit-server/pkg/routing" + "github.com/livekit/livekit-server/pkg/rtc" "github.com/livekit/livekit-server/pkg/service" + "github.com/livekit/livekit-server/pkg/telemetry/prometheus" "github.com/livekit/livekit-server/version" ) @@ -49,7 +51,7 @@ var baseFlags = []cli.Flag{ &cli.StringFlag{ Name: "config-body", Usage: "LiveKit config in YAML, typically passed in as an environment var in a container", - EnvVars: []string{"LIVEKIT_CONFIG"}, + Sources: cli.EnvVars("LIVEKIT_CONFIG"), }, &cli.StringFlag{ Name: "key-file", @@ -58,44 +60,47 @@ var baseFlags = []cli.Flag{ &cli.StringFlag{ Name: "keys", Usage: "api keys (key: secret\\n)", - EnvVars: []string{"LIVEKIT_KEYS"}, + Sources: cli.EnvVars("LIVEKIT_KEYS"), }, &cli.StringFlag{ Name: "region", Usage: "region of the current node. Used by regionaware node selector", - EnvVars: []string{"LIVEKIT_REGION"}, + Sources: cli.EnvVars("LIVEKIT_REGION"), }, &cli.StringFlag{ Name: "node-ip", Usage: "IP address of the current node, used to advertise to clients. Automatically determined by default", - EnvVars: []string{"NODE_IP"}, + Sources: cli.EnvVars("NODE_IP"), }, &cli.StringFlag{ Name: "udp-port", Usage: "UDP port(s) to use for WebRTC traffic", - EnvVars: []string{"UDP_PORT"}, + Sources: cli.EnvVars("UDP_PORT"), }, &cli.StringFlag{ Name: "redis-host", Usage: "host (incl. port) to redis server", - EnvVars: []string{"REDIS_HOST"}, + Sources: cli.EnvVars("REDIS_HOST"), }, &cli.StringFlag{ Name: "redis-password", Usage: "password to redis", - EnvVars: []string{"REDIS_PASSWORD"}, + Sources: cli.EnvVars("REDIS_PASSWORD"), }, &cli.StringFlag{ Name: "turn-cert", Usage: "tls cert file for TURN server", - EnvVars: []string{"LIVEKIT_TURN_CERT"}, + Sources: cli.EnvVars("LIVEKIT_TURN_CERT"), }, &cli.StringFlag{ Name: "turn-key", Usage: "tls key file for TURN server", - EnvVars: []string{"LIVEKIT_TURN_KEY"}, + Sources: cli.EnvVars("LIVEKIT_TURN_KEY"), + }, + &cli.StringFlag{ + Name: "cpuprofile", + Usage: "write CPU profile to `file`", }, - // debugging flags &cli.StringFlag{ Name: "memprofile", Usage: "write memory profile to `file`", @@ -127,7 +132,7 @@ func main() { fmt.Println(err) } - app := &cli.App{ + cmd := &cli.Command{ Name: "livekit-server", Usage: "High performance WebRTC server", Description: "run without subcommands to start the server", @@ -182,21 +187,18 @@ func main() { Version: version.Version, } - if err := app.Run(os.Args); err != nil { + if err := cmd.Run(context.Background(), os.Args); err != nil { fmt.Println(err) } } -func getConfig(c *cli.Context) (*config.Config, error) { +func getConfig(c *cli.Command) (*config.Config, error) { confString, err := getConfigString(c.String("config"), c.String("config-body")) if err != nil { return nil, err } - strictMode := true - if c.Bool("disable-strict-config") { - strictMode = false - } + strictMode := !c.Bool("disable-strict-config") conf, err := config.NewConfig(confString, strictMode, c, baseFlags) if err != nil { @@ -242,11 +244,14 @@ func getConfig(c *cli.Context) (*config.Config, error) { return conf, nil } -func startServer(c *cli.Context) error { +func startServer(ctx context.Context, c *cli.Command) error { conf, err := getConfig(c) if err != nil { return err } + if url := conf.Trace.JaegerURL; url != "" { + jaeger.Configure(ctx, url, "livekit") + } // validate API key length err = conf.ValidateKeys() @@ -254,6 +259,25 @@ func startServer(c *cli.Context) error { return err } + if err = conf.LoadTURNSecrets(); err != nil { + return err + } + + if cpuProfile := c.String("cpuprofile"); cpuProfile != "" { + if f, err := os.Create(cpuProfile); err != nil { + return err + } else { + if err := pprof.StartCPUProfile(f); err != nil { + f.Close() + return err + } + defer func() { + pprof.StopCPUProfile() + f.Close() + }() + } + } + if memProfile := c.String("memprofile"); memProfile != "" { if f, err := os.Create(memProfile); err != nil { return err @@ -285,7 +309,7 @@ func startServer(c *cli.Context) error { signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) go func() { - for i := 0; i < 2; i++ { + for i := range 2 { sig := <-sigChan force := i > 0 logger.Infow("exit requested, shutting down", "signal", sig, "force", force) diff --git a/config-sample.yaml b/config-sample.yaml index 6a64782..f264e72 100644 --- a/config-sample.yaml +++ b/config-sample.yaml @@ -65,6 +65,19 @@ rtc: # this is useful for cloud environments such as AWS & Google where hosts have an internal IP # that maps to an external one use_external_ip: true + # # when set to true, advertises both mapped external and internal IPs to clients as server candidates. + # # useful when clients connect from both private and public networks. + # # works only when `use_external_ip` is set to true. + # # when both this and `external_ip_only` are set, SFU advertises all private IPs with their mapped external IPs and skips + # # private IPs that do not have a mapped external IP. + # advertise_internal_ip: true + # # when set to true, SFU skips external IP accessibility checks via self-ping. + # # enable this when SFU is behind NAT that does not allow self-ping. only works when `use_external_ip` is set to true. + # skip_external_ip_validation: true + # # there are cases where the public IP determined via STUN is not the correct one + # # in such cases, use this setting to set the public IP of the node + # # use_external_ip takes precedence, for this to take effect, set use_external_ip to false + # node_ip: # # when set, LiveKit will attempt to use a UDP mux so all UDP traffic goes through # # listed port(s). To maximize system performance, we recommend using a range of ports # # greater or equal to the number of vCPUs on the machine. @@ -83,6 +96,13 @@ rtc: # port: 443 # # tls, tcp, or udp # protocol: tls + # # Shared secret for TURN server authentication + # secret: "" + # # Path for file containing shared secret for TURN server authentication + # # When both secret and secret_file are set, secret takes precedence + # secret_file: "/var/lib/coturn/secret" + # ttl: 14400 # seconds + # # Insecure username/password authentication # username: "" # credential: "" # # allows LiveKit to monitor congestion when sending streams and automatically @@ -94,6 +114,12 @@ rtc: # allow_pause: true # # allows automatic connection fallback to TCP and TURN/TLS (if configured) when UDP has been unstable, default true # allow_tcp_fallback: true + # # signaling RTT (in milliseconds) below which ICE/TCP is attempted on a UDP failure; at or above it, + # # supporting clients fall back directly to TURN/TLS. 0 (default) disables the check (ICE/TCP always tried). + # # a positive value also enables allow_udp_unstable_fallback. + # tcp_fallback_rtt_threshold: 0 + # # migrate an established-but-lossy UDP connection to ICE/TCP or TURN/TLS. requires tcp_fallback_rtt_threshold > 0, default false + # allow_udp_unstable_fallback: false # # number of packets to buffer in the SFU for video, defaults to 500 # packet_buffer_size_video: 500 # # number of packets to buffer in the SFU for audio, defaults to 200 @@ -269,6 +295,21 @@ keys: # # optional (set only if not using external TLS termination) # # cert_file: /path/to/cert.pem # # key_file: /path/to/key.pem +# # TTL of the TURN credentials in seconds - defaults to 300 +# ttl_seconds: 300 +# # list of restricted peer CIDRs (loopback, link-local (unicast, multicast), multicast, private, unspecified) to allow access to. +# # By default (i. e. empty list), all restricted peer CIDRs are denied access. +# # When not empty, only the specified CIDRs are allowed access. +# # Note that this check is applied to restricted peer CIDRs only. +# allow_restricted_peer_cidrs: +# - 10.0.0.0/8 +# - 192.168.0.0/16 +# # list of peer CIDRs to deny access to. +# # This applies to all peer CIDRs, including restricted ones. +# # Deny list takes precedence over allow list. +# deny_peer_cidrs: +# - 10.0.0.0/8 +# - 192.168.0.0/16 # ingress server # ingress: @@ -287,6 +328,9 @@ keys: # # priority used for selection of node when multiple are available # # default: random. valid values: random, sysload, cpuload, rooms, clients, tracks, bytespersec # sort_by: sysload +# # algorithm used to govern selecting from sorted nodes +# # default: lowest. valid values: lowest, twochoice +# algorithm: lowest # # used in sysload and regionaware # # do not assign room to node if load per CPU exceeds sysload_limit # sysload_limit: 0.7 diff --git a/go.mod b/go.mod index e583c01..bddcbc3 100644 --- a/go.mod +++ b/go.mod @@ -1,144 +1,160 @@ module github.com/livekit/livekit-server -go 1.23 - -toolchain go1.23.6 +go 1.26 require ( github.com/bep/debounce v1.2.1 github.com/d5/tengo/v2 v2.17.0 - github.com/dennwc/iters v1.0.1 + github.com/dennwc/iters v1.2.2 github.com/dustin/go-humanize v1.0.1 - github.com/elliotchance/orderedmap/v2 v2.7.0 - github.com/florianl/go-tc v0.4.4 + github.com/elliotchance/orderedmap/v3 v3.1.0 + github.com/florianl/go-tc v0.4.8 github.com/frostbyte73/core v0.1.1 - github.com/gammazero/deque v1.0.0 - github.com/gammazero/workerpool v1.1.3 + github.com/gammazero/deque v1.2.1 + github.com/gammazero/workerpool v1.2.1 github.com/google/uuid v1.6.0 - github.com/google/wire v0.6.0 + github.com/google/wire v0.7.0 github.com/gorilla/websocket v1.5.3 - github.com/hashicorp/go-version v1.7.0 + github.com/hashicorp/go-version v1.9.0 github.com/hashicorp/golang-lru/v2 v2.0.7 - github.com/jellydator/ttlcache/v3 v3.3.0 + github.com/jellydator/ttlcache/v3 v3.4.0 github.com/jxskiss/base62 v1.1.0 - github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 - github.com/livekit/mediatransportutil v0.0.0-20241220010243-a2bdee945564 - github.com/livekit/protocol v1.34.0 - github.com/livekit/psrpc v0.6.1-0.20250205181828-a0beed2e4126 - github.com/mackerelio/go-osstat v0.2.5 - github.com/magefile/mage v1.15.0 - github.com/maxbrunsfeld/counterfeiter/v6 v6.11.2 + github.com/livekit/mageutil v0.0.0-20250511045019-0f1ff63f7731 + github.com/livekit/mediatransportutil v0.0.0-20260608063931-a3417d38cda0 + github.com/livekit/protocol v1.46.7-0.20260605212020-c0615a2f6f84 + github.com/livekit/psrpc v0.7.2 + github.com/mackerelio/go-osstat v0.2.7 + github.com/magefile/mage v1.17.2 + github.com/maxbrunsfeld/counterfeiter/v6 v6.12.2 github.com/mitchellh/go-homedir v1.1.0 - github.com/olekukonko/tablewriter v0.0.5 - github.com/ory/dockertest/v3 v3.11.0 - github.com/pion/datachannel v1.5.10 - github.com/pion/dtls/v3 v3.0.4 - github.com/pion/ice/v4 v4.0.7 - github.com/pion/interceptor v0.1.37 - github.com/pion/rtcp v1.2.15 - github.com/pion/rtp v1.8.11 - github.com/pion/sctp v1.8.36 - github.com/pion/sdp/v3 v3.0.10 - github.com/pion/transport/v3 v3.0.7 - github.com/pion/turn/v4 v4.0.0 - github.com/pion/webrtc/v4 v4.0.11 + github.com/moby/moby/client v0.4.1 + github.com/olekukonko/tablewriter v1.1.4 + github.com/ory/dockertest/v4 v4.0.0 + github.com/pion/datachannel v1.6.0 + github.com/pion/dtls/v3 v3.1.3 + github.com/pion/ice/v4 v4.2.7 + github.com/pion/interceptor v0.1.45 + github.com/pion/rtcp v1.2.16 + github.com/pion/rtp v1.10.2 + github.com/pion/sctp v1.9.5 + github.com/pion/sdp/v3 v3.0.18 + github.com/pion/transport/v4 v4.0.2 + github.com/pion/turn/v5 v5.0.8 + github.com/pion/webrtc/v4 v4.2.11 github.com/pkg/errors v0.9.1 - github.com/prometheus/client_golang v1.20.5 - github.com/redis/go-redis/v9 v9.7.0 + github.com/prometheus/client_golang v1.23.2 + github.com/redis/go-redis/v9 v9.20.0 github.com/rs/cors v1.11.1 - github.com/stretchr/testify v1.10.0 + github.com/stretchr/testify v1.11.1 github.com/thoas/go-funk v0.9.3 + github.com/tomnomnom/linkheader v0.0.0-20250811210735-e5fe3b51442e github.com/twitchtv/twirp v8.1.3+incompatible - github.com/ua-parser/uap-go v0.0.0-20250126222208-a52596c19dff - github.com/urfave/cli/v2 v2.27.5 + github.com/ua-parser/uap-go v0.0.0-20260529044130-17c35e68e58c github.com/urfave/negroni/v3 v3.1.1 go.uber.org/atomic v1.11.0 go.uber.org/multierr v1.11.0 - go.uber.org/zap v1.27.0 - golang.org/x/exp v0.0.0-20250210185358-939b2ce775ac - golang.org/x/sync v0.11.0 - google.golang.org/protobuf v1.36.5 + go.uber.org/zap v1.28.0 + golang.org/x/mod v0.36.0 + golang.org/x/sync v0.20.0 + google.golang.org/protobuf v1.36.11 gopkg.in/yaml.v3 v3.0.1 ) require ( - buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.0-20241127180247-a33202765966.1 // indirect - buf.build/go/protoyaml v0.3.1 // indirect - cel.dev/expr v0.19.0 // indirect - dario.cat/mergo v1.0.0 // indirect - github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect + github.com/cenkalti/backoff/v5 v5.0.3 // indirect + github.com/cilium/ebpf v0.16.0 // indirect + github.com/clipperhouse/displaywidth v0.11.0 // indirect + github.com/clipperhouse/uax29/v2 v2.7.0 // indirect + github.com/containerd/errdefs v1.0.0 // indirect + github.com/containerd/errdefs/pkg v0.3.0 // indirect + github.com/distribution/reference v0.6.0 // indirect + github.com/fatih/color v1.19.0 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/goccy/go-json v0.10.6 // indirect + github.com/golang-jwt/jwt/v5 v5.3.1 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect + github.com/mattn/go-colorable v0.1.15 // indirect + github.com/mattn/go-isatty v0.0.22 // indirect + github.com/moby/moby/api v1.54.2 // indirect + github.com/nyaruka/phonenumbers v1.8.0 // indirect + github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6 // indirect + github.com/olekukonko/errors v1.3.0 // indirect + github.com/olekukonko/ll v0.1.8 // indirect + github.com/puzpuzpuz/xsync/v4 v4.5.0 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 // indirect + go.opentelemetry.io/otel v1.44.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.44.0 // indirect + go.opentelemetry.io/otel/metric v1.44.0 // indirect + go.opentelemetry.io/otel/sdk v1.44.0 // indirect + go.opentelemetry.io/otel/trace v1.44.0 // indirect + go.opentelemetry.io/proto/otlp v1.10.0 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/exp v0.0.0-20260603202125-055de637280b // indirect + golang.org/x/time v0.15.0 // indirect +) + +require ( + buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.11-20260415201107-50325440f8f2.1 // indirect + buf.build/go/protovalidate v1.2.0 // indirect + buf.build/go/protoyaml v0.7.0 // indirect + cel.dev/expr v0.25.2 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect - github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect - github.com/antlr4-go/antlr/v4 v4.13.0 // indirect + github.com/antlr4-go/antlr/v4 v4.13.1 // indirect github.com/benbjohnson/clock v1.3.5 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/bufbuild/protovalidate-go v0.8.0 // indirect - github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/containerd/continuity v0.4.3 // indirect - github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect - github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect - github.com/docker/cli v26.1.4+incompatible // indirect - github.com/docker/docker v27.1.1+incompatible // indirect - github.com/docker/go-connections v0.5.0 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/docker/go-connections v0.7.0 // indirect github.com/docker/go-units v0.5.0 // indirect - github.com/fsnotify/fsnotify v1.8.0 // indirect - github.com/go-jose/go-jose/v3 v3.0.3 // indirect - github.com/go-logr/logr v1.4.2 // indirect - github.com/gogo/protobuf v1.3.2 // indirect - github.com/google/cel-go v0.22.1 // indirect - github.com/google/go-cmp v0.6.0 // indirect - github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect + github.com/fsnotify/fsnotify v1.10.1 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/google/cel-go v0.28.1 // indirect + github.com/google/go-cmp v0.7.0 // indirect github.com/google/subcommands v1.2.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect - github.com/hashicorp/go-retryablehttp v0.7.7 // indirect - github.com/hashicorp/golang-lru v0.5.4 // indirect + github.com/hashicorp/go-retryablehttp v0.7.8 // indirect + github.com/hashicorp/golang-lru v1.0.2 // indirect github.com/josharian/native v1.1.0 // indirect - github.com/klauspost/compress v1.17.11 // indirect - github.com/klauspost/cpuid/v2 v2.2.6 // indirect + github.com/klauspost/compress v1.18.6 // indirect + github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/lithammer/shortuuid/v4 v4.2.0 // indirect - github.com/mattn/go-runewidth v0.0.9 // indirect - github.com/mdlayher/netlink v1.7.1 // indirect - github.com/mdlayher/socket v0.4.0 // indirect - github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/mattn/go-runewidth v0.0.24 // indirect + github.com/mdlayher/netlink v1.11.2 // indirect + github.com/mdlayher/socket v0.6.1 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect - github.com/moby/term v0.5.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/nats-io/nats.go v1.39.0 // indirect - github.com/nats-io/nkeys v0.4.10 // indirect + github.com/nats-io/nats.go v1.52.0 // indirect + github.com/nats-io/nkeys v0.4.16 // indirect github.com/nats-io/nuid v1.0.1 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/image-spec v1.1.0 // indirect - github.com/opencontainers/runc v1.1.14 // indirect - github.com/pion/logging v0.2.3 // indirect - github.com/pion/mdns/v2 v2.0.7 // indirect + github.com/opencontainers/image-spec v1.1.1 // indirect + github.com/pion/logging v0.2.4 // indirect + github.com/pion/mdns/v2 v2.1.0 // indirect github.com/pion/randutil v0.1.0 // indirect - github.com/pion/srtp/v3 v3.0.4 // indirect - github.com/pion/stun/v3 v3.0.0 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_model v0.6.1 // indirect - github.com/prometheus/common v0.55.0 // indirect - github.com/prometheus/procfs v0.15.1 // indirect - github.com/puzpuzpuz/xsync/v3 v3.5.0 // indirect - github.com/russross/blackfriday/v2 v2.1.0 // indirect - github.com/sirupsen/logrus v1.9.3 // indirect - github.com/stoewer/go-strcase v1.3.0 // indirect + github.com/pion/srtp/v3 v3.0.11 // indirect + github.com/pion/stun/v3 v3.1.4 + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.68.1 // indirect + github.com/prometheus/procfs v0.20.1 // indirect + github.com/urfave/cli/v3 v3.9.0 github.com/wlynxg/anet v0.0.5 // indirect - github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect - github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect - github.com/xeipuuv/gojsonschema v1.2.0 // indirect - github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect - github.com/zeebo/xxh3 v1.0.2 // indirect + github.com/zeebo/xxh3 v1.1.0 // indirect go.uber.org/zap/exp v0.3.0 // indirect - golang.org/x/crypto v0.33.0 // indirect - golang.org/x/mod v0.23.0 // indirect - golang.org/x/net v0.35.0 // indirect - golang.org/x/sys v0.30.0 // indirect - golang.org/x/text v0.22.0 // indirect - golang.org/x/tools v0.30.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20241202173237-19429a94021a // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20250212204824-5a70512c5d8b // indirect - google.golang.org/grpc v1.70.0 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect + golang.org/x/crypto v0.52.0 // indirect + golang.org/x/net v0.55.0 // indirect + golang.org/x/sys v0.45.0 // indirect + golang.org/x/text v0.37.0 // indirect + golang.org/x/tools v0.45.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect + google.golang.org/grpc v1.81.1 // indirect ) + +// for gst whipsink compatibility +// it is unclear what in pion/dtls v3.1.3 breaks compatibility, +// but both sides sends their first flights and do not make further progress +replace github.com/pion/dtls/v3 => github.com/pion/dtls/v3 v3.1.2 diff --git a/go.sum b/go.sum index c5c38ce..0e03e47 100644 --- a/go.sum +++ b/go.sum @@ -1,99 +1,92 @@ -buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.0-20241127180247-a33202765966.1 h1:ntAj16eF7AtUyzOOAFk5gvbAO52QmUKPKk7GmsIEORo= -buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.0-20241127180247-a33202765966.1/go.mod h1:AxRT+qTj5PJCz2nyQzsR/qxAcveW5USRhJTt/edTO5w= -buf.build/go/protoyaml v0.3.1 h1:ucyzE7DRnjX+mQ6AH4JzN0Kg50ByHHu+yrSKbgQn2D4= -buf.build/go/protoyaml v0.3.1/go.mod h1:0TzNpFQDXhwbkXb/ajLvxIijqbve+vMQvWY/b3/Dzxg= -cel.dev/expr v0.19.0 h1:lXuo+nDhpyJSpWxpPVi5cPUwzKb+dsdOiw6IreM5yt0= -cel.dev/expr v0.19.0/go.mod h1:MrpN08Q+lEBs+bGYdLxxHkZoUSsCp0nSKTs0nTymJgw= -dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= -dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= -filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= -filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= -github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= -github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.11-20260415201107-50325440f8f2.1 h1:s6hzCXtND/ICdGPTMGk7C+/BFlr2Jg5GyH0NKf4XGXg= +buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.11-20260415201107-50325440f8f2.1/go.mod h1:tvtbpgaVXZX4g6Pn+AnzFycuRK3MOz5HJfEGeEllXYM= +buf.build/go/protovalidate v1.2.0 h1:DQVrUWkmGTBij+kOYv/x2LLxwcLaGKMdzShj1/6/3H0= +buf.build/go/protovalidate v1.2.0/go.mod h1:7rYiQEhqvAipoazpVNBBH2S2f8bjG4huMVy1V2Yofn4= +buf.build/go/protoyaml v0.7.0 h1:z4oVoFicbpPefhT7WAykxUdfp0yEQlhMQ2mCZOY5V38= +buf.build/go/protoyaml v0.7.0/go.mod h1:+a0cavd0uMvirb87xdu2ZMMmjlIQoiH/N2Ich5MGSQ0= +cel.dev/expr v0.25.2 h1:K6j46C81hXtZQfuX60cVWQFBJahKSE2gfRbNuvr5bFs= +cel.dev/expr v0.25.2/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= -github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw= -github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk= -github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= -github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= +github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= +github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o= github.com/benbjohnson/clock v1.3.5/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bep/debounce v1.2.1 h1:v67fRdBA9UQu2NhLFXrSg0Brw7CexQekrBwDMM8bzeY= github.com/bep/debounce v1.2.1/go.mod h1:H8yggRPQKLUhUoqrJC1bO2xNya7vanpDl7xR3ISbCJ0= +github.com/brianvoe/gofakeit/v6 v6.28.0 h1:Xib46XXuQfmlLS2EXRuJpqcw8St6qSZz75OUo0tgAW4= +github.com/brianvoe/gofakeit/v6 v6.28.0/go.mod h1:Xj58BMSnFqcn/fAQeSK+/PLtC5kSb7FJIq4JyGa8vEs= github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= -github.com/bufbuild/protovalidate-go v0.8.0 h1:Xs3kCLCJ4tQiogJ0iOXm+ClKw/KviW3nLAryCGW2I3Y= -github.com/bufbuild/protovalidate-go v0.8.0/go.mod h1:JPWZInGm2y2NBg3vKDKdDIkvDjyLv31J3hLH5GIFc/Q= -github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= -github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cilium/ebpf v0.5.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA= -github.com/cilium/ebpf v0.8.1 h1:bLSSEbBLqGPXxls55pGr5qWZaTqcmfDJHhou7t254ao= github.com/cilium/ebpf v0.8.1/go.mod h1:f5zLIM0FSNuAkSyLAN7X+Hy6yznlF1mNiWUMfxMtrgk= -github.com/containerd/continuity v0.4.3 h1:6HVkalIp+2u1ZLH1J/pYX2oBVXlJZvh1X1A7bEZ9Su8= -github.com/containerd/continuity v0.4.3/go.mod h1:F6PTNCKepoxEaXLQp3wDAjygEnImnZ/7o4JzpodfroQ= -github.com/cpuguy83/go-md2man/v2 v2.0.5 h1:ZtcqGrnekaHpVLArFSe4HK5DoKx1T0rq2DwVB0alcyc= -github.com/cpuguy83/go-md2man/v2 v2.0.5/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cilium/ebpf v0.16.0 h1:+BiEnHL6Z7lXnlGUsXQPPAE7+kenAd4ES8MQ5min0Ok= +github.com/cilium/ebpf v0.16.0/go.mod h1:L7u2Blt2jMM/vLAVgjxluxtBKlz3/GWjB0dMOEngfwE= +github.com/clipperhouse/displaywidth v0.11.0 h1:lBc6kY44VFw+TDx4I8opi/EtL9m20WSEFgwIwO+UVM8= +github.com/clipperhouse/displaywidth v0.11.0/go.mod h1:bkrFNkf81G8HyVqmKGxsPufD3JhNl3dSqnGhOoSD/o0= +github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk= +github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= +github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= +github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= +github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= +github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= -github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/d5/tengo/v2 v2.17.0 h1:BWUN9NoJzw48jZKiYDXDIF3QrIVZRm1uV1gTzeZ2lqM= github.com/d5/tengo/v2 v2.17.0/go.mod h1:XRGjEs5I9jYIKTxly6HCF8oiiilk5E/RYXOZ5b0DZC8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dennwc/iters v1.0.1 h1:XwMudE6xtS0ugEdum4HQ+iRi+5HSvaeKxJPM/VI3pJs= -github.com/dennwc/iters v1.0.1/go.mod h1:M9KuuMBeyEXYTmB7EnI9SCyALFCmPWOIxn5W1L0CjGg= -github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= -github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= -github.com/docker/cli v26.1.4+incompatible h1:I8PHdc0MtxEADqYJZvhBrW9bo8gawKwwenxRM7/rLu8= -github.com/docker/cli v26.1.4+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= -github.com/docker/docker v27.1.1+incompatible h1:hO/M4MtV36kzKldqnA37IWhebRA+LnqqcqDja6kVaKY= -github.com/docker/docker v27.1.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= -github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dennwc/iters v1.2.2 h1:XH2/Etihiy9ZvPOVCR+icQXeYlhbvS7k0qro4x/2qQo= +github.com/dennwc/iters v1.2.2/go.mod h1:M9KuuMBeyEXYTmB7EnI9SCyALFCmPWOIxn5W1L0CjGg= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/docker/go-connections v0.7.0 h1:6SsRfJddP22WMrCkj19x9WKjEDTB+ahsdiGYf0mN39c= +github.com/docker/go-connections v0.7.0/go.mod h1:no1qkHdjq7kLMGUXYAduOhYPSJxxvgWBh7ogVvptn3Q= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/elliotchance/orderedmap/v2 v2.7.0 h1:WHuf0DRo63uLnldCPp9ojm3gskYwEdIIfAUVG5KhoOc= -github.com/elliotchance/orderedmap/v2 v2.7.0/go.mod h1:85lZyVbpGaGvHvnKa7Qhx7zncAdBIBq6u56Hb1PRU5Q= -github.com/envoyproxy/protoc-gen-validate v1.1.0 h1:tntQDh69XqOCOZsDz0lVJQez/2L6Uu2PdjCQwWCJ3bM= -github.com/envoyproxy/protoc-gen-validate v1.1.0/go.mod h1:sXRDRVmzEbkM7CVcM06s9shE/m23dg3wzjl0UWqJ2q4= -github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= -github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= -github.com/florianl/go-tc v0.4.4 h1:q6lhEWEfyhGffRzdl3eIcNqX/yVIw0IJwXqa9Rdcctw= -github.com/florianl/go-tc v0.4.4/go.mod h1:uvp6pIlOw7Z8hhfnT5M4+V1hHVgZWRZwwMS8Z0JsRxc= +github.com/elliotchance/orderedmap/v3 v3.1.0 h1:j4DJ5ObEmMBt/lcwIecKcoRxIQUEnw0L804lXYDt/pg= +github.com/elliotchance/orderedmap/v3 v3.1.0/go.mod h1:G+Hc2RwaZvJMcS4JpGCOyViCnGeKf0bTYCGTO4uhjSo= +github.com/fatih/color v1.19.0 h1:Zp3PiM21/9Ld6FzSKyL5c/BULoe/ONr9KlbYVOfG8+w= +github.com/fatih/color v1.19.0/go.mod h1:zNk67I0ZUT1bEGsSGyCZYZNrHuTkJJB+r6Q9VuMi0LE= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/florianl/go-tc v0.4.8 h1:hgmakUX1Nm0Ba1I0ZkbUl9CH6HbRwqSiwipnpmYp3Es= +github.com/florianl/go-tc v0.4.8/go.mod h1:B8GeOEnmrbOnxZtaCvsYJcgIzzmM8c/AIhtfCZsDj3Q= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= github.com/frostbyte73/core v0.1.1 h1:ChhJOR7bAKOCPbA+lqDLE2cGKlCG5JXsDvvQr4YaJIA= github.com/frostbyte73/core v0.1.1/go.mod h1:mhfOtR+xWAvwXiwor7jnqPMnu4fxbv1F2MwZ0BEpzZo= -github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= -github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= -github.com/gammazero/deque v1.0.0 h1:LTmimT8H7bXkkCy6gZX7zNLtkbz4NdS2z8LZuor3j34= -github.com/gammazero/deque v1.0.0/go.mod h1:iflpYvtGfM3U8S8j+sZEKIak3SAKYpA5/SQewgfXDKo= -github.com/gammazero/workerpool v1.1.3 h1:WixN4xzukFoN0XSeXF6puqEqFTl2mECI9S6W44HWy9Q= -github.com/gammazero/workerpool v1.1.3/go.mod h1:wPjyBLDbyKnUn2XwwyD3EEwo9dHutia9/fwNmSHWACc= -github.com/go-jose/go-jose/v3 v3.0.3 h1:fFKWeig/irsp7XD2zBxvnmA/XaRWp5V3CBsZXJF7G7k= -github.com/go-jose/go-jose/v3 v3.0.3/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQrVfLAMboGkQ= -github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= -github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho= +github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo= +github.com/gammazero/deque v1.2.1 h1:9fnQVFCCZ9/NOc7ccTNqzoKd1tCWOqeI05/lPqFPMGQ= +github.com/gammazero/deque v1.2.1/go.mod h1:5nSFkzVm+afG9+gy0VIowlqVAW4N8zNcMne+CMQVD2g= +github.com/gammazero/workerpool v1.2.1 h1:MEDvUJsNYGuCvl1RwIXNKu2YtQtHqCSF9XWF04N7lqs= +github.com/gammazero/workerpool v1.2.1/go.mod h1:E32GVRUanF4d6QtRmdss3AScgaDkIyrvPtgRQUWgmx4= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= -github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/goccy/go-json v0.10.6 h1:p8HrPJzOakx/mn/bQtjgNjdTcN+/S6FcG2CTtQOrHVU= +github.com/goccy/go-json v0.10.6/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= +github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/cel-go v0.22.1 h1:AfVXx3chM2qwoSbM7Da8g8hX8OVSkBFwX+rz2+PcK40= -github.com/google/cel-go v0.22.1/go.mod h1:BuznPXXfQDpXKWQ9sPW3TzlAJN5zzFe+i9tIs0yC4s8= +github.com/google/cel-go v0.28.1 h1:YWIwi77J4xIsYUwAF/iIuS6haffzIHS8yWI8glSbLWM= +github.com/google/cel-go v0.28.1/go.mod h1:X0bD6iVNR8pkROSOoHVdgTkzmRcosof7WQqCD6wcMc8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -103,32 +96,33 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= -github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/subcommands v1.2.0 h1:vWQspBTo2nEqTUFita5/KeEWlUL8kQObDFbub/EN9oE= github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/wire v0.6.0 h1:HBkoIh4BdSxoyo9PveV8giw7ZsaBOvzWKfcg/6MrVwI= -github.com/google/wire v0.6.0/go.mod h1:F4QhpQ9EDIdJ1Mbop/NZBRB+5yrR6qg3BnctaoUk6NA= +github.com/google/wire v0.7.0 h1:JxUKI6+CVBgCO2WToKy/nQk0sS+amI9z9EjVmdaocj4= +github.com/google/wire v0.7.0/go.mod h1:n6YbUQD9cPKTnHXEBN2DXlOp/mVADhVErcMFb0v3J18= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF28c5ZQfqCBQ5g2xfk= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= -github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU= -github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk= -github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY= -github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= -github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= -github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/go-retryablehttp v0.7.8 h1:ylXZWnqa7Lhqpk0L1P1LzDtGcCR0rPVUrx/c8Unxc48= +github.com/hashicorp/go-retryablehttp v0.7.8/go.mod h1:rjiScheydd+CxvumBsIrFKlx3iS0jrZ7LvzFGFmuKbw= +github.com/hashicorp/go-version v1.9.0 h1:CeOIz6k+LoN3qX9Z0tyQrPtiB1DFYRPfCIBtaXPSCnA= +github.com/hashicorp/go-version v1.9.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= +github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= -github.com/jellydator/ttlcache/v3 v3.3.0 h1:BdoC9cE81qXfrxeb9eoJi9dWrdhSuwXMAnHTbnBm4Wc= -github.com/jellydator/ttlcache/v3 v3.3.0/go.mod h1:bj2/e0l4jRnQdrnSTaGTsh4GSXvMjQcy41i7th0GVGw= +github.com/jellydator/ttlcache/v3 v3.4.0 h1:YS4P125qQS0tNhtL6aeYkheEaB/m8HCqdMMP4mnWdTY= +github.com/jellydator/ttlcache/v3 v3.4.0/go.mod h1:Hw9EgjymziQD3yGsQdf1FqFdpp7YjFMd4Srg5EJlgD4= github.com/josharian/native v0.0.0-20200817173448-b6b71def0850/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= github.com/josharian/native v1.0.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= github.com/josharian/native v1.1.0 h1:uuaP0hAbW7Y4l0ZRQ6C9zfb7Mg1mbFKry/xzDAfmtLA= @@ -145,12 +139,10 @@ github.com/jsimonetti/rtnetlink v0.0.0-20211022192332-93da33804786 h1:N527AHMa79 github.com/jsimonetti/rtnetlink v0.0.0-20211022192332-93da33804786/go.mod h1:v4hqbTdfQngbVSZJVWUhGE/lbTFf9jb+ygmNUDQMuOs= github.com/jxskiss/base62 v1.1.0 h1:A5zbF8v8WXx2xixnAKD2w+abC+sIzYJX+nxmhA6HWFw= github.com/jxskiss/base62 v1.1.0/go.mod h1:HhWAlUXvxKThfOlZbcuFzsqwtF5TcqS9ru3y5GfjWAc= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= -github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= -github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc= -github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao= +github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= +github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= @@ -162,30 +154,28 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= -github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lithammer/shortuuid/v4 v4.2.0 h1:LMFOzVB3996a7b8aBuEXxqOBflbfPQAiVzkIcHO0h8c= github.com/lithammer/shortuuid/v4 v4.2.0/go.mod h1:D5noHZ2oFw/YaKCfGy0YxyE7M0wMbezmMjPdhyEFe6Y= -github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkDaKb5iXdynYrzB84ErPPO4LbRASk58= -github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= -github.com/livekit/mediatransportutil v0.0.0-20241220010243-a2bdee945564 h1:GX7KF/V9ExmcfT/2Bdia8aROjkxrgx7WpyH7w9MB4J4= -github.com/livekit/mediatransportutil v0.0.0-20241220010243-a2bdee945564/go.mod h1:36s+wwmU3O40IAhE+MjBWP3W71QRiEE9SfooSBvtBqY= -github.com/livekit/protocol v1.34.0 h1:hbIXgNW+JPiTcGjzNg1XgQg3Wqa2R5dBhzuy+LLEIS4= -github.com/livekit/protocol v1.34.0/go.mod h1:yXuQ7ucrLj91nbxL6/AHgtxdha1DGzLj1LkgvnT90So= -github.com/livekit/psrpc v0.6.1-0.20250205181828-a0beed2e4126 h1:fzuYpAQbCid7ySPpQWWePfQOWUrs8x6dJ0T3Wl07n+Y= -github.com/livekit/psrpc v0.6.1-0.20250205181828-a0beed2e4126/go.mod h1:X5WtEZ7OnEs72Fi5/J+i0on3964F1aynQpCalcgMqRo= -github.com/mackerelio/go-osstat v0.2.5 h1:+MqTbZUhoIt4m8qzkVoXUJg1EuifwlAJSk4Yl2GXh+o= -github.com/mackerelio/go-osstat v0.2.5/go.mod h1:atxwWF+POUZcdtR1wnsUcQxTytoHG4uhl2AKKzrOajY= -github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg= -github.com/magefile/mage v1.15.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= -github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= -github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= -github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= -github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= -github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= -github.com/maxbrunsfeld/counterfeiter/v6 v6.11.2 h1:yVCLo4+ACVroOEr4iFU1iH46Ldlzz2rTuu18Ra7M8sU= -github.com/maxbrunsfeld/counterfeiter/v6 v6.11.2/go.mod h1:VzB2VoMh1Y32/QqDfg9ZJYHj99oM4LiGtqPZydTiQSQ= +github.com/livekit/mageutil v0.0.0-20250511045019-0f1ff63f7731 h1:9x+U2HGLrSw5ATTo469PQPkqzdoU7be46ryiCDO3boc= +github.com/livekit/mageutil v0.0.0-20250511045019-0f1ff63f7731/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= +github.com/livekit/mediatransportutil v0.0.0-20260608063931-a3417d38cda0 h1:XHNNzebIKZRkLimla/hFGrAIX5EMWHctrgt3hLw7s+I= +github.com/livekit/mediatransportutil v0.0.0-20260608063931-a3417d38cda0/go.mod h1:o8CFmAdrVwzJNOCsQCLUzXRjokkufNshnQHOe4fRaqU= +github.com/livekit/protocol v1.46.7-0.20260605212020-c0615a2f6f84 h1:dkHHthyor9dwxxdBmbeG1ZUI4bPHpTEk9DjYJSSSIl4= +github.com/livekit/protocol v1.46.7-0.20260605212020-c0615a2f6f84/go.mod h1:jO+y05AU9Ec4JswDyuzKCZ4bhziOS0CzMqgnbj60Dzs= +github.com/livekit/psrpc v0.7.2 h1:6oZ+NODJ2pLyaT6VqDq1F4Qc/3TpDUSpyphj/P9MhQc= +github.com/livekit/psrpc v0.7.2/go.mod h1:rAI+m2+/cb4x9RXhLRtUx5ZwdfjjXOl4zi46IjEetaw= +github.com/mackerelio/go-osstat v0.2.7 h1:TCavZi10wF49bT6iQZ9eT2keGZQpC69MTDfdJej5e94= +github.com/mackerelio/go-osstat v0.2.7/go.mod h1:dwpYh5pIPmvk+IEwBKNIWRFMB92mrC08CmXOhDC7nQk= +github.com/magefile/mage v1.17.2 h1:fyXVu1eadI8Ap1HCCNgEhJ5McIWiYhLR8uol64ZZc40= +github.com/magefile/mage v1.17.2/go.mod h1:Yj51kqllmsgFpvvSzgrZPK9WtluG3kUhFaBUVLo4feA= +github.com/mattn/go-colorable v0.1.15 h1:+u9SLTRGnXv73cEsnsmoZBom+dMU88B2M0aDcWy0/jY= +github.com/mattn/go-colorable v0.1.15/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= +github.com/mattn/go-isatty v0.0.22 h1:j8l17JJ9i6VGPUFUYoTUKPSgKe/83EYU2zBC7YNKMw4= +github.com/mattn/go-isatty v0.0.22/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4= +github.com/mattn/go-runewidth v0.0.24 h1:cpokDiIn0MGnhdHwuWnJBITySJ20QyNGnY2kR/ay2DU= +github.com/mattn/go-runewidth v0.0.24/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= +github.com/maxbrunsfeld/counterfeiter/v6 v6.12.2 h1:V23nK2R2B63g2GhygF9zVGpnigmhvoZoH8d0hrZwMGY= +github.com/maxbrunsfeld/counterfeiter/v6 v6.12.2/go.mod h1:Mr897yU9FmyKaQDPtRlVKibrjz40XXyOHUfyZBPSyZU= github.com/mdlayher/ethtool v0.0.0-20210210192532-2b88debcdd43/go.mod h1:+t7E0lkKfbBsebllff1xdTmyJt8lH37niI6kwFk9OTo= github.com/mdlayher/genetlink v1.0.0/go.mod h1:0rJ0h4itni50A86M2kHcgS85ttZazNt7a8H2a2cw0Gc= github.com/mdlayher/netlink v0.0.0-20190409211403-11939a169225/go.mod h1:eQB3mZE4aiYnlUsyGGCOpPETfdQq4Jhsgf1fk3cwQaA= @@ -198,216 +188,195 @@ github.com/mdlayher/netlink v1.2.2-0.20210123213345-5cc92139ae3e/go.mod h1:bacnN github.com/mdlayher/netlink v1.3.0/go.mod h1:xK/BssKuwcRXHrtN04UBkwQ6dY9VviGGuriDdoPSWys= github.com/mdlayher/netlink v1.4.0/go.mod h1:dRJi5IABcZpBD2A3D0Mv/AiX8I9uDEu5oGkAVrekmf8= github.com/mdlayher/netlink v1.4.1/go.mod h1:e4/KuJ+s8UhfUpO9z00/fDZZmhSrs+oxyqAS9cNgn6Q= -github.com/mdlayher/netlink v1.6.0/go.mod h1:0o3PlBmGst1xve7wQ7j/hwpNaFaH4qCRyWCdcZk8/vA= -github.com/mdlayher/netlink v1.7.1 h1:FdUaT/e33HjEXagwELR8R3/KL1Fq5x3G5jgHLp/BTmg= -github.com/mdlayher/netlink v1.7.1/go.mod h1:nKO5CSjE/DJjVhk/TNp6vCE1ktVxEA8VEh8drhZzxsQ= +github.com/mdlayher/netlink v1.6.2/go.mod h1:O1HXX2sIWSMJ3Qn1BYZk1yZM+7iMki/uYGGiwGyq/iU= +github.com/mdlayher/netlink v1.11.2 h1:HKh2jqe+omdSWcQ88nrT7INE61B0NXfiSPFdgL4YbNI= +github.com/mdlayher/netlink v1.11.2/go.mod h1:uT2Yc/QLaZubzDpZIBi9d4GoeLwtp3x1AMeqSRrK2sA= github.com/mdlayher/socket v0.0.0-20210307095302-262dc9984e00/go.mod h1:GAFlyu4/XV68LkQKYzKhIo/WW7j3Zi0YRAz/BOoanUc= -github.com/mdlayher/socket v0.1.1/go.mod h1:mYV5YIZAfHh4dzDVzI8x8tWLWCliuX8Mon5Awbj+qDs= -github.com/mdlayher/socket v0.4.0 h1:280wsy40IC9M9q1uPGcLBwXpcTQDtoGwVt+BNoITxIw= -github.com/mdlayher/socket v0.4.0/go.mod h1:xxFqz5GRCUN3UEOm9CZqEJsAbe1C8OwSK46NlmWuVoc= +github.com/mdlayher/socket v0.2.3/go.mod h1:bz12/FozYNH/VbvC3q7TRIK/Y6dH1kCKsXaUeXi/FmY= +github.com/mdlayher/socket v0.6.1 h1:M7uj2NtuujUY4mYr1C57NmfNiRHbkKpnBxO856lsc3A= +github.com/mdlayher/socket v0.6.1/go.mod h1:+/SGtqc9V+5dAuRgQsU0fGBI+oRDiW7O2Obx10OIWfg= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= -github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= -github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= -github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= +github.com/moby/moby/api v1.54.2 h1:wiat9QAhnDQjA7wk1kh/TqHz2I1uUA7M7t9SAl/JNXg= +github.com/moby/moby/api v1.54.2/go.mod h1:+RQ6wluLwtYaTd1WnPLykIDPekkuyD/ROWQClE83pzs= +github.com/moby/moby/client v0.4.1 h1:DMQgisVoMkmMs7fp3ROSdiBnoAu8+vo3GggFl06M/wY= +github.com/moby/moby/client v0.4.1/go.mod h1:z52C9O2POPOsnxZAy//WtKcQ32P+jT/NGeXu/7nfjGQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/nats-io/nats.go v1.39.0 h1:2/yg2JQjiYYKLwDuBzV0FbB2sIV+eFNkEevlRi4n9lI= -github.com/nats-io/nats.go v1.39.0/go.mod h1:MgRb8oOdigA6cYpEPhXJuRVH6UE/V4jblJ2jQ27IXYM= -github.com/nats-io/nkeys v0.4.10 h1:glmRrpCmYLHByYcePvnTBEAwawwapjCPMjy2huw20wc= -github.com/nats-io/nkeys v0.4.10/go.mod h1:OjRrnIKnWBFl+s4YK5ChQfvHP2fxqZexrKJoVVyWB3U= +github.com/nats-io/nats.go v1.52.0 h1:n3avV4VBsCgsdwh71TppsTwtv+QdPs7ntSKM8qJLGsc= +github.com/nats-io/nats.go v1.52.0/go.mod h1:26HypzazeOkyO3/mqd1zZd53STJN0EjCYF9Uy2ZOBno= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= -github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= -github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= -github.com/onsi/gomega v1.36.1 h1:bJDPBO7ibjxcbHMgSCoo4Yj18UWbKDlLwX1x9sybDcw= -github.com/onsi/gomega v1.36.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= +github.com/nyaruka/phonenumbers v1.8.0 h1:TrXNJmbwcAHajzDqin3mLWw57vqLUA6ZjVdeNds0heQ= +github.com/nyaruka/phonenumbers v1.8.0/go.mod h1:fsKPJ70O9JetEA4ggnJadYTFWwtGPvu/lETTXNXq6Cs= +github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6 h1:zrbMGy9YXpIeTnGj4EljqMiZsIcE09mmF8XsD5AYOJc= +github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6/go.mod h1:rEKTHC9roVVicUIfZK7DYrdIoM0EOr8mK1Hj5s3JjH0= +github.com/olekukonko/errors v1.3.0 h1:teJvgLGUEqMzBUms+Dj3/3szNqCG/Jdw9iDbum8fR6U= +github.com/olekukonko/errors v1.3.0/go.mod h1:ppzxA5jBKcO1vIpCXQ9ZqgDh8iwODz6OXIGKU8r5m4Y= +github.com/olekukonko/ll v0.1.8 h1:ysHCJRGHYKzmBSdz9w5AySztx7lG8SQY+naTGYUbsz8= +github.com/olekukonko/ll v0.1.8/go.mod h1:RPRC6UcscfFZgjo1nulkfMH5IM0QAYim0LfnMvUuozw= +github.com/olekukonko/tablewriter v1.1.4 h1:ORUMI3dXbMnRlRggJX3+q7OzQFDdvgbN9nVWj1drm6I= +github.com/olekukonko/tablewriter v1.1.4/go.mod h1:+kedxuyTtgoZLwif3P1Em4hARJs+mVnzKxmsCL/C5RY= +github.com/onsi/gomega v1.39.1 h1:1IJLAad4zjPn2PsnhH70V4DKRFlrCzGBNrNaru+Vf28= +github.com/onsi/gomega v1.39.1/go.mod h1:hL6yVALoTOxeWudERyfppUcZXjMwIMLnuSfruD2lcfg= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= -github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= -github.com/opencontainers/runc v1.1.14 h1:rgSuzbmgz5DUJjeSnw337TxDbRuqjs6iqQck/2weR6w= -github.com/opencontainers/runc v1.1.14/go.mod h1:E4C2z+7BxR7GHXp0hAY53mek+x49X1LjPNeMTfRGvOA= -github.com/ory/dockertest/v3 v3.11.0 h1:OiHcxKAvSDUwsEVh2BjxQQc/5EHz9n0va9awCtNGuyA= -github.com/ory/dockertest/v3 v3.11.0/go.mod h1:VIPxS1gwT9NpPOrfD3rACs8Y9Z7yhzO4SB194iUDnUI= -github.com/pion/datachannel v1.5.10 h1:ly0Q26K1i6ZkGf42W7D4hQYR90pZwzFOjTq5AuCKk4o= -github.com/pion/datachannel v1.5.10/go.mod h1:p/jJfC9arb29W7WrxyKbepTU20CFgyx5oLo8Rs4Py/M= -github.com/pion/dtls/v3 v3.0.4 h1:44CZekewMzfrn9pmGrj5BNnTMDCFwr+6sLH+cCuLM7U= -github.com/pion/dtls/v3 v3.0.4/go.mod h1:R373CsjxWqNPf6MEkfdy3aSe9niZvL/JaKlGeFphtMg= -github.com/pion/ice/v4 v4.0.7 h1:mnwuT3n3RE/9va41/9QJqN5+Bhc0H/x/ZyiVlWMw35M= -github.com/pion/ice/v4 v4.0.7/go.mod h1:y3M18aPhIxLlcO/4dn9X8LzLLSma84cx6emMSu14FGw= -github.com/pion/interceptor v0.1.37 h1:aRA8Zpab/wE7/c0O3fh1PqY0AJI3fCSEM5lRWJVorwI= -github.com/pion/interceptor v0.1.37/go.mod h1:JzxbJ4umVTlZAf+/utHzNesY8tmRkM2lVmkS82TTj8Y= -github.com/pion/logging v0.2.3 h1:gHuf0zpoh1GW67Nr6Gj4cv5Z9ZscU7g/EaoC/Ke/igI= -github.com/pion/logging v0.2.3/go.mod h1:z8YfknkquMe1csOrxK5kc+5/ZPAzMxbKLX5aXpbpC90= -github.com/pion/mdns/v2 v2.0.7 h1:c9kM8ewCgjslaAmicYMFQIde2H9/lrZpjBkN8VwoVtM= -github.com/pion/mdns/v2 v2.0.7/go.mod h1:vAdSYNAT0Jy3Ru0zl2YiW3Rm/fJCwIeM0nToenfOJKA= +github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= +github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= +github.com/ory/dockertest/v4 v4.0.0 h1:i19aFsO/VXE0VrMk4ifnKW4G/KIJ93PCjLOslxXoPME= +github.com/ory/dockertest/v4 v4.0.0/go.mod h1:b5Ofu8VIxWNhXFvQcLu17pRNQdoUBKtXBW74G4Ygzx8= +github.com/pion/datachannel v1.6.0 h1:XecBlj+cvsxhAMZWFfFcPyUaDZtd7IJvrXqlXD/53i0= +github.com/pion/datachannel v1.6.0/go.mod h1:ur+wzYF8mWdC+Mkis5Thosk+u/VOL287apDNEbFpsIk= +github.com/pion/dtls/v3 v3.1.2 h1:gqEdOUXLtCGW+afsBLO0LtDD8GnuBBjEy6HRtyofZTc= +github.com/pion/dtls/v3 v3.1.2/go.mod h1:Hw/igcX4pdY69z1Hgv5x7wJFrUkdgHwAn/Q/uo7YHRo= +github.com/pion/ice/v4 v4.2.7 h1:zDEbC6MiEdhQpF8TxBOTws+NU6ZgGpveHrQq4Lc1kao= +github.com/pion/ice/v4 v4.2.7/go.mod h1:9SNPaq0c7El/ki8leJzyCkK10zsskprR3zTNbO3monY= +github.com/pion/interceptor v0.1.45 h1:6PUo/5829bIfRFIPPJQzuDn8EjxRTSB/CSD7QVCOaqo= +github.com/pion/interceptor v0.1.45/go.mod h1:gNDYM/uFKcLe/B3gS2/7+aw6z+RDiMy2qKTnF1LO31w= +github.com/pion/logging v0.2.4 h1:tTew+7cmQ+Mc1pTBLKH2puKsOvhm32dROumOZ655zB8= +github.com/pion/logging v0.2.4/go.mod h1:DffhXTKYdNZU+KtJ5pyQDjvOAh/GsNSyv1lbkFbe3so= +github.com/pion/mdns/v2 v2.1.0 h1:3IJ9+Xio6tWYjhN6WwuY142P/1jA0D5ERaIqawg/fOY= +github.com/pion/mdns/v2 v2.1.0/go.mod h1:pcez23GdynwcfRU1977qKU0mDxSeucttSHbCSfFOd9A= github.com/pion/randutil v0.1.0 h1:CFG1UdESneORglEsnimhUjf33Rwjubwj6xfiOXBa3mA= github.com/pion/randutil v0.1.0/go.mod h1:XcJrSMMbbMRhASFVOlj/5hQial/Y8oH/HVo7TBZq+j8= -github.com/pion/rtcp v1.2.15 h1:LZQi2JbdipLOj4eBjK4wlVoQWfrZbh3Q6eHtWtJBZBo= -github.com/pion/rtcp v1.2.15/go.mod h1:jlGuAjHMEXwMUHK78RgX0UmEJFV4zUKOFHR7OP+D3D0= -github.com/pion/rtp v1.8.11 h1:17xjnY5WO5hgO6SD3/NTIUPvSFw/PbLsIJyz1r1yNIk= -github.com/pion/rtp v1.8.11/go.mod h1:8uMBJj32Pa1wwx8Fuv/AsFhn8jsgw+3rUC2PfoBZ8p4= -github.com/pion/sctp v1.8.36 h1:owNudmnz1xmhfYje5L/FCav3V9wpPRePHle3Zi+P+M0= -github.com/pion/sctp v1.8.36/go.mod h1:cNiLdchXra8fHQwmIoqw0MbLLMs+f7uQ+dGMG2gWebE= -github.com/pion/sdp/v3 v3.0.10 h1:6MChLE/1xYB+CjumMw+gZ9ufp2DPApuVSnDT8t5MIgA= -github.com/pion/sdp/v3 v3.0.10/go.mod h1:88GMahN5xnScv1hIMTqLdu/cOcUkj6a9ytbncwMCq2E= -github.com/pion/srtp/v3 v3.0.4 h1:2Z6vDVxzrX3UHEgrUyIGM4rRouoC7v+NiF1IHtp9B5M= -github.com/pion/srtp/v3 v3.0.4/go.mod h1:1Jx3FwDoxpRaTh1oRV8A/6G1BnFL+QI82eK4ms8EEJQ= -github.com/pion/stun/v3 v3.0.0 h1:4h1gwhWLWuZWOJIJR9s2ferRO+W3zA/b6ijOI6mKzUw= -github.com/pion/stun/v3 v3.0.0/go.mod h1:HvCN8txt8mwi4FBvS3EmDghW6aQJ24T+y+1TKjB5jyU= -github.com/pion/transport/v3 v3.0.7 h1:iRbMH05BzSNwhILHoBoAPxoB9xQgOaJk+591KC9P1o0= -github.com/pion/transport/v3 v3.0.7/go.mod h1:YleKiTZ4vqNxVwh77Z0zytYi7rXHl7j6uPLGhhz9rwo= -github.com/pion/turn/v4 v4.0.0 h1:qxplo3Rxa9Yg1xXDxxH8xaqcyGUtbHYw4QSCvmFWvhM= -github.com/pion/turn/v4 v4.0.0/go.mod h1:MuPDkm15nYSklKpN8vWJ9W2M0PlyQZqYt1McGuxG7mA= -github.com/pion/webrtc/v4 v4.0.11 h1:0i7BNFH2n8LVp08q/dqM5iyZBXW4TITbD1+RwNqk/iY= -github.com/pion/webrtc/v4 v4.0.11/go.mod h1:C+5JA7KiyLyoKyGh7hVFD/HCAon3IB/tfniocpZ9JoU= +github.com/pion/rtcp v1.2.16 h1:fk1B1dNW4hsI78XUCljZJlC4kZOPk67mNRuQ0fcEkSo= +github.com/pion/rtcp v1.2.16/go.mod h1:/as7VKfYbs5NIb4h6muQ35kQF/J0ZVNz2Z3xKoCBYOo= +github.com/pion/rtp v1.10.2 h1:l+f6tTDcAH6xwepaAoW791ddhuYsJlqRATOzirO04Mo= +github.com/pion/rtp v1.10.2/go.mod h1:Au8fc6cEByy8RLTwKTQTEeQqDB/SJDxwL4mZuxYA5Pk= +github.com/pion/sctp v1.9.5 h1:QoSFB/drmAsmSeSFNQNI3xx010nW4HsycCZckRVWWag= +github.com/pion/sctp v1.9.5/go.mod h1:N20Dq6LY+JvJDAh9VVh1JELngb2rQ8dPgds5yBWiPgw= +github.com/pion/sdp/v3 v3.0.18 h1:l0bAXazKHpepazVdp+tPYnrsy9dfh7ZbT8DxesH5ZnI= +github.com/pion/sdp/v3 v3.0.18/go.mod h1:ZREGo6A9ZygQ9XkqAj5xYCQtQpif0i6Pa81HOiAdqQ8= +github.com/pion/srtp/v3 v3.0.11 h1:GiESUr54/K4UuPigfq/CvWUed80JenQAHXn0C2MQQIQ= +github.com/pion/srtp/v3 v3.0.11/go.mod h1:EeZOi/sd6glM1EXapg051gdNWO9yWT1YSsgQ4SlJkns= +github.com/pion/stun/v3 v3.1.4 h1:/7ZL0j0dmLroKOq4GfkyKQ6asByYqntwyHSp5sYLcGY= +github.com/pion/stun/v3 v3.1.4/go.mod h1:ET7PFiXo1nrD2ZNVpbEHDuT0kCPVXhKmyWdiePNMw/U= +github.com/pion/transport/v3 v3.1.1 h1:Tr684+fnnKlhPceU+ICdrw6KKkTms+5qHMgw6bIkYOM= +github.com/pion/transport/v3 v3.1.1/go.mod h1:+c2eewC5WJQHiAA46fkMMzoYZSuGzA/7E2FPrOYHctQ= +github.com/pion/transport/v4 v4.0.2 h1:ifYlPqNwsy6aKQ9y8yzxXlHae5431ZrH2avkD/Rn6Tk= +github.com/pion/transport/v4 v4.0.2/go.mod h1:06hFI+jCFcok2X2MekVufNZ/uzNZXivGBPfviSVcjgM= +github.com/pion/turn/v4 v4.1.4 h1:EU11yMXKIsK43FhcUnjLlrhE4nboHZq+TXBIi3QpcxQ= +github.com/pion/turn/v4 v4.1.4/go.mod h1:ES1DXVFKnOhuDkqn9hn5VJlSWmZPaRJLyBXoOeO/BmQ= +github.com/pion/turn/v5 v5.0.8 h1:pZUCtmwWCMkrRKqh/8pL3WoGADXBe0/lOPkN7oqFjK8= +github.com/pion/turn/v5 v5.0.8/go.mod h1:1VwvxElZaOdJU0liJ/WUSm/Tsh+n2OxS5ISSDxgOWxU= +github.com/pion/webrtc/v4 v4.2.11 h1:QUX1QZKlNIn4O7U5JxLPGP0sV5RTncZkzu9SPR3jVNU= +github.com/pion/webrtc/v4 v4.2.11/go.mod h1:s/rAiyy77GyRFrZMx+Ls6aua26dIBPudH8/ZHYbIRWY= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= -github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= -github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= -github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= -github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= -github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= -github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= -github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= -github.com/puzpuzpuz/xsync/v3 v3.5.0 h1:i+cMcpEDY1BkNm7lPDkCtE4oElsYLn+EKF8kAu2vXT4= -github.com/puzpuzpuz/xsync/v3 v3.5.0/go.mod h1:VjzYrABPabuM4KyBh1Ftq6u8nhwY5tBPKP9jpmh0nnA= -github.com/redis/go-redis/v9 v9.7.0 h1:HhLSs+B6O021gwzl+locl0zEDnyNkxMtf/Z3NNBMa9E= -github.com/redis/go-redis/v9 v9.7.0/go.mod h1:f6zhXITC7JUJIlPEiBOTXxJgPLdZcA93GewI7inzyWw= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.68.1 h1:omjRRl4QP4komogpXuhfeOiisQg7xdy8VM1UY+pStaY= +github.com/prometheus/common v0.68.1/go.mod h1:ZzL3f6u94qUxh9p+tJTrF+FvBS1XXbbRAZCQkytAL0Y= +github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc= +github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo= +github.com/puzpuzpuz/xsync/v4 v4.5.0 h1:vOSWu6b57/emh+L/Cw0BeQfvxa/cogFywXHeGUxQxAg= +github.com/puzpuzpuz/xsync/v4 v4.5.0/go.mod h1:VJDmTCJMBt8igNxnkQd86r+8KUeN1quSfNKu5bLYFQo= +github.com/redis/go-redis/v9 v9.20.0 h1:WnQYxLkgO2xiXTCJY0ldIiI8dNqCDlQAG+AtaH7a2a0= +github.com/redis/go-redis/v9 v9.20.0/go.mod h1:v/M13XI1PVCDcm01VtPFOADfZtHf8YW3baQf57KlIkA= +github.com/rodaine/protogofakeit v0.1.1 h1:ZKouljuRM3A+TArppfBqnH8tGZHOwM/pjvtXe9DaXH8= +github.com/rodaine/protogofakeit v0.1.1/go.mod h1:pXn/AstBYMaSfc1/RqH3N82pBuxtWgejz1AlYpY1mI0= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= -github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/rogpeppe/go-internal v1.15.0 h1:D0RCU5rMAp+SpgkiNdrjfJ+LX4J1M32V2NeCY7EJ6hc= +github.com/rogpeppe/go-internal v1.15.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs= github.com/rs/cors v1.11.1 h1:eU3gRzXLRK57F5rKMGMZURNdIG4EoAmX8k94r9wXWHA= github.com/rs/cors v1.11.1/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= -github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= -github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sclevine/spec v1.4.0 h1:z/Q9idDcay5m5irkZ28M7PtQM4aOISzOpj4bUPkDee8= github.com/sclevine/spec v1.4.0/go.mod h1:LvpgJaFyvQzRvc1kaDs0bulYwzC70PbiYjC4QnFHkOM= github.com/shoenig/test v1.7.0 h1:eWcHtTXa6QLnBvm0jgEabMRN/uJ4DMV3M8xUGgRkZmk= github.com/shoenig/test v1.7.0/go.mod h1:UxJ6u/x2v/TNs/LoLxBNJRV9DiwBBKYxXSyczsBHFoI= -github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= -github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs= -github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/thoas/go-funk v0.9.3 h1:7+nAEx3kn5ZJcnDm2Bh23N2yOtweO14bi//dvRtgLpw= github.com/thoas/go-funk v0.9.3/go.mod h1:+IWnUfUmFO1+WVYQWQtIJHeRRdaIyyYglZN7xzUPe4Q= +github.com/tomnomnom/linkheader v0.0.0-20250811210735-e5fe3b51442e h1:tD38/4xg4nuQCASJ/JxcvCHNb46w0cdAaJfkzQOO1bA= +github.com/tomnomnom/linkheader v0.0.0-20250811210735-e5fe3b51442e/go.mod h1:krvJ5AY/MjdPkTeRgMYbIDhbbbVvnPQPzsIsDJO8xrY= github.com/twitchtv/twirp v8.1.3+incompatible h1:+F4TdErPgSUbMZMwp13Q/KgDVuI7HJXP61mNV3/7iuU= github.com/twitchtv/twirp v8.1.3+incompatible/go.mod h1:RRJoFSAmTEh2weEqWtpPE3vFK5YBhA6bqp2l1kfCC5A= -github.com/ua-parser/uap-go v0.0.0-20250126222208-a52596c19dff h1:NwMEGwb7JJ8wPjT8OPKP5hO1Xz6AQ7Z00+GLSJfW21s= -github.com/ua-parser/uap-go v0.0.0-20250126222208-a52596c19dff/go.mod h1:BUbeWZiieNxAuuADTBNb3/aeje6on3DhU3rpWsQSB1E= -github.com/urfave/cli/v2 v2.27.5 h1:WoHEJLdsXr6dDWoJgMq/CboDmyY/8HMMH1fTECbih+w= -github.com/urfave/cli/v2 v2.27.5/go.mod h1:3Sevf16NykTbInEnD0yKkjDAeZDS0A6bzhBH5hrMvTQ= +github.com/ua-parser/uap-go v0.0.0-20260529044130-17c35e68e58c h1:XbG4n3OWA1PcRTpbBA22E2ChPLvJCuwYRXO12tIyVL0= +github.com/ua-parser/uap-go v0.0.0-20260529044130-17c35e68e58c/go.mod h1:gwANdYmo9R8LLwGnyDFWK2PMsaXXX2HhAvCnb/UhZsM= +github.com/urfave/cli/v3 v3.9.0 h1:AV9lIiPv3ukYnxunaCUsHnEozptYmDN2F0+yWqLMn/c= +github.com/urfave/cli/v3 v3.9.0/go.mod h1:ysVLtOEmg2tOy6PknnYVhDoouyC/6N42TMeoMzskhso= github.com/urfave/negroni/v3 v3.1.1 h1:6MS4nG9Jk/UuCACaUlNXCbiKa0ywF9LXz5dGu09v8hw= github.com/urfave/negroni/v3 v3.1.1/go.mod h1:jWvnX03kcSjDBl/ShB0iHvx5uOs7mAzZXW+JvJ5XYAs= github.com/wlynxg/anet v0.0.5 h1:J3VJGi1gvo0JwZ/P1/Yc/8p63SoW98B5dHkYDmpgvvU= github.com/wlynxg/anet v0.0.5/go.mod h1:eay5PRQr7fIVAMbTbchTnO9gG65Hg/uYGdc7mguHxoA= -github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= -github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= -github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= -github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= -github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= -github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= -github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= -github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= -github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= -github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= -github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= -go.opentelemetry.io/otel v1.32.0 h1:WnBN+Xjcteh0zdk01SVqV55d/m62NJLJdIyb4y/WO5U= -go.opentelemetry.io/otel v1.32.0/go.mod h1:00DCVSB0RQcnzlwyTfqtxSm+DRr9hpYrHjNGiBHVQIg= -go.opentelemetry.io/otel/metric v1.32.0 h1:xV2umtmNcThh2/a/aCP+h64Xx5wsj8qqnkYZktzNa0M= -go.opentelemetry.io/otel/metric v1.32.0/go.mod h1:jH7CIbbK6SH2V2wE16W05BHCtIDzauciCRLoc/SyMv8= -go.opentelemetry.io/otel/sdk v1.32.0 h1:RNxepc9vK59A8XsgZQouW8ue8Gkb4jpWtJm9ge5lEG4= -go.opentelemetry.io/otel/sdk v1.32.0/go.mod h1:LqgegDBjKMmb2GC6/PrTnteJG39I8/vJCAP9LlJXEjU= -go.opentelemetry.io/otel/sdk/metric v1.32.0 h1:rZvFnvmvawYb0alrYkjraqJq0Z4ZUJAiyYCU9snn1CU= -go.opentelemetry.io/otel/sdk/metric v1.32.0/go.mod h1:PWeZlq0zt9YkYAp3gjKZ0eicRYvOh1Gd+X99x6GHpCQ= -go.opentelemetry.io/otel/trace v1.32.0 h1:WIC9mYrXf8TmY/EXuULKc8hR17vE+Hjv2cssQDe03fM= -go.opentelemetry.io/otel/trace v1.32.0/go.mod h1:+i4rkvCraA+tG6AzwloGaCtkx53Fa+L+V8e9a7YvhT8= +github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= +github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 h1:8tvICD4vSTOOsNrsI4Ljf6C+6UKvpTEH5XY3JMoyPoo= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0/go.mod h1:z9+yiacE0IHRqM4qFfkbt/JYlmYXgss8GY/jXoNuPJI= +go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= +go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0 h1:4YsVu3B8+3qtWYYrsUYgn0OG78pN0rnNPRGX4SbokQI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0/go.mod h1:+wnlSn0mD1ADVMe3v9Z/WIaiz6q6gL2J/ejaAmdmv80= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.44.0 h1:lgh3PiVrRUWMLOVSkQicxzZll5NjF1r+AtsX1XRIHw0= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.44.0/go.mod h1:5Cnhth3m/AgOeTgE3ex12pPmiu/gGtZit03kSzx9X7s= +go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= +go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= +go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= +go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= +go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= +go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= +go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= +go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= +go.opentelemetry.io/proto/otlp v1.10.0 h1:IQRWgT5srOCYfiWnpqUYz9CVmbO8bFmKcwYxpuCSL2g= +go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= -go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.uber.org/zap v1.28.0 h1:IZzaP1Fv73/T/pBMLk4VutPl36uNC+OSUh3JLG3FIjo= +go.uber.org/zap v1.28.0/go.mod h1:rDLpOi171uODNm/mxFcuYWxDsqWSAVkFdX4XojSKg/Q= go.uber.org/zap/exp v0.3.0 h1:6JYzdifzYkGmTdRR59oYH+Ng7k49H9qVpWwNSsGJj3U= go.uber.org/zap/exp v0.3.0/go.mod h1:5I384qq7XGxYyByIhHm6jg5CHkGY0nsTfbDLgDDlgJQ= +go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= +go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= -golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= -golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= -golang.org/x/crypto v0.33.0 h1:IOBPskki6Lysi0lo9qQvbxiQ+FvsCC/YWOecCHAixus= -golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M= -golang.org/x/exp v0.0.0-20250210185358-939b2ce775ac h1:l5+whBCLH3iH2ZNHYLbAe58bo7yrN4mVcnkHDYz5vvs= -golang.org/x/exp v0.0.0-20250210185358-939b2ce775ac/go.mod h1:hH+7mtFmImwwcMvScyxUhjuVHR3HGaDPMn9rMSUUbxo= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/mod v0.23.0 h1:Zb7khfcRGKk+kqfxFaP5tZqCnDZMjC5VtUBs87Hr6QM= -golang.org/x/mod v0.23.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= +golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988= +golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc= +golang.org/x/exp v0.0.0-20260603202125-055de637280b h1:v1uXiEBHo8QA0LiGCo7UgHMzHT4Kdfpl2zmtH5vaP1Q= +golang.org/x/exp v0.0.0-20260603202125-055de637280b/go.mod h1:d2fgXJLVs4dYDHUk5lwMIfzRzSrWCfGZb0ZqeLa/Vcw= +golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4= +golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191007182048-72f939374954/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201216054612-986b41b23924/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210928044308-7d9f5e0b762b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= -golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= -golang.org/x/net v0.35.0 h1:T5GQRQb2y08kTAByq9L4/bz8cipCdA8FbRTXewonqY8= -golang.org/x/net v0.35.0/go.mod h1:EglIi67kWsHKlRzzVMUD93VMSWGFOMSZgxFjparz1Qk= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/net v0.0.0-20220923203811-8be639271d50/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= +golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= -golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w= -golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.0.0-20220923202941-7f9b1623fab7/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190411185658-b44545bcd369/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -428,71 +397,45 @@ golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210525143221-35b2ab0089ea/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= -golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= +golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= -golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= -golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= -golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= +golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= +golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= -golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps= -golang.org/x/tools v0.30.0 h1:BgcpHewrV5AUp2G9MebG4XPFI1E2W41zU1SaqVA9vJY= -golang.org/x/tools v0.30.0/go.mod h1:c347cR/OJfw5TI+GfX7RUPNMdDRRbjvYTS0jPyvsVtY= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8= +golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/api v0.0.0-20241202173237-19429a94021a h1:OAiGFfOiA0v9MRYsSidp3ubZaBnteRUyn3xB2ZQ5G/E= -google.golang.org/genproto/googleapis/api v0.0.0-20241202173237-19429a94021a/go.mod h1:jehYqy3+AhJU9ve55aNOaSml7wUXjF9x6z2LcCfpAhY= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250212204824-5a70512c5d8b h1:FQtJ1MxbXoIIrZHZ33M+w5+dAP9o86rgpjoKr/ZmT7k= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250212204824-5a70512c5d8b/go.mod h1:8BS3B93F/U1juMFq9+EDk+qOT5CO1R9IzXxG3PTqiRk= -google.golang.org/grpc v1.70.0 h1:pWFv03aZoHzlRKHWicjsZytKAiYCtNS0dHbXnIdq7jQ= -google.golang.org/grpc v1.70.0/go.mod h1:ofIJqVKDXx/JiXrwr2IG4/zwdH9txy3IlF40RmcJSQw= -google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= -google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa h1:Kjn0N0tCrDgiAFW+lGO4JZ3ck44CehvJQMAwj9QF0G8= +google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:q4lMZS6kskjT5HvCPrnnypcDPVJqT/f4nfxmkE7gryY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.81.1 h1:VnnIIZ88UzOOKLukQi+ImGz8O1Wdp8nAGGnvOfEIWQQ= +google.golang.org/grpc v1.81.1/go.mod h1:xGH9GfzOyMTGIOXBJmXt+BX/V0kcdQbdcuwQ/zNw42I= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU= -gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= +gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= +gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= +pgregory.net/rapid v1.2.0 h1:keKAYRcjm+e1F0oAuU5F5+YPAWcyxNNRK2wud503Gnk= +pgregory.net/rapid v1.2.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04= diff --git a/install-livekit.sh b/install-livekit.sh index e0b243a..ea1dca8 100755 --- a/install-livekit.sh +++ b/install-livekit.sh @@ -21,7 +21,7 @@ set -o errexit set -o pipefail REPO="livekit" -INSTALL_PATH="/usr/local/bin" +INSTALL_PATH="${INSTALL_PATH:-/usr/local/bin}" log() { printf "%b\n" "$*"; } abort() { diff --git a/magefile.go b/magefile.go index 629e0e6..4423b01 100644 --- a/magefile.go +++ b/magefile.go @@ -76,7 +76,7 @@ func Build() error { return nil } -// builds binary that runs on linux amd64 +// builds binary that runs on linux func BuildLinux() error { mg.Deps(generateWire) if !checksummer.IsChanged() { @@ -88,10 +88,14 @@ func BuildLinux() error { if err := os.MkdirAll("bin", 0755); err != nil { return err } - cmd := mageutil.CommandDir(context.Background(), "cmd/server", "go build -buildvcs=false -o ../../bin/livekit-server-amd64") + buildArch := os.Getenv("GOARCH") + if len(buildArch) == 0 { + buildArch = "amd64" + } + cmd := mageutil.CommandDir(context.Background(), "cmd/server", "go build -buildvcs=false -o ../../bin/livekit-server-" + buildArch) cmd.Env = []string{ "GOOS=linux", - "GOARCH=amd64", + "GOARCH=" + buildArch, "HOME=" + os.Getenv("HOME"), "GOPATH=" + os.Getenv("GOPATH"), } @@ -173,6 +177,14 @@ func TestAll() error { return mageutil.Run(context.Background(), "go test ./... -count=1 -timeout=4m -v") } +// runs golangci-lint +func Lint() error { + if _, err := exec.LookPath("golangci-lint"); err != nil { + return errors.New("golangci-lint is not installed, install instructions: https://golangci-lint.run/docs/welcome/install/") + } + return mageutil.Run(context.Background(), "golangci-lint run ./...") +} + // cleans up builds func Clean() { fmt.Println("cleaning...") diff --git a/pkg/agent/agent_test.go b/pkg/agent/agent_test.go index 98d0407..447ca79 100644 --- a/pkg/agent/agent_test.go +++ b/pkg/agent/agent_test.go @@ -12,6 +12,7 @@ import ( "github.com/livekit/livekit-server/pkg/agent" "github.com/livekit/livekit-server/pkg/agent/testutils" + "github.com/livekit/protocol/auth" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/rpc" "github.com/livekit/protocol/utils/guid" @@ -20,6 +21,7 @@ import ( ) func TestAgent(t *testing.T) { + testAgentName := "test_agent" t.Run("dispatched jobs are assigned to a worker", func(t *testing.T) { bus := psrpc.NewLocalMessageBus() @@ -28,7 +30,7 @@ func TestAgent(t *testing.T) { t.Cleanup(server.Close) worker := server.SimulateAgentWorker() - worker.Register("test", livekit.JobType_JT_ROOM) + worker.Register(testAgentName, livekit.JobType_JT_ROOM) jobAssignments := worker.JobAssignments.Observe() job := &livekit.Job{ @@ -36,14 +38,19 @@ func TestAgent(t *testing.T) { DispatchId: guid.New(guid.AgentDispatchPrefix), Type: livekit.JobType_JT_ROOM, Room: &livekit.Room{}, - AgentName: "test", + AgentName: testAgentName, } - _, err := client.JobRequest(context.Background(), "test", agent.RoomAgentTopic, job) + _, err := client.JobRequest(context.Background(), testAgentName, agent.RoomAgentTopic, job) require.NoError(t, err) select { case a := <-jobAssignments.Events(): require.EqualValues(t, job.Id, a.Job.Id) + v, err := auth.ParseAPIToken(a.Token) + require.NoError(t, err) + _, claims, err := v.Verify(server.TestAPISecret) + require.NoError(t, err) + require.Equal(t, testAgentName, claims.Attributes[agent.AgentNameAttributeKey]) case <-time.After(time.Second): require.Fail(t, "job assignment timeout") } @@ -108,7 +115,7 @@ func TestAgentLoadBalancing(t *testing.T) { t.Cleanup(server.Close) agents := make([]*testutils.AgentWorker, totalWorkers) - for i := 0; i < totalWorkers; i++ { + for i := range totalWorkers { agents[i] = server.SimulateAgentWorker( testutils.WithLabel(fmt.Sprintf("agent-%d", i)), testutils.WithJobLoad(testutils.NewStableJobLoad(0.01)), @@ -128,7 +135,7 @@ func TestAgentLoadBalancing(t *testing.T) { } // check that jobs are distributed normally - for i := 0; i < totalWorkers; i++ { + for i := range totalWorkers { label := fmt.Sprintf("agent-%d", i) require.GreaterOrEqual(t, jobCount[label], 0) require.Less(t, jobCount[label], 35) // three std deviations from the mean is 32 @@ -147,7 +154,7 @@ func TestAgentLoadBalancing(t *testing.T) { t.Cleanup(server.Close) agents := make([]*testutils.AgentWorker, totalWorkers) - for i := 0; i < totalWorkers; i++ { + for i := range totalWorkers { label := fmt.Sprintf("agent-%d", i) if i%2 == 0 { // make sure we have some workers that can accept jobs @@ -169,7 +176,7 @@ func TestAgentLoadBalancing(t *testing.T) { jobCount[w.Label] = len(w.Jobs()) } - for i := 0; i < totalWorkers; i++ { + for i := range totalWorkers { label := fmt.Sprintf("agent-%d", i) if i%2 == 0 { @@ -181,3 +188,33 @@ func TestAgentLoadBalancing(t *testing.T) { } }) } + +func TestConnectionClosedOnDispatchError(t *testing.T) { + t.Run("connection closed when unknown message type received", func(t *testing.T) { + bus := psrpc.NewLocalMessageBus() + server := testutils.NewTestServer(bus) + t.Cleanup(server.Close) + + // register agent + worker := server.SimulateAgentWorker() + worker.Register("test_agent", livekit.JobType_JT_ROOM) + responses := worker.RegisterWorkerResponses.Observe() + select { + case <-responses.Events(): + // registered + case <-time.After(time.Second): + require.Fail(t, "registration timeout") + } + responses.Stop() + + // send invalid message (nil Message field triggers ErrUnknownWorkerSignal) + worker.SendMessage(&livekit.WorkerMessage{Message: nil}) + + select { + case <-worker.Closed(): + // connection closed + case <-time.After(time.Second): + require.Fail(t, "connection should have been closed after dispatch error") + } + }) +} diff --git a/pkg/agent/client.go b/pkg/agent/client.go index 925ca10..51129d0 100644 --- a/pkg/agent/client.go +++ b/pkg/agent/client.go @@ -62,11 +62,13 @@ type JobRequest struct { // only set for participant jobs Participant *livekit.ParticipantInfo Metadata string - AgentName string + AgentName string + Deployment string } type agentClient struct { client rpc.AgentInternalClient + config Config mu sync.RWMutex @@ -87,7 +89,7 @@ type agentClient struct { subDone chan struct{} } -func NewAgentClient(bus psrpc.MessageBus) (Client, error) { +func NewAgentClient(bus psrpc.MessageBus, config Config) (Client, error) { client, err := rpc.NewAgentInternalClient(bus) if err != nil { return nil, err @@ -95,6 +97,7 @@ func NewAgentClient(bus psrpc.MessageBus) (Client, error) { c := &agentClient{ client: client, + config: config, workers: workerpool.New(50), subDone: make(chan struct{}), } @@ -152,21 +155,23 @@ func (c *agentClient) LaunchJob(ctx context.Context, desc *JobRequest) *serverut } dispatcher.ForEach(func(curNs string) { - topic := GetAgentTopic(desc.AgentName, curNs) + topic := GetAgentTopic(desc.AgentName, curNs, desc.Deployment) wg.Add(1) c.workers.Submit(func() { defer wg.Done() // The cached agent parameters do not provide the exact combination of available job type/agent name/namespace, so some of the JobRequest RPC may not trigger any worker job := &livekit.Job{ - Id: utils.NewGuid(utils.AgentJobPrefix), - DispatchId: desc.DispatchId, - Type: desc.JobType, - Room: desc.Room, - Participant: desc.Participant, - Namespace: curNs, - AgentName: desc.AgentName, - Metadata: desc.Metadata, + Id: utils.NewGuid(utils.AgentJobPrefix), + DispatchId: desc.DispatchId, + Type: desc.JobType, + Room: desc.Room, + Participant: desc.Participant, + Namespace: curNs, + AgentName: desc.AgentName, + Metadata: desc.Metadata, + EnableRecording: c.config.EnableUserDataRecording, + Deployment: desc.Deployment, } resp, err := c.client.JobRequest(context.Background(), topic, jobTypeTopic, job) if err != nil { @@ -323,14 +328,19 @@ func (c *agentClient) Stop() error { return nil } -func GetAgentTopic(agentName, namespace string) string { +func GetAgentTopic(agentName, namespace, deployment string) string { + var topic string if agentName == "" { // Backward compatibility - return namespace + topic = namespace } else if namespace == "" { // Forward compatibility once the namespace field is removed from the worker SDK - return agentName + topic = agentName } else { - return fmt.Sprintf("%s_%s", agentName, namespace) + topic = fmt.Sprintf("%s_%s", agentName, namespace) } + if deployment != "" { + topic += "_" + deployment + } + return topic } diff --git a/pkg/agent/config.go b/pkg/agent/config.go new file mode 100644 index 0000000..5a36331 --- /dev/null +++ b/pkg/agent/config.go @@ -0,0 +1,8 @@ +package agent + +const DefaultTargetLoad = 0.7 + +type Config struct { + EnableUserDataRecording bool `yaml:"enable_user_data_recording"` + TargetLoad float32 `yaml:"target_load,omitempty"` +} diff --git a/pkg/agent/testutils/server.go b/pkg/agent/testutils/server.go index 04de89f..090e376 100644 --- a/pkg/agent/testutils/server.go +++ b/pkg/agent/testutils/server.go @@ -4,14 +4,15 @@ import ( "context" "errors" "io" + "maps" "math" "math/rand/v2" + "slices" "sync" "time" "github.com/frostbyte73/core" "github.com/gammazero/deque" - "golang.org/x/exp/maps" "github.com/livekit/livekit-server/pkg/agent" "github.com/livekit/livekit-server/pkg/config" @@ -27,18 +28,25 @@ import ( ) type AgentService interface { - HandleConnection(context.Context, agent.SignalConn, agent.WorkerProtocolVersion) + HandleConnection(context.Context, agent.SignalConn, agent.WorkerRegistration) DrainConnections(time.Duration) } type TestServer struct { AgentService + TestAPIKey string + TestAPISecret string } func NewTestServer(bus psrpc.MessageBus) *TestServer { localNode, _ := routing.NewLocalNode(nil) return NewTestServerWithService(must.Get(service.NewAgentService( - &config.Config{Region: "test"}, + &config.Config{ + Region: "test", + Agents: agent.Config{ + TargetLoad: agent.DefaultTargetLoad, + }, + }, localNode, bus, auth.NewSimpleKeyProvider("test", "verysecretsecret"), @@ -46,7 +54,7 @@ func NewTestServer(bus psrpc.MessageBus) *TestServer { } func NewTestServerWithService(s AgentService) *TestServer { - return &TestServer{s} + return &TestServer{s, "test", "verysecretsecret"} } type SimulatedWorkerOptions struct { @@ -126,7 +134,7 @@ func (h *TestServer) SimulateAgentWorker(opts ...SimulatedWorkerOption) *AgentWo } ctx := service.WithAPIKey(o.Context, &auth.ClaimGrants{}, "test") - go h.HandleConnection(ctx, w, agent.CurrentProtocol) + go h.HandleConnection(ctx, w, agent.MakeWorkerRegistration()) return w } @@ -204,6 +212,15 @@ func (w *AgentWorker) Close() error { return nil } +func (w *AgentWorker) CloseWithReason(_ string) error { + return w.Close() +} + +// Closed returns a channel that is closed when the connection is closed by the server +func (w *AgentWorker) Closed() <-chan struct{} { + return w.fuse.Watch() +} + func (w *AgentWorker) SetReadDeadline(t time.Time) error { w.mu.Lock() defer w.mu.Unlock() @@ -315,7 +332,7 @@ func (w *AgentWorker) handlePong(m *livekit.WorkerPong) { w.WorkerPongs.Emit(m) } -func (w *AgentWorker) sendMessage(m *livekit.WorkerMessage) { +func (w *AgentWorker) SendMessage(m *livekit.WorkerMessage) { select { case <-w.fuse.Watch(): case w.workerMessages <- m: @@ -323,43 +340,43 @@ func (w *AgentWorker) sendMessage(m *livekit.WorkerMessage) { } func (w *AgentWorker) SendRegister(m *livekit.RegisterWorkerRequest) { - w.sendMessage(&livekit.WorkerMessage{Message: &livekit.WorkerMessage_Register{ + w.SendMessage(&livekit.WorkerMessage{Message: &livekit.WorkerMessage_Register{ Register: m, }}) } func (w *AgentWorker) SendAvailability(m *livekit.AvailabilityResponse) { - w.sendMessage(&livekit.WorkerMessage{Message: &livekit.WorkerMessage_Availability{ + w.SendMessage(&livekit.WorkerMessage{Message: &livekit.WorkerMessage_Availability{ Availability: m, }}) } func (w *AgentWorker) SendUpdateWorker(m *livekit.UpdateWorkerStatus) { - w.sendMessage(&livekit.WorkerMessage{Message: &livekit.WorkerMessage_UpdateWorker{ + w.SendMessage(&livekit.WorkerMessage{Message: &livekit.WorkerMessage_UpdateWorker{ UpdateWorker: m, }}) } func (w *AgentWorker) SendUpdateJob(m *livekit.UpdateJobStatus) { - w.sendMessage(&livekit.WorkerMessage{Message: &livekit.WorkerMessage_UpdateJob{ + w.SendMessage(&livekit.WorkerMessage{Message: &livekit.WorkerMessage_UpdateJob{ UpdateJob: m, }}) } func (w *AgentWorker) SendPing(m *livekit.WorkerPing) { - w.sendMessage(&livekit.WorkerMessage{Message: &livekit.WorkerMessage_Ping{ + w.SendMessage(&livekit.WorkerMessage{Message: &livekit.WorkerMessage_Ping{ Ping: m, }}) } func (w *AgentWorker) SendSimulateJob(m *livekit.SimulateJobRequest) { - w.sendMessage(&livekit.WorkerMessage{Message: &livekit.WorkerMessage_SimulateJob{ + w.SendMessage(&livekit.WorkerMessage{Message: &livekit.WorkerMessage_SimulateJob{ SimulateJob: m, }}) } func (w *AgentWorker) SendMigrateJob(m *livekit.MigrateJobRequest) { - w.sendMessage(&livekit.WorkerMessage{Message: &livekit.WorkerMessage_MigrateJob{ + w.SendMessage(&livekit.WorkerMessage{Message: &livekit.WorkerMessage_MigrateJob{ MigrateJob: m, }}) } @@ -411,7 +428,7 @@ func (w *AgentWorker) SimulateRoomJob(roomName string) { func (w *AgentWorker) Jobs() []*AgentJob { w.mu.Lock() defer w.mu.Unlock() - return maps.Values(w.jobs) + return slices.Collect(maps.Values(w.jobs)) } type stableJobLoad struct { diff --git a/pkg/agent/worker.go b/pkg/agent/worker.go index fefbe54..b93c334 100644 --- a/pkg/agent/worker.go +++ b/pkg/agent/worker.go @@ -18,12 +18,14 @@ import ( "context" "errors" "fmt" + "maps" "sync" "time" + "go.uber.org/multierr" "google.golang.org/protobuf/proto" - pagent "github.com/livekit/protocol/agent" + protoagent "github.com/livekit/protocol/agent" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" "github.com/livekit/protocol/rpc" @@ -43,6 +45,8 @@ var ( ErrDuplicateJobAssignment = errors.New("duplicate job assignment") ) +const AgentNameAttributeKey = "lk.agent_name" + type WorkerProtocolVersion int const CurrentProtocol = 1 @@ -57,12 +61,15 @@ type SignalConn interface { ReadWorkerMessage() (*livekit.WorkerMessage, int, error) SetReadDeadline(time.Time) error Close() error + CloseWithReason(reason string) error } func JobStatusIsEnded(s livekit.JobStatus) bool { return s == livekit.JobStatus_JS_SUCCESS || s == livekit.JobStatus_JS_FAILED } +type AssignmentHook func(next func(*livekit.JobAssignment) error) func(*livekit.JobAssignment) error + type WorkerSignalHandler interface { HandleRegister(*livekit.RegisterWorkerRequest) error HandleAvailability(*livekit.AvailabilityResponse) error @@ -141,10 +148,20 @@ type WorkerRegistration struct { Protocol WorkerProtocolVersion ID string Version string + AgentID string AgentName string Namespace string JobType livekit.JobType Permissions *livekit.ParticipantPermission + ClientIP string + Deployment string +} + +func MakeWorkerRegistration() WorkerRegistration { + return WorkerRegistration{ + ID: guid.New(guid.AgentWorkerPrefix), + Protocol: CurrentProtocol, + } } var _ WorkerSignalHandler = (*WorkerRegisterer)(nil) @@ -152,18 +169,17 @@ var _ WorkerSignalHandler = (*WorkerRegisterer)(nil) type WorkerRegisterer struct { WorkerPingHandler serverInfo *livekit.ServerInfo - protocol WorkerProtocolVersion deadline time.Time registration WorkerRegistration registered bool } -func NewWorkerRegisterer(conn SignalConn, serverInfo *livekit.ServerInfo, protocol WorkerProtocolVersion) *WorkerRegisterer { +func NewWorkerRegisterer(conn SignalConn, serverInfo *livekit.ServerInfo, base WorkerRegistration) *WorkerRegisterer { return &WorkerRegisterer{ WorkerPingHandler: WorkerPingHandler{conn: conn}, serverInfo: serverInfo, - protocol: protocol, + registration: base, deadline: time.Now().Add(RegisterTimeout), } } @@ -185,6 +201,10 @@ func (h *WorkerRegisterer) HandleRegister(req *livekit.RegisterWorkerRequest) er return ErrUnknownJobType } + if err := protoagent.ValidateDeployment(req.GetDeployment()); err != nil { + return err + } + permissions := req.AllowedPermissions if permissions == nil { permissions = &livekit.ParticipantPermission{ @@ -195,15 +215,12 @@ func (h *WorkerRegisterer) HandleRegister(req *livekit.RegisterWorkerRequest) er } } - h.registration = WorkerRegistration{ - Protocol: h.protocol, - ID: guid.New(guid.AgentWorkerPrefix), - Version: req.Version, - AgentName: req.AgentName, - Namespace: req.GetNamespace(), - JobType: req.GetType(), - Permissions: permissions, - } + h.registration.Version = req.Version + h.registration.AgentName = req.AgentName + h.registration.Namespace = req.GetNamespace() + h.registration.JobType = req.GetType() + h.registration.Permissions = permissions + h.registration.Deployment = req.GetDeployment() h.registered = true _, err := h.conn.WriteServerMessage(&livekit.ServerMessage{ @@ -256,6 +273,9 @@ func NewWorker( logger: logger.WithValues( "workerID", registration.ID, "agentName", registration.AgentName, + "deployment", registration.Deployment, + "agentID", registration.AgentID, + "version", registration.Version, "jobType", registration.JobType.String(), ), @@ -268,10 +288,8 @@ func NewWorker( } } -func (w *Worker) sendRequest(req *livekit.ServerMessage) { - if _, err := w.conn.WriteServerMessage(req); err != nil { - w.logger.Warnw("error writing to websocket", err) - } +func (w *Worker) APIKey() string { + return w.apiKey } func (w *Worker) Status() livekit.WorkerStatus { @@ -293,11 +311,7 @@ func (w *Worker) Logger() logger.Logger { func (w *Worker) RunningJobs() map[livekit.JobID]*livekit.Job { w.mu.Lock() defer w.mu.Unlock() - jobs := make(map[livekit.JobID]*livekit.Job, len(w.runningJobs)) - for k, v := range w.runningJobs { - jobs[k] = v - } - return jobs + return maps.Clone(w.runningJobs) } func (w *Worker) RunningJobCount() int { @@ -316,7 +330,7 @@ func (w *Worker) GetJobState(jobID livekit.JobID) (*livekit.JobState, error) { return utils.CloneProto(j.State), nil } -func (w *Worker) AssignJob(ctx context.Context, job *livekit.Job) (*livekit.JobState, error) { +func (w *Worker) AssignJob(ctx context.Context, job *livekit.Job, hook AssignmentHook) (*livekit.JobState, error) { availCh := make(chan *livekit.AvailabilityResponse, 1) job = utils.CloneProto(job) jobID := livekit.JobID(job.Id) @@ -340,13 +354,17 @@ func (w *Worker) AssignJob(ctx context.Context, job *livekit.Job) (*livekit.JobS job.State = &livekit.JobState{} } now := time.Now() + job.State.WorkerId = w.ID + job.State.AgentId = w.AgentID job.State.UpdatedAt = now.UnixNano() job.State.StartedAt = now.UnixNano() job.State.Status = livekit.JobStatus_JS_RUNNING - w.sendRequest(&livekit.ServerMessage{Message: &livekit.ServerMessage_Availability{ + if _, err := w.conn.WriteServerMessage(&livekit.ServerMessage{Message: &livekit.ServerMessage_Availability{ Availability: &livekit.AvailabilityRequest{Job: job}, - }}) + }}); err != nil { + return nil, err + } timeout := time.NewTimer(AssignJobTimeout) defer timeout.Stop() @@ -354,20 +372,31 @@ func (w *Worker) AssignJob(ctx context.Context, job *livekit.Job) (*livekit.JobS // See handleAvailability for the response select { case res := <-availCh: + if res.Terminate { + job.State.EndedAt = now.UnixNano() + job.State.Status = livekit.JobStatus_JS_SUCCESS + return job.State, nil + } + if !res.Available { return nil, ErrWorkerNotAvailable } job.State.ParticipantIdentity = res.ParticipantIdentity + attributes := res.ParticipantAttributes + if attributes == nil { + attributes = make(map[string]string) + } + attributes[AgentNameAttributeKey] = w.AgentName - token, err := pagent.BuildAgentToken( + token, err := protoagent.BuildAgentToken( w.apiKey, w.apiSecret, job.Room.Name, res.ParticipantIdentity, res.ParticipantName, res.ParticipantMetadata, - res.ParticipantAttributes, + attributes, w.Permissions, ) if err != nil { @@ -375,10 +404,18 @@ func (w *Worker) AssignJob(ctx context.Context, job *livekit.Job) (*livekit.JobS return nil, err } - // In OSS, Url is nil, and the used API Key is the same as the one used to connect the worker - w.sendRequest(&livekit.ServerMessage{Message: &livekit.ServerMessage_Assignment{ - Assignment: &livekit.JobAssignment{Job: job, Url: nil, Token: token}, - }}) + send := func(a *livekit.JobAssignment) error { + _, err := w.conn.WriteServerMessage(&livekit.ServerMessage{ + Message: &livekit.ServerMessage_Assignment{Assignment: a}, + }) + return err + } + if hook != nil { + send = hook(send) + } + if err := send(&livekit.JobAssignment{Job: job, Token: token}); err != nil { + return nil, err + } state := utils.CloneProto(job.State) @@ -407,7 +444,7 @@ func (w *Worker) TerminateJob(jobID livekit.JobID, reason rpc.JobTerminateReason return nil, ErrJobNotFound } - w.sendRequest(&livekit.ServerMessage{Message: &livekit.ServerMessage_Termination{ + _, writeErr := w.conn.WriteServerMessage(&livekit.ServerMessage{Message: &livekit.ServerMessage_Termination{ Termination: &livekit.JobTermination{ JobId: string(jobID), }, @@ -420,11 +457,12 @@ func (w *Worker) TerminateJob(jobID livekit.JobID, reason rpc.JobTerminateReason errorStr = "agent worker left the room" } - return w.UpdateJobStatus(&livekit.UpdateJobStatus{ + state, updateErr := w.UpdateJobStatus(&livekit.UpdateJobStatus{ JobId: string(jobID), Status: status, Error: errorStr, }) + return state, multierr.Combine(writeErr, updateErr) } func (w *Worker) UpdateMetadata(metadata string) { @@ -530,7 +568,7 @@ func (w *Worker) HandleSimulateJob(simulate *livekit.SimulateJobRequest) error { } go func() { - _, err := w.AssignJob(w.ctx, job) + _, err := w.AssignJob(w.ctx, job, nil) if err != nil { w.logger.Errorw("unable to simulate job", err, "jobID", job.Id) } diff --git a/pkg/clientconfiguration/conf.go b/pkg/clientconfiguration/conf.go index 992cca9..7fb5ae8 100644 --- a/pkg/clientconfiguration/conf.go +++ b/pkg/clientconfiguration/conf.go @@ -15,30 +15,48 @@ package clientconfiguration import ( - "github.com/livekit/livekit-server/pkg/sfu/mime" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/utils/must" ) // StaticConfigurations list specific device-side limitations that should be disabled at a global level var StaticConfigurations = []ConfigurationItem{ // { - // Match: &ScriptMatch{Expr: `c.protocol <= 5 || c.browser == "firefox"`}, + // Match: must.Get(NewScriptMatch(`c.protocol <= 5 || c.browser == "firefox"`)), // Configuration: &livekit.ClientConfiguration{ResumeConnection: livekit.ClientConfigSetting_DISABLED}, // Merge: false, // }, { - Match: &ScriptMatch{Expr: `c.browser == "safari"`}, - Configuration: &livekit.ClientConfiguration{DisabledCodecs: &livekit.DisabledCodecs{Codecs: []*livekit.Codec{ - {Mime: mime.MimeTypeAV1.String()}, - }}}, - Merge: false, - }, - { - Match: &ScriptMatch{Expr: `(c.device_model == "xiaomi 2201117ti" && c.os == "android") || - ((c.browser == "firefox" || c.browser == "firefox mobile") && (c.os == "linux" || c.os == "android"))`}, + Match: must.Get(NewScriptMatch(`c.browser == "safari"`)), Configuration: &livekit.ClientConfiguration{ DisabledCodecs: &livekit.DisabledCodecs{ - Publish: []*livekit.Codec{{Mime: mime.MimeTypeH264.String()}}, + Codecs: []*livekit.Codec{ + {Mime: mime.MimeTypeAV1.String()}, + }, + }, + }, + Merge: true, + }, + { + Match: must.Get(NewScriptMatch(`c.browser == "safari" && c.browser_version > "18.3"`)), + Configuration: &livekit.ClientConfiguration{ + DisabledCodecs: &livekit.DisabledCodecs{ + Publish: []*livekit.Codec{ + {Mime: mime.MimeTypeVP9.String()}, + }, + }, + }, + Merge: true, + }, + { + Match: must.Get(NewScriptMatch(`(c.device_model == "xiaomi 2201117ti" && c.os == "android") || + ((c.browser == "firefox" || c.browser == "firefox mobile") && (c.os == "linux" || c.os == "android"))`)), + Configuration: &livekit.ClientConfiguration{ + DisabledCodecs: &livekit.DisabledCodecs{ + Publish: []*livekit.Codec{ + {Mime: mime.MimeTypeH264.String()}, + }, }, }, Merge: false, diff --git a/pkg/clientconfiguration/conf_test.go b/pkg/clientconfiguration/conf_test.go index 9c3554d..453d57c 100644 --- a/pkg/clientconfiguration/conf_test.go +++ b/pkg/clientconfiguration/conf_test.go @@ -20,13 +20,14 @@ import ( "github.com/stretchr/testify/require" "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/utils/must" ) func TestScriptMatchConfiguration(t *testing.T) { t.Run("no merge", func(t *testing.T) { confs := []ConfigurationItem{ { - Match: &ScriptMatch{Expr: `c.protocol > 5 && c.browser != "firefox"`}, + Match: must.Get(NewScriptMatch(`c.protocol > 5 && c.browser != "firefox"`)), Configuration: &livekit.ClientConfiguration{ ResumeConnection: livekit.ClientConfigSetting_ENABLED, }, @@ -48,14 +49,14 @@ func TestScriptMatchConfiguration(t *testing.T) { t.Run("merge", func(t *testing.T) { confs := []ConfigurationItem{ { - Match: &ScriptMatch{Expr: `c.protocol > 5 && c.browser != "firefox"`}, + Match: must.Get(NewScriptMatch(`c.protocol > 5 && c.browser != "firefox"`)), Configuration: &livekit.ClientConfiguration{ ResumeConnection: livekit.ClientConfigSetting_ENABLED, }, Merge: true, }, { - Match: &ScriptMatch{Expr: `c.sdk == "android"`}, + Match: must.Get(NewScriptMatch(`c.sdk == "android"`)), Configuration: &livekit.ClientConfiguration{ Video: &livekit.VideoConfiguration{ HardwareEncoder: livekit.ClientConfigSetting_DISABLED, @@ -81,10 +82,12 @@ func TestScriptMatchConfiguration(t *testing.T) { func TestScriptMatch(t *testing.T) { client := &livekit.ClientInfo{ - Protocol: 6, - Browser: "chrome", - Sdk: 3, // android - DeviceModel: "12345", + Protocol: 6, + Browser: "chrome", + Sdk: 3, // android + DeviceModel: "12345", + BrowserVersion: "13.2", + Version: "2.17.1", } type testcase struct { @@ -99,12 +102,26 @@ func TestScriptMatch(t *testing.T) { {name: "invalid expr", expr: `cc.protocol > 5`, err: true}, {name: "unexist field", expr: `c.protocols > 5`, err: true}, {name: "combined condition", expr: `c.protocol > 5 && (c.sdk=="android" || c.sdk=="ios")`, result: true}, - {name: "combined condition2", expr: `(c.device_model == "xiaomi 2201117ti" && c.os == "android) || ((c.browser == "firefox" || c.browser == "firefox mobile") && (c.os == "linux" || c.os == "android"))`, result: false}, + {name: "combined condition2", expr: `(c.device_model == "xiaomi 2201117ti" && c.os == "android") || ((c.browser == "firefox" || c.browser == "firefox mobile") && (c.os == "linux" || c.os == "android"))`, result: false}, + {name: "string lesser", expr: `c.browser_version < "11.3"`, result: false}, + {name: "string lesser eq", expr: `c.browser_version <= "13.2"`, result: true}, + {name: "string greater", expr: `c.browser_version > "11.3"`, result: true}, + {name: "string greater eq", expr: `c.browser_version >= "13.2"`, result: true}, + {name: "semantic lesser", expr: `c.version < "2.16.10"`, result: false}, + {name: "semantic lesser eq", expr: `c.version <= "2.17.1"`, result: true}, + {name: "semantic greater", expr: `c.version > "2.16.10"`, result: true}, + {name: "semantic greater eq", expr: `c.version >= "2.17.1"`, result: true}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { - match := &ScriptMatch{Expr: c.expr} + match, err := NewScriptMatch(c.expr) + if err != nil { + if !c.err { + require.NoError(t, err) + } + return + } m, err := match.Match(client) if c.err { require.Error(t, err) diff --git a/pkg/clientconfiguration/match.go b/pkg/clientconfiguration/match.go index b915c2c..c0cf25f 100644 --- a/pkg/clientconfiguration/match.go +++ b/pkg/clientconfiguration/match.go @@ -15,11 +15,13 @@ package clientconfiguration import ( - "context" "errors" + "fmt" "strings" "github.com/d5/tengo/v2" + "github.com/d5/tengo/v2/token" + "golang.org/x/mod/semver" "github.com/livekit/protocol/livekit" ) @@ -29,7 +31,19 @@ type Match interface { } type ScriptMatch struct { - Expr string + compiled *tengo.Compiled +} + +func NewScriptMatch(expr string) (*ScriptMatch, error) { + script := tengo.NewScript(fmt.Appendf(nil, "__res__ := (%s)", expr)) + if err := script.Add("c", &clientObject{}); err != nil { + return nil, err + } + compiled, err := script.Compile() + if err != nil { + return nil, err + } + return &ScriptMatch{compiled}, nil } // use result of eval script expression for match. @@ -38,17 +52,23 @@ type ScriptMatch struct { // browser if firefox: c.browser == "firefox" // combined rule : c.protocol > 5 && c.browser == "firefox" func (m *ScriptMatch) Match(clientInfo *livekit.ClientInfo) (bool, error) { - res, err := tengo.Eval(context.TODO(), m.Expr, map[string]interface{}{"c": &clientObject{info: clientInfo}}) - if err != nil { + clone := m.compiled.Clone() + if err := clone.Set("c", &clientObject{info: clientInfo}); err != nil { + return false, err + } + if err := clone.Run(); err != nil { return false, err } + res := clone.Get("__res__").Value() if val, ok := res.(bool); ok { return val, nil } return false, errors.New("invalid match expression result") } +// ------------------------------------------------ + type clientObject struct { tengo.ObjectImpl info *livekit.ClientInfo @@ -72,7 +92,7 @@ func (c *clientObject) IndexGet(index tengo.Object) (res tengo.Object, err error case "sdk": return &tengo.String{Value: strings.ToLower(c.info.Sdk.String())}, nil case "version": - return &tengo.String{Value: c.info.Version}, nil + return &ruleSdkVersion{sdkVersion: c.info.Version}, nil case "protocol": return &tengo.Int{Value: int64(c.info.Protocol)}, nil case "os": @@ -84,9 +104,74 @@ func (c *clientObject) IndexGet(index tengo.Object) (res tengo.Object, err error case "browser": return &tengo.String{Value: strings.ToLower(c.info.Browser)}, nil case "browser_version": - return &tengo.String{Value: c.info.BrowserVersion}, nil + return &ruleSdkVersion{sdkVersion: c.info.BrowserVersion}, nil case "address": return &tengo.String{Value: c.info.Address}, nil } return &tengo.Undefined{}, nil } + +// ------------------------------------------ + +type ruleSdkVersion struct { + tengo.ObjectImpl + sdkVersion string +} + +func (r *ruleSdkVersion) TypeName() string { + return "sdkVersion" +} + +func (r *ruleSdkVersion) String() string { + return r.sdkVersion +} + +func (r *ruleSdkVersion) BinaryOp(op token.Token, rhs tengo.Object) (tengo.Object, error) { + if rhs, ok := rhs.(*tengo.String); ok { + cmp := r.compare(rhs.Value) + + isMatch := false + switch op { + case token.Greater: + isMatch = cmp > 0 + case token.GreaterEq: + isMatch = cmp >= 0 + case token.Less: + isMatch = cmp < 0 + case token.LessEq: + isMatch = cmp <= 0 + default: + return nil, tengo.ErrInvalidOperator + } + + if isMatch { + return tengo.TrueValue, nil + } + return tengo.FalseValue, nil + } + + return nil, tengo.ErrInvalidOperator +} + +func (r *ruleSdkVersion) Equals(rhs tengo.Object) bool { + if rhs, ok := rhs.(*tengo.String); ok { + return r.compare(rhs.Value) == 0 + } + + return false +} + +func (r *ruleSdkVersion) compare(rhsSdkVersion string) int { + if !semver.IsValid("v"+r.sdkVersion) || !semver.IsValid("v"+rhsSdkVersion) { + // if not valid semver, do string compare + switch { + case r.sdkVersion < rhsSdkVersion: + return -1 + case r.sdkVersion > rhsSdkVersion: + return 1 + } + } else { + return semver.Compare("v"+r.sdkVersion, "v"+rhsSdkVersion) + } + return 0 +} diff --git a/pkg/config/config.go b/pkg/config/config.go index 5c04c5f..68d3e8a 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -18,25 +18,30 @@ import ( "fmt" "os" "reflect" + "strconv" "strings" "time" "github.com/mitchellh/go-homedir" "github.com/pkg/errors" - "github.com/urfave/cli/v2" + "github.com/urfave/cli/v3" "gopkg.in/yaml.v3" - "github.com/livekit/livekit-server/pkg/metric" - "github.com/livekit/livekit-server/pkg/sfu" - "github.com/livekit/livekit-server/pkg/sfu/bwe/remotebwe" - "github.com/livekit/livekit-server/pkg/sfu/bwe/sendsidebwe" - "github.com/livekit/livekit-server/pkg/sfu/mime" - "github.com/livekit/livekit-server/pkg/sfu/streamallocator" "github.com/livekit/mediatransportutil/pkg/rtcconfig" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" redisLiveKit "github.com/livekit/protocol/redis" "github.com/livekit/protocol/rpc" + "github.com/livekit/protocol/webhook" + + "github.com/livekit/livekit-server/pkg/agent" + "github.com/livekit/livekit-server/pkg/metric" + "github.com/livekit/livekit-server/pkg/sfu" + "github.com/livekit/livekit-server/pkg/sfu/bwe/remotebwe" + "github.com/livekit/livekit-server/pkg/sfu/bwe/sendsidebwe" + "github.com/livekit/livekit-server/pkg/sfu/pacer" + "github.com/livekit/livekit-server/pkg/sfu/streamallocator" ) const ( @@ -44,8 +49,9 @@ const ( ) var ( - ErrKeyFileIncorrectPermission = errors.New("key file others permissions must be set to 0") - ErrKeysNotSet = errors.New("one of key-file or keys must be provided") + ErrKeyFileIncorrectPermission = errors.New("key file others permissions must be set to 0") + ErrTURNSecretFileIncorrectPermission = errors.New("turn secret file others permissions must be set to 0") + ErrKeysNotSet = errors.New("one of key-file or keys must be provided") ) type Config struct { @@ -62,7 +68,7 @@ type Config struct { TURN TURNConfig `yaml:"turn,omitempty"` Ingress IngressConfig `yaml:"ingress,omitempty"` SIP SIPConfig `yaml:"sip,omitempty"` - WebHook WebHookConfig `yaml:"webhook,omitempty"` + WebHook webhook.WebHookConfig `yaml:"webhook,omitempty"` NodeSelector NodeSelectorConfig `yaml:"node_selector,omitempty"` KeyFile string `yaml:"key_file,omitempty"` Keys map[string]string `yaml:"keys,omitempty"` @@ -73,10 +79,18 @@ type Config struct { LogLevel string `yaml:"log_level,omitempty"` Logging LoggingConfig `yaml:"logging,omitempty"` Limit LimitConfig `yaml:"limit,omitempty"` + Agents agent.Config `yaml:"agents,omitempty"` Development bool `yaml:"development,omitempty"` Metric metric.MetricConfig `yaml:"metric,omitempty"` + Trace TracingConfig `yaml:"trace,omitempty"` + + NodeStats NodeStatsConfig `yaml:"node_stats,omitempty"` + + EnableDataTracks bool `yaml:"enable_data_tracks,omitempty"` + + API APIConfig `yaml:"api,omitempty"` } type RTCConfig struct { @@ -102,6 +116,18 @@ type RTCConfig struct { // allow TCP and TURN/TLS fallback AllowTCPFallback *bool `yaml:"allow_tcp_fallback,omitempty"` + // Signaling RTT threshold (in milliseconds) governing ICE/TCP fallback. On a UDP + // failure, ICE/TCP is attempted only while the measured signaling RTT is below this + // value; at or above it, supporting clients fall back directly to TURN/TLS. When 0 + // (the default), the RTT check is disabled and ICE/TCP is always attempted (for + // clients that support it). A positive value also gates allow_udp_unstable_fallback. + TCPFallbackRTTThreshold int `yaml:"tcp_fallback_rtt_threshold,omitempty"` + + // When enabled, an established UDP connection reporting sustained high packet loss is + // migrated to ICE/TCP or TURN/TLS. Requires tcp_fallback_rtt_threshold to be set + // (> 0). Disabled by default. + AllowUDPUnstableFallback bool `yaml:"allow_udp_unstable_fallback,omitempty"` + // force a reconnect on a publication error ReconnectOnPublicationError *bool `yaml:"reconnect_on_publication_error,omitempty"` @@ -118,7 +144,13 @@ type RTCConfig struct { // be dropped for a slow data channel to avoid blocking the room. DatachannelSlowThreshold int `yaml:"datachannel_slow_threshold,omitempty"` + // Target latency for lossy data channels, used to drop packets to reduce latency. + DatachannelLossyTargetLatency time.Duration `yaml:"datachannel_lossy_target_latency,omitempty"` + ForwardStats ForwardStatsConfig `yaml:"forward_stats,omitempty"` + + // enable rtp stream restart detection for published tracks + EnableRTPStreamRestartDetection bool `yaml:"enable_rtp_stream_restart_detection,omitempty"` } type TURNServer struct { @@ -127,6 +159,14 @@ type TURNServer struct { Protocol string `yaml:"protocol,omitempty"` Username string `yaml:"username,omitempty"` Credential string `yaml:"credential,omitempty"` + // Secret is used for TURN static auth secrets mechanism. When provided, + // dynamic credentials are generated using HMAC-SHA1 instead of static Username/Credential + Secret string `yaml:"secret,omitempty"` + // File containing the secret + SecretFile string `yaml:"secret_file,omitempty"` + // TTL is the time-to-live in seconds for generated credentials when using Secret. + // Defaults to 14400 seconds (4 hours) if not specified + TTL int `yaml:"ttl,omitempty"` } type CongestionControlConfig struct { @@ -139,8 +179,9 @@ type CongestionControlConfig struct { UseSendSideBWEInterceptor bool `yaml:"use_send_side_bwe_interceptor,omitempty"` - UseSendSideBWE bool `yaml:"use_send_side_bwe,omitempty"` - SendSideBWE sendsidebwe.SendSideBWEConfig `yaml:"send_side_bwe,omitempty"` + UseSendSideBWE bool `yaml:"use_send_side_bwe,omitempty"` + SendSideBWEPacer string `yaml:"send_side_bwe_pacer,omitempty"` + SendSideBWE sendsidebwe.SendSideBWEConfig `yaml:"send_side_bwe,omitempty"` } type PlayoutDelayConfig struct { @@ -152,6 +193,8 @@ type PlayoutDelayConfig struct { type VideoConfig struct { DynacastPauseDelay time.Duration `yaml:"dynacast_pause_delay,omitempty"` StreamTrackerManager sfu.StreamTrackerManagerConfig `yaml:"stream_tracker_manager,omitempty"` + + CodecRegressionThreshold int `yaml:"codec_regression_threshold,omitempty"` } type RoomConfig struct { @@ -167,6 +210,8 @@ type RoomConfig struct { CreateRoomEnabled bool `yaml:"create_room_enabled,omitempty"` CreateRoomTimeout time.Duration `yaml:"create_room_timeout,omitempty"` CreateRoomAttempts int `yaml:"create_room_attempts,omitempty"` + // target room participant update batch chunk size in bytes + UpdateBatchTargetSize int `yaml:"update_batch_target_size,omitempty"` // deprecated, moved to limits MaxMetadataSize uint32 `yaml:"max_metadata_size,omitempty"` // deprecated, moved to limits @@ -187,26 +232,33 @@ type LoggingConfig struct { } type TURNConfig struct { - Enabled bool `yaml:"enabled,omitempty"` - Domain string `yaml:"domain,omitempty"` - CertFile string `yaml:"cert_file,omitempty"` - KeyFile string `yaml:"key_file,omitempty"` - TLSPort int `yaml:"tls_port,omitempty"` - UDPPort int `yaml:"udp_port,omitempty"` - RelayPortRangeStart uint16 `yaml:"relay_range_start,omitempty"` - RelayPortRangeEnd uint16 `yaml:"relay_range_end,omitempty"` - ExternalTLS bool `yaml:"external_tls,omitempty"` -} - -type WebHookConfig struct { - URLs []string `yaml:"urls,omitempty"` - // key to use for webhook - APIKey string `yaml:"api_key,omitempty"` + Enabled bool `yaml:"enabled,omitempty"` + Domain string `yaml:"domain,omitempty"` + CertFile string `yaml:"cert_file,omitempty"` + KeyFile string `yaml:"key_file,omitempty"` + TLSPort int `yaml:"tls_port,omitempty"` + UDPPort int `yaml:"udp_port,omitempty"` + RelayPortRangeStart uint16 `yaml:"relay_range_start,omitempty"` + RelayPortRangeEnd uint16 `yaml:"relay_range_end,omitempty"` + ExternalTLS bool `yaml:"external_tls,omitempty"` + BindAddresses []string `yaml:"bind_addresses,omitempty"` + // TTL of the TURN credentials in seconds - defaults to 300 + TTLSeconds int `yaml:"ttl_seconds,omitempty"` + // list of restricted peer CIDRs (loopback, link-local (unicast, multicast), multicast, private, unspecified) to allow access to. + // By default (i. e. empty list), all restricted peer CIDRs are denied access. + // When not empty, only the specified CIDRs are allowed access. + // Note that this check is applied to restricted peer CIDRs only. + AllowRestrictedPeerCIDRs []string `yaml:"allow_restricted_peer_cidrs,omitempty"` + // list of peer CIDRs to deny access to + // This applies to all peer CIDRs, including restricted ones. + // Deny list takes precedence over allow list. + DenyPeerCIDRs []string `yaml:"deny_peer_cidrs,omitempty"` } type NodeSelectorConfig struct { Kind string `yaml:"kind,omitempty"` SortBy string `yaml:"sort_by,omitempty"` + Algorithm string `yaml:"algorithm,omitempty"` CPULoadLimit float32 `yaml:"cpu_load_limit,omitempty"` SysloadLimit float32 `yaml:"sysload_limit,omitempty"` Regions []RegionConfig `yaml:"regions,omitempty"` @@ -245,6 +297,10 @@ func (l LimitConfig) CheckRoomNameLength(name string) bool { return l.MaxRoomNameLength == 0 || len(name) <= l.MaxRoomNameLength } +func (l LimitConfig) CheckParticipantIdentityLength(identity string) bool { + return l.MaxParticipantIdentityLength == 0 || len(identity) <= l.MaxParticipantIdentityLength +} + func (l LimitConfig) CheckParticipantNameLength(name string) bool { return l.MaxParticipantNameLength == 0 || len(name) <= l.MaxParticipantNameLength } @@ -281,6 +337,9 @@ type APIConfig struct { // max amount of time to wait before checking for operation complete MaxCheckInterval time.Duration `yaml:"max_check_interval,omitempty"` + + // Backwards compatibility for room service api calls, will enable by default and remove in a future release + EnablePsrpcForGetListParticpants bool `yaml:"enable_psrpc_for_get_list_participants,omitempty"` } type PrometheusConfig struct { @@ -295,6 +354,13 @@ type ForwardStatsConfig struct { ReportWindow time.Duration `yaml:"report_window,omitempty"` } +type TracingConfig struct { + // JaegerURL configures Jaeger as a global tracer. + // + // The following formats are supported: , :, http(s):/// + JaegerURL string `yaml:"jaeger_url,omitempty"` +} + func DefaultAPIConfig() APIConfig { return APIConfig{ ExecutionTimeout: 2 * time.Second, @@ -303,6 +369,18 @@ func DefaultAPIConfig() APIConfig { } } +type NodeStatsConfig struct { + StatsUpdateInterval time.Duration `yaml:"stats_update_interval,omitempty"` + StatsRateMeasurementIntervals []time.Duration `yaml:"stats_rate_measurement_intervals,omitempty"` + StatsMaxDelay time.Duration `yaml:"stats_max_delay,omitempty"` +} + +var DefaultNodeStatsConfig = NodeStatsConfig{ + StatsUpdateInterval: 2 * time.Second, + StatsRateMeasurementIntervals: []time.Duration{10 * time.Second}, + StatsMaxDelay: 30 * time.Second, +} + var DefaultConfig = Config{ Port: 7880, RTC: RTCConfig{ @@ -324,31 +402,37 @@ var DefaultConfig = Config{ RemoteBWE: remotebwe.DefaultRemoteBWEConfig, UseSendSideBWEInterceptor: false, UseSendSideBWE: false, + SendSideBWEPacer: string(pacer.PacerBehaviorNoQueue), SendSideBWE: sendsidebwe.DefaultSendSideBWEConfig, }, }, Audio: sfu.DefaultAudioConfig, Video: VideoConfig{ - DynacastPauseDelay: 5 * time.Second, - StreamTrackerManager: sfu.DefaultStreamTrackerManagerConfig, + DynacastPauseDelay: 5 * time.Second, + StreamTrackerManager: sfu.DefaultStreamTrackerManagerConfig, + CodecRegressionThreshold: 5, }, Redis: redisLiveKit.RedisConfig{}, Room: RoomConfig{ AutoCreate: true, EnabledCodecs: []CodecSpec{ + {Mime: mime.MimeTypePCMU.String()}, + {Mime: mime.MimeTypePCMA.String()}, {Mime: mime.MimeTypeOpus.String()}, {Mime: mime.MimeTypeRED.String()}, {Mime: mime.MimeTypeVP8.String()}, {Mime: mime.MimeTypeH264.String()}, {Mime: mime.MimeTypeVP9.String()}, {Mime: mime.MimeTypeAV1.String()}, + {Mime: mime.MimeTypeH265.String()}, {Mime: mime.MimeTypeRTX.String()}, }, - EmptyTimeout: 5 * 60, - DepartureTimeout: 20, - CreateRoomEnabled: true, - CreateRoomTimeout: 10 * time.Second, - CreateRoomAttempts: 3, + EmptyTimeout: 5 * 60, + DepartureTimeout: 20, + CreateRoomEnabled: true, + CreateRoomTimeout: 10 * time.Second, + CreateRoomAttempts: 3, + UpdateBatchTargetSize: 128 * 1024, }, Limit: LimitConfig{ MaxMetadataSize: 64000, @@ -361,13 +445,16 @@ var DefaultConfig = Config{ PionLevel: "error", }, TURN: TURNConfig{ - Enabled: false, + Enabled: false, + BindAddresses: []string{"0.0.0.0"}, + TTLSeconds: 300, }, NodeSelector: NodeSelectorConfig{ Kind: "any", SortBy: "random", SysloadLimit: 0.9, CPULoadLimit: 0.9, + Algorithm: "lowest", }, SignalRelay: SignalRelayConfig{ RetryTimeout: 7500 * time.Millisecond, @@ -376,12 +463,19 @@ var DefaultConfig = Config{ StreamBufferSize: 1000, ConnectAttempts: 3, }, - PSRPC: rpc.DefaultPSRPCConfig, - Keys: map[string]string{}, - Metric: metric.DefaultMetricConfig, + Agents: agent.Config{ + TargetLoad: agent.DefaultTargetLoad, + }, + PSRPC: rpc.DefaultPSRPCConfig, + Keys: map[string]string{}, + Metric: metric.DefaultMetricConfig, + WebHook: webhook.DefaultWebHookConfig, + NodeStats: DefaultNodeStatsConfig, + API: DefaultAPIConfig(), + EnableDataTracks: true, } -func NewConfig(confString string, strictMode bool, c *cli.Context, baseFlags []cli.Flag) (*Config, error) { +func NewConfig(confString string, strictMode bool, c *cli.Command, baseFlags []cli.Flag) (*Config, error) { // start with defaults marshalled, err := yaml.Marshal(&DefaultConfig) if err != nil { @@ -563,74 +657,110 @@ func (conf *Config) ValidateKeys() error { return nil } +func (conf *Config) LoadTURNSecrets() error { + var otherFilter os.FileMode = 0o007 + for i, s := range conf.RTC.TURNServers { + if s.SecretFile == "" { + continue + } + if s.Secret != "" { + logger.Warnw("both secret and secret_file are set for TURN server, the hardcoded secret will be used", nil, + "host", s.Host, "port", s.Port) + continue + } + st, err := os.Stat(s.SecretFile) + if err != nil { + return err + } + if st.Mode().Perm()&otherFilter != 0o000 { + return ErrTURNSecretFileIncorrectPermission + } + data, err := os.ReadFile(s.SecretFile) + if err != nil { + return fmt.Errorf("reading turn secret file %q: %w", s.SecretFile, err) + } + conf.RTC.TURNServers[i].Secret = strings.TrimSpace(string(data)) + } + return nil +} + func GenerateCLIFlags(existingFlags []cli.Flag, hidden bool) ([]cli.Flag, error) { - blankConfig := &Config{} + defaultConfig := &DefaultConfig flags := make([]cli.Flag, 0) - for name, value := range blankConfig.ToCLIFlagNames(existingFlags) { + for name, value := range (defaultConfig).ToCLIFlagNames(existingFlags) { kind := value.Kind() if kind == reflect.Ptr { kind = value.Type().Elem().Kind() } var flag cli.Flag - envVar := fmt.Sprintf("LIVEKIT_%s", strings.ToUpper(strings.Replace(name, ".", "_", -1))) + envVar := fmt.Sprintf("LIVEKIT_%s", strings.ToUpper(strings.ReplaceAll(name, ".", "_"))) + defaultText := cliDefaultText(value) switch kind { case reflect.Bool: flag = &cli.BoolFlag{ - Name: name, - EnvVars: []string{envVar}, - Usage: generatedCLIFlagUsage, - Hidden: hidden, + Name: name, + Sources: cli.EnvVars(envVar), + Usage: generatedCLIFlagUsage, + DefaultText: defaultText, + Hidden: hidden, } case reflect.String: flag = &cli.StringFlag{ - Name: name, - EnvVars: []string{envVar}, - Usage: generatedCLIFlagUsage, - Hidden: hidden, + Name: name, + Sources: cli.EnvVars(envVar), + Usage: generatedCLIFlagUsage, + DefaultText: defaultText, + Hidden: hidden, } case reflect.Int, reflect.Int32: flag = &cli.IntFlag{ - Name: name, - EnvVars: []string{envVar}, - Usage: generatedCLIFlagUsage, - Hidden: hidden, + Name: name, + Sources: cli.EnvVars(envVar), + Usage: generatedCLIFlagUsage, + DefaultText: defaultText, + Hidden: hidden, } case reflect.Int64: flag = &cli.Int64Flag{ - Name: name, - EnvVars: []string{envVar}, - Usage: generatedCLIFlagUsage, - Hidden: hidden, + Name: name, + Sources: cli.EnvVars(envVar), + Usage: generatedCLIFlagUsage, + DefaultText: defaultText, + Hidden: hidden, } case reflect.Uint8, reflect.Uint16, reflect.Uint32: flag = &cli.UintFlag{ - Name: name, - EnvVars: []string{envVar}, - Usage: generatedCLIFlagUsage, - Hidden: hidden, + Name: name, + Sources: cli.EnvVars(envVar), + Usage: generatedCLIFlagUsage, + DefaultText: defaultText, + Hidden: hidden, } case reflect.Uint64: flag = &cli.Uint64Flag{ - Name: name, - EnvVars: []string{envVar}, - Usage: generatedCLIFlagUsage, - Hidden: hidden, + Name: name, + Sources: cli.EnvVars(envVar), + Usage: generatedCLIFlagUsage, + DefaultText: defaultText, + Hidden: hidden, } case reflect.Float32: flag = &cli.Float64Flag{ - Name: name, - EnvVars: []string{envVar}, - Usage: generatedCLIFlagUsage, - Hidden: hidden, + Name: name, + Sources: cli.EnvVars(envVar), + Usage: generatedCLIFlagUsage, + DefaultText: defaultText, + Hidden: hidden, } case reflect.Float64: flag = &cli.Float64Flag{ - Name: name, - EnvVars: []string{envVar}, - Usage: generatedCLIFlagUsage, - Hidden: hidden, + Name: name, + Sources: cli.EnvVars(envVar), + Usage: generatedCLIFlagUsage, + DefaultText: defaultText, + Hidden: hidden, } case reflect.Slice: // TODO @@ -651,13 +781,39 @@ func GenerateCLIFlags(existingFlags []cli.Flag, hidden bool) ([]cli.Flag, error) return flags, nil } -func (conf *Config) updateFromCLI(c *cli.Context, baseFlags []cli.Flag) error { +func cliDefaultText(value reflect.Value) string { + if value.Kind() == reflect.Ptr { + if value.IsNil() { + return "" + } + value = value.Elem() + } + + switch value.Kind() { + case reflect.Bool: + return strconv.FormatBool(value.Bool()) + case reflect.String: + return value.String() + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + if value.Type() == reflect.TypeOf(time.Duration(0)) { + return value.Interface().(time.Duration).String() + } + return strconv.FormatInt(value.Int(), 10) + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + return strconv.FormatUint(value.Uint(), 10) + case reflect.Float32, reflect.Float64: + return strconv.FormatFloat(value.Float(), 'f', -1, 64) + default: + return "" + } +} + +func (conf *Config) updateFromCLI(c *cli.Command, baseFlags []cli.Flag) error { generatedFlagNames := conf.ToCLIFlagNames(baseFlags) - for _, flag := range c.App.Flags { + for _, flag := range c.Flags { flagName := flag.Names()[0] - // the `c.App.Name != "test"` check is needed because `c.IsSet(...)` is always false in unit tests - if !c.IsSet(flagName) && c.App.Name != "test" { + if !c.IsSet(flagName) { continue } @@ -666,34 +822,16 @@ func (conf *Config) updateFromCLI(c *cli.Context, baseFlags []cli.Flag) error { continue } - kind := configValue.Kind() - if kind == reflect.Ptr { - // instantiate value to be set + if configValue.Kind() == reflect.Ptr { configValue.Set(reflect.New(configValue.Type().Elem())) - - kind = configValue.Type().Elem().Kind() configValue = configValue.Elem() } - switch kind { - case reflect.Bool: - configValue.SetBool(c.Bool(flagName)) - case reflect.String: - configValue.SetString(c.String(flagName)) - case reflect.Int, reflect.Int32, reflect.Int64: - configValue.SetInt(c.Int64(flagName)) - case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: - configValue.SetUint(c.Uint64(flagName)) - case reflect.Float32: - configValue.SetFloat(c.Float64(flagName)) - case reflect.Float64: - configValue.SetFloat(c.Float64(flagName)) - // case reflect.Slice: - // // TODO - // case reflect.Map: - // // TODO - default: - return fmt.Errorf("unsupported generated cli flag type for config: %s is a %s", flagName, kind.String()) + value := reflect.ValueOf(c.Value(flagName)) + if value.CanConvert(configValue.Type()) { + configValue.Set(value.Convert(configValue.Type())) + } else { + return fmt.Errorf("unsupported generated cli flag type for config: %s (expected %s, got %s)", flagName, configValue.Type(), value.Type()) } } @@ -724,7 +862,7 @@ func (conf *Config) updateFromCLI(c *cli.Context, baseFlags []cli.Flag) error { conf.TURN.KeyFile = c.String("turn-key") } if c.IsSet("node-ip") { - conf.RTC.NodeIP = c.String("node-ip") + conf.RTC.NodeIP.UnmarshalString(c.String("node-ip")) } if c.IsSet("udp-port") { conf.RTC.UDPPort.UnmarshalString(c.String("udp-port")) @@ -736,7 +874,7 @@ func (conf *Config) updateFromCLI(c *cli.Context, baseFlags []cli.Flag) error { } func (conf *Config) unmarshalKeys(keys string) error { - temp := make(map[string]interface{}) + temp := make(map[string]any) if err := yaml.Unmarshal([]byte(keys), temp); err != nil { return err } diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index cf5fb23..da2b2e6 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -15,11 +15,10 @@ package config import ( - "flag" "testing" "github.com/stretchr/testify/require" - "github.com/urfave/cli/v2" + "github.com/urfave/cli/v3" "github.com/livekit/livekit-server/pkg/config/configtest" ) @@ -53,19 +52,17 @@ func TestGeneratedFlags(t *testing.T) { generatedFlags, err := GenerateCLIFlags(nil, false) require.NoError(t, err) - app := cli.NewApp() - app.Name = "test" - app.Flags = append(app.Flags, generatedFlags...) + c := &cli.Command{} + c.Name = "test" + c.Flags = append(c.Flags, generatedFlags...) - set := flag.NewFlagSet("test", 0) - set.Bool("rtc.use_ice_lite", true, "") // bool - set.String("redis.address", "localhost:6379", "") // string - set.Uint("prometheus.port", 9999, "") // uint32 - set.Bool("rtc.allow_tcp_fallback", true, "") // pointer - set.Bool("rtc.reconnect_on_publication_error", true, "") // pointer - set.Bool("rtc.reconnect_on_subscription_error", false, "") // pointer + c.Set("rtc.use_ice_lite", "true") + c.Set("redis.address", "localhost:6379") + c.Set("prometheus.port", "9999") + c.Set("rtc.allow_tcp_fallback", "true") + c.Set("rtc.reconnect_on_publication_error", "true") + c.Set("rtc.reconnect_on_subscription_error", "false") - c := cli.NewContext(app, set, nil) conf, err := NewConfig("", true, c, nil) require.NoError(t, err) diff --git a/pkg/metric/metric_timestamper.go b/pkg/metric/metric_timestamper.go index c14f935..0ee0959 100644 --- a/pkg/metric/metric_timestamper.go +++ b/pkg/metric/metric_timestamper.go @@ -18,7 +18,7 @@ import ( "sync" "time" - "github.com/livekit/livekit-server/pkg/sfu/utils" + "github.com/livekit/mediatransportutil/pkg/latency" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" "github.com/livekit/protocol/utils/mono" @@ -49,7 +49,7 @@ type MetricTimestamperParams struct { type MetricTimestamper struct { params MetricTimestamperParams lock sync.Mutex - owdEstimator *utils.OWDEstimator + owdEstimator *latency.OWDEstimator lastOWDEstimatorRunAt time.Time batchesSinceLastOWDEstimatorRun int } @@ -57,7 +57,7 @@ type MetricTimestamper struct { func NewMetricTimestamper(params MetricTimestamperParams) *MetricTimestamper { return &MetricTimestamper{ params: params, - owdEstimator: utils.NewOWDEstimator(utils.OWDEstimatorParamsDefault), + owdEstimator: latency.NewOWDEstimator(latency.OWDEstimatorParamsDefault), lastOWDEstimatorRunAt: time.Now().Add(-params.Config.OneWayDelayEstimatorMinInterval), } } diff --git a/pkg/routing/interfaces.go b/pkg/routing/interfaces.go index 0312c93..6b2e1b6 100644 --- a/pkg/routing/interfaces.go +++ b/pkg/routing/interfaces.go @@ -23,6 +23,7 @@ import ( "go.uber.org/zap/zapcore" "google.golang.org/protobuf/proto" + "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/livekit-server/pkg/utils" "github.com/livekit/protocol/auth" "github.com/livekit/protocol/livekit" @@ -94,7 +95,7 @@ type NullMessageSource struct { func NewNullMessageSource(connID livekit.ConnectionID) *NullMessageSource { return &NullMessageSource{ connID: connID, - msgChan: make(chan proto.Message, 0), + msgChan: make(chan proto.Message), } } @@ -152,8 +153,13 @@ type StartParticipantSignalResults struct { type MessageRouter interface { // CreateRoom starts an rtc room CreateRoom(ctx context.Context, req *livekit.CreateRoomRequest) (res *livekit.Room, err error) + // StartParticipantSignal participant signal connection is ready to start - StartParticipantSignal(ctx context.Context, roomName livekit.RoomName, pi ParticipantInit) (res StartParticipantSignalResults, err error) + StartParticipantSignal( + ctx context.Context, + roomName livekit.RoomName, + pi ParticipantInit, + ) (res StartParticipantSignalResults, err error) } func CreateRouter( @@ -162,8 +168,9 @@ func CreateRouter( signalClient SignalClient, roomManagerClient RoomManagerClient, kps rpc.KeepalivePubSub, + nodeStatsConfig config.NodeStatsConfig, ) Router { - lr := NewLocalRouter(node, signalClient, roomManagerClient) + lr := NewLocalRouter(node, signalClient, roomManagerClient, nodeStatsConfig) if rc != nil { return NewRedisRouter(lr, rc, kps) @@ -177,19 +184,24 @@ func CreateRouter( // ------------------------------------------------ type ParticipantInit struct { - Identity livekit.ParticipantIdentity - Name livekit.ParticipantName - Reconnect bool - ReconnectReason livekit.ReconnectReason - AutoSubscribe bool - Client *livekit.ClientInfo - Grants *auth.ClaimGrants - Region string - AdaptiveStream bool - ID livekit.ParticipantID - SubscriberAllowPause *bool - DisableICELite bool - CreateRoom *livekit.CreateRoomRequest + Identity livekit.ParticipantIdentity + Name livekit.ParticipantName + Reconnect bool + ReconnectReason livekit.ReconnectReason + AutoSubscribe bool + AutoSubscribeDataTrack *bool + Client *livekit.ClientInfo + Grants *auth.ClaimGrants + Region string + AdaptiveStream bool + ID livekit.ParticipantID + SubscriberAllowPause *bool + DisableICELite bool + CreateRoom *livekit.CreateRoomRequest + AddTrackRequests []*livekit.AddTrackRequest + PublisherOffer *livekit.SessionDescription + SyncState *livekit.SyncState + UseSinglePeerConnection bool } func (pi *ParticipantInit) MarshalLogObject(e zapcore.ObjectEncoder) error { @@ -209,6 +221,7 @@ func (pi *ParticipantInit) MarshalLogObject(e zapcore.ObjectEncoder) error { logBoolPtr("Reconnect", &pi.Reconnect) e.AddString("ReconnectReason", pi.ReconnectReason.String()) logBoolPtr("AutoSubscribe", &pi.AutoSubscribe) + logBoolPtr("AutoSubscribeDataTrack", pi.AutoSubscribeDataTrack) e.AddObject("Client", logger.Proto(utils.ClientInfoWithoutAddress(pi.Client))) e.AddObject("Grants", pi.Grants) e.AddString("Region", pi.Region) @@ -217,6 +230,10 @@ func (pi *ParticipantInit) MarshalLogObject(e zapcore.ObjectEncoder) error { logBoolPtr("SubscriberAllowPause", pi.SubscriberAllowPause) logBoolPtr("DisableICELite", &pi.DisableICELite) e.AddObject("CreateRoom", logger.Proto(pi.CreateRoom)) + e.AddArray("AddTrackRequests", logger.ProtoSlice(pi.AddTrackRequests)) + e.AddObject("PublisherOffer", logger.Proto(pi.PublisherOffer)) + e.AddObject("SyncState", logger.Proto(pi.SyncState)) + logBoolPtr("UseSinglePeerConnection", &pi.UseSinglePeerConnection) return nil } @@ -227,20 +244,27 @@ func (pi *ParticipantInit) ToStartSession(roomName livekit.RoomName, connectionI } ss := &livekit.StartSession{ - RoomName: string(roomName), - Identity: string(pi.Identity), - Name: string(pi.Name), - // connection id is to allow the RTC node to identify where to route the message back to - ConnectionId: string(connectionID), - Reconnect: pi.Reconnect, - ReconnectReason: pi.ReconnectReason, - AutoSubscribe: pi.AutoSubscribe, - Client: pi.Client, - GrantsJson: string(claims), - AdaptiveStream: pi.AdaptiveStream, - ParticipantId: string(pi.ID), - DisableIceLite: pi.DisableICELite, - CreateRoom: pi.CreateRoom, + RoomName: string(roomName), + Identity: string(pi.Identity), + Name: string(pi.Name), + ConnectionId: string(connectionID), + Reconnect: pi.Reconnect, + ReconnectReason: pi.ReconnectReason, + AutoSubscribe: pi.AutoSubscribe, + Client: pi.Client, + GrantsJson: string(claims), + AdaptiveStream: pi.AdaptiveStream, + ParticipantId: string(pi.ID), + DisableIceLite: pi.DisableICELite, + CreateRoom: pi.CreateRoom, + AddTrackRequests: pi.AddTrackRequests, + PublisherOffer: pi.PublisherOffer, + SyncState: pi.SyncState, + UseSinglePeerConnection: pi.UseSinglePeerConnection, + } + if pi.AutoSubscribeDataTrack != nil { + autoSubscribeDataTrack := *pi.AutoSubscribeDataTrack + ss.AutoSubscribeDataTrack = &autoSubscribeDataTrack } if pi.SubscriberAllowPause != nil { subscriberAllowPause := *pi.SubscriberAllowPause @@ -257,18 +281,26 @@ func ParticipantInitFromStartSession(ss *livekit.StartSession, region string) (* } pi := &ParticipantInit{ - Identity: livekit.ParticipantIdentity(ss.Identity), - Name: livekit.ParticipantName(ss.Name), - Reconnect: ss.Reconnect, - ReconnectReason: ss.ReconnectReason, - Client: ss.Client, - AutoSubscribe: ss.AutoSubscribe, - Grants: claims, - Region: region, - AdaptiveStream: ss.AdaptiveStream, - ID: livekit.ParticipantID(ss.ParticipantId), - DisableICELite: ss.DisableIceLite, - CreateRoom: ss.CreateRoom, + Identity: livekit.ParticipantIdentity(ss.Identity), + Name: livekit.ParticipantName(ss.Name), + Reconnect: ss.Reconnect, + ReconnectReason: ss.ReconnectReason, + Client: ss.Client, + AutoSubscribe: ss.AutoSubscribe, + Grants: claims, + Region: region, + AdaptiveStream: ss.AdaptiveStream, + ID: livekit.ParticipantID(ss.ParticipantId), + DisableICELite: ss.DisableIceLite, + CreateRoom: ss.CreateRoom, + AddTrackRequests: ss.AddTrackRequests, + PublisherOffer: ss.PublisherOffer, + SyncState: ss.SyncState, + UseSinglePeerConnection: ss.UseSinglePeerConnection, + } + if ss.AutoSubscribeDataTrack != nil { + autoSubscribeDataTrack := *ss.AutoSubscribeDataTrack + pi.AutoSubscribeDataTrack = &autoSubscribeDataTrack } if ss.SubscriberAllowPause != nil { subscriberAllowPause := *ss.SubscriberAllowPause diff --git a/pkg/routing/localrouter.go b/pkg/routing/localrouter.go index 8baba75..b3e2db1 100644 --- a/pkg/routing/localrouter.go +++ b/pkg/routing/localrouter.go @@ -16,11 +16,11 @@ package routing import ( "context" - "sync" "time" "go.uber.org/atomic" + "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" ) @@ -32,8 +32,8 @@ type LocalRouter struct { currentNode LocalNode signalClient SignalClient roomManagerClient RoomManagerClient + nodeStatsConfig config.NodeStatsConfig - lock sync.RWMutex // channels for each participant requestChannels map[string]*MessageChannel responseChannels map[string]*MessageChannel @@ -44,11 +44,13 @@ func NewLocalRouter( currentNode LocalNode, signalClient SignalClient, roomManagerClient RoomManagerClient, + nodeStatsConfig config.NodeStatsConfig, ) *LocalRouter { return &LocalRouter{ currentNode: currentNode, signalClient: signalClient, roomManagerClient: roomManagerClient, + nodeStatsConfig: nodeStatsConfig, requestChannels: make(map[string]*MessageChannel), responseChannels: make(map[string]*MessageChannel), } @@ -106,7 +108,8 @@ func (r *LocalRouter) StartParticipantSignal(ctx context.Context, roomName livek func (r *LocalRouter) StartParticipantSignalWithNodeID(ctx context.Context, roomName livekit.RoomName, pi ParticipantInit, nodeID livekit.NodeID) (res StartParticipantSignalResults, err error) { connectionID, reqSink, resSource, err := r.signalClient.StartParticipantSignal(ctx, roomName, pi, nodeID) if err != nil { - logger.Errorw("could not handle new participant", err, + logger.Errorw( + "could not handle new participant", err, "room", roomName, "participant", pi.Identity, "connID", connectionID, @@ -146,8 +149,7 @@ func (r *LocalRouter) statsWorker() { if !r.isStarted.Load() { return } - // update every 10 seconds - <-time.After(statsUpdateInterval) + <-time.After(r.nodeStatsConfig.StatsUpdateInterval) r.currentNode.UpdateNodeStats() } } diff --git a/pkg/routing/messagechannel_test.go b/pkg/routing/messagechannel_test.go index bac68ef..ab0ab81 100644 --- a/pkg/routing/messagechannel_test.go +++ b/pkg/routing/messagechannel_test.go @@ -38,7 +38,7 @@ func TestMessageChannel_WriteMessageClosed(t *testing.T) { wg.Add(1) go func() { defer wg.Done() - for i := 0; i < 100; i++ { + for range 100 { _ = m.WriteMessage(&livekit.SignalRequest{}) } }() diff --git a/pkg/routing/node.go b/pkg/routing/node.go index bc41158..df47226 100644 --- a/pkg/routing/node.go +++ b/pkg/routing/node.go @@ -20,12 +20,10 @@ import ( "time" "github.com/livekit/protocol/livekit" - "github.com/livekit/protocol/logger" "github.com/livekit/protocol/utils" "github.com/livekit/protocol/utils/guid" "github.com/livekit/livekit-server/pkg/config" - "github.com/livekit/livekit-server/pkg/telemetry/prometheus" ) type LocalNode interface { @@ -45,30 +43,35 @@ type LocalNodeImpl struct { lock sync.RWMutex node *livekit.Node - // previous stats for computing averages - prevStats *livekit.NodeStats + nodeStats *NodeStats } func NewLocalNode(conf *config.Config) (*LocalNodeImpl, error) { nodeID := guid.New(utils.NodePrefix) - if conf != nil && conf.RTC.NodeIP == "" { + if conf != nil && conf.RTC.NodeIP.IsEmpty() { return nil, ErrIPNotSet } + nowUnix := time.Now().Unix() l := &LocalNodeImpl{ node: &livekit.Node{ Id: nodeID, NumCpus: uint32(runtime.NumCPU()), State: livekit.NodeState_SERVING, Stats: &livekit.NodeStats{ - StartedAt: time.Now().Unix(), - UpdatedAt: time.Now().Unix(), + StartedAt: nowUnix, + UpdatedAt: nowUnix, }, }, } + var nsc *config.NodeStatsConfig if conf != nil { - l.node.Ip = conf.RTC.NodeIP + l.node.Ip = conf.RTC.NodeIP.PrimaryIP() l.node.Region = conf.Region + + nsc = &conf.NodeStats } + l.nodeStats = NewNodeStats(nsc, nowUnix) + return l, nil } @@ -138,18 +141,12 @@ func (l *LocalNodeImpl) UpdateNodeStats() bool { l.lock.Lock() defer l.lock.Unlock() - if l.prevStats == nil { - l.prevStats = l.node.Stats - } - updated, computedAvg, err := prometheus.GetUpdatedNodeStats(l.node.Stats, l.prevStats) + stats, err := l.nodeStats.UpdateAndGetNodeStats() if err != nil { - logger.Errorw("could not update node stats", err) return false } - l.node.Stats = updated - if computedAvg { - l.prevStats = updated - } + + l.node.Stats = stats return true } diff --git a/pkg/routing/nodestats.go b/pkg/routing/nodestats.go new file mode 100644 index 0000000..999ae9c --- /dev/null +++ b/pkg/routing/nodestats.go @@ -0,0 +1,82 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package routing + +import ( + "sync" + "time" + + "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/logger" + + "github.com/livekit/livekit-server/pkg/config" + "github.com/livekit/livekit-server/pkg/telemetry/prometheus" +) + +type NodeStats struct { + config config.NodeStatsConfig + startedAt int64 + + lock sync.Mutex + statsHistory []*livekit.NodeStats + statsHistoryWritePtr int +} + +func NewNodeStats(conf *config.NodeStatsConfig, startedAt int64) *NodeStats { + n := &NodeStats{ + startedAt: startedAt, + } + n.UpdateConfig(conf) + return n +} + +func (n *NodeStats) UpdateConfig(conf *config.NodeStatsConfig) { + n.lock.Lock() + defer n.lock.Unlock() + + if conf == nil { + conf = &config.DefaultNodeStatsConfig + } + n.config = *conf + + // set up stats history to be able to measure different rate windows + var maxInterval time.Duration + for _, rateInterval := range conf.StatsRateMeasurementIntervals { + if rateInterval > maxInterval { + maxInterval = rateInterval + } + } + n.statsHistory = make([]*livekit.NodeStats, (maxInterval+conf.StatsUpdateInterval-1)/conf.StatsUpdateInterval) + n.statsHistoryWritePtr = 0 +} + +func (n *NodeStats) UpdateAndGetNodeStats() (*livekit.NodeStats, error) { + n.lock.Lock() + defer n.lock.Unlock() + + stats, err := prometheus.GetNodeStats( + n.startedAt, + append(n.statsHistory[n.statsHistoryWritePtr:], n.statsHistory[0:n.statsHistoryWritePtr]...), + n.config.StatsRateMeasurementIntervals, + ) + if err != nil { + logger.Errorw("could not update node stats", err) + return nil, err + } + + n.statsHistory[n.statsHistoryWritePtr] = stats + n.statsHistoryWritePtr = (n.statsHistoryWritePtr + 1) % len(n.statsHistory) + return stats, nil +} diff --git a/pkg/routing/redisrouter.go b/pkg/routing/redisrouter.go index 6dbe28e..44859b4 100644 --- a/pkg/routing/redisrouter.go +++ b/pkg/routing/redisrouter.go @@ -33,11 +33,6 @@ import ( ) const ( - // expire participant mappings after a day - participantMappingTTL = 24 * time.Hour - statsUpdateInterval = 2 * time.Second - statsMaxDelaySeconds = float64(30) - // hash of node_id => Node proto NodesKey = "nodes" @@ -209,11 +204,11 @@ func (r *RedisRouter) statsWorker() { for r.ctx.Err() == nil { // update periodically select { - case <-time.After(statsUpdateInterval): + case <-time.After(r.nodeStatsConfig.StatsUpdateInterval): r.kps.PublishPing(r.ctx, r.currentNode.NodeID(), &rpc.KeepalivePing{Timestamp: time.Now().Unix()}) delaySeconds := r.currentNode.SecondsSinceNodeStatsUpdate() - if delaySeconds > statsMaxDelaySeconds { + if delaySeconds > r.nodeStatsConfig.StatsMaxDelay.Seconds() { if !goroutineDumped { goroutineDumped = true buf := bytes.NewBuffer(nil) @@ -240,7 +235,7 @@ func (r *RedisRouter) keepaliveWorker(startedChan chan error) { close(startedChan) for ping := range pings.Channel() { - if time.Since(time.Unix(ping.Timestamp, 0)) > statsUpdateInterval { + if time.Since(time.Unix(ping.Timestamp, 0)) > r.nodeStatsConfig.StatsUpdateInterval { logger.Infow("keep alive too old, skipping", "timestamp", ping.Timestamp) continue } diff --git a/pkg/routing/routingfakes/fake_message_sink.go b/pkg/routing/routingfakes/fake_message_sink.go index 70c4172..5069f8b 100644 --- a/pkg/routing/routingfakes/fake_message_sink.go +++ b/pkg/routing/routingfakes/fake_message_sink.go @@ -243,14 +243,6 @@ func (fake *FakeMessageSink) WriteMessageReturnsOnCall(i int, result1 error) { func (fake *FakeMessageSink) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() - fake.closeMutex.RLock() - defer fake.closeMutex.RUnlock() - fake.connectionIDMutex.RLock() - defer fake.connectionIDMutex.RUnlock() - fake.isClosedMutex.RLock() - defer fake.isClosedMutex.RUnlock() - fake.writeMessageMutex.RLock() - defer fake.writeMessageMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} for key, value := range fake.invocations { copiedInvocations[key] = value diff --git a/pkg/routing/routingfakes/fake_message_source.go b/pkg/routing/routingfakes/fake_message_source.go index 02161ab..37eec2f 100644 --- a/pkg/routing/routingfakes/fake_message_source.go +++ b/pkg/routing/routingfakes/fake_message_source.go @@ -234,14 +234,6 @@ func (fake *FakeMessageSource) ReadChanReturnsOnCall(i int, result1 <-chan proto func (fake *FakeMessageSource) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() - fake.closeMutex.RLock() - defer fake.closeMutex.RUnlock() - fake.connectionIDMutex.RLock() - defer fake.connectionIDMutex.RUnlock() - fake.isClosedMutex.RLock() - defer fake.isClosedMutex.RUnlock() - fake.readChanMutex.RLock() - defer fake.readChanMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} for key, value := range fake.invocations { copiedInvocations[key] = value diff --git a/pkg/routing/routingfakes/fake_room_manager_client.go b/pkg/routing/routingfakes/fake_room_manager_client.go index d164ac1..6e7e652 100644 --- a/pkg/routing/routingfakes/fake_room_manager_client.go +++ b/pkg/routing/routingfakes/fake_room_manager_client.go @@ -129,10 +129,6 @@ func (fake *FakeRoomManagerClient) CreateRoomReturnsOnCall(i int, result1 *livek func (fake *FakeRoomManagerClient) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() - fake.closeMutex.RLock() - defer fake.closeMutex.RUnlock() - fake.createRoomMutex.RLock() - defer fake.createRoomMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} for key, value := range fake.invocations { copiedInvocations[key] = value diff --git a/pkg/routing/routingfakes/fake_router.go b/pkg/routing/routingfakes/fake_router.go index ca83775..48fec0b 100644 --- a/pkg/routing/routingfakes/fake_router.go +++ b/pkg/routing/routingfakes/fake_router.go @@ -845,32 +845,6 @@ func (fake *FakeRouter) UnregisterNodeReturnsOnCall(i int, result1 error) { func (fake *FakeRouter) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() - fake.clearRoomStateMutex.RLock() - defer fake.clearRoomStateMutex.RUnlock() - fake.createRoomMutex.RLock() - defer fake.createRoomMutex.RUnlock() - fake.drainMutex.RLock() - defer fake.drainMutex.RUnlock() - fake.getNodeForRoomMutex.RLock() - defer fake.getNodeForRoomMutex.RUnlock() - fake.getRegionMutex.RLock() - defer fake.getRegionMutex.RUnlock() - fake.listNodesMutex.RLock() - defer fake.listNodesMutex.RUnlock() - fake.registerNodeMutex.RLock() - defer fake.registerNodeMutex.RUnlock() - fake.removeDeadNodesMutex.RLock() - defer fake.removeDeadNodesMutex.RUnlock() - fake.setNodeForRoomMutex.RLock() - defer fake.setNodeForRoomMutex.RUnlock() - fake.startMutex.RLock() - defer fake.startMutex.RUnlock() - fake.startParticipantSignalMutex.RLock() - defer fake.startParticipantSignalMutex.RUnlock() - fake.stopMutex.RLock() - defer fake.stopMutex.RUnlock() - fake.unregisterNodeMutex.RLock() - defer fake.unregisterNodeMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} for key, value := range fake.invocations { copiedInvocations[key] = value diff --git a/pkg/routing/routingfakes/fake_signal_client.go b/pkg/routing/routingfakes/fake_signal_client.go index 0562b7c..c7eeff1 100644 --- a/pkg/routing/routingfakes/fake_signal_client.go +++ b/pkg/routing/routingfakes/fake_signal_client.go @@ -173,10 +173,6 @@ func (fake *FakeSignalClient) StartParticipantSignalReturnsOnCall(i int, result1 func (fake *FakeSignalClient) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() - fake.activeCountMutex.RLock() - defer fake.activeCountMutex.RUnlock() - fake.startParticipantSignalMutex.RLock() - defer fake.startParticipantSignalMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} for key, value := range fake.invocations { copiedInvocations[key] = value diff --git a/pkg/routing/selector/any.go b/pkg/routing/selector/any.go index 71f09ba..592e2ed 100644 --- a/pkg/routing/selector/any.go +++ b/pkg/routing/selector/any.go @@ -20,7 +20,8 @@ import ( // AnySelector selects any available node with no limitations type AnySelector struct { - SortBy string + SortBy string + Algorithm string } func (s *AnySelector) SelectNode(nodes []*livekit.Node) (*livekit.Node, error) { @@ -29,5 +30,5 @@ func (s *AnySelector) SelectNode(nodes []*livekit.Node) (*livekit.Node, error) { return nil, ErrNoAvailableNodes } - return SelectSortedNode(nodes, s.SortBy) + return SelectSortedNode(nodes, s.SortBy, s.Algorithm) } diff --git a/pkg/routing/selector/any_test.go b/pkg/routing/selector/any_test.go new file mode 100644 index 0000000..e7c6d36 --- /dev/null +++ b/pkg/routing/selector/any_test.go @@ -0,0 +1,287 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package selector + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/livekit/protocol/livekit" +) + +func createTestNode(id string, cpuLoad float32, numRooms int32, numClients int32, state livekit.NodeState) *livekit.Node { + return &livekit.Node{ + Id: id, + State: state, + Stats: &livekit.NodeStats{ + UpdatedAt: time.Now().Unix() - 1, // Recent update to be considered available + CpuLoad: cpuLoad, + NumRooms: numRooms, + NumClients: numClients, + NumCpus: 4, + LoadAvgLast1Min: cpuLoad * 4, // Simulate system load + }, + } +} + +func TestAnySelector_SelectNode_TwoChoice(t *testing.T) { + tests := []struct { + name string + sortBy string + algorithm string + nodes []*livekit.Node + wantErr string + expected string + notExpected string + }{ + { + name: "successful selection with cpuload sorting", + sortBy: "cpuload", + algorithm: "twochoice", + nodes: []*livekit.Node{ + createTestNode("node1", 0.8, 5, 10, livekit.NodeState_SERVING), + createTestNode("node2", 0.3, 2, 5, livekit.NodeState_SERVING), + createTestNode("node3", 0.6, 3, 8, livekit.NodeState_SERVING), + createTestNode("node4", 0.9, 6, 12, livekit.NodeState_SERVING), + }, + wantErr: "", + expected: "", // Not determinstic selection, so no specific expected node + notExpected: "node4", // Node with highest load should not be selected + }, + { + name: "successful selection with rooms sorting", + sortBy: "rooms", + algorithm: "twochoice", + nodes: []*livekit.Node{ + createTestNode("node1", 0.5, 8, 15, livekit.NodeState_SERVING), + createTestNode("node2", 0.4, 2, 5, livekit.NodeState_SERVING), + createTestNode("node3", 0.6, 12, 20, livekit.NodeState_SERVING), + }, + wantErr: "", + expected: "", // Not determinstic selection, so no specific expected node + notExpected: "node3", // Node with highest room count should not be selected + }, + { + name: "successful selection with clients sorting", + sortBy: "clients", + algorithm: "twochoice", + nodes: []*livekit.Node{ + createTestNode("node1", 0.5, 3, 25, livekit.NodeState_SERVING), + createTestNode("node2", 0.4, 2, 5, livekit.NodeState_SERVING), + createTestNode("node3", 0.6, 4, 30, livekit.NodeState_SERVING), + }, + wantErr: "", + expected: "", // Not determinstic selection, so no specific expected node + notExpected: "node3", // Node with highest clients should not be selected + }, + { + name: "empty nodes list", + sortBy: "cpuload", + algorithm: "twochoice", + nodes: []*livekit.Node{}, + wantErr: "could not find any available nodes", + }, + { + name: "no available nodes - all unavailable", + sortBy: "cpuload", + algorithm: "twochoice", + nodes: []*livekit.Node{ + { + Id: "node1", + State: livekit.NodeState_SERVING, + Stats: &livekit.NodeStats{ + UpdatedAt: time.Now().Unix() - 10, // Too old + CpuLoad: 0.3, + }, + }, + }, + wantErr: "could not find any available nodes", + }, + { + name: "no available nodes - not serving", + sortBy: "cpuload", + algorithm: "twochoice", + nodes: []*livekit.Node{ + { + Id: "node1", + State: livekit.NodeState_SHUTTING_DOWN, + Stats: &livekit.NodeStats{ + UpdatedAt: time.Now().Unix() - 1, + CpuLoad: 0.3, + }, + }, + }, + wantErr: "could not find any available nodes", + }, + { + name: "single available node", + sortBy: "cpuload", + algorithm: "twochoice", + nodes: []*livekit.Node{ + createTestNode("node1", 0.5, 3, 10, livekit.NodeState_SERVING), + }, + wantErr: "", + expected: "node1", // Should select the only available node + notExpected: "", // No other nodes to compare against + }, + { + name: "two available nodes", + sortBy: "cpuload", + algorithm: "twochoice", + nodes: []*livekit.Node{ + createTestNode("node1", 0.8, 5, 15, livekit.NodeState_SERVING), + createTestNode("node2", 0.3, 2, 5, livekit.NodeState_SERVING), + }, + wantErr: "", + expected: "node2", // Should select the node with lower load + notExpected: "node1", // Should not select the node with higher load + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + selector := &AnySelector{ + SortBy: tt.sortBy, + Algorithm: tt.algorithm, + } + + node, err := selector.SelectNode(tt.nodes) + + if tt.wantErr != "" { + require.Error(t, err) + require.Contains(t, err.Error(), tt.wantErr) + require.Nil(t, node) + } else { + require.NoError(t, err) + require.NotNil(t, node) + require.NotEmpty(t, node.Id) + + // Verify the selected node is one of the available nodes + found := false + availableNodes := GetAvailableNodes(tt.nodes) + for _, availableNode := range availableNodes { + if availableNode.Id == node.Id { + found = true + break + } + } + require.True(t, found, "Selected node should be one of the available nodes") + + if tt.expected != "" { + require.Equal(t, tt.expected, node.Id, "Selected node should match expected") + } + if tt.notExpected != "" { + require.NotEqual(t, tt.notExpected, node.Id, "Selected node should not match not expected") + } + } + }) + } +} + +func TestAnySelector_SelectNode_TwoChoice_Probabilistic_Behavior(t *testing.T) { + // Test that two-choice algorithm favors nodes with lower metrics + // This test runs multiple iterations to increase confidence in the probabilistic behavior + selector := &AnySelector{ + SortBy: "cpuload", + Algorithm: "twochoice", + } + + // Create nodes where node2 has significantly lower CPU load + nodes := []*livekit.Node{ + createTestNode("node1", 0.95, 10, 20, livekit.NodeState_SERVING), // Very high load + createTestNode("node2", 0.1, 1, 2, livekit.NodeState_SERVING), // Low load + createTestNode("node3", 0.5, 9, 18, livekit.NodeState_SERVING), // Medium load + createTestNode("node4", 0.85, 8, 16, livekit.NodeState_SERVING), // High load + } + + // Run multiple selections and count how often the low-load node is selected + iterations := 1000 + lowLoadSelections := 0 + higestLoadSelections := 0 + + for range iterations { + node, err := selector.SelectNode(nodes) + require.NoError(t, err) + require.NotNil(t, node) + + if node.Id == "node2" { + lowLoadSelections++ + } + if node.Id == "node1" { + higestLoadSelections++ + } + } + + // The low-load node should be selected more often than pure random (25%) + // Due to the two-choice algorithm favoring the better node + selectionRate := float64(lowLoadSelections) / float64(iterations) + require.Greater(t, selectionRate, 0.4, "Two-choice algorithm should favor the low-load node more than random selection") + require.Equal(t, higestLoadSelections, 0, "Two-choice algorithm should never favor the highest load node") +} + +func TestAnySelector_SelectNode_InvalidParameters(t *testing.T) { + nodes := []*livekit.Node{ + createTestNode("node1", 0.5, 3, 10, livekit.NodeState_SERVING), + } + + tests := []struct { + name string + sortBy string + algorithm string + wantErr string + }{ + { + name: "empty sortBy", + sortBy: "", + algorithm: "twochoice", + wantErr: "sort by option cannot be blank", + }, + { + name: "empty algorithm", + sortBy: "cpuload", + algorithm: "", + wantErr: "node selector algorithm option cannot be blank", + }, + { + name: "unknown sortBy", + sortBy: "invalid", + algorithm: "twochoice", + wantErr: "unknown sort by option", + }, + { + name: "unknown algorithm", + sortBy: "cpuload", + algorithm: "invalid", + wantErr: "unknown node selector algorithm option", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + selector := &AnySelector{ + SortBy: tt.sortBy, + Algorithm: tt.algorithm, + } + + node, err := selector.SelectNode(nodes) + + require.Error(t, err) + require.Contains(t, err.Error(), tt.wantErr) + require.Nil(t, node) + }) + } +} diff --git a/pkg/routing/selector/cpuload.go b/pkg/routing/selector/cpuload.go index 1cd04c4..f84b4a3 100644 --- a/pkg/routing/selector/cpuload.go +++ b/pkg/routing/selector/cpuload.go @@ -23,6 +23,7 @@ import ( type CPULoadSelector struct { CPULoadLimit float32 SortBy string + Algorithm string } func (s *CPULoadSelector) filterNodes(nodes []*livekit.Node) ([]*livekit.Node, error) { @@ -50,5 +51,5 @@ func (s *CPULoadSelector) SelectNode(nodes []*livekit.Node) (*livekit.Node, erro return nil, err } - return SelectSortedNode(nodes, s.SortBy) + return SelectSortedNode(nodes, s.SortBy, s.Algorithm) } diff --git a/pkg/routing/selector/cpuload_test.go b/pkg/routing/selector/cpuload_test.go index 33bca47..b267dcf 100644 --- a/pkg/routing/selector/cpuload_test.go +++ b/pkg/routing/selector/cpuload_test.go @@ -25,7 +25,7 @@ import ( ) func TestCPULoadSelector_SelectNode(t *testing.T) { - sel := selector.CPULoadSelector{CPULoadLimit: 0.8, SortBy: "random"} + sel := selector.CPULoadSelector{CPULoadLimit: 0.8, SortBy: "random", Algorithm: "lowest"} var nodes []*livekit.Node _, err := sel.SelectNode(nodes) @@ -39,7 +39,7 @@ func TestCPULoadSelector_SelectNode(t *testing.T) { // Select a node with low load when available nodes = []*livekit.Node{nodeLoadLow, nodeLoadHigh} - for i := 0; i < 5; i++ { + for range 5 { node, err := sel.SelectNode(nodes) if err != nil { t.Error(err) diff --git a/pkg/routing/selector/errors.go b/pkg/routing/selector/errors.go index c011f67..690153a 100644 --- a/pkg/routing/selector/errors.go +++ b/pkg/routing/selector/errors.go @@ -21,5 +21,7 @@ var ( ErrCurrentRegionNotSet = errors.New("current region cannot be blank") ErrCurrentRegionUnknownLatLon = errors.New("unknown lat and lon for the current region") ErrSortByNotSet = errors.New("sort by option cannot be blank") + ErrAlgorithmNotSet = errors.New("node selector algorithm option cannot be blank") ErrSortByUnknown = errors.New("unknown sort by option") + ErrAlgorithmUnknown = errors.New("unknown node selector algorithm option") ) diff --git a/pkg/routing/selector/interfaces.go b/pkg/routing/selector/interfaces.go index 60d001e..6fc223e 100644 --- a/pkg/routing/selector/interfaces.go +++ b/pkg/routing/selector/interfaces.go @@ -37,19 +37,21 @@ func CreateNodeSelector(conf *config.Config) (NodeSelector, error) { } switch kind { case "any": - return &AnySelector{conf.NodeSelector.SortBy}, nil + return &AnySelector{conf.NodeSelector.SortBy, conf.NodeSelector.Algorithm}, nil case "cpuload": return &CPULoadSelector{ CPULoadLimit: conf.NodeSelector.CPULoadLimit, - SortBy: conf.NodeSelector.SortBy, + SortBy: conf.NodeSelector.SortBy, + Algorithm: conf.NodeSelector.Algorithm, }, nil case "sysload": return &SystemLoadSelector{ SysloadLimit: conf.NodeSelector.SysloadLimit, - SortBy: conf.NodeSelector.SortBy, + SortBy: conf.NodeSelector.SortBy, + Algorithm: conf.NodeSelector.Algorithm, }, nil case "regionaware": - s, err := NewRegionAwareSelector(conf.Region, conf.NodeSelector.Regions, conf.NodeSelector.SortBy) + s, err := NewRegionAwareSelector(conf.Region, conf.NodeSelector.Regions, conf.NodeSelector.SortBy, conf.NodeSelector.Algorithm) if err != nil { return nil, err } @@ -57,7 +59,7 @@ func CreateNodeSelector(conf *config.Config) (NodeSelector, error) { return s, nil case "random": logger.Warnw("random node selector is deprecated, please switch to \"any\" or another selector", nil) - return &AnySelector{conf.NodeSelector.SortBy}, nil + return &AnySelector{conf.NodeSelector.SortBy, conf.NodeSelector.Algorithm}, nil default: return nil, ErrUnsupportedSelector } diff --git a/pkg/routing/selector/regionaware.go b/pkg/routing/selector/regionaware.go index 61f494f..756781a 100644 --- a/pkg/routing/selector/regionaware.go +++ b/pkg/routing/selector/regionaware.go @@ -29,9 +29,10 @@ type RegionAwareSelector struct { regionDistances map[string]float64 regions []config.RegionConfig SortBy string + Algorithm string } -func NewRegionAwareSelector(currentRegion string, regions []config.RegionConfig, sortBy string) (*RegionAwareSelector, error) { +func NewRegionAwareSelector(currentRegion string, regions []config.RegionConfig, sortBy string, algorithm string) (*RegionAwareSelector, error) { if currentRegion == "" { return nil, ErrCurrentRegionNotSet } @@ -41,6 +42,7 @@ func NewRegionAwareSelector(currentRegion string, regions []config.RegionConfig, regionDistances: make(map[string]float64), regions: regions, SortBy: sortBy, + Algorithm: algorithm, } var currentRC *config.RegionConfig @@ -94,7 +96,7 @@ func (s *RegionAwareSelector) SelectNode(nodes []*livekit.Node) (*livekit.Node, nodes = nearestNodes } - return SelectSortedNode(nodes, s.SortBy) + return SelectSortedNode(nodes, s.SortBy, s.Algorithm) } // haversine(θ) function diff --git a/pkg/routing/selector/regionaware_test.go b/pkg/routing/selector/regionaware_test.go index bc4b203..7433101 100644 --- a/pkg/routing/selector/regionaware_test.go +++ b/pkg/routing/selector/regionaware_test.go @@ -34,6 +34,7 @@ const ( regionEast = "us-east" regionSeattle = "seattle" sortBy = "random" + algorithm = "lowest" ) func TestRegionAwareRouting(t *testing.T) { @@ -58,7 +59,7 @@ func TestRegionAwareRouting(t *testing.T) { nodes := []*livekit.Node{ newTestNodeInRegion("", false), } - s, err := selector.NewRegionAwareSelector(regionEast, nil, sortBy) + s, err := selector.NewRegionAwareSelector(regionEast, nil, sortBy, algorithm) require.NoError(t, err) node, err := s.SelectNode(nodes) @@ -74,7 +75,7 @@ func TestRegionAwareRouting(t *testing.T) { expectedNode, newTestNodeInRegion(regionEast, false), } - s, err := selector.NewRegionAwareSelector(regionEast, rc, sortBy) + s, err := selector.NewRegionAwareSelector(regionEast, rc, sortBy, algorithm) require.NoError(t, err) s.SysloadLimit = loadLimit @@ -91,7 +92,7 @@ func TestRegionAwareRouting(t *testing.T) { newTestNodeInRegion(regionWest, true), newTestNodeInRegion(regionEast, false), } - s, err := selector.NewRegionAwareSelector(regionEast, rc, sortBy) + s, err := selector.NewRegionAwareSelector(regionEast, rc, sortBy, algorithm) require.NoError(t, err) s.SysloadLimit = loadLimit @@ -107,7 +108,7 @@ func TestRegionAwareRouting(t *testing.T) { expectedNode, newTestNodeInRegion(regionEast, true), } - s, err := selector.NewRegionAwareSelector(regionSeattle, rc, sortBy) + s, err := selector.NewRegionAwareSelector(regionSeattle, rc, sortBy, algorithm) require.NoError(t, err) s.SysloadLimit = loadLimit @@ -125,7 +126,7 @@ func TestRegionAwareRouting(t *testing.T) { expectedNode, expectedNode, } - s, err := selector.NewRegionAwareSelector(regionSeattle, rc, sortBy) + s, err := selector.NewRegionAwareSelector(regionSeattle, rc, sortBy, algorithm) require.NoError(t, err) s.SysloadLimit = loadLimit @@ -138,7 +139,7 @@ func TestRegionAwareRouting(t *testing.T) { nodes := []*livekit.Node{ newTestNodeInRegion(regionWest, true), } - s, err := selector.NewRegionAwareSelector(regionEast, rc, sortBy) + s, err := selector.NewRegionAwareSelector(regionEast, rc, sortBy, algorithm) require.NoError(t, err) node, err := s.SelectNode(nodes) diff --git a/pkg/routing/selector/sortby_test.go b/pkg/routing/selector/sortby_test.go index 31de002..57260e4 100644 --- a/pkg/routing/selector/sortby_test.go +++ b/pkg/routing/selector/sortby_test.go @@ -23,10 +23,10 @@ import ( ) func SortByTest(t *testing.T, sortBy string) { - sel := selector.SystemLoadSelector{SortBy: sortBy} + sel := selector.SystemLoadSelector{SortBy: sortBy, Algorithm: "lowest"} nodes := []*livekit.Node{nodeLoadLow, nodeLoadMedium, nodeLoadHigh} - for i := 0; i < 5; i++ { + for range 5 { node, err := sel.SelectNode(nodes) if err != nil { t.Error(err) @@ -38,7 +38,7 @@ func SortByTest(t *testing.T, sortBy string) { } func TestSortByErrors(t *testing.T) { - sel := selector.SystemLoadSelector{} + sel := selector.SystemLoadSelector{Algorithm: "lowest"} nodes := []*livekit.Node{nodeLoadLow, nodeLoadMedium, nodeLoadHigh} // Test unset sort by option error diff --git a/pkg/routing/selector/sysload.go b/pkg/routing/selector/sysload.go index 909311a..285fca4 100644 --- a/pkg/routing/selector/sysload.go +++ b/pkg/routing/selector/sysload.go @@ -23,6 +23,7 @@ import ( type SystemLoadSelector struct { SysloadLimit float32 SortBy string + Algorithm string } func (s *SystemLoadSelector) filterNodes(nodes []*livekit.Node) ([]*livekit.Node, error) { @@ -49,5 +50,5 @@ func (s *SystemLoadSelector) SelectNode(nodes []*livekit.Node) (*livekit.Node, e return nil, err } - return SelectSortedNode(nodes, s.SortBy) + return SelectSortedNode(nodes, s.SortBy, s.Algorithm) } diff --git a/pkg/routing/selector/sysload_test.go b/pkg/routing/selector/sysload_test.go index ac7d59a..f2dd342 100644 --- a/pkg/routing/selector/sysload_test.go +++ b/pkg/routing/selector/sysload_test.go @@ -33,12 +33,16 @@ var ( NumCpus: 1, CpuLoad: 0.1, LoadAvgLast1Min: 0.0, - NumRooms: 1, - NumClients: 2, - NumTracksIn: 4, - NumTracksOut: 8, - BytesInPerSec: 1000, - BytesOutPerSec: 2000, + NumRooms: 1, + NumClients: 2, + NumTracksIn: 4, + NumTracksOut: 8, + Rates: []*livekit.NodeStatsRate{ + { + BytesIn: 1000, + BytesOut: 2000, + }, + }, }, } @@ -49,12 +53,16 @@ var ( NumCpus: 1, CpuLoad: 0.5, LoadAvgLast1Min: 0.5, - NumRooms: 5, - NumClients: 10, - NumTracksIn: 20, - NumTracksOut: 200, - BytesInPerSec: 5000, - BytesOutPerSec: 10000, + NumRooms: 5, + NumClients: 10, + NumTracksIn: 20, + NumTracksOut: 200, + Rates: []*livekit.NodeStatsRate{ + { + BytesIn: 5000, + BytesOut: 10000, + }, + }, }, } @@ -65,18 +73,22 @@ var ( NumCpus: 1, CpuLoad: 0.99, LoadAvgLast1Min: 2.0, - NumRooms: 10, - NumClients: 20, - NumTracksIn: 40, - NumTracksOut: 800, - BytesInPerSec: 10000, - BytesOutPerSec: 40000, + NumRooms: 10, + NumClients: 20, + NumTracksIn: 40, + NumTracksOut: 800, + Rates: []*livekit.NodeStatsRate{ + { + BytesIn: 10000, + BytesOut: 40000, + }, + }, }, } ) func TestSystemLoadSelector_SelectNode(t *testing.T) { - sel := selector.SystemLoadSelector{SysloadLimit: 1.0, SortBy: "random"} + sel := selector.SystemLoadSelector{SysloadLimit: 1.0, SortBy: "random", Algorithm: "lowest"} var nodes []*livekit.Node _, err := sel.SelectNode(nodes) @@ -90,7 +102,7 @@ func TestSystemLoadSelector_SelectNode(t *testing.T) { // Select a node with low load when available nodes = []*livekit.Node{nodeLoadLow, nodeLoadHigh} - for i := 0; i < 5; i++ { + for range 5 { node, err := sel.SelectNode(nodes) if err != nil { t.Error(err) diff --git a/pkg/routing/selector/utils.go b/pkg/routing/selector/utils.go index 2ba0b38..cf6738b 100644 --- a/pkg/routing/selector/utils.go +++ b/pkg/routing/selector/utils.go @@ -15,7 +15,8 @@ package selector import ( - "sort" + "math/rand/v2" + "slices" "time" "github.com/thoas/go-funk" @@ -23,6 +24,7 @@ import ( "github.com/livekit/protocol/livekit" "github.com/livekit/livekit-server/pkg/config" + "github.com/livekit/livekit-server/pkg/utils" ) const AvailableSeconds = 5 @@ -62,54 +64,116 @@ func LimitsReached(limitConfig config.LimitConfig, nodeStats *livekit.NodeStats) if limitConfig.NumTracks > 0 && limitConfig.NumTracks <= nodeStats.NumTracksIn+nodeStats.NumTracksOut { return true } - if limitConfig.BytesPerSec > 0 && limitConfig.BytesPerSec <= nodeStats.BytesInPerSec+nodeStats.BytesOutPerSec { + + rate := &livekit.NodeStatsRate{} + if len(nodeStats.Rates) > 0 { + rate = nodeStats.Rates[0] + } + if limitConfig.BytesPerSec > 0 && limitConfig.BytesPerSec <= rate.BytesIn+rate.BytesOut { return true } return false } -func SelectSortedNode(nodes []*livekit.Node, sortBy string) (*livekit.Node, error) { +func SelectSortedNode(nodes []*livekit.Node, sortBy string, algorithm string) (*livekit.Node, error) { if sortBy == "" { return nil, ErrSortByNotSet } + if algorithm == "" { + return nil, ErrAlgorithmNotSet + } + switch algorithm { + case "lowest": // examine all nodes and select the lowest based on sort criteria + return selectLowestSortedNode(nodes, sortBy) + case "twochoice": // randomly select two nodes and return the lowest based on sort criteria "Power of Two Random Choices" + return selectTwoChoiceSortedNode(nodes, sortBy) + default: + return nil, ErrAlgorithmUnknown + } +} + +func selectTwoChoiceSortedNode(nodes []*livekit.Node, sortBy string) (*livekit.Node, error) { + if len(nodes) <= 2 { + return selectLowestSortedNode(nodes, sortBy) + } + + // randomly select two nodes + node1, node2, err := selectTwoRandomNodes(nodes) + if err != nil { + return nil, err + } + + // compare the two nodes based on the sort criteria + if node1 == nil || node2 == nil { + return nil, ErrNoAvailableNodes + } + + selectedNode, err := selectLowestSortedNode([]*livekit.Node{node1, node2}, sortBy) + if err != nil { + return nil, err + } + + return selectedNode, nil +} + +func selectLowestSortedNode(nodes []*livekit.Node, sortBy string) (*livekit.Node, error) { // Return a node based on what it should be sorted by for priority switch sortBy { case "random": idx := funk.RandomInt(0, len(nodes)) return nodes[idx], nil case "sysload": - sort.Slice(nodes, func(i, j int) bool { - return GetNodeSysload(nodes[i]) < GetNodeSysload(nodes[j]) + slices.SortFunc(nodes, func(a, b *livekit.Node) int { + return utils.Signum(GetNodeSysload(a) - GetNodeSysload(b)) }) return nodes[0], nil case "cpuload": - sort.Slice(nodes, func(i, j int) bool { - return nodes[i].Stats.CpuLoad < nodes[j].Stats.CpuLoad + slices.SortFunc(nodes, func(a, b *livekit.Node) int { + return utils.Signum(a.Stats.CpuLoad - b.Stats.CpuLoad) }) return nodes[0], nil case "rooms": - sort.Slice(nodes, func(i, j int) bool { - return nodes[i].Stats.NumRooms < nodes[j].Stats.NumRooms + slices.SortFunc(nodes, func(a, b *livekit.Node) int { + return utils.Signum(a.Stats.NumRooms - b.Stats.NumRooms) }) return nodes[0], nil case "clients": - sort.Slice(nodes, func(i, j int) bool { - return nodes[i].Stats.NumClients < nodes[j].Stats.NumClients + slices.SortFunc(nodes, func(a, b *livekit.Node) int { + return utils.Signum(a.Stats.NumClients - b.Stats.NumClients) }) return nodes[0], nil case "tracks": - sort.Slice(nodes, func(i, j int) bool { - return nodes[i].Stats.NumTracksIn+nodes[i].Stats.NumTracksOut < nodes[j].Stats.NumTracksIn+nodes[j].Stats.NumTracksOut + slices.SortFunc(nodes, func(a, b *livekit.Node) int { + return utils.Signum((a.Stats.NumTracksIn + a.Stats.NumTracksOut) - (b.Stats.NumTracksIn + b.Stats.NumTracksOut)) }) return nodes[0], nil case "bytespersec": - sort.Slice(nodes, func(i, j int) bool { - return nodes[i].Stats.BytesInPerSec+nodes[i].Stats.BytesOutPerSec < nodes[j].Stats.BytesInPerSec+nodes[j].Stats.BytesOutPerSec + slices.SortFunc(nodes, func(a, b *livekit.Node) int { + ratea := &livekit.NodeStatsRate{} + if len(a.Stats.Rates) > 0 { + ratea = a.Stats.Rates[0] + } + + rateb := &livekit.NodeStatsRate{} + if len(b.Stats.Rates) > 0 { + rateb = b.Stats.Rates[0] + } + return utils.Signum((ratea.BytesIn + ratea.BytesOut) - (rateb.BytesIn + rateb.BytesOut)) }) return nodes[0], nil default: return nil, ErrSortByUnknown } } + +func selectTwoRandomNodes(nodes []*livekit.Node) (*livekit.Node, *livekit.Node, error) { + if len(nodes) < 2 { + return nil, nil, ErrNoAvailableNodes + } + + shuffledIndices := rand.Perm(len(nodes)) + + return nodes[shuffledIndices[0]], nodes[shuffledIndices[1]], nil +} diff --git a/pkg/routing/signal.go b/pkg/routing/signal.go index ce7c5f4..7bb25f0 100644 --- a/pkg/routing/signal.go +++ b/pkg/routing/signal.go @@ -25,6 +25,7 @@ import ( "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/livekit-server/pkg/telemetry/prometheus" + "github.com/livekit/livekit-server/pkg/utils" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" "github.com/livekit/protocol/rpc" @@ -50,7 +51,7 @@ type signalClient struct { } func NewSignalClient(nodeID livekit.NodeID, bus psrpc.MessageBus, config config.SignalRelayConfig) (SignalClient, error) { - c, err := rpc.NewTypedSignalClient( + client, err := rpc.NewTypedSignalClient( nodeID, bus, middleware.WithClientMetrics(rpc.PSRPCMetricsObserver{}), @@ -63,7 +64,7 @@ func NewSignalClient(nodeID livekit.NodeID, bus psrpc.MessageBus, config config. return &signalClient{ nodeID: nodeID, config: config, - client: c, + client: client, }, nil } @@ -88,25 +89,27 @@ func (r *signalClient) StartParticipantSignal( return } - l := logger.GetLogger().WithValues( + l := utils.GetLogger(ctx).WithValues( "room", roomName, "reqNodeID", nodeID, "participant", pi.Identity, "connID", connectionID, + "participantInit", &pi, + "startSession", logger.Proto(ss), ) l.Debugw("starting signal connection") stream, err := r.client.RelaySignal(ctx, nodeID) if err != nil { - prometheus.MessageCounter.WithLabelValues("signal", "failure").Add(1) + prometheus.RecordSignalRequestFailure() return } err = stream.Send(&rpc.RelaySignalRequest{StartSession: ss}) if err != nil { stream.Close(err) - prometheus.MessageCounter.WithLabelValues("signal", "failure").Add(1) + prometheus.RecordSignalRequestFailure() return } @@ -130,6 +133,8 @@ func (r *signalClient) StartParticipantSignal( resChan, signalResponseMessageReader{}, r.config, + prometheus.RecordSignalResponseSuccess, + prometheus.RecordSignalResponseFailure, ) l.Debugw("signal stream closed", "error", err) @@ -139,6 +144,8 @@ func (r *signalClient) StartParticipantSignal( return connectionID, sink, resChan, nil } +// ------------------------------ + type signalRequestMessageWriter struct{} func (e signalRequestMessageWriter) Write(seq uint64, close bool, msgs []proto.Message) *rpc.RelaySignalRequest { @@ -153,6 +160,8 @@ func (e signalRequestMessageWriter) Write(seq uint64, close bool, msgs []proto.M return r } +// ------------------------------- + type signalResponseMessageReader struct{} func (e signalResponseMessageReader) Read(rm *rpc.RelaySignalResponse) ([]proto.Message, error) { @@ -163,6 +172,8 @@ func (e signalResponseMessageReader) Read(rm *rpc.RelaySignalResponse) ([]proto. return msgs, nil } +// ----------------------------------------- + type RelaySignalMessage interface { proto.Message GetSeq() uint64 @@ -182,6 +193,8 @@ func CopySignalStreamToMessageChannel[SendType, RecvType RelaySignalMessage]( ch *MessageChannel, reader SignalMessageReader[RecvType], config config.SignalRelayConfig, + promSignalSuccess func(), + promSignalFailure func(), ) error { r := &signalMessageReader[SendType, RecvType]{ reader: reader, @@ -190,16 +203,16 @@ func CopySignalStreamToMessageChannel[SendType, RecvType RelaySignalMessage]( for msg := range stream.Channel() { res, err := r.Read(msg) if err != nil { - prometheus.MessageCounter.WithLabelValues("signal", "failure").Add(1) + promSignalFailure() return err } for _, r := range res { if err = ch.WriteMessage(r); err != nil { - prometheus.MessageCounter.WithLabelValues("signal", "failure").Add(1) + promSignalFailure() return err } - prometheus.MessageCounter.WithLabelValues("signal", "success").Add(1) + promSignalSuccess() } if msg.GetClose() { @@ -209,6 +222,8 @@ func CopySignalStreamToMessageChannel[SendType, RecvType RelaySignalMessage]( return stream.Err() } +// ---------------------------------------- + type signalMessageReader[SendType, RecvType RelaySignalMessage] struct { seq uint64 reader SignalMessageReader[RecvType] @@ -236,6 +251,8 @@ func (r *signalMessageReader[SendType, RecvType]) Read(msg RecvType) ([]proto.Me return res, nil } +// ---------------------------------------- + type SignalSinkParams[SendType, RecvType RelaySignalMessage] struct { Stream psrpc.Stream[SendType, RecvType] Logger logger.Logger diff --git a/pkg/rtc/clientinfo.go b/pkg/rtc/clientinfo.go index 463e32d..899d60a 100644 --- a/pkg/rtc/clientinfo.go +++ b/pkg/rtc/clientinfo.go @@ -15,6 +15,7 @@ package rtc import ( + "slices" "strconv" "strings" @@ -45,11 +46,15 @@ func (c ClientInfo) isAndroid() bool { return c.ClientInfo != nil && strings.EqualFold(c.ClientInfo.Os, "android") } +func (c ClientInfo) isOBS() bool { + return c.ClientInfo != nil && strings.Contains(c.ClientInfo.Browser, "OBS") +} + func (c ClientInfo) SupportsAudioRED() bool { return !c.isFirefox() && !c.isSafari() } -func (c ClientInfo) SupportPrflxOverRelay() bool { +func (c ClientInfo) SupportsPrflxOverRelay() bool { return !c.isFirefox() } @@ -93,21 +98,36 @@ func (c ClientInfo) SupportsChangeRTPSenderEncodingActive() bool { } func (c ClientInfo) ComplyWithCodecOrderInSDPAnswer() bool { - return !((c.isLinux() || c.isAndroid()) && c.isFirefox()) + return (!c.isLinux() && !c.isAndroid()) || !c.isFirefox() } // Rust SDK can't decode unknown signal message (TrackSubscribed and ErrorResponse) -func (c ClientInfo) SupportTrackSubscribedEvent() bool { - return !(c.ClientInfo.GetSdk() == livekit.ClientInfo_RUST && c.ClientInfo.GetProtocol() < 10) +func (c ClientInfo) SupportsTrackSubscribedEvent() bool { + return c.ClientInfo.GetSdk() != livekit.ClientInfo_RUST || c.ClientInfo.GetProtocol() >= 10 } -func (c ClientInfo) SupportErrorResponse() bool { - return c.SupportTrackSubscribedEvent() +func (c ClientInfo) SupportsRequestResponse() bool { + return c.SupportsTrackSubscribedEvent() } -func (c ClientInfo) SupportSctpZeroChecksum() bool { - return !(c.ClientInfo.GetSdk() == livekit.ClientInfo_UNKNOWN || - (c.isGo() && c.compareVersion("2.4.0") < 0)) +func (c ClientInfo) SupportsSctpZeroChecksum() bool { + return c.ClientInfo.GetSdk() != livekit.ClientInfo_UNKNOWN && + (!c.isGo() || c.compareVersion("2.4.0") >= 0) +} + +func (c ClientInfo) SupportsTransceiverReuse() bool { + return !c.isSafari() +} + +func (c ClientInfo) HasCapability(cap livekit.ClientInfo_Capability) bool { + if c.ClientInfo == nil { + return false + } + return slices.Contains(c.ClientInfo.Capabilities, cap) +} + +func (c ClientInfo) SupportsPacketTrailer() bool { + return c.HasCapability(livekit.ClientInfo_CAP_PACKET_TRAILER) } // compareVersion compares a semver against the current client SDK version @@ -121,7 +141,7 @@ func (c ClientInfo) compareVersion(version string) int { parts1 := strings.Split(version, ".") ints0 := make([]int, 3) ints1 := make([]int, 3) - for i := 0; i < 3; i++ { + for i := range 3 { if len(parts0) > i { ints0[i], _ = strconv.Atoi(parts0[i]) } diff --git a/pkg/rtc/config.go b/pkg/rtc/config.go index 1a87a0f..2f30060 100644 --- a/pkg/rtc/config.go +++ b/pkg/rtc/config.go @@ -20,13 +20,14 @@ import ( "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/livekit-server/pkg/sfu/buffer" + act "github.com/livekit/livekit-server/pkg/sfu/rtpextension/abscapturetime" dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor" "github.com/livekit/mediatransportutil/pkg/rtcconfig" ) const ( - frameMarking = "urn:ietf:params:rtp-hdrext:framemarking" - repairedRTPStreamID = "urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id" + frameMarkingURI = "urn:ietf:params:rtp-hdrext:framemarking" + repairedRTPStreamIDURI = "urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id" ) type WebRTCConfig struct { @@ -79,23 +80,82 @@ func NewWebRTCConfig(conf *config.Config) (*WebRTCConfig, error) { rtcConf.PacketBufferSizeAudio = rtcConf.PacketBufferSize } - // publisher configuration - publisherConfig := DirectionConfig{ + return &WebRTCConfig{ + WebRTCConfig: *webRTCConfig, + Receiver: ReceiverConfig{ + PacketBufferSizeVideo: rtcConf.PacketBufferSizeVideo, + PacketBufferSizeAudio: rtcConf.PacketBufferSizeAudio, + }, + Publisher: getPublisherConfig(false), + Subscriber: getSubscriberConfig(rtcConf.CongestionControl.UseSendSideBWEInterceptor || rtcConf.CongestionControl.UseSendSideBWE), + }, nil +} + +func (c *WebRTCConfig) UpdatePublisherConfig(consolidated bool) { + c.Publisher = getPublisherConfig(consolidated) +} + +func (c *WebRTCConfig) UpdateSubscriberConfig(ccConf config.CongestionControlConfig) { + c.Subscriber = getSubscriberConfig(ccConf.UseSendSideBWEInterceptor || ccConf.UseSendSideBWE) +} + +func (c *WebRTCConfig) SetBufferFactory(factory *buffer.Factory) { + c.BufferFactory = factory + c.SettingEngine.BufferFactory = factory.GetOrNew +} + +func getPublisherConfig(consolidated bool) DirectionConfig { + if consolidated { + return DirectionConfig{ + RTPHeaderExtension: RTPHeaderExtensionConfig{ + Audio: []string{ + sdp.SDESMidURI, + sdp.SDESRTPStreamIDURI, + sdp.AudioLevelURI, + act.AbsCaptureTimeURI, + }, + Video: []string{ + sdp.SDESMidURI, + sdp.SDESRTPStreamIDURI, + sdp.TransportCCURI, + sdp.ABSSendTimeURI, + frameMarkingURI, + dd.ExtensionURI, + repairedRTPStreamIDURI, + act.AbsCaptureTimeURI, + }, + }, + RTCPFeedback: RTCPFeedbackConfig{ + Audio: []webrtc.RTCPFeedback{ + {Type: webrtc.TypeRTCPFBNACK}, + }, + Video: []webrtc.RTCPFeedback{ + {Type: webrtc.TypeRTCPFBTransportCC}, + {Type: webrtc.TypeRTCPFBGoogREMB}, + {Type: webrtc.TypeRTCPFBCCM, Parameter: "fir"}, + {Type: webrtc.TypeRTCPFBNACK}, + {Type: webrtc.TypeRTCPFBNACK, Parameter: "pli"}, + }, + }, + } + } + + return DirectionConfig{ RTPHeaderExtension: RTPHeaderExtensionConfig{ Audio: []string{ sdp.SDESMidURI, sdp.SDESRTPStreamIDURI, sdp.AudioLevelURI, - //act.AbsCaptureTimeURI, + act.AbsCaptureTimeURI, }, Video: []string{ sdp.SDESMidURI, sdp.SDESRTPStreamIDURI, sdp.TransportCCURI, - frameMarking, + frameMarkingURI, dd.ExtensionURI, - repairedRTPStreamID, - //act.AbsCaptureTimeURI, + repairedRTPStreamIDURI, + act.AbsCaptureTimeURI, }, }, RTCPFeedback: RTCPFeedbackConfig{ @@ -110,25 +170,6 @@ func NewWebRTCConfig(conf *config.Config) (*WebRTCConfig, error) { }, }, } - - return &WebRTCConfig{ - WebRTCConfig: *webRTCConfig, - Receiver: ReceiverConfig{ - PacketBufferSizeVideo: rtcConf.PacketBufferSizeVideo, - PacketBufferSizeAudio: rtcConf.PacketBufferSizeAudio, - }, - Publisher: publisherConfig, - Subscriber: getSubscriberConfig(rtcConf.CongestionControl.UseSendSideBWEInterceptor || rtcConf.CongestionControl.UseSendSideBWE), - }, nil -} - -func (c *WebRTCConfig) UpdateCongestionControl(ccConf config.CongestionControlConfig) { - c.Subscriber = getSubscriberConfig(ccConf.UseSendSideBWEInterceptor || ccConf.UseSendSideBWE) -} - -func (c *WebRTCConfig) SetBufferFactory(factory *buffer.Factory) { - c.BufferFactory = factory - c.SettingEngine.BufferFactory = factory.GetOrNew } func getSubscriberConfig(enableTWCC bool) DirectionConfig { @@ -136,10 +177,10 @@ func getSubscriberConfig(enableTWCC bool) DirectionConfig { RTPHeaderExtension: RTPHeaderExtensionConfig{ Video: []string{ dd.ExtensionURI, - //act.AbsCaptureTimeURI, + act.AbsCaptureTimeURI, }, Audio: []string{ - //act.AbsCaptureTimeURI, + act.AbsCaptureTimeURI, }, }, RTCPFeedback: RTCPFeedbackConfig{ diff --git a/pkg/rtc/datadowntrack.go b/pkg/rtc/datadowntrack.go new file mode 100644 index 0000000..89c069f --- /dev/null +++ b/pkg/rtc/datadowntrack.go @@ -0,0 +1,109 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rtc + +import ( + "fmt" + "time" + + "github.com/livekit/livekit-server/pkg/rtc/datatrack" + "github.com/livekit/livekit-server/pkg/rtc/types" + "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/logger" +) + +var _ types.DataDownTrack = (*DataDownTrack)(nil) +var _ types.DataTrackSender = (*DataDownTrack)(nil) + +type DataDownTrackParams struct { + Logger logger.Logger + SubscriberID livekit.ParticipantID + PublishDataTrack types.DataTrack + Handle uint16 + Transport types.DataTrackTransport + BytesTrackStats *BytesTrackStats +} + +type DataDownTrack struct { + params DataDownTrackParams + logger logger.Logger + createdAt int64 +} + +func NewDataDownTrack(params DataDownTrackParams) (*DataDownTrack, error) { + d := &DataDownTrack{ + params: params, + createdAt: time.Now().UnixNano(), + } + d.logger = params.Logger.WithValues("name", d.Name(), "handle", d.Handle()) + + if err := d.params.PublishDataTrack.AddDataDownTrack(d); err != nil { + d.logger.Warnw("could not add data down track", err) + return nil, err + } + + d.logger.Infow("created data down track") + return d, nil +} + +func (d *DataDownTrack) Close() { + d.logger.Infow("closing data down track") + if d.params.BytesTrackStats != nil { + d.params.BytesTrackStats.Stop() + } + d.params.PublishDataTrack.DeleteDataDownTrack(d.SubscriberID()) +} + +func (d *DataDownTrack) Handle() uint16 { + return d.params.Handle +} + +func (d *DataDownTrack) PublishDataTrack() types.DataTrack { + return d.params.PublishDataTrack +} + +func (d *DataDownTrack) ID() livekit.TrackID { + return d.params.PublishDataTrack.ID() +} + +func (d *DataDownTrack) Name() string { + return d.params.PublishDataTrack.Name() +} + +func (d *DataDownTrack) SubscriberID() livekit.ParticipantID { + // add `createdAt` to ensure repeated subscriptions from same subscriber to same publisher does not collide + return livekit.ParticipantID(fmt.Sprintf("%s:%d", d.params.SubscriberID, d.createdAt)) +} + +func (d *DataDownTrack) WritePacket(data []byte, packet *datatrack.Packet, _arrivalTime int64) { + forwardedPacket := *packet + forwardedPacket.Handle = d.params.Handle + buf, err := forwardedPacket.Marshal() + if err != nil { + d.logger.Warnw("could not marshal data track message", err) + return + } + if err := d.params.Transport.SendDataTrackMessage(buf); err != nil { + d.logger.Warnw("could not send data track message", err) + return + } + if d.params.BytesTrackStats != nil { + d.params.BytesTrackStats.AddBytes(uint64(len(buf)), true) + } +} + +func (d *DataDownTrack) UpdateSubscriptionOptions(subscriptionOptions *livekit.DataTrackSubscriptionOptions) { + // DT-TODO +} diff --git a/pkg/rtc/datatrack.go b/pkg/rtc/datatrack.go new file mode 100644 index 0000000..1e8c4d5 --- /dev/null +++ b/pkg/rtc/datatrack.go @@ -0,0 +1,190 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rtc + +import ( + "errors" + "sync" + + "github.com/frostbyte73/core" + "github.com/livekit/livekit-server/pkg/rtc/datatrack" + "github.com/livekit/livekit-server/pkg/rtc/types" + sfuutils "github.com/livekit/livekit-server/pkg/sfu/utils" + "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/logger" + "github.com/livekit/protocol/utils" +) + +var ( + errReceiverClosed = errors.New("datatrack is closed") +) + +var _ types.DataTrack = (*DataTrack)(nil) + +type DataTrackParams struct { + Logger logger.Logger + ParticipantID func() livekit.ParticipantID + ParticipantIdentity livekit.ParticipantIdentity + BytesTrackStats *BytesTrackStats +} + +type DataTrack struct { + params DataTrackParams + + logger logger.Logger + + lock sync.Mutex + dti *livekit.DataTrackInfo + subscribedTracks map[livekit.ParticipantID]types.DataDownTrack + + downTrackSpreader *sfuutils.DownTrackSpreader[types.DataTrackSender] + + stats *dataTrackStats + + closed core.Fuse +} + +func NewDataTrack(params DataTrackParams, dti *livekit.DataTrackInfo) *DataTrack { + d := &DataTrack{ + params: params, + dti: dti, + subscribedTracks: make(map[livekit.ParticipantID]types.DataDownTrack), + } + d.logger = params.Logger.WithValues("name", d.Name(), "handle", dti.PubHandle) + d.downTrackSpreader = sfuutils.NewDownTrackSpreader[types.DataTrackSender](sfuutils.DownTrackSpreaderParams{ + Threshold: 20, + Logger: d.logger, + }) + d.stats = newDataTrackStats(dataTrackStatsParams{Logger: d.logger}) + d.logger.Infow("created data track", "dataTrackInfo", logger.Proto(d.dti)) + return d +} + +func (d *DataTrack) Close() { + d.logger.Infow("closing data track") + d.closed.Break() + + d.stats.Close() + if d.params.BytesTrackStats != nil { + d.params.BytesTrackStats.Stop() + } +} + +func (d *DataTrack) PublisherID() livekit.ParticipantID { + return d.params.ParticipantID() +} + +func (d *DataTrack) PublisherIdentity() livekit.ParticipantIdentity { + return d.params.ParticipantIdentity +} + +func (d *DataTrack) ToProto() *livekit.DataTrackInfo { + return utils.CloneProto(d.dti) +} + +func (d *DataTrack) PubHandle() uint16 { + return uint16(d.dti.PubHandle) +} + +func (d *DataTrack) ID() livekit.TrackID { + return livekit.TrackID(d.dti.Sid) +} + +func (d *DataTrack) Name() string { + return d.dti.Name +} + +func (d *DataTrack) AddSubscriber(sub types.LocalParticipant) (types.DataDownTrack, error) { + d.lock.Lock() + defer d.lock.Unlock() + + if _, ok := d.subscribedTracks[sub.ID()]; ok { + return nil, errAlreadySubscribed + } + + bytesStats := NewBytesTrackStats( + sub.GetCountry(), + d.ID(), + sub.ID(), + sub.Kind(), + sub.KindDetails(), + sub.GetTelemetryListener(), + sub.GetReporter(), + ) + dataDownTrack, err := NewDataDownTrack(DataDownTrackParams{ + Logger: sub.GetLogger().WithValues("trackID", d.ID()), + SubscriberID: sub.ID(), + PublishDataTrack: d, + Handle: sub.GetNextSubscribedDataTrackHandle(), + Transport: sub.GetDataTrackTransport(), + BytesTrackStats: bytesStats, + }) + if err != nil { + bytesStats.Stop() + return nil, err + } + + d.subscribedTracks[sub.ID()] = dataDownTrack + return dataDownTrack, nil +} + +func (d *DataTrack) RemoveSubscriber(subID livekit.ParticipantID) { + d.lock.Lock() + dataDownTrack, ok := d.subscribedTracks[subID] + delete(d.subscribedTracks, subID) + d.lock.Unlock() + + if ok { + dataDownTrack.Close() + } +} + +func (d *DataTrack) IsSubscriber(subID livekit.ParticipantID) bool { + d.lock.Lock() + defer d.lock.Unlock() + + _, ok := d.subscribedTracks[subID] + return ok +} + +func (d *DataTrack) AddDataDownTrack(dts types.DataTrackSender) error { + if d.closed.IsBroken() { + return errReceiverClosed + } + + if d.downTrackSpreader.HasDownTrack(dts.SubscriberID()) { + d.logger.Infow("subscriberID already exists, replacing data downtrack", "subscriberID", dts.SubscriberID()) + } + + d.downTrackSpreader.Store(dts) + d.logger.Infow("data downtrack added", "subscriberID", dts.SubscriberID()) + return nil +} + +func (d *DataTrack) DeleteDataDownTrack(subscriberID livekit.ParticipantID) { + d.downTrackSpreader.Free(subscriberID) + d.logger.Infow("data downtrack deleted", "subscriberID", subscriberID) +} + +func (d *DataTrack) HandlePacket(data []byte, packet *datatrack.Packet, arrivalTime int64) { + d.stats.Update(packet, arrivalTime, len(data)) + if d.params.BytesTrackStats != nil { + d.params.BytesTrackStats.AddBytes(uint64(len(data)), false) + } + + d.downTrackSpreader.Broadcast(func(dts types.DataTrackSender) { + dts.WritePacket(data, packet, arrivalTime) + }) +} diff --git a/pkg/rtc/datatrack/extension_participant_sid.go b/pkg/rtc/datatrack/extension_participant_sid.go new file mode 100644 index 0000000..35ada18 --- /dev/null +++ b/pkg/rtc/datatrack/extension_participant_sid.go @@ -0,0 +1,59 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package datatrack + +import ( + "errors" + + "github.com/livekit/protocol/livekit" +) + +type ExtensionParticipantSid struct { + participantID livekit.ParticipantID +} + +func NewExtensionParticipantSid(participantID livekit.ParticipantID) (*ExtensionParticipantSid, error) { + if len(participantID) >= 256 { + return nil, errors.New("participantID too long") + } + + return &ExtensionParticipantSid{participantID}, nil +} + +func (e *ExtensionParticipantSid) ParticipantID() livekit.ParticipantID { + return e.participantID +} + +func (e *ExtensionParticipantSid) Marshal() (Extension, error) { + data := make([]byte, len(e.participantID)) + copy(data, e.participantID) + return Extension{ + id: uint8(livekit.DataTrackExtensionID_DTEI_PARTICIPANT_SID), + data: data, + }, nil +} + +func (e *ExtensionParticipantSid) Unmarshal(ext Extension) error { + if ext.id != uint8(livekit.DataTrackExtensionID_DTEI_PARTICIPANT_SID) { + return errors.New("invalid extension ID") + } + + if len(ext.data) == 0 { + return errors.New("empty extension data") + } + + e.participantID = livekit.ParticipantID(ext.data) + return nil +} diff --git a/pkg/rtc/datatrack/extension_participant_sid_test.go b/pkg/rtc/datatrack/extension_participant_sid_test.go new file mode 100644 index 0000000..7128e8b --- /dev/null +++ b/pkg/rtc/datatrack/extension_participant_sid_test.go @@ -0,0 +1,46 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package datatrack + +import ( + "testing" + + "github.com/livekit/protocol/livekit" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestExtensionParticipantSid(t *testing.T) { + longTestParticipantID := livekit.ParticipantID(make([]byte, 256)) + _, err := NewExtensionParticipantSid(longTestParticipantID) + require.Error(t, err) + + testParticipantID := livekit.ParticipantID("test") + extParticipantSid, err := NewExtensionParticipantSid(testParticipantID) + require.NoError(t, err) + + expectedExt := Extension{ + id: uint8(livekit.DataTrackExtensionID_DTEI_PARTICIPANT_SID), + data: []byte{'t', 'e', 's', 't'}, + } + ext, err := extParticipantSid.Marshal() + require.NoError(t, err) + require.Equal(t, expectedExt, ext) + + var unmarshaled ExtensionParticipantSid + err = unmarshaled.Unmarshal(ext) + require.NoError(t, err) + assert.Equal(t, testParticipantID, unmarshaled.ParticipantID()) +} diff --git a/pkg/rtc/datatrack/packet.go b/pkg/rtc/datatrack/packet.go new file mode 100644 index 0000000..487a1c6 --- /dev/null +++ b/pkg/rtc/datatrack/packet.go @@ -0,0 +1,301 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package datatrack + +import ( + "encoding/binary" + "errors" + "fmt" +) + +var ( + errHeaderSizeInsufficient = errors.New("data track packet header size insufficient") + errBufferSizeInsufficient = errors.New("data track packet buffer size insufficient") + errExtensionSizeInsufficient = errors.New("data track packet extension size insufficient") + errExtensionNotFound = errors.New("data track packet extension not found") + errExtensionSizeTooBig = errors.New("extension size is too big") +) + +const ( + headerLength = 12 + + versionShift = 5 + versionMask = (1 << 3) - 1 + + startOfFrameShift = 4 + startOfFrameMask = (1 << 1) - 1 + + finalOfFrameShift = 3 + finalOfFrameMask = (1 << 1) - 1 + + extensionsShift = 2 + extensionsMask = (1 << 1) - 1 + + handleOffset = 2 + handleLength = 2 + + seqNumOffset = 4 + seqNumLength = 2 + + frameNumOffset = 6 + frameNumLength = 2 + + timestampOffset = 8 + timestampLength = 4 + + extensionsSizeOffset = headerLength + extensionsSizeLength = 2 + + extensionIDLength = 1 + extensionSizeLength = 1 +) + +type Extension struct { + id uint8 + data []byte +} + +type Header struct { + Version uint8 + IsStartOfFrame bool + IsFinalOfFrame bool + HasExtensions bool + Handle uint16 + SequenceNumber uint16 + FrameNumber uint16 + Timestamp uint32 + ExtensionsSize uint16 + Extensions []Extension +} + +/* + ┆* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + ┆* 0 1 2 3 + ┆* 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + ┆* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + ┆* |V |S|F|X| reserved | handle | + ┆* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + ┆* | sequence number | frame number | + ┆* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + ┆* | timestamp | + ┆* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + |* Extensions Size if X=1 | Extensions... | + ┆* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + + Each extension + ┆* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + ┆* 0 1 2 3 + ┆* 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + ┆* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + ┆* | Extension ID | Extension size| Extension payload | + ┆* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + + End of all extensions + ┆* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + |* padded to 4 byte boundary if aggregate of `Extensions Size` | + |* field and all extensions do not end on a 4 byte boundary | + ┆* +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +*/ + +func (h *Header) Unmarshal(buf []byte) (int, error) { + if len(buf) < headerLength { + return 0, fmt.Errorf("%w: %d < %d", errHeaderSizeInsufficient, len(buf), headerLength) + } + + hdrSize := headerLength + h.Version = buf[0] >> versionShift & versionMask + h.IsStartOfFrame = (buf[0] >> startOfFrameShift & startOfFrameMask) > 0 + h.IsFinalOfFrame = (buf[0] >> finalOfFrameShift & finalOfFrameMask) > 0 + h.HasExtensions = (buf[0] >> extensionsShift & extensionsMask) > 0 + + h.Handle = binary.BigEndian.Uint16(buf[handleOffset : handleOffset+handleLength]) + h.SequenceNumber = binary.BigEndian.Uint16(buf[seqNumOffset : seqNumOffset+seqNumLength]) + h.FrameNumber = binary.BigEndian.Uint16(buf[frameNumOffset : frameNumOffset+frameNumLength]) + h.Timestamp = binary.BigEndian.Uint32(buf[timestampOffset : timestampOffset+timestampLength]) + + if h.HasExtensions { + extensionsSize := (binary.BigEndian.Uint16(buf[extensionsSizeOffset:extensionsSizeOffset+extensionsSizeLength])+1)*4 - extensionsSizeLength + hdrSize += extensionsSizeLength + + extensionHeaderSize := extensionIDLength + extensionSizeLength + remainingSize := int(extensionsSize) + idx := extensionsSizeOffset + extensionsSizeLength + for remainingSize != 0 { + // read extension header + if len(buf[idx:]) < extensionIDLength || remainingSize < extensionIDLength { + return 0, fmt.Errorf("%w: %d/%d < %d", errExtensionSizeInsufficient, remainingSize, len(buf[idx:]), extensionIDLength) + } + id := buf[idx] + if id == 0 { + // end of extensions, padding has started + hdrSize += remainingSize + break + } + + if len(buf[idx+1:]) < extensionSizeLength || remainingSize < extensionSizeLength { + return 0, fmt.Errorf("%w: %d/%d < %d", errExtensionSizeInsufficient, remainingSize, len(buf[idx:]), extensionSizeLength) + } + size := int(buf[idx+1]) + + remainingSize -= extensionHeaderSize + idx += extensionHeaderSize + hdrSize += extensionHeaderSize + + // read extension data + if len(buf[idx:]) < size || remainingSize < size { + return 0, fmt.Errorf("%w: %d/%d < %d", errExtensionSizeInsufficient, remainingSize, len(buf[idx:]), size) + } + h.Extensions = append(h.Extensions, Extension{id: id, data: buf[idx : idx+size]}) + + remainingSize -= size + idx += size + hdrSize += size + } + h.ExtensionsSize = extensionsSize - uint16(remainingSize) + } + + return hdrSize, nil +} + +func (h *Header) MarshalSize() int { + extensionsSize := 0 + if h.HasExtensions { + extensionsSize += extensionsSizeLength + for _, ext := range h.Extensions { + extensionsSize += len(ext.data) + extensionIDLength + extensionSizeLength + } + } + + return headerLength + (extensionsSize+3)/4*4 +} + +func (h *Header) MarshalTo(buf []byte) (int, error) { + if len(buf) < headerLength { + return 0, fmt.Errorf("%w: %d < %d", errHeaderSizeInsufficient, len(buf), headerLength) + } + + hdrSize := headerLength + buf[0] = h.Version << versionShift + if h.IsStartOfFrame { + buf[0] |= (1 << startOfFrameShift) + } + if h.IsFinalOfFrame { + buf[0] |= (1 << finalOfFrameShift) + } + if h.HasExtensions { + buf[0] |= (1 << extensionsShift) + } + + binary.BigEndian.PutUint16(buf[handleOffset:handleOffset+handleLength], h.Handle) + binary.BigEndian.PutUint16(buf[seqNumOffset:seqNumOffset+seqNumLength], h.SequenceNumber) + binary.BigEndian.PutUint16(buf[frameNumOffset:frameNumOffset+frameNumLength], h.FrameNumber) + binary.BigEndian.PutUint32(buf[timestampOffset:timestampOffset+timestampLength], h.Timestamp) + + if h.HasExtensions { + extensionsSize := (extensionsSizeLength + h.ExtensionsSize + 3) / 4 * 4 + binary.BigEndian.PutUint16(buf[extensionsSizeOffset:extensionsSizeOffset+extensionsSizeLength], (extensionsSize/4)-1) + hdrSize += extensionsSizeLength + + addedSize := 0 + idx := extensionsSizeOffset + extensionsSizeLength + for _, ext := range h.Extensions { + buf[idx] = ext.id + if len(ext.data) > 255 { + return 0, fmt.Errorf("%w: %d > 255", errExtensionSizeTooBig, len(ext.data)) + } + buf[idx+extensionIDLength] = byte(len(ext.data)) + copy(buf[idx+extensionIDLength+extensionSizeLength:], ext.data) + + extSize := len(ext.data) + extensionIDLength + extensionSizeLength + idx += extSize + hdrSize += extSize + addedSize += extSize + } + + paddingSize := extensionsSize - extensionsSizeLength - uint16(addedSize) + for i := range paddingSize { + buf[idx+int(i)] = 0 + } + idx += int(paddingSize) + hdrSize += int(paddingSize) + } + + return hdrSize, nil +} + +func (h *Header) AddExtension(ext Extension) { + for i, existingExt := range h.Extensions { + if existingExt.id == ext.id { + h.ExtensionsSize -= uint16(len(existingExt.data) + extensionIDLength + extensionSizeLength) + h.Extensions[i].data = ext.data + h.ExtensionsSize += uint16(len(h.Extensions[i].data) + extensionIDLength + extensionSizeLength) + return + } + } + + h.Extensions = append(h.Extensions, ext) + h.ExtensionsSize += uint16(len(ext.data) + extensionIDLength + extensionSizeLength) + h.HasExtensions = true +} + +func (h *Header) GetExtension(id uint8) (Extension, error) { + for _, ext := range h.Extensions { + if ext.id == id { + return ext, nil + } + } + return Extension{}, fmt.Errorf("%w, id: %d", errExtensionNotFound, id) +} + +// ---------------------------------------------------- + +type Packet struct { + Header + Payload []byte +} + +func (p *Packet) Unmarshal(buf []byte) error { + hdrSize, err := p.Header.Unmarshal(buf) + if err != nil { + return err + } + + p.Payload = buf[hdrSize:] + return nil +} + +func (p *Packet) Marshal() ([]byte, error) { + buf := make([]byte, p.Header.MarshalSize()+len(p.Payload)) + if err := p.MarshalTo(buf); err != nil { + return nil, err + } + + return buf, nil +} + +func (p *Packet) MarshalTo(buf []byte) error { + size := p.Header.MarshalSize() + len(p.Payload) + if len(buf) < size { + return fmt.Errorf("%w: %d < %d", errBufferSizeInsufficient, len(buf), size) + } + + hdrSize, err := p.Header.MarshalTo(buf) + if err != nil { + return err + } + + copy(buf[hdrSize:], p.Payload) + return nil +} diff --git a/pkg/rtc/datatrack/packet_test.go b/pkg/rtc/datatrack/packet_test.go new file mode 100644 index 0000000..4d8b2c6 --- /dev/null +++ b/pkg/rtc/datatrack/packet_test.go @@ -0,0 +1,259 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package datatrack + +import ( + "testing" + + "github.com/livekit/protocol/livekit" + "github.com/stretchr/testify/require" +) + +func TestPacket(t *testing.T) { + t.Run("without extension", func(t *testing.T) { + payload := make([]byte, 6) + for i := range len(payload) { + payload[i] = byte(255 - i) + } + packet := &Packet{ + Header: Header{ + Version: 0, + IsStartOfFrame: true, + IsFinalOfFrame: true, + Handle: 3333, + SequenceNumber: 6666, + FrameNumber: 9999, + Timestamp: 0xdeadbeef, + }, + Payload: payload, + } + rawPacket, err := packet.Marshal() + require.NoError(t, err) + + expectedRawPacket := []byte{ + 0x18, 0x00, 0x0d, 0x05, 0x1a, 0x0a, 0x27, 0x0f, + 0xde, 0xad, 0xbe, 0xef, 0xff, 0xfe, 0xfd, 0xfc, + 0xfb, 0xfa, + } + require.Equal(t, expectedRawPacket, rawPacket) + + var unmarshaled Packet + err = unmarshaled.Unmarshal(rawPacket) + require.NoError(t, err) + require.Equal(t, packet, &unmarshaled) + }) + + t.Run("with extension", func(t *testing.T) { + payload := make([]byte, 4) + for i := range len(payload) { + payload[i] = byte(255 - i) + } + packet := &Packet{ + Header: Header{ + Version: 0, + IsStartOfFrame: true, + IsFinalOfFrame: false, + Handle: 3333, + SequenceNumber: 6666, + FrameNumber: 9999, + Timestamp: 0xdeadbeef, + }, + Payload: payload, + } + if extParticipantSid, err := NewExtensionParticipantSid("test_participant"); err == nil { + if ext, err := extParticipantSid.Marshal(); err == nil { + packet.AddExtension(ext) + } + } + rawPacket, err := packet.Marshal() + require.NoError(t, err) + + expectedRawPacket := []byte{ + 0x14, 0x00, 0x0d, 0x05, 0x1a, 0x0a, 0x27, 0x0f, + 0xde, 0xad, 0xbe, 0xef, 0x00, 0x04, 0x01, 0x10, + 0x74, 0x65, 0x73, 0x74, 0x5f, 0x70, 0x61, 0x72, + 0x74, 0x69, 0x63, 0x69, 0x70, 0x61, 0x6e, 0x74, + 0xff, 0xfe, 0xfd, 0xfc, + } + require.Equal(t, expectedRawPacket, rawPacket) + + var unmarshaled Packet + err = unmarshaled.Unmarshal(rawPacket) + require.NoError(t, err) + require.Equal(t, packet, &unmarshaled) + + ext, err := unmarshaled.GetExtension(uint8(livekit.DataTrackExtensionID_DTEI_PARTICIPANT_SID)) + require.NoError(t, err) + + var extParticipantSid ExtensionParticipantSid + require.NoError(t, extParticipantSid.Unmarshal(ext)) + require.Equal(t, livekit.ParticipantID("test_participant"), extParticipantSid.ParticipantID()) + }) + + t.Run("with extension padding", func(t *testing.T) { + payload := make([]byte, 4) + for i := range len(payload) { + payload[i] = byte(255 - i) + } + packet := &Packet{ + Header: Header{ + Version: 0, + IsStartOfFrame: true, + IsFinalOfFrame: false, + Handle: 3333, + SequenceNumber: 6666, + FrameNumber: 9999, + Timestamp: 0xdeadbeef, + }, + Payload: payload, + } + if extParticipantSid, err := NewExtensionParticipantSid("participant"); err == nil { + if ext, err := extParticipantSid.Marshal(); err == nil { + packet.AddExtension(ext) + } + } + rawPacket, err := packet.Marshal() + require.NoError(t, err) + + expectedRawPacket := []byte{ + 0x14, 0x00, 0x0d, 0x05, 0x1a, 0x0a, 0x27, 0x0f, + 0xde, 0xad, 0xbe, 0xef, 0x00, 0x03, 0x01, 0x0b, + 0x70, 0x61, 0x72, 0x74, 0x69, 0x63, 0x69, 0x70, + 0x61, 0x6e, 0x74, 0x00, 0xff, 0xfe, 0xfd, 0xfc, + } + require.Equal(t, expectedRawPacket, rawPacket) + + var unmarshaled Packet + err = unmarshaled.Unmarshal(rawPacket) + require.NoError(t, err) + require.Equal(t, packet, &unmarshaled) + + ext, err := unmarshaled.GetExtension(uint8(livekit.DataTrackExtensionID_DTEI_PARTICIPANT_SID)) + require.NoError(t, err) + + var extParticipantSid ExtensionParticipantSid + require.NoError(t, extParticipantSid.Unmarshal(ext)) + require.Equal(t, livekit.ParticipantID("participant"), extParticipantSid.ParticipantID()) + }) + + t.Run("replace extension", func(t *testing.T) { + payload := make([]byte, 4) + for i := range len(payload) { + payload[i] = byte(255 - i) + } + packet := &Packet{ + Header: Header{ + Version: 0, + IsStartOfFrame: true, + IsFinalOfFrame: false, + Handle: 3333, + SequenceNumber: 6666, + FrameNumber: 9999, + Timestamp: 0xdeadbeef, + }, + Payload: payload, + } + if extParticipantSid, err := NewExtensionParticipantSid("participant"); err == nil { + if ext, err := extParticipantSid.Marshal(); err == nil { + packet.AddExtension(ext) + } + } + rawPacket, err := packet.Marshal() + require.NoError(t, err) + + expectedRawPacket := []byte{ + 0x14, 0x00, 0x0d, 0x05, 0x1a, 0x0a, 0x27, 0x0f, + 0xde, 0xad, 0xbe, 0xef, 0x00, 0x03, 0x01, 0x0b, + 0x70, 0x61, 0x72, 0x74, 0x69, 0x63, 0x69, 0x70, + 0x61, 0x6e, 0x74, 0x00, 0xff, 0xfe, 0xfd, 0xfc, + } + require.Equal(t, expectedRawPacket, rawPacket) + + // replace existing extension ID and ensure that marshalled packet is updated + if extParticipantSid, err := NewExtensionParticipantSid("test_participant"); err == nil { + if ext, err := extParticipantSid.Marshal(); err == nil { + packet.AddExtension(ext) + } + } + rawPacket, err = packet.Marshal() + require.NoError(t, err) + + expectedRawPacket = []byte{ + 0x14, 0x00, 0x0d, 0x05, 0x1a, 0x0a, 0x27, 0x0f, + 0xde, 0xad, 0xbe, 0xef, 0x00, 0x04, 0x01, 0x10, + 0x74, 0x65, 0x73, 0x74, 0x5f, 0x70, 0x61, 0x72, + 0x74, 0x69, 0x63, 0x69, 0x70, 0x61, 0x6e, 0x74, + 0xff, 0xfe, 0xfd, 0xfc, + } + require.Equal(t, expectedRawPacket, rawPacket) + + var unmarshaled Packet + err = unmarshaled.Unmarshal(rawPacket) + require.NoError(t, err) + require.Equal(t, packet, &unmarshaled) + + ext, err := unmarshaled.GetExtension(uint8(livekit.DataTrackExtensionID_DTEI_PARTICIPANT_SID)) + require.NoError(t, err) + + var extParticipantSid ExtensionParticipantSid + require.NoError(t, extParticipantSid.Unmarshal(ext)) + require.Equal(t, livekit.ParticipantID("test_participant"), extParticipantSid.ParticipantID()) + }) + + t.Run("bad packet", func(t *testing.T) { + var unmarshaled Packet + // extensions size too small + badPacket := []byte{ + 0x14, 0x00, 0x0d, 0x05, 0x1a, 0x0a, 0x27, 0x0f, + 0xde, 0xad, 0xbe, 0xef, 0x00, 0x02, 0x01, 0x0b, + 0x70, 0x61, 0x72, 0x74, 0x69, 0x63, 0x69, 0x70, + 0x61, 0x6e, 0x74, 0x00, 0xff, 0xfe, 0xfd, 0xfc, + } + err := unmarshaled.Unmarshal(badPacket) + require.Error(t, err) + + // get an invalid extension id + badPacket = []byte{ + 0x14, 0x00, 0x0d, 0x05, 0x1a, 0x0a, 0x27, 0x0f, + 0xde, 0xad, 0xbe, 0xef, 0x00, 0x03, 0x02, 0x0b, + 0x70, 0x61, 0x72, 0x74, 0x69, 0x63, 0x69, 0x70, + 0x61, 0x6e, 0x74, 0x00, 0xff, 0xfe, 0xfd, 0xfc, + } + err = unmarshaled.Unmarshal(badPacket) + require.NoError(t, err) + _, err = unmarshaled.GetExtension(uint8(livekit.DataTrackExtensionID_DTEI_PARTICIPANT_SID)) + require.Error(t, err) + + // extension payload size bigger than payload + badPacket = []byte{ + 0x14, 0x00, 0x0d, 0x05, 0x1a, 0x0a, 0x27, 0x0f, + 0xde, 0xad, 0xbe, 0xef, 0x00, 0x03, 0x01, 0x0d, + 0x70, 0x61, 0x72, 0x74, 0x69, 0x63, 0x69, 0x70, + 0x61, 0x6e, 0x74, 0x00, 0xff, 0xfe, 0xfd, 0xfc, + } + err = unmarshaled.Unmarshal(badPacket) + require.Error(t, err) + + // extension payload size smaller than payload + badPacket = []byte{ + 0x14, 0x00, 0x0d, 0x05, 0x1a, 0x0a, 0x27, 0x0f, + 0xde, 0xad, 0xbe, 0xef, 0x00, 0x03, 0x01, 0x07, + 0x70, 0x61, 0x72, 0x74, 0x69, 0x63, 0x69, 0x70, + 0x61, 0x6e, 0x74, 0x00, 0xff, 0xfe, 0xfd, 0xfc, + } + err = unmarshaled.Unmarshal(badPacket) + require.Error(t, err) + }) +} diff --git a/pkg/rtc/datatrack/testutils.go b/pkg/rtc/datatrack/testutils.go new file mode 100644 index 0000000..e7df0bb --- /dev/null +++ b/pkg/rtc/datatrack/testutils.go @@ -0,0 +1,74 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package datatrack + +import ( + "math/rand" + "time" +) + +func GenerateRawDataPackets(handle uint16, seqNum uint16, frameNum uint16, numFrames int, frameSize int, frameDuration time.Duration) [][]byte { + if seqNum == 0 { + seqNum = uint16(rand.Intn(256) + 1) + } + if frameNum == 0 { + frameNum = uint16(rand.Intn(256) + 1) + } + timestamp := uint32(rand.Intn(1024)) + + packetsPerFrame := (frameSize + 255) / 256 // using 256 bytes of payload per packet + if packetsPerFrame == 0 { + return nil + } + numPackets := packetsPerFrame * numFrames + rawPackets := make([][]byte, 0, numPackets) + for range numFrames { + remainingSize := frameSize + for packetIdx := range packetsPerFrame { + payloadSize := min(remainingSize, 256) + payload := make([]byte, payloadSize) + for i := range len(payload) { + payload[i] = byte(255 - i) + } + packet := &Packet{ + Header: Header{ + Version: 0, + IsStartOfFrame: packetIdx == 0, + IsFinalOfFrame: packetIdx == packetsPerFrame-1, + Handle: handle, + SequenceNumber: seqNum, + FrameNumber: frameNum, + Timestamp: timestamp, + }, + Payload: payload, + } + if extParticipantSid, err := NewExtensionParticipantSid("test_participant"); err == nil { + if ext, err := extParticipantSid.Marshal(); err == nil { + packet.AddExtension(ext) + } + } + rawPacket, err := packet.Marshal() + if err == nil { + rawPackets = append(rawPackets, rawPacket) + } + seqNum++ + remainingSize -= payloadSize + } + frameNum++ + timestamp += uint32(90000 * frameDuration.Seconds()) + } + + return rawPackets +} diff --git a/pkg/rtc/datatrack_stats.go b/pkg/rtc/datatrack_stats.go new file mode 100644 index 0000000..5bde6f5 --- /dev/null +++ b/pkg/rtc/datatrack_stats.go @@ -0,0 +1,110 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rtc + +import ( + "sync" + "time" + + "github.com/livekit/livekit-server/pkg/rtc/datatrack" + "github.com/livekit/protocol/logger" + "github.com/livekit/protocol/utils/mono" +) + +type dataTrackStatsParams struct { + Logger logger.Logger +} + +type dataTrackStats struct { + params dataTrackStatsParams + + lock sync.Mutex + startTime int64 + endTime int64 + highestSequenceNumber uint16 + numPackets int + numPacketsLost int + numPacketsOutOfOrder int + numFrames int // count of `F` tagged packets, i. e. packets with final packet of frame marker + numBytes int +} + +func newDataTrackStats(params dataTrackStatsParams) *dataTrackStats { + return &dataTrackStats{ + params: params, + } +} + +func (d *dataTrackStats) Update(packet *datatrack.Packet, arrivalTime int64, payloadLength int) { + d.lock.Lock() + defer d.lock.Unlock() + + d.numBytes += payloadLength + + if d.endTime != 0 { + return + } + + if d.startTime == 0 { + d.startTime = arrivalTime + d.highestSequenceNumber = packet.SequenceNumber + d.numPackets = 1 + } else { + diff := packet.SequenceNumber - d.highestSequenceNumber + switch { + case diff == 0: // duplicate + return + + case diff > (1 << 15): // out of order + d.numPackets++ + d.numPacketsOutOfOrder++ + if d.numPacketsLost > 0 { + d.numPacketsLost-- + } + + default: // in order + d.numPackets++ + d.numPacketsLost += int(diff) - 1 + d.highestSequenceNumber = packet.SequenceNumber + } + } + + if packet.IsFinalOfFrame { + d.numFrames++ + } +} + +func (d *dataTrackStats) Close() { + d.lock.Lock() + defer d.lock.Unlock() + + d.endTime = mono.UnixNano() + + if d.startTime != 0 { + duration := time.Duration(d.endTime - d.startTime).Seconds() + fps := float64(d.numFrames) / duration + + d.params.Logger.Infow( + "data track stats", + "duration", duration, + "numPackets", d.numPackets, + "numPacketsLost", d.numPacketsLost, + "numPacketsOutOfOrder", d.numPacketsOutOfOrder, + "numFrames", d.numFrames, + "fps", fps, + "numBytes", d.numBytes, + ) + } +} diff --git a/pkg/rtc/dynacast/dynacastmanager_test.go b/pkg/rtc/dynacast/dynacastmanager_test.go index 8a30d7e..6d48e67 100644 --- a/pkg/rtc/dynacast/dynacastmanager_test.go +++ b/pkg/rtc/dynacast/dynacastmanager_test.go @@ -15,7 +15,8 @@ package dynacast import ( - "sort" + "slices" + "strings" "sync" "testing" "time" @@ -23,20 +24,41 @@ import ( "github.com/stretchr/testify/require" "github.com/livekit/livekit-server/pkg/rtc/types" - "github.com/livekit/livekit-server/pkg/sfu/mime" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" ) -func TestSubscribedMaxQuality(t *testing.T) { +type testDynacastManagerListener struct { + onSubscribedMaxQualityChange func(subscribedQualties []*livekit.SubscribedCodec) + onSubscribedAudioCodecChange func(subscribedCodecs []*livekit.SubscribedAudioCodec) +} +func (t *testDynacastManagerListener) OnDynacastSubscribedMaxQualityChange( + subscribedQualities []*livekit.SubscribedCodec, + _maxSubscribedQualities []types.SubscribedCodecQuality, +) { + t.onSubscribedMaxQualityChange(subscribedQualities) +} + +func (t *testDynacastManagerListener) OnDynacastSubscribedAudioCodecChange( + codecs []*livekit.SubscribedAudioCodec, +) { + t.onSubscribedAudioCodecChange(codecs) +} + +func TestSubscribedMaxQuality(t *testing.T) { t.Run("subscribers muted", func(t *testing.T) { - dm := NewDynacastManager(DynacastManagerParams{}) var lock sync.Mutex actualSubscribedQualities := make([]*livekit.SubscribedCodec, 0) - dm.OnSubscribedMaxQualityChange(func(subscribedQualities []*livekit.SubscribedCodec, _maxSubscribedQualities []types.SubscribedCodecQuality) { - lock.Lock() - actualSubscribedQualities = subscribedQualities - lock.Unlock() + + dm := NewDynacastManagerVideo(DynacastManagerVideoParams{ + Listener: &testDynacastManagerListener{ + onSubscribedMaxQualityChange: func(subscribedQualities []*livekit.SubscribedCodec) { + lock.Lock() + actualSubscribedQualities = subscribedQualities + lock.Unlock() + }, + }, }) dm.NotifySubscriberMaxQuality("s1", mime.MimeTypeVP8, livekit.VideoQuality_HIGH) @@ -72,21 +94,20 @@ func TestSubscribedMaxQuality(t *testing.T) { }) t.Run("subscribers max quality", func(t *testing.T) { - dm := NewDynacastManager(DynacastManagerParams{ - DynacastPauseDelay: 100 * time.Millisecond, - }) - lock := sync.RWMutex{} - lock.Lock() actualSubscribedQualities := make([]*livekit.SubscribedCodec, 0) - lock.Unlock() - dm.OnSubscribedMaxQualityChange(func(subscribedQualities []*livekit.SubscribedCodec, _maxSubscribedQualities []types.SubscribedCodecQuality) { - lock.Lock() - actualSubscribedQualities = subscribedQualities - lock.Unlock() + + dm := NewDynacastManagerVideo(DynacastManagerVideoParams{ + Listener: &testDynacastManagerListener{ + onSubscribedMaxQualityChange: func(subscribedQualities []*livekit.SubscribedCodec) { + lock.Lock() + actualSubscribedQualities = subscribedQualities + lock.Unlock() + }, + }, }) - dm.maxSubscribedQuality = map[mime.MimeType]livekit.VideoQuality{ + dm.(*dynacastManagerVideo).maxSubscribedQuality = map[mime.MimeType]livekit.VideoQuality{ mime.MimeTypeVP8: livekit.VideoQuality_LOW, mime.MimeTypeAV1: livekit.VideoQuality_LOW, } @@ -279,98 +300,224 @@ func TestSubscribedMaxQuality(t *testing.T) { } func TestCodecRegression(t *testing.T) { - dm := NewDynacastManager(DynacastManagerParams{}) - var lock sync.Mutex - actualSubscribedQualities := make([]*livekit.SubscribedCodec, 0) - dm.OnSubscribedMaxQualityChange(func(subscribedQualities []*livekit.SubscribedCodec, _maxSubscribedQualities []types.SubscribedCodecQuality) { - lock.Lock() - actualSubscribedQualities = subscribedQualities - lock.Unlock() + t.Run("codec regression video", func(t *testing.T) { + var lock sync.Mutex + actualSubscribedQualities := make([]*livekit.SubscribedCodec, 0) + + dm := NewDynacastManagerVideo(DynacastManagerVideoParams{ + Listener: &testDynacastManagerListener{ + onSubscribedMaxQualityChange: func(subscribedQualities []*livekit.SubscribedCodec) { + lock.Lock() + actualSubscribedQualities = subscribedQualities + lock.Unlock() + }, + }, + }) + + dm.NotifySubscriberMaxQuality("s1", mime.MimeTypeAV1, livekit.VideoQuality_HIGH) + + expectedSubscribedQualities := []*livekit.SubscribedCodec{ + { + Codec: mime.MimeTypeAV1.String(), + Qualities: []*livekit.SubscribedQuality{ + {Quality: livekit.VideoQuality_LOW, Enabled: true}, + {Quality: livekit.VideoQuality_MEDIUM, Enabled: true}, + {Quality: livekit.VideoQuality_HIGH, Enabled: true}, + }, + }, + } + require.Eventually(t, func() bool { + lock.Lock() + defer lock.Unlock() + + return subscribedCodecsAsString(expectedSubscribedQualities) == subscribedCodecsAsString(actualSubscribedQualities) + }, 10*time.Second, 100*time.Millisecond) + + dm.HandleCodecRegression(mime.MimeTypeAV1, mime.MimeTypeVP8) + + expectedSubscribedQualities = []*livekit.SubscribedCodec{ + { + Codec: mime.MimeTypeAV1.String(), + Qualities: []*livekit.SubscribedQuality{ + {Quality: livekit.VideoQuality_LOW, Enabled: false}, + {Quality: livekit.VideoQuality_MEDIUM, Enabled: false}, + {Quality: livekit.VideoQuality_HIGH, Enabled: false}, + }, + }, + { + Codec: mime.MimeTypeVP8.String(), + Qualities: []*livekit.SubscribedQuality{ + {Quality: livekit.VideoQuality_LOW, Enabled: true}, + {Quality: livekit.VideoQuality_MEDIUM, Enabled: true}, + {Quality: livekit.VideoQuality_HIGH, Enabled: true}, + }, + }, + } + require.Eventually(t, func() bool { + lock.Lock() + defer lock.Unlock() + + return subscribedCodecsAsString(expectedSubscribedQualities) == subscribedCodecsAsString(actualSubscribedQualities) + }, 10*time.Second, 100*time.Millisecond) + + // av1 quality change should be forwarded to vp8 + // av1 quality change of node should be ignored + dm.NotifySubscriberMaxQuality("s1", mime.MimeTypeAV1, livekit.VideoQuality_MEDIUM) + dm.NotifySubscriberNodeMaxQuality("n1", []types.SubscribedCodecQuality{ + {CodecMime: mime.MimeTypeAV1, Quality: livekit.VideoQuality_HIGH}, + }) + expectedSubscribedQualities = []*livekit.SubscribedCodec{ + { + Codec: mime.MimeTypeAV1.String(), + Qualities: []*livekit.SubscribedQuality{ + {Quality: livekit.VideoQuality_LOW, Enabled: false}, + {Quality: livekit.VideoQuality_MEDIUM, Enabled: false}, + {Quality: livekit.VideoQuality_HIGH, Enabled: false}, + }, + }, + { + Codec: mime.MimeTypeVP8.String(), + Qualities: []*livekit.SubscribedQuality{ + {Quality: livekit.VideoQuality_LOW, Enabled: true}, + {Quality: livekit.VideoQuality_MEDIUM, Enabled: true}, + {Quality: livekit.VideoQuality_HIGH, Enabled: false}, + }, + }, + } + require.Eventually(t, func() bool { + lock.Lock() + defer lock.Unlock() + + return subscribedCodecsAsString(expectedSubscribedQualities) == subscribedCodecsAsString(actualSubscribedQualities) + }, 10*time.Second, 100*time.Millisecond) }) - dm.NotifySubscriberMaxQuality("s1", mime.MimeTypeAV1, livekit.VideoQuality_HIGH) + t.Run("codec regression audio", func(t *testing.T) { + var lock sync.Mutex + actualSubscribedCodecs := make([]*livekit.SubscribedAudioCodec, 0) - expectedSubscribedQualities := []*livekit.SubscribedCodec{ - { - Codec: mime.MimeTypeAV1.String(), - Qualities: []*livekit.SubscribedQuality{ - {Quality: livekit.VideoQuality_LOW, Enabled: true}, - {Quality: livekit.VideoQuality_MEDIUM, Enabled: true}, - {Quality: livekit.VideoQuality_HIGH, Enabled: true}, + dm := NewDynacastManagerAudio(DynacastManagerAudioParams{ + Listener: &testDynacastManagerListener{ + onSubscribedAudioCodecChange: func(subscribedCodecs []*livekit.SubscribedAudioCodec) { + lock.Lock() + actualSubscribedCodecs = subscribedCodecs + lock.Unlock() + }, }, - }, - } - require.Eventually(t, func() bool { - lock.Lock() - defer lock.Unlock() + }) - return subscribedCodecsAsString(expectedSubscribedQualities) == subscribedCodecsAsString(actualSubscribedQualities) - }, 10*time.Second, 100*time.Millisecond) + dm.NotifySubscription("s1", mime.MimeTypeRED, true) - dm.HandleCodecRegression(mime.MimeTypeAV1, mime.MimeTypeVP8) - - expectedSubscribedQualities = []*livekit.SubscribedCodec{ - { - Codec: mime.MimeTypeAV1.String(), - Qualities: []*livekit.SubscribedQuality{ - {Quality: livekit.VideoQuality_LOW, Enabled: false}, - {Quality: livekit.VideoQuality_MEDIUM, Enabled: false}, - {Quality: livekit.VideoQuality_HIGH, Enabled: false}, + expectedSubscribedCodecs := []*livekit.SubscribedAudioCodec{ + { + Codec: mime.MimeTypeRED.String(), + Enabled: true, }, - }, - { - Codec: mime.MimeTypeVP8.String(), - Qualities: []*livekit.SubscribedQuality{ - {Quality: livekit.VideoQuality_LOW, Enabled: true}, - {Quality: livekit.VideoQuality_MEDIUM, Enabled: true}, - {Quality: livekit.VideoQuality_HIGH, Enabled: true}, + } + require.Eventually(t, func() bool { + lock.Lock() + defer lock.Unlock() + + return subscribedAudioCodecsAsString(expectedSubscribedCodecs) == subscribedAudioCodecsAsString(actualSubscribedCodecs) + }, 10*time.Second, 100*time.Millisecond) + + dm.HandleCodecRegression(mime.MimeTypeRED, mime.MimeTypeOpus) + + expectedSubscribedCodecs = []*livekit.SubscribedAudioCodec{ + { + Codec: mime.MimeTypeRED.String(), + Enabled: false, }, - }, - } - require.Eventually(t, func() bool { - lock.Lock() - defer lock.Unlock() + { + Codec: mime.MimeTypeOpus.String(), + Enabled: true, + }, + } + require.Eventually(t, func() bool { + lock.Lock() + defer lock.Unlock() - return subscribedCodecsAsString(expectedSubscribedQualities) == subscribedCodecsAsString(actualSubscribedQualities) - }, 10*time.Second, 100*time.Millisecond) + return subscribedAudioCodecsAsString(expectedSubscribedCodecs) == subscribedAudioCodecsAsString(actualSubscribedCodecs) + }, 10*time.Second, 100*time.Millisecond) + + // RED disable as subscriber or subscriber node should be ignored as it has been regressed + dm.NotifySubscription("s1", mime.MimeTypeRED, false) + dm.NotifySubscriptionNode("n1", []*livekit.SubscribedAudioCodec{ + {Codec: mime.MimeTypeRED.String(), Enabled: false}, + }) + require.Eventually(t, func() bool { + lock.Lock() + defer lock.Unlock() + + return subscribedAudioCodecsAsString(expectedSubscribedCodecs) == subscribedAudioCodecsAsString(actualSubscribedCodecs) + }, 10*time.Second, 100*time.Millisecond) + + // `s1` unsubscription should turn off `opus` + dm.NotifySubscription("s1", mime.MimeTypeOpus, false) + expectedSubscribedCodecs = []*livekit.SubscribedAudioCodec{ + { + Codec: mime.MimeTypeRED.String(), + Enabled: false, + }, + { + Codec: mime.MimeTypeOpus.String(), + Enabled: false, + }, + } + require.Eventually(t, func() bool { + lock.Lock() + defer lock.Unlock() + + return subscribedAudioCodecsAsString(expectedSubscribedCodecs) == subscribedAudioCodecsAsString(actualSubscribedCodecs) + }, 10*time.Second, 100*time.Millisecond) + + // a node subscription should turn `opus` back on + dm.NotifySubscriptionNode("n1", []*livekit.SubscribedAudioCodec{ + { + Codec: mime.MimeTypeOpus.String(), + Enabled: true, + }, + }) + expectedSubscribedCodecs = []*livekit.SubscribedAudioCodec{ + { + Codec: mime.MimeTypeRED.String(), + Enabled: false, + }, + { + Codec: mime.MimeTypeOpus.String(), + Enabled: true, + }, + } + require.Eventually(t, func() bool { + lock.Lock() + defer lock.Unlock() + + return subscribedAudioCodecsAsString(expectedSubscribedCodecs) == subscribedAudioCodecsAsString(actualSubscribedCodecs) + }, 10*time.Second, 100*time.Millisecond) - // av1 quality change should be forwarded to vp8 - // av1 quality change of node should be ignored - dm.NotifySubscriberMaxQuality("s1", mime.MimeTypeAV1, livekit.VideoQuality_MEDIUM) - dm.NotifySubscriberNodeMaxQuality("n1", []types.SubscribedCodecQuality{ - {CodecMime: mime.MimeTypeAV1, Quality: livekit.VideoQuality_HIGH}, }) - expectedSubscribedQualities = []*livekit.SubscribedCodec{ - { - Codec: mime.MimeTypeAV1.String(), - Qualities: []*livekit.SubscribedQuality{ - {Quality: livekit.VideoQuality_LOW, Enabled: false}, - {Quality: livekit.VideoQuality_MEDIUM, Enabled: false}, - {Quality: livekit.VideoQuality_HIGH, Enabled: false}, - }, - }, - { - Codec: mime.MimeTypeVP8.String(), - Qualities: []*livekit.SubscribedQuality{ - {Quality: livekit.VideoQuality_LOW, Enabled: true}, - {Quality: livekit.VideoQuality_MEDIUM, Enabled: true}, - {Quality: livekit.VideoQuality_HIGH, Enabled: false}, - }, - }, - } - require.Eventually(t, func() bool { - lock.Lock() - defer lock.Unlock() - - return subscribedCodecsAsString(expectedSubscribedQualities) == subscribedCodecsAsString(actualSubscribedQualities) - }, 10*time.Second, 100*time.Millisecond) } func subscribedCodecsAsString(c1 []*livekit.SubscribedCodec) string { - sort.Slice(c1, func(i, j int) bool { return c1[i].Codec < c1[j].Codec }) - var s1 string + slices.SortFunc(c1, func(a, b *livekit.SubscribedCodec) int { + return strings.Compare(a.Codec, b.Codec) + }) + + var s1 strings.Builder for _, c := range c1 { - s1 += c.String() + s1.WriteString(c.String()) } - return s1 + return s1.String() +} + +func subscribedAudioCodecsAsString(c1 []*livekit.SubscribedAudioCodec) string { + slices.SortFunc(c1, func(a, b *livekit.SubscribedAudioCodec) int { + return strings.Compare(a.Codec, b.Codec) + }) + + var s1 strings.Builder + for _, c := range c1 { + s1.WriteString(c.String()) + } + return s1.String() } diff --git a/pkg/rtc/dynacast/dynacastmanageraudio.go b/pkg/rtc/dynacast/dynacastmanageraudio.go new file mode 100644 index 0000000..2e9bf6e --- /dev/null +++ b/pkg/rtc/dynacast/dynacastmanageraudio.go @@ -0,0 +1,198 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dynacast + +import ( + "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/logger" + + "github.com/livekit/protocol/codecs/mime" +) + +var _ DynacastManager = (*dynacastManagerAudio)(nil) +var _ dynacastQualityListener = (*dynacastManagerAudio)(nil) + +type DynacastManagerAudioParams struct { + Listener DynacastManagerListener + Logger logger.Logger +} + +type dynacastManagerAudio struct { + params DynacastManagerAudioParams + + subscribedCodecs map[mime.MimeType]bool + committedSubscribedCodecs map[mime.MimeType]bool + + isClosed bool + + *dynacastManagerBase +} + +func NewDynacastManagerAudio(params DynacastManagerAudioParams) DynacastManager { + if params.Logger == nil { + params.Logger = logger.GetLogger() + } + d := &dynacastManagerAudio{ + params: params, + subscribedCodecs: make(map[mime.MimeType]bool), + committedSubscribedCodecs: make(map[mime.MimeType]bool), + } + d.dynacastManagerBase = newDynacastManagerBase(dynacastManagerBaseParams{ + Logger: params.Logger, + OpsQueueDepth: 4, + OnRestart: func() { + d.committedSubscribedCodecs = make(map[mime.MimeType]bool) + }, + OnDynacastQualityCreate: func(mimeType mime.MimeType) dynacastQuality { + dq := newDynacastQualityAudio(dynacastQualityAudioParams{ + MimeType: mimeType, + Listener: d, + Logger: d.params.Logger, + }) + return dq + }, + OnRegressCodec: func(fromMime, toMime mime.MimeType) { + d.subscribedCodecs[fromMime] = false + + // if the new codec is not added, notify the publisher to start publishing + if _, ok := d.subscribedCodecs[toMime]; !ok { + d.subscribedCodecs[toMime] = true + } + }, + OnUpdateNeeded: d.update, + }) + return d +} + +// It is possible for tracks to be in pending close state. When track +// is waiting to be closed, a node is not streaming a track. This can +// be used to force an update announcing that subscribed codec is disabled, +// i.e. indicating not pulling track any more. +func (d *dynacastManagerAudio) ForceEnable(enabled bool) { + d.lock.Lock() + defer d.lock.Unlock() + + for mime := range d.committedSubscribedCodecs { + d.committedSubscribedCodecs[mime] = enabled + } + + d.enqueueSubscribedChange() +} + +func (d *dynacastManagerAudio) NotifySubscription( + subscriberID livekit.ParticipantID, + mime mime.MimeType, + enabled bool, +) { + dq := d.getOrCreateDynacastQuality(mime) + if dq != nil { + dq.NotifySubscription(subscriberID, enabled) + } +} + +func (d *dynacastManagerAudio) NotifySubscriptionNode( + nodeID livekit.NodeID, + codecs []*livekit.SubscribedAudioCodec, +) { + for _, codec := range codecs { + dq := d.getOrCreateDynacastQuality(mime.NormalizeMimeType(codec.Codec)) + if dq != nil { + dq.NotifySubscriptionNode(nodeID, codec.Enabled) + } + } +} + +func (d *dynacastManagerAudio) OnUpdateAudioCodecForMime(mime mime.MimeType, enabled bool) { + d.lock.Lock() + if _, ok := d.regressedCodec[mime]; !ok { + d.subscribedCodecs[mime] = enabled + } + d.lock.Unlock() + + d.update(false) +} + +func (d *dynacastManagerAudio) update(force bool) { + d.lock.Lock() + + d.params.Logger.Debugw( + "processing subscribed codec change", + "force", force, + "committedSubscribedCodecs", d.committedSubscribedCodecs, + "subscribedCodecs", d.subscribedCodecs, + ) + + if len(d.subscribedCodecs) == 0 { + // no mime has been added, nothing to update + d.lock.Unlock() + return + } + + // add or remove of a mime triggers an update + changed := len(d.subscribedCodecs) != len(d.committedSubscribedCodecs) + if !changed { + for mime, enabled := range d.subscribedCodecs { + if ce, ok := d.committedSubscribedCodecs[mime]; ok { + if ce != enabled { + changed = true + break + } + } + } + } + + if !force && !changed { + d.lock.Unlock() + return + } + + d.params.Logger.Debugw( + "committing subscribed codec change", + "force", force, + "committedSubscribedCoecs", d.committedSubscribedCodecs, + "subscribedcodecs", d.subscribedCodecs, + ) + + // commit change + d.committedSubscribedCodecs = make(map[mime.MimeType]bool, len(d.subscribedCodecs)) + for mime, enabled := range d.subscribedCodecs { + d.committedSubscribedCodecs[mime] = enabled + } + + d.enqueueSubscribedChange() + d.lock.Unlock() +} + +func (d *dynacastManagerAudio) enqueueSubscribedChange() { + if d.isClosed || d.params.Listener == nil { + return + } + + subscribedCodecs := make([]*livekit.SubscribedAudioCodec, 0, len(d.committedSubscribedCodecs)) + for mime, enabled := range d.committedSubscribedCodecs { + subscribedCodecs = append(subscribedCodecs, &livekit.SubscribedAudioCodec{ + Codec: mime.String(), + Enabled: enabled, + }) + } + + d.params.Logger.Debugw( + "subscribedAudioCodecChange", + "subscribedCodecs", logger.ProtoSlice(subscribedCodecs), + ) + d.notifyOpsQueue.Enqueue(func() { + d.params.Listener.OnDynacastSubscribedAudioCodecChange(subscribedCodecs) + }) +} diff --git a/pkg/rtc/dynacast/dynacastmanagerbase.go b/pkg/rtc/dynacast/dynacastmanagerbase.go new file mode 100644 index 0000000..dff3b9c --- /dev/null +++ b/pkg/rtc/dynacast/dynacastmanagerbase.go @@ -0,0 +1,165 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dynacast + +import ( + "maps" + "slices" + "sync" + + "github.com/livekit/protocol/codecs/mime" + "github.com/livekit/protocol/logger" + + "github.com/livekit/livekit-server/pkg/utils" +) + +type dynacastManagerBaseParams struct { + Logger logger.Logger + OpsQueueDepth uint + OnRestart func() + OnDynacastQualityCreate func(mimeType mime.MimeType) dynacastQuality + OnRegressCodec func(fromMime, toMime mime.MimeType) + OnUpdateNeeded func(force bool) +} + +type dynacastManagerBase struct { + params dynacastManagerBaseParams + + lock sync.RWMutex + regressedCodec map[mime.MimeType]struct{} + dynacastQuality map[mime.MimeType]dynacastQuality + + notifyOpsQueue *utils.OpsQueue + + isClosed bool + + dynacastManagerNull + dynacastQualityListenerNull +} + +func newDynacastManagerBase(params dynacastManagerBaseParams) *dynacastManagerBase { + if params.OpsQueueDepth == 0 { + params.OpsQueueDepth = 4 + } + d := &dynacastManagerBase{ + params: params, + regressedCodec: make(map[mime.MimeType]struct{}), + dynacastQuality: make(map[mime.MimeType]dynacastQuality), + notifyOpsQueue: utils.NewOpsQueue(utils.OpsQueueParams{ + Name: "dynacast-notify", + MinSize: params.OpsQueueDepth, + FlushOnStop: true, + Logger: params.Logger, + }), + } + d.notifyOpsQueue.Start() + return d +} + +func (d *dynacastManagerBase) AddCodec(mime mime.MimeType) { + d.getOrCreateDynacastQuality(mime) +} + +func (d *dynacastManagerBase) HandleCodecRegression(fromMime, toMime mime.MimeType) { + fromDq := d.getOrCreateDynacastQuality(fromMime) + + d.lock.Lock() + if d.isClosed { + d.lock.Unlock() + return + } + + if fromDq == nil { + // should not happen as we have added the codec on setup receiver + d.params.Logger.Warnw("regression from codec not found", nil, "mime", fromMime, "toMime", toMime) + d.lock.Unlock() + return + } + d.regressedCodec[fromMime] = struct{}{} + d.params.OnRegressCodec(fromMime, toMime) + d.lock.Unlock() + + d.params.OnUpdateNeeded(false) + + fromDq.Stop() + fromDq.RegressTo(d.getOrCreateDynacastQuality(toMime)) +} + +func (d *dynacastManagerBase) Restart() { + d.lock.Lock() + d.params.OnRestart() + + dqs := d.getDynacastQualitiesLocked() + d.lock.Unlock() + + for _, dq := range dqs { + dq.Restart() + } +} + +func (d *dynacastManagerBase) Close() { + d.notifyOpsQueue.Stop() + + d.lock.Lock() + dqs := d.getDynacastQualitiesLocked() + d.dynacastQuality = make(map[mime.MimeType]dynacastQuality) + + d.isClosed = true + d.lock.Unlock() + + for _, dq := range dqs { + dq.Stop() + } +} + +// There are situations like track unmute or streaming from a different node +// where subscription changes needs to sent to the provider immediately. +// This bypasses any debouncing and forces a subscription change update +// with immediate effect. +func (d *dynacastManagerBase) ForceUpdate() { + d.params.OnUpdateNeeded(true) +} + +func (d *dynacastManagerBase) ClearSubscriberNodes() { + d.lock.Lock() + dqs := d.getDynacastQualitiesLocked() + d.lock.Unlock() + for _, dq := range dqs { + dq.ClearSubscriberNodes() + } +} + +func (d *dynacastManagerBase) getOrCreateDynacastQuality(mimeType mime.MimeType) dynacastQuality { + d.lock.Lock() + defer d.lock.Unlock() + + if d.isClosed || mimeType == mime.MimeTypeUnknown { + return nil + } + + if dq := d.dynacastQuality[mimeType]; dq != nil { + return dq + } + + dq := d.params.OnDynacastQualityCreate(mimeType) + dq.Start() + + d.dynacastQuality[mimeType] = dq + return dq +} + +func (d *dynacastManagerBase) getDynacastQualitiesLocked() []dynacastQuality { + return slices.Collect(maps.Values(d.dynacastQuality)) +} diff --git a/pkg/rtc/dynacast/dynacastmanager.go b/pkg/rtc/dynacast/dynacastmanagervideo.go similarity index 55% rename from pkg/rtc/dynacast/dynacastmanager.go rename to pkg/rtc/dynacast/dynacastmanagervideo.go index f7697a8..8178b46 100644 --- a/pkg/rtc/dynacast/dynacastmanager.go +++ b/pkg/rtc/dynacast/dynacastmanagervideo.go @@ -15,148 +15,85 @@ package dynacast import ( - "sync" + "maps" "time" "github.com/bep/debounce" - "golang.org/x/exp/maps" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" "github.com/livekit/livekit-server/pkg/rtc/types" - "github.com/livekit/livekit-server/pkg/sfu/mime" - "github.com/livekit/livekit-server/pkg/utils" ) -type DynacastManagerParams struct { +var _ DynacastManager = (*dynacastManagerVideo)(nil) +var _ dynacastQualityListener = (*dynacastManagerVideo)(nil) + +type DynacastManagerVideoParams struct { DynacastPauseDelay time.Duration + Listener DynacastManagerListener Logger logger.Logger } -type DynacastManager struct { - params DynacastManagerParams +type dynacastManagerVideo struct { + params DynacastManagerVideoParams - lock sync.RWMutex - regressedCodec map[mime.MimeType]struct{} - dynacastQuality map[mime.MimeType]*DynacastQuality maxSubscribedQuality map[mime.MimeType]livekit.VideoQuality committedMaxSubscribedQuality map[mime.MimeType]livekit.VideoQuality maxSubscribedQualityDebounce func(func()) maxSubscribedQualityDebouncePending bool - qualityNotifyOpQueue *utils.OpsQueue - isClosed bool - onSubscribedMaxQualityChange func(subscribedQualities []*livekit.SubscribedCodec, maxSubscribedQualities []types.SubscribedCodecQuality) + *dynacastManagerBase } -func NewDynacastManager(params DynacastManagerParams) *DynacastManager { +func NewDynacastManagerVideo(params DynacastManagerVideoParams) DynacastManager { if params.Logger == nil { params.Logger = logger.GetLogger() } - d := &DynacastManager{ + d := &dynacastManagerVideo{ params: params, - regressedCodec: make(map[mime.MimeType]struct{}), - dynacastQuality: make(map[mime.MimeType]*DynacastQuality), maxSubscribedQuality: make(map[mime.MimeType]livekit.VideoQuality), committedMaxSubscribedQuality: make(map[mime.MimeType]livekit.VideoQuality), - qualityNotifyOpQueue: utils.NewOpsQueue(utils.OpsQueueParams{ - Name: "quality-notify", - MinSize: 64, - FlushOnStop: true, - Logger: params.Logger, - }), } if params.DynacastPauseDelay > 0 { d.maxSubscribedQualityDebounce = debounce.New(params.DynacastPauseDelay) } - d.qualityNotifyOpQueue.Start() + d.dynacastManagerBase = newDynacastManagerBase(dynacastManagerBaseParams{ + Logger: params.Logger, + OpsQueueDepth: 64, + OnRestart: func() { + d.committedMaxSubscribedQuality = make(map[mime.MimeType]livekit.VideoQuality) + }, + OnDynacastQualityCreate: func(mimeType mime.MimeType) dynacastQuality { + dq := newDynacastQualityVideo(dynacastQualityVideoParams{ + MimeType: mimeType, + Listener: d, + Logger: d.params.Logger, + }) + return dq + }, + OnRegressCodec: func(fromMime, toMime mime.MimeType) { + d.maxSubscribedQuality[fromMime] = livekit.VideoQuality_OFF + + // if the new codec is not added, notify the publisher to start publishing + if _, ok := d.maxSubscribedQuality[toMime]; !ok { + d.maxSubscribedQuality[toMime] = livekit.VideoQuality_HIGH + } + }, + OnUpdateNeeded: d.update, + }) return d } -func (d *DynacastManager) OnSubscribedMaxQualityChange(f func(subscribedQualities []*livekit.SubscribedCodec, maxSubscribedQualities []types.SubscribedCodecQuality)) { - d.lock.Lock() - d.onSubscribedMaxQualityChange = f - d.lock.Unlock() -} - -func (d *DynacastManager) AddCodec(mime mime.MimeType) { - d.getOrCreateDynacastQuality(mime) -} - -func (d *DynacastManager) HandleCodecRegression(fromMime, toMime mime.MimeType) { - fromDq := d.getOrCreateDynacastQuality(fromMime) - - d.lock.Lock() - if d.isClosed { - d.lock.Unlock() - return - } - - if fromDq == nil { - // should not happen as we have added the codec on setup receiver - d.params.Logger.Warnw("regression from codec not found", nil, "mime", fromMime) - d.lock.Unlock() - return - } - d.regressedCodec[fromMime] = struct{}{} - d.maxSubscribedQuality[fromMime] = livekit.VideoQuality_OFF - - // if the new codec is not added, notify the publisher to start publishing - if _, ok := d.maxSubscribedQuality[toMime]; !ok { - d.maxSubscribedQuality[toMime] = livekit.VideoQuality_HIGH - } - - d.lock.Unlock() - d.update(false) - - fromDq.Stop() - fromDq.RegressTo(d.getOrCreateDynacastQuality(toMime)) -} - -func (d *DynacastManager) Restart() { - d.lock.Lock() - d.committedMaxSubscribedQuality = make(map[mime.MimeType]livekit.VideoQuality) - - dqs := d.getDynacastQualitiesLocked() - d.lock.Unlock() - - for _, dq := range dqs { - dq.Restart() - } -} - -func (d *DynacastManager) Close() { - d.qualityNotifyOpQueue.Stop() - - d.lock.Lock() - dqs := d.getDynacastQualitiesLocked() - d.dynacastQuality = make(map[mime.MimeType]*DynacastQuality) - - d.isClosed = true - d.lock.Unlock() - - for _, dq := range dqs { - dq.Stop() - } -} - -// THere are situations like track unmute or streaming from a different node -// where subscribed quality needs to sent to the provider immediately. -// This bypasses any debouncing and forces a subscribed quality update -// with immediate effect. -func (d *DynacastManager) ForceUpdate() { - d.update(true) -} - // It is possible for tracks to be in pending close state. When track // is waiting to be closed, a node is not streaming a track. This can // be used to force an update announcing that subscribed quality is OFF, // i.e. indicating not pulling track any more. -func (d *DynacastManager) ForceQuality(quality livekit.VideoQuality) { +func (d *dynacastManagerVideo) ForceQuality(quality livekit.VideoQuality) { d.lock.Lock() defer d.lock.Unlock() @@ -167,14 +104,21 @@ func (d *DynacastManager) ForceQuality(quality livekit.VideoQuality) { d.enqueueSubscribedQualityChange() } -func (d *DynacastManager) NotifySubscriberMaxQuality(subscriberID livekit.ParticipantID, mime mime.MimeType, quality livekit.VideoQuality) { +func (d *dynacastManagerVideo) NotifySubscriberMaxQuality( + subscriberID livekit.ParticipantID, + mime mime.MimeType, + quality livekit.VideoQuality, +) { dq := d.getOrCreateDynacastQuality(mime) if dq != nil { dq.NotifySubscriberMaxQuality(subscriberID, quality) } } -func (d *DynacastManager) NotifySubscriberNodeMaxQuality(nodeID livekit.NodeID, qualities []types.SubscribedCodecQuality) { +func (d *dynacastManagerVideo) NotifySubscriberNodeMaxQuality( + nodeID livekit.NodeID, + qualities []types.SubscribedCodecQuality, +) { for _, quality := range qualities { dq := d.getOrCreateDynacastQuality(quality.CodecMime) if dq != nil { @@ -183,34 +127,10 @@ func (d *DynacastManager) NotifySubscriberNodeMaxQuality(nodeID livekit.NodeID, } } -func (d *DynacastManager) getOrCreateDynacastQuality(mimeType mime.MimeType) *DynacastQuality { - d.lock.Lock() - defer d.lock.Unlock() - - if d.isClosed || mimeType == mime.MimeTypeUnknown { - return nil - } - - if dq := d.dynacastQuality[mimeType]; dq != nil { - return dq - } - - dq := NewDynacastQuality(DynacastQualityParams{ - MimeType: mimeType, - Logger: d.params.Logger, - }) - dq.OnSubscribedMaxQualityChange(d.updateMaxQualityForMime) - dq.Start() - - d.dynacastQuality[mimeType] = dq - return dq -} - -func (d *DynacastManager) getDynacastQualitiesLocked() []*DynacastQuality { - return maps.Values(d.dynacastQuality) -} - -func (d *DynacastManager) updateMaxQualityForMime(mime mime.MimeType, maxQuality livekit.VideoQuality) { +func (d *dynacastManagerVideo) OnUpdateMaxQualityForMime( + mime mime.MimeType, + maxQuality livekit.VideoQuality, +) { d.lock.Lock() if _, ok := d.regressedCodec[mime]; !ok { d.maxSubscribedQuality[mime] = maxQuality @@ -220,10 +140,11 @@ func (d *DynacastManager) updateMaxQualityForMime(mime mime.MimeType, maxQuality d.update(false) } -func (d *DynacastManager) update(force bool) { +func (d *dynacastManagerVideo) update(force bool) { d.lock.Lock() - d.params.Logger.Debugw("processing quality change", + d.params.Logger.Debugw( + "processing quality change", "force", force, "committedMaxSubscribedQuality", d.committedMaxSubscribedQuality, "maxSubscribedQuality", d.maxSubscribedQuality, @@ -260,7 +181,8 @@ func (d *DynacastManager) update(force bool) { if downgradesOnly && d.maxSubscribedQualityDebounce != nil { if !d.maxSubscribedQualityDebouncePending { - d.params.Logger.Debugw("debouncing quality downgrade", + d.params.Logger.Debugw( + "debouncing quality downgrade", "committedMaxSubscribedQuality", d.committedMaxSubscribedQuality, "maxSubscribedQuality", d.maxSubscribedQuality, ) @@ -269,7 +191,8 @@ func (d *DynacastManager) update(force bool) { }) d.maxSubscribedQualityDebouncePending = true } else { - d.params.Logger.Debugw("quality downgrade waiting for debounce", + d.params.Logger.Debugw( + "quality downgrade waiting for debounce", "committedMaxSubscribedQuality", d.committedMaxSubscribedQuality, "maxSubscribedQuality", d.maxSubscribedQuality, ) @@ -285,7 +208,8 @@ func (d *DynacastManager) update(force bool) { d.maxSubscribedQualityDebouncePending = false } - d.params.Logger.Debugw("committing quality change", + d.params.Logger.Debugw( + "committing quality change", "force", force, "committedMaxSubscribedQuality", d.committedMaxSubscribedQuality, "maxSubscribedQuality", d.maxSubscribedQuality, @@ -293,16 +217,14 @@ func (d *DynacastManager) update(force bool) { // commit change d.committedMaxSubscribedQuality = make(map[mime.MimeType]livekit.VideoQuality, len(d.maxSubscribedQuality)) - for mime, quality := range d.maxSubscribedQuality { - d.committedMaxSubscribedQuality[mime] = quality - } + maps.Copy(d.committedMaxSubscribedQuality, d.maxSubscribedQuality) d.enqueueSubscribedQualityChange() d.lock.Unlock() } -func (d *DynacastManager) enqueueSubscribedQualityChange() { - if d.isClosed || d.onSubscribedMaxQualityChange == nil { +func (d *dynacastManagerVideo) enqueueSubscribedQualityChange() { + if d.isClosed || d.params.Listener == nil { return } @@ -343,7 +265,7 @@ func (d *DynacastManager) enqueueSubscribedQualityChange() { "subscribedCodecs", subscribedCodecs, "maxSubscribedQualities", maxSubscribedQualities, ) - d.qualityNotifyOpQueue.Enqueue(func() { - d.onSubscribedMaxQualityChange(subscribedCodecs, maxSubscribedQualities) + d.notifyOpsQueue.Enqueue(func() { + d.params.Listener.OnDynacastSubscribedMaxQualityChange(subscribedCodecs, maxSubscribedQualities) }) } diff --git a/pkg/rtc/dynacast/dynacastqualityaudio.go b/pkg/rtc/dynacast/dynacastqualityaudio.go new file mode 100644 index 0000000..817481d --- /dev/null +++ b/pkg/rtc/dynacast/dynacastqualityaudio.go @@ -0,0 +1,168 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dynacast + +import ( + "sync" + + "github.com/livekit/protocol/codecs/mime" + "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/logger" +) + +var _ dynacastQuality = (*dynacastQualityAudio)(nil) + +type dynacastQualityAudioParams struct { + MimeType mime.MimeType + Listener dynacastQualityListener + Logger logger.Logger +} + +// dynacastQualityAudio manages enable a single receiver of a media track +type dynacastQualityAudio struct { + params dynacastQualityAudioParams + + // quality level enable/disable + lock sync.RWMutex + initialized bool + subscriberEnables map[livekit.ParticipantID]bool + subscriberNodeEnables map[livekit.NodeID]bool + enabled bool + regressTo dynacastQuality + + dynacastQualityNull +} + +func newDynacastQualityAudio(params dynacastQualityAudioParams) dynacastQuality { + return &dynacastQualityAudio{ + params: params, + subscriberEnables: make(map[livekit.ParticipantID]bool), + subscriberNodeEnables: make(map[livekit.NodeID]bool), + } +} + +func (d *dynacastQualityAudio) Start() { + d.reset() +} + +func (d *dynacastQualityAudio) Restart() { + d.reset() +} + +func (d *dynacastQualityAudio) Stop() { +} + +func (d *dynacastQualityAudio) NotifySubscription(subscriberID livekit.ParticipantID, enabled bool) { + d.params.Logger.Debugw( + "setting subscriber codec enable", + "mime", d.params.MimeType, + "subscriberID", subscriberID, + "enabled", enabled, + ) + + d.lock.Lock() + if r := d.regressTo; r != nil { + d.lock.Unlock() + return + } + + if !enabled { + delete(d.subscriberEnables, subscriberID) + } else { + d.subscriberEnables[subscriberID] = true + } + d.lock.Unlock() + + d.updateQualityChange(false) +} + +func (d *dynacastQualityAudio) NotifySubscriptionNode(nodeID livekit.NodeID, enabled bool) { + d.params.Logger.Debugw( + "setting subscriber node codec enabled", + "mime", d.params.MimeType, + "subscriberNodeID", nodeID, + "enabled", enabled, + ) + + d.lock.Lock() + if r := d.regressTo; r != nil { + // the downstream node will synthesize correct enable (its dynacast manager has codec regression), just ignore it + d.params.Logger.Debugw( + "ignoring node codec change, regressed to another dynacast quality", + "mime", d.params.MimeType, + "regressedMime", d.regressTo.Mime(), + ) + d.lock.Unlock() + return + } + + if !enabled { + delete(d.subscriberNodeEnables, nodeID) + } else { + d.subscriberNodeEnables[nodeID] = true + } + d.lock.Unlock() + + d.updateQualityChange(false) +} + +func (d *dynacastQualityAudio) ClearSubscriberNodes() { + d.lock.Lock() + d.subscriberNodeEnables = make(map[livekit.NodeID]bool) + d.lock.Unlock() + + d.updateQualityChange(false) +} + +func (d *dynacastQualityAudio) Mime() mime.MimeType { + return d.params.MimeType +} + +func (d *dynacastQualityAudio) RegressTo(other dynacastQuality) { + d.lock.Lock() + d.regressTo = other + d.lock.Unlock() + + other.Restart() +} + +func (d *dynacastQualityAudio) reset() { + d.lock.Lock() + d.initialized = false + d.lock.Unlock() +} + +func (d *dynacastQualityAudio) updateQualityChange(force bool) { + d.lock.Lock() + enabled := len(d.subscriberEnables) != 0 || len(d.subscriberNodeEnables) != 0 + if enabled == d.enabled && d.initialized && !force { + d.lock.Unlock() + return + } + + d.initialized = true + d.enabled = enabled + d.params.Logger.Debugw( + "notifying enabled change", + "mime", d.params.MimeType, + "enabled", d.enabled, + "subscriberNodeEnables", d.subscriberNodeEnables, + "subscribedEnables", d.subscriberEnables, + "force", force, + ) + d.lock.Unlock() + + d.params.Listener.OnUpdateAudioCodecForMime(d.params.MimeType, enabled) +} diff --git a/pkg/rtc/dynacast/dynacastquality.go b/pkg/rtc/dynacast/dynacastqualityvideo.go similarity index 65% rename from pkg/rtc/dynacast/dynacastquality.go rename to pkg/rtc/dynacast/dynacastqualityvideo.go index 77b6aaa..d209504 100644 --- a/pkg/rtc/dynacast/dynacastquality.go +++ b/pkg/rtc/dynacast/dynacastqualityvideo.go @@ -18,23 +18,26 @@ import ( "sync" "time" - "github.com/livekit/livekit-server/pkg/sfu/mime" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" ) +var _ dynacastQuality = (*dynacastQualityVideo)(nil) + const ( initialQualityUpdateWait = 10 * time.Second ) -type DynacastQualityParams struct { +type dynacastQualityVideoParams struct { MimeType mime.MimeType + Listener dynacastQualityListener Logger logger.Logger } -// DynacastQuality manages max subscribed quality of a single receiver of a media track -type DynacastQuality struct { - params DynacastQualityParams +// dynacastQualityVideo manages max subscribed quality of a single receiver of a media track +type dynacastQualityVideo struct { + params dynacastQualityVideoParams // quality level enable/disable lock sync.RWMutex @@ -43,38 +46,32 @@ type DynacastQuality struct { maxSubscriberNodeQuality map[livekit.NodeID]livekit.VideoQuality maxSubscribedQuality livekit.VideoQuality maxQualityTimer *time.Timer - regressTo *DynacastQuality + regressTo dynacastQuality - onSubscribedMaxQualityChange func(mimeType mime.MimeType, maxSubscribedQuality livekit.VideoQuality) + dynacastQualityNull } -func NewDynacastQuality(params DynacastQualityParams) *DynacastQuality { - return &DynacastQuality{ +func newDynacastQualityVideo(params dynacastQualityVideoParams) dynacastQuality { + return &dynacastQualityVideo{ params: params, maxSubscriberQuality: make(map[livekit.ParticipantID]livekit.VideoQuality), maxSubscriberNodeQuality: make(map[livekit.NodeID]livekit.VideoQuality), } } -func (d *DynacastQuality) Start() { +func (d *dynacastQualityVideo) Start() { d.reset() } -func (d *DynacastQuality) Restart() { +func (d *dynacastQualityVideo) Restart() { d.reset() } -func (d *DynacastQuality) Stop() { +func (d *dynacastQualityVideo) Stop() { d.stopMaxQualityTimer() } -func (d *DynacastQuality) OnSubscribedMaxQualityChange(f func(mimeType mime.MimeType, maxSubscribedQuality livekit.VideoQuality)) { - d.lock.Lock() - defer d.lock.Unlock() - d.onSubscribedMaxQualityChange = f -} - -func (d *DynacastQuality) NotifySubscriberMaxQuality(subscriberID livekit.ParticipantID, quality livekit.VideoQuality) { +func (d *dynacastQualityVideo) NotifySubscriberMaxQuality(subscriberID livekit.ParticipantID, quality livekit.VideoQuality) { d.params.Logger.Debugw( "setting subscriber max quality", "mime", d.params.MimeType, @@ -99,7 +96,7 @@ func (d *DynacastQuality) NotifySubscriberMaxQuality(subscriberID livekit.Partic d.updateQualityChange(false) } -func (d *DynacastQuality) NotifySubscriberNodeMaxQuality(nodeID livekit.NodeID, quality livekit.VideoQuality) { +func (d *dynacastQualityVideo) NotifySubscriberNodeMaxQuality(nodeID livekit.NodeID, quality livekit.VideoQuality) { d.params.Logger.Debugw( "setting subscriber node max quality", "mime", d.params.MimeType, @@ -110,8 +107,12 @@ func (d *DynacastQuality) NotifySubscriberNodeMaxQuality(nodeID livekit.NodeID, d.lock.Lock() if r := d.regressTo; r != nil { // the downstream node will synthesize correct quality notify (its dynacast manager has codec regression), just ignore it + d.params.Logger.Debugw( + "ignoring node quality change, regressed to another dynacast quality", + "mime", d.params.MimeType, + "regressedMime", d.regressTo.Mime(), + ) d.lock.Unlock() - r.params.Logger.Debugw("ignoring node quality change, regressed to another dynacast quality", "mime", d.params.MimeType) return } @@ -125,7 +126,19 @@ func (d *DynacastQuality) NotifySubscriberNodeMaxQuality(nodeID livekit.NodeID, d.updateQualityChange(false) } -func (d *DynacastQuality) RegressTo(other *DynacastQuality) { +func (d *dynacastQualityVideo) ClearSubscriberNodes() { + d.lock.Lock() + d.maxSubscriberNodeQuality = make(map[livekit.NodeID]livekit.VideoQuality) + d.lock.Unlock() + + d.updateQualityChange(false) +} + +func (d *dynacastQualityVideo) Mime() mime.MimeType { + return d.params.MimeType +} + +func (d *dynacastQualityVideo) RegressTo(other dynacastQuality) { d.lock.Lock() d.regressTo = other maxSubscriberQuality := d.maxSubscriberQuality @@ -134,33 +147,41 @@ func (d *DynacastQuality) RegressTo(other *DynacastQuality) { d.maxSubscriberNodeQuality = make(map[livekit.NodeID]livekit.VideoQuality) d.lock.Unlock() - other.lock.Lock() + other.Replace(maxSubscriberQuality, maxSubscriberNodeQuality) +} + +func (d *dynacastQualityVideo) Replace( + maxSubscriberQuality map[livekit.ParticipantID]livekit.VideoQuality, + maxSubscriberNodeQuality map[livekit.NodeID]livekit.VideoQuality, +) { + d.lock.Lock() for subID, quality := range maxSubscriberQuality { - if otherQuality, ok := other.maxSubscriberQuality[subID]; ok { + if oldQuality, ok := d.maxSubscriberQuality[subID]; ok { // no QUALITY_OFF in the map - if quality > otherQuality { - other.maxSubscriberQuality[subID] = quality + if quality > oldQuality { + d.maxSubscriberQuality[subID] = quality } } else { - other.maxSubscriberQuality[subID] = quality + d.maxSubscriberQuality[subID] = quality } } for nodeID, quality := range maxSubscriberNodeQuality { - if otherQuality, ok := other.maxSubscriberNodeQuality[nodeID]; ok { + if oldQuality, ok := d.maxSubscriberNodeQuality[nodeID]; ok { // no QUALITY_OFF in the map - if quality > otherQuality { - other.maxSubscriberNodeQuality[nodeID] = quality + if quality > oldQuality { + d.maxSubscriberNodeQuality[nodeID] = quality } } else { - other.maxSubscriberNodeQuality[nodeID] = quality + d.maxSubscriberNodeQuality[nodeID] = quality } } - other.lock.Unlock() - other.Restart() + d.lock.Unlock() + + d.Restart() } -func (d *DynacastQuality) reset() { +func (d *dynacastQualityVideo) reset() { d.lock.Lock() d.initialized = false d.lock.Unlock() @@ -168,7 +189,7 @@ func (d *DynacastQuality) reset() { d.startMaxQualityTimer() } -func (d *DynacastQuality) updateQualityChange(force bool) { +func (d *dynacastQualityVideo) updateQualityChange(force bool) { d.lock.Lock() maxSubscribedQuality := livekit.VideoQuality_OFF for _, subQuality := range d.maxSubscriberQuality { @@ -189,22 +210,20 @@ func (d *DynacastQuality) updateQualityChange(force bool) { d.initialized = true d.maxSubscribedQuality = maxSubscribedQuality - d.params.Logger.Debugw("notifying quality change", + d.params.Logger.Debugw( + "notifying quality change", "mime", d.params.MimeType, "maxSubscriberQuality", d.maxSubscriberQuality, "maxSubscriberNodeQuality", d.maxSubscriberNodeQuality, "maxSubscribedQuality", d.maxSubscribedQuality, "force", force, ) - onSubscribedMaxQualityChange := d.onSubscribedMaxQualityChange d.lock.Unlock() - if onSubscribedMaxQualityChange != nil { - onSubscribedMaxQualityChange(d.params.MimeType, maxSubscribedQuality) - } + d.params.Listener.OnUpdateMaxQualityForMime(d.params.MimeType, maxSubscribedQuality) } -func (d *DynacastQuality) startMaxQualityTimer() { +func (d *dynacastQualityVideo) startMaxQualityTimer() { d.lock.Lock() defer d.lock.Unlock() @@ -219,7 +238,7 @@ func (d *DynacastQuality) startMaxQualityTimer() { }) } -func (d *DynacastQuality) stopMaxQualityTimer() { +func (d *dynacastQualityVideo) stopMaxQualityTimer() { d.lock.Lock() defer d.lock.Unlock() diff --git a/pkg/rtc/dynacast/interfaces.go b/pkg/rtc/dynacast/interfaces.go new file mode 100644 index 0000000..c544ff5 --- /dev/null +++ b/pkg/rtc/dynacast/interfaces.go @@ -0,0 +1,185 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dynacast + +import ( + "github.com/livekit/protocol/codecs/mime" + "github.com/livekit/protocol/livekit" + + "github.com/livekit/livekit-server/pkg/rtc/types" +) + +type DynacastManagerListener interface { + OnDynacastSubscribedMaxQualityChange( + subscribedQualities []*livekit.SubscribedCodec, + maxSubscribedQualities []types.SubscribedCodecQuality, + ) + + OnDynacastSubscribedAudioCodecChange(codecs []*livekit.SubscribedAudioCodec) +} + +var _ DynacastManagerListener = (*DynacastManagerListenerNull)(nil) + +type DynacastManagerListenerNull struct { +} + +func (d *DynacastManagerListenerNull) OnDynacastSubscribedMaxQualityChange( + subscribedQualities []*livekit.SubscribedCodec, + maxSubscribedQualities []types.SubscribedCodecQuality, +) { +} +func (d *DynacastManagerListenerNull) OnDynacastSubscribedAudioCodecChange( + codecs []*livekit.SubscribedAudioCodec, +) { +} + +// ----------------------------------------- + +type DynacastManager interface { + AddCodec(mime mime.MimeType) + HandleCodecRegression(fromMime, toMime mime.MimeType) + Restart() + Close() + ForceUpdate() + ForceQuality(quality livekit.VideoQuality) + ForceEnable(enabled bool) + + NotifySubscriberMaxQuality( + subscriberID livekit.ParticipantID, + mime mime.MimeType, + quality livekit.VideoQuality, + ) + NotifySubscription( + subscriberID livekit.ParticipantID, + mime mime.MimeType, + enabled bool, + ) + + NotifySubscriberNodeMaxQuality( + nodeID livekit.NodeID, + qualities []types.SubscribedCodecQuality, + ) + NotifySubscriptionNode( + nodeID livekit.NodeID, + codecs []*livekit.SubscribedAudioCodec, + ) + ClearSubscriberNodes() +} + +var _ DynacastManager = (*dynacastManagerNull)(nil) + +type dynacastManagerNull struct { +} + +func (d *dynacastManagerNull) AddCodec(mime mime.MimeType) {} +func (d *dynacastManagerNull) HandleCodecRegression(fromMime, toMime mime.MimeType) {} +func (d *dynacastManagerNull) Restart() {} +func (d *dynacastManagerNull) Close() {} +func (d *dynacastManagerNull) ForceUpdate() {} +func (d *dynacastManagerNull) ForceQuality(quality livekit.VideoQuality) {} +func (d *dynacastManagerNull) ForceEnable(enabled bool) {} +func (d *dynacastManagerNull) NotifySubscriberMaxQuality( + subscriberID livekit.ParticipantID, + mime mime.MimeType, + quality livekit.VideoQuality, +) { +} +func (d *dynacastManagerNull) NotifySubscription( + subscriberID livekit.ParticipantID, + mime mime.MimeType, + enabled bool, +) { +} +func (d *dynacastManagerNull) NotifySubscriberNodeMaxQuality( + nodeID livekit.NodeID, + qualities []types.SubscribedCodecQuality, +) { +} +func (d *dynacastManagerNull) NotifySubscriptionNode( + nodeID livekit.NodeID, + codecs []*livekit.SubscribedAudioCodec, +) { +} +func (d *dynacastManagerNull) ClearSubscriberNodes() {} + +// ------------------------------------------------ + +type dynacastQualityListener interface { + OnUpdateMaxQualityForMime(mimeType mime.MimeType, maxQuality livekit.VideoQuality) + OnUpdateAudioCodecForMime(mimeType mime.MimeType, enabled bool) +} + +var _ dynacastQualityListener = (*dynacastQualityListenerNull)(nil) + +type dynacastQualityListenerNull struct { +} + +func (d *dynacastQualityListenerNull) OnUpdateMaxQualityForMime( + mimeType mime.MimeType, + maxQuality livekit.VideoQuality, +) { +} + +func (d *dynacastQualityListenerNull) OnUpdateAudioCodecForMime( + mimeType mime.MimeType, + enabled bool, +) { +} + +// ------------------------------------------------ + +type dynacastQuality interface { + Start() + Restart() + Stop() + + NotifySubscriberMaxQuality(subscriberID livekit.ParticipantID, quality livekit.VideoQuality) + NotifySubscription(subscriberID livekit.ParticipantID, enabled bool) + + NotifySubscriberNodeMaxQuality(nodeID livekit.NodeID, quality livekit.VideoQuality) + NotifySubscriptionNode(nodeID livekit.NodeID, enabled bool) + ClearSubscriberNodes() + + Replace( + maxSubscriberQuality map[livekit.ParticipantID]livekit.VideoQuality, + maxSubscriberNodeQuality map[livekit.NodeID]livekit.VideoQuality, + ) + + Mime() mime.MimeType + RegressTo(other dynacastQuality) +} + +var _ dynacastQuality = (*dynacastQualityNull)(nil) + +type dynacastQualityNull struct { +} + +func (d *dynacastQualityNull) Start() {} +func (d *dynacastQualityNull) Restart() {} +func (d *dynacastQualityNull) Stop() {} +func (d *dynacastQualityNull) NotifySubscriberMaxQuality(subscriberID livekit.ParticipantID, quality livekit.VideoQuality) { +} +func (d *dynacastQualityNull) NotifySubscription(subscriberID livekit.ParticipantID, enabled bool) {} +func (d *dynacastQualityNull) NotifySubscriberNodeMaxQuality(nodeID livekit.NodeID, quality livekit.VideoQuality) { +} +func (d *dynacastQualityNull) NotifySubscriptionNode(nodeID livekit.NodeID, enabled bool) {} +func (d *dynacastQualityNull) ClearSubscriberNodes() {} +func (d *dynacastQualityNull) Replace( + maxSubscriberQuality map[livekit.ParticipantID]livekit.VideoQuality, + maxSubscriberNodeQuality map[livekit.NodeID]livekit.VideoQuality, +) { +} +func (d *dynacastQualityNull) Mime() mime.MimeType { return mime.MimeTypeUnknown } +func (d *dynacastQualityNull) RegressTo(other dynacastQuality) {} diff --git a/pkg/rtc/egress.go b/pkg/rtc/egress.go index b3fc017..f403ea6 100644 --- a/pkg/rtc/egress.go +++ b/pkg/rtc/egress.go @@ -29,6 +29,7 @@ import ( type EgressLauncher interface { StartEgress(context.Context, *rpc.StartEgressRequest) (*livekit.EgressInfo, error) + StopEgress(context.Context, *livekit.StopEgressRequest) (*livekit.EgressInfo, error) } func StartParticipantEgress( @@ -42,16 +43,17 @@ func StartParticipantEgress( ) error { if req, err := startParticipantEgress(ctx, launcher, opts, identity, roomName, roomID); err != nil { // send egress failed webhook - ts.NotifyEvent(ctx, &livekit.WebhookEvent{ - Event: webhook.EventEgressEnded, - EgressInfo: &livekit.EgressInfo{ - RoomId: string(roomID), - RoomName: string(roomName), - Status: livekit.EgressStatus_EGRESS_FAILED, - Error: err.Error(), - Request: &livekit.EgressInfo_Participant{Participant: req}, - }, - }) + + info := &livekit.EgressInfo{ + RoomId: string(roomID), + RoomName: string(roomName), + Status: livekit.EgressStatus_EGRESS_FAILED, + Error: err.Error(), + Request: &livekit.EgressInfo_Participant{Participant: req}, + } + + ts.NotifyEgressEvent(ctx, webhook.EventEgressEnded, info) + return err } return nil @@ -103,16 +105,16 @@ func StartTrackEgress( ) error { if req, err := startTrackEgress(ctx, launcher, opts, track, roomName, roomID); err != nil { // send egress failed webhook - ts.NotifyEvent(ctx, &livekit.WebhookEvent{ - Event: webhook.EventEgressEnded, - EgressInfo: &livekit.EgressInfo{ - RoomId: string(roomID), - RoomName: string(roomName), - Status: livekit.EgressStatus_EGRESS_FAILED, - Error: err.Error(), - Request: &livekit.EgressInfo_Track{Track: req}, - }, - }) + + info := &livekit.EgressInfo{ + RoomId: string(roomID), + RoomName: string(roomName), + Status: livekit.EgressStatus_EGRESS_FAILED, + Error: err.Error(), + Request: &livekit.EgressInfo_Track{Track: req}, + } + ts.NotifyEgressEvent(ctx, webhook.EventEgressEnded, info) + return err } return nil diff --git a/pkg/rtc/errors.go b/pkg/rtc/errors.go index d66fc00..bbe3906 100644 --- a/pkg/rtc/errors.go +++ b/pkg/rtc/errors.go @@ -32,9 +32,6 @@ var ( ErrEmptyParticipantID = errors.New("participant ID cannot be empty") ErrMissingGrants = errors.New("VideoGrant is missing") ErrInternalError = errors.New("internal error") - ErrNameExceedsLimits = errors.New("name length exceeds limits") - ErrMetadataExceedsLimits = errors.New("metadata size exceeds limits") - ErrAttributesExceedsLimits = errors.New("attributes size exceeds limits") // Track subscription related ErrNoTrackPermission = errors.New("participant is not allowed to subscribe to this track") diff --git a/pkg/rtc/mediaengine.go b/pkg/rtc/mediaengine.go index a603446..cfc53ea 100644 --- a/pkg/rtc/mediaengine.go +++ b/pkg/rtc/mediaengine.go @@ -20,146 +20,67 @@ import ( "github.com/pion/webrtc/v4" - "github.com/livekit/livekit-server/pkg/sfu/mime" + protoCodecs "github.com/livekit/protocol/codecs" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" ) -var OpusCodecCapability = webrtc.RTPCodecCapability{ - MimeType: mime.MimeTypeOpus.String(), - ClockRate: 48000, - Channels: 2, - SDPFmtpLine: "minptime=10;useinbandfec=1", -} -var RedCodecCapability = webrtc.RTPCodecCapability{ - MimeType: mime.MimeTypeRED.String(), - ClockRate: 48000, - Channels: 2, - SDPFmtpLine: "111/111", -} -var videoRTX = webrtc.RTPCodecCapability{ - MimeType: mime.MimeTypeRTX.String(), - ClockRate: 90000, -} - func registerCodecs(me *webrtc.MediaEngine, codecs []*livekit.Codec, rtcpFeedback RTCPFeedbackConfig, filterOutH264HighProfile bool) error { - opusCodec := OpusCodecCapability - opusCodec.RTCPFeedback = rtcpFeedback.Audio - var opusPayload webrtc.PayloadType - if IsCodecEnabled(codecs, opusCodec) { - opusPayload = 111 - if err := me.RegisterCodec(webrtc.RTPCodecParameters{ - RTPCodecCapability: opusCodec, - PayloadType: opusPayload, - }, webrtc.RTPCodecTypeAudio); err != nil { + // audio codecs + if IsCodecEnabled(codecs, protoCodecs.OpusCodecParameters.RTPCodecCapability) { + cp := protoCodecs.OpusCodecParameters + cp.RTPCodecCapability.RTCPFeedback = rtcpFeedback.Audio + if err := me.RegisterCodec(cp, webrtc.RTPCodecTypeAudio); err != nil { return err } - if IsCodecEnabled(codecs, RedCodecCapability) { - if err := me.RegisterCodec(webrtc.RTPCodecParameters{ - RTPCodecCapability: RedCodecCapability, - PayloadType: 63, - }, webrtc.RTPCodecTypeAudio); err != nil { + if IsCodecEnabled(codecs, protoCodecs.RedCodecParameters.RTPCodecCapability) { + if err := me.RegisterCodec(protoCodecs.RedCodecParameters, webrtc.RTPCodecTypeAudio); err != nil { return err } } } - rtxEnabled := IsCodecEnabled(codecs, videoRTX) + for _, codec := range []webrtc.RTPCodecParameters{protoCodecs.PCMUCodecParameters, protoCodecs.PCMACodecParameters} { + if !IsCodecEnabled(codecs, codec.RTPCodecCapability) { + continue + } - h264HighProfileFmtp := "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640032" - for _, codec := range []webrtc.RTPCodecParameters{ - { - RTPCodecCapability: webrtc.RTPCodecCapability{ - MimeType: mime.MimeTypeVP8.String(), - ClockRate: 90000, - RTCPFeedback: rtcpFeedback.Video, - }, - PayloadType: 96, - }, - { - RTPCodecCapability: webrtc.RTPCodecCapability{ - MimeType: mime.MimeTypeVP9.String(), - ClockRate: 90000, - SDPFmtpLine: "profile-id=0", - RTCPFeedback: rtcpFeedback.Video, - }, - PayloadType: 98, - }, - { - RTPCodecCapability: webrtc.RTPCodecCapability{ - MimeType: mime.MimeTypeVP9.String(), - ClockRate: 90000, - SDPFmtpLine: "profile-id=1", - RTCPFeedback: rtcpFeedback.Video, - }, - PayloadType: 100, - }, - { - RTPCodecCapability: webrtc.RTPCodecCapability{ - MimeType: mime.MimeTypeH264.String(), - ClockRate: 90000, - SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f", - RTCPFeedback: rtcpFeedback.Video, - }, - PayloadType: 125, - }, - { - RTPCodecCapability: webrtc.RTPCodecCapability{ - MimeType: mime.MimeTypeH264.String(), - ClockRate: 90000, - SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42e01f", - RTCPFeedback: rtcpFeedback.Video, - }, - PayloadType: 108, - }, - { - RTPCodecCapability: webrtc.RTPCodecCapability{ - MimeType: mime.MimeTypeH264.String(), - ClockRate: 90000, - SDPFmtpLine: h264HighProfileFmtp, - RTCPFeedback: rtcpFeedback.Video, - }, - PayloadType: 123, - }, - { - RTPCodecCapability: webrtc.RTPCodecCapability{ - MimeType: mime.MimeTypeAV1.String(), - ClockRate: 90000, - RTCPFeedback: rtcpFeedback.Video, - }, - PayloadType: 35, - }, - { - RTPCodecCapability: webrtc.RTPCodecCapability{ - MimeType: webrtc.MimeTypeH265, - ClockRate: 90000, - RTCPFeedback: rtcpFeedback.Video, - }, - PayloadType: 116, - }, - } { - if filterOutH264HighProfile && codec.RTPCodecCapability.SDPFmtpLine == h264HighProfileFmtp { + cp := codec + cp.RTPCodecCapability.RTCPFeedback = rtcpFeedback.Audio + if err := me.RegisterCodec(cp, webrtc.RTPCodecTypeAudio); err != nil { + return err + } + } + + // video codecs + rtxEnabled := IsCodecEnabled(codecs, protoCodecs.VideoRTXCodecParameters.RTPCodecCapability) + for _, codec := range protoCodecs.VideoCodecsParameters { + if filterOutH264HighProfile && codec.RTPCodecCapability.SDPFmtpLine == protoCodecs.H264HighProfileFmtp { continue } if mime.IsMimeTypeStringRTX(codec.MimeType) { continue } - if IsCodecEnabled(codecs, codec.RTPCodecCapability) { - if err := me.RegisterCodec(codec, webrtc.RTPCodecTypeVideo); err != nil { - return err - } - if rtxEnabled { - if err := me.RegisterCodec(webrtc.RTPCodecParameters{ - RTPCodecCapability: webrtc.RTPCodecCapability{ - MimeType: mime.MimeTypeRTX.String(), - ClockRate: 90000, - SDPFmtpLine: fmt.Sprintf("apt=%d", codec.PayloadType), - }, - PayloadType: codec.PayloadType + 1, - }, webrtc.RTPCodecTypeVideo); err != nil { - return err - } - } + if !IsCodecEnabled(codecs, codec.RTPCodecCapability) { + continue + } + + cp := codec + cp.RTPCodecCapability.RTCPFeedback = rtcpFeedback.Video + if err := me.RegisterCodec(cp, webrtc.RTPCodecTypeVideo); err != nil { + return err + } + + if !rtxEnabled { + continue + } + + cp = protoCodecs.VideoRTXCodecParameters + cp.RTPCodecCapability.SDPFmtpLine = fmt.Sprintf("apt=%d", codec.PayloadType) + cp.PayloadType = codec.PayloadType + 1 + if err := me.RegisterCodec(cp, webrtc.RTPCodecTypeVideo); err != nil { + return err } } return nil @@ -215,3 +136,77 @@ func selectAlternativeVideoCodec(enabledCodecs []*livekit.Codec) string { // no viable codec in the list of enabled codecs, fall back to the most widely supported codec return mime.MimeTypeVP8.String() } + +func selectAlternativeAudioCodec(enabledCodecs []*livekit.Codec) string { + for _, c := range enabledCodecs { + if mime.IsMimeTypeStringAudio(c.Mime) { + return c.Mime + } + } + // no viable codec in the list of enabled codecs, fall back to the most widely supported codec + return mime.MimeTypeOpus.String() +} + +// mergeCodecsByMime returns a union b, deduplicated by mime type. +func mergeCodecsByMime(a, b []*livekit.Codec) []*livekit.Codec { + merged := make([]*livekit.Codec, 0, len(a)+len(b)) + merged = append(merged, a...) + for _, c := range b { + seen := false + for _, existing := range a { + if mime.IsMimeTypeStringEqual(c.Mime, existing.Mime) { + seen = true + break + } + } + if !seen { + merged = append(merged, c) + } + } + return merged +} + +func filterCodecs( + codecs []webrtc.RTPCodecParameters, + enabledCodecs []*livekit.Codec, + rtcpFeedbackConfig RTCPFeedbackConfig, + filterOutH264HighProfile bool, +) []webrtc.RTPCodecParameters { + filteredCodecs := make([]webrtc.RTPCodecParameters, 0, len(codecs)) + for _, c := range codecs { + if filterOutH264HighProfile && isH264HighProfile(c.RTPCodecCapability.SDPFmtpLine) { + continue + } + + for _, enabledCodec := range enabledCodecs { + if mime.NormalizeMimeType(enabledCodec.Mime) == mime.NormalizeMimeType(c.RTPCodecCapability.MimeType) { + if !mime.IsMimeTypeStringEqual(c.RTPCodecCapability.MimeType, mime.MimeTypeRTX.String()) { + if mime.IsMimeTypeStringVideo(c.RTPCodecCapability.MimeType) { + c.RTPCodecCapability.RTCPFeedback = rtcpFeedbackConfig.Video + } else { + c.RTPCodecCapability.RTCPFeedback = rtcpFeedbackConfig.Audio + } + } + filteredCodecs = append(filteredCodecs, c) + break + } + } + } + return filteredCodecs +} + +func isH264HighProfile(fmtp string) bool { + params := strings.Split(fmtp, ";") + for _, param := range params { + parts := strings.Split(param, "=") + if len(parts) == 2 { + if parts[0] == "profile-level-id" { + // https://datatracker.ietf.org/doc/html/rfc6184#section-8.1 + // hex value 0x64 for profile_idc is high profile + return strings.HasPrefix(parts[1], "64") + } + } + } + + return false +} diff --git a/pkg/rtc/mediaengine_test.go b/pkg/rtc/mediaengine_test.go index 122861a..aa70368 100644 --- a/pkg/rtc/mediaengine_test.go +++ b/pkg/rtc/mediaengine_test.go @@ -20,7 +20,7 @@ import ( "github.com/pion/webrtc/v4" "github.com/stretchr/testify/require" - "github.com/livekit/livekit-server/pkg/sfu/mime" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" ) diff --git a/pkg/rtc/mediatrack.go b/pkg/rtc/mediatrack.go index cd1bd72..eefa1c0 100644 --- a/pkg/rtc/mediatrack.go +++ b/pkg/rtc/mediatrack.go @@ -15,7 +15,6 @@ package rtc import ( - "context" "math" "sync" "time" @@ -24,8 +23,11 @@ import ( "github.com/pion/webrtc/v4" "go.uber.org/atomic" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" + "github.com/livekit/protocol/observability/roomobs" + "github.com/livekit/protocol/utils/mono" "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/livekit-server/pkg/rtc/dynacast" @@ -33,77 +35,100 @@ import ( "github.com/livekit/livekit-server/pkg/sfu" "github.com/livekit/livekit-server/pkg/sfu/buffer" "github.com/livekit/livekit-server/pkg/sfu/connectionquality" - "github.com/livekit/livekit-server/pkg/sfu/mime" + "github.com/livekit/livekit-server/pkg/sfu/interceptor" "github.com/livekit/livekit-server/pkg/telemetry" util "github.com/livekit/mediatransportutil" ) +var _ types.LocalMediaTrack = (*MediaTrack)(nil) + // MediaTrack represents a WebRTC track that needs to be forwarded // Implements MediaTrack and PublishedTrack interface type MediaTrack struct { params MediaTrackParams - numUpTracks atomic.Uint32 buffer *buffer.Buffer everSubscribed atomic.Bool *MediaTrackReceiver *MediaLossProxy - dynacastManager *dynacast.DynacastManager + dynacastManager dynacast.DynacastManager lock sync.RWMutex rttFromXR atomic.Bool - enableRegression bool + backupCodecPolicy livekit.BackupCodecPolicy regressionTargetCodec mime.MimeType regressionTargetCodecReceived bool + + onSubscribedMaxQualityChange func( + trackID livekit.TrackID, + trackInfo *livekit.TrackInfo, + subscribedQualities []*livekit.SubscribedCodec, + maxSubscribedQualities []types.SubscribedCodecQuality, + ) error + onSubscribedAudioCodecChange func( + trackID livekit.TrackID, + codecs []*livekit.SubscribedAudioCodec, + ) error } type MediaTrackParams struct { - SignalCid string - SdpCid string - ParticipantID livekit.ParticipantID - ParticipantIdentity livekit.ParticipantIdentity - ParticipantVersion uint32 - BufferFactory *buffer.Factory - ReceiverConfig ReceiverConfig - SubscriberConfig DirectionConfig - PLIThrottleConfig sfu.PLIThrottleConfig - AudioConfig sfu.AudioConfig - VideoConfig config.VideoConfig - Telemetry telemetry.TelemetryService - Logger logger.Logger - SimTracks map[uint32]SimulcastTrackInfo - OnRTCP func([]rtcp.Packet) - ForwardStats *sfu.ForwardStats - OnTrackEverSubscribed func(livekit.TrackID) + ParticipantID func() livekit.ParticipantID + ParticipantIdentity livekit.ParticipantIdentity + ParticipantVersion uint32 + ParticipantCountry string + ParticipantKind livekit.ParticipantInfo_Kind + ParticipantKindDetails []livekit.ParticipantInfo_KindDetail + BufferFactory *buffer.Factory + ReceiverConfig ReceiverConfig + SubscriberConfig DirectionConfig + PLIThrottleConfig sfu.PLIThrottleConfig + AudioConfig sfu.AudioConfig + VideoConfig config.VideoConfig + TelemetryListener types.ParticipantTelemetryListener + Logger logger.Logger + Reporter roomobs.TrackReporter + SimTracks map[uint32]interceptor.SimulcastTrackInfo + OnRTCP func([]rtcp.Packet) + ForwardStats *sfu.ForwardStats + OnTrackEverSubscribed func(livekit.TrackID) + ShouldRegressCodec func() bool + PreferVideoSizeFromMedia bool + EnableRTPStreamRestartDetection bool + UpdateTrackInfoByVideoSizeChange bool + ForceBackupCodecPolicySimulcast bool } func NewMediaTrack(params MediaTrackParams, ti *livekit.TrackInfo) *MediaTrack { t := &MediaTrack{ - params: params, + params: params, + backupCodecPolicy: ti.BackupCodecPolicy, } - // TODO: disable codec regression until simulcast-codec clients knows that - if ti.BackupCodecPolicy == livekit.BackupCodecPolicy_REGRESSION && len(ti.Codecs) > 1 { - t.enableRegression = true + if t.params.ForceBackupCodecPolicySimulcast { + t.backupCodecPolicy = livekit.BackupCodecPolicy_SIMULCAST + } + + if t.backupCodecPolicy != livekit.BackupCodecPolicy_SIMULCAST && len(ti.Codecs) > 1 { t.regressionTargetCodec = mime.NormalizeMimeType(ti.Codecs[1].MimeType) t.params.Logger.Debugw("track enabled codec regression", "regressionCodec", t.regressionTargetCodec) } t.MediaTrackReceiver = NewMediaTrackReceiver(MediaTrackReceiverParams{ - MediaTrack: t, - IsRelayed: false, - ParticipantID: params.ParticipantID, - ParticipantIdentity: params.ParticipantIdentity, - ParticipantVersion: params.ParticipantVersion, - ReceiverConfig: params.ReceiverConfig, - SubscriberConfig: params.SubscriberConfig, - AudioConfig: params.AudioConfig, - Telemetry: params.Telemetry, - Logger: params.Logger, - RegressionTargetCodec: t.regressionTargetCodec, + MediaTrack: t, + IsRelayed: false, + ParticipantID: params.ParticipantID, + ParticipantIdentity: params.ParticipantIdentity, + ParticipantVersion: params.ParticipantVersion, + ReceiverConfig: params.ReceiverConfig, + SubscriberConfig: params.SubscriberConfig, + AudioConfig: params.AudioConfig, + TelemetryListener: params.TelemetryListener, + Logger: params.Logger, + RegressionTargetCodec: t.regressionTargetCodec, + PreferVideoSizeFromMedia: params.PreferVideoSizeFromMedia, }, ti) if ti.Type == livekit.TrackType_AUDIO { @@ -118,27 +143,57 @@ func NewMediaTrack(params MediaTrackParams, ti *livekit.TrackInfo) *MediaTrack { t.MediaTrackReceiver.OnMediaLossFeedback(t.MediaLossProxy.HandleMaxLossFeedback) } - if ti.Type == livekit.TrackType_VIDEO { - t.dynacastManager = dynacast.NewDynacastManager(dynacast.DynacastManagerParams{ + switch ti.Type { + case livekit.TrackType_VIDEO: + t.dynacastManager = dynacast.NewDynacastManagerVideo(dynacast.DynacastManagerVideoParams{ DynacastPauseDelay: params.VideoConfig.DynacastPauseDelay, + Listener: t, Logger: params.Logger, }) - t.MediaTrackReceiver.OnSetupReceiver(func(mime mime.MimeType) { + + case livekit.TrackType_AUDIO: + if len(ti.Codecs) > 1 { + t.dynacastManager = dynacast.NewDynacastManagerAudio(dynacast.DynacastManagerAudioParams{ + Listener: t, + Logger: params.Logger, + }) + } + } + t.MediaTrackReceiver.OnSetupReceiver(func(mime mime.MimeType) { + if t.dynacastManager != nil { t.dynacastManager.AddCodec(mime) - }) - t.MediaTrackReceiver.OnSubscriberMaxQualityChange( - func(subscriberID livekit.ParticipantID, mimeType mime.MimeType, layer int32) { + } + }) + t.MediaTrackReceiver.OnSubscriberMaxQualityChange( + func(subscriberID livekit.ParticipantID, mimeType mime.MimeType, layer int32) { + if t.dynacastManager != nil { t.dynacastManager.NotifySubscriberMaxQuality( subscriberID, mimeType, - buffer.SpatialLayerToVideoQuality(layer, t.MediaTrackReceiver.TrackInfo()), + buffer.GetVideoQualityForSpatialLayer( + mimeType, + layer, + t.MediaTrackReceiver.TrackInfo(), + ), ) - }, - ) - t.MediaTrackReceiver.OnCodecRegression(func(old, new webrtc.RTPCodecParameters) { - t.dynacastManager.HandleCodecRegression(mime.NormalizeMimeType(old.MimeType), mime.NormalizeMimeType(new.MimeType)) - }) - } + } + }, + ) + t.MediaTrackReceiver.OnSubscriberAudioCodecChange( + func(subscriberID livekit.ParticipantID, mimeType mime.MimeType, enabled bool) { + if t.dynacastManager != nil { + t.dynacastManager.NotifySubscription(subscriberID, mimeType, enabled) + } + }, + ) + t.MediaTrackReceiver.OnCodecRegression(func(old, new webrtc.RTPCodecParameters) { + if t.dynacastManager != nil { + t.dynacastManager.HandleCodecRegression( + mime.NormalizeMimeType(old.MimeType), + mime.NormalizeMimeType(new.MimeType), + ) + } + }) return t } @@ -151,24 +206,20 @@ func (t *MediaTrack) OnSubscribedMaxQualityChange( maxSubscribedQualities []types.SubscribedCodecQuality, ) error, ) { - if t.dynacastManager == nil { - return - } + t.lock.Lock() + t.onSubscribedMaxQualityChange = f + t.lock.Unlock() +} - handler := func(subscribedQualities []*livekit.SubscribedCodec, maxSubscribedQualities []types.SubscribedCodecQuality) { - if f != nil && !t.IsMuted() { - _ = f(t.ID(), t.ToProto(), subscribedQualities, maxSubscribedQualities) - } - - for _, q := range maxSubscribedQualities { - receiver := t.Receiver(q.CodecMime) - if receiver != nil { - receiver.SetMaxExpectedSpatialLayer(buffer.VideoQualityToSpatialLayer(q.Quality, t.MediaTrackReceiver.TrackInfo())) - } - } - } - - t.dynacastManager.OnSubscribedMaxQualityChange(handler) +func (t *MediaTrack) OnSubscribedAudioCodecChange( + f func( + trackID livekit.TrackID, + codecs []*livekit.SubscribedAudioCodec, + ) error, +) { + t.lock.Lock() + t.onSubscribedAudioCodecChange = f + t.lock.Unlock() } func (t *MediaTrack) NotifySubscriberNodeMaxQuality(nodeID livekit.NodeID, qualities []types.SubscribedCodecQuality) { @@ -177,47 +228,84 @@ func (t *MediaTrack) NotifySubscriberNodeMaxQuality(nodeID livekit.NodeID, quali } } -func (t *MediaTrack) SignalCid() string { - return t.params.SignalCid +func (t *MediaTrack) NotifySubscriptionNode(nodeID livekit.NodeID, codecs []*livekit.SubscribedAudioCodec) { + if t.dynacastManager != nil { + t.dynacastManager.NotifySubscriptionNode(nodeID, codecs) + } } -func (t *MediaTrack) HasSdpCid(cid string) bool { - if t.params.SdpCid == cid { - return true +func (t *MediaTrack) ClearSubscriberNodes() { + if t.dynacastManager != nil { + t.dynacastManager.ClearSubscriberNodes() } +} - ti := t.MediaTrackReceiver.TrackInfoClone() - for _, c := range ti.Codecs { - if c.Cid == cid { - return true +func (t *MediaTrack) HasSignalCid(cid string) bool { + if cid != "" { + ti := t.MediaTrackReceiver.TrackInfoClone() + for _, c := range ti.Codecs { + if c.Cid == cid { + return true + } } } return false } +func (t *MediaTrack) HasSdpCid(cid string) bool { + if cid != "" { + ti := t.MediaTrackReceiver.TrackInfoClone() + for _, c := range ti.Codecs { + if c.Cid == cid || c.SdpCid == cid { + return true + } + } + } + return false +} + +func (t *MediaTrack) GetMimeTypeForSdpCid(cid string) mime.MimeType { + if cid != "" { + ti := t.MediaTrackReceiver.TrackInfoClone() + for _, c := range ti.Codecs { + if c.Cid == cid || c.SdpCid == cid { + return mime.NormalizeMimeType(c.MimeType) + } + } + } + return mime.MimeTypeUnknown +} + +func (t *MediaTrack) GetCidsForMimeType(mimeType mime.MimeType) (string, string) { + ti := t.MediaTrackReceiver.TrackInfoClone() + for _, c := range ti.Codecs { + if mime.NormalizeMimeType(c.MimeType) == mimeType { + return c.Cid, c.SdpCid + } + } + return "", "" +} + func (t *MediaTrack) ToProto() *livekit.TrackInfo { return t.MediaTrackReceiver.TrackInfoClone() } -func (t *MediaTrack) UpdateCodecCid(codecs []*livekit.SimulcastCodec) { - t.MediaTrackReceiver.UpdateCodecCid(codecs) -} - // AddReceiver adds a new RTP receiver to the track, returns true when receiver represents a new codec -func (t *MediaTrack) AddReceiver(receiver *webrtc.RTPReceiver, track sfu.TrackRemote, mid string) bool { +// and if a receiver was added successfully +func (t *MediaTrack) AddReceiver(receiver *webrtc.RTPReceiver, track sfu.TrackRemote, mid string) (bool, bool) { var newCodec bool ssrc := uint32(track.SSRC()) buff, rtcpReader := t.params.BufferFactory.GetBufferPair(ssrc) if buff == nil || rtcpReader == nil { t.params.Logger.Errorw("could not retrieve buffer pair", nil) - return newCodec + return newCodec, false } var lastRR uint32 rtcpReader.OnPacket(func(bytes []byte) { pkts, err := rtcp.Unmarshal(bytes) if err != nil { - t.params.Logger.Errorw("could not unmarshal RTCP", err) + t.params.Logger.Errorw("could not unmarshal RTCP", err, "size", len(bytes)) return } @@ -226,7 +314,13 @@ func (t *MediaTrack) AddReceiver(receiver *webrtc.RTPReceiver, track sfu.TrackRe case *rtcp.SourceDescription: case *rtcp.SenderReport: if pkt.SSRC == uint32(track.SSRC()) { - buff.SetSenderReportData(pkt.RTPTime, pkt.NTPTime, pkt.PacketCount, pkt.OctetCount) + buff.SetSenderReportData(&livekit.RTCPSenderReportState{ + RtpTimestamp: pkt.RTPTime, + NtpTimestamp: pkt.NTPTime, + Packets: pkt.PacketCount, + Octets: uint64(pkt.OctetCount), + At: mono.UnixNano(), + }) } case *rtcp.ExtendedReport: rttFromXR: @@ -255,13 +349,27 @@ func (t *MediaTrack) AddReceiver(receiver *webrtc.RTPReceiver, track sfu.TrackRe t.lock.Lock() var regressCodec bool mimeType := mime.NormalizeMimeType(track.Codec().MimeType) - layer := buffer.RidToSpatialLayer(track.RID(), ti) + layer := buffer.GetSpatialLayerForRid(mimeType, track.RID(), ti) + if layer < 0 { + t.params.Logger.Warnw( + "AddReceiver failed due to negative layer", nil, + "rid", track.RID(), + "layer", layer, + "ssrc", track.SSRC(), + "codec", track.Codec(), + "trackInfo", logger.Proto(ti), + ) + t.lock.Unlock() + return newCodec, false + } + t.params.Logger.Debugw( "AddReceiver", "rid", track.RID(), "layer", layer, "ssrc", track.SSRC(), "codec", track.Codec(), + "trackInfo", logger.Proto(ti), ) wr := t.MediaTrackReceiver.Receiver(mimeType) if wr == nil { @@ -276,6 +384,11 @@ func (t *MediaTrack) AddReceiver(receiver *webrtc.RTPReceiver, track sfu.TrackRe switch len(ti.Codecs) { case 0: // audio track + t.params.Logger.Warnw( + "unexpected 0 codecs in track info", nil, + "mime", mimeType, + "track", logger.Proto(ti), + ) priority = 0 case 1: // older clients or non simulcast-codec, mime type only set later @@ -285,9 +398,13 @@ func (t *MediaTrack) AddReceiver(receiver *webrtc.RTPReceiver, track sfu.TrackRe } } if priority < 0 { - t.params.Logger.Warnw("could not find codec for webrtc receiver", nil, "webrtcCodec", mimeType, "track", logger.Proto(ti)) + t.params.Logger.Warnw( + "could not find codec for webrtc receiver", nil, + "mime", mimeType, + "track", logger.Proto(ti), + ) t.lock.Unlock() - return false + return newCodec, false } newWR := sfu.NewWebRTCReceiver( @@ -300,11 +417,11 @@ func (t *MediaTrack) AddReceiver(receiver *webrtc.RTPReceiver, track sfu.TrackRe sfu.WithPliThrottleConfig(t.params.PLIThrottleConfig), sfu.WithAudioConfig(t.params.AudioConfig), sfu.WithLoadBalanceThreshold(20), - sfu.WithStreamTrackers(), sfu.WithForwardStats(t.params.ForwardStats), + sfu.WithEnableRTPStreamRestartDetection(t.params.EnableRTPStreamRestartDetection), ) newWR.OnCloseHandler(func() { - t.MediaTrackReceiver.SetClosing() + t.MediaTrackReceiver.SetClosing(false) t.MediaTrackReceiver.ClearReceiver(mimeType, false) if t.MediaTrackReceiver.TryClose() { if t.dynacastManager != nil { @@ -312,15 +429,61 @@ func (t *MediaTrack) AddReceiver(receiver *webrtc.RTPReceiver, track sfu.TrackRe } } }) - // SIMULCAST-CODEC-TODO: these need to be receiver/mime aware, setting it up only for primary now - if priority == 0 { - newWR.OnStatsUpdate(func(_ *sfu.WebRTCReceiver, stat *livekit.AnalyticsStat) { - key := telemetry.StatsKeyForTrack(livekit.StreamType_UPSTREAM, t.PublisherID(), t.ID(), ti.Source, ti.Type) - t.params.Telemetry.TrackStats(key, stat) - }) - newWR.OnMaxLayerChange(t.onMaxLayerChange) + // SIMULCAST-CODEC-TODO: these need to be receiver/mime aware, setting it up only for primary now + statsKey := telemetry.StatsKeyForTrack( + t.params.ParticipantCountry, + livekit.StreamType_UPSTREAM, + t.PublisherID(), + t.ID(), + ti.Source, + ti.Type, + ) + for _, c := range ti.Codecs { + for _, l := range c.Layers { + t.params.Reporter.ReportLayer(roomobs.PackTrackLayer(l.Height, l.Width)) + } } + newWR.OnStatsUpdate(func(_ *sfu.WebRTCReceiver, stat *livekit.AnalyticsStat) { + // send for only one codec, either primary (priority == 0) OR regressed codec + t.lock.RLock() + regressionTargetCodecReceived := t.regressionTargetCodecReceived + t.lock.RUnlock() + if priority == 0 || regressionTargetCodecReceived { + t.params.TelemetryListener.OnTrackStats(statsKey, stat) + + if cs, ok := telemetry.CondenseStat(stat); ok { + t.params.Reporter.Tx(func(tx roomobs.TrackTx) { + tx.ParticipantSession().ReportKind(t.params.ParticipantKind.String()) + tx.ParticipantSession().ReportKindCode(roomobs.ParticipantKindCode(t.params.ParticipantKind)) + tx.ParticipantSession().ReportKindDetailsCodes(roomobs.ParticipantKindDetailsCodes(t.params.ParticipantKindDetails)) + tx.ReportName(ti.Name) + tx.ReportKind(roomobs.TrackKindPub) + tx.ReportType(roomobs.TrackTypeFromProto(ti.Type)) + tx.ReportSource(roomobs.TrackSourceFromProto(ti.Source)) + tx.ReportMime(mime.NormalizeMimeType(ti.MimeType).ReporterType()) + tx.ReportDuration(uint16(cs.EndTime.Sub(cs.StartTime).Milliseconds())) + tx.ReportFrames(uint16(cs.Frames)) + tx.ReportRecvBytes(uint32(cs.Bytes)) + tx.ReportRecvPackets(cs.Packets) + tx.ReportPacketsLost(cs.PacketsLost) + tx.ReportScore(stat.Score) + }) + } + } + }) + + newWR.OnMaxLayerChange(func(mimeType mime.MimeType, maxLayer int32) { + // send for only one codec, either primary (priority == 0) OR regressed codec + t.lock.RLock() + regressionTargetCodecReceived := t.regressionTargetCodecReceived + t.lock.RUnlock() + if priority == 0 || regressionTargetCodecReceived { + t.MediaTrackReceiver.NotifyMaxLayerChange(mimeType, maxLayer) + } + }) + // SIMULCAST-CODEC-TODO END: these need to be receiver/mime aware, setting it up only for primary now + if t.PrimaryReceiver() == nil { // primary codec published, set potential codecs potentialCodecs := make([]webrtc.RTPCodecParameters, 0, len(ti.Codecs)) @@ -345,8 +508,8 @@ func (t *MediaTrack) AddReceiver(receiver *webrtc.RTPReceiver, track sfu.TrackRe t.MediaTrackReceiver.SetupReceiver(newWR, priority, mid) for ssrc, info := range t.params.SimTracks { - if info.Mid == mid { - t.MediaTrackReceiver.SetLayerSsrc(mimeType, info.Rid, ssrc) + if info.Mid == mid && !info.IsRepairStream { + t.MediaTrackReceiver.SetLayerSsrcsForRid(mimeType, info.StreamID, ssrc, info.RepairSSRC) } } wr = newWR @@ -355,9 +518,18 @@ func (t *MediaTrack) AddReceiver(receiver *webrtc.RTPReceiver, track sfu.TrackRe newWR.AddOnCodecStateChange(func(codec webrtc.RTPCodecParameters, state sfu.ReceiverCodecState) { t.MediaTrackReceiver.HandleReceiverCodecChange(newWR, codec, state) }) + + // update subscriber video layers when video size changes + newWR.OnVideoSizeChanged(func() { + if t.params.UpdateTrackInfoByVideoSizeChange { + t.MediaTrackReceiver.UpdateVideoSize(mimeType, newWR.VideoSizes()) + } + + t.MediaTrackSubscriptions.UpdateVideoLayers() + }) } - if newCodec && t.enableRegression { + if newCodec && t.enableRegression() { if mimeType == t.regressionTargetCodec { t.params.Logger.Infow("regression target codec received", "codec", mimeType) t.regressionTargetCodecReceived = true @@ -377,21 +549,28 @@ func (t *MediaTrack) AddReceiver(receiver *webrtc.RTPReceiver, track sfu.TrackRe "newCodec", newCodec, ) buff.Close() - return false + return newCodec, false } - // LK-TODO: can remove this completely when VideoLayers protocol becomes the default as it has info from client or if we decide to use TrackInfo.Simulcast - if t.numUpTracks.Inc() > 1 || track.RID() != "" { - // cannot only rely on numUpTracks since we fire metadata events immediately after the first layer - t.SetSimulcast(true) + var expectedBitrate int + layers := buffer.GetVideoLayersForMimeType(mimeType, ti) + if layer >= 0 && len(layers) > int(layer) { + expectedBitrate = int(layers[layer].GetBitrate()) + } + if err := buff.Bind(receiver.GetParameters(), track.Codec().RTPCodecCapability, expectedBitrate); err != nil { + t.params.Logger.Warnw( + "binding buffer failed", err, + "rid", track.RID(), + "layer", layer, + "ssrc", track.SSRC(), + "newCodec", newCodec, + ) + buff.Close() + return newCodec, false } - var bitrates int - if len(ti.Layers) > int(layer) { - bitrates = int(ti.Layers[layer].GetBitrate()) - } - - t.MediaTrackReceiver.SetLayerSsrc(mimeType, track.RID(), uint32(track.SSRC())) + t.MediaTrackReceiver.MaybeSetSimulcast() + t.MediaTrackReceiver.SetLayerSsrcsForRid(mimeType, track.RID(), uint32(track.SSRC()), 0) if regressCodec { for _, c := range ti.Codecs { @@ -408,7 +587,7 @@ func (t *MediaTrack) AddReceiver(receiver *webrtc.RTPReceiver, track sfu.TrackRe } } - buff.Bind(receiver.GetParameters(), track.Codec().RTPCodecCapability, bitrates) + buff.OnNotifyRTX(t.MediaTrackReceiver.setLayerRtxInfo) // if subscriber request fps before fps calculated, update them after fps updated. buff.OnFpsChanged(func() { @@ -416,20 +595,19 @@ func (t *MediaTrack) AddReceiver(receiver *webrtc.RTPReceiver, track sfu.TrackRe }) buff.OnFinalRtpStats(func(stats *livekit.RTPStats) { - t.params.Telemetry.TrackPublishRTPStats( - context.Background(), - t.params.ParticipantID, + t.params.TelemetryListener.OnTrackPublishRTPStats( + t.params.ParticipantID(), t.ID(), mimeType, int(layer), stats, ) }) - return newCodec + return newCodec, true } func (t *MediaTrack) GetConnectionScoreAndQuality() (float32, livekit.ConnectionQuality) { - receiver := t.PrimaryReceiver() + receiver := t.ActiveReceiver() if rtcReceiver, ok := receiver.(*sfu.WebRTCReceiver); ok { return rtcReceiver.GetConnectionScoreAndQuality() } @@ -447,10 +625,6 @@ func (t *MediaTrack) HasPendingCodec() bool { return t.MediaTrackReceiver.PrimaryReceiver() == nil } -func (t *MediaTrack) onMaxLayerChange(maxLayer int32) { - t.MediaTrackReceiver.NotifyMaxLayerChange(maxLayer) -} - func (t *MediaTrack) Restart() { t.MediaTrackReceiver.Restart() @@ -460,11 +634,10 @@ func (t *MediaTrack) Restart() { } func (t *MediaTrack) Close(isExpectedToResume bool) { - t.MediaTrackReceiver.SetClosing() + t.MediaTrackReceiver.SetClosing(isExpectedToResume) if t.dynacastManager != nil { t.dynacastManager.Close() } - t.MediaTrackReceiver.ClearAllReceivers(isExpectedToResume) t.MediaTrackReceiver.Close(isExpectedToResume) } @@ -486,3 +659,56 @@ func (t *MediaTrack) OnTrackSubscribed() { go t.params.OnTrackEverSubscribed(t.ID()) } } + +func (t *MediaTrack) enableRegression() bool { + return t.backupCodecPolicy == livekit.BackupCodecPolicy_REGRESSION || + (t.backupCodecPolicy == livekit.BackupCodecPolicy_PREFER_REGRESSION && t.params.ShouldRegressCodec()) +} + +func (t *MediaTrack) Logger() logger.Logger { + return t.params.Logger +} + +// dynacast.DynacastManagerListtener implementation +var _ dynacast.DynacastManagerListener = (*MediaTrack)(nil) + +func (t *MediaTrack) OnDynacastSubscribedMaxQualityChange( + subscribedQualities []*livekit.SubscribedCodec, + maxSubscribedQualities []types.SubscribedCodecQuality, +) { + t.lock.RLock() + onSubscribedMaxQualityChange := t.onSubscribedMaxQualityChange + t.lock.RUnlock() + + if onSubscribedMaxQualityChange != nil && !t.IsMuted() { + _ = onSubscribedMaxQualityChange( + t.ID(), + t.ToProto(), + subscribedQualities, + maxSubscribedQualities, + ) + } + + for _, q := range maxSubscribedQualities { + receiver := t.Receiver(q.CodecMime) + if receiver != nil { + receiver.SetMaxExpectedSpatialLayer( + buffer.GetSpatialLayerForVideoQuality( + q.CodecMime, + q.Quality, + t.MediaTrackReceiver.TrackInfo(), + ), + ) + } + } +} + +func (t *MediaTrack) OnDynacastSubscribedAudioCodecChange(codecs []*livekit.SubscribedAudioCodec) { + t.lock.RLock() + onSubscribedAudioCodecChange := t.onSubscribedAudioCodecChange + t.lock.RUnlock() + + if onSubscribedAudioCodecChange != nil { + _ = onSubscribedAudioCodecChange(t.ID(), codecs) + } +} diff --git a/pkg/rtc/mediatrack_test.go b/pkg/rtc/mediatrack_test.go index 8d96bba..bd3b09d 100644 --- a/pkg/rtc/mediatrack_test.go +++ b/pkg/rtc/mediatrack_test.go @@ -19,7 +19,9 @@ import ( "github.com/stretchr/testify/require" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/logger" ) func TestTrackInfo(t *testing.T) { @@ -47,128 +49,149 @@ func TestTrackInfo(t *testing.T) { require.Equal(t, ti.Width, outInfo.Width) require.Equal(t, ti.Height, outInfo.Height) require.Equal(t, ti.Simulcast, outInfo.Simulcast) - - // make it simulcasted - mt.SetSimulcast(true) - require.True(t, mt.ToProto().Simulcast) } func TestGetQualityForDimension(t *testing.T) { t.Run("landscape source", func(t *testing.T) { - mt := NewMediaTrack(MediaTrackParams{}, &livekit.TrackInfo{ + mt := NewMediaTrack(MediaTrackParams{ + Logger: logger.GetLogger(), + }, &livekit.TrackInfo{ Type: livekit.TrackType_VIDEO, Width: 1080, Height: 720, }) - require.Equal(t, livekit.VideoQuality_LOW, mt.GetQualityForDimension(120, 120)) - require.Equal(t, livekit.VideoQuality_LOW, mt.GetQualityForDimension(300, 200)) - require.Equal(t, livekit.VideoQuality_MEDIUM, mt.GetQualityForDimension(200, 250)) - require.Equal(t, livekit.VideoQuality_HIGH, mt.GetQualityForDimension(700, 480)) - require.Equal(t, livekit.VideoQuality_HIGH, mt.GetQualityForDimension(500, 1000)) + require.Equal(t, livekit.VideoQuality_LOW, mt.GetQualityForDimension(mime.MimeTypeVP8, 120, 120)) + require.Equal(t, livekit.VideoQuality_LOW, mt.GetQualityForDimension(mime.MimeTypeVP8, 300, 200)) + require.Equal(t, livekit.VideoQuality_MEDIUM, mt.GetQualityForDimension(mime.MimeTypeVP8, 200, 250)) + require.Equal(t, livekit.VideoQuality_HIGH, mt.GetQualityForDimension(mime.MimeTypeVP8, 700, 480)) + require.Equal(t, livekit.VideoQuality_HIGH, mt.GetQualityForDimension(mime.MimeTypeVP8, 500, 1000)) }) t.Run("portrait source", func(t *testing.T) { - mt := NewMediaTrack(MediaTrackParams{}, &livekit.TrackInfo{ + mt := NewMediaTrack(MediaTrackParams{ + Logger: logger.GetLogger(), + }, &livekit.TrackInfo{ Type: livekit.TrackType_VIDEO, Width: 540, Height: 960, }) - require.Equal(t, livekit.VideoQuality_LOW, mt.GetQualityForDimension(200, 400)) - require.Equal(t, livekit.VideoQuality_MEDIUM, mt.GetQualityForDimension(400, 400)) - require.Equal(t, livekit.VideoQuality_MEDIUM, mt.GetQualityForDimension(400, 700)) - require.Equal(t, livekit.VideoQuality_HIGH, mt.GetQualityForDimension(600, 900)) + require.Equal(t, livekit.VideoQuality_LOW, mt.GetQualityForDimension(mime.MimeTypeVP8, 200, 400)) + require.Equal(t, livekit.VideoQuality_MEDIUM, mt.GetQualityForDimension(mime.MimeTypeVP8, 400, 400)) + require.Equal(t, livekit.VideoQuality_MEDIUM, mt.GetQualityForDimension(mime.MimeTypeVP8, 400, 700)) + require.Equal(t, livekit.VideoQuality_HIGH, mt.GetQualityForDimension(mime.MimeTypeVP8, 600, 900)) }) t.Run("layers provided", func(t *testing.T) { - mt := NewMediaTrack(MediaTrackParams{}, &livekit.TrackInfo{ + mt := NewMediaTrack(MediaTrackParams{ + Logger: logger.GetLogger(), + }, &livekit.TrackInfo{ Type: livekit.TrackType_VIDEO, Width: 1080, Height: 720, - Layers: []*livekit.VideoLayer{ + Codecs: []*livekit.SimulcastCodecInfo{ { - Quality: livekit.VideoQuality_LOW, - Width: 480, - Height: 270, - }, - { - Quality: livekit.VideoQuality_MEDIUM, - Width: 960, - Height: 540, - }, - { - Quality: livekit.VideoQuality_HIGH, - Width: 1080, - Height: 720, + MimeType: mime.MimeTypeH264.String(), + Layers: []*livekit.VideoLayer{ + { + Quality: livekit.VideoQuality_LOW, + Width: 480, + Height: 270, + }, + { + Quality: livekit.VideoQuality_MEDIUM, + Width: 960, + Height: 540, + }, + { + Quality: livekit.VideoQuality_HIGH, + Width: 1080, + Height: 720, + }, + }, }, }, }) - require.Equal(t, livekit.VideoQuality_LOW, mt.GetQualityForDimension(120, 120)) - require.Equal(t, livekit.VideoQuality_LOW, mt.GetQualityForDimension(300, 300)) - require.Equal(t, livekit.VideoQuality_MEDIUM, mt.GetQualityForDimension(800, 500)) - require.Equal(t, livekit.VideoQuality_HIGH, mt.GetQualityForDimension(1000, 700)) + require.Equal(t, livekit.VideoQuality_LOW, mt.GetQualityForDimension(mime.MimeTypeH264, 120, 120)) + require.Equal(t, livekit.VideoQuality_LOW, mt.GetQualityForDimension(mime.MimeTypeH264, 300, 300)) + require.Equal(t, livekit.VideoQuality_MEDIUM, mt.GetQualityForDimension(mime.MimeTypeH264, 800, 500)) + require.Equal(t, livekit.VideoQuality_HIGH, mt.GetQualityForDimension(mime.MimeTypeH264, 1000, 700)) }) t.Run("highest layer with smallest dimensions", func(t *testing.T) { - mt := NewMediaTrack(MediaTrackParams{}, &livekit.TrackInfo{ + mt := NewMediaTrack(MediaTrackParams{ + Logger: logger.GetLogger(), + }, &livekit.TrackInfo{ Type: livekit.TrackType_VIDEO, Width: 1080, Height: 720, - Layers: []*livekit.VideoLayer{ + Codecs: []*livekit.SimulcastCodecInfo{ { - Quality: livekit.VideoQuality_LOW, - Width: 480, - Height: 270, - }, - { - Quality: livekit.VideoQuality_MEDIUM, - Width: 1080, - Height: 720, - }, - { - Quality: livekit.VideoQuality_HIGH, - Width: 1080, - Height: 720, + MimeType: mime.MimeTypeH264.String(), + Layers: []*livekit.VideoLayer{ + { + Quality: livekit.VideoQuality_LOW, + Width: 480, + Height: 270, + }, + { + Quality: livekit.VideoQuality_MEDIUM, + Width: 1080, + Height: 720, + }, + { + Quality: livekit.VideoQuality_HIGH, + Width: 1080, + Height: 720, + }, + }, }, }, }) - require.Equal(t, livekit.VideoQuality_LOW, mt.GetQualityForDimension(120, 120)) - require.Equal(t, livekit.VideoQuality_LOW, mt.GetQualityForDimension(300, 300)) - require.Equal(t, livekit.VideoQuality_HIGH, mt.GetQualityForDimension(800, 500)) - require.Equal(t, livekit.VideoQuality_HIGH, mt.GetQualityForDimension(1000, 700)) - require.Equal(t, livekit.VideoQuality_HIGH, mt.GetQualityForDimension(1200, 800)) + require.Equal(t, livekit.VideoQuality_LOW, mt.GetQualityForDimension(mime.MimeTypeH264, 120, 120)) + require.Equal(t, livekit.VideoQuality_LOW, mt.GetQualityForDimension(mime.MimeTypeH264, 300, 300)) + require.Equal(t, livekit.VideoQuality_HIGH, mt.GetQualityForDimension(mime.MimeTypeH264, 800, 500)) + require.Equal(t, livekit.VideoQuality_HIGH, mt.GetQualityForDimension(mime.MimeTypeH264, 1000, 700)) + require.Equal(t, livekit.VideoQuality_HIGH, mt.GetQualityForDimension(mime.MimeTypeH264, 1200, 800)) - mt = NewMediaTrack(MediaTrackParams{}, &livekit.TrackInfo{ + mt = NewMediaTrack(MediaTrackParams{ + Logger: logger.GetLogger(), + }, &livekit.TrackInfo{ Type: livekit.TrackType_VIDEO, Width: 1080, Height: 720, - Layers: []*livekit.VideoLayer{ + Codecs: []*livekit.SimulcastCodecInfo{ { - Quality: livekit.VideoQuality_LOW, - Width: 480, - Height: 270, - }, - { - Quality: livekit.VideoQuality_MEDIUM, - Width: 480, - Height: 270, - }, - { - Quality: livekit.VideoQuality_HIGH, - Width: 1080, - Height: 720, + MimeType: mime.MimeTypeH264.String(), + Layers: []*livekit.VideoLayer{ + { + Quality: livekit.VideoQuality_LOW, + Width: 480, + Height: 270, + }, + { + Quality: livekit.VideoQuality_MEDIUM, + Width: 480, + Height: 270, + }, + { + Quality: livekit.VideoQuality_HIGH, + Width: 1080, + Height: 720, + }, + }, }, }, }) - require.Equal(t, livekit.VideoQuality_MEDIUM, mt.GetQualityForDimension(120, 120)) - require.Equal(t, livekit.VideoQuality_MEDIUM, mt.GetQualityForDimension(300, 300)) - require.Equal(t, livekit.VideoQuality_HIGH, mt.GetQualityForDimension(800, 500)) - require.Equal(t, livekit.VideoQuality_HIGH, mt.GetQualityForDimension(1000, 700)) - require.Equal(t, livekit.VideoQuality_HIGH, mt.GetQualityForDimension(1200, 800)) + require.Equal(t, livekit.VideoQuality_MEDIUM, mt.GetQualityForDimension(mime.MimeTypeH264, 120, 120)) + require.Equal(t, livekit.VideoQuality_MEDIUM, mt.GetQualityForDimension(mime.MimeTypeH264, 300, 300)) + require.Equal(t, livekit.VideoQuality_HIGH, mt.GetQualityForDimension(mime.MimeTypeH264, 800, 500)) + require.Equal(t, livekit.VideoQuality_HIGH, mt.GetQualityForDimension(mime.MimeTypeH264, 1000, 700)) + require.Equal(t, livekit.VideoQuality_HIGH, mt.GetQualityForDimension(mime.MimeTypeH264, 1200, 800)) }) } diff --git a/pkg/rtc/mediatrackreceiver.go b/pkg/rtc/mediatrackreceiver.go index 8b62900..cffde83 100644 --- a/pkg/rtc/mediatrackreceiver.go +++ b/pkg/rtc/mediatrackreceiver.go @@ -15,19 +15,19 @@ package rtc import ( - "context" "errors" "fmt" - "sort" + "slices" "strings" "sync" "github.com/pion/rtcp" "github.com/pion/webrtc/v4" "go.uber.org/atomic" - "golang.org/x/exp/slices" "google.golang.org/protobuf/proto" + "github.com/livekit/mediatransportutil/pkg/codec" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" "github.com/livekit/protocol/utils" @@ -35,9 +35,8 @@ import ( "github.com/livekit/livekit-server/pkg/rtc/types" "github.com/livekit/livekit-server/pkg/sfu" "github.com/livekit/livekit-server/pkg/sfu/buffer" - "github.com/livekit/livekit-server/pkg/sfu/mime" "github.com/livekit/livekit-server/pkg/sfu/rtpstats" - "github.com/livekit/livekit-server/pkg/telemetry" + sutils "github.com/livekit/livekit-server/pkg/utils" ) const ( @@ -118,17 +117,18 @@ func (r *simulcastReceiver) IsRegressed() bool { // ----------------------------------------------------- type MediaTrackReceiverParams struct { - MediaTrack types.MediaTrack - IsRelayed bool - ParticipantID livekit.ParticipantID - ParticipantIdentity livekit.ParticipantIdentity - ParticipantVersion uint32 - ReceiverConfig ReceiverConfig - SubscriberConfig DirectionConfig - AudioConfig sfu.AudioConfig - Telemetry telemetry.TelemetryService - Logger logger.Logger - RegressionTargetCodec mime.MimeType + MediaTrack types.MediaTrack + IsRelayed bool + ParticipantID func() livekit.ParticipantID + ParticipantIdentity livekit.ParticipantIdentity + ParticipantVersion uint32 + ReceiverConfig ReceiverConfig + SubscriberConfig DirectionConfig + AudioConfig sfu.AudioConfig + TelemetryListener types.ParticipantTelemetryListener + Logger logger.Logger + RegressionTargetCodec mime.MimeType + PreferVideoSizeFromMedia bool } type MediaTrackReceiver struct { @@ -161,21 +161,15 @@ func NewMediaTrackReceiver(params MediaTrackReceiverParams, ti *livekit.TrackInf IsRelayed: params.IsRelayed, ReceiverConfig: params.ReceiverConfig, SubscriberConfig: params.SubscriberConfig, - Telemetry: params.Telemetry, Logger: params.Logger, }) t.MediaTrackSubscriptions.OnDownTrackCreated(t.onDownTrackCreated) - - if ti.Muted { - t.SetMuted(true) - } return t } func (t *MediaTrackReceiver) Restart() { - hq := buffer.VideoQualityToSpatialLayer(livekit.VideoQuality_HIGH, t.TrackInfo()) - for _, receiver := range t.loadReceivers() { + hq := buffer.GetSpatialLayerForVideoQuality(receiver.Mime(), livekit.VideoQuality_HIGH, t.TrackInfo()) receiver.SetMaxExpectedSpatialLayer(hq) } } @@ -213,8 +207,8 @@ func (t *MediaTrackReceiver) SetupReceiver(receiver sfu.TrackReceiver, priority receivers = append(receivers, &simulcastReceiver{TrackReceiver: receiver, priority: priority}) } - sort.Slice(receivers, func(i, j int) bool { - return receivers[i].Priority() < receivers[j].Priority() + slices.SortFunc(receivers, func(a, b *simulcastReceiver) int { + return sutils.Signum(a.Priority() - b.Priority()) }) if mid != "" { @@ -308,12 +302,16 @@ func (t *MediaTrackReceiver) HandleReceiverCodecChange(r sfu.TrackReceiver, code return } - t.params.Logger.Infow("regressing codec", "from", codec.MimeType, "to", backupCodecReceiver.Codec().MimeType) + t.params.Logger.Infow( + "regressing codec", + "from", codec.MimeType, + "to", backupCodecReceiver.Codec().MimeType, + ) // remove old codec from potential codecs for i, c := range t.potentialCodecs { if strings.EqualFold(c.MimeType, codec.MimeType) { - slices.Delete(t.potentialCodecs, i, i+1) + t.potentialCodecs = slices.Delete(t.potentialCodecs, i, i+1) break } } @@ -351,8 +349,8 @@ func (t *MediaTrackReceiver) SetPotentialCodecs(codecs []webrtc.RTPCodecParamete }) } } - sort.Slice(receivers, func(i, j int) bool { - return receivers[i].Priority() < receivers[j].Priority() + slices.SortFunc(receivers, func(a, b *simulcastReceiver) int { + return sutils.Signum(a.Priority() - b.Priority()) }) t.receivers = receivers t.lock.Unlock() @@ -376,7 +374,7 @@ func (t *MediaTrackReceiver) ClearReceiver(mime mime.MimeType, isExpectedToResum } func (t *MediaTrackReceiver) ClearAllReceivers(isExpectedToResume bool) { - t.params.Logger.Debugw("clearing all receivers") + t.params.Logger.Debugw("clearing all receivers", "isExpectedToResume", isExpectedToResume) t.lock.Lock() receivers := t.receivers t.receivers = nil @@ -408,12 +406,15 @@ func (t *MediaTrackReceiver) IsOpen() bool { return true } -func (t *MediaTrackReceiver) SetClosing() { +func (t *MediaTrackReceiver) SetClosing(isExpectedToResume bool) { t.lock.Lock() defer t.lock.Unlock() + if t.state == mediaTrackReceiverStateOpen { t.state = mediaTrackReceiverStateClosing } + + t.isExpectedToResume = isExpectedToResume } func (t *MediaTrackReceiver) TryClose() bool { @@ -444,6 +445,8 @@ func (t *MediaTrackReceiver) TryClose() bool { } func (t *MediaTrackReceiver) Close(isExpectedToResume bool) { + t.ClearAllReceivers(isExpectedToResume) + t.lock.Lock() if t.state == mediaTrackReceiverStateClosed { t.lock.Unlock() @@ -476,7 +479,7 @@ func (t *MediaTrackReceiver) Stream() string { } func (t *MediaTrackReceiver) PublisherID() livekit.ParticipantID { - return t.params.ParticipantID + return t.params.ParticipantID() } func (t *MediaTrackReceiver) PublisherIdentity() livekit.ParticipantIdentity { @@ -487,19 +490,6 @@ func (t *MediaTrackReceiver) PublisherVersion() uint32 { return t.params.ParticipantVersion } -func (t *MediaTrackReceiver) IsSimulcast() bool { - return t.TrackInfo().Simulcast -} - -func (t *MediaTrackReceiver) SetSimulcast(simulcast bool) { - t.lock.Lock() - defer t.lock.Unlock() - - trackInfo := t.TrackInfoClone() - trackInfo.Simulcast = simulcast - t.trackInfo.Store(trackInfo) -} - func (t *MediaTrackReceiver) Name() string { return t.TrackInfo().Name } @@ -513,21 +503,19 @@ func (t *MediaTrackReceiver) SetMuted(muted bool) { trackInfo := t.TrackInfoClone() trackInfo.Muted = muted t.trackInfo.Store(trackInfo) - - receivers := t.receivers t.lock.Unlock() - for _, receiver := range receivers { - receiver.SetUpTrackPaused(muted) - } - - t.MediaTrackSubscriptions.SetMuted(muted) + t.updateTrackInfoOfReceivers() } func (t *MediaTrackReceiver) IsEncrypted() bool { return t.TrackInfo().Encryption != livekit.Encryption_NONE } +func (t *MediaTrackReceiver) HasPacketTrailer() bool { + return len(t.TrackInfo().GetPacketTrailerFeatures()) > 0 +} + func (t *MediaTrackReceiver) AddOnClose(f func(isExpectedToResume bool)) { if f == nil { return @@ -588,8 +576,10 @@ func (t *MediaTrackReceiver) AddSubscriber(sub types.LocalParticipant) (types.Su StreamId: streamId, UpstreamCodecs: potentialCodecs, Logger: tLogger, - DisableRed: t.TrackInfo().GetDisableRed() || !t.params.AudioConfig.ActiveREDEncoding, + DisableRed: !IsRedEnabled(t.TrackInfo()) || !t.params.AudioConfig.ActiveREDEncoding, + IsEncrypted: t.IsEncrypted(), }) + subID := sub.ID() subTrack, err := t.MediaTrackSubscriptions.AddSubscriber(sub, wr) // media track could have been closed while adding subscription @@ -603,7 +593,12 @@ func (t *MediaTrackReceiver) AddSubscriber(sub types.LocalParticipant) (types.Su t.lock.RUnlock() if remove { - _ = t.MediaTrackSubscriptions.RemoveSubscriber(sub.ID(), isExpectedToResume) + t.params.Logger.Debugw( + "removing subscriber on a not-open track", + "subscriberID", subID, + "isExpectedToResume", isExpectedToResume, + ) + _ = t.MediaTrackSubscriptions.RemoveSubscriber(subID, isExpectedToResume) return nil, ErrNotOpen } @@ -617,7 +612,7 @@ func (t *MediaTrackReceiver) RemoveSubscriber(subscriberID livekit.ParticipantID } func (t *MediaTrackReceiver) removeAllSubscribersForMime(mime mime.MimeType, isExpectedToResume bool) { - t.params.Logger.Debugw("removing all subscribers for mime", "mime", mime) + t.params.Logger.Debugw("removing all subscribers for mime", "mime", mime, "isExpectedToResume", isExpectedToResume) for _, subscriberID := range t.MediaTrackSubscriptions.GetAllSubscribersForMime(mime) { t.RemoveSubscriber(subscriberID, isExpectedToResume) } @@ -632,14 +627,7 @@ func (t *MediaTrackReceiver) RevokeDisallowedSubscribers(allowedSubscriberIdenti continue } - found := false - for _, allowedIdentity := range allowedSubscriberIdentities { - if subTrack.SubscriberIdentity() == allowedIdentity { - found = true - break - } - } - + found := slices.Contains(allowedSubscriberIdentities, subTrack.SubscriberIdentity()) if !found { t.params.Logger.Infow("revoking subscription", "subscriber", subTrack.SubscriberIdentity(), @@ -658,17 +646,42 @@ func (t *MediaTrackReceiver) updateTrackInfoOfReceivers() { for _, r := range t.loadReceivers() { r.UpdateTrackInfo(ti) } + + t.MediaTrackSubscriptions.SetMuted(ti.GetMuted()) } -func (t *MediaTrackReceiver) SetLayerSsrc(mimeType mime.MimeType, rid string, ssrc uint32) { +func (t *MediaTrackReceiver) MaybeSetSimulcast() { + // only primary receiver (i.e. receiver at index 0) for legacy use case + primaryReceiver := t.PrimaryReceiver() + if primaryReceiver == nil { + return + } + if wr, ok := primaryReceiver.(*sfu.WebRTCReceiver); !ok || wr.NumUpTracks() < 2 { + return + } + t.lock.Lock() trackInfo := t.TrackInfoClone() - layer := buffer.RidToSpatialLayer(rid, trackInfo) + if trackInfo.Simulcast { + t.lock.Unlock() + return + } + trackInfo.Simulcast = true + t.trackInfo.Store(trackInfo) + t.lock.Unlock() + + t.updateTrackInfoOfReceivers() +} + +func (t *MediaTrackReceiver) SetLayerSsrcsForRid(mimeType mime.MimeType, rid string, ssrc uint32, repairSSRC uint32) { + t.lock.Lock() + trackInfo := t.TrackInfoClone() + layer := buffer.GetSpatialLayerForRid(mimeType, rid, trackInfo) if layer == buffer.InvalidLayerSpatial { // non-simulcast case will not have `rid` layer = 0 } - quality := buffer.SpatialLayerToVideoQuality(layer, trackInfo) + quality := buffer.GetVideoQualityForSpatialLayer(mimeType, layer, trackInfo) // set video layer ssrc info for i, ci := range trackInfo.Codecs { if mime.NormalizeMimeType(ci.MimeType) != mimeType { @@ -689,6 +702,20 @@ func (t *MediaTrackReceiver) SetLayerSsrc(mimeType mime.MimeType, rid string, ss } if !ssrcFound && matchingLayer != nil { matchingLayer.Ssrc = ssrc + if repairSSRC != 0 { + matchingLayer.RepairSsrc = repairSSRC + } + } + if ssrcFound && (matchingLayer.Ssrc != ssrc || matchingLayer.RepairSsrc != repairSSRC) { + t.params.Logger.Warnw( + "not overriding ssrc", nil, + "rid", rid, + "ssrc", ssrc, + "existingSSRC", matchingLayer.Ssrc, + "repairSSRC", repairSSRC, + "existingRepairSSRC", matchingLayer.RepairSsrc, + "trackInfo", logger.Proto(trackInfo), + ) } // for client don't use simulcast codecs (old client version or single codec) @@ -703,13 +730,99 @@ func (t *MediaTrackReceiver) SetLayerSsrc(mimeType mime.MimeType, rid string, ss t.updateTrackInfoOfReceivers() } -func (t *MediaTrackReceiver) UpdateCodecCid(codecs []*livekit.SimulcastCodec) { +func (t *MediaTrackReceiver) setLayerRtxInfo(ssrc uint32, repairSSRC uint32, rsid string) { + t.params.Logger.Debugw("rtx notification", "ssrc", ssrc, "repairSSRC", repairSSRC, "rsid", rsid) + if ssrc == 0 || repairSSRC == 0 || rsid == "" { + return + } + + t.lock.Lock() + trackInfo := t.TrackInfoClone() + +done: + for _, ci := range trackInfo.Codecs { + for _, l := range ci.Layers { + if l.Ssrc == ssrc { + if (l.RepairSsrc != 0 && l.RepairSsrc != repairSSRC) || (l.Rid != "" && l.Rid != rsid) { + t.params.Logger.Warnw( + "not overriding rtx info", nil, + "ssrc", ssrc, + "repairSSRC", repairSSRC, + "existingRepairSSRC", l.RepairSsrc, + "rsid", rsid, + "existingRid", l.Rid, + "trackInfo", logger.Proto(trackInfo), + ) + } else { + l.RepairSsrc = repairSSRC + t.params.Logger.Debugw( + "set rtx info", + "ssrc", ssrc, + "repairSSRC", repairSSRC, + "rsid", rsid, + "trackInfo", logger.Proto(trackInfo), + ) + } + break done + } + } + } + + // backwards compatibility + for _, l := range trackInfo.Layers { + if l.Ssrc == ssrc { + if (l.RepairSsrc != 0 && l.RepairSsrc != repairSSRC) || (l.Rid != "" && l.Rid != rsid) { + t.params.Logger.Warnw( + "not overriding rtx info", nil, + "ssrc", ssrc, + "repairSSRC", repairSSRC, + "existingRepairSSRC", l.RepairSsrc, + "rsid", rsid, + "existingRid", l.Rid, + "trackInfo", logger.Proto(trackInfo), + ) + } else { + l.RepairSsrc = repairSSRC + t.params.Logger.Debugw( + "set rtx info", + "ssrc", ssrc, + "repairSSRC", repairSSRC, + "rsid", rsid, + "trackInfo", logger.Proto(trackInfo), + ) + } + break + } + } + + t.trackInfo.Store(trackInfo) + t.lock.Unlock() + + // change not propagated as it is internal +} + +func (t *MediaTrackReceiver) UpdateCodecInfo(codecs []*livekit.SimulcastCodec) { t.lock.Lock() trackInfo := t.TrackInfoClone() for _, c := range codecs { for _, origin := range trackInfo.Codecs { if mime.GetMimeTypeCodec(origin.MimeType) == mime.NormalizeMimeTypeCodec(c.Codec) { origin.Cid = c.Cid + + if len(c.Layers) != 0 { + clonedLayers := make([]*livekit.VideoLayer, 0, len(c.Layers)) + for _, l := range c.Layers { + clonedLayers = append(clonedLayers, utils.CloneProto(l)) + } + origin.Layers = clonedLayers + + mimeType := mime.NormalizeMimeType(origin.MimeType) + for _, layer := range origin.Layers { + layer.SpatialLayer = buffer.VideoQualityToSpatialLayer(mimeType, layer.Quality, trackInfo) + layer.Rid = buffer.VideoQualityToRid(mimeType, layer.Quality, trackInfo, buffer.DefaultVideoLayersRid) + } + } + break } } @@ -720,13 +833,49 @@ func (t *MediaTrackReceiver) UpdateCodecCid(codecs []*livekit.SimulcastCodec) { t.updateTrackInfoOfReceivers() } +func (t *MediaTrackReceiver) UpdateCodecSdpCid(mimeType mime.MimeType, sdpCid string) { + t.lock.Lock() + trackInfo := t.TrackInfoClone() + for _, origin := range trackInfo.Codecs { + if mime.NormalizeMimeType(origin.MimeType) == mimeType { + if sdpCid != origin.Cid { + origin.SdpCid = sdpCid + } + } + } + t.trackInfo.Store(trackInfo) + t.lock.Unlock() + + t.updateTrackInfoOfReceivers() +} + +func (t *MediaTrackReceiver) UpdateCodecRids(mimeType mime.MimeType, rids buffer.VideoLayersRid) { + t.lock.Lock() + trackInfo := t.TrackInfoClone() + for _, origin := range trackInfo.Codecs { + originMimeType := mime.NormalizeMimeType(origin.MimeType) + if originMimeType != mimeType { + continue + } + + for _, layer := range origin.Layers { + layer.SpatialLayer = buffer.VideoQualityToSpatialLayer(mimeType, layer.Quality, trackInfo) + layer.Rid = buffer.VideoQualityToRid(mimeType, layer.Quality, trackInfo, rids) + } + break + } + t.trackInfo.Store(trackInfo) + t.lock.Unlock() + + t.updateTrackInfoOfReceivers() +} + func (t *MediaTrackReceiver) UpdateTrackInfo(ti *livekit.TrackInfo) { - updateMute := false clonedInfo := utils.CloneProto(ti) t.lock.Lock() trackInfo := t.TrackInfo() - // patch Mid and SSRC of codecs/layers by keeping original if available + // patch Mid/Rid and Ssrc/RtxSsrc of codecs/layers by keeping original if available for i, ci := range clonedInfo.Codecs { for _, originCi := range trackInfo.Codecs { if !mime.IsMimeTypeStringEqual(ci.MimeType, originCi.MimeType) { @@ -743,6 +892,13 @@ func (t *MediaTrackReceiver) UpdateTrackInfo(ti *livekit.TrackInfo) { if originLayer.Ssrc != 0 { layer.Ssrc = originLayer.Ssrc } + if originLayer.Rid != "" { + layer.Rid = originLayer.Rid + } + + if originLayer.RepairSsrc != 0 { + layer.RepairSsrc = originLayer.RepairSsrc + } break } } @@ -755,16 +911,9 @@ func (t *MediaTrackReceiver) UpdateTrackInfo(ti *livekit.TrackInfo) { clonedInfo.Layers = ci.Layers } } - if trackInfo.Muted != clonedInfo.Muted { - updateMute = true - } t.trackInfo.Store(clonedInfo) t.lock.Unlock() - if updateMute { - t.SetMuted(clonedInfo.Muted) - } - t.updateTrackInfoOfReceivers() } @@ -776,7 +925,9 @@ func (t *MediaTrackReceiver) UpdateAudioTrack(update *livekit.UpdateLocalAudioTr t.lock.Lock() trackInfo := t.TrackInfo() clonedInfo := utils.CloneProto(trackInfo) - clonedInfo.AudioFeatures = update.Features + + clonedInfo.AudioFeatures = sutils.DedupeSlice(update.Features) + clonedInfo.Stereo = false clonedInfo.DisableDtx = false for _, feature := range update.Features { @@ -787,6 +938,7 @@ func (t *MediaTrackReceiver) UpdateAudioTrack(update *livekit.UpdateLocalAudioTr clonedInfo.DisableDtx = true } } + if proto.Equal(trackInfo, clonedInfo) { t.lock.Unlock() return @@ -797,7 +949,7 @@ func (t *MediaTrackReceiver) UpdateAudioTrack(update *livekit.UpdateLocalAudioTr t.updateTrackInfoOfReceivers() - t.params.Telemetry.TrackPublishedUpdate(context.Background(), t.PublisherID(), clonedInfo) + t.params.TelemetryListener.OnTrackPublishedUpdate(t.PublisherID(), clonedInfo) t.params.Logger.Debugw("updated audio track", "before", logger.Proto(trackInfo), "after", logger.Proto(clonedInfo)) } @@ -821,10 +973,56 @@ func (t *MediaTrackReceiver) UpdateVideoTrack(update *livekit.UpdateLocalVideoTr t.updateTrackInfoOfReceivers() - t.params.Telemetry.TrackPublishedUpdate(context.Background(), t.PublisherID(), clonedInfo) + t.params.TelemetryListener.OnTrackPublishedUpdate(t.PublisherID(), clonedInfo) t.params.Logger.Debugw("updated video track", "before", logger.Proto(trackInfo), "after", logger.Proto(clonedInfo)) } +func (t *MediaTrackReceiver) UpdateVideoSize(mimeType mime.MimeType, sizes []codec.VideoSize) { + var changed bool + t.lock.Lock() + trackInfo := t.TrackInfo() + clonedInfo := utils.CloneProto(trackInfo) + var maxWidth, maxHeight uint32 + for _, size := range sizes { + if size.Width > maxWidth { + maxWidth = size.Width + maxHeight = size.Height + } + } + + if clonedInfo.Width != maxWidth || clonedInfo.Height != maxHeight { + clonedInfo.Width = maxWidth + clonedInfo.Height = maxHeight + changed = true + } + + for _, c := range clonedInfo.Codecs { + if mime.NormalizeMimeType(c.MimeType) == mimeType { + for i, l := range c.Layers { + if i < len(sizes) && (sizes[i].Width != 0 || sizes[i].Height != 0) && + (l.Width != sizes[i].Width || l.Height != sizes[i].Height) { + l.Width = sizes[i].Width + l.Height = sizes[i].Height + changed = true + } + } + } + } + + if !changed { + t.lock.Unlock() + return + } + + t.trackInfo.Store(clonedInfo) + t.lock.Unlock() + + t.updateTrackInfoOfReceivers() + + t.params.TelemetryListener.OnTrackPublishedUpdate(t.PublisherID(), clonedInfo) + t.params.Logger.Debugw("updated video sizes", "before", logger.Proto(trackInfo), "after", logger.Proto(clonedInfo)) +} + func (t *MediaTrackReceiver) TrackInfo() *livekit.TrackInfo { return t.trackInfo.Load() } @@ -833,16 +1031,17 @@ func (t *MediaTrackReceiver) TrackInfoClone() *livekit.TrackInfo { return utils.CloneProto(t.TrackInfo()) } -func (t *MediaTrackReceiver) NotifyMaxLayerChange(maxLayer int32) { +func (t *MediaTrackReceiver) NotifyMaxLayerChange(mimeType mime.MimeType, maxLayer int32) { trackInfo := t.TrackInfo() - quality := buffer.SpatialLayerToVideoQuality(maxLayer, trackInfo) + quality := buffer.GetVideoQualityForSpatialLayer(mimeType, maxLayer, trackInfo) ti := &livekit.TrackInfo{ Sid: trackInfo.Sid, Type: trackInfo.Type, Layers: []*livekit.VideoLayer{{Quality: quality}}, } if quality != livekit.VideoQuality_OFF { - for _, layer := range trackInfo.Layers { + layers := buffer.GetVideoLayersForMimeType(mimeType, trackInfo) + for _, layer := range layers { if layer.Quality == quality { ti.Layers[0].Width = layer.Width ti.Layers[0].Height = layer.Height @@ -851,12 +1050,12 @@ func (t *MediaTrackReceiver) NotifyMaxLayerChange(maxLayer int32) { } } - t.params.Telemetry.TrackPublishedUpdate(context.Background(), t.PublisherID(), ti) + t.params.TelemetryListener.OnTrackPublishedUpdate(t.PublisherID(), ti) } // GetQualityForDimension finds the closest quality to use for desired dimensions // affords a 20% tolerance on dimension -func (t *MediaTrackReceiver) GetQualityForDimension(width, height uint32) livekit.VideoQuality { +func (t *MediaTrackReceiver) GetQualityForDimension(mimeType mime.MimeType, width, height uint32) livekit.VideoQuality { quality := livekit.VideoQuality_HIGH if t.Kind() == livekit.TrackType_AUDIO { return quality @@ -864,7 +1063,12 @@ func (t *MediaTrackReceiver) GetQualityForDimension(width, height uint32) liveki trackInfo := t.TrackInfo() - if trackInfo.Height == 0 { + var mediaSizes []codec.VideoSize + if receiver := t.Receiver(mimeType); receiver != nil { + mediaSizes = receiver.VideoSizes() + } + + if trackInfo.Height == 0 && len(mediaSizes) == 0 { return quality } origSize := trackInfo.Height @@ -875,19 +1079,38 @@ func (t *MediaTrackReceiver) GetQualityForDimension(width, height uint32) liveki requestedSize = width } + if origSize == 0 { + for i := len(mediaSizes) - 1; i >= 0; i-- { + if mediaSizes[i].Height > 0 { + origSize = min(mediaSizes[i].Width, mediaSizes[i].Height) + break + } + } + } + // default sizes representing qualities low - high layerSizes := []uint32{180, 360, origSize} var providedSizes []uint32 - for _, layer := range trackInfo.Layers { + for _, layer := range buffer.GetVideoLayersForMimeType(mimeType, trackInfo) { providedSizes = append(providedSizes, layer.Height) } + + if len(providedSizes) == 0 || providedSizes[0] == 0 || t.params.PreferVideoSizeFromMedia { + if len(mediaSizes) > 0 { + providedSizes = providedSizes[:0] + for _, size := range mediaSizes { + providedSizes = append(providedSizes, size.Height) + } + } else { + t.params.Logger.Debugw("no video sizes provided by receiver, using track info sizes") + } + } + if len(providedSizes) > 0 { layerSizes = providedSizes // comparing height always requestedSize = height - sort.Slice(layerSizes, func(i, j int) bool { - return layerSizes[i] < layerSizes[j] - }) + slices.Sort(layerSizes) } // finds the highest layer with smallest dimensions that still satisfy client demands @@ -905,7 +1128,7 @@ func (t *MediaTrackReceiver) GetQualityForDimension(width, height uint32) liveki } func (t *MediaTrackReceiver) GetAudioLevel() (float64, bool) { - receiver := t.PrimaryReceiver() + receiver := t.ActiveReceiver() if receiver == nil { return 0, false } @@ -923,8 +1146,8 @@ func (t *MediaTrackReceiver) onDownTrackCreated(downTrack *sfu.DownTrack) { } } -func (t *MediaTrackReceiver) DebugInfo() map[string]interface{} { - info := map[string]interface{}{ +func (t *MediaTrackReceiver) DebugInfo() map[string]any { + info := map[string]any{ "ID": t.ID(), "Kind": t.Kind().String(), "PubMuted": t.IsMuted(), @@ -950,6 +1173,16 @@ func (t *MediaTrackReceiver) PrimaryReceiver() sfu.TrackReceiver { return receivers[0].TrackReceiver } +func (t *MediaTrackReceiver) ActiveReceiver() sfu.TrackReceiver { + for _, r := range t.loadReceivers() { + if r.IsRegressed() { + return r.TrackReceiver + } + } + + return t.PrimaryReceiver() +} + func (t *MediaTrackReceiver) Receiver(mime mime.MimeType) sfu.TrackReceiver { for _, r := range t.loadReceivers() { if r.Mime() == mime { @@ -985,8 +1218,8 @@ func (t *MediaTrackReceiver) SetRTT(rtt uint32) { } } -func (t *MediaTrackReceiver) GetTemporalLayerForSpatialFps(spatial int32, fps uint32, mime mime.MimeType) int32 { - receiver := t.Receiver(mime) +func (t *MediaTrackReceiver) GetTemporalLayerForSpatialFps(mimeType mime.MimeType, spatial int32, fps uint32) int32 { + receiver := t.Receiver(mimeType) if receiver == nil { return buffer.DefaultMaxLayerTemporal } diff --git a/pkg/rtc/mediatracksubscriptions.go b/pkg/rtc/mediatracksubscriptions.go index 88a9df9..69575fe 100644 --- a/pkg/rtc/mediatracksubscriptions.go +++ b/pkg/rtc/mediatracksubscriptions.go @@ -16,21 +16,17 @@ package rtc import ( "errors" + "slices" "sync" - "github.com/pion/rtcp" + "github.com/livekit/protocol/codecs/mime" + "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/logger" "github.com/pion/webrtc/v4" "go.uber.org/atomic" - "github.com/livekit/livekit-server/pkg/sfu/buffer" - "github.com/livekit/livekit-server/pkg/sfu/mime" - sutils "github.com/livekit/livekit-server/pkg/utils" - "github.com/livekit/protocol/livekit" - "github.com/livekit/protocol/logger" - "github.com/livekit/livekit-server/pkg/rtc/types" "github.com/livekit/livekit-server/pkg/sfu" - "github.com/livekit/livekit-server/pkg/telemetry" ) var ( @@ -47,6 +43,7 @@ type MediaTrackSubscriptions struct { onDownTrackCreated func(downTrack *sfu.DownTrack) onSubscriberMaxQualityChange func(subscriberID livekit.ParticipantID, mime mime.MimeType, layer int32) + onSubscriberAudioCodecChange func(subscriberID livekit.ParticipantID, mime mime.MimeType, enabled bool) } type MediaTrackSubscriptionsParams struct { @@ -56,8 +53,6 @@ type MediaTrackSubscriptionsParams struct { ReceiverConfig ReceiverConfig SubscriberConfig DirectionConfig - Telemetry telemetry.TelemetryService - Logger logger.Logger } @@ -76,6 +71,10 @@ func (t *MediaTrackSubscriptions) OnSubscriberMaxQualityChange(f func(subscriber t.onSubscriberMaxQualityChange = f } +func (t *MediaTrackSubscriptions) OnSubscriberAudioCodecChange(f func(subscriberID livekit.ParticipantID, mime mime.MimeType, enabled bool)) { + t.onSubscriberAudioCodecChange = f +} + func (t *MediaTrackSubscriptions) SetMuted(muted bool) { // update mute of all subscribed tracks for _, st := range t.getAllSubscribedTracks() { @@ -104,86 +103,32 @@ func (t *MediaTrackSubscriptions) AddSubscriber(sub types.LocalParticipant, wr * } t.subscribedTracksMu.Unlock() - var rtcpFeedback []webrtc.RTCPFeedback - var maxTrack int - switch t.params.MediaTrack.Kind() { - case livekit.TrackType_AUDIO: - rtcpFeedback = t.params.SubscriberConfig.RTCPFeedback.Audio - maxTrack = t.params.ReceiverConfig.PacketBufferSizeAudio - case livekit.TrackType_VIDEO: - rtcpFeedback = t.params.SubscriberConfig.RTCPFeedback.Video - maxTrack = t.params.ReceiverConfig.PacketBufferSizeVideo - } - codecs := wr.Codecs() - for _, c := range codecs { - c.RTCPFeedback = rtcpFeedback - } - - streamID := wr.StreamID() - if sub.SupportsSyncStreamID() && t.params.MediaTrack.Stream() != "" { - streamID = PackSyncStreamID(t.params.MediaTrack.PublisherID(), t.params.MediaTrack.Stream()) - } - - var trailer []byte - if t.params.MediaTrack.IsEncrypted() { - trailer = sub.GetTrailer() - } - - downTrack, err := sfu.NewDownTrack(sfu.DowntrackParams{ - Codecs: codecs, - Source: t.params.MediaTrack.Source(), - Receiver: wr, - BufferFactory: sub.GetBufferFactory(), - SubID: subscriberID, - StreamID: streamID, - MaxTrack: maxTrack, - PlayoutDelayLimit: sub.GetPlayoutDelayConfig(), - Pacer: sub.GetPacer(), - Trailer: trailer, - Logger: LoggerWithTrack(sub.GetLogger().WithComponent(sutils.ComponentSub), trackID, t.params.IsRelayed), - RTCPWriter: sub.WriteSubscriberRTCP, - DisableSenderReportPassThrough: sub.GetDisableSenderReportPassThrough(), - SupportsCodecChange: sub.SupportsCodecChange(), + subTrack, err := NewSubscribedTrack(SubscribedTrackParams{ + ReceiverConfig: t.params.ReceiverConfig, + SubscriberConfig: t.params.SubscriberConfig, + Subscriber: sub, + MediaTrack: t.params.MediaTrack, + AdaptiveStream: sub.GetAdaptiveStream(), + TelemetryListener: sub.GetTelemetryListener(), + WrappedReceiver: wr, + IsRelayed: t.params.IsRelayed, + OnDownTrackCreated: t.onDownTrackCreated, + OnDownTrackClosed: func(subscriberID livekit.ParticipantID) { + t.subscribedTracksMu.Lock() + delete(t.subscribedTracks, subscriberID) + t.subscribedTracksMu.Unlock() + }, + OnSubscriberMaxQualityChange: t.onSubscriberMaxQualityChange, + OnSubscriberAudioCodecChange: t.onSubscriberAudioCodecChange, }) if err != nil { return nil, err } - if t.onDownTrackCreated != nil { - t.onDownTrackCreated(downTrack) - } - - subTrack := NewSubscribedTrack(SubscribedTrackParams{ - PublisherID: t.params.MediaTrack.PublisherID(), - PublisherIdentity: t.params.MediaTrack.PublisherIdentity(), - PublisherVersion: t.params.MediaTrack.PublisherVersion(), - Subscriber: sub, - MediaTrack: t.params.MediaTrack, - DownTrack: downTrack, - AdaptiveStream: sub.GetAdaptiveStream(), - }) - - if !sub.Hidden() { - subTrack.AddOnBind(func(err error) { - if err == nil { - t.params.MediaTrack.OnTrackSubscribed() - } - }) - } - // Bind callback can happen from replaceTrack, so set it up early var reusingTransceiver atomic.Bool var dtState sfu.DownTrackState - downTrack.OnCodecNegotiated(func(codec webrtc.RTPCodecCapability) { - if !wr.DetermineReceiver(codec) { - if t.onSubscriberMaxQualityChange != nil { - go func() { - spatial := buffer.VideoQualityToSpatialLayer(livekit.VideoQuality_HIGH, t.params.MediaTrack.ToProto()) - t.onSubscriberMaxQualityChange(downTrack.SubscriberID(), mime.NormalizeMimeType(codec.MimeType), spatial) - }() - } - } - }) + downTrack := subTrack.DownTrack() downTrack.OnBinding(func(err error) { if err != nil { go subTrack.Bound(err) @@ -207,25 +152,6 @@ func (t *MediaTrackSubscriptions) AddSubscriber(sub types.LocalParticipant, wr * subTrack.SetPublisherMuted(t.params.MediaTrack.IsMuted()) }) - downTrack.OnStatsUpdate(func(_ *sfu.DownTrack, stat *livekit.AnalyticsStat) { - key := telemetry.StatsKeyForTrack(livekit.StreamType_DOWNSTREAM, subscriberID, trackID, t.params.MediaTrack.Source(), t.params.MediaTrack.Kind()) - t.params.Telemetry.TrackStats(key, stat) - }) - - downTrack.OnMaxLayerChanged(func(dt *sfu.DownTrack, layer int32) { - if t.onSubscriberMaxQualityChange != nil { - t.onSubscriberMaxQualityChange(dt.SubscriberID(), dt.Mime(), layer) - } - }) - - downTrack.OnRttUpdate(func(_ *sfu.DownTrack, rtt uint32) { - go sub.UpdateMediaRTT(rtt) - }) - - downTrack.AddReceiverReportListener(func(dt *sfu.DownTrack, report *rtcp.ReceiverReport) { - sub.HandleReceiverReport(dt, report) - }) - var transceiver *webrtc.RTPTransceiver var sender *webrtc.RTPSender @@ -277,15 +203,16 @@ func (t *MediaTrackSubscriptions) AddSubscriber(sub types.LocalParticipant, wr * if transceiver == nil { info := t.params.MediaTrack.ToProto() addTrackParams := types.AddTrackParams{ - Stereo: info.Stereo, - Red: !info.DisableRed, + Stereo: slices.Contains(info.AudioFeatures, livekit.AudioTrackFeature_TF_STEREO), + Red: IsRedEnabled(info), } + codecs := wr.Codecs() if addTrackParams.Red && (len(codecs) == 1 && mime.IsMimeTypeStringOpus(codecs[0].MimeType)) { addTrackParams.Red = false } sub.VerifySubscribeParticipantInfo(subTrack.PublisherID(), subTrack.PublisherVersion()) - if sub.SupportsTransceiverReuse() { + if sub.SupportsTransceiverReuse(t.params.MediaTrack) { // // AddTrack will create a new transceiver or re-use an unused one // if the attributes match. This prevents SDP from bloating @@ -320,6 +247,7 @@ func (t *MediaTrackSubscriptions) AddSubscriber(sub types.LocalParticipant, wr * // But, the subscription could be removed early if the published track is closed // while adding subscription. In those cases, subscription manager would not have set // the `OnClose` callback. So, set it here to handle cases of early close. + // Subscription manager will reset this if this subscription proceeds till that point. subTrack.OnClose(func(isExpectedToResume bool) { if !isExpectedToResume { if err := sub.RemoveTrackLocal(sender); err != nil { @@ -330,10 +258,6 @@ func (t *MediaTrackSubscriptions) AddSubscriber(sub types.LocalParticipant, wr * downTrack.SetTransceiver(transceiver) - downTrack.OnCloseHandler(func(isExpectedToResume bool) { - t.downTrackClosed(sub, subTrack, isExpectedToResume) - }) - t.subscribedTracksMu.Lock() t.subscribedTracks[subscriberID] = subTrack t.subscribedTracksMu.Unlock() @@ -361,10 +285,10 @@ func (t *MediaTrackSubscriptions) closeSubscribedTrack(subTrack types.Subscribed } if isExpectedToResume { - dt.CloseWithFlush(false) + dt.CloseWithFlush(false, false) } else { // flushing blocks, avoid blocking when publisher removes all its subscribers - go dt.CloseWithFlush(true) + go dt.CloseWithFlush(true, true) } } @@ -429,8 +353,8 @@ func (t *MediaTrackSubscriptions) getAllSubscribedTracksLocked() []types.Subscri return subTracks } -func (t *MediaTrackSubscriptions) DebugInfo() []map[string]interface{} { - subscribedTrackInfo := make([]map[string]interface{}, 0) +func (t *MediaTrackSubscriptions) DebugInfo() []map[string]any { + subscribedTrackInfo := make([]map[string]any, 0) for _, val := range t.getAllSubscribedTracks() { if st, ok := val.(*SubscribedTrack); ok { subscribedTrackInfo = append(subscribedTrackInfo, st.DownTrack().DebugInfo()) @@ -439,27 +363,3 @@ func (t *MediaTrackSubscriptions) DebugInfo() []map[string]interface{} { return subscribedTrackInfo } - -func (t *MediaTrackSubscriptions) downTrackClosed( - sub types.LocalParticipant, - subTrack types.SubscribedTrack, - isExpectedToResume bool, -) { - // Cache transceiver for potential re-use on resume. - // To ensure subscription manager does not re-subscribe before caching, - // delete the subscribed track only after caching. - if isExpectedToResume { - dt := subTrack.DownTrack() - tr := dt.GetTransceiver() - if tr != nil { - sub.CacheDownTrack(subTrack.ID(), tr, dt.GetState()) - } - } - - go func() { - t.subscribedTracksMu.Lock() - delete(t.subscribedTracks, sub.ID()) - t.subscribedTracksMu.Unlock() - subTrack.Close(isExpectedToResume) - }() -} diff --git a/pkg/rtc/migrationdatacache.go b/pkg/rtc/migrationdatacache.go new file mode 100644 index 0000000..8c44eac --- /dev/null +++ b/pkg/rtc/migrationdatacache.go @@ -0,0 +1,59 @@ +package rtc + +import ( + "time" + + "github.com/livekit/protocol/livekit" +) + +type MigrationDataCacheState int + +const ( + MigrationDataCacheStateWaiting MigrationDataCacheState = iota + MigrationDataCacheStateTimeout + MigrationDataCacheStateDone +) + +type MigrationDataCache struct { + lastSeq uint32 + pkts []*livekit.DataPacket + state MigrationDataCacheState + expiredAt time.Time +} + +func NewMigrationDataCache(lastSeq uint32, expiredAt time.Time) *MigrationDataCache { + return &MigrationDataCache{ + lastSeq: lastSeq, + expiredAt: expiredAt, + } +} + +// Add adds a message to the cache if there is a gap between the last sequence number and cached messages then return the cache State: +// - MigrationDataCacheStateWaiting: waiting for the next packet (lastSeq + 1) of last sequence from old node +// - MigrationDataCacheStateTimeout: the next packet is not received before the expiredAt, participant will +// continue to process the reliable messages, subscribers will see the gap after the publisher migration +// - MigrationDataCacheStateDone: the next packet is received, participant can continue to process the reliable messages +func (c *MigrationDataCache) Add(pkt *livekit.DataPacket) MigrationDataCacheState { + if c.state == MigrationDataCacheStateDone || c.state == MigrationDataCacheStateTimeout { + return c.state + } + + if pkt.Sequence <= c.lastSeq { + return c.state + } + + if pkt.Sequence == c.lastSeq+1 { + c.state = MigrationDataCacheStateDone + return c.state + } + + c.pkts = append(c.pkts, pkt) + if time.Now().After(c.expiredAt) { + c.state = MigrationDataCacheStateTimeout + } + return c.state +} + +func (c *MigrationDataCache) Get() []*livekit.DataPacket { + return c.pkts +} diff --git a/pkg/rtc/migrationdatacache_test.go b/pkg/rtc/migrationdatacache_test.go new file mode 100644 index 0000000..046b2dd --- /dev/null +++ b/pkg/rtc/migrationdatacache_test.go @@ -0,0 +1,38 @@ +package rtc + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/livekit/protocol/livekit" +) + +func TestMigrationDataCache_Add(t *testing.T) { + expiredAt := time.Now().Add(100 * time.Millisecond) + cache := NewMigrationDataCache(10, expiredAt) + + pkt1 := &livekit.DataPacket{Sequence: 9} + state := cache.Add(pkt1) + require.Equal(t, MigrationDataCacheStateWaiting, state) + require.Empty(t, cache.Get()) + + pkt2 := &livekit.DataPacket{Sequence: 11} + state = cache.Add(pkt2) + require.Equal(t, MigrationDataCacheStateDone, state) + require.Empty(t, cache.Get()) + + pkt3 := &livekit.DataPacket{Sequence: 12} + state = cache.Add(pkt3) + require.Equal(t, MigrationDataCacheStateDone, state) + require.Empty(t, cache.Get()) + + cache2 := NewMigrationDataCache(20, time.Now().Add(10*time.Millisecond)) + pkt4 := &livekit.DataPacket{Sequence: 22} + time.Sleep(20 * time.Millisecond) + state = cache2.Add(pkt4) + require.Equal(t, MigrationDataCacheStateTimeout, state) + require.Len(t, cache2.Get(), 1) + require.Equal(t, uint32(22), cache2.Get()[0].Sequence) +} diff --git a/pkg/rtc/participant.go b/pkg/rtc/participant.go index 6f3d9b0..2b71174 100644 --- a/pkg/rtc/participant.go +++ b/pkg/rtc/participant.go @@ -15,41 +15,52 @@ package rtc import ( - "context" "fmt" "io" + "maps" + "math/rand" "os" "slices" - "strconv" "strings" "sync" "time" + "github.com/frostbyte73/core" + "github.com/google/uuid" lru "github.com/hashicorp/golang-lru/v2" "github.com/pion/rtcp" "github.com/pion/sdp/v3" "github.com/pion/webrtc/v4" "github.com/pkg/errors" "go.uber.org/atomic" + "go.uber.org/zap/zapcore" "google.golang.org/protobuf/proto" "github.com/livekit/mediatransportutil/pkg/twcc" "github.com/livekit/protocol/auth" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" + "github.com/livekit/protocol/observability" + "github.com/livekit/protocol/observability/roomobs" + protosdp "github.com/livekit/protocol/sdp" + protosignalling "github.com/livekit/protocol/signalling" "github.com/livekit/protocol/utils" "github.com/livekit/protocol/utils/guid" + "github.com/livekit/protocol/utils/pointer" + "github.com/livekit/psrpc" "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/livekit-server/pkg/metric" "github.com/livekit/livekit-server/pkg/routing" + "github.com/livekit/livekit-server/pkg/rtc/signalling" "github.com/livekit/livekit-server/pkg/rtc/supervisor" "github.com/livekit/livekit-server/pkg/rtc/transport" "github.com/livekit/livekit-server/pkg/rtc/types" "github.com/livekit/livekit-server/pkg/sfu" "github.com/livekit/livekit-server/pkg/sfu/buffer" "github.com/livekit/livekit-server/pkg/sfu/connectionquality" - "github.com/livekit/livekit-server/pkg/sfu/mime" + "github.com/livekit/livekit-server/pkg/sfu/interceptor" "github.com/livekit/livekit-server/pkg/sfu/pacer" "github.com/livekit/livekit-server/pkg/sfu/streamallocator" "github.com/livekit/livekit-server/pkg/telemetry" @@ -57,28 +68,53 @@ import ( sutils "github.com/livekit/livekit-server/pkg/utils" ) +var _ types.LocalParticipant = (*ParticipantImpl)(nil) + const ( sdBatchSize = 30 rttUpdateInterval = 5 * time.Second - disconnectCleanupDuration = 5 * time.Second - migrationWaitDuration = 3 * time.Second + disconnectCleanupDuration = 5 * time.Second + migrationWaitDuration = 3 * time.Second + migrationWaitContinuousMsgDuration = 2 * time.Second PingIntervalSeconds = 5 PingTimeoutSeconds = 15 ) +var ( + ErrMoveOldClientVersion = errors.New("participant client version does not support moving") +) + +// ------------------------------------------------- + type pendingTrackInfo struct { trackInfos []*livekit.TrackInfo + sdpRids buffer.VideoLayersRid migrated bool createdAt time.Time // indicates if this track is queued for publishing to avoid a track has been published - // before the previous track is unpublished(closed) because client is allowed to neogtiate + // before the previous track is unpublished(closed) because client is allowed to negotiate // webrtc track before AddTrackRequest return to speed up the publishing process queued bool } +func (p *pendingTrackInfo) MarshalLogObject(e zapcore.ObjectEncoder) error { + if p == nil { + return nil + } + + e.AddArray("trackInfos", logger.ProtoSlice(p.trackInfos)) + e.AddArray("sdpRids", logger.StringSlice(p.sdpRids[:])) + e.AddBool("migrated", p.migrated) + e.AddTime("createdAt", p.createdAt) + e.AddBool("queued", p.queued) + return nil +} + +// -------------------------------------------------- + type pendingRemoteTrack struct { track *webrtc.TrackRemote receiver *webrtc.RTPReceiver @@ -107,8 +143,20 @@ func (p participantUpdateInfo) String() string { return fmt.Sprintf("identity: %s, version: %d, state: %s, updatedAt: %s", p.identity, p.version, p.state.String(), p.updatedAt.String()) } +type reliableDataInfo struct { + joiningMessageLock sync.Mutex + joiningMessageFirstSeqs map[livekit.ParticipantID]uint32 + joiningMessageLastWrittenSeqs map[livekit.ParticipantID]uint32 + lastPubReliableSeq atomic.Uint32 + stopReliableByMigrateOut atomic.Bool + canWriteReliable bool + migrateInPubDataCache atomic.Pointer[MigrationDataCache] +} + // --------------------------------------------------------------- +var _ types.LocalParticipant = (*ParticipantImpl)(nil) + type ParticipantParams struct { Identity livekit.ParticipantIdentity Name livekit.ParticipantName @@ -120,51 +168,62 @@ type ParticipantParams struct { LimitConfig config.LimitConfig ProtocolVersion types.ProtocolVersion SessionStartTime time.Time - Telemetry telemetry.TelemetryService + SessionTimer *observability.SessionTimer + TelemetryListener types.ParticipantTelemetryListener Trailer []byte PLIThrottleConfig sfu.PLIThrottleConfig CongestionControlConfig config.CongestionControlConfig // codecs that are enabled for this room - PublishEnabledCodecs []*livekit.Codec - SubscribeEnabledCodecs []*livekit.Codec - Logger logger.Logger - SimTracks map[uint32]SimulcastTrackInfo - Grants *auth.ClaimGrants - InitialVersion uint32 - ClientConf *livekit.ClientConfiguration - ClientInfo ClientInfo - Region string - Migration bool - Reconnect bool - AdaptiveStream bool - AllowTCPFallback bool - TCPFallbackRTTThreshold int - AllowUDPUnstableFallback bool - TURNSEnabled bool - GetParticipantInfo func(pID livekit.ParticipantID) *livekit.ParticipantInfo - GetRegionSettings func(ip string) *livekit.RegionSettings - GetSubscriberForwarderState func(p types.LocalParticipant) (map[livekit.TrackID]*livekit.RTPForwarderState, error) - DisableSupervisor bool - ReconnectOnPublicationError bool - ReconnectOnSubscriptionError bool - ReconnectOnDataChannelError bool - VersionGenerator utils.TimedVersionGenerator - TrackResolver types.MediaTrackResolver - DisableDynacast bool - SubscriberAllowPause bool - SubscriptionLimitAudio int32 - SubscriptionLimitVideo int32 - PlayoutDelay *livekit.PlayoutDelay - SyncStreams bool - ForwardStats *sfu.ForwardStats - DisableSenderReportPassThrough bool - MetricConfig metric.MetricConfig - UseOneShotSignallingMode bool - EnableMetrics bool - DataChannelMaxBufferedAmount uint64 - DatachannelSlowThreshold int - FireOnTrackBySdp bool - DisableCodecRegression bool + PublishEnabledCodecs []*livekit.Codec + SubscribeEnabledCodecs []*livekit.Codec + Logger logger.Logger + LoggerResolver logger.DeferredFieldResolver + Reporter roomobs.ParticipantSessionReporter + ReporterResolver roomobs.ParticipantReporterResolver + SimTracks map[uint32]interceptor.SimulcastTrackInfo + Grants *auth.ClaimGrants + InitialVersion uint32 + ClientConf *livekit.ClientConfiguration + ClientInfo ClientInfo + Region string + Migration bool + Reconnect bool + AdaptiveStream bool + AllowTCPFallback bool + TCPFallbackRTTThreshold int + AllowUDPUnstableFallback bool + TURNSEnabled bool + ParticipantListener types.LocalParticipantListener + ParticipantHelper types.LocalParticipantHelper + DisableSupervisor bool + ReconnectOnPublicationError bool + ReconnectOnSubscriptionError bool + ReconnectOnDataChannelError bool + VersionGenerator utils.TimedVersionGenerator + DisableDynacast bool + SubscriberAllowPause bool + SubscriptionLimitAudio int32 + SubscriptionLimitVideo int32 + PlayoutDelay *livekit.PlayoutDelay + SyncStreams bool + ForwardStats *sfu.ForwardStats + DisableSenderReportPassThrough bool + MetricConfig metric.MetricConfig + UseOneShotSignallingMode bool + EnableMetrics bool + DataChannelMaxBufferedAmount uint64 + DatachannelSlowThreshold int + DatachannelLossyTargetLatency time.Duration + FireOnTrackBySdp bool + DisableCodecRegression bool + LastPubReliableSeq uint32 + Country string + PreferVideoSizeFromMedia bool + UseSinglePeerConnection bool + EnableDataTracks bool + EnableRTPStreamRestartDetection bool + ForceBackupCodecPolicySimulcast bool + DisableTransceiverReuseForE2EE bool } type ParticipantImpl struct { @@ -174,22 +233,24 @@ type ParticipantImpl struct { params ParticipantParams + participantListener atomic.Pointer[types.LocalParticipantListener] + participantHelper atomic.Value // types.LocalParticipantHelper + id atomic.Value // types.ParticipantID + isClosed atomic.Bool closeReason atomic.Value // types.ParticipantCloseReason state atomic.Value // livekit.ParticipantInfo_State disconnected chan struct{} - resSinkMu sync.Mutex - resSink routing.MessageSink - grants atomic.Pointer[auth.ClaimGrants] isPublisher atomic.Bool sessionStartRecorded atomic.Bool lastActiveAt atomic.Pointer[time.Time] // when first connected - connectedAt time.Time + connectedAt time.Time + disconnectedAt atomic.Pointer[time.Time] // timer that's set when disconnect is detected on primary PC disconnectTimer *time.Timer migrationTimer *time.Timer @@ -211,42 +272,41 @@ type ParticipantImpl struct { *TransportManager *UpTrackManager + *UpDataTrackManager *SubscriptionManager + nextSubscribedDataTrackHandle uint16 + icQueue [2]atomic.Pointer[webrtc.ICECandidate] requireBroadcast bool // queued participant updates before join response is sent // guarded by updateLock queuedUpdates []*livekit.ParticipantInfo - // cache of recently sent updates, to ensuring ordering by version + // cache of recently sent updates, to ensure ordering by version // guarded by updateLock updateCache *lru.Cache[livekit.ParticipantID, participantUpdateInfo] updateLock utils.Mutex - dataChannelStats *telemetry.BytesTrackStats + dataChannelStats *BytesTrackStats + + reliableDataInfo reliableDataInfo rttUpdatedAt time.Time lastRTT uint32 + // idempotent reference guard for telemetry stats worker + telemetryGuard *telemetry.ReferenceGuard + lock utils.RWMutex dirty atomic.Bool version atomic.Uint32 - // callbacks & handlers - onTrackPublished func(types.LocalParticipant, types.MediaTrack) - onTrackUpdated func(types.LocalParticipant, types.MediaTrack) - onTrackUnpublished func(types.LocalParticipant, types.MediaTrack) - onStateChange func(p types.LocalParticipant, state livekit.ParticipantInfo_State) - onMigrateStateChange func(p types.LocalParticipant, migrateState types.MigrateState) - onParticipantUpdate func(types.LocalParticipant) - onDataPacket func(types.LocalParticipant, livekit.DataPacket_Kind, *livekit.DataPacket) - onMetrics func(types.Participant, *livekit.DataPacket) + migrateState atomic.Value // types.MigrateState + migratedTracksPublishedFuse core.Fuse - migrateState atomic.Value // types.MigrateState - - onClose func(types.LocalParticipant) + onClose map[string]func(types.LocalParticipant) onClaimsChanged func(participant types.LocalParticipant) onICEConfigChanged func(participant types.LocalParticipant, iceConfig *livekit.ICEConfig) @@ -261,9 +321,17 @@ type ParticipantImpl struct { metricsCollector *metric.MetricsCollector metricsReporter *metric.MetricsReporter + signalling signalling.ParticipantSignalling + signalHandler signalling.ParticipantSignalHandler + signaller signalling.ParticipantSignaller + // loggers for publisher and subscriber pubLogger logger.Logger subLogger logger.Logger + + rpcLock sync.Mutex + rpcPendingAcks map[string]*utils.DataChannelRpcPendingAckHandler + rpcPendingResponses map[string]*utils.DataChannelRpcPendingResponseHandler } func NewParticipant(params ParticipantParams) (*ParticipantImpl, error) { @@ -289,31 +357,77 @@ func NewParticipant(params ParticipantParams) (*ParticipantImpl, error) { connectedAt: time.Now().Truncate(time.Millisecond), rttUpdatedAt: time.Now(), cachedDownTracks: make(map[livekit.TrackID]*downTrackState), - dataChannelStats: telemetry.NewBytesTrackStats( - telemetry.BytesTrackIDForParticipantID(telemetry.BytesTrackTypeData, params.SID), - params.SID, - params.Telemetry, - ), - connectionQuality: livekit.ConnectionQuality_EXCELLENT, - pubLogger: params.Logger.WithComponent(sutils.ComponentPub), - subLogger: params.Logger.WithComponent(sutils.ComponentSub), + connectionQuality: livekit.ConnectionQuality_EXCELLENT, + pubLogger: params.Logger.WithComponent(sutils.ComponentPub), + subLogger: params.Logger.WithComponent(sutils.ComponentSub), + reliableDataInfo: reliableDataInfo{ + joiningMessageFirstSeqs: make(map[livekit.ParticipantID]uint32), + joiningMessageLastWrittenSeqs: make(map[livekit.ParticipantID]uint32), + }, + rpcPendingAcks: make(map[string]*utils.DataChannelRpcPendingAckHandler), + rpcPendingResponses: make(map[string]*utils.DataChannelRpcPendingResponseHandler), + onClose: make(map[string]func(types.LocalParticipant)), + telemetryGuard: &telemetry.ReferenceGuard{}, + nextSubscribedDataTrackHandle: uint16(rand.Intn(256)), + requireBroadcast: params.Grants.Metadata != "" || len(params.Grants.Attributes) != 0, } + p.setupSignalling() + + p.id.Store(params.SID) + p.dataChannelStats = NewBytesTrackStats( + p.params.Country, + BytesTrackIDForParticipantID(BytesTrackTypeData, p.ID()), + p.ID(), + params.Grants.GetParticipantKind(), + params.Grants.GetKindDetails(), + params.TelemetryListener, + params.Reporter, + ) + p.reliableDataInfo.lastPubReliableSeq.Store(params.LastPubReliableSeq) + p.setListener(params.ParticipantListener) + p.participantHelper.Store(params.ParticipantHelper) if !params.DisableSupervisor { p.supervisor = supervisor.NewParticipantSupervisor(supervisor.ParticipantSupervisorParams{Logger: params.Logger}) } p.closeReason.Store(types.ParticipantCloseReasonNone) p.version.Store(params.InitialVersion) p.timedVersion.Update(params.VersionGenerator.Next()) + p.migrateState.Store(types.MigrateStateInit) + p.state.Store(livekit.ParticipantInfo_JOINING) p.grants.Store(params.Grants.Clone()) - p.SetResponseSink(params.Sink) + p.SwapResponseSink(params.Sink, types.SignallingCloseReasonUnknown) p.setupEnabledCodecs(params.PublishEnabledCodecs, params.SubscribeEnabledCodecs, params.ClientConf.GetDisabledCodecs()) if p.supervisor != nil { p.supervisor.OnPublicationError(p.onPublicationError) } + params.Reporter.RegisterFunc(func(ts time.Time, tx roomobs.ParticipantSessionTx) bool { + // Don't publish duration if participant never became active. Otherwise short-lived + // JOINING/JOINED -> DISCONNECTED transitions would still get rounded up to a + // minute by the session timer and inflate billed/reported duration. + if p.lastActiveAt.Load() == nil { + return !p.IsClosed() + } + + if dts := p.disconnectedAt.Load(); dts != nil { + ts = *dts + tx.ReportEndTime(ts) + } + + tx.ReportKindCode(roomobs.ParticipantKindCode(p.Kind())) + tx.ReportKindDetailsCodes(roomobs.ParticipantKindDetailsCodes(p.KindDetails())) + + millis, secs, mins := p.params.SessionTimer.Advance(ts) + tx.ReportDuration(uint16(millis)) + tx.ReportDurationSeconds(uint16(secs)) + tx.ReportDurationMinutes(uint8(mins)) + + return !p.IsClosed() + }) + var err error // keep last participants and when updates were sent if p.updateCache, err = lru.New[livekit.ParticipantID, participantUpdateInfo](128); err != nil { @@ -326,12 +440,40 @@ func NewParticipant(params ParticipantParams) (*ParticipantImpl, error) { } p.setupUpTrackManager() + p.setupUpDataTrackManager() p.setupSubscriptionManager() p.setupMetrics() return p, nil } +func (p *ParticipantImpl) setListener(listener types.LocalParticipantListener) { + if listener == nil { + p.participantListener.Store(nil) + return + } + p.participantListener.Store(&listener) +} + +func (p *ParticipantImpl) listener() types.LocalParticipantListener { + if l := p.participantListener.Load(); l != nil { + return *l + } + return &types.NullLocalParticipantListener{} +} + +func (p *ParticipantImpl) GetParticipantListener() types.ParticipantListener { + return p.listener() +} + +func (p *ParticipantImpl) ClearParticipantListener() { + p.setListener(nil) +} + +func (p *ParticipantImpl) GetCountry() string { + return p.params.Country +} + func (p *ParticipantImpl) GetTrailer() []byte { trailer := make([]byte, len(p.params.Trailer)) copy(trailer, p.params.Trailer) @@ -342,6 +484,18 @@ func (p *ParticipantImpl) GetLogger() logger.Logger { return p.params.Logger } +func (p *ParticipantImpl) GetLoggerResolver() logger.DeferredFieldResolver { + return p.params.LoggerResolver +} + +func (p *ParticipantImpl) GetReporter() roomobs.ParticipantSessionReporter { + return p.params.Reporter +} + +func (p *ParticipantImpl) GetReporterResolver() roomobs.ParticipantReporterResolver { + return p.params.ReporterResolver +} + func (p *ParticipantImpl) GetAdaptiveStream() bool { return p.params.AdaptiveStream } @@ -355,7 +509,7 @@ func (p *ParticipantImpl) GetDisableSenderReportPassThrough() bool { } func (p *ParticipantImpl) ID() livekit.ParticipantID { - return p.params.SID + return p.id.Load().(livekit.ParticipantID) } func (p *ParticipantImpl) Identity() livekit.ParticipantIdentity { @@ -370,6 +524,10 @@ func (p *ParticipantImpl) Kind() livekit.ParticipantInfo_Kind { return p.grants.Load().GetParticipantKind() } +func (p *ParticipantImpl) KindDetails() []livekit.ParticipantInfo_KindDetail { + return p.grants.Load().GetKindDetails() +} + func (p *ParticipantImpl) IsRecorder() bool { grants := p.grants.Load() return grants.GetParticipantKind() == livekit.ParticipantInfo_EGRESS || grants.Video.Recorder @@ -430,6 +588,14 @@ func (p *ParticipantImpl) ConnectedAt() time.Time { return p.connectedAt } +func (p *ParticipantImpl) ActiveAt() time.Time { + if activeAt := p.lastActiveAt.Load(); activeAt != nil { + return *activeAt + } + + return time.Time{} +} + func (p *ParticipantImpl) GetClientInfo() *livekit.ClientInfo { p.lock.RLock() defer p.lock.RUnlock() @@ -446,27 +612,87 @@ func (p *ParticipantImpl) GetBufferFactory() *buffer.Factory { return p.params.Config.BufferFactory } -// CheckMetadataLimits check if name/metadata/attributes of a participant is within configured limits -func (p *ParticipantImpl) CheckMetadataLimits( +// checkMetadataLimits check if name/metadata/attributes of a participant is within configured limits +func (p *ParticipantImpl) checkMetadataLimits( name string, metadata string, attributes map[string]string, ) error { if !p.params.LimitConfig.CheckParticipantNameLength(name) { - return ErrNameExceedsLimits + return signalling.ErrNameExceedsLimits } if !p.params.LimitConfig.CheckMetadataSize(metadata) { - return ErrMetadataExceedsLimits + return signalling.ErrMetadataExceedsLimits } if !p.params.LimitConfig.CheckAttributesSize(attributes) { - return ErrAttributesExceedsLimits + return signalling.ErrAttributesExceedsLimits } return nil } +func (p *ParticipantImpl) UpdateMetadata(update *livekit.UpdateParticipantMetadata, fromAdmin bool) error { + lgr := p.params.Logger.WithUnlikelyValues( + "update", logger.Proto(update), + "fromAdmin", fromAdmin, + ) + lgr.Debugw("updating participant metadata") + + var err error + requestResponse := &livekit.RequestResponse{ + RequestId: update.RequestId, + } + sendRequestResponse := func() error { + if !fromAdmin || (update.RequestId != 0 || err != nil) { + requestResponse.Request = &livekit.RequestResponse_UpdateMetadata{ + UpdateMetadata: utils.CloneProto(update), + } + p.sendRequestResponse(requestResponse) + } + if err != nil { + lgr.Warnw("could not update metadata", err) + } + return err + } + + if !fromAdmin && !p.ClaimGrants().Video.GetCanUpdateOwnMetadata() { + requestResponse.Reason = livekit.RequestResponse_NOT_ALLOWED + requestResponse.Message = "does not have permission to update own metadata" + err = signalling.ErrUpdateOwnMetadataNotAllowed + return sendRequestResponse() + } + + if err = p.checkMetadataLimits(update.Name, update.Metadata, update.Attributes); err != nil { + switch err { + case signalling.ErrNameExceedsLimits: + requestResponse.Reason = livekit.RequestResponse_LIMIT_EXCEEDED + requestResponse.Message = "exceeds name length limit" + + case signalling.ErrMetadataExceedsLimits: + requestResponse.Reason = livekit.RequestResponse_LIMIT_EXCEEDED + requestResponse.Message = "exceeds metadata size limit" + + case signalling.ErrAttributesExceedsLimits: + requestResponse.Reason = livekit.RequestResponse_LIMIT_EXCEEDED + requestResponse.Message = "exceeds attributes size limit" + } + return sendRequestResponse() + } + + if update.Name != "" { + p.SetName(update.Name) + } + if update.Metadata != "" { + p.SetMetadata(update.Metadata) + } + if update.Attributes != nil { + p.SetAttributes(update.Attributes) + } + return sendRequestResponse() +} + // SetName attaches name to the participant func (p *ParticipantImpl) SetName(name string) { p.lock.Lock() @@ -481,13 +707,11 @@ func (p *ParticipantImpl) SetName(name string) { p.grants.Store(grants) p.dirty.Store(true) - onParticipantUpdate := p.onParticipantUpdate onClaimsChanged := p.onClaimsChanged p.lock.Unlock() - if onParticipantUpdate != nil { - onParticipantUpdate(p) - } + p.listener().OnParticipantUpdate(p) + if onClaimsChanged != nil { onClaimsChanged(p) } @@ -508,13 +732,11 @@ func (p *ParticipantImpl) SetMetadata(metadata string) { p.requireBroadcast = p.requireBroadcast || metadata != "" p.dirty.Store(true) - onParticipantUpdate := p.onParticipantUpdate onClaimsChanged := p.onClaimsChanged p.lock.Unlock() - if onParticipantUpdate != nil { - onParticipantUpdate(p) - } + p.listener().OnParticipantUpdate(p) + if onClaimsChanged != nil { onClaimsChanged(p) } @@ -545,13 +767,11 @@ func (p *ParticipantImpl) SetAttributes(attrs map[string]string) { p.requireBroadcast = true // already checked above p.dirty.Store(true) - onParticipantUpdate := p.onParticipantUpdate onClaimsChanged := p.onClaimsChanged p.lock.Unlock() - if onParticipantUpdate != nil { - onParticipantUpdate(p) - } + p.listener().OnParticipantUpdate(p) + if onClaimsChanged != nil { onClaimsChanged(p) } @@ -583,7 +803,6 @@ func (p *ParticipantImpl) SetPermission(permission *livekit.ParticipantPermissio canPublish := grants.Video.GetCanPublish() canSubscribe := grants.Video.GetCanSubscribe() - onParticipantUpdate := p.onParticipantUpdate onClaimsChanged := p.onClaimsChanged isPublisher := canPublish && p.TransportManager.IsPublisherEstablished() @@ -607,12 +826,17 @@ func (p *ParticipantImpl) SetPermission(permission *livekit.ParticipantPermissio } } + if !grants.Video.GetCanPublishData() { + for _, dt := range p.UpDataTrackManager.GetPublishedDataTracks() { + p.UpDataTrackManager.RemovePublishedDataTrack(dt) + } + } + // update isPublisher attribute p.isPublisher.Store(isPublisher) - if onParticipantUpdate != nil { - onParticipantUpdate(p) - } + p.listener().OnParticipantUpdate(p) + if onClaimsChanged != nil { onClaimsChanged(p) } @@ -652,8 +876,12 @@ func (p *ParticipantImpl) ToProtoWithVersion() (*livekit.ParticipantInfo, utils. v := p.version.Load() piv := p.timedVersion + var clientProtocol int32 + if p.params.ClientInfo.ClientInfo != nil { + clientProtocol = p.params.ClientInfo.ClientInfo.GetClientProtocol() + } pi := &livekit.ParticipantInfo{ - Sid: string(p.params.SID), + Sid: string(p.ID()), Identity: string(p.params.Identity), Name: grants.Name, State: p.State(), @@ -666,7 +894,9 @@ func (p *ParticipantImpl) ToProtoWithVersion() (*livekit.ParticipantInfo, utils. Region: p.params.Region, IsPublisher: p.IsPublisher(), Kind: grants.GetParticipantKind(), + KindDetails: grants.GetKindDetails(), DisconnectReason: p.CloseReason().ToDisconnectReason(), + ClientProtocol: clientProtocol, } p.lock.RUnlock() @@ -700,6 +930,8 @@ func (p *ParticipantImpl) ToProtoWithVersion() (*livekit.ParticipantInfo, utils. } p.pendingTracksLock.RUnlock() + pi.DataTracks = p.UpDataTrackManager.ToProto() + return pi, piv } @@ -708,101 +940,33 @@ func (p *ParticipantImpl) ToProto() *livekit.ParticipantInfo { return pi } -// callbacks for clients - -func (p *ParticipantImpl) OnTrackPublished(callback func(types.LocalParticipant, types.MediaTrack)) { - p.lock.Lock() - p.onTrackPublished = callback - p.lock.Unlock() -} - -func (p *ParticipantImpl) getOnTrackPublished() func(types.LocalParticipant, types.MediaTrack) { +func (p *ParticipantImpl) TelemetryGuard() *telemetry.ReferenceGuard { p.lock.RLock() defer p.lock.RUnlock() - return p.onTrackPublished + + return p.telemetryGuard } -func (p *ParticipantImpl) OnTrackUnpublished(callback func(types.LocalParticipant, types.MediaTrack)) { +func (p *ParticipantImpl) GetTelemetryListener() types.ParticipantTelemetryListener { + if p.params.TelemetryListener == nil { + return &types.NullParticipantTelemetryListener{} + } + + return p.params.TelemetryListener +} + +func (p *ParticipantImpl) AddOnClose(key string, callback func(types.LocalParticipant)) { + if p.isClosed.Load() { + go callback(p) + return + } + p.lock.Lock() - p.onTrackUnpublished = callback - p.lock.Unlock() -} - -func (p *ParticipantImpl) getOnTrackUnpublished() func(types.LocalParticipant, types.MediaTrack) { - p.lock.RLock() - defer p.lock.RUnlock() - return p.onTrackUnpublished -} - -func (p *ParticipantImpl) OnStateChange(callback func(p types.LocalParticipant, state livekit.ParticipantInfo_State)) { - p.lock.Lock() - p.onStateChange = callback - p.lock.Unlock() -} - -func (p *ParticipantImpl) getOnStateChange() func(p types.LocalParticipant, state livekit.ParticipantInfo_State) { - p.lock.RLock() - defer p.lock.RUnlock() - return p.onStateChange -} - -func (p *ParticipantImpl) OnMigrateStateChange(callback func(p types.LocalParticipant, state types.MigrateState)) { - p.lock.Lock() - p.onMigrateStateChange = callback - p.lock.Unlock() -} - -func (p *ParticipantImpl) getOnMigrateStateChange() func(p types.LocalParticipant, state types.MigrateState) { - p.lock.RLock() - defer p.lock.RUnlock() - return p.onMigrateStateChange -} - -func (p *ParticipantImpl) OnTrackUpdated(callback func(types.LocalParticipant, types.MediaTrack)) { - p.lock.Lock() - p.onTrackUpdated = callback - p.lock.Unlock() -} - -func (p *ParticipantImpl) getOnTrackUpdated() func(types.LocalParticipant, types.MediaTrack) { - p.lock.RLock() - defer p.lock.RUnlock() - return p.onTrackUpdated -} - -func (p *ParticipantImpl) OnParticipantUpdate(callback func(types.LocalParticipant)) { - p.lock.Lock() - p.onParticipantUpdate = callback - p.lock.Unlock() -} - -func (p *ParticipantImpl) OnDataPacket(callback func(types.LocalParticipant, livekit.DataPacket_Kind, *livekit.DataPacket)) { - p.lock.Lock() - p.onDataPacket = callback - p.lock.Unlock() -} - -func (p *ParticipantImpl) getOnDataPacket() func(types.LocalParticipant, livekit.DataPacket_Kind, *livekit.DataPacket) { - p.lock.RLock() - defer p.lock.RUnlock() - return p.onDataPacket -} - -func (p *ParticipantImpl) OnMetrics(callback func(types.Participant, *livekit.DataPacket)) { - p.lock.Lock() - p.onMetrics = callback - p.lock.Unlock() -} - -func (p *ParticipantImpl) getOnMetrics() func(types.Participant, *livekit.DataPacket) { - p.lock.RLock() - defer p.lock.RUnlock() - return p.onMetrics -} - -func (p *ParticipantImpl) OnClose(callback func(types.LocalParticipant)) { - p.lock.Lock() - p.onClose = callback + if callback == nil { + delete(p.onClose, key) + } else { + p.onClose[key] = callback + } p.lock.Unlock() } @@ -820,111 +984,268 @@ func (p *ParticipantImpl) HandleSignalSourceClose() { } } -func (p *ParticipantImpl) synthesizeAddTrackRequests(offer webrtc.SessionDescription) error { - parsed, err := offer.Unmarshal() - if err != nil { - return err - } - - for _, m := range parsed.MediaDescriptions { - if !strings.EqualFold(m.MediaName.Media, "audio") { - // ONE-SHOT-SIGNALLING-MODE-TODO: support video +func (p *ParticipantImpl) synthesizeAddTrackRequests(parsedOffer *sdp.SessionDescription) error { + for _, m := range parsedOffer.MediaDescriptions { + if !strings.EqualFold(m.MediaName.Media, "audio") && !strings.EqualFold(m.MediaName.Media, "video") { continue } - trackID := "" - - msid, ok := m.Attribute(sdp.AttrKeyMsid) - if ok { - if split := strings.Split(msid, " "); len(split) == 2 { - trackID = split[1] - } + cid := protosdp.GetMediaStreamTrack(m) + if cid == "" { + cid = guid.New(utils.TrackPrefix) } - if trackID == "" { - attr, ok := m.Attribute(sdp.AttrKeySSRC) - if ok { - split := strings.Split(attr, " ") - if len(split) == 3 && strings.HasPrefix(split[1], "msid:") { - trackID = split[2] - } - } - } + rids, ridsOk := protosdp.GetSimulcastRids(m) - if trackID == "" { - trackID = guid.New(utils.TrackPrefix) + var ( + name string + trackSource livekit.TrackSource + trackType livekit.TrackType + ) + if strings.EqualFold(m.MediaName.Media, "audio") { + name = "synthesized-microphone" + trackSource = livekit.TrackSource_MICROPHONE + trackType = livekit.TrackType_AUDIO + } else { + name = "synthesized-camera" + trackSource = livekit.TrackSource_CAMERA + trackType = livekit.TrackType_VIDEO } - req := &livekit.AddTrackRequest{ - Cid: trackID, - Name: "synthesized-microphone", - Source: livekit.TrackSource_MICROPHONE, - Type: livekit.TrackType_AUDIO, + Cid: cid, + Name: name, + Source: trackSource, + Type: trackType, DisableDtx: true, Stereo: false, Stream: "camera", } + if strings.EqualFold(m.MediaName.Media, "video") { + if ridsOk { + // add simulcast layers, NOTE: only quality can be set as dimensions/fps is not available + n := min(len(rids), int(buffer.DefaultMaxLayerSpatial)+1) + for i := range n { + // WARN: casting int -> protobuf enum + req.Layers = append(req.Layers, &livekit.VideoLayer{Quality: livekit.VideoQuality(i)}) + } + } else { + // dummy layer to ensure at least one layer is available + req.Layers = []*livekit.VideoLayer{{}} + } + } p.AddTrack(req) } return nil } +func (p *ParticipantImpl) updateRidsFromSDP(parsed *sdp.SessionDescription, unmatchVideos []*sdp.MediaDescription) { + for _, m := range parsed.MediaDescriptions { + if m.MediaName.Media != "video" || !slices.Contains(unmatchVideos, m) { + continue + } + + mst := protosdp.GetMediaStreamTrack(m) + if mst == "" { + continue + } + + getRids := func(inRids buffer.VideoLayersRid) buffer.VideoLayersRid { + var outRids buffer.VideoLayersRid + rids, ok := protosdp.GetSimulcastRids(m) + if ok { + n := min(len(rids), len(inRids)) + for i := range n { + // disabled layers will have a `~` prefix, remove it while determining actual rid + if len(rids[i]) != 0 && rids[i][0] == '~' { + outRids[i] = rids[i][1:] + } else { + outRids[i] = rids[i] + } + } + for i := n; i < len(inRids); i++ { + outRids[i] = "" + } + outRids = buffer.NormalizeVideoLayersRid(outRids) + } else { + for i := range len(inRids) { + outRids[i] = "" + } + } + + return outRids + } + + p.pendingTracksLock.Lock() + pti := p.getPendingTrackPrimaryBySdpCid(mst) + if pti != nil { + pti.sdpRids = getRids(pti.sdpRids) + p.pubLogger.Debugw( + "pending track rids updated", + "trackID", pti.trackInfos[0].Sid, + "pendingTrack", pti, + ) + + ti := pti.trackInfos[0] + for _, codec := range ti.Codecs { + if codec.Cid == mst || codec.SdpCid == mst { + mimeType := mime.NormalizeMimeType(codec.MimeType) + for _, layer := range codec.Layers { + layer.SpatialLayer = buffer.VideoQualityToSpatialLayer(mimeType, layer.Quality, ti) + layer.Rid = buffer.VideoQualityToRid(mimeType, layer.Quality, ti, pti.sdpRids) + } + } + } + } + p.pendingTracksLock.Unlock() + + if pti == nil { + // track could already be published, but this could be back up codec offer, + // so check in published tracks also + mt := p.getPublishedTrackBySdpCid(mst) + if mt != nil { + mimeType := mt.(*MediaTrack).GetMimeTypeForSdpCid(mst) + if mimeType != mime.MimeTypeUnknown { + rids := getRids(buffer.DefaultVideoLayersRid) + mt.(*MediaTrack).UpdateCodecRids(mimeType, rids) + p.pubLogger.Debugw( + "published track rids updated", + "trackID", mt.ID(), + "mime", mimeType, + "track", logger.Proto(mt.ToProto()), + ) + } else { + p.pubLogger.Warnw( + "could not get mime type for sdp cid", nil, + "trackID", mt.ID(), + "sdpCid", mst, + "track", logger.Proto(mt.ToProto()), + ) + } + } + } + } +} + +func (p *ParticipantImpl) HandleICETrickle(trickleRequest *livekit.TrickleRequest) { + candidateInit, err := protosignalling.FromProtoTrickle(trickleRequest) + if err != nil { + p.params.Logger.Warnw("could not decode trickle", err) + p.sendRequestResponse(&livekit.RequestResponse{ + Reason: livekit.RequestResponse_UNCLASSIFIED_ERROR, + Message: err.Error(), + Request: &livekit.RequestResponse_Trickle{ + Trickle: utils.CloneProto(trickleRequest), + }, + }) + return + } + + p.TransportManager.AddICECandidate(candidateInit, trickleRequest.Target) +} + // HandleOffer an offer from remote participant, used when clients make the initial connection -func (p *ParticipantImpl) HandleOffer(offer webrtc.SessionDescription) error { - p.pubLogger.Debugw("received offer", "transport", livekit.SignalTarget_PUBLISHER, "offer", offer) +func (p *ParticipantImpl) HandleOffer(sd *livekit.SessionDescription) error { + offer, offerId, _ := protosignalling.FromProtoSessionDescription(sd) + lgr := p.pubLogger.WithUnlikelyValues( + "transport", livekit.SignalTarget_PUBLISHER, + "offer", offer, + "offerId", offerId, + ) + + lgr.Debugw("received offer") + + parsedOffer, err := offer.Unmarshal() + if err != nil { + lgr.Warnw("could not parse offer", err) + return err + } if p.params.UseOneShotSignallingMode { - if err := p.synthesizeAddTrackRequests(offer); err != nil { + if err := p.synthesizeAddTrackRequests(parsedOffer); err != nil { + lgr.Warnw("could not synthesize add track requests", err) return err } } - shouldPend := false - if p.MigrateState() == types.MigrateStateInit { - shouldPend = true + err = p.TransportManager.HandleOffer(offer, offerId, p.MigrateState() == types.MigrateStateInit) + if err != nil { + lgr.Warnw("could not handle offer", err, "mungedOffer", offer) + return err } - offer = p.setCodecPreferencesForPublisher(offer) - err := p.TransportManager.HandleOffer(offer, shouldPend) if p.params.UseOneShotSignallingMode { - p.updateState(livekit.ParticipantInfo_ACTIVE) + go p.listener().OnSubscriberReady(p) } - return err + + p.handlePendingRemoteTracks() + return nil } -func (p *ParticipantImpl) onPublisherAnswer(answer webrtc.SessionDescription) error { +func (p *ParticipantImpl) onPublisherSetRemoteDescription() { + offer := p.TransportManager.LastPublisherOfferPending() + if offer == nil { + return + } + parsedOffer, err := offer.Unmarshal() + if err != nil { + p.pubLogger.Warnw("could not parse offer", err) + return + } + + // set publish codec preferences after remote description is set + // and required transceivers are created + unmatchAudios, unmatchVideos := p.populateSdpCid(parsedOffer) + p.setCodecPreferencesForPublisher(parsedOffer, unmatchAudios, unmatchVideos) + p.updateRidsFromSDP(parsedOffer, unmatchVideos) +} + +func (p *ParticipantImpl) onPublisherAnswer(answer webrtc.SessionDescription, answerId uint32, midToTrackID map[string]string) error { if p.IsClosed() || p.IsDisconnected() { return nil } answer = p.configurePublisherAnswer(answer) - p.pubLogger.Debugw("sending answer", "transport", livekit.SignalTarget_PUBLISHER, "answer", answer) - return p.writeMessage(&livekit.SignalResponse{ - Message: &livekit.SignalResponse_Answer{ - Answer: ToProtoSessionDescription(answer), - }, - }) + p.pubLogger.Debugw( + "sending answer", + "transport", livekit.SignalTarget_PUBLISHER, + "answer", answer, + "answerId", answerId, + "midToTrackID", midToTrackID, + ) + + return p.sendSdpAnswer(answer, answerId, midToTrackID) } -func (p *ParticipantImpl) GetAnswer() (webrtc.SessionDescription, error) { +func (p *ParticipantImpl) GetAnswer() (webrtc.SessionDescription, uint32, error) { if p.IsClosed() || p.IsDisconnected() { - return webrtc.SessionDescription{}, ErrParticipantSessionClosed + return webrtc.SessionDescription{}, 0, ErrParticipantSessionClosed } - answer, err := p.TransportManager.GetAnswer() + answer, answerId, err := p.TransportManager.GetAnswer() if err != nil { - return answer, err + return answer, answerId, err } answer = p.configurePublisherAnswer(answer) - p.pubLogger.Debugw("returning answer", "transport", livekit.SignalTarget_PUBLISHER, "answer", answer) - return answer, nil + p.pubLogger.Debugw( + "returning answer", + "transport", livekit.SignalTarget_PUBLISHER, + "answer", answer, + "answerId", answerId, + ) + return answer, answerId, nil } // HandleAnswer handles a client answer response, with subscriber PC, server initiates the // offer and client answers -func (p *ParticipantImpl) HandleAnswer(answer webrtc.SessionDescription) { - p.subLogger.Debugw("received answer", "transport", livekit.SignalTarget_SUBSCRIBER, "answer", answer) +func (p *ParticipantImpl) HandleAnswer(sd *livekit.SessionDescription) { + answer, answerId, _ := protosignalling.FromProtoSessionDescription(sd) + p.subLogger.Debugw( + "received answer", + "transport", livekit.SignalTarget_SUBSCRIBER, + "answer", answer, + "answerId", answerId, + ) /* from server received join request to client answer * 1. server send join response & offer @@ -934,10 +1255,10 @@ func (p *ParticipantImpl) HandleAnswer(answer webrtc.SessionDescription) { signalConnCost := time.Since(p.ConnectedAt()).Milliseconds() p.TransportManager.UpdateSignalingRTT(uint32(signalConnCost)) - p.TransportManager.HandleAnswer(answer) + p.TransportManager.HandleAnswer(answer, answerId) } -func (p *ParticipantImpl) handleMigrateTracks() { +func (p *ParticipantImpl) handleMigrateTracks() []*MediaTrack { // muted track won't send rtp packet, so it is required to add mediatrack manually. // But, synthesising track publish for unmuted tracks keeps a consistent path. // In both cases (muted and unmuted), when publisher sends media packets, OnTrack would register and go from there. @@ -949,7 +1270,12 @@ func (p *ParticipantImpl) handleMigrateTracks() { } if len(pti.trackInfos) > 1 { - p.pubLogger.Warnw("too many pending migrated tracks", nil, "trackID", pti.trackInfos[0].Sid, "count", len(pti.trackInfos), "cid", cid) + p.pubLogger.Warnw( + "too many pending migrated tracks", nil, + "trackID", pti.trackInfos[0].Sid, + "count", len(pti.trackInfos), + "cid", cid, + ) } mt := p.addMigratedTrack(cid, pti.trackInfos[0]) @@ -959,7 +1285,7 @@ func (p *ParticipantImpl) handleMigrateTracks() { p.pubLogger.Warnw("could not find migrated track, migration failed", nil, "cid", cid) p.pendingTracksLock.Unlock() p.IssueFullReconnect(types.ParticipantCloseReasonMigrateCodecMismatch) - return + return nil } } @@ -968,20 +1294,32 @@ func (p *ParticipantImpl) handleMigrateTracks() { } p.pendingTracksLock.Unlock() - // launch callbacks in goroutine since they could block. - // callbacks handle webhooks as well as db persistence - go func() { - for _, t := range addedTracks { - p.handleTrackPublished(t) - } - }() + return addedTracks } // AddTrack is called when client intends to publish track. // records track details and lets client know it's ok to proceed func (p *ParticipantImpl) AddTrack(req *livekit.AddTrackRequest) { + p.params.Logger.Debugw("add track request", "trackID", req.Cid, "request", logger.Proto(req)) if !p.CanPublishSource(req.Source) { - p.pubLogger.Warnw("no permission to publish track", nil) + p.pubLogger.Warnw("no permission to publish track", nil, "trackID", req.Sid, "kind", req.Type) + p.sendRequestResponse(&livekit.RequestResponse{ + Reason: livekit.RequestResponse_NOT_ALLOWED, + Request: &livekit.RequestResponse_AddTrack{ + AddTrack: utils.CloneProto(req), + }, + }) + return + } + + if req.Type != livekit.TrackType_AUDIO && req.Type != livekit.TrackType_VIDEO { + p.pubLogger.Warnw("unsupported track type", nil, "trackID", req.Sid, "kind", req.Type) + p.sendRequestResponse(&livekit.RequestResponse{ + Reason: livekit.RequestResponse_UNSUPPORTED_TYPE, + Request: &livekit.RequestResponse_AddTrack{ + AddTrack: utils.CloneProto(req), + }, + }) return } @@ -1001,6 +1339,8 @@ func (p *ParticipantImpl) SetMigrateInfo( previousOffer, previousAnswer *webrtc.SessionDescription, mediaTracks []*livekit.TrackPublishedResponse, dataChannels []*livekit.DataChannelInfo, + dataChannelReceiveState []*livekit.DataChannelReceiveState, + dataTracks []*livekit.PublishDataTrackResponse, ) { p.pendingTracksLock.Lock() for _, t := range mediaTracks { @@ -1011,15 +1351,52 @@ func (p *ParticipantImpl) SetMigrateInfo( p.supervisor.SetPublicationMute(livekit.TrackID(ti.Sid), ti.Muted) } - p.pendingTracks[t.GetCid()] = &pendingTrackInfo{trackInfos: []*livekit.TrackInfo{ti}, migrated: true, createdAt: time.Now()} - p.pubLogger.Infow("pending track added (migration)", "trackID", ti.Sid, "track", logger.Proto(ti)) + p.pendingTracks[t.GetCid()] = &pendingTrackInfo{ + trackInfos: []*livekit.TrackInfo{ti}, + migrated: true, + createdAt: time.Now(), + } + p.pubLogger.Infow( + "pending track added (migration)", + "trackID", ti.Sid, + "cid", t.GetCid(), + "pendingTrack", p.pendingTracks[t.GetCid()], + ) } p.pendingTracksLock.Unlock() - if len(mediaTracks) != 0 { + for _, t := range dataTracks { + dti := t.GetInfo() + dt := NewDataTrack( + DataTrackParams{ + Logger: p.params.Logger.WithValues("trackID", dti.Sid), + ParticipantID: p.ID, + ParticipantIdentity: p.params.Identity, + BytesTrackStats: NewBytesTrackStats( + p.params.Country, + livekit.TrackID(dti.Sid), + p.ID(), + p.Kind(), + p.KindDetails(), + p.params.TelemetryListener, + p.params.Reporter, + ), + }, + dti, + ) + p.UpDataTrackManager.AddPublishedDataTrack(dt) + } + + if len(mediaTracks) != 0 || len(dataTracks) != 0 { p.setIsPublisher(true) } + p.reliableDataInfo.joiningMessageLock.Lock() + for _, state := range dataChannelReceiveState { + p.reliableDataInfo.joiningMessageFirstSeqs[livekit.ParticipantID(state.PublisherSid)] = state.LastSeq + 1 + } + p.reliableDataInfo.joiningMessageLock.Unlock() + p.TransportManager.SetMigrateInfo(previousOffer, previousAnswer, dataChannels) } @@ -1033,11 +1410,18 @@ func (p *ParticipantImpl) Close(sendLeave bool, reason types.ParticipantCloseRea return nil } + var sessionDuration time.Duration + if activeAt := p.ActiveAt(); !activeAt.IsZero() { + sessionDuration = time.Since(activeAt) + } p.params.Logger.Infow( "participant closing", "sendLeave", sendLeave, "reason", reason.String(), "isExpectedToResume", isExpectedToResume, + "clientInfo", logger.Proto(sutils.ClientInfoWithoutAddress(p.GetClientInfo())), + "kind", p.Kind(), + "sessionDuration", sessionDuration, ) p.closeReason.Store(reason) p.clearDisconnectTimer() @@ -1057,11 +1441,26 @@ func (p *ParticipantImpl) Close(sendLeave bool, reason types.ParticipantCloseRea } p.pendingTracksLock.Lock() + for _, pti := range p.pendingTracks { + if len(pti.trackInfos) == 0 { + continue + } + prometheus.RecordTrackPublishCancels(pti.trackInfos[0].Type.String(), int32(len(pti.trackInfos))) + } p.pendingTracks = make(map[string]*pendingTrackInfo) p.pendingPublishingTracks = make(map[livekit.TrackID]*pendingTrackInfo) p.pendingTracksLock.Unlock() p.UpTrackManager.Close(isExpectedToResume) + p.UpDataTrackManager.Close() + + p.rpcLock.Lock() + clear(p.rpcPendingAcks) + for _, handler := range p.rpcPendingResponses { + handler.Resolve("", utils.DataChannelRpcErrorFromBuiltInCodes(utils.DataChannelRpcRecipientDisconnected, "")) + } + p.rpcPendingResponses = make(map[string]*utils.DataChannelRpcPendingResponseHandler) + p.rpcLock.Unlock() p.updateState(livekit.ParticipantInfo_DISCONNECTED) close(p.disconnected) @@ -1069,10 +1468,10 @@ func (p *ParticipantImpl) Close(sendLeave bool, reason types.ParticipantCloseRea // ensure this is synchronized p.CloseSignalConnection(types.SignallingCloseReasonParticipantClose) p.lock.RLock() - onClose := p.onClose + onClose := slices.Collect(maps.Values(p.onClose)) p.lock.RUnlock() - if onClose != nil { - onClose(p) + for _, cb := range onClose { + cb(p) } // Close peer connections without blocking participant Close. If peer connections are gathering candidates @@ -1119,6 +1518,10 @@ func (p *ParticipantImpl) clearMigrationTimer() { } func (p *ParticipantImpl) setupMigrationTimerLocked() { + if p.params.UseSinglePeerConnection { + return + } + // // On subscriber peer connection, remote side will try ICE on both // pre- and post-migration ICE candidates as the migrating out @@ -1151,7 +1554,7 @@ func (p *ParticipantImpl) setupMigrationTimerLocked() { } func (p *ParticipantImpl) MaybeStartMigration(force bool, onStart func()) bool { - if p.params.UseOneShotSignallingMode { + if p.IsClosed() || p.params.UseOneShotSignallingMode { return false } @@ -1203,8 +1606,9 @@ func (p *ParticipantImpl) SetMigrateState(s types.MigrateState) { } p.params.Logger.Debugw("SetMigrateState", "state", s) + var migratedTracks []*MediaTrack if s == types.MigrateStateComplete { - p.handleMigrateTracks() + migratedTracks = p.handleMigrateTracks() } p.migrateState.Store(s) p.dirty.Store(true) @@ -1214,14 +1618,58 @@ func (p *ParticipantImpl) SetMigrateState(s types.MigrateState) { p.TransportManager.ProcessPendingPublisherOffer() case types.MigrateStateComplete: - p.params.Logger.Infow("migration complete") + if preState == types.MigrateStateSync { + p.params.Logger.Infow("migration complete") + + if p.params.LastPubReliableSeq > 0 { + p.reliableDataInfo.migrateInPubDataCache.Store(NewMigrationDataCache(p.params.LastPubReliableSeq, time.Now().Add(migrationWaitContinuousMsgDuration))) + } + } p.TransportManager.ProcessPendingPublisherDataChannels() go p.cacheForwarderState() } - if onMigrateStateChange := p.getOnMigrateStateChange(); onMigrateStateChange != nil { - go onMigrateStateChange(p, s) - } + go func() { + // launch callbacks in goroutine since they could block. + // callbacks handle webhooks as well as db persistence + for _, t := range migratedTracks { + p.handleTrackPublished(t, true) + } + + if s == types.MigrateStateComplete { + // wait for all migrated track to be published, + // it is possible that synthesized track publish above could + // race with actual publish from client and the above synthesized + // one could actually be a no-op because the actual publish path is active. + // + // if the actual publish path has not finished, the migration state change + // callback could close the remote participant/tracks before the local track + // is fully active. + // + // that could lead to subscribers unsubscribing due to source + // track going away, i. e. in this case, the remote track close would have + // notified the subscription manager, the subscription manager would + // re-resolve to check if the track is still active and unsubscribe if none + // is active, as local track is in the process of completing publish, + // the check would have resolved to an empty track leading to unsubscription. + go func() { + startTime := time.Now() + for { + if !p.hasPendingMigratedTrack() || p.IsDisconnected() || time.Since(startTime) > 15*time.Second { + // a time out just to be safe, but it should not be needed + p.migratedTracksPublishedFuse.Break() + return + } + + time.Sleep(20 * time.Millisecond) + } + }() + + <-p.migratedTracksPublishedFuse.Watch() + } + + p.listener().OnMigrateStateChange(p, s) + }() } func (p *ParticipantImpl) MigrateState() types.MigrateState { @@ -1344,10 +1792,8 @@ func (p *ParticipantImpl) VerifySubscribeParticipantInfo(pID livekit.Participant return } - if f := p.params.GetParticipantInfo; f != nil { - if info := f(pID); info != nil { - _ = p.SendParticipantUpdate([]*livekit.ParticipantInfo{info}) - } + if info := p.helper().GetParticipantInfo(pID); info != nil { + _ = p.SendParticipantUpdate([]*livekit.ParticipantInfo{info}) } } @@ -1384,22 +1830,6 @@ func (p *ParticipantImpl) onTrackUnsubscribed(subTrack types.SubscribedTrack) { p.TransportManager.RemoveSubscribedTrack(subTrack) } -func (p *ParticipantImpl) SubscriptionPermissionUpdate(publisherID livekit.ParticipantID, trackID livekit.TrackID, allowed bool) { - p.subLogger.Debugw("sending subscription permission update", "publisherID", publisherID, "trackID", trackID, "allowed", allowed) - err := p.writeMessage(&livekit.SignalResponse{ - Message: &livekit.SignalResponse_SubscriptionPermissionUpdate{ - SubscriptionPermissionUpdate: &livekit.SubscriptionPermissionUpdate{ - ParticipantSid: string(publisherID), - TrackSid: string(trackID), - Allowed: allowed, - }, - }, - }) - if err != nil { - p.subLogger.Errorw("could not send subscription permission update", err) - } -} - func (p *ParticipantImpl) UpdateMediaRTT(rtt uint32) { now := time.Now() p.lock.Lock() @@ -1419,6 +1849,8 @@ func (p *ParticipantImpl) UpdateMediaRTT(rtt uint32) { // ---------------------------------------------------------- +var _ transport.Handler = (*AnyTransportHandler)(nil) + type AnyTransportHandler struct { transport.UnimplementedHandler p *ParticipantImpl @@ -1442,8 +1874,12 @@ type PublisherTransportHandler struct { AnyTransportHandler } -func (h PublisherTransportHandler) OnAnswer(sd webrtc.SessionDescription) error { - return h.p.onPublisherAnswer(sd) +func (h PublisherTransportHandler) OnSetRemoteDescriptionOffer() { + h.p.onPublisherSetRemoteDescription() +} + +func (h PublisherTransportHandler) OnAnswer(sd webrtc.SessionDescription, answerId uint32, midToTrackID map[string]string) error { + return h.p.onPublisherAnswer(sd, answerId, midToTrackID) } func (h PublisherTransportHandler) OnTrack(track *webrtc.TrackRemote, rtpReceiver *webrtc.RTPReceiver) { @@ -1454,8 +1890,24 @@ func (h PublisherTransportHandler) OnInitialConnected() { h.p.onPublisherInitialConnected() } -func (h PublisherTransportHandler) OnDataPacket(kind livekit.DataPacket_Kind, data []byte) { - h.p.onDataMessage(kind, data) +func (h PublisherTransportHandler) OnDataMessage(kind livekit.DataPacket_Kind, data []byte) { + h.p.onReceivedDataMessage(kind, data) +} + +func (h PublisherTransportHandler) OnDataMessageUnlabeled(data []byte) { + h.p.onReceivedDataMessageUnlabeled(data) +} + +func (h PublisherTransportHandler) OnDataTrackMessage(data []byte, arrivalTime int64) { + h.p.onReceivedDataTrackMessage(data, arrivalTime) +} + +func (h PublisherTransportHandler) OnDataSendError(err error) { + h.p.onDataSendError(err) +} + +func (h PublisherTransportHandler) OnUnmatchedMedia(numAudios uint32, numVideos uint32) error { + return h.p.sendMediaSectionsRequirement(numAudios, numVideos) } // ---------------------------------------------------------- @@ -1464,8 +1916,8 @@ type SubscriberTransportHandler struct { AnyTransportHandler } -func (h SubscriberTransportHandler) OnOffer(sd webrtc.SessionDescription) error { - return h.p.onSubscriberOffer(sd) +func (h SubscriberTransportHandler) OnOffer(sd webrtc.SessionDescription, offerId uint32, midToTrackID map[string]string) error { + return h.p.onSubscriberOffer(sd, offerId, midToTrackID) } func (h SubscriberTransportHandler) OnStreamStateChange(update *streamallocator.StreamStateUpdate) error { @@ -1498,6 +1950,20 @@ func (h PrimaryTransportHandler) OnFullyEstablished() { // ---------------------------------------------------------- +func (p *ParticipantImpl) setupSignalling() { + p.signalling = signalling.NewSignalling(signalling.SignallingParams{ + Logger: p.params.Logger, + }) + p.signalHandler = signalling.NewSignalHandler(signalling.SignalHandlerParams{ + Logger: p.params.Logger, + Participant: p, + }) + p.signaller = signalling.NewSignallerAsync(signalling.SignallerAsyncParams{ + Logger: p.params.Logger, + Participant: p, + }) +} + func (p *ParticipantImpl) setupTransportManager() error { p.twcc = twcc.NewTransportWideCCResponder() p.twcc.OnFeedback(func(pkts []rtcp.Packet) { @@ -1507,7 +1973,7 @@ func (p *ParticipantImpl) setupTransportManager() error { var pth transport.Handler = PublisherTransportHandler{ath} var sth transport.Handler = SubscriberTransportHandler{ath} - subscriberAsPrimary := p.ProtocolVersion().SubscriberAsPrimary() && p.CanSubscribe() && !p.params.UseOneShotSignallingMode + subscriberAsPrimary := !p.params.UseOneShotSignallingMode && (p.ProtocolVersion().SubscriberAsPrimary() && p.CanSubscribe()) && !p.params.UseSinglePeerConnection if subscriberAsPrimary { sth = PrimaryTransportHandler{sth, p} } else { @@ -1515,33 +1981,34 @@ func (p *ParticipantImpl) setupTransportManager() error { } params := TransportManagerParams{ - Identity: p.params.Identity, - SID: p.params.SID, // primary connection does not change, canSubscribe can change if permission was updated // after the participant has joined - SubscriberAsPrimary: subscriberAsPrimary, - Config: p.params.Config, - Twcc: p.twcc, - ProtocolVersion: p.params.ProtocolVersion, - CongestionControlConfig: p.params.CongestionControlConfig, - EnabledPublishCodecs: p.enabledPublishCodecs, - EnabledSubscribeCodecs: p.enabledSubscribeCodecs, - SimTracks: p.params.SimTracks, - ClientInfo: p.params.ClientInfo, - Migration: p.params.Migration, - AllowTCPFallback: p.params.AllowTCPFallback, - TCPFallbackRTTThreshold: p.params.TCPFallbackRTTThreshold, - AllowUDPUnstableFallback: p.params.AllowUDPUnstableFallback, - TURNSEnabled: p.params.TURNSEnabled, - AllowPlayoutDelay: p.params.PlayoutDelay.GetEnabled(), - DataChannelMaxBufferedAmount: p.params.DataChannelMaxBufferedAmount, - DatachannelSlowThreshold: p.params.DatachannelSlowThreshold, - Logger: p.params.Logger.WithComponent(sutils.ComponentTransport), - PublisherHandler: pth, - SubscriberHandler: sth, - DataChannelStats: p.dataChannelStats, - UseOneShotSignallingMode: p.params.UseOneShotSignallingMode, - FireOnTrackBySdp: p.params.FireOnTrackBySdp, + SubscriberAsPrimary: subscriberAsPrimary, + UseSinglePeerConnection: p.params.UseSinglePeerConnection, + Config: p.params.Config, + Twcc: p.twcc, + ProtocolVersion: p.params.ProtocolVersion, + CongestionControlConfig: p.params.CongestionControlConfig, + EnabledPublishCodecs: p.enabledPublishCodecs, + EnabledSubscribeCodecs: p.enabledSubscribeCodecs, + SimTracks: p.params.SimTracks, + ClientInfo: p.params.ClientInfo, + Migration: p.params.Migration, + AllowTCPFallback: p.params.AllowTCPFallback, + TCPFallbackRTTThreshold: p.params.TCPFallbackRTTThreshold, + AllowUDPUnstableFallback: p.params.AllowUDPUnstableFallback, + TURNSEnabled: p.params.TURNSEnabled, + AllowPlayoutDelay: p.params.PlayoutDelay.GetEnabled(), + DataChannelMaxBufferedAmount: p.params.DataChannelMaxBufferedAmount, + DatachannelSlowThreshold: p.params.DatachannelSlowThreshold, + DatachannelLossyTargetLatency: p.params.DatachannelLossyTargetLatency, + Logger: p.params.Logger.WithComponent(sutils.ComponentTransport), + PublisherHandler: pth, + SubscriberHandler: sth, + DataChannelStats: p.dataChannelStats, + UseOneShotSignallingMode: p.params.UseOneShotSignallingMode, + FireOnTrackBySdp: p.params.FireOnTrackBySdp, + EnableDataTracks: p.params.EnableDataTracks, } if p.params.SyncStreams && p.params.PlayoutDelay.GetEnabled() && p.params.ClientInfo.isFirefox() { // we will disable playout delay for Firefox if the user is expecting @@ -1586,20 +2053,30 @@ func (p *ParticipantImpl) setupUpTrackManager() { p.UpTrackManager.OnPublishedTrackUpdated(func(track types.MediaTrack) { p.dirty.Store(true) - if onTrackUpdated := p.getOnTrackUpdated(); onTrackUpdated != nil { - onTrackUpdated(p, track) - } + p.listener().OnTrackUpdated(p, track) }) p.UpTrackManager.OnUpTrackManagerClose(p.onUpTrackManagerClose) } +func (p *ParticipantImpl) setupUpDataTrackManager() { + p.UpDataTrackManager = NewUpDataTrackManager(UpDataTrackManagerParams{ + Logger: p.pubLogger, + Participant: p, + }) +} + func (p *ParticipantImpl) setupSubscriptionManager() { p.SubscriptionManager = NewSubscriptionManager(SubscriptionManagerParams{ - Participant: p, - Logger: p.subLogger.WithoutSampler(), - TrackResolver: p.params.TrackResolver, - Telemetry: p.params.Telemetry, + Participant: p, + Logger: p.subLogger.WithoutSampler(), + TrackResolver: func(lp types.LocalParticipant, ti livekit.TrackID) types.MediaResolverResult { + return p.helper().ResolveMediaTrack(lp, ti) + }, + DataTrackResolver: func(lp types.LocalParticipant, ti livekit.TrackID) types.DataResolverResult { + return p.helper().ResolveDataTrack(lp, ti) + }, + TelemetryListener: p.params.TelemetryListener, OnTrackSubscribed: p.onTrackSubscribed, OnTrackUnsubscribed: p.onTrackUnsubscribed, OnSubscriptionError: p.onSubscriptionError, @@ -1622,14 +2099,12 @@ func (p *ParticipantImpl) MetricsCollectorTimeToCollectMetrics() { } func (p *ParticipantImpl) MetricsCollectorBatchReady(mb *livekit.MetricsBatch) { - if onMetrics := p.getOnMetrics(); onMetrics != nil { - onMetrics(p, &livekit.DataPacket{ - ParticipantIdentity: string(p.Identity()), - Value: &livekit.DataPacket_Metrics{ - Metrics: mb, - }, - }) - } + p.listener().OnMetrics(p, &livekit.DataPacket{ + ParticipantIdentity: string(p.Identity()), + Value: &livekit.DataPacket_Metrics{ + Metrics: mb, + }, + }) } func (p *ParticipantImpl) MetricsReporterBatchReady(mb *livekit.MetricsBatch) { @@ -1644,7 +2119,7 @@ func (p *ParticipantImpl) MetricsReporterBatchReady(mb *livekit.MetricsBatch) { return } - p.TransportManager.SendDataPacket(livekit.DataPacket_RELIABLE, dpData) + p.TransportManager.SendDataMessage(livekit.DataPacket_RELIABLE, dpData) } func (p *ParticipantImpl) setupMetrics() { @@ -1671,23 +2146,29 @@ func (p *ParticipantImpl) setupMetrics() { } func (p *ParticipantImpl) updateState(state livekit.ParticipantInfo_State) { - if state == livekit.ParticipantInfo_ACTIVE { - t := time.Now() - p.lastActiveAt.CompareAndSwap(nil, &t) - } - oldState := p.state.Swap(state).(livekit.ParticipantInfo_State) - if oldState == state { - return + var oldState livekit.ParticipantInfo_State + for { + oldState = p.state.Load().(livekit.ParticipantInfo_State) + if state <= oldState { + p.params.Logger.Debugw("ignoring out of order participant state", "state", state.String()) + return + } + if state == livekit.ParticipantInfo_ACTIVE { + p.lastActiveAt.CompareAndSwap(nil, pointer.To(time.Now())) + } + if p.state.CompareAndSwap(oldState, state) { + break + } } - if state == livekit.ParticipantInfo_DISCONNECTED && oldState == livekit.ParticipantInfo_ACTIVE { - prometheus.RecordSessionDuration(int(p.ProtocolVersion()), time.Since(*p.lastActiveAt.Load())) - } p.params.Logger.Debugw("updating participant state", "state", state.String()) p.dirty.Store(true) - if onStateChange := p.getOnStateChange(); onStateChange != nil { - go onStateChange(p, state) + go p.listener().OnStateChange(p) + + if state == livekit.ParticipantInfo_DISCONNECTED && oldState == livekit.ParticipantInfo_ACTIVE { + p.disconnectedAt.Store(pointer.To(time.Now())) + prometheus.RecordSessionDuration(int(p.ProtocolVersion()), time.Since(*p.lastActiveAt.Load())) } } @@ -1704,28 +2185,24 @@ func (p *ParticipantImpl) setIsPublisher(isPublisher bool) { // trigger update as well if participant is already fully connected if p.State() == livekit.ParticipantInfo_ACTIVE { - p.lock.RLock() - onParticipantUpdate := p.onParticipantUpdate - p.lock.RUnlock() - - if onParticipantUpdate != nil { - onParticipantUpdate(p) - } + p.listener().OnParticipantUpdate(p) } } // when the server has an offer for participant -func (p *ParticipantImpl) onSubscriberOffer(offer webrtc.SessionDescription) error { - p.subLogger.Debugw("sending offer", "transport", livekit.SignalTarget_SUBSCRIBER, "offer", offer) - return p.writeMessage(&livekit.SignalResponse{ - Message: &livekit.SignalResponse_Offer{ - Offer: ToProtoSessionDescription(offer), - }, - }) +func (p *ParticipantImpl) onSubscriberOffer(offer webrtc.SessionDescription, offerId uint32, midToTrackID map[string]string) error { + p.subLogger.Debugw( + "sending offer", + "transport", livekit.SignalTarget_SUBSCRIBER, + "offer", offer, + "offerId", offerId, + "midToTrackID", midToTrackID, + ) + return p.sendSdpOffer(offer, offerId, midToTrackID) } func (p *ParticipantImpl) removePublishedTrack(track types.MediaTrack) { - p.RemovePublishedTrack(track, false, true) + p.RemovePublishedTrack(track, false) if p.ProtocolVersion().SupportsUnpublish() { p.sendTrackUnpublished(track.ID()) } else { @@ -1750,7 +2227,11 @@ func (p *ParticipantImpl) onMediaTrack(rtcTrack *webrtc.TrackRemote, rtpReceiver _, _, err := rtcTrack.Read(bytes) if err != nil { if !errors.Is(err, io.EOF) { - p.params.Logger.Warnw("could not read first packet to determine codec, track will be ignored", err, "trackID", rtcTrack.ID(), "StreamID", rtcTrack.StreamID()) + p.params.Logger.Warnw( + "could not read first packet to determine codec, track will be ignored", err, + "trackID", rtcTrack.ID(), + "StreamID", rtcTrack.StreamID(), + ) } return } @@ -1763,24 +2244,37 @@ func (p *ParticipantImpl) onMediaTrack(rtcTrack *webrtc.TrackRemote, rtpReceiver // track fired by sdp codecs := rtpReceiver.GetParameters().Codecs if len(codecs) == 0 { - p.pubLogger.Errorw("no negotiated codecs for track, track will be ignored", nil, "trackID", rtcTrack.ID(), "StreamID", rtcTrack.StreamID()) + p.pubLogger.Errorw( + "no negotiated codecs for track, track will be ignored", nil, + "trackID", rtcTrack.ID(), + "StreamID", rtcTrack.StreamID(), + ) return } codec = codecs[0] fromSdp = true } - p.params.Logger.Debugw("onMediaTrack", "codec", codec, "payloadType", codec.PayloadType, "fromSdp", fromSdp, "parameters", rtpReceiver.GetParameters()) + p.params.Logger.Debugw( + "onMediaTrack", + "codec", codec, + "payloadType", codec.PayloadType, + "fromSdp", fromSdp, + "parameters", rtpReceiver.GetParameters(), + ) var track sfu.TrackRemote = sfu.NewTrackRemoteFromSdp(rtcTrack, codec) - publishedTrack, isNewTrack := p.mediaTrackReceived(track, rtpReceiver) + publishedTrack, isNewTrack, isReceiverAdded, sdpRids := p.mediaTrackReceived(track, rtpReceiver) if publishedTrack == nil { p.pubLogger.Debugw( - "webrtc Track published but can't find MediaTrack, add to pendingTracks", + "webrtc track published but can't find MediaTrack in pendingTracks", "kind", track.Kind().String(), "webrtcTrackID", track.ID(), "rid", track.RID(), - "SSRC", track.SSRC(), + "ssrc", track.SSRC(), + "rtxSsrc", track.RtxSSRC(), "mime", mime.NormalizeMimeType(codec.MimeType), + "isReceiverAdded", isReceiverAdded, + "sdpRids", logger.StringSlice(sdpRids[:]), ) return } @@ -1793,24 +2287,33 @@ func (p *ParticipantImpl) onMediaTrack(rtcTrack *webrtc.TrackRemote, rtpReceiver return } + p.TransportManager.RTPStreamPublished( + uint32(track.SSRC()), + p.TransportManager.GetPublisherMid(rtpReceiver), + track.RID(), + ) + p.setIsPublisher(true) p.dirty.Store(true) - p.pubLogger.Infow("mediaTrack published", + p.pubLogger.Infow( + "mediaTrack published", "kind", track.Kind().String(), "trackID", publishedTrack.ID(), "webrtcTrackID", track.ID(), "rid", track.RID(), - "SSRC", track.SSRC(), + "ssrc", track.SSRC(), + "rtxSsrc", track.RtxSSRC(), "mime", mime.NormalizeMimeType(codec.MimeType), "trackInfo", logger.Proto(publishedTrack.ToProto()), "fromSdp", fromSdp, + "isReceiverAdded", isReceiverAdded, + "sdpRids", logger.StringSlice(sdpRids[:]), ) if !isNewTrack && !publishedTrack.HasPendingCodec() && p.IsReady() { - if onTrackUpdated := p.getOnTrackUpdated(); onTrackUpdated != nil { - onTrackUpdated(p, publishedTrack) - } + p.dirty.Store(true) + p.listener().OnTrackUpdated(p, publishedTrack) } } @@ -1824,7 +2327,7 @@ func (p *ParticipantImpl) handlePendingRemoteTracks() { } } -func (p *ParticipantImpl) onDataMessage(kind livekit.DataPacket_Kind, data []byte) { +func (p *ParticipantImpl) onReceivedDataMessage(kind livekit.DataPacket_Kind, data []byte) { if p.IsDisconnected() || !p.CanPublishData() { return } @@ -1837,6 +2340,64 @@ func (p *ParticipantImpl) onDataMessage(kind livekit.DataPacket_Kind, data []byt return } + dp.ParticipantSid = string(p.ID()) + if kind == livekit.DataPacket_RELIABLE && dp.Sequence > 0 { + if p.reliableDataInfo.stopReliableByMigrateOut.Load() { + return + } + + if migrationCache := p.reliableDataInfo.migrateInPubDataCache.Load(); migrationCache != nil { + switch migrationCache.Add(dp) { + case MigrationDataCacheStateWaiting: + // waiting for the reliable sequence to continue from last node + return + + case MigrationDataCacheStateTimeout: + p.reliableDataInfo.migrateInPubDataCache.Store(nil) + // waiting time out, handle all cached messages + cachedMsgs := migrationCache.Get() + if len(cachedMsgs) == 0 { + p.pubLogger.Warnw( + "migration data cache timed out, no cached messages received", nil, + "lastPubReliableSeq", p.params.LastPubReliableSeq, + ) + } else { + p.pubLogger.Warnw( + "migration data cache timed out, handling cached messages", nil, + "cachedFirstSeq", cachedMsgs[0].Sequence, + "cachedLastSeq", cachedMsgs[len(cachedMsgs)-1].Sequence, + "lastPubReliableSeq", p.params.LastPubReliableSeq, + ) + } + for _, cachedDp := range cachedMsgs { + p.handleReceivedDataMessage(kind, cachedDp) + } + return + + case MigrationDataCacheStateDone: + // see the continuous message, drop the cache + p.reliableDataInfo.migrateInPubDataCache.Store(nil) + } + } + } + + p.handleReceivedDataMessage(kind, dp) +} + +func (p *ParticipantImpl) handleReceivedDataMessage(kind livekit.DataPacket_Kind, dp *livekit.DataPacket) { + if kind == livekit.DataPacket_RELIABLE && dp.Sequence > 0 { + if p.reliableDataInfo.lastPubReliableSeq.Load() >= dp.Sequence { + p.params.Logger.Infow( + "received out of order reliable data packet", + "lastPubReliableSeq", p.reliableDataInfo.lastPubReliableSeq.Load(), + "dpSequence", dp.Sequence, + ) + return + } + + p.reliableDataInfo.lastPubReliableSeq.Store(dp.Sequence) + } + // trust the channel that it came in as the source of truth dp.Kind = kind @@ -1854,7 +2415,7 @@ func (p *ParticipantImpl) onDataMessage(kind livekit.DataPacket_Kind, data []byt u.ParticipantSid = "" u.ParticipantIdentity = "" } else { - u.ParticipantSid = string(p.params.SID) + u.ParticipantSid = string(p.ID()) u.ParticipantIdentity = string(p.params.Identity) } if len(dp.DestinationIdentities) != 0 { @@ -1892,15 +2453,42 @@ func (p *ParticipantImpl) onDataMessage(kind livekit.DataPacket_Kind, data []byt if payload.RpcRequest == nil { return } - p.pubLogger.Infow("received RPC request data packet", "method", payload.RpcRequest.Method, "rpc_request_id", payload.RpcRequest.Id) + p.pubLogger.Debugw( + "received RPC request", + "method", payload.RpcRequest.Method, + "rpc_request_id", payload.RpcRequest.Id, + "destinationIdentities", dp.DestinationIdentities, + ) case *livekit.DataPacket_RpcResponse: if payload.RpcResponse == nil { return } + p.pubLogger.Debugw( + "received RPC response", + "rpc_request_id", payload.RpcResponse.RequestId, + ) + + rpcResponse := payload.RpcResponse + switch res := rpcResponse.Value.(type) { + case *livekit.RpcResponse_Payload: + shouldForwardData = !p.handleIncomingRpcResponse(payload.RpcResponse.GetRequestId(), res.Payload, nil) + case *livekit.RpcResponse_Error: + shouldForwardData = !p.handleIncomingRpcResponse(payload.RpcResponse.GetRequestId(), "", &utils.DataChannelRpcError{ + Code: utils.DataChannelRpcErrorCode(res.Error.GetCode()), + Message: res.Error.GetMessage(), + Data: res.Error.GetData(), + }) + } case *livekit.DataPacket_RpcAck: if payload.RpcAck == nil { return } + p.pubLogger.Debugw( + "received RPC ack", + "rpc_request_id", payload.RpcAck.RequestId, + ) + + shouldForwardData = !p.handleIncomingRpcAck(payload.RpcAck.GetRequestId()) case *livekit.DataPacket_StreamHeader: if payload.StreamHeader == nil { return @@ -1925,28 +2513,43 @@ func (p *ParticipantImpl) onDataMessage(kind livekit.DataPacket_Kind, data []byt if payload.StreamTrailer == nil { return } + case *livekit.DataPacket_EncryptedPacket: + if payload.EncryptedPacket == nil { + return + } default: p.pubLogger.Warnw("received unsupported data packet", nil, "payload", payload) } - if p.Hidden() { - dp.ParticipantIdentity = "" - } else if overrideSenderIdentity { - dp.ParticipantIdentity = string(p.params.Identity) + // SFU typically asserts the sender's identity. However, agents are able to + // publish data on behalf of the participant in case of transcriptions/text streams + // in those cases we'd leave the existing identity on the data packet alone. + if overrideSenderIdentity { + if p.Hidden() { + dp.ParticipantIdentity = "" + } else { + dp.ParticipantIdentity = string(p.params.Identity) + } } if shouldForwardData { - if onDataPacket := p.getOnDataPacket(); onDataPacket != nil { - onDataPacket(p, kind, dp) - } + p.listener().OnDataMessage(p, kind, dp) } if shouldForwardMetrics { - if onMetrics := p.getOnMetrics(); onMetrics != nil { - onMetrics(p, dp) - } + p.listener().OnMetrics(p, dp) } } +func (p *ParticipantImpl) onReceivedDataMessageUnlabeled(data []byte) { + if p.IsDisconnected() || !p.CanPublishData() { + return + } + + p.dataChannelStats.AddBytes(uint64(len(data)), false) + + p.listener().OnDataMessageUnlabeled(p, data) +} + func (p *ParticipantImpl) onICECandidate(c *webrtc.ICECandidate, target livekit.SignalTarget) error { if p.IsDisconnected() || p.IsClosed() { return nil @@ -1966,7 +2569,7 @@ func (p *ParticipantImpl) onPublisherInitialConnected() { p.supervisor.SetPublisherPeerConnectionConnected(true) } - if p.params.UseOneShotSignallingMode { + if p.params.UseOneShotSignallingMode || p.params.UseSinglePeerConnection { go p.subscriberRTCPWorker() p.setDownTracksConnected() @@ -1987,15 +2590,17 @@ func (p *ParticipantImpl) onPrimaryTransportInitialConnected() { // else, wait for all tracks to be published and publisher peer connection established p.SetMigrateState(types.MigrateStateComplete) } -} -func (p *ParticipantImpl) onPrimaryTransportFullyEstablished() { if !p.sessionStartRecorded.Swap(true) { prometheus.RecordSessionStartTime(int(p.ProtocolVersion()), time.Since(p.params.SessionStartTime)) } p.updateState(livekit.ParticipantInfo_ACTIVE) } +func (p *ParticipantImpl) onPrimaryTransportFullyEstablished() { + p.replayJoiningReliableMessages() +} + func (p *ParticipantImpl) clearDisconnectTimer() { p.lock.Lock() if p.disconnectTimer != nil { @@ -2125,11 +2730,7 @@ func (p *ParticipantImpl) onStreamStateChange(update *streamallocator.StreamStat }) } - return p.writeMessage(&livekit.SignalResponse{ - Message: &livekit.SignalResponse_StreamStateUpdate{ - StreamStateUpdate: streamStateUpdate, - }, - }) + return p.sendStreamStateUpdate(streamStateUpdate) } func (p *ParticipantImpl) onSubscribedMaxQualityChange( @@ -2152,15 +2753,14 @@ func (p *ParticipantImpl) onSubscribedMaxQualityChange( Sid: trackInfo.Sid, Type: trackInfo.Type, } - for _, layer := range trackInfo.Layers { + for _, layer := range buffer.GetVideoLayersForMimeType(maxSubscribedQuality.CodecMime, trackInfo) { if layer.Quality == maxSubscribedQuality.Quality { ti.Width = layer.Width ti.Height = layer.Height break } } - p.params.Telemetry.TrackMaxSubscribedVideoQuality( - context.Background(), + p.params.TelemetryListener.OnTrackMaxSubscribedVideoQuality( p.ID(), ti, maxSubscribedQuality.CodecMime, @@ -2185,11 +2785,36 @@ func (p *ParticipantImpl) onSubscribedMaxQualityChange( "qualities", subscribedQualities, "max", maxSubscribedQualities, ) - return p.writeMessage(&livekit.SignalResponse{ - Message: &livekit.SignalResponse_SubscribedQualityUpdate{ - SubscribedQualityUpdate: subscribedQualityUpdate, - }, - }) + return p.sendSubscribedQualityUpdate(subscribedQualityUpdate) +} + +func (p *ParticipantImpl) onSubscribedAudioCodecChange( + trackID livekit.TrackID, + codecs []*livekit.SubscribedAudioCodec, +) error { + if p.params.DisableDynacast { + return nil + } + + if len(codecs) == 0 { + return nil + } + + // normalize the codec name + for _, codec := range codecs { + codec.Codec = strings.ToLower(strings.TrimPrefix(codec.Codec, mime.MimeTypePrefixAudio)) + } + + subscribedAudioCodecUpdate := &livekit.SubscribedAudioCodecUpdate{ + TrackSid: string(trackID), + SubscribedAudioCodecs: codecs, + } + p.pubLogger.Debugw( + "sending subscribed audio codec update", + "trackID", trackID, + "update", logger.Proto(subscribedAudioCodecUpdate), + ) + return p.sendSubscribedAudioCodecUpdate(subscribedAudioCodecUpdate) } func (p *ParticipantImpl) addPendingTrackLocked(req *livekit.AddTrackRequest) *livekit.TrackInfo { @@ -2197,38 +2822,63 @@ func (p *ParticipantImpl) addPendingTrackLocked(req *livekit.AddTrackRequest) *l track := p.GetPublishedTrack(livekit.TrackID(req.Sid)) if track == nil { p.pubLogger.Infow("could not find existing track for multi-codec simulcast", "trackID", req.Sid) + p.sendRequestResponse(&livekit.RequestResponse{ + Reason: livekit.RequestResponse_NOT_FOUND, + Request: &livekit.RequestResponse_AddTrack{ + AddTrack: utils.CloneProto(req), + }, + }) return nil } - track.(*MediaTrack).UpdateCodecCid(req.SimulcastCodecs) - ti := track.ToProto() - return ti + track.(*MediaTrack).UpdateCodecInfo(req.SimulcastCodecs) + return track.ToProto() } backupCodecPolicy := req.BackupCodecPolicy - if backupCodecPolicy == livekit.BackupCodecPolicy_REGRESSION && p.params.DisableCodecRegression { + + // enable simulcast codec for audio by default + if (backupCodecPolicy != livekit.BackupCodecPolicy_REGRESSION && req.Type == livekit.TrackType_AUDIO) || + (backupCodecPolicy != livekit.BackupCodecPolicy_SIMULCAST && p.params.DisableCodecRegression) { backupCodecPolicy = livekit.BackupCodecPolicy_SIMULCAST } - ti := &livekit.TrackInfo{ - Type: req.Type, - Name: req.Name, - Width: req.Width, - Height: req.Height, - Muted: req.Muted, - DisableDtx: req.DisableDtx, - Source: req.Source, - Layers: req.Layers, - DisableRed: req.DisableRed, - Stereo: req.Stereo, - Encryption: req.Encryption, - Stream: req.Stream, - BackupCodecPolicy: backupCodecPolicy, + cloneLayers := func(layers []*livekit.VideoLayer) []*livekit.VideoLayer { + if len(layers) == 0 { + return nil + } + + clonedLayers := make([]*livekit.VideoLayer, 0, len(layers)) + for _, l := range layers { + clonedLayers = append(clonedLayers, utils.CloneProto(l)) + } + slices.SortFunc(clonedLayers, func(i, j *livekit.VideoLayer) int { + return int(i.Quality) - int(j.Quality) + }) + return clonedLayers } - if req.Stereo { + + ti := &livekit.TrackInfo{ + Type: req.Type, + Name: req.Name, + Width: req.Width, + Height: req.Height, + Muted: req.Muted, + DisableDtx: req.DisableDtx, + Source: req.Source, + Layers: cloneLayers(req.Layers), + DisableRed: req.DisableRed, + Stereo: req.Stereo, + Encryption: req.Encryption, + Stream: req.Stream, + BackupCodecPolicy: backupCodecPolicy, + AudioFeatures: sutils.DedupeSlice(req.AudioFeatures), + PacketTrailerFeatures: sutils.DedupeSlice(req.PacketTrailerFeatures), + } + if req.Stereo && !slices.Contains(ti.AudioFeatures, livekit.AudioTrackFeature_TF_STEREO) { ti.AudioFeatures = append(ti.AudioFeatures, livekit.AudioTrackFeature_TF_STEREO) } - if req.DisableDtx { + if req.DisableDtx && !slices.Contains(ti.AudioFeatures, livekit.AudioTrackFeature_TF_NO_DTX) { ti.AudioFeatures = append(ti.AudioFeatures, livekit.AudioTrackFeature_TF_NO_DTX) } if ti.Stream == "" { @@ -2237,33 +2887,75 @@ func (p *ParticipantImpl) addPendingTrackLocked(req *livekit.AddTrackRequest) *l p.setTrackID(req.Cid, ti) if len(req.SimulcastCodecs) == 0 { - if req.Type == livekit.TrackType_VIDEO { - // clients not supporting simulcast codecs, synthesise a codec - ti.Codecs = append(ti.Codecs, &livekit.SimulcastCodecInfo{ - Cid: req.Cid, - Layers: req.Layers, - }) + // clients not supporting simulcast codecs, synthesise a codec + videoLayerMode := livekit.VideoLayer_ONE_SPATIAL_LAYER_PER_STREAM + if p.params.ClientInfo.isOBS() { + videoLayerMode = livekit.VideoLayer_ONE_SPATIAL_LAYER_PER_STREAM_INCOMPLETE_RTCP_SR } + ti.Codecs = append(ti.Codecs, &livekit.SimulcastCodecInfo{ + Cid: req.Cid, + Layers: cloneLayers(req.Layers), + VideoLayerMode: videoLayerMode, + }) } else { seenCodecs := make(map[string]struct{}) for _, codec := range req.SimulcastCodecs { + if codec.Codec == "" { + p.pubLogger.Warnw( + "simulcast codec without mime type", nil, + "trackID", ti.Sid, + "track", logger.Proto(ti), + "addTrackRequest", logger.Proto(req), + ) + } + mimeType := codec.Codec - if req.Type == livekit.TrackType_VIDEO { + videoLayerMode := codec.VideoLayerMode + switch req.Type { + case livekit.TrackType_VIDEO: if !mime.IsMimeTypeStringVideo(mimeType) { mimeType = mime.MimeTypePrefixVideo + mimeType } if !IsCodecEnabled(p.enabledPublishCodecs, webrtc.RTPCodecCapability{MimeType: mimeType}) { altCodec := selectAlternativeVideoCodec(p.enabledPublishCodecs) - p.pubLogger.Infow("falling back to alternative codec", + p.pubLogger.Infow( + "falling back to alternative video codec", "codec", mimeType, "altCodec", altCodec, + "enabledPublishCodecs", logger.ProtoSlice(p.enabledPublishCodecs), + "trackID", ti.Sid, + ) + // select an alternative MIME type that's generally supported + mimeType = altCodec + } + if videoLayerMode == livekit.VideoLayer_MODE_UNUSED { + if mime.IsMimeTypeStringSVCCapable(mimeType) { + videoLayerMode = livekit.VideoLayer_MULTIPLE_SPATIAL_LAYERS_PER_STREAM + } else { + if p.params.ClientInfo.isOBS() { + videoLayerMode = livekit.VideoLayer_ONE_SPATIAL_LAYER_PER_STREAM_INCOMPLETE_RTCP_SR + } else { + videoLayerMode = livekit.VideoLayer_ONE_SPATIAL_LAYER_PER_STREAM + } + } + } + + case livekit.TrackType_AUDIO: + if !mime.IsMimeTypeStringAudio(mimeType) { + mimeType = mime.MimeTypePrefixAudio + mimeType + } + if !IsCodecEnabled(p.enabledPublishCodecs, webrtc.RTPCodecCapability{MimeType: mimeType}) { + altCodec := selectAlternativeAudioCodec(p.enabledPublishCodecs) + p.pubLogger.Infow( + "falling back to alternative audio codec", + "codec", mimeType, + "altCodec", altCodec, + "enabledPublishCodecs", logger.ProtoSlice(p.enabledPublishCodecs), "trackID", ti.Sid, ) // select an alternative MIME type that's generally supported mimeType = altCodec } - } else if req.Type == livekit.TrackType_AUDIO && !mime.IsMimeTypeStringAudio(mimeType) { - mimeType = mime.MimeTypePrefixAudio + mimeType } if _, ok := seenCodecs[mimeType]; ok || mimeType == "" { @@ -2271,35 +2963,93 @@ func (p *ParticipantImpl) addPendingTrackLocked(req *livekit.AddTrackRequest) *l } seenCodecs[mimeType] = struct{}{} - clonedLayers := make([]*livekit.VideoLayer, 0, len(req.Layers)) - for _, l := range req.Layers { - clonedLayers = append(clonedLayers, utils.CloneProto(l)) - } ti.Codecs = append(ti.Codecs, &livekit.SimulcastCodecInfo{ - MimeType: mimeType, - Cid: codec.Cid, - Layers: clonedLayers, + MimeType: mimeType, + Cid: codec.Cid, + VideoLayerMode: videoLayerMode, }) } + + // set up layers with codec specific layers, + // fall back to common layers if codec specific layer is not available + for idx, codec := range ti.Codecs { + found := false + for _, simulcastCodec := range req.SimulcastCodecs { + if mime.GetMimeTypeCodec(codec.MimeType) != mime.NormalizeMimeTypeCodec(simulcastCodec.Codec) { + continue + } + + if len(simulcastCodec.Layers) != 0 { + codec.Layers = cloneLayers(simulcastCodec.Layers) + } else { + codec.Layers = cloneLayers(req.Layers) + } + found = true + break + } + + if !found { + // could happen if an alternate codec is selected and that is not in the simulcast codecs list + codec.Layers = cloneLayers(req.Layers) + } + + // populate simulcast flag for compatibility, true if primary codec is not SVC and has multiple layers + if idx == 0 && codec.VideoLayerMode != livekit.VideoLayer_MULTIPLE_SPATIAL_LAYERS_PER_STREAM && len(codec.Layers) > 1 { + ti.Simulcast = true + } + } } - p.params.Telemetry.TrackPublishRequested(context.Background(), p.ID(), p.Identity(), utils.CloneProto(ti)) + p.params.TelemetryListener.OnTrackPublishRequested(p.ID(), p.Identity(), utils.CloneProto(ti)) + if p.supervisor != nil { p.supervisor.AddPublication(livekit.TrackID(ti.Sid)) p.supervisor.SetPublicationMute(livekit.TrackID(ti.Sid), ti.Muted) } + if p.getPublishedTrackBySignalCid(req.Cid) != nil || p.getPublishedTrackBySdpCid(req.Cid) != nil || p.pendingTracks[req.Cid] != nil { if p.pendingTracks[req.Cid] == nil { - p.pendingTracks[req.Cid] = &pendingTrackInfo{trackInfos: []*livekit.TrackInfo{ti}, createdAt: time.Now(), queued: true} + pti := &pendingTrackInfo{ + trackInfos: []*livekit.TrackInfo{ti}, + createdAt: time.Now(), + queued: true, + } + if ti.Type == livekit.TrackType_VIDEO { + pti.sdpRids = buffer.DefaultVideoLayersRid // could get updated from SDP + } + p.pendingTracks[req.Cid] = pti } else { p.pendingTracks[req.Cid].trackInfos = append(p.pendingTracks[req.Cid].trackInfos, ti) } - p.pubLogger.Infow("pending track queued", "trackID", ti.Sid, "track", logger.Proto(ti), "request", logger.Proto(req)) + p.pubLogger.Infow( + "pending track queued", + "trackID", ti.Sid, + "request", logger.Proto(req), + "pendingTrack", p.pendingTracks[req.Cid], + ) + p.sendRequestResponse(&livekit.RequestResponse{ + Reason: livekit.RequestResponse_QUEUED, + Request: &livekit.RequestResponse_AddTrack{ + AddTrack: utils.CloneProto(req), + }, + }) return nil } - p.pendingTracks[req.Cid] = &pendingTrackInfo{trackInfos: []*livekit.TrackInfo{ti}, createdAt: time.Now()} - p.pubLogger.Debugw("pending track added", "trackID", ti.Sid, "track", logger.Proto(ti), "request", logger.Proto(req)) + pti := &pendingTrackInfo{ + trackInfos: []*livekit.TrackInfo{ti}, + createdAt: time.Now(), + } + if ti.Type == livekit.TrackType_VIDEO { + pti.sdpRids = buffer.DefaultVideoLayersRid // could get updated from SDP + } + p.pendingTracks[req.Cid] = pti + p.pubLogger.Debugw( + "pending track added", + "trackID", ti.Sid, + "request", logger.Proto(req), + "pendingTrack", p.pendingTracks[req.Cid], + ) return ti } @@ -2320,70 +3070,69 @@ func (p *ParticipantImpl) HasConnected() bool { return p.TransportManager.HasSubscriberEverConnected() || p.TransportManager.HasPublisherEverConnected() } -func (p *ParticipantImpl) sendTrackPublished(cid string, ti *livekit.TrackInfo) { - p.pubLogger.Debugw("sending track published", "cid", cid, "trackInfo", logger.Proto(ti)) - _ = p.writeMessage(&livekit.SignalResponse{ - Message: &livekit.SignalResponse_TrackPublished{ - TrackPublished: &livekit.TrackPublishedResponse{ - Cid: cid, - Track: ti, - }, - }, - }) -} - -func (p *ParticipantImpl) SetTrackMuted(trackID livekit.TrackID, muted bool, fromAdmin bool) *livekit.TrackInfo { +func (p *ParticipantImpl) SetTrackMuted(mute *livekit.MuteTrackRequest, fromAdmin bool) *livekit.TrackInfo { // when request is coming from admin, send message to current participant if fromAdmin { - p.sendTrackMuted(trackID, muted) + p.sendTrackMuted(livekit.TrackID(mute.Sid), mute.Muted) } - return p.setTrackMuted(trackID, muted) + return p.setTrackMuted(mute, fromAdmin) } -func (p *ParticipantImpl) setTrackMuted(trackID livekit.TrackID, muted bool) *livekit.TrackInfo { +func (p *ParticipantImpl) setTrackMuted(mute *livekit.MuteTrackRequest, fromAdmin bool) *livekit.TrackInfo { + trackID := livekit.TrackID(mute.Sid) p.dirty.Store(true) if p.supervisor != nil { - p.supervisor.SetPublicationMute(trackID, muted) + p.supervisor.SetPublicationMute(trackID, mute.Muted) } - track := p.UpTrackManager.SetPublishedTrackMuted(trackID, muted) + track, changed := p.UpTrackManager.SetPublishedTrackMuted(trackID, mute.Muted) var trackInfo *livekit.TrackInfo if track != nil { trackInfo = track.ToProto() } - isPending := false + // update mute status in any pending/queued add track requests too p.pendingTracksLock.RLock() for _, pti := range p.pendingTracks { for i, ti := range pti.trackInfos { if livekit.TrackID(ti.Sid) == trackID { ti = utils.CloneProto(ti) - ti.Muted = muted + changed = changed || ti.Muted != mute.Muted + ti.Muted = mute.Muted pti.trackInfos[i] = ti - isPending = true - trackInfo = ti + if trackInfo == nil { + trackInfo = ti + } } } } p.pendingTracksLock.RUnlock() - if trackInfo != nil { - if muted { - p.params.Telemetry.TrackMuted(context.Background(), p.ID(), trackInfo) + if trackInfo != nil && changed { + if mute.Muted { + p.params.TelemetryListener.OnTrackMuted(p.ID(), trackInfo) } else { - p.params.Telemetry.TrackUnmuted(context.Background(), p.ID(), trackInfo) + p.params.TelemetryListener.OnTrackUnmuted(p.ID(), trackInfo) } } - if !isPending && track == nil { - p.pubLogger.Debugw("could not locate track", "trackID", trackID) + if trackInfo == nil && !fromAdmin { + p.sendRequestResponse(&livekit.RequestResponse{ + Reason: livekit.RequestResponse_NOT_FOUND, + Request: &livekit.RequestResponse_Mute{ + Mute: utils.CloneProto(mute), + }, + }) } return trackInfo } -func (p *ParticipantImpl) mediaTrackReceived(track sfu.TrackRemote, rtpReceiver *webrtc.RTPReceiver) (*MediaTrack, bool) { +func (p *ParticipantImpl) mediaTrackReceived( + track sfu.TrackRemote, + rtpReceiver *webrtc.RTPReceiver, +) (*MediaTrack, bool, bool, buffer.VideoLayersRid) { p.pendingTracksLock.Lock() newTrack := false @@ -2393,27 +3142,38 @@ func (p *ParticipantImpl) mediaTrackReceived(track sfu.TrackRemote, rtpReceiver "kind", track.Kind().String(), "trackID", track.ID(), "rid", track.RID(), - "SSRC", track.SSRC(), + "ssrc", track.SSRC(), + "rtxSsrc", track.RtxSSRC(), "mime", mime.NormalizeMimeType(track.Codec().MimeType), "mid", mid, ) if mid == "" { - p.pendingRemoteTracks = append(p.pendingRemoteTracks, &pendingRemoteTrack{track: track.RTCTrack(), receiver: rtpReceiver}) + p.pendingRemoteTracks = append( + p.pendingRemoteTracks, + &pendingRemoteTrack{track: track.RTCTrack(), receiver: rtpReceiver}, + ) p.pendingTracksLock.Unlock() p.pubLogger.Warnw("could not get mid for track", nil, "trackID", track.ID()) - return nil, false + return nil, false, false, buffer.VideoLayersRid{} } // use existing media track to handle simulcast var pubTime time.Duration + var isMigrated bool + var ridsFromSdp buffer.VideoLayersRid mt, ok := p.getPublishedTrackBySdpCid(track.ID()).(*MediaTrack) if !ok { - signalCid, ti, migrated, createdAt := p.getPendingTrack(track.ID(), ToProtoTrackKind(track.Kind()), true) + signalCid, ti, sdpRids, migrated, createdAt := p.getPendingTrack(track.ID(), ToProtoTrackKind(track.Kind()), true) + ridsFromSdp = sdpRids if ti == nil { - p.pendingRemoteTracks = append(p.pendingRemoteTracks, &pendingRemoteTrack{track: track.RTCTrack(), receiver: rtpReceiver}) + p.pendingRemoteTracks = append( + p.pendingRemoteTracks, + &pendingRemoteTrack{track: track.RTCTrack(), receiver: rtpReceiver}, + ) p.pendingTracksLock.Unlock() - return nil, false + return nil, false, false, ridsFromSdp } + isMigrated = migrated // check if the migrated track has correct codec if migrated && len(ti.Codecs) > 0 { @@ -2431,7 +3191,7 @@ func (p *ParticipantImpl) mediaTrackReceived(track sfu.TrackRemote, rtpReceiver p.pubLogger.Warnw("migrated track codec mismatched", nil, "track", logger.Proto(ti), "webrtcCodec", parameters) p.pendingTracksLock.Unlock() p.IssueFullReconnect(types.ParticipantCloseReasonMigrateCodecMismatch) - return nil, false + return nil, false, false, ridsFromSdp } } @@ -2444,22 +3204,28 @@ func (p *ParticipantImpl) mediaTrackReceived(track sfu.TrackRemote, rtpReceiver // only assign version on a fresh publish, i. e. avoid updating version in scenarios like migration ti.Version = p.params.VersionGenerator.Next().ToProto() } - mt = p.addMediaTrack(signalCid, track.ID(), ti) + + mimeType := mime.NormalizeMimeType(ti.MimeType) + for _, layer := range ti.Layers { + layer.SpatialLayer = buffer.VideoQualityToSpatialLayer(mimeType, layer.Quality, ti) + layer.Rid = buffer.VideoQualityToRid(mimeType, layer.Quality, ti, sdpRids) + } + + mt = p.addMediaTrack(signalCid, ti) newTrack = true // if the addTrackRequest is sent before participant active then it means the client tries to publish // before fully connected, in this case we only record the time when the participant is active since - // we want this metric to represent the time cost by pubilshing. + // we want this metric to represent the time cost by publishing. if activeAt := p.lastActiveAt.Load(); activeAt != nil && createdAt.Before(*activeAt) { createdAt = *activeAt } pubTime = time.Since(createdAt) p.dirty.Store(true) } - p.pendingTracksLock.Unlock() - mt.AddReceiver(rtpReceiver, track, mid) + _, isReceiverAdded := mt.AddReceiver(rtpReceiver, track, mid) if newTrack { go func() { @@ -2472,6 +3238,7 @@ func (p *ParticipantImpl) mediaTrackReceived(track sfu.TrackRemote, rtpReceiver "cost", pubTime.Milliseconds(), "rid", track.RID(), "mime", track.Codec().MimeType, + "isMigrated", isMigrated, ) } else { p.pubLogger.Debugw( @@ -2479,26 +3246,38 @@ func (p *ParticipantImpl) mediaTrackReceived(track sfu.TrackRemote, rtpReceiver "trackID", mt.ID(), "track", logger.Proto(mt.ToProto()), "cost", pubTime.Milliseconds(), + "isMigrated", isMigrated, ) } - prometheus.RecordPublishTime(mt.Source(), mt.Kind(), pubTime, p.GetClientInfo().GetSdk(), p.Kind()) - p.handleTrackPublished(mt) + prometheus.RecordPublishTime( + p.params.Country, + mt.Source(), + mt.Kind(), + pubTime, + p.GetClientInfo().GetSdk(), + p.Kind(), + ) + p.handleTrackPublished(mt, isMigrated) }() } - return mt, newTrack + return mt, newTrack, isReceiverAdded, ridsFromSdp } func (p *ParticipantImpl) addMigratedTrack(cid string, ti *livekit.TrackInfo) *MediaTrack { p.pubLogger.Infow("add migrated track", "cid", cid, "trackID", ti.Sid, "track", logger.Proto(ti)) rtpReceiver := p.TransportManager.GetPublisherRTPReceiver(ti.Mid) if rtpReceiver == nil { - p.pubLogger.Errorw("could not find receiver for migrated track", nil, "trackID", ti.Sid, "mid", ti.Mid) + p.pubLogger.Errorw( + "could not find receiver for migrated track", nil, + "trackID", ti.Sid, + "mid", ti.Mid, + ) return nil } - mt := p.addMediaTrack(cid, cid, ti) + mt := p.addMediaTrack(cid, ti) potentialCodecs := make([]webrtc.RTPCodecParameters, 0, len(ti.Codecs)) parameters := rtpReceiver.GetParameters() @@ -2532,39 +3311,47 @@ func (p *ParticipantImpl) addMigratedTrack(cid string, ti *livekit.TrackInfo) *M for _, codec := range ti.Codecs { for ssrc, info := range p.params.SimTracks { - if info.Mid == codec.Mid { - mt.SetLayerSsrc(mime.NormalizeMimeType(codec.MimeType), info.Rid, ssrc) + if info.Mid == codec.Mid && !info.IsRepairStream { + mt.SetLayerSsrcsForRid(mime.NormalizeMimeType(codec.MimeType), info.StreamID, ssrc, info.RepairSSRC) } } } - mt.SetSimulcast(ti.Simulcast) - mt.SetMuted(ti.Muted) return mt } -func (p *ParticipantImpl) addMediaTrack(signalCid string, sdpCid string, ti *livekit.TrackInfo) *MediaTrack { +func (p *ParticipantImpl) addMediaTrack(signalCid string, ti *livekit.TrackInfo) *MediaTrack { mt := NewMediaTrack(MediaTrackParams{ - SignalCid: signalCid, - SdpCid: sdpCid, - ParticipantID: p.params.SID, - ParticipantIdentity: p.params.Identity, - ParticipantVersion: p.version.Load(), - BufferFactory: p.params.Config.BufferFactory, - ReceiverConfig: p.params.Config.Receiver, - AudioConfig: p.params.AudioConfig, - VideoConfig: p.params.VideoConfig, - Telemetry: p.params.Telemetry, - Logger: LoggerWithTrack(p.pubLogger, livekit.TrackID(ti.Sid), false), - SubscriberConfig: p.params.Config.Subscriber, - PLIThrottleConfig: p.params.PLIThrottleConfig, - SimTracks: p.params.SimTracks, - OnRTCP: p.postRtcp, - ForwardStats: p.params.ForwardStats, - OnTrackEverSubscribed: p.sendTrackHasBeenSubscribed, + ParticipantID: p.ID, + ParticipantIdentity: p.params.Identity, + ParticipantVersion: p.version.Load(), + ParticipantCountry: p.params.Country, + ParticipantKind: p.Kind(), + ParticipantKindDetails: p.KindDetails(), + BufferFactory: p.params.Config.BufferFactory, + ReceiverConfig: p.params.Config.Receiver, + AudioConfig: p.params.AudioConfig, + VideoConfig: p.params.VideoConfig, + TelemetryListener: p.params.TelemetryListener, + Logger: LoggerWithTrack(p.pubLogger, livekit.TrackID(ti.Sid), false), + Reporter: p.params.Reporter.WithTrack(ti.Sid), + SubscriberConfig: p.params.Config.Subscriber, + PLIThrottleConfig: p.params.PLIThrottleConfig, + SimTracks: p.params.SimTracks, + OnRTCP: p.postRtcp, + ForwardStats: p.params.ForwardStats, + OnTrackEverSubscribed: p.sendTrackHasBeenSubscribed, + ShouldRegressCodec: func() bool { + return p.helper().ShouldRegressCodec() + }, + PreferVideoSizeFromMedia: p.params.PreferVideoSizeFromMedia, + EnableRTPStreamRestartDetection: p.params.EnableRTPStreamRestartDetection, + UpdateTrackInfoByVideoSizeChange: p.params.UseOneShotSignallingMode, + ForceBackupCodecPolicySimulcast: p.params.ForceBackupCodecPolicySimulcast, }, ti) mt.OnSubscribedMaxQualityChange(p.onSubscribedMaxQualityChange) + mt.OnSubscribedAudioCodecChange(p.onSubscribedAudioCodecChange) // add to published and clean up pending if p.supervisor != nil { @@ -2592,17 +3379,16 @@ func (p *ParticipantImpl) addMediaTrack(signalCid string, sdpCid string, ti *liv } trackID := livekit.TrackID(ti.Sid) - mt.AddOnClose(func(_isExpectedToRsume bool) { + mt.AddOnClose(func(isExpectedToResume bool) { if p.supervisor != nil { p.supervisor.ClearPublishedTrack(trackID, mt) } - p.params.Telemetry.TrackUnpublished( - context.Background(), + p.params.TelemetryListener.OnTrackUnpublished( p.ID(), p.Identity(), mt.ToProto(), - true, + !isExpectedToResume, ) p.pendingTracksLock.Lock() @@ -2615,27 +3401,28 @@ func (p *ParticipantImpl) addMediaTrack(signalCid string, sdpCid string, ti *liv p.dirty.Store(true) - p.pubLogger.Debugw("track unpublished", "trackID", ti.Sid, "track", logger.Proto(ti)) - if onTrackUnpublished := p.getOnTrackUnpublished(); onTrackUnpublished != nil { - onTrackUnpublished(p, mt) - } + p.pubLogger.Debugw( + "track unpublished", + "trackID", ti.Sid, + "expectedToResume", isExpectedToResume, + "track", logger.Proto(ti), + ) + p.listener().OnTrackUnpublished(p, mt) }) return mt } -func (p *ParticipantImpl) handleTrackPublished(track types.MediaTrack) { - if onTrackPublished := p.getOnTrackPublished(); onTrackPublished != nil { - onTrackPublished(p, track) - } +func (p *ParticipantImpl) handleTrackPublished(track types.MediaTrack, isMigrated bool) { + p.listener().OnTrackPublished(p, track) // send webhook after callbacks are complete, persistence and state handling happens // in `onTrackPublished` cb - p.params.Telemetry.TrackPublished( - context.Background(), + p.params.TelemetryListener.OnTrackPublished( p.ID(), p.Identity(), track.ToProto(), + !isMigrated, ) p.pendingTracksLock.Lock() @@ -2666,7 +3453,7 @@ func (p *ParticipantImpl) onUpTrackManagerClose() { p.pubRTCPQueue.Stop() } -func (p *ParticipantImpl) getPendingTrack(clientId string, kind livekit.TrackType, skipQueued bool) (string, *livekit.TrackInfo, bool, time.Time) { +func (p *ParticipantImpl) getPendingTrack(clientId string, kind livekit.TrackType, skipQueued bool) (string, *livekit.TrackInfo, buffer.VideoLayersRid, bool, time.Time) { signalCid := clientId pendingInfo := p.pendingTracks[clientId] if pendingInfo == nil { @@ -2701,17 +3488,31 @@ func (p *ParticipantImpl) getPendingTrack(clientId string, kind livekit.TrackTyp // if still not found, we are done if pendingInfo == nil || (skipQueued && pendingInfo.queued) { - return signalCid, nil, false, time.Time{} + return signalCid, nil, buffer.VideoLayersRid{}, false, time.Time{} } - return signalCid, utils.CloneProto(pendingInfo.trackInfos[0]), pendingInfo.migrated, pendingInfo.createdAt + return signalCid, utils.CloneProto(pendingInfo.trackInfos[0]), pendingInfo.sdpRids, pendingInfo.migrated, pendingInfo.createdAt +} + +func (p *ParticipantImpl) getPendingTrackPrimaryBySdpCid(sdpCid string) *pendingTrackInfo { + for _, pti := range p.pendingTracks { + ti := pti.trackInfos[0] + if len(ti.Codecs) == 0 { + continue + } + if ti.Codecs[0].Cid == sdpCid || ti.Codecs[0].SdpCid == sdpCid { + return pti + } + } + + return nil } // setTrackID either generates a new TrackID for an AddTrackRequest func (p *ParticipantImpl) setTrackID(cid string, info *livekit.TrackInfo) { var trackID string // if already pending, use the same SID - // should not happen as this means multiple `AddTrack` requests have been called, but check anyway + // it is possible to have multiple AddTrackRequests for the same track if pti := p.pendingTracks[cid]; pti != nil { trackID = pti.trackInfos[0].Sid } @@ -2719,11 +3520,13 @@ func (p *ParticipantImpl) setTrackID(cid string, info *livekit.TrackInfo) { // otherwise generate if trackID == "" { trackPrefix := utils.TrackPrefix - if info.Type == livekit.TrackType_VIDEO { + switch info.Type { + case livekit.TrackType_VIDEO: trackPrefix += "V" - } else if info.Type == livekit.TrackType_AUDIO { + case livekit.TrackType_AUDIO: trackPrefix += "A" } + switch info.Source { case livekit.TrackSource_CAMERA: trackPrefix += "C" @@ -2741,7 +3544,8 @@ func (p *ParticipantImpl) setTrackID(cid string, info *livekit.TrackInfo) { func (p *ParticipantImpl) getPublishedTrackBySignalCid(clientId string) types.MediaTrack { for _, publishedTrack := range p.GetPublishedTracks() { - if publishedTrack.(types.LocalMediaTrack).SignalCid() == clientId { + if publishedTrack.(types.LocalMediaTrack).HasSignalCid(clientId) { + p.pubLogger.Debugw("found track by signal cid", "signalCid", clientId, "trackID", publishedTrack.ID()) return publishedTrack } } @@ -2760,13 +3564,13 @@ func (p *ParticipantImpl) getPublishedTrackBySdpCid(clientId string) types.Media return nil } -func (p *ParticipantImpl) DebugInfo() map[string]interface{} { - info := map[string]interface{}{ - "ID": p.params.SID, +func (p *ParticipantImpl) DebugInfo() map[string]any { + info := map[string]any{ + "ID": p.ID(), "State": p.State().String(), } - pendingTrackInfo := make(map[string]interface{}) + pendingTrackInfo := make(map[string]any) p.pendingTracksLock.RLock() for clientID, pti := range p.pendingTracks { var trackInfos []string @@ -2774,7 +3578,7 @@ func (p *ParticipantImpl) DebugInfo() map[string]interface{} { trackInfos = append(trackInfos, ti.String()) } - pendingTrackInfo[clientID] = map[string]interface{}{ + pendingTrackInfo[clientID] = map[string]any{ "TrackInfos": trackInfos, "Migrated": pti.migrated, } @@ -2825,16 +3629,14 @@ func (p *ParticipantImpl) setDownTracksConnected() { func (p *ParticipantImpl) cacheForwarderState() { // if migrating in, get forwarder state from migrating out node to facilitate resume - if f := p.params.GetSubscriberForwarderState; f != nil { - if fs, err := f(p); err == nil { - p.lock.Lock() - p.forwarderState = fs - p.lock.Unlock() + if fs, err := p.helper().GetSubscriberForwarderState(p); err == nil && fs != nil { + p.lock.Lock() + p.forwarderState = fs + p.lock.Unlock() - for _, t := range p.SubscriptionManager.GetSubscribedTracks() { - if dt := t.DownTrack(); dt != nil { - dt.SeedState(sfu.DownTrackState{ForwarderState: p.getAndDeleteForwarderState(t.ID())}) - } + for _, t := range p.SubscriptionManager.GetSubscribedTracks() { + if dt := t.DownTrack(); dt != nil { + dt.SeedState(sfu.DownTrackState{ForwarderState: p.getAndDeleteForwarderState(t.ID())}) } } } @@ -2925,14 +3727,7 @@ func (p *ParticipantImpl) onSubscriptionError(trackID livekit.TrackID, fatal boo signalErr = livekit.SubscriptionError_SE_TRACK_NOTFOUND } - _ = p.writeMessage(&livekit.SignalResponse{ - Message: &livekit.SignalResponse_SubscriptionResponse{ - SubscriptionResponse: &livekit.SubscriptionResponse{ - TrackSid: string(trackID), - Err: signalErr, - }, - }, - }) + p.sendSubscriptionResponse(trackID, signalErr) if p.params.ReconnectOnSubscriptionError && fatal { p.subLogger.Infow("issuing full reconnect on subscription error", "trackID", trackID) @@ -2958,6 +3753,17 @@ func (p *ParticipantImpl) UpdateSubscribedQuality(nodeID livekit.NodeID, trackID return nil } +func (p *ParticipantImpl) UpdateSubscribedAudioCodecs(nodeID livekit.NodeID, trackID livekit.TrackID, codecs []*livekit.SubscribedAudioCodec) error { + track := p.GetPublishedTrack(trackID) + if track == nil { + p.pubLogger.Debugw("could not find track", "trackID", trackID) + return errors.New("could not find published track") + } + + track.(types.LocalMediaTrack).NotifySubscriptionNode(nodeID, codecs) + return nil +} + func (p *ParticipantImpl) UpdateMediaLoss(nodeID livekit.NodeID, trackID livekit.TrackID, fractionalLoss uint32) error { track := p.GetPublishedTrack(trackID) if track == nil { @@ -2974,23 +3780,60 @@ func (p *ParticipantImpl) GetPlayoutDelayConfig() *livekit.PlayoutDelay { } func (p *ParticipantImpl) SupportsSyncStreamID() bool { - return p.ProtocolVersion().SupportSyncStreamID() && !p.params.ClientInfo.isFirefox() && p.params.SyncStreams + return p.ProtocolVersion().SupportsSyncStreamID() && !p.params.ClientInfo.isFirefox() && p.params.SyncStreams } -func (p *ParticipantImpl) SupportsTransceiverReuse() bool { +func (p *ParticipantImpl) SupportsTransceiverReuse(mt types.MediaTrack) bool { + if !p.params.ClientInfo.SupportsTransceiverReuse() { + return false + } + if p.params.UseOneShotSignallingMode { return p.ProtocolVersion().SupportsTransceiverReuse() } - return p.ProtocolVersion().SupportsTransceiverReuse() && !p.SupportsSyncStreamID() + return p.ProtocolVersion().SupportsTransceiverReuse() && !p.SupportsSyncStreamID() && (!mt.IsEncrypted() || !p.params.DisableTransceiverReuseForE2EE) } -func (p *ParticipantImpl) SendDataPacket(kind livekit.DataPacket_Kind, encoded []byte) error { +func (p *ParticipantImpl) SendDataMessage(kind livekit.DataPacket_Kind, data []byte, sender livekit.ParticipantID, seq uint32) error { + if sender == "" || kind != livekit.DataPacket_RELIABLE || seq == 0 { + if p.State() != livekit.ParticipantInfo_ACTIVE { + return ErrDataChannelUnavailable + } + return p.TransportManager.SendDataMessage(kind, data) + } + + p.reliableDataInfo.joiningMessageLock.Lock() + if !p.reliableDataInfo.canWriteReliable { + if _, ok := p.reliableDataInfo.joiningMessageFirstSeqs[sender]; !ok { + p.reliableDataInfo.joiningMessageFirstSeqs[sender] = seq + } + p.reliableDataInfo.joiningMessageLock.Unlock() + return nil + } + + lastWrittenSeq, ok := p.reliableDataInfo.joiningMessageLastWrittenSeqs[sender] + if ok { + if seq <= lastWrittenSeq { + // already sent by replayJoiningReliableMessages + p.reliableDataInfo.joiningMessageLock.Unlock() + return nil + } else { + delete(p.reliableDataInfo.joiningMessageLastWrittenSeqs, sender) + } + } + + p.reliableDataInfo.joiningMessageLock.Unlock() + + return p.TransportManager.SendDataMessage(kind, data) +} + +func (p *ParticipantImpl) SendDataMessageUnlabeled(data []byte, useRaw bool, sender livekit.ParticipantIdentity) error { if p.State() != livekit.ParticipantInfo_ACTIVE { return ErrDataChannelUnavailable } - return p.TransportManager.SendDataPacket(kind, encoded) + return p.TransportManager.SendDataMessageUnlabeled(data, useRaw, sender) } func (p *ParticipantImpl) onDataSendError(err error) { @@ -3011,7 +3854,8 @@ func (p *ParticipantImpl) setupEnabledCodecs(publishEnabledCodecs []*livekit.Cod return false } - publishCodecs := make([]*livekit.Codec, 0, len(publishEnabledCodecs)) + publishCodecsAudio := make([]*livekit.Codec, 0, len(publishEnabledCodecs)) + publishCodecsVideo := make([]*livekit.Codec, 0, len(publishEnabledCodecs)) for _, c := range publishEnabledCodecs { if shouldDisable(c, disabledCodecs.GetCodecs()) || shouldDisable(c, disabledCodecs.GetPublish()) { continue @@ -3027,10 +3871,16 @@ func (p *ParticipantImpl) setupEnabledCodecs(publishEnabledCodecs []*livekit.Cod } else if mime.IsMimeTypeStringH264(c.Mime) { p.enabledPublishCodecs = append(p.enabledPublishCodecs, c) } else { - publishCodecs = append(publishCodecs, c) + if mime.IsMimeTypeStringAudio(c.Mime) { + publishCodecsAudio = append(publishCodecsAudio, c) + } else { + publishCodecsVideo = append(publishCodecsVideo, c) + } } } - p.enabledPublishCodecs = append(p.enabledPublishCodecs, publishCodecs...) + // list all video first and then audio to work around a client side issue with Flutter SDK 2.4.2 + p.enabledPublishCodecs = append(p.enabledPublishCodecs, publishCodecsVideo...) + p.enabledPublishCodecs = append(p.enabledPublishCodecs, publishCodecsAudio...) subscribeCodecs := make([]*livekit.Codec, 0, len(subscribeEnabledCodecs)) for _, c := range subscribeEnabledCodecs { @@ -3040,6 +3890,30 @@ func (p *ParticipantImpl) setupEnabledCodecs(publishEnabledCodecs []*livekit.Cod subscribeCodecs = append(subscribeCodecs, c) } p.enabledSubscribeCodecs = subscribeCodecs + p.params.Logger.Debugw( + "setup enabled codecs", + "publish", logger.ProtoSlice(p.enabledPublishCodecs), + "subscribe", logger.ProtoSlice(p.enabledSubscribeCodecs), + "disabled", logger.Proto(disabledCodecs), + ) +} + +func (p *ParticipantImpl) replayJoiningReliableMessages() { + p.reliableDataInfo.joiningMessageLock.Lock() + for _, msgCache := range p.helper().GetCachedReliableDataMessage(p.reliableDataInfo.joiningMessageFirstSeqs) { + if len(msgCache.DestIdentities) != 0 && !slices.Contains(msgCache.DestIdentities, p.Identity()) { + continue + } + if lastSeq, ok := p.reliableDataInfo.joiningMessageLastWrittenSeqs[msgCache.SenderID]; !ok || lastSeq < msgCache.Seq { + p.reliableDataInfo.joiningMessageLastWrittenSeqs[msgCache.SenderID] = msgCache.Seq + } + + p.TransportManager.SendDataMessage(livekit.DataPacket_RELIABLE, msgCache.Data) + } + + p.reliableDataInfo.joiningMessageFirstSeqs = make(map[livekit.ParticipantID]uint32) + p.reliableDataInfo.canWriteReliable = true + p.reliableDataInfo.joiningMessageLock.Unlock() } func (p *ParticipantImpl) GetEnabledPublishCodecs() []*livekit.Codec { @@ -3065,7 +3939,7 @@ func (p *ParticipantImpl) UpdateAudioTrack(update *livekit.UpdateLocalAudioTrack if ti.Sid == update.TrackSid { isPending = true - ti.AudioFeatures = update.Features + ti.AudioFeatures = sutils.DedupeSlice(update.Features) ti.Stereo = false ti.DisableDtx = false for _, feature := range update.Features { @@ -3087,6 +3961,12 @@ func (p *ParticipantImpl) UpdateAudioTrack(update *livekit.UpdateLocalAudioTrack } p.pubLogger.Debugw("could not locate track", "trackID", update.TrackSid) + p.sendRequestResponse(&livekit.RequestResponse{ + Reason: livekit.RequestResponse_NOT_FOUND, + Request: &livekit.RequestResponse_UpdateAudioTrack{ + UpdateAudioTrack: utils.CloneProto(update), + }, + }) return errors.New("could not find track") } @@ -3113,6 +3993,12 @@ func (p *ParticipantImpl) UpdateVideoTrack(update *livekit.UpdateLocalVideoTrack } p.pubLogger.Debugw("could not locate track", "trackID", update.TrackSid) + p.sendRequestResponse(&livekit.RequestResponse{ + Reason: livekit.RequestResponse_NOT_FOUND, + Request: &livekit.RequestResponse_UpdateVideoTrack{ + UpdateVideoTrack: utils.CloneProto(update), + }, + }) return errors.New("could not find track") } @@ -3137,29 +4023,262 @@ func (p *ParticipantImpl) SupportsCodecChange() bool { return p.params.ClientInfo.SupportsCodecChange() } -// ---------------------------------------------- - -func codecsFromMediaDescription(m *sdp.MediaDescription) (out []sdp.Codec, err error) { - s := &sdp.SessionDescription{ - MediaDescriptions: []*sdp.MediaDescription{m}, +func (p *ParticipantImpl) SupportsMoving() error { + if !p.ProtocolVersion().SupportsMoving() { + return ErrMoveOldClientVersion } - for _, payloadStr := range m.MediaName.Formats { - payloadType, err := strconv.ParseUint(payloadStr, 10, 8) - if err != nil { - return nil, err - } - - codec, err := s.GetCodecForPayloadType(uint8(payloadType)) - if err != nil { - if payloadType == 0 { - continue - } - return nil, err - } - - out = append(out, codec) + if kind := p.Kind(); kind == livekit.ParticipantInfo_EGRESS || kind == livekit.ParticipantInfo_AGENT || p.params.UseOneShotSignallingMode { + return fmt.Errorf("%s participants cannot be moved, one-shot signaling mode: %t", kind.String(), p.params.UseOneShotSignallingMode) } - return out, nil + return nil +} + +func (p *ParticipantImpl) MoveToRoom(params types.MoveToRoomParams) { + for _, track := range p.GetPublishedTracks() { + for _, sub := range track.GetAllSubscribers() { + track.RemoveSubscriber(sub, false) + } + + // clear the subscriber node max quality/audio codecs as the remote quality notify + // from source room would not reach the moving out participant. + track.(types.LocalMediaTrack).ClearSubscriberNodes() + + trackInfo := track.ToProto() + p.params.TelemetryListener.OnTrackUnpublished( + p.ID(), + p.Identity(), + trackInfo, + true, + ) + } + + // fire onClose callback for original room + p.lock.Lock() + onClose := p.onClose + p.onClose = make(map[string]func(types.LocalParticipant)) + p.lock.Unlock() + for _, cb := range onClose { + cb(p) + } + + p.params.Logger.Infow("move participant to new room", "newRoomName", params.RoomName, "newID", params.ParticipantID) + + p.lock.Lock() + p.telemetryGuard = &telemetry.ReferenceGuard{} + p.lock.Unlock() + + p.params.Reporter.ReportEndTime(time.Now()) + p.params.LoggerResolver.Reset() + p.params.ReporterResolver.Reset() + p.setListener(params.Listener) + p.participantHelper.Store(params.Helper) + p.SubscriptionManager.ClearAllSubscriptions() + p.id.Store(params.ParticipantID) + grants := p.grants.Load().Clone() + grants.Video.Room = string(params.RoomName) + p.grants.Store(grants) +} + +func (p *ParticipantImpl) helper() types.LocalParticipantHelper { + return p.participantHelper.Load().(types.LocalParticipantHelper) +} + +func (p *ParticipantImpl) GetLastReliableSequence(migrateOut bool) uint32 { + if migrateOut { + p.reliableDataInfo.stopReliableByMigrateOut.Store(true) + } + return p.reliableDataInfo.lastPubReliableSeq.Load() +} + +func (p *ParticipantImpl) HandleUpdateSubscriptions( + trackIDs []livekit.TrackID, + participantTracks []*livekit.ParticipantTracks, + subscribe bool, +) { + p.listener().OnUpdateSubscriptions(p, trackIDs, participantTracks, subscribe) +} + +func (p *ParticipantImpl) HandleUpdateSubscriptionPermission(subscriptionPermission *livekit.SubscriptionPermission) error { + return p.listener().OnUpdateSubscriptionPermission(p, subscriptionPermission) +} + +func (p *ParticipantImpl) HandleSyncState(syncState *livekit.SyncState) error { + return p.listener().OnSyncState(p, syncState) +} + +func (p *ParticipantImpl) HandleSimulateScenario(simulateScenario *livekit.SimulateScenario) error { + return p.listener().OnSimulateScenario(p, simulateScenario) +} + +func (p *ParticipantImpl) HandleLeaveRequest(reason types.ParticipantCloseReason) { + p.listener().OnLeave(p, reason) +} + +func (p *ParticipantImpl) HandleSignalMessage(msg proto.Message) error { + return p.signalHandler.HandleMessage(msg) +} + +func (p *ParticipantImpl) IsUsingSinglePeerConnection() bool { + return p.params.UseSinglePeerConnection +} + +func (p *ParticipantImpl) AddTrackLocal( + trackLocal webrtc.TrackLocal, + params types.AddTrackParams, +) (*webrtc.RTPSender, *webrtc.RTPTransceiver, error) { + if p.params.UseSinglePeerConnection { + return p.TransportManager.AddTrackLocal( + trackLocal, + params, + p.enabledSubscribeCodecs, + p.params.Config.Subscriber.RTCPFeedback, + ) + } else { + return p.TransportManager.AddTrackLocal(trackLocal, params, nil, RTCPFeedbackConfig{}) + } +} + +func (p *ParticipantImpl) AddTransceiverFromTrackLocal( + trackLocal webrtc.TrackLocal, + params types.AddTrackParams, +) (*webrtc.RTPSender, *webrtc.RTPTransceiver, error) { + if p.params.UseSinglePeerConnection { + return p.TransportManager.AddTransceiverFromTrackLocal( + trackLocal, + params, + p.enabledSubscribeCodecs, + p.params.Config.Subscriber.RTCPFeedback, + ) + } else { + return p.TransportManager.AddTransceiverFromTrackLocal( + trackLocal, + params, + nil, + RTCPFeedbackConfig{}, + ) + } +} + +func (p *ParticipantImpl) handleIncomingRpcAck(requestId string) bool { + p.rpcLock.Lock() + defer p.rpcLock.Unlock() + + handler, ok := p.rpcPendingAcks[requestId] + if !ok { + return false + } + + handler.Resolve() + delete(p.rpcPendingAcks, requestId) + return true +} + +func (p *ParticipantImpl) handleIncomingRpcResponse(requestId string, payload string, err *utils.DataChannelRpcError) bool { + p.rpcLock.Lock() + defer p.rpcLock.Unlock() + + handler, ok := p.rpcPendingResponses[requestId] + if !ok { + return false + } + + handler.Resolve(payload, err) + delete(p.rpcPendingResponses, requestId) + return true +} + +func (p *ParticipantImpl) PerformRpc(req *livekit.PerformRpcRequest, resultCh chan string, errorCh chan error) { + responseTimeout := req.GetResponseTimeoutMs() + if responseTimeout <= 0 { + responseTimeout = uint32(utils.DataChannelRpcDefaultResponseTimeout.Milliseconds()) + } + + go func() { + if len([]byte(req.GetPayload())) > utils.DataChannelRpcMaxPayloadBytes { + errorCh <- utils.DataChannelRpcErrorFromBuiltInCodes(utils.DataChannelRpcRequestPayloadTooLarge, "").PsrpcError() + return + } + + id := uuid.NewString() + + responseTimer := time.AfterFunc(time.Duration(responseTimeout)*time.Millisecond, func() { + p.rpcLock.Lock() + delete(p.rpcPendingResponses, id) + p.rpcLock.Unlock() + + select { + case errorCh <- utils.DataChannelRpcErrorFromBuiltInCodes(utils.DataChannelRpcResponseTimeout, "").PsrpcError(): + default: + } + }) + ackTimer := time.AfterFunc(utils.DataChannelRpcMaxRoundTripLatency, func() { + p.rpcLock.Lock() + delete(p.rpcPendingAcks, id) + delete(p.rpcPendingResponses, id) + p.rpcLock.Unlock() + responseTimer.Stop() + + select { + case errorCh <- utils.DataChannelRpcErrorFromBuiltInCodes(utils.DataChannelRpcConnectionTimeout, "").PsrpcError(): + default: + } + }) + + rpcRequest := &livekit.DataPacket{ + Kind: livekit.DataPacket_RELIABLE, + ParticipantIdentity: id, + Value: &livekit.DataPacket_RpcRequest{ + RpcRequest: &livekit.RpcRequest{ + Id: id, + Method: req.GetMethod(), + Payload: req.GetPayload(), + ResponseTimeoutMs: responseTimeout - p.lastRTT, + Version: 1, + }, + }, + } + data, err := proto.Marshal(rpcRequest) + if err != nil { + ackTimer.Stop() + responseTimer.Stop() + errorCh <- psrpc.NewError(psrpc.Internal, err) + return + } + + // using RPC ID as the unique ID for server to identify the response + err = p.SendDataMessage(livekit.DataPacket_RELIABLE, data, livekit.ParticipantID(id), 0) + if err != nil { + ackTimer.Stop() + responseTimer.Stop() + errorCh <- psrpc.NewError(psrpc.Internal, err) + return + } + + p.rpcLock.Lock() + p.rpcPendingAcks[id] = &utils.DataChannelRpcPendingAckHandler{ + Resolve: func() { + ackTimer.Stop() + }, + ParticipantIdentity: req.GetDestinationIdentity(), + } + p.rpcPendingResponses[id] = &utils.DataChannelRpcPendingResponseHandler{ + Resolve: func(payload string, error *utils.DataChannelRpcError) { + responseTimer.Stop() + if _, ok := p.rpcPendingAcks[id]; ok { + p.rpcPendingAcks[id].Resolve() + ackTimer.Stop() + } + + if error != nil { + errorCh <- error.PsrpcError() + } else { + resultCh <- payload + } + }, + ParticipantIdentity: req.GetDestinationIdentity(), + } + p.rpcLock.Unlock() + }() } diff --git a/pkg/rtc/participant_data_track.go b/pkg/rtc/participant_data_track.go new file mode 100644 index 0000000..829beaf --- /dev/null +++ b/pkg/rtc/participant_data_track.go @@ -0,0 +1,169 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rtc + +import ( + "github.com/livekit/livekit-server/pkg/rtc/datatrack" + "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/logger" + "github.com/livekit/protocol/utils" + "github.com/livekit/protocol/utils/guid" +) + +func (p *ParticipantImpl) HandlePublishDataTrackRequest(req *livekit.PublishDataTrackRequest) { + if !p.CanPublishData() || !p.params.EnableDataTracks { + p.pubLogger.Warnw("no permission to publish data track", nil, "req", logger.Proto(req)) + p.sendRequestResponse(&livekit.RequestResponse{ + Reason: livekit.RequestResponse_NOT_ALLOWED, + Message: "does not have permission to publish data", + Request: &livekit.RequestResponse_PublishDataTrack{ + PublishDataTrack: utils.CloneProto(req), + }, + }) + return + } + + if req.PubHandle == 0 || req.PubHandle > 65535 { + p.pubLogger.Warnw("invalid data track handle", nil, "req", logger.Proto(req)) + p.sendRequestResponse(&livekit.RequestResponse{ + Reason: livekit.RequestResponse_INVALID_HANDLE, + Message: "handle should be > 0 AND < 65536", + Request: &livekit.RequestResponse_PublishDataTrack{ + PublishDataTrack: utils.CloneProto(req), + }, + }) + return + } + + if len(req.Name) == 0 || len(req.Name) > 256 { + p.pubLogger.Warnw("invalid data track name", nil, "req", logger.Proto(req)) + p.sendRequestResponse(&livekit.RequestResponse{ + Reason: livekit.RequestResponse_INVALID_NAME, + Message: "name should not be empty and should not exceed 256 characters", + Request: &livekit.RequestResponse_PublishDataTrack{ + PublishDataTrack: utils.CloneProto(req), + }, + }) + return + } + + publishedDataTracks := p.UpDataTrackManager.GetPublishedDataTracks() + for _, dt := range publishedDataTracks { + message := "" + reason := livekit.RequestResponse_OK + switch { + case dt.PubHandle() == uint16(req.PubHandle): + message = "a data track with same handle already exists" + reason = livekit.RequestResponse_DUPLICATE_HANDLE + case dt.Name() == req.Name: + message = "a data track with same name already exists" + reason = livekit.RequestResponse_DUPLICATE_NAME + } + if message != "" { + p.pubLogger.Warnw( + "cannot publish duplicate data track", nil, + "req", logger.Proto(req), + "existing", logger.Proto(dt.ToProto()), + ) + p.sendRequestResponse(&livekit.RequestResponse{ + Reason: reason, + Message: message, + Request: &livekit.RequestResponse_PublishDataTrack{ + PublishDataTrack: utils.CloneProto(req), + }, + }) + return + } + } + + dti := &livekit.DataTrackInfo{ + PubHandle: req.PubHandle, + Sid: guid.New(utils.DataTrackPrefix), + Name: req.Name, + Encryption: req.Encryption, + } + dt := NewDataTrack( + DataTrackParams{ + Logger: p.params.Logger.WithValues("trackID", dti.Sid), + ParticipantID: p.ID, + ParticipantIdentity: p.params.Identity, + BytesTrackStats: NewBytesTrackStats( + p.params.Country, + livekit.TrackID(dti.Sid), + p.ID(), + p.Kind(), + p.KindDetails(), + p.params.TelemetryListener, + p.params.Reporter, + ), + }, + dti, + ) + + p.UpDataTrackManager.AddPublishedDataTrack(dt) + + p.sendPublishDataTrackResponse(dti) + + p.setIsPublisher(true) + p.dirty.Store(true) +} + +func (p *ParticipantImpl) HandleUnpublishDataTrackRequest(req *livekit.UnpublishDataTrackRequest) { + dt := p.UpDataTrackManager.GetPublishedDataTrack(uint16(req.PubHandle)) + if dt == nil { + p.pubLogger.Warnw("unpublish data track not found", nil, "req", logger.Proto(req)) + p.sendRequestResponse(&livekit.RequestResponse{ + Reason: livekit.RequestResponse_NOT_FOUND, + Request: &livekit.RequestResponse_UnpublishDataTrack{ + UnpublishDataTrack: utils.CloneProto(req), + }, + }) + return + } + + p.UpDataTrackManager.RemovePublishedDataTrack(dt) + + p.sendUnpublishDataTrackResponse(dt.ToProto()) + + p.dirty.Store(true) +} + +func (p *ParticipantImpl) HandleUpdateDataSubscription(req *livekit.UpdateDataSubscription) { + p.listener().OnUpdateDataSubscriptions(p, req) +} + +func (p *ParticipantImpl) onReceivedDataTrackMessage(data []byte, arrivalTime int64) { + var packet datatrack.Packet + if err := packet.Unmarshal(data); err != nil { + p.params.Logger.Errorw("could not unmarshal data track message", err) + return + } + + p.UpDataTrackManager.HandleReceivedDataTrackMessage(data, &packet, arrivalTime) + + p.listener().OnDataTrackMessage(p, data, &packet) +} + +func (p *ParticipantImpl) GetNextSubscribedDataTrackHandle() uint16 { + p.lock.Lock() + defer p.lock.Unlock() + + p.nextSubscribedDataTrackHandle++ + if p.nextSubscribedDataTrackHandle == 0 { + p.nextSubscribedDataTrackHandle++ + } + + return p.nextSubscribedDataTrackHandle +} diff --git a/pkg/rtc/participant_internal_test.go b/pkg/rtc/participant_internal_test.go index db9ad1e..781747a 100644 --- a/pkg/rtc/participant_internal_test.go +++ b/pkg/rtc/participant_internal_test.go @@ -26,11 +26,14 @@ import ( "google.golang.org/protobuf/proto" "github.com/livekit/livekit-server/pkg/sfu/buffer" - "github.com/livekit/livekit-server/pkg/sfu/mime" - "github.com/livekit/livekit-server/pkg/telemetry/telemetryfakes" "github.com/livekit/protocol/auth" + protoCodecs "github.com/livekit/protocol/codecs" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" + "github.com/livekit/protocol/observability/roomobs" + lksdp "github.com/livekit/protocol/sdp" + "github.com/livekit/protocol/signalling" "github.com/livekit/protocol/utils" "github.com/livekit/protocol/utils/guid" @@ -81,14 +84,14 @@ func TestTrackPublishing(t *testing.T) { track.IDReturns("id") published := false updated := false - p.OnTrackUpdated(func(p types.LocalParticipant, track types.MediaTrack) { + p.listener().(*typesfakes.FakeLocalParticipantListener).OnTrackUpdatedCalls(func(p types.Participant, track types.MediaTrack) { updated = true }) - p.OnTrackPublished(func(p types.LocalParticipant, track types.MediaTrack) { + p.listener().(*typesfakes.FakeLocalParticipantListener).OnTrackPublishedCalls(func(p types.Participant, track types.MediaTrack) { published = true }) p.UpTrackManager.AddPublishedTrack(track) - p.handleTrackPublished(track) + p.handleTrackPublished(track, false) require.True(t, published) require.False(t, updated) require.Len(t, p.UpTrackManager.publishedTracks, 1) @@ -129,7 +132,8 @@ func TestTrackPublishing(t *testing.T) { Type: livekit.TrackType_AUDIO, }) - require.Equal(t, 1, sink.WriteMessageCallCount()) + // error response on duplicate adds a message + require.Equal(t, 2, sink.WriteMessageCallCount()) }) t.Run("should queue adding of duplicate tracks if already published by client id in signalling", func(t *testing.T) { @@ -137,7 +141,7 @@ func TestTrackPublishing(t *testing.T) { sink := p.params.Sink.(*routingfakes.FakeMessageSink) track := &typesfakes.FakeLocalMediaTrack{} - track.SignalCidReturns("cid") + track.HasSignalCidCalls(func(s string) bool { return s == "cid" }) track.ToProtoReturns(&livekit.TrackInfo{}) // directly add to publishedTracks without lock - for testing purpose only p.UpTrackManager.publishedTracks["cid"] = track @@ -147,7 +151,8 @@ func TestTrackPublishing(t *testing.T) { Name: "webcam", Type: livekit.TrackType_VIDEO, }) - require.Equal(t, 0, sink.WriteMessageCallCount()) + // `queued` `RequestResponse` should add a message + require.Equal(t, 1, sink.WriteMessageCallCount()) require.Equal(t, 1, len(p.pendingTracks["cid"].trackInfos)) // add again - it should be added to the queue @@ -156,7 +161,8 @@ func TestTrackPublishing(t *testing.T) { Name: "webcam", Type: livekit.TrackType_VIDEO, }) - require.Equal(t, 0, sink.WriteMessageCallCount()) + // `queued` `RequestResponse`s should have been sent for duplicate additions + require.Equal(t, 2, sink.WriteMessageCallCount()) require.Equal(t, 2, len(p.pendingTracks["cid"].trackInfos)) // check SID is the same @@ -178,7 +184,8 @@ func TestTrackPublishing(t *testing.T) { Name: "webcam", Type: livekit.TrackType_VIDEO, }) - require.Equal(t, 0, sink.WriteMessageCallCount()) + // `queued` `RequestResponse` should add a message + require.Equal(t, 1, sink.WriteMessageCallCount()) require.Equal(t, 1, len(p.pendingTracks["cid"].trackInfos)) // add again - it should be added to the queue @@ -187,7 +194,8 @@ func TestTrackPublishing(t *testing.T) { Name: "webcam", Type: livekit.TrackType_VIDEO, }) - require.Equal(t, 0, sink.WriteMessageCallCount()) + // `queued` `RequestResponse`s should have been sent for duplicate additions + require.Equal(t, 2, sink.WriteMessageCallCount()) require.Equal(t, 2, len(p.pendingTracks["cid"].trackInfos)) // check SID is the same @@ -217,7 +225,8 @@ func TestTrackPublishing(t *testing.T) { Type: livekit.TrackType_AUDIO, Source: livekit.TrackSource_MICROPHONE, }) - require.Equal(t, 1, sink.WriteMessageCallCount()) + // an error response for disallowed source should send a `RequestResponse`. + require.Equal(t, 2, sink.WriteMessageCallCount()) }) } @@ -225,7 +234,7 @@ func TestOutOfOrderUpdates(t *testing.T) { p := newParticipantForTest("test") p.updateState(livekit.ParticipantInfo_JOINED) p.SetMetadata("initial metadata") - sink := p.getResponseSink().(*routingfakes.FakeMessageSink) + sink := p.GetResponseSink().(*routingfakes.FakeMessageSink) pi1 := p.ToProto() p.SetMetadata("second update") pi2 := p.ToProto() @@ -255,7 +264,7 @@ func TestDisconnectTiming(t *testing.T) { }() track := &typesfakes.FakeMediaTrack{} p.UpTrackManager.AddPublishedTrack(track) - p.handleTrackPublished(track) + p.handleTrackPublished(track, false) // close channel and then try to Negotiate msg.Close() @@ -275,7 +284,10 @@ func TestMuteSetting(t *testing.T) { ti := &livekit.TrackInfo{Sid: "testTrack"} p.pendingTracks["cid"] = &pendingTrackInfo{trackInfos: []*livekit.TrackInfo{ti}} - p.SetTrackMuted(livekit.TrackID(ti.Sid), true, false) + p.SetTrackMuted(&livekit.MuteTrackRequest{ + Sid: ti.Sid, + Muted: true, + }, false) require.True(t, p.pendingTracks["cid"].trackInfos[0].Muted) }) @@ -287,7 +299,7 @@ func TestMuteSetting(t *testing.T) { Muted: true, }) - _, ti, _, _ := p.getPendingTrack("cid", livekit.TrackType_AUDIO, false) + _, ti, _, _, _ := p.getPendingTrack("cid", livekit.TrackType_AUDIO, false) require.NotNil(t, ti) require.True(t, ti.Muted) }) @@ -362,28 +374,36 @@ func TestDisableCodecs(t *testing.T) { } } require.True(t, found264) + offerId := uint32(42) // negotiated codec should not contain h264 sink := &routingfakes.FakeMessageSink{} - participant.SetResponseSink(sink) + participant.SwapResponseSink(sink, types.SignallingCloseReasonUnknown) var answer webrtc.SessionDescription + var answerId uint32 var answerReceived atomic.Bool + var answerIdReceived atomic.Uint32 sink.WriteMessageCalls(func(msg proto.Message) error { if res, ok := msg.(*livekit.SignalResponse); ok { if res.GetAnswer() != nil { - answer = FromProtoSessionDescription(res.GetAnswer()) + answer, answerId, _ = signalling.FromProtoSessionDescription(res.GetAnswer()) answerReceived.Store(true) + answerIdReceived.Store(answerId) } } return nil }) - participant.HandleOffer(sdp) + participant.HandleOffer(&livekit.SessionDescription{ + Type: webrtc.SDPTypeOffer.String(), + Sdp: sdp.SDP, + Id: offerId, + }) testutils.WithTimeout(t, func() string { - if answerReceived.Load() { + if answerReceived.Load() && answerIdReceived.Load() == offerId { return "" } else { - return "answer not received" + return "answer not received OR answer id mismatch" } }) require.NoError(t, pc.SetRemoteDescription(answer), answer.SDP, sdp.SDP) @@ -415,7 +435,7 @@ func TestDisablePublishCodec(t *testing.T) { } sink := &routingfakes.FakeMessageSink{} - participant.SetResponseSink(sink) + participant.SwapResponseSink(sink, types.SignallingCloseReasonUnknown) var publishReceived atomic.Bool sink.WriteMessageCalls(func(msg proto.Message) error { if res, ok := msg.(*livekit.SignalResponse); ok { @@ -463,89 +483,138 @@ func TestDisablePublishCodec(t *testing.T) { require.Eventually(t, func() bool { return publishReceived.Load() }, 5*time.Second, 10*time.Millisecond) } -func TestPreferVideoCodecForPublisher(t *testing.T) { - participant := newParticipantForTestWithOpts("123", &participantOpts{ - publisher: true, - }) - participant.SetMigrateState(types.MigrateStateComplete) - - pc, err := webrtc.NewPeerConnection(webrtc.Configuration{}) - require.NoError(t, err) - defer pc.Close() - - for i := 0; i < 2; i++ { - // publish h264 track without client preferred codec - trackCid := fmt.Sprintf("preferh264video%d", i) - participant.AddTrack(&livekit.AddTrackRequest{ - Type: livekit.TrackType_VIDEO, - Name: "video", - Width: 1280, - Height: 720, - Source: livekit.TrackSource_CAMERA, - SimulcastCodecs: []*livekit.SimulcastCodec{ - { - Codec: "h264", - Cid: trackCid, - }, +func TestPreferMediaCodecForPublisher(t *testing.T) { + testCases := []struct { + name string + mediaKind string + trackBaseCid string + preferredCodec string + addTrack *livekit.AddTrackRequest + mimeTypeStringChecker func(string) bool + mimeTypeCodecStringChecker func(string) bool + transceiverMimeType mime.MimeType + }{ + { + name: "video", + mediaKind: "video", + trackBaseCid: "preferH264Video", + preferredCodec: "h264", + addTrack: &livekit.AddTrackRequest{ + Type: livekit.TrackType_VIDEO, + Name: "video", + Width: 1280, + Height: 720, + Source: livekit.TrackSource_CAMERA, }, - }) + mimeTypeStringChecker: mime.IsMimeTypeStringH264, + mimeTypeCodecStringChecker: mime.IsMimeTypeCodecStringH264, + transceiverMimeType: mime.MimeTypeVP8, + }, + { + name: "audio", + mediaKind: "audio", + trackBaseCid: "preferPCMAAudio", + preferredCodec: "pcma", + addTrack: &livekit.AddTrackRequest{ + Type: livekit.TrackType_AUDIO, + Name: "audio", + Source: livekit.TrackSource_MICROPHONE, + }, + mimeTypeStringChecker: mime.IsMimeTypeStringPCMA, + mimeTypeCodecStringChecker: mime.IsMimeTypeCodecStringPCMA, + transceiverMimeType: mime.MimeTypeOpus, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + participant := newParticipantForTestWithOpts("123", &participantOpts{ + publisher: true, + }) + participant.SetMigrateState(types.MigrateStateComplete) - track, err := webrtc.NewTrackLocalStaticRTP(webrtc.RTPCodecCapability{MimeType: "video/vp8"}, trackCid, trackCid) - require.NoError(t, err) - transceiver, err := pc.AddTransceiverFromTrack(track, webrtc.RTPTransceiverInit{Direction: webrtc.RTPTransceiverDirectionSendrecv}) - require.NoError(t, err) - codecs := transceiver.Receiver().GetParameters().Codecs + pc, err := webrtc.NewPeerConnection(webrtc.Configuration{}) + require.NoError(t, err) + defer pc.Close() - if i > 0 { - // the negotiated codecs order could be updated by first negotiation, reorder to make h264 not preferred - for mime.IsMimeTypeStringH264(codecs[0].MimeType) { - codecs = append(codecs[1:], codecs[0]) - } - } - // h264 should not be preferred - require.False(t, mime.IsMimeTypeStringH264(codecs[0].MimeType), "codecs", codecs) - - sdp, err := pc.CreateOffer(nil) - require.NoError(t, err) - require.NoError(t, pc.SetLocalDescription(sdp)) - - sink := &routingfakes.FakeMessageSink{} - participant.SetResponseSink(sink) - var answer webrtc.SessionDescription - var answerReceived atomic.Bool - sink.WriteMessageCalls(func(msg proto.Message) error { - if res, ok := msg.(*livekit.SignalResponse); ok { - if res.GetAnswer() != nil { - answer = FromProtoSessionDescription(res.GetAnswer()) - pc.SetRemoteDescription(answer) - answerReceived.Store(true) + for i := 0; i < 2; i++ { + // publish preferred track without client using setCodecPreferences() + trackCid := fmt.Sprintf("%s-%d", tc.trackBaseCid, i) + req := utils.CloneProto(tc.addTrack) + req.SimulcastCodecs = []*livekit.SimulcastCodec{ + { + Codec: tc.preferredCodec, + Cid: trackCid, + }, } - } - return nil - }) - participant.HandleOffer(sdp) + participant.AddTrack(req) - require.Eventually(t, func() bool { return answerReceived.Load() }, 5*time.Second, 10*time.Millisecond) + track, err := webrtc.NewTrackLocalStaticRTP(webrtc.RTPCodecCapability{MimeType: tc.transceiverMimeType.String()}, trackCid, trackCid) + require.NoError(t, err) + transceiver, err := pc.AddTransceiverFromTrack(track, webrtc.RTPTransceiverInit{Direction: webrtc.RTPTransceiverDirectionSendrecv}) + require.NoError(t, err) + codecs := transceiver.Receiver().GetParameters().Codecs - var h264Preferred bool - parsed, err := answer.Unmarshal() - require.NoError(t, err) - var videoSectionIndex int - for _, m := range parsed.MediaDescriptions { - if m.MediaName.Media == "video" { - if videoSectionIndex == i { - codecs, err := codecsFromMediaDescription(m) - require.NoError(t, err) - if mime.IsMimeTypeCodecStringH264(codecs[0].Name) { - h264Preferred = true - break + if i > 0 { + // the negotiated codecs order could be updated by first negotiation, + // reorder to make tested preferred codec not preferred + for tc.mimeTypeStringChecker(codecs[0].MimeType) { + codecs = append(codecs[1:], codecs[0]) } } - videoSectionIndex++ - } - } + // preferred codec should not be preferred in `offer` + require.False(t, tc.mimeTypeStringChecker(codecs[0].MimeType), "codecs", codecs) - require.Truef(t, h264Preferred, "h264 should be preferred for video section %d, answer sdp: \n%s", i, answer.SDP) + sdp, err := pc.CreateOffer(nil) + require.NoError(t, err) + require.NoError(t, pc.SetLocalDescription(sdp)) + offerId := uint32(23) + + sink := &routingfakes.FakeMessageSink{} + participant.SwapResponseSink(sink, types.SignallingCloseReasonUnknown) + var answer webrtc.SessionDescription + var answerId uint32 + var answerReceived atomic.Bool + var answerIdReceived atomic.Uint32 + sink.WriteMessageCalls(func(msg proto.Message) error { + if res, ok := msg.(*livekit.SignalResponse); ok { + if res.GetAnswer() != nil { + answer, answerId, _ = signalling.FromProtoSessionDescription(res.GetAnswer()) + pc.SetRemoteDescription(answer) + answerReceived.Store(true) + answerIdReceived.Store(answerId) + } + } + return nil + }) + participant.HandleOffer(&livekit.SessionDescription{ + Type: webrtc.SDPTypeOffer.String(), + Sdp: sdp.SDP, + Id: offerId, + }) + + require.Eventually(t, func() bool { return answerReceived.Load() && answerIdReceived.Load() == offerId }, 5*time.Second, 10*time.Millisecond) + + var havePreferred bool + parsed, err := answer.Unmarshal() + require.NoError(t, err) + var mediaSectionIndex int + for _, m := range parsed.MediaDescriptions { + if m.MediaName.Media == tc.mediaKind { + if mediaSectionIndex == i { + codecs, err := lksdp.CodecsFromMediaDescription(m) + require.NoError(t, err) + if tc.mimeTypeCodecStringChecker(codecs[0].Name) { + havePreferred = true + break + } + } + mediaSectionIndex++ + } + } + + require.Truef(t, havePreferred, "%s should be preferred for %s section %d, answer sdp: \n%s", tc.preferredCodec, tc.mediaKind, i, answer.SDP) + } + }) } } @@ -556,33 +625,59 @@ func TestPreferAudioCodecForRed(t *testing.T) { participant.SetMigrateState(types.MigrateStateComplete) me := webrtc.MediaEngine{} - me.RegisterDefaultCodecs() - require.NoError(t, me.RegisterCodec(webrtc.RTPCodecParameters{ - RTPCodecCapability: RedCodecCapability, - PayloadType: 63, - }, webrtc.RTPCodecTypeAudio)) + opusCodecParameters := protoCodecs.OpusCodecParameters + opusCodecParameters.RTPCodecCapability.RTCPFeedback = []webrtc.RTCPFeedback{{Type: webrtc.TypeRTCPFBNACK}} + require.NoError(t, me.RegisterCodec(opusCodecParameters, webrtc.RTPCodecTypeAudio)) + redCodecParameters := protoCodecs.RedCodecParameters + redCodecParameters.RTPCodecCapability.RTCPFeedback = []webrtc.RTCPFeedback{{Type: webrtc.TypeRTCPFBNACK}} + require.NoError(t, me.RegisterCodec(redCodecParameters, webrtc.RTPCodecTypeAudio)) api := webrtc.NewAPI(webrtc.WithMediaEngine(&me)) pc, err := api.NewPeerConnection(webrtc.Configuration{}) require.NoError(t, err) defer pc.Close() - for i, disableRed := range []bool{false, true} { + for idx, disableRed := range []bool{false, true, false, true} { t.Run(fmt.Sprintf("disableRed=%v", disableRed), func(t *testing.T) { - trackCid := fmt.Sprintf("audiotrack%d", i) - participant.AddTrack(&livekit.AddTrackRequest{ - Type: livekit.TrackType_AUDIO, - DisableRed: disableRed, - Cid: trackCid, - }) - track, err := webrtc.NewTrackLocalStaticRTP(webrtc.RTPCodecCapability{MimeType: "audio/opus"}, trackCid, trackCid) + trackCid := fmt.Sprintf("audiotrack%d", idx) + req := &livekit.AddTrackRequest{ + Type: livekit.TrackType_AUDIO, + Cid: trackCid, + } + if idx < 2 { + req.DisableRed = disableRed + } else { + codec := "red" + if disableRed { + codec = "opus" + } + req.SimulcastCodecs = []*livekit.SimulcastCodec{ + { + Codec: codec, + Cid: trackCid, + }, + } + } + participant.AddTrack(req) + + track, err := webrtc.NewTrackLocalStaticRTP( + webrtc.RTPCodecCapability{MimeType: "audio/opus"}, + trackCid, + trackCid, + ) require.NoError(t, err) - transceiver, err := pc.AddTransceiverFromTrack(track, webrtc.RTPTransceiverInit{Direction: webrtc.RTPTransceiverDirectionSendrecv}) + + transceiver, err := pc.AddTransceiverFromTrack( + track, + webrtc.RTPTransceiverInit{Direction: webrtc.RTPTransceiverDirectionSendrecv}, + ) require.NoError(t, err) codecs := transceiver.Sender().GetParameters().Codecs for i, c := range codecs { - if c.MimeType == "audio/opus" && i != 0 { - codecs[0], codecs[i] = codecs[i], codecs[0] + if c.MimeType == "audio/opus" { + if i != 0 { + codecs[0], codecs[i] = codecs[i], codecs[0] + } break } } @@ -592,24 +687,38 @@ func TestPreferAudioCodecForRed(t *testing.T) { pc.SetLocalDescription(sdp) // opus should be preferred require.Equal(t, codecs[0].MimeType, "audio/opus", sdp) + offerId := uint32(0xffffff) sink := &routingfakes.FakeMessageSink{} - participant.SetResponseSink(sink) + participant.SwapResponseSink(sink, types.SignallingCloseReasonUnknown) var answer webrtc.SessionDescription + var answerId uint32 var answerReceived atomic.Bool + var answerIdReceived atomic.Uint32 sink.WriteMessageCalls(func(msg proto.Message) error { if res, ok := msg.(*livekit.SignalResponse); ok { if res.GetAnswer() != nil { - answer = FromProtoSessionDescription(res.GetAnswer()) + answer, answerId, _ = signalling.FromProtoSessionDescription(res.GetAnswer()) pc.SetRemoteDescription(answer) answerReceived.Store(true) + answerIdReceived.Store(answerId) } } return nil }) - participant.HandleOffer(sdp) - - require.Eventually(t, func() bool { return answerReceived.Load() }, 5*time.Second, 10*time.Millisecond) + participant.HandleOffer(&livekit.SessionDescription{ + Type: webrtc.SDPTypeOffer.String(), + Sdp: sdp.SDP, + Id: offerId, + }) + require.Eventually( + t, + func() bool { + return answerReceived.Load() && answerIdReceived.Load() == offerId + }, + 5*time.Second, + 10*time.Millisecond, + ) var redPreferred bool parsed, err := answer.Unmarshal() @@ -617,8 +726,8 @@ func TestPreferAudioCodecForRed(t *testing.T) { var audioSectionIndex int for _, m := range parsed.MediaDescriptions { if m.MediaName.Media == "audio" { - if audioSectionIndex == i { - codecs, err := codecsFromMediaDescription(m) + if audioSectionIndex == idx { + codecs, err := lksdp.CodecsFromMediaDescription(m) require.NoError(t, err) // nack is always enabled. if red is preferred, server will not generate nack request var nackEnabled bool @@ -702,8 +811,11 @@ func newParticipantForTestWithOpts(identity livekit.ParticipantIdentity, opts *p ClientConf: opts.clientConf, ClientInfo: ClientInfo{ClientInfo: opts.clientInfo}, Logger: LoggerWithParticipant(logger.GetLogger(), identity, sid, false), - Telemetry: &telemetryfakes.FakeTelemetryService{}, + Reporter: roomobs.NewNoopParticipantSessionReporter(), + TelemetryListener: &typesfakes.FakeParticipantTelemetryListener{}, VersionGenerator: utils.NewDefaultTimedVersionGenerator(), + ParticipantListener: &typesfakes.FakeLocalParticipantListener{}, + ParticipantHelper: &typesfakes.FakeLocalParticipantHelper{}, }) p.isPublisher.Store(opts.publisher) p.updateState(livekit.ParticipantInfo_ACTIVE) diff --git a/pkg/rtc/participant_sdp.go b/pkg/rtc/participant_sdp.go index 59d75ad..79cb595 100644 --- a/pkg/rtc/participant_sdp.go +++ b/pkg/rtc/participant_sdp.go @@ -16,44 +16,279 @@ package rtc import ( "fmt" + "slices" "strconv" "strings" "github.com/pion/sdp/v3" "github.com/pion/webrtc/v4" - "github.com/livekit/livekit-server/pkg/sfu/mime" + "github.com/livekit/livekit-server/pkg/rtc/types" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/logger" lksdp "github.com/livekit/protocol/sdp" + "github.com/livekit/protocol/utils" ) -func (p *ParticipantImpl) setCodecPreferencesForPublisher(offer webrtc.SessionDescription) webrtc.SessionDescription { - offer = p.setCodecPreferencesOpusRedForPublisher(offer) - offer = p.setCodecPreferencesVideoForPublisher(offer) - return offer -} +func (p *ParticipantImpl) populateSdpCid(parsedOffer *sdp.SessionDescription) ([]*sdp.MediaDescription, []*sdp.MediaDescription) { + processUnmatch := func(unmatches []*sdp.MediaDescription, trackType livekit.TrackType) { + for _, unmatch := range unmatches { + streamID, ok := lksdp.ExtractStreamID(unmatch) + if !ok { + continue + } -func (p *ParticipantImpl) setCodecPreferencesOpusRedForPublisher(offer webrtc.SessionDescription) webrtc.SessionDescription { - parsed, unmatchAudios, err := p.TransportManager.GetUnmatchMediaForOffer(offer, "audio") - if err != nil || len(unmatchAudios) == 0 { - return offer + sdpCodecs, err := lksdp.CodecsFromMediaDescription(unmatch) + if err != nil || len(sdpCodecs) == 0 { + p.pubLogger.Errorw( + "extract codecs from media section failed", err, + "media", unmatch, + "parsedOffer", parsedOffer, + ) + continue + } + + p.pendingTracksLock.Lock() + signalCid, info, _, migrated, _ := p.getPendingTrack(streamID, trackType, false) + if migrated { + p.pendingTracksLock.Unlock() + continue + } + + if info == nil { + p.pendingTracksLock.Unlock() + + // could be already published track and the unmatch could be a back up codec publish + numUnmatchedTracks := 0 + var unmatchedTrack types.MediaTrack + var unmatchedSdpMimeType mime.MimeType + + found := false + for _, sdpCodec := range sdpCodecs { + sdpMimeType := mime.NormalizeMimeTypeCodec(sdpCodec.Name).ToMimeType() + for _, publishedTrack := range p.GetPublishedTracks() { + if sigCid, sdpCid := publishedTrack.(*MediaTrack).GetCidsForMimeType(sdpMimeType); sigCid != "" && sdpCid == "" { + // a back up codec has a SDP cid match + if sigCid == streamID { + found = true + break + } else { + numUnmatchedTracks++ + unmatchedTrack = publishedTrack + unmatchedSdpMimeType = sdpMimeType + } + } + } + if found { + break + } + } + if !found && unmatchedTrack != nil { + if numUnmatchedTracks != 1 { + p.pubLogger.Warnw( + "too many unmatched tracks", nil, + "media", unmatch, + "parsedOffer", parsedOffer, + ) + } + unmatchedTrack.(*MediaTrack).UpdateCodecSdpCid(unmatchedSdpMimeType, streamID) + p.pubLogger.Debugw( + "published track SDP cid updated", + "trackID", unmatchedTrack.ID(), + "track", logger.Proto(unmatchedTrack.ToProto()), + ) + } + continue + } + + if len(info.Codecs) == 0 { + p.pendingTracksLock.Unlock() + p.pubLogger.Warnw( + "track without codecs", nil, + "trackID", info.Sid, + "pendingTrack", p.pendingTracks[signalCid], + "media", unmatch, + "parsedOffer", parsedOffer, + ) + continue + } + + found := false + updated := false + for _, sdpCodec := range sdpCodecs { + if mime.NormalizeMimeTypeCodec(sdpCodec.Name) == mime.GetMimeTypeCodec(info.Codecs[0].MimeType) { + // set SdpCid only if different from SignalCid + if streamID != info.Codecs[0].Cid { + info.Codecs[0].SdpCid = streamID + updated = true + } + found = true + break + } + if found { + break + } + } + + if !found { + // not using SimulcastCodec, i. e. mime type not available till track publish + if len(info.Codecs) == 1 { + // set SdpCid only if different from SignalCid + if streamID != info.Codecs[0].Cid { + info.Codecs[0].SdpCid = streamID + updated = true + } + } + } + + if updated { + p.pendingTracks[signalCid].trackInfos[0] = utils.CloneProto(info) + p.pubLogger.Debugw( + "pending track SDP cid updated", + "signalCid", signalCid, + "trackID", info.Sid, + "pendingTrack", p.pendingTracks[signalCid], + ) + } + p.pendingTracksLock.Unlock() + } } + unmatchAudios, err := p.TransportManager.GetUnmatchMediaForOffer(parsedOffer, "audio") + if err != nil { + p.pubLogger.Warnw("could not get unmatch audios", err) + return nil, nil + } + + unmatchVideos, err := p.TransportManager.GetUnmatchMediaForOffer(parsedOffer, "video") + if err != nil { + p.pubLogger.Warnw("could not get unmatch videos", err) + return nil, nil + } + + processUnmatch(unmatchAudios, livekit.TrackType_AUDIO) + processUnmatch(unmatchVideos, livekit.TrackType_VIDEO) + return unmatchAudios, unmatchVideos +} + +func (p *ParticipantImpl) setCodecPreferencesForPublisher( + parsedOffer *sdp.SessionDescription, + unmatchAudios []*sdp.MediaDescription, + unmatchVideos []*sdp.MediaDescription, +) { + unprocessedUnmatchAudios := p.setCodecPreferencesForPublisherMedia( + parsedOffer, + unmatchAudios, + livekit.TrackType_AUDIO, + ) + p.setCodecPreferencesOpusRedForPublisher(parsedOffer, unprocessedUnmatchAudios) + _ = p.setCodecPreferencesForPublisherMedia( + parsedOffer, + unmatchVideos, + livekit.TrackType_VIDEO, + ) +} + +func (p *ParticipantImpl) setCodecPreferencesForPublisherMedia( + parsedOffer *sdp.SessionDescription, + unmatches []*sdp.MediaDescription, + trackType livekit.TrackType, +) []*sdp.MediaDescription { + unprocessed := make([]*sdp.MediaDescription, 0, len(unmatches)) + for _, unmatch := range unmatches { + var ti *livekit.TrackInfo + var mimeType string + + mid := lksdp.GetMidValue(unmatch) + if mid == "" { + unprocessed = append(unprocessed, unmatch) + continue + } + transceiver := p.TransportManager.GetPublisherRTPTransceiver(mid) + if transceiver == nil { + unprocessed = append(unprocessed, unmatch) + continue + } + + streamID, ok := lksdp.ExtractStreamID(unmatch) + if !ok { + unprocessed = append(unprocessed, unmatch) + continue + } + + p.pendingTracksLock.RLock() + mt := p.getPublishedTrackBySdpCid(streamID) + if mt != nil { + ti = mt.ToProto() + } else { + _, ti, _, _, _ = p.getPendingTrack(streamID, trackType, false) + } + p.pendingTracksLock.RUnlock() + + if ti == nil { + unprocessed = append(unprocessed, unmatch) + continue + } + + for _, c := range ti.Codecs { + if c.Cid == streamID || c.SdpCid == streamID { + mimeType = c.MimeType + break + } + } + if mimeType == "" && len(ti.Codecs) > 0 { + mimeType = ti.Codecs[0].MimeType + } + + if mimeType == "" { + unprocessed = append(unprocessed, unmatch) + continue + } + + configureReceiverCodecs( + transceiver, + mimeType, + p.params.ClientInfo.ComplyWithCodecOrderInSDPAnswer(), + ) + } + + return unprocessed +} + +func (p *ParticipantImpl) setCodecPreferencesOpusRedForPublisher( + parsedOffer *sdp.SessionDescription, + unmatchAudios []*sdp.MediaDescription, +) { for _, unmatchAudio := range unmatchAudios { + mid := lksdp.GetMidValue(unmatchAudio) + if mid == "" { + continue + } + transceiver := p.TransportManager.GetPublisherRTPTransceiver(mid) + if transceiver == nil { + continue + } + streamID, ok := lksdp.ExtractStreamID(unmatchAudio) if !ok { continue } p.pendingTracksLock.RLock() - _, info, _, _ := p.getPendingTrack(streamID, livekit.TrackType_AUDIO, false) - // if RED is disabled for this track, don't prefer RED codec in offer - disableRed := info != nil && info.DisableRed + _, ti, _, _, _ := p.getPendingTrack(streamID, livekit.TrackType_AUDIO, false) p.pendingTracksLock.RUnlock() + if ti == nil { + continue + } - codecs, err := codecsFromMediaDescription(unmatchAudio) + codecs, err := lksdp.CodecsFromMediaDescription(unmatchAudio) if err != nil { - p.pubLogger.Errorw("extract codecs from media section failed", err, "media", unmatchAudio) + p.pubLogger.Errorw( + "extract codecs from media section failed", err, + "media", unmatchAudio, + "parsedOffer", parsedOffer, + ) continue } @@ -68,122 +303,17 @@ func (p *ParticipantImpl) setCodecPreferencesOpusRedForPublisher(offer webrtc.Se continue } - var preferredCodecs, leftCodecs []string + preferRED := IsRedEnabled(ti) + // if RED is enabled for this track, prefer RED codec in offer for _, codec := range codecs { // codec contain opus/red - if !disableRed && mime.IsMimeTypeCodecStringRED(codec.Name) && strings.Contains(codec.Fmtp, strconv.FormatInt(int64(opusPayload), 10)) { - preferredCodecs = append(preferredCodecs, strconv.FormatInt(int64(codec.PayloadType), 10)) - } else { - leftCodecs = append(leftCodecs, strconv.FormatInt(int64(codec.PayloadType), 10)) - } - } - - // ensure nack enabled for audio in publisher offer - var nackFound bool - for _, attr := range unmatchAudio.Attributes { - if attr.Key == "rtcp-fb" && strings.Contains(attr.Value, fmt.Sprintf("%d nack", opusPayload)) { - nackFound = true + if preferRED && + mime.IsMimeTypeCodecStringRED(codec.Name) && + strings.Contains(codec.Fmtp, strconv.FormatInt(int64(opusPayload), 10)) { + configureReceiverCodecs(transceiver, "audio/red", true) break } } - if !nackFound { - unmatchAudio.Attributes = append(unmatchAudio.Attributes, sdp.Attribute{ - Key: "rtcp-fb", - Value: fmt.Sprintf("%d nack", opusPayload), - }) - } - - // no opus/red found - if len(preferredCodecs) == 0 { - continue - } - - unmatchAudio.MediaName.Formats = append(unmatchAudio.MediaName.Formats[:0], preferredCodecs...) - unmatchAudio.MediaName.Formats = append(unmatchAudio.MediaName.Formats, leftCodecs...) - } - - bytes, err := parsed.Marshal() - if err != nil { - p.pubLogger.Errorw("failed to marshal offer", err) - return offer - } - - return webrtc.SessionDescription{ - Type: offer.Type, - SDP: string(bytes), - } -} - -func (p *ParticipantImpl) setCodecPreferencesVideoForPublisher(offer webrtc.SessionDescription) webrtc.SessionDescription { - parsed, unmatchVideos, err := p.TransportManager.GetUnmatchMediaForOffer(offer, "video") - if err != nil || len(unmatchVideos) == 0 { - return offer - } - // unmatched video is pending for publish, set codec preference - for _, unmatchVideo := range unmatchVideos { - streamID, ok := lksdp.ExtractStreamID(unmatchVideo) - if !ok { - continue - } - - var info *livekit.TrackInfo - p.pendingTracksLock.RLock() - mt := p.getPublishedTrackBySdpCid(streamID) - if mt != nil { - info = mt.ToProto() - } else { - _, info, _, _ = p.getPendingTrack(streamID, livekit.TrackType_VIDEO, false) - } - - if info == nil { - p.pendingTracksLock.RUnlock() - continue - } - var mimeType string - for _, c := range info.Codecs { - if c.Cid == streamID { - mimeType = c.MimeType - break - } - } - if mimeType == "" && len(info.Codecs) > 0 { - mimeType = info.Codecs[0].MimeType - } - p.pendingTracksLock.RUnlock() - - if mimeType != "" { - codecs, err := codecsFromMediaDescription(unmatchVideo) - if err != nil { - p.pubLogger.Errorw("extract codecs from media section failed", err, "media", unmatchVideo) - continue - } - - var preferredCodecs, leftCodecs []string - for _, c := range codecs { - if mime.GetMimeTypeCodec(mimeType) == mime.NormalizeMimeTypeCodec(c.Name) { - preferredCodecs = append(preferredCodecs, strconv.FormatInt(int64(c.PayloadType), 10)) - } else { - leftCodecs = append(leftCodecs, strconv.FormatInt(int64(c.PayloadType), 10)) - } - } - - unmatchVideo.MediaName.Formats = append(unmatchVideo.MediaName.Formats[:0], preferredCodecs...) - // if the client don't comply with codec order in SDP answer, only keep preferred codecs to force client to use it - if p.params.ClientInfo.ComplyWithCodecOrderInSDPAnswer() { - unmatchVideo.MediaName.Formats = append(unmatchVideo.MediaName.Formats, leftCodecs...) - } - } - } - - bytes, err := parsed.Marshal() - if err != nil { - p.pubLogger.Errorw("failed to marshal offer", err) - return offer - } - - return webrtc.SessionDescription{ - Type: offer.Type, - SDP: string(bytes), } } @@ -195,12 +325,12 @@ func (p *ParticipantImpl) configurePublisherAnswer(answer webrtc.SessionDescript return answer } - parsed, err := answer.Unmarshal() + parsedAnswer, err := answer.Unmarshal() if err != nil { return answer } - for _, m := range parsed.MediaDescriptions { + for _, m := range parsedAnswer.MediaDescriptions { switch m.MediaName.Media { case "audio": _, ok := m.Attribute(sdp.AttrKeyInactive) @@ -227,7 +357,7 @@ func (p *ParticipantImpl) configurePublisherAnswer(answer webrtc.SessionDescript track, _ := p.getPublishedTrackBySdpCid(streamID).(*MediaTrack) if track == nil { p.pendingTracksLock.RLock() - _, ti, _, _ = p.getPendingTrack(streamID, livekit.TrackType_AUDIO, false) + _, ti, _, _, _ = p.getPendingTrack(streamID, livekit.TrackType_AUDIO, false) p.pendingTracksLock.RUnlock() } else { ti = track.ToProto() @@ -236,12 +366,12 @@ func (p *ParticipantImpl) configurePublisherAnswer(answer webrtc.SessionDescript } } - if ti == nil || (ti.DisableDtx && !ti.Stereo) { + if ti == nil { // no need to configure continue } - opusPT, err := parsed.GetPayloadTypeForCodec(sdp.Codec{Name: mime.MimeTypeCodecOpus.String()}) + opusPT, err := parsedAnswer.GetPayloadTypeForCodec(sdp.Codec{Name: mime.MimeTypeCodecOpus.String()}) if err != nil { p.pubLogger.Infow("failed to get opus payload type", "error", err, "trackID", ti.Sid) continue @@ -249,11 +379,15 @@ func (p *ParticipantImpl) configurePublisherAnswer(answer webrtc.SessionDescript for i, attr := range m.Attributes { if strings.HasPrefix(attr.String(), fmt.Sprintf("fmtp:%d", opusPT)) { - if !ti.DisableDtx { + if !slices.Contains(ti.AudioFeatures, livekit.AudioTrackFeature_TF_NO_DTX) { attr.Value += ";usedtx=1" + } else { + attr.Value = strings.ReplaceAll(attr.Value, ";usedtx=1", "") } - if ti.Stereo { + if slices.Contains(ti.AudioFeatures, livekit.AudioTrackFeature_TF_STEREO) { attr.Value += ";stereo=1;maxaveragebitrate=510000" + } else { + attr.Value = strings.ReplaceAll(attr.Value, ";stereo=1", "") } m.Attributes[i] = attr } @@ -264,7 +398,7 @@ func (p *ParticipantImpl) configurePublisherAnswer(answer webrtc.SessionDescript } } - bytes, err := parsed.Marshal() + bytes, err := parsedAnswer.Marshal() if err != nil { p.pubLogger.Infow("failed to marshal answer", "error", err) return answer diff --git a/pkg/rtc/participant_signal.go b/pkg/rtc/participant_signal.go index 81fda0a..3fa388a 100644 --- a/pkg/rtc/participant_signal.go +++ b/pkg/rtc/participant_signal.go @@ -15,30 +15,28 @@ package rtc import ( - "fmt" "time" "github.com/pion/webrtc/v4" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" - "github.com/livekit/protocol/utils" - "github.com/livekit/psrpc" + protosignalling "github.com/livekit/protocol/signalling" "github.com/livekit/livekit-server/pkg/routing" "github.com/livekit/livekit-server/pkg/rtc/types" ) -func (p *ParticipantImpl) getResponseSink() routing.MessageSink { - p.resSinkMu.Lock() - defer p.resSinkMu.Unlock() - return p.resSink +func (p *ParticipantImpl) SwapResponseSink(sink routing.MessageSink, reason types.SignallingCloseReason) { + p.signaller.SwapResponseSink(sink, reason) } -func (p *ParticipantImpl) SetResponseSink(sink routing.MessageSink) { - p.resSinkMu.Lock() - defer p.resSinkMu.Unlock() - p.resSink = sink +func (p *ParticipantImpl) GetResponseSink() routing.MessageSink { + return p.signaller.GetResponseSink() +} + +func (p *ParticipantImpl) CloseSignalConnection(reason types.SignallingCloseReason) { + p.signaller.CloseSignalConnection(reason) } func (p *ParticipantImpl) SendJoinResponse(joinResponse *livekit.JoinResponse) error { @@ -55,11 +53,7 @@ func (p *ParticipantImpl) SendJoinResponse(joinResponse *livekit.JoinResponse) e p.updateLock.Unlock() // send Join response - err := p.writeMessage(&livekit.SignalResponse{ - Message: &livekit.SignalResponse_Join{ - Join: joinResponse, - }, - }) + err := p.signaller.WriteMessage(p.signalling.SignalJoinResponse(joinResponse)) if err != nil { return err } @@ -111,7 +105,7 @@ func (p *ParticipantImpl) SendParticipantUpdate(participantsToUpdate []*livekit. isValid = false } } - if pi.Permission != nil && pi.Permission.Hidden && pi.Sid != string(p.params.SID) { + if pi.Permission != nil && pi.Permission.Hidden && pi.Sid != string(p.ID()) { p.params.Logger.Debugw("skipping hidden participant update", "otherParticipant", pi.Identity) isValid = false } @@ -127,17 +121,7 @@ func (p *ParticipantImpl) SendParticipantUpdate(participantsToUpdate []*livekit. } p.updateLock.Unlock() - if len(validUpdates) == 0 { - return nil - } - - return p.writeMessage(&livekit.SignalResponse{ - Message: &livekit.SignalResponse_Update{ - Update: &livekit.ParticipantUpdate{ - Participants: validUpdates, - }, - }, - }) + return p.signaller.WriteMessage(p.signalling.SignalParticipantUpdate(validUpdates)) } // SendSpeakerUpdate notifies participant changes to speakers. only send members that have changed since last update @@ -158,47 +142,23 @@ func (p *ParticipantImpl) SendSpeakerUpdate(speakers []*livekit.SpeakerInfo, for } } - if len(scopedSpeakers) == 0 { - return nil - } - - return p.writeMessage(&livekit.SignalResponse{ - Message: &livekit.SignalResponse_SpeakersChanged{ - SpeakersChanged: &livekit.SpeakersChanged{ - Speakers: scopedSpeakers, - }, - }, - }) + return p.signaller.WriteMessage(p.signalling.SignalSpeakerUpdate(scopedSpeakers)) } func (p *ParticipantImpl) SendRoomUpdate(room *livekit.Room) error { - return p.writeMessage(&livekit.SignalResponse{ - Message: &livekit.SignalResponse_RoomUpdate{ - RoomUpdate: &livekit.RoomUpdate{ - Room: room, - }, - }, - }) + return p.signaller.WriteMessage(p.signalling.SignalRoomUpdate(room)) } func (p *ParticipantImpl) SendConnectionQualityUpdate(update *livekit.ConnectionQualityUpdate) error { - return p.writeMessage(&livekit.SignalResponse{ - Message: &livekit.SignalResponse_ConnectionQuality{ - ConnectionQuality: update, - }, - }) + return p.signaller.WriteMessage(p.signalling.SignalConnectionQualityUpdate(update)) } func (p *ParticipantImpl) SendRefreshToken(token string) error { - return p.writeMessage(&livekit.SignalResponse{ - Message: &livekit.SignalResponse_RefreshToken{ - RefreshToken: token, - }, - }) + return p.signaller.WriteMessage(p.signalling.SignalRefreshToken(token)) } -func (p *ParticipantImpl) SendRequestResponse(requestResponse *livekit.RequestResponse) error { - if requestResponse.RequestId == 0 || !p.params.ClientInfo.SupportErrorResponse() { +func (p *ParticipantImpl) sendRequestResponse(requestResponse *livekit.RequestResponse) error { + if !p.params.ClientInfo.SupportsRequestResponse() { return nil } @@ -206,11 +166,11 @@ func (p *ParticipantImpl) SendRequestResponse(requestResponse *livekit.RequestRe return nil } - return p.writeMessage(&livekit.SignalResponse{ - Message: &livekit.SignalResponse_RequestResponse{ - RequestResponse: requestResponse, - }, - }) + return p.signaller.WriteMessage(p.signalling.SignalRequestResponse(requestResponse)) +} + +func (p *ParticipantImpl) SendRoomMovedResponse(roomMovedResponse *livekit.RoomMovedResponse) error { + return p.signaller.WriteMessage(p.signalling.SignalRoomMovedResponse(roomMovedResponse)) } func (p *ParticipantImpl) HandleReconnectAndSendResponse(reconnectReason livekit.ReconnectReason, reconnectResponse *livekit.ReconnectResponse) error { @@ -219,15 +179,11 @@ func (p *ParticipantImpl) HandleReconnectAndSendResponse(reconnectReason livekit if !p.params.ClientInfo.CanHandleReconnectResponse() { return nil } - if err := p.writeMessage(&livekit.SignalResponse{ - Message: &livekit.SignalResponse_Reconnect{ - Reconnect: reconnectResponse, - }, - }); err != nil { + if err := p.signaller.WriteMessage(p.signalling.SignalReconnectResponse(reconnectResponse)); err != nil { return err } - if p.params.ProtocolVersion.SupportHandlesDisconnectedUpdate() { + if p.params.ProtocolVersion.SupportsDisconnectedUpdate() { return p.sendDisconnectUpdatesForReconnect() } @@ -255,17 +211,7 @@ func (p *ParticipantImpl) sendDisconnectUpdatesForReconnect() error { } p.updateLock.Unlock() - if len(disconnectedParticipants) == 0 { - return nil - } - - return p.writeMessage(&livekit.SignalResponse{ - Message: &livekit.SignalResponse_Update{ - Update: &livekit.ParticipantUpdate{ - Participants: disconnectedParticipants, - }, - }, - }) + return p.signaller.WriteMessage(p.signalling.SignalParticipantUpdate(disconnectedParticipants)) } func (p *ParticipantImpl) sendICECandidate(ic *webrtc.ICECandidate, target livekit.SignalTarget) error { @@ -274,90 +220,43 @@ func (p *ParticipantImpl) sendICECandidate(ic *webrtc.ICECandidate, target livek return nil } - trickle := ToProtoTrickle(prevIC.ToJSON(), target, ic == nil) + trickle := protosignalling.ToProtoTrickle(prevIC.ToJSON(), target, ic == nil) p.params.Logger.Debugw("sending ICE candidate", "transport", target, "trickle", logger.Proto(trickle)) - return p.writeMessage(&livekit.SignalResponse{ - Message: &livekit.SignalResponse_Trickle{ - Trickle: trickle, - }, - }) + return p.signaller.WriteMessage(p.signalling.SignalICECandidate(trickle)) } func (p *ParticipantImpl) sendTrackMuted(trackID livekit.TrackID, muted bool) { - _ = p.writeMessage(&livekit.SignalResponse{ - Message: &livekit.SignalResponse_Mute{ - Mute: &livekit.MuteTrackRequest{ - Sid: string(trackID), - Muted: muted, - }, - }, - }) + _ = p.signaller.WriteMessage(p.signalling.SignalTrackMuted(&livekit.MuteTrackRequest{ + Sid: string(trackID), + Muted: muted, + })) +} + +func (p *ParticipantImpl) sendTrackPublished(cid string, ti *livekit.TrackInfo) error { + p.pubLogger.Debugw("sending track published", "cid", cid, "trackInfo", logger.Proto(ti)) + return p.signaller.WriteMessage(p.signalling.SignalTrackPublished(&livekit.TrackPublishedResponse{ + Cid: cid, + Track: ti, + })) } func (p *ParticipantImpl) sendTrackUnpublished(trackID livekit.TrackID) { - _ = p.writeMessage(&livekit.SignalResponse{ - Message: &livekit.SignalResponse_TrackUnpublished{ - TrackUnpublished: &livekit.TrackUnpublishedResponse{ - TrackSid: string(trackID), - }, - }, - }) + _ = p.signaller.WriteMessage(p.signalling.SignalTrackUnpublished(&livekit.TrackUnpublishedResponse{ + TrackSid: string(trackID), + })) } func (p *ParticipantImpl) sendTrackHasBeenSubscribed(trackID livekit.TrackID) { - if !p.params.ClientInfo.SupportTrackSubscribedEvent() { + if !p.params.ClientInfo.SupportsTrackSubscribedEvent() { return } - _ = p.writeMessage(&livekit.SignalResponse{ - Message: &livekit.SignalResponse_TrackSubscribed{ - TrackSubscribed: &livekit.TrackSubscribed{ - TrackSid: string(trackID), - }, - }, - }) + _ = p.signaller.WriteMessage(p.signalling.SignalTrackSubscribed(&livekit.TrackSubscribed{ + TrackSid: string(trackID), + })) p.params.Logger.Debugw("track has been subscribed", "trackID", trackID) } -func (p *ParticipantImpl) writeMessage(msg *livekit.SignalResponse) error { - if p.IsDisconnected() || (!p.IsReady() && msg.GetJoin() == nil) { - return nil - } - - sink := p.getResponseSink() - if sink == nil { - p.params.Logger.Debugw("could not send message to participant", "messageType", fmt.Sprintf("%T", msg.Message)) - return nil - } - - err := sink.WriteMessage(msg) - if utils.ErrorIsOneOf(err, psrpc.Canceled, routing.ErrChannelClosed) { - p.params.Logger.Debugw( - "could not send message to participant", - "error", err, - "messageType", fmt.Sprintf("%T", msg.Message), - ) - return nil - } else if err != nil { - p.params.Logger.Warnw( - "could not send message to participant", err, - "messageType", fmt.Sprintf("%T", msg.Message), - ) - return err - } - return nil -} - -// closes signal connection to notify client to resume/reconnect -func (p *ParticipantImpl) CloseSignalConnection(reason types.SignallingCloseReason) { - sink := p.getResponseSink() - if sink != nil { - p.params.Logger.Debugw("closing signal connection", "reason", reason, "connID", sink.ConnectionID()) - sink.Close() - p.SetResponseSink(nil) - } -} - func (p *ParticipantImpl) sendLeaveRequest( reason types.ParticipantCloseReason, isExpectedToResume bool, @@ -377,9 +276,9 @@ func (p *ParticipantImpl) sendLeaveRequest( default: leave.Action = livekit.LeaveRequest_DISCONNECT } - if leave.Action != livekit.LeaveRequest_DISCONNECT && p.params.GetRegionSettings != nil { + if leave.Action != livekit.LeaveRequest_DISCONNECT { // sending region settings even for RESUME just in case client wants to a full reconnect despite server saying RESUME - leave.Regions = p.params.GetRegionSettings(p.params.ClientInfo.Address) + leave.Regions = p.helper().GetRegionSettings(p.params.ClientInfo.Address) } } else { if !sendOnlyIfSupportingLeaveRequestWithAction { @@ -390,12 +289,82 @@ func (p *ParticipantImpl) sendLeaveRequest( } } if leave != nil { - return p.writeMessage(&livekit.SignalResponse{ - Message: &livekit.SignalResponse_Leave{ - Leave: leave, - }, - }) + return p.signaller.WriteMessage(p.signalling.SignalLeaveRequest(leave)) } return nil } + +func (p *ParticipantImpl) sendSdpAnswer(answer webrtc.SessionDescription, answerId uint32, midToTrackID map[string]string) error { + return p.signaller.WriteMessage(p.signalling.SignalSdpAnswer(protosignalling.ToProtoSessionDescription(answer, answerId, midToTrackID))) +} + +func (p *ParticipantImpl) sendSdpOffer(offer webrtc.SessionDescription, offerId uint32, midToTrackID map[string]string) error { + return p.signaller.WriteMessage(p.signalling.SignalSdpOffer(protosignalling.ToProtoSessionDescription(offer, offerId, midToTrackID))) +} + +func (p *ParticipantImpl) sendStreamStateUpdate(streamStateUpdate *livekit.StreamStateUpdate) error { + return p.signaller.WriteMessage(p.signalling.SignalStreamStateUpdate(streamStateUpdate)) +} + +func (p *ParticipantImpl) sendSubscribedQualityUpdate(subscribedQualityUpdate *livekit.SubscribedQualityUpdate) error { + return p.signaller.WriteMessage(p.signalling.SignalSubscribedQualityUpdate(subscribedQualityUpdate)) +} + +func (p *ParticipantImpl) sendSubscribedAudioCodecUpdate(subscribedAudioCodecUpdate *livekit.SubscribedAudioCodecUpdate) error { + return p.signaller.WriteMessage(p.signalling.SignalSubscribedAudioCodecUpdate(subscribedAudioCodecUpdate)) +} + +func (p *ParticipantImpl) sendSubscriptionResponse(trackID livekit.TrackID, subErr livekit.SubscriptionError) error { + return p.signaller.WriteMessage(p.signalling.SignalSubscriptionResponse(&livekit.SubscriptionResponse{ + TrackSid: string(trackID), + Err: subErr, + })) +} + +func (p *ParticipantImpl) SendSubscriptionPermissionUpdate(publisherID livekit.ParticipantID, trackID livekit.TrackID, allowed bool) error { + p.subLogger.Debugw("sending subscription permission update", "publisherID", publisherID, "trackID", trackID, "allowed", allowed) + err := p.signaller.WriteMessage(p.signalling.SignalSubscriptionPermissionUpdate(&livekit.SubscriptionPermissionUpdate{ + ParticipantSid: string(publisherID), + TrackSid: string(trackID), + Allowed: allowed, + })) + if err != nil { + p.subLogger.Errorw("could not send subscription permission update", err) + } + return err +} + +func (p *ParticipantImpl) sendMediaSectionsRequirement(numAudios uint32, numVideos uint32) error { + p.pubLogger.Debugw( + "sending media sections requirement", + "numAudios", numAudios, + "numVideos", numVideos, + ) + err := p.signaller.WriteMessage(p.signalling.SignalMediaSectionsRequirement(&livekit.MediaSectionsRequirement{ + NumAudios: numAudios, + NumVideos: numVideos, + })) + if err != nil { + p.subLogger.Errorw("could not send media sections requirement", err) + } + return err +} + +func (p *ParticipantImpl) sendPublishDataTrackResponse(dti *livekit.DataTrackInfo) error { + return p.signaller.WriteMessage(p.signalling.SignalPublishDataTrackResponse(&livekit.PublishDataTrackResponse{ + Info: dti, + })) +} + +func (p *ParticipantImpl) sendUnpublishDataTrackResponse(dti *livekit.DataTrackInfo) error { + return p.signaller.WriteMessage(p.signalling.SignalUnpublishDataTrackResponse(&livekit.UnpublishDataTrackResponse{ + Info: dti, + })) +} + +func (p *ParticipantImpl) SendDataTrackSubscriberHandles(handles map[uint32]*livekit.DataTrackSubscriberHandles_PublishedDataTrack) error { + return p.signaller.WriteMessage(p.signalling.SignalDataTrackSubscriberHandles(&livekit.DataTrackSubscriberHandles{ + SubHandles: handles, + })) +} diff --git a/pkg/rtc/room.go b/pkg/rtc/room.go index cd33e4f..e696abd 100644 --- a/pkg/rtc/room.go +++ b/pkg/rtc/room.go @@ -17,19 +17,21 @@ package rtc import ( "context" "fmt" + "maps" "math" "slices" - "sort" "strings" "sync" "time" "go.uber.org/atomic" - "golang.org/x/exp/maps" "google.golang.org/protobuf/proto" + protoagent "github.com/livekit/protocol/agent" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" + "github.com/livekit/protocol/observability/roomobs" "github.com/livekit/protocol/rpc" "github.com/livekit/protocol/utils" "github.com/livekit/protocol/utils/guid" @@ -38,6 +40,7 @@ import ( "github.com/livekit/livekit-server/pkg/agent" "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/livekit-server/pkg/routing" + "github.com/livekit/livekit-server/pkg/rtc/datatrack" "github.com/livekit/livekit-server/pkg/rtc/types" "github.com/livekit/livekit-server/pkg/sfu" "github.com/livekit/livekit-server/pkg/sfu/buffer" @@ -55,6 +58,9 @@ const ( dataForwardLoadBalanceThreshold = 4 simulateDisconnectSignalTimeout = 5 * time.Second + + dataMessageCacheTTL = 2 * time.Second + dataMessageCacheSize = 100_000 ) var ( @@ -79,12 +85,6 @@ type broadcastOptions struct { immediate bool } -type participantUpdate struct { - pi *livekit.ParticipantInfo - isSynthesizedDisconnect bool - closeReason types.ParticipantCloseReason -} - type disconnectSignalOnResumeNoMessages struct { expiry time.Time closedCount int @@ -105,9 +105,10 @@ type Room struct { protoRoom *livekit.Room internal *livekit.RoomInternal protoProxy *utils.ProtoProxy[*livekit.Room] - Logger logger.Logger + logger logger.Logger config WebRTCConfig + roomConfig config.RoomConfig audioConfig *sfu.AudioConfig serverInfo *livekit.ServerInfo telemetry telemetry.TelemetryService @@ -128,14 +129,14 @@ type Room struct { bufferFactory *buffer.FactoryOfBufferFactory // batch update participant info for non-publishers - batchedUpdates map[livekit.ParticipantIdentity]*participantUpdate + batchedUpdates map[livekit.ParticipantIdentity]*ParticipantUpdate batchedUpdatesMu sync.Mutex closed chan struct{} trailer []byte - onParticipantChanged func(p types.LocalParticipant) + onParticipantChanged func(p types.Participant) onRoomUpdated func() onClose func() @@ -144,10 +145,17 @@ type Room struct { disconnectSignalOnResumeNoMessagesParticipants map[livekit.ParticipantIdentity]*disconnectSignalOnResumeNoMessages userPacketDeduper *UserPacketDeduper + + dataMessageCache *utils.TimeSizeCache[types.DataMessageCache] + + onStateChangeMu sync.Mutex + localParticipantListener types.LocalParticipantListener + participantTelemetryListener types.ParticipantTelemetryListener } type ParticipantOptions struct { - AutoSubscribe bool + AutoSubscribe bool + AutoSubscribeDataTrack bool } type agentDispatch struct { @@ -188,7 +196,7 @@ func (ad *agentDispatch) jobsLaunching() (jobsLaunched func()) { func (ad *agentDispatch) waitForPendingJobs() { ad.lock.Lock() - cs := maps.Keys(ad.pending) + cs := slices.Collect(maps.Keys(ad.pending)) ad.lock.Unlock() for _, c := range cs { @@ -247,19 +255,19 @@ func NewRoom( r := &Room{ protoRoom: utils.CloneProto(room), internal: internal, - Logger: LoggerWithRoom( + logger: LoggerWithRoom( logger.GetLogger().WithComponent(sutils.ComponentRoom), livekit.RoomName(room.Name), livekit.RoomID(room.Sid), ), config: config, + roomConfig: roomConfig, audioConfig: audioConfig, telemetry: telemetry, egressLauncher: egressLauncher, agentClient: agentClient, agentStore: agentStore, agentDispatches: make(map[string]*agentDispatch), - trackManager: NewRoomTrackManager(), serverInfo: serverInfo, participants: make(map[livekit.ParticipantIdentity]types.LocalParticipant), participantOpts: make(map[livekit.ParticipantIdentity]*ParticipantOptions), @@ -267,13 +275,20 @@ func NewRoom( hasPublished: make(map[livekit.ParticipantIdentity]bool), agentParticpants: make(map[livekit.ParticipantIdentity]*agentJob), bufferFactory: buffer.NewFactoryOfBufferFactory(config.Receiver.PacketBufferSizeVideo, config.Receiver.PacketBufferSizeAudio), - batchedUpdates: make(map[livekit.ParticipantIdentity]*participantUpdate), + batchedUpdates: make(map[livekit.ParticipantIdentity]*ParticipantUpdate), closed: make(chan struct{}), trailer: []byte(utils.RandomSecret()), disconnectSignalOnResumeParticipants: make(map[livekit.ParticipantIdentity]time.Time), disconnectSignalOnResumeNoMessagesParticipants: make(map[livekit.ParticipantIdentity]*disconnectSignalOnResumeNoMessages), userPacketDeduper: NewUserPacketDeduper(), + dataMessageCache: utils.NewTimeSizeCache[types.DataMessageCache](utils.TimeSizeCacheParams{ + TTL: dataMessageCacheTTL, + MaxSize: dataMessageCacheSize, + }), } + r.trackManager = NewRoomTrackManager(r.logger) + r.localParticipantListener = &localParticipantListener{room: r} + r.participantTelemetryListener = &participantTelemetryListener{room: r} if r.protoRoom.EmptyTimeout == 0 { r.protoRoom.EmptyTimeout = roomConfig.EmptyTimeout @@ -286,11 +301,11 @@ func NewRoom( r.protoRoom.CreationTime = now.Unix() r.protoRoom.CreationTimeMs = now.UnixMilli() } - r.protoProxy = utils.NewProtoProxy[*livekit.Room](roomUpdateInterval, r.updateProto) + r.protoProxy = utils.NewProtoProxy(roomUpdateInterval, r.updateProto) r.createAgentDispatchesFromRoomAgent() - r.launchRoomAgents(maps.Values(r.agentDispatches)) + r.launchRoomAgents(slices.Collect(maps.Values(r.agentDispatches))) go r.audioUpdateWorker() go r.connectionQualityWorker() @@ -300,6 +315,10 @@ func NewRoom( return r } +func (r *Room) Logger() logger.Logger { + return r.logger +} + func (r *Room) ToProto() *livekit.Room { return r.protoProxy.Get() } @@ -324,6 +343,7 @@ func (r *Room) Trailer() []byte { func (r *Room) GetParticipant(identity livekit.ParticipantIdentity) types.LocalParticipant { r.lock.RLock() defer r.lock.RUnlock() + return r.participants[identity] } @@ -344,7 +364,7 @@ func (r *Room) GetParticipants() []types.LocalParticipant { r.lock.RLock() defer r.lock.RUnlock() - return maps.Values(r.participants) + return slices.Collect(maps.Values(r.participants)) } func (r *Room) GetLocalParticipants() []types.LocalParticipant { @@ -373,8 +393,8 @@ func (r *Room) GetActiveSpeakers() []*livekit.SpeakerInfo { }) } - sort.Slice(speakers, func(i, j int) bool { - return speakers[i].Level > speakers[j].Level + slices.SortFunc(speakers, func(a, b *livekit.SpeakerInfo) int { + return sutils.Signum(b.Level - a.Level) }) // quantize to smooth out small changes @@ -417,7 +437,12 @@ func (r *Room) Release() { r.holds.Dec() } -func (r *Room) Join(participant types.LocalParticipant, requestSource routing.MessageSource, opts *ParticipantOptions, iceServers []*livekit.ICEServer) error { +func (r *Room) Join( + participant types.LocalParticipant, + requestSource routing.MessageSource, + opts *ParticipantOptions, + iceServers []*livekit.ICEServer, +) error { r.lock.Lock() defer r.lock.Unlock() @@ -444,86 +469,11 @@ func (r *Room) Join(participant types.LocalParticipant, requestSource routing.Me r.joinedAt.Store(time.Now().Unix()) } - participant.OnStateChange(func(p types.LocalParticipant, state livekit.ParticipantInfo_State) { - if r.onParticipantChanged != nil { - r.onParticipantChanged(p) - } - r.broadcastParticipantState(p, broadcastOptions{skipSource: true}) + r.launchTargetAgents(slices.Collect(maps.Values(r.agentDispatches)), participant, livekit.JobType_JT_PARTICIPANT) - if state == livekit.ParticipantInfo_ACTIVE { - // subscribe participant to existing published tracks - r.subscribeToExistingTracks(p) - - meta := &livekit.AnalyticsClientMeta{ - ClientConnectTime: uint32(time.Since(p.ConnectedAt()).Milliseconds()), - } - infos := p.GetICEConnectionInfo() - for _, info := range infos { - if info.Type != types.ICEConnectionTypeUnknown { - meta.ConnectionType = string(info.Type) - break - } - } - r.telemetry.ParticipantActive(context.Background(), - r.ToProto(), - p.ToProto(), - meta, - false, - ) - - p.GetLogger().Infow("participant active", connectionDetailsFields(infos)...) - } else if state == livekit.ParticipantInfo_DISCONNECTED { - // remove participant from room - // participant should already be closed and have a close reason, so NONE is fine here - go r.RemoveParticipant(p.Identity(), p.ID(), types.ParticipantCloseReasonNone) - } - }) - // it's important to set this before connection, we don't want to miss out on any published tracks - participant.OnTrackPublished(r.onTrackPublished) - participant.OnTrackUpdated(r.onTrackUpdated) - participant.OnTrackUnpublished(r.onTrackUnpublished) - participant.OnParticipantUpdate(r.onParticipantUpdate) - participant.OnDataPacket(r.onDataPacket) - participant.OnMetrics(r.onMetrics) - participant.OnSubscribeStatusChanged(func(publisherID livekit.ParticipantID, subscribed bool) { - if subscribed { - pub := r.GetParticipantByID(publisherID) - if pub != nil && pub.State() == livekit.ParticipantInfo_ACTIVE { - // when a participant subscribes to another participant, - // send speaker update if the subscribed to participant is active. - level, active := pub.GetAudioLevel() - if active { - _ = participant.SendSpeakerUpdate([]*livekit.SpeakerInfo{ - { - Sid: string(pub.ID()), - Level: float32(level), - Active: active, - }, - }, false) - } - - if cq := pub.GetConnectionQuality(); cq != nil { - update := &livekit.ConnectionQualityUpdate{} - update.Updates = append(update.Updates, cq) - _ = participant.SendConnectionQualityUpdate(update) - } - } - } else { - // no longer subscribed to the publisher, clear speaker status - _ = participant.SendSpeakerUpdate([]*livekit.SpeakerInfo{ - { - Sid: string(publisherID), - Level: 0, - Active: false, - }, - }, true) - } - }) - - r.launchTargetAgents(maps.Values(r.agentDispatches), participant, livekit.JobType_JT_PARTICIPANT) - - r.Logger.Debugw("new participant joined", - "pID", participant.ID(), + r.logger.Debugw( + "new participant joined", + "participantID", participant.ID(), "participant", participant.Identity(), "clientInfo", logger.Proto(participant.GetClientInfo()), "options", opts, @@ -553,7 +503,7 @@ func (r *Room) Join(participant types.LocalParticipant, requestSource routing.Me joinResponse := r.createJoinResponseLocked(participant, iceServers) if err := participant.SendJoinResponse(joinResponse); err != nil { - prometheus.ServiceOperationCounter.WithLabelValues("participant_join", "error", "send_response").Add(1) + prometheus.RecordServiceOperationError("participant_join", "send_response") return err } @@ -563,15 +513,19 @@ func (r *Room) Join(participant types.LocalParticipant, requestSource routing.Me // initiates sub connection as primary if participant.ProtocolVersion().SupportFastStart() { go func() { - r.subscribeToExistingTracks(participant) + r.subscribeToExistingTracks(participant, false) participant.Negotiate(true) }() } else { participant.Negotiate(true) } + } else { + if participant.IsUsingSinglePeerConnection() { + go r.subscribeToExistingTracks(participant, false) + } } - prometheus.ServiceOperationCounter.WithLabelValues("participant_join", "success", "").Add(1) + prometheus.RecordServiceOperationSuccess("participant_join") return nil } @@ -601,8 +555,7 @@ func (r *Room) ResumeParticipant( ) error { r.ReplaceParticipantRequestSource(p.Identity(), requestSource) // close previous sink, and link to new one - p.CloseSignalConnection(types.SignallingCloseReasonResume) - p.SetResponseSink(responseSink) + p.SwapResponseSink(responseSink, types.SignallingCloseReasonResume) p.SetSignalSourceValid(true) @@ -629,12 +582,14 @@ func (r *Room) ResumeParticipant( if err := p.HandleReconnectAndSendResponse(reason, &livekit.ReconnectResponse{ IceServers: iceServers, ClientConfiguration: p.GetClientConfiguration(), + ServerInfo: r.serverInfo, + LastMessageSeq: p.GetLastReliableSequence(false), }); err != nil { return err } // include the local participant's info as well, since metadata could have been changed - updates := r.getOtherParticipantInfo("") + updates := GetOtherParticipantInfo(nil, false, toParticipants(r.GetParticipants()), false) if err := p.SendParticipantUpdate(updates); err != nil { return err } @@ -655,124 +610,15 @@ func (r *Room) ResumeParticipant( return nil } -func (r *Room) RemoveParticipant(identity livekit.ParticipantIdentity, pID livekit.ParticipantID, reason types.ParticipantCloseReason) { - r.lock.Lock() - p, ok := r.participants[identity] - if !ok { - r.lock.Unlock() - return +func (r *Room) HandleSyncState(participant types.LocalParticipant, state *livekit.SyncState) error { + if state != nil { + return r.onSyncState(participant, state) } - if pID != "" && p.ID() != pID { - // participant session has been replaced - r.lock.Unlock() - return - } - - agentJob := r.agentParticpants[identity] - - delete(r.participants, identity) - delete(r.participantOpts, identity) - delete(r.participantRequestSources, identity) - delete(r.hasPublished, identity) - delete(r.agentParticpants, identity) - if !p.Hidden() { - r.protoRoom.NumParticipants-- - } - - immediateChange := false - if p.IsRecorder() { - activeRecording := false - for _, op := range r.participants { - if op.IsRecorder() { - activeRecording = true - break - } - } - - if r.protoRoom.ActiveRecording != activeRecording { - r.protoRoom.ActiveRecording = activeRecording - immediateChange = true - } - } - r.lock.Unlock() - r.protoProxy.MarkDirty(immediateChange) - - if !p.HasConnected() { - fields := append(connectionDetailsFields(p.GetICEConnectionInfo()), - "reason", reason.String(), - ) - p.GetLogger().Infow("removing participant without connection", fields...) - } - - // send broadcast only if it's not already closed - sendUpdates := !p.IsDisconnected() - - // remove all published tracks - for _, t := range p.GetPublishedTracks() { - r.trackManager.RemoveTrack(t) - } - - if agentJob != nil { - agentJob.participantLeft() - - go func() { - _, err := r.agentClient.TerminateJob(context.Background(), agentJob.Id, rpc.JobTerminateReason_AGENT_LEFT_ROOM) - if err != nil { - r.Logger.Infow("failed sending TerminateJob RPC", "error", err, "jobID", agentJob.Id, "participant", identity) - } - }() - } - - p.OnTrackUpdated(nil) - p.OnTrackPublished(nil) - p.OnTrackUnpublished(nil) - p.OnStateChange(nil) - p.OnParticipantUpdate(nil) - p.OnDataPacket(nil) - p.OnMetrics(nil) - p.OnSubscribeStatusChanged(nil) - - // close participant as well - _ = p.Close(true, reason, false) - - r.leftAt.Store(time.Now().Unix()) - - if sendUpdates { - if r.onParticipantChanged != nil { - r.onParticipantChanged(p) - } - r.broadcastParticipantState(p, broadcastOptions{skipSource: true}) - } + return nil } -func (r *Room) UpdateSubscriptions( - participant types.LocalParticipant, - trackIDs []livekit.TrackID, - participantTracks []*livekit.ParticipantTracks, - subscribe bool, -) { - // handle subscription changes - for _, trackID := range trackIDs { - if subscribe { - participant.SubscribeToTrack(trackID) - } else { - participant.UnsubscribeFromTrack(trackID) - } - } - - for _, pt := range participantTracks { - for _, trackID := range livekit.StringsAsIDs[livekit.TrackID](pt.TrackSids) { - if subscribe { - participant.SubscribeToTrack(trackID) - } else { - participant.UnsubscribeFromTrack(trackID) - } - } - } -} - -func (r *Room) SyncState(participant types.LocalParticipant, state *livekit.SyncState) error { +func (r *Room) onSyncState(participant types.LocalParticipant, state *livekit.SyncState) error { pLogger := participant.GetLogger() pLogger.Infow("setting sync state", "state", logger.Proto(state)) @@ -805,6 +651,32 @@ func (r *Room) SyncState(participant types.LocalParticipant, state *livekit.Sync break } } + + pubDataTracks := state.GetPublishDataTracks() + existingPubDataTracks := participant.GetPublishedDataTracks() + for _, pubDataTrack := range pubDataTracks { + // client may not have sent DataTrackInfo for each published data track + dti := pubDataTrack.Info + if dti == nil { + pLogger.Warnw("DataTrackInfo not sent during resume", nil) + shouldReconnect = true + break + } + + found := false + for _, existingPubDataTrack := range existingPubDataTracks { + if existingPubDataTrack.ID() == livekit.TrackID(dti.Sid) { + found = true + break + } + } + if !found { + pLogger.Warnw("unknown data track during resume", nil, "trackID", dti.Sid) + shouldReconnect = true + break + } + } + if shouldReconnect { pLogger.Warnw("unable to resume due to missing published tracks, starting full reconnect", nil) participant.IssueFullReconnect(types.ParticipantCloseReasonPublicationError) @@ -812,22 +684,21 @@ func (r *Room) SyncState(participant types.LocalParticipant, state *livekit.Sync } // synthesize a track setting for each disabled track, - // can be set before addding subscriptions, + // can be set before adding subscriptions, // in fact it is done before so that setting can be updated immediately upon subscription. - for _, trackSid := range state.TrackSidsDisabled { + for _, trackSid := range state.GetTrackSidsDisabled() { participant.UpdateSubscribedTrackSettings(livekit.TrackID(trackSid), &livekit.UpdateTrackSettings{Disabled: true}) } - r.UpdateSubscriptions( - participant, - livekit.StringsAsIDs[livekit.TrackID](state.Subscription.TrackSids), - state.Subscription.ParticipantTracks, - state.Subscription.Subscribe, + participant.HandleUpdateSubscriptions( + livekit.StringsAsIDs[livekit.TrackID](state.GetSubscription().GetTrackSids()), + state.GetSubscription().GetParticipantTracks(), + state.GetSubscription().GetSubscribe(), ) return nil } -func (r *Room) UpdateSubscriptionPermission(participant types.LocalParticipant, subscriptionPermission *livekit.SubscriptionPermission) error { +func (r *Room) onUpdateSubscriptionPermission(participant types.LocalParticipant, subscriptionPermission *livekit.SubscriptionPermission) error { if err := participant.UpdateSubscriptionPermission(subscriptionPermission, utils.TimedVersion(0), r.GetParticipantByID); err != nil { return err } @@ -861,6 +732,23 @@ func (r *Room) ResolveMediaTrackForSubscriber(sub types.LocalParticipant, trackI return res } +func (r *Room) ResolveDataTrackForSubscriber(sub types.LocalParticipant, trackID livekit.TrackID) types.DataResolverResult { + res := types.DataResolverResult{} + + info := r.trackManager.GetDataTrackInfo(trackID) + res.TrackChangedNotifier = r.trackManager.GetOrCreateTrackChangeNotifier(trackID) + + if info == nil { + return res + } + + res.DataTrack = info.DataTrack + res.TrackRemovedNotifier = r.trackManager.GetOrCreateTrackRemoveNotifier(trackID) + res.PublisherIdentity = info.PublisherIdentity + res.PublisherID = info.PublisherID + return res +} + func (r *Room) IsClosed() bool { select { case <-r.closed: @@ -903,7 +791,7 @@ func (r *Room) CloseIfEmpty() { if elapsed >= int64(timeout) { r.Close(types.ParticipantCloseReasonRoomClosed) - r.Logger.Infow("closing idle room", "reason", reason) + r.logger.Infow("closing idle room", "reason", reason) } } @@ -919,7 +807,7 @@ func (r *Room) Close(reason types.ParticipantCloseReason) { close(r.closed) r.lock.Unlock() - r.Logger.Infow("closing room") + r.logger.Infow("closing room") for _, p := range r.GetParticipants() { _ = p.Close(true, reason, false) } @@ -935,12 +823,12 @@ func (r *Room) OnClose(f func()) { r.onClose = f } -func (r *Room) OnParticipantChanged(f func(participant types.LocalParticipant)) { +func (r *Room) OnParticipantChanged(f func(participant types.Participant)) { r.onParticipantChanged = f } func (r *Room) SendDataPacket(dp *livekit.DataPacket, kind livekit.DataPacket_Kind) { - r.onDataPacket(nil, kind, dp) + r.onDataMessage(nil, kind, dp) } func (r *Room) SetMetadata(metadata string) <-chan struct{} { @@ -960,7 +848,7 @@ func (r *Room) sendRoomUpdate() { err := p.SendRoomUpdate(roomInfo) if err != nil { - r.Logger.Warnw("failed to send room update", err, "participant", p.Identity()) + r.logger.Warnw("failed to send room update", err, "participant", p.Identity()) } } } @@ -1038,7 +926,7 @@ func (r *Room) DeleteAgentDispatch(dispatchID string) (*livekit.AgentDispatch, e if agentJob != nil { err := agentJob.waitForParticipantLeaving() if err == ErrJobShutdownTimeout { - r.Logger.Infow("Agent Worker did not disconnect after 3s") + r.logger.Infow("Agent Worker did not disconnect after 3s") } } r.RemoveParticipant(p.Identity(), p.ID(), types.ParticipantCloseReasonServiceRequestRemoveParticipant) @@ -1057,10 +945,10 @@ func (r *Room) OnRoomUpdated(f func()) { r.onRoomUpdated = f } -func (r *Room) SimulateScenario(participant types.LocalParticipant, simulateScenario *livekit.SimulateScenario) error { +func (r *Room) onSimulateScenario(participant types.LocalParticipant, simulateScenario *livekit.SimulateScenario) error { switch scenario := simulateScenario.Scenario.(type) { case *livekit.SimulateScenario_SpeakerUpdate: - r.Logger.Infow("simulating speaker update", "participant", participant.Identity(), "duration", scenario.SpeakerUpdate) + r.logger.Infow("simulating speaker update", "participant", participant.Identity(), "duration", scenario.SpeakerUpdate) go func() { <-time.After(time.Duration(scenario.SpeakerUpdate) * time.Second) r.sendSpeakerChanges([]*livekit.SpeakerInfo{{ @@ -1075,33 +963,33 @@ func (r *Room) SimulateScenario(participant types.LocalParticipant, simulateScen Level: 0.9, }}) case *livekit.SimulateScenario_Migration: - r.Logger.Infow("simulating migration", "participant", participant.Identity()) + r.logger.Infow("simulating migration", "participant", participant.Identity()) // drop participant without necessarily cleaning up if err := participant.Close(false, types.ParticipantCloseReasonSimulateMigration, true); err != nil { return err } case *livekit.SimulateScenario_NodeFailure: - r.Logger.Infow("simulating node failure", "participant", participant.Identity()) + r.logger.Infow("simulating node failure", "participant", participant.Identity()) // drop participant without necessarily cleaning up if err := participant.Close(false, types.ParticipantCloseReasonSimulateNodeFailure, true); err != nil { return err } case *livekit.SimulateScenario_ServerLeave: - r.Logger.Infow("simulating server leave", "participant", participant.Identity()) + r.logger.Infow("simulating server leave", "participant", participant.Identity()) if err := participant.Close(true, types.ParticipantCloseReasonSimulateServerLeave, false); err != nil { return err } case *livekit.SimulateScenario_SwitchCandidateProtocol: - r.Logger.Infow("simulating switch candidate protocol", "participant", participant.Identity()) + r.logger.Infow("simulating switch candidate protocol", "participant", participant.Identity()) participant.ICERestart(&livekit.ICEConfig{ PreferenceSubscriber: livekit.ICECandidateType(scenario.SwitchCandidateProtocol), PreferencePublisher: livekit.ICECandidateType(scenario.SwitchCandidateProtocol), }) case *livekit.SimulateScenario_SubscriberBandwidth: if scenario.SubscriberBandwidth > 0 { - r.Logger.Infow("simulating subscriber bandwidth start", "participant", participant.Identity(), "bandwidth", scenario.SubscriberBandwidth) + r.logger.Infow("simulating subscriber bandwidth start", "participant", participant.Identity(), "bandwidth", scenario.SubscriberBandwidth) } else { - r.Logger.Infow("simulating subscriber bandwidth end", "participant", participant.Identity()) + r.logger.Infow("simulating subscriber bandwidth end", "participant", participant.Identity()) } participant.SetSubscriberChannelCapacity(scenario.SubscriberBandwidth) case *livekit.SimulateScenario_DisconnectSignalOnResume: @@ -1120,19 +1008,7 @@ func (r *Room) SimulateScenario(participant types.LocalParticipant, simulateScen return nil } -func (r *Room) getOtherParticipantInfo(identity livekit.ParticipantIdentity) []*livekit.ParticipantInfo { - participants := r.GetParticipants() - pi := make([]*livekit.ParticipantInfo, 0, len(participants)) - for _, p := range participants { - if !p.Hidden() && p.Identity() != identity { - pi = append(pi, p.ToProto()) - } - } - - return pi -} - -// checks if participant should be autosubscribed to new tracks, assumes lock is already acquired +// checks if participant should be auto subscribed to new tracks, assumes lock is already acquired func (r *Room) autoSubscribe(participant types.LocalParticipant) bool { opts := r.participantOpts[participant.Identity()] // default to true if no options are set @@ -1142,22 +1018,32 @@ func (r *Room) autoSubscribe(participant types.LocalParticipant) bool { return true } -func (r *Room) createJoinResponseLocked(participant types.LocalParticipant, iceServers []*livekit.ICEServer) *livekit.JoinResponse { - // gather other participants and send join response - otherParticipants := make([]*livekit.ParticipantInfo, 0, len(r.participants)) - for _, p := range r.participants { - if p.ID() != participant.ID() && !p.Hidden() { - otherParticipants = append(otherParticipants, p.ToProto()) - } +// checks if participant should be auto subscribed to new data tracks, assumes lock is already acquired +func (r *Room) autoSubscribeDataTrack(participant types.LocalParticipant) bool { + opts := r.participantOpts[participant.Identity()] + // default to true if no options are set + if opts != nil && !opts.AutoSubscribeDataTrack { + return false } + return true +} +func (r *Room) createJoinResponseLocked( + participant types.LocalParticipant, + iceServers []*livekit.ICEServer, +) *livekit.JoinResponse { iceConfig := participant.GetICEConfig() hasICEFallback := iceConfig.GetPreferencePublisher() != livekit.ICECandidateType_ICT_NONE || iceConfig.GetPreferenceSubscriber() != livekit.ICECandidateType_ICT_NONE return &livekit.JoinResponse{ - Room: r.ToProto(), - Participant: participant.ToProto(), - OtherParticipants: otherParticipants, - IceServers: iceServers, + Room: r.ToProto(), + Participant: participant.ToProto(), + OtherParticipants: GetOtherParticipantInfo( + participant, + false, // isMigratingIn + toParticipants(slices.Collect(maps.Values(r.participants))), + false, // skipSubscriberBroadcast + ), + IceServers: iceServers, // indicates both server and client support subscriber as primary SubscriberPrimary: participant.SubscriberAsPrimary(), ClientConfiguration: participant.GetClientConfiguration(), @@ -1174,7 +1060,9 @@ func (r *Room) createJoinResponseLocked(participant types.LocalParticipant, iceS } // a ParticipantImpl in the room added a new track, subscribe other participants to it -func (r *Room) onTrackPublished(participant types.LocalParticipant, track types.MediaTrack) { +func (r *Room) onTrackPublished(participant types.Participant, track types.MediaTrack) { + r.trackManager.AddTrack(track, participant.Identity(), participant.ID()) + // publish participant update, since track state is changed r.broadcastParticipantState(participant, broadcastOptions{skipSource: true}) @@ -1193,13 +1081,13 @@ func (r *Room) onTrackPublished(participant types.LocalParticipant, track types. continue } - r.Logger.Debugw("subscribing to new track", - "participant", existingParticipant.Identity(), - "pID", existingParticipant.ID(), + existingParticipant.GetLogger().Debugw( + "subscribing to new track", "publisher", participant.Identity(), "publisherID", participant.ID(), - "trackID", track.ID()) - existingParticipant.SubscribeToTrack(track.ID()) + "trackID", track.ID(), + ) + existingParticipant.SubscribeToTrack(track.ID(), false) } onParticipantChanged := r.onParticipantChanged r.lock.RUnlock() @@ -1208,8 +1096,6 @@ func (r *Room) onTrackPublished(participant types.LocalParticipant, track types. onParticipantChanged(participant) } - r.trackManager.AddTrack(track, participant.Identity(), participant.ID()) - // launch jobs r.lock.Lock() hasPublished := r.hasPublished[participant.Identity()] @@ -1218,7 +1104,7 @@ func (r *Room) onTrackPublished(participant types.LocalParticipant, track types. if !hasPublished { r.lock.RLock() - r.launchTargetAgents(maps.Values(r.agentDispatches), participant, livekit.JobType_JT_PUBLISHER) + r.launchTargetAgents(slices.Collect(maps.Values(r.agentDispatches)), participant, livekit.JobType_JT_PUBLISHER) r.lock.RUnlock() if r.internal != nil && r.internal.ParticipantEgress != nil { go func() { @@ -1231,7 +1117,7 @@ func (r *Room) onTrackPublished(participant types.LocalParticipant, track types. r.Name(), r.ID(), ); err != nil { - r.Logger.Errorw("failed to launch participant egress", err) + r.logger.Errorw("failed to launch participant egress", err) } }() } @@ -1247,13 +1133,13 @@ func (r *Room) onTrackPublished(participant types.LocalParticipant, track types. r.Name(), r.ID(), ); err != nil { - r.Logger.Errorw("failed to launch track egress", err) + r.logger.Errorw("failed to launch track egress", err) } }() } } -func (r *Room) onTrackUpdated(p types.LocalParticipant, _ types.MediaTrack) { +func (r *Room) onTrackUpdated(p types.Participant, _ types.MediaTrack) { // send track updates to everyone, especially if track was updated by admin r.broadcastParticipantState(p, broadcastOptions{}) if r.onParticipantChanged != nil { @@ -1261,7 +1147,7 @@ func (r *Room) onTrackUpdated(p types.LocalParticipant, _ types.MediaTrack) { } } -func (r *Room) onTrackUnpublished(p types.LocalParticipant, track types.MediaTrack) { +func (r *Room) onTrackUnpublished(p types.Participant, track types.MediaTrack) { r.trackManager.RemoveTrack(track) if !p.IsClosed() { r.broadcastParticipantState(p, broadcastOptions{skipSource: true}) @@ -1271,7 +1157,54 @@ func (r *Room) onTrackUnpublished(p types.LocalParticipant, track types.MediaTra } } -func (r *Room) onParticipantUpdate(p types.LocalParticipant) { +func (r *Room) onDataTrackPublished(participant types.Participant, dt types.DataTrack) { + r.trackManager.AddDataTrack(dt, participant.Identity(), participant.ID()) + + // publish participant update, since a new data track was published + r.broadcastParticipantState(participant, broadcastOptions{skipSource: true}) + + r.lock.RLock() + // subscribe all existing participants to this DataTrack + for _, existingParticipant := range r.participants { + if existingParticipant == participant { + // skip publishing participant + continue + } + if existingParticipant.State() != livekit.ParticipantInfo_ACTIVE { + // not fully joined. don't subscribe yet + continue + } + if !r.autoSubscribeDataTrack(existingParticipant) { + continue + } + + existingParticipant.GetLogger().Debugw( + "subscribing to new data track", + "publisher", participant.Identity(), + "publisherID", participant.ID(), + "trackID", dt.ID(), + ) + existingParticipant.SubscribeToDataTrack(dt.ID()) + } + onParticipantChanged := r.onParticipantChanged + r.lock.RUnlock() + + if onParticipantChanged != nil { + onParticipantChanged(participant) + } +} + +func (r *Room) onDataTrackUnpublished(p types.Participant, dt types.DataTrack) { + r.trackManager.RemoveDataTrack(dt) + if !p.IsClosed() { + r.broadcastParticipantState(p, broadcastOptions{skipSource: true}) + } + if r.onParticipantChanged != nil { + r.onParticipantChanged(p) + } +} + +func (r *Room) onParticipantUpdate(p types.Participant) { r.protoProxy.MarkDirty(false) // immediately notify when permissions or metadata changed r.broadcastParticipantState(p, broadcastOptions{immediate: true}) @@ -1280,22 +1213,272 @@ func (r *Room) onParticipantUpdate(p types.LocalParticipant) { } } -func (r *Room) onDataPacket(source types.LocalParticipant, kind livekit.DataPacket_Kind, dp *livekit.DataPacket) { - BroadcastDataPacketForRoom(r, source, kind, dp, r.Logger) +func (r *Room) onStateChange(p types.LocalParticipant) { + if r.onParticipantChanged != nil { + r.onParticipantChanged(p) + } + r.broadcastParticipantState(p, broadcastOptions{skipSource: true}) + + r.onStateChangeMu.Lock() + defer r.onStateChangeMu.Unlock() + + switch p.State() { + case livekit.ParticipantInfo_ACTIVE: + // subscribe participant to existing published tracks + r.subscribeToExistingTracks(p, false) + + connectTime := time.Since(p.ConnectedAt()) + meta := &livekit.AnalyticsClientMeta{ + ClientConnectTime: uint32(connectTime.Milliseconds()), + } + infos := p.GetICEConnectionInfo() + var connectionType roomobs.ConnectionType + for _, info := range infos { + if info.Type != types.ICEConnectionTypeUnknown { + meta.ConnectionType = info.Type.String() + connectionType = info.Type.ReporterType() + break + } + } + r.telemetry.ParticipantActive(context.Background(), + r.ToProto(), + p.ToProto(), + meta, + false, + p.TelemetryGuard(), + ) + + p.GetReporter().Tx(func(tx roomobs.ParticipantSessionTx) { + tx.ReportClientConnectTime(uint16(connectTime.Milliseconds())) + tx.ReportConnectResult(roomobs.ConnectionResultSuccess) + tx.ReportConnectionType(connectionType) + }) + + fields := append( + connectionDetailsFields(infos), + "clientInfo", logger.Proto(sutils.ClientInfoWithoutAddress(p.GetClientInfo())), + "connectTime", connectTime, + ) + p.GetLogger().Infow("participant active", fields...) + + case livekit.ParticipantInfo_DISCONNECTED: + // remove participant from room + go r.RemoveParticipant(p.Identity(), p.ID(), p.CloseReason()) + } +} + +func (r *Room) onDataMessage(source types.LocalParticipant, kind livekit.DataPacket_Kind, dp *livekit.DataPacket) { + if kind == livekit.DataPacket_RELIABLE && source != nil && dp.GetSequence() > 0 { + data, err := proto.Marshal(dp) + if err != nil { + r.logger.Errorw("failed to marshal data packet for cache", err, "participant", source.Identity(), "seq", dp.GetSequence()) + return + } + r.dataMessageCache.Add(&types.DataMessageCache{ + SenderID: source.ID(), + Seq: dp.GetSequence(), + Data: data, + DestIdentities: livekit.StringsAsIDs[livekit.ParticipantIdentity](dp.DestinationIdentities), + }, len(data)) + } + BroadcastDataPacketForRoom(r, source, kind, dp, r.logger) +} + +func (r *Room) onDataMessageUnlabeled(source types.LocalParticipant, data []byte) { + BroadcastDataMessageForRoom(r, source, data, r.logger) } func (r *Room) onMetrics(source types.Participant, dp *livekit.DataPacket) { - BroadcastMetricsForRoom(r, source, dp, r.Logger) + BroadcastMetricsForRoom(r, source, dp, r.logger) } -func (r *Room) subscribeToExistingTracks(p types.LocalParticipant) { - r.lock.RLock() - shouldSubscribe := r.autoSubscribe(p) - r.lock.RUnlock() - if !shouldSubscribe { +func (r *Room) onSubscribeStatusChanged(participant types.LocalParticipant, publisherID livekit.ParticipantID, subscribed bool) { + if subscribed { + pub := r.GetParticipantByID(publisherID) + if pub != nil && pub.State() == livekit.ParticipantInfo_ACTIVE { + // when a participant subscribes to another participant, + // send speaker update if the subscribed to participant is active. + level, active := pub.GetAudioLevel() + if active { + _ = participant.SendSpeakerUpdate([]*livekit.SpeakerInfo{ + { + Sid: string(pub.ID()), + Level: float32(level), + Active: active, + }, + }, false) + } + + if cq := pub.GetConnectionQuality(); cq != nil { + update := &livekit.ConnectionQualityUpdate{} + update.Updates = append(update.Updates, cq) + _ = participant.SendConnectionQualityUpdate(update) + } + } + } else { + // no longer subscribed to the publisher, clear speaker status + _ = participant.SendSpeakerUpdate([]*livekit.SpeakerInfo{ + { + Sid: string(publisherID), + Level: 0, + Active: false, + }, + }, true) + } +} + +func (r *Room) onUpdateSubscriptions( + participant types.LocalParticipant, + trackIDs []livekit.TrackID, + participantTracks []*livekit.ParticipantTracks, + subscribe bool, +) { + r.UpdateSubscriptions(participant, trackIDs, participantTracks, subscribe) +} + +func (r *Room) UpdateSubscriptions( + participant types.LocalParticipant, + trackIDs []livekit.TrackID, + participantTracks []*livekit.ParticipantTracks, + subscribe bool, +) { + for _, trackID := range trackIDs { + if subscribe { + participant.SubscribeToTrack(trackID, false) + } else { + participant.UnsubscribeFromTrack(trackID) + } + } + + for _, pt := range participantTracks { + for _, trackID := range livekit.StringsAsIDs[livekit.TrackID](pt.TrackSids) { + if subscribe { + participant.SubscribeToTrack(trackID, false) + } else { + participant.UnsubscribeFromTrack(trackID) + } + } + } +} + +func (r *Room) onUpdateDataSubscriptions(participant types.LocalParticipant, req *livekit.UpdateDataSubscription) { + for _, update := range req.Updates { + trackID := livekit.TrackID(update.TrackSid) + if update.Subscribe { + participant.SubscribeToDataTrack(trackID) + participant.UpdateDataTrackSubscriptionOptions(trackID, update.Options) + } else { + participant.UnsubscribeFromDataTrack(trackID) + participant.UpdateDataTrackSubscriptionOptions(trackID, nil) + } + } +} + +func (r *Room) onLeave(p types.LocalParticipant, reason types.ParticipantCloseReason) { + r.RemoveParticipant(p.Identity(), p.ID(), reason) +} + +func (r *Room) RemoveParticipant( + identity livekit.ParticipantIdentity, + pID livekit.ParticipantID, + reason types.ParticipantCloseReason, +) { + r.lock.Lock() + p, ok := r.participants[identity] + if !ok { + r.lock.Unlock() return } + if pID != "" && p.ID() != pID { + // participant session has been replaced + r.lock.Unlock() + return + } + + agentJob := r.agentParticpants[identity] + + delete(r.participants, identity) + delete(r.participantOpts, identity) + delete(r.participantRequestSources, identity) + delete(r.hasPublished, identity) + delete(r.agentParticpants, identity) + if !p.Hidden() { + r.protoRoom.NumParticipants-- + } + + immediateChange := false + if p.IsRecorder() { + activeRecording := false + for _, op := range r.participants { + if op.IsRecorder() { + activeRecording = true + break + } + } + + if r.protoRoom.ActiveRecording != activeRecording { + r.protoRoom.ActiveRecording = activeRecording + immediateChange = true + } + } + r.lock.Unlock() + r.protoProxy.MarkDirty(immediateChange) + + if !p.HasConnected() { + fields := append( + connectionDetailsFields(p.GetICEConnectionInfo()), + "reason", reason.String(), + "clientInfo", logger.Proto(sutils.ClientInfoWithoutAddress(p.GetClientInfo())), + ) + p.GetLogger().Infow("removing participant without connection", fields...) + } + + // send broadcast only if it's not already closed + sendUpdates := !p.IsDisconnected() + + // remove all published tracks + for _, t := range p.GetPublishedTracks() { + r.trackManager.RemoveTrack(t) + } + + // remove all published data tracks + for _, t := range p.GetPublishedDataTracks() { + r.trackManager.RemoveDataTrack(t) + } + + if agentJob != nil { + agentJob.participantLeft() + + go func() { + _, err := r.agentClient.TerminateJob(context.Background(), agentJob.Id, rpc.JobTerminateReason_AGENT_LEFT_ROOM) + if err != nil { + r.logger.Infow("failed sending TerminateJob RPC", "error", err, "jobID", agentJob.Id, "participant", identity) + } + }() + } + + p.ClearParticipantListener() + + // close participant as well + _ = p.Close(true, reason, false) + + r.leftAt.Store(time.Now().Unix()) + + if sendUpdates { + if r.onParticipantChanged != nil { + r.onParticipantChanged(p) + } + r.broadcastParticipantState(p, broadcastOptions{skipSource: true}) + } +} + +func (r *Room) subscribeToExistingTracks(p types.LocalParticipant, isSync bool) { + r.lock.RLock() + autoSubscribe := r.autoSubscribe(p) + autoSubscribeDataTrack := r.autoSubscribeDataTrack(p) + r.lock.RUnlock() + var trackIDs []livekit.TrackID for _, op := range r.GetParticipants() { if p.ID() == op.ID() { @@ -1304,69 +1487,64 @@ func (r *Room) subscribeToExistingTracks(p types.LocalParticipant) { } // subscribe to all - for _, track := range op.GetPublishedTracks() { - trackIDs = append(trackIDs, track.ID()) - p.SubscribeToTrack(track.ID()) + if autoSubscribe { + for _, track := range op.GetPublishedTracks() { + trackIDs = append(trackIDs, track.ID()) + p.SubscribeToTrack(track.ID(), isSync) + } + } + + if autoSubscribeDataTrack { + for _, track := range op.GetPublishedDataTracks() { + trackIDs = append(trackIDs, track.ID()) + p.SubscribeToDataTrack(track.ID()) + } } } if len(trackIDs) > 0 { - r.Logger.Debugw("subscribed participant to existing tracks", "trackID", trackIDs) + p.GetLogger().Debugw("subscribed participant to existing tracks", "trackID", trackIDs) } } // broadcast an update about participant p -func (r *Room) broadcastParticipantState(p types.LocalParticipant, opts broadcastOptions) { +func (r *Room) broadcastParticipantState(p types.Participant, opts broadcastOptions) { pi := p.ToProto() - if p.Hidden() { - if !opts.skipSource { - // send update only to hidden participant - err := p.SendParticipantUpdate([]*livekit.ParticipantInfo{pi}) - if err != nil { - p.GetLogger().Errorw("could not send update to participant", err) + // send it to the same participant immediately + selfSent := false + if !opts.skipSource { + defer func() { + if selfSent { + return } - } + + if lp, ok := p.(types.LocalParticipant); ok { + err := lp.SendParticipantUpdate([]*livekit.ParticipantInfo{pi}) + if err != nil { + lp.GetLogger().Errorw("could not send update to participant", err) + } + } + }() + } + + if p.Hidden() { + // hidden participant updates are sent only to the hidden participant itself, + // these could be things like metadata update return } - updates := r.pushAndDequeueUpdates(pi, p.CloseReason(), opts.immediate) - r.sendParticipantUpdates(updates) -} - -func (r *Room) sendParticipantUpdates(updates []*participantUpdate) { - if len(updates) == 0 { - return - } - - // For filtered updates, skip - // 1. synthesized DISCONNECT - this happens on SID change - // 2. close reasons of DUPLICATE_IDENTITY/STALE - A newer session for that identity exists. - // - // Filtered updates are used with clients that can handle identity based reconnect and hence those - // conditions can be skipped. - var filteredUpdates []*livekit.ParticipantInfo - for _, update := range updates { - if update.isSynthesizedDisconnect || IsCloseNotifySkippable(update.closeReason) { - continue - } - filteredUpdates = append(filteredUpdates, update.pi) - } - - var fullUpdates []*livekit.ParticipantInfo - for _, update := range updates { - fullUpdates = append(fullUpdates, update.pi) - } - - for _, op := range r.GetParticipants() { - var err error - if op.ProtocolVersion().SupportsIdentityBasedReconnection() { - err = op.SendParticipantUpdate(filteredUpdates) - } else { - err = op.SendParticipantUpdate(fullUpdates) - } - if err != nil { - op.GetLogger().Errorw("could not send update to participant", err) - } + r.batchedUpdatesMu.Lock() + updates := PushAndDequeueUpdates( + pi, + p.CloseReason(), + opts.immediate, + r.GetParticipant(livekit.ParticipantIdentity(pi.Identity)), + r.batchedUpdates, + ) + r.batchedUpdatesMu.Unlock() + if len(updates) != 0 { + selfSent = true + SendParticipantUpdates(updates, r.GetParticipants(), r.roomConfig.UpdateBatchTargetSize) } } @@ -1379,68 +1557,6 @@ func (r *Room) sendSpeakerChanges(speakers []*livekit.SpeakerInfo) { } } -// push a participant update for batched broadcast, optionally returning immediate updates to broadcast. -// it handles the following scenarios -// * subscriber-only updates will be queued for batch updates -// * publisher & immediate updates will be returned without queuing -// * when the SID changes, it will return both updates, with the earlier participant set to disconnected -func (r *Room) pushAndDequeueUpdates( - pi *livekit.ParticipantInfo, - closeReason types.ParticipantCloseReason, - isImmediate bool, -) []*participantUpdate { - r.batchedUpdatesMu.Lock() - defer r.batchedUpdatesMu.Unlock() - - var updates []*participantUpdate - identity := livekit.ParticipantIdentity(pi.Identity) - existing := r.batchedUpdates[identity] - shouldSend := isImmediate || pi.IsPublisher - - if existing != nil { - if pi.Sid == existing.pi.Sid { - // same participant session - if pi.Version < existing.pi.Version { - // out of order update - return nil - } - } else { - // different participant sessions - if CompareParticipant(existing.pi, pi) < 0 { - // existing is older, synthesize a DISCONNECT for older and - // send immediately along with newer session to signal switch - shouldSend = true - existing.pi.State = livekit.ParticipantInfo_DISCONNECTED - existing.isSynthesizedDisconnect = true - updates = append(updates, existing) - } else { - // older session update, newer session has already become active, so nothing to do - return nil - } - } - } else { - ep := r.GetParticipant(identity) - if ep != nil { - epi := ep.ToProto() - if CompareParticipant(epi, pi) > 0 { - // older session update, newer session has already become active, so nothing to do - return nil - } - } - } - - if shouldSend { - // include any queued update, and return - delete(r.batchedUpdates, identity) - updates = append(updates, &participantUpdate{pi: pi, closeReason: closeReason}) - } else { - // enqueue for batch - r.batchedUpdates[identity] = &participantUpdate{pi: pi, closeReason: closeReason} - } - - return updates -} - func (r *Room) updateProto() *livekit.Room { r.lock.RLock() room := utils.CloneProto(r.protoRoom) @@ -1464,6 +1580,8 @@ func (r *Room) changeUpdateWorker() { subTicker := time.NewTicker(subscriberUpdateInterval) defer subTicker.Stop() + cleanDataMessageTicker := time.NewTicker(dataMessageCacheTTL) + for !r.IsClosed() { select { case <-r.closed: @@ -1475,15 +1593,18 @@ func (r *Room) changeUpdateWorker() { r.sendRoomUpdate() case <-subTicker.C: r.batchedUpdatesMu.Lock() - updatesMap := r.batchedUpdates - r.batchedUpdates = make(map[livekit.ParticipantIdentity]*participantUpdate) - r.batchedUpdatesMu.Unlock() - - if len(updatesMap) == 0 { + if len(r.batchedUpdates) == 0 { + r.batchedUpdatesMu.Unlock() continue } + updatesMap := r.batchedUpdates + r.batchedUpdates = make(map[livekit.ParticipantIdentity]*ParticipantUpdate) + r.batchedUpdatesMu.Unlock() - r.sendParticipantUpdates(maps.Values(updatesMap)) + SendParticipantUpdates(slices.Collect(maps.Values(updatesMap)), r.GetParticipants(), r.roomConfig.UpdateBatchTargetSize) + + case <-cleanDataMessageTicker.C: + r.dataMessageCache.Prune() } } } @@ -1598,7 +1719,7 @@ func (r *Room) connectionQualityWorker() { continue } if err := op.SendConnectionQualityUpdate(update); err != nil { - r.Logger.Warnw("could not send connection quality update", err, + r.logger.Warnw("could not send connection quality update", err, "participant", op.Identity()) } } @@ -1647,6 +1768,7 @@ func (r *Room) launchRoomAgents(ads []*agentDispatch) { Metadata: ad.Metadata, AgentName: ad.AgentName, DispatchId: ad.Id, + Deployment: ad.Deployment, }) r.handleNewJobs(ad.AgentDispatch, inc) done() @@ -1670,6 +1792,7 @@ func (r *Room) launchTargetAgents(ads []*agentDispatch, p types.Participant, job Metadata: ad.Metadata, AgentName: ad.AgentName, DispatchId: ad.Id, + Deployment: ad.Deployment, }) r.handleNewJobs(ad.AgentDispatch, inc) done() @@ -1689,15 +1812,15 @@ func (r *Room) handleNewJobs(ad *livekit.AgentDispatch, inc *sutils.IncrementalD }) } -func (r *Room) DebugInfo() map[string]interface{} { - info := map[string]interface{}{ +func (r *Room) DebugInfo() map[string]any { + info := map[string]any{ "Name": r.protoRoom.Name, "Sid": r.protoRoom.Sid, "CreatedAt": r.protoRoom.CreationTime, } participants := r.GetParticipants() - participantInfo := make(map[string]interface{}) + participantInfo := make(map[string]any) for _, p := range participants { participantInfo[string(p.Identity())] = p.DebugInfo() } @@ -1712,9 +1835,9 @@ func (r *Room) createAgentDispatch(dispatch *livekit.AgentDispatch) (*agentDispa } ad := newAgentDispatch(dispatch) - r.lock.RLock() + r.lock.Lock() r.agentDispatches[ad.Id] = ad - r.lock.RUnlock() + r.lock.Unlock() if r.agentStore != nil { err := r.agentStore.StoreAgentDispatch(context.Background(), ad.AgentDispatch) if err != nil { @@ -1725,12 +1848,17 @@ func (r *Room) createAgentDispatch(dispatch *livekit.AgentDispatch) (*agentDispa return ad, nil } -func (r *Room) createAgentDispatchFromParams(agentName string, metadata string) (*agentDispatch, error) { +func (r *Room) createAgentDispatchFromRoomDispatch(rad *livekit.RoomAgentDispatch) (*agentDispatch, error) { + if err := protoagent.ValidateDeployment(rad.GetDeployment()); err != nil { + return nil, err + } return r.createAgentDispatch(&livekit.AgentDispatch{ - Id: guid.New(guid.AgentDispatchPrefix), - AgentName: agentName, - Metadata: metadata, - Room: r.protoRoom.Name, + Id: guid.New(guid.AgentDispatchPrefix), + AgentName: rad.GetAgentName(), + Metadata: rad.GetMetadata(), + Room: r.protoRoom.Name, + RestartPolicy: rad.GetRestartPolicy(), + Deployment: rad.GetDeployment(), }) } @@ -1746,9 +1874,9 @@ func (r *Room) createAgentDispatchesFromRoomAgent() { } for _, ag := range roomDisp { - _, err := r.createAgentDispatchFromParams(ag.AgentName, ag.Metadata) + _, err := r.createAgentDispatchFromRoomDispatch(ag) if err != nil { - r.Logger.Warnw("failed storing room dispatch", err) + r.logger.Warnw("failed storing room dispatch", err) } } } @@ -1757,9 +1885,194 @@ func (r *Room) IsDataMessageUserPacketDuplicate(up *livekit.UserPacket) bool { return r.userPacketDeduper.IsDuplicate(up) } +func (r *Room) GetCachedReliableDataMessage(seqs map[livekit.ParticipantID]uint32) []*types.DataMessageCache { + msgs := make([]*types.DataMessageCache, 0, len(seqs)*10) + for _, msg := range r.dataMessageCache.Get() { + seq, ok := seqs[msg.SenderID] + if ok && msg.Seq >= seq { + msgs = append(msgs, msg) + } + } + return msgs +} + +func (r *Room) LocalParticipantListener() types.LocalParticipantListener { + return r.localParticipantListener +} + +func (r *Room) ParticipantTelemetryListener() types.ParticipantTelemetryListener { + return r.participantTelemetryListener +} + // ------------------------------------------------------------ -func BroadcastDataPacketForRoom(r types.Room, source types.LocalParticipant, kind livekit.DataPacket_Kind, dp *livekit.DataPacket, logger logger.Logger) { +var _ types.LocalParticipantListener = (*localParticipantListener)(nil) + +type localParticipantListener struct { + room *Room +} + +func (l *localParticipantListener) OnParticipantUpdate(p types.Participant) { + l.room.onParticipantUpdate(p) +} + +func (l *localParticipantListener) OnTrackPublished(p types.Participant, track types.MediaTrack) { + l.room.onTrackPublished(p, track) +} + +func (l *localParticipantListener) OnTrackUpdated(p types.Participant, track types.MediaTrack) { + l.room.onTrackUpdated(p, track) +} + +func (l *localParticipantListener) OnTrackUnpublished(p types.Participant, track types.MediaTrack) { + l.room.onTrackUnpublished(p, track) +} + +func (l *localParticipantListener) OnDataTrackPublished(p types.Participant, track types.DataTrack) { + l.room.onDataTrackPublished(p, track) +} + +func (l *localParticipantListener) OnDataTrackUnpublished(p types.Participant, track types.DataTrack) { + l.room.onDataTrackUnpublished(p, track) +} + +func (l *localParticipantListener) OnDataTrackMessage(_p types.Participant, _data []byte, _packet *datatrack.Packet) { +} + +func (l *localParticipantListener) OnMetrics(p types.Participant, dp *livekit.DataPacket) { + l.room.onMetrics(p, dp) +} + +func (l *localParticipantListener) OnStateChange(p types.LocalParticipant) { + l.room.onStateChange(p) +} + +func (l *localParticipantListener) OnSubscriberReady(p types.LocalParticipant) { + l.room.subscribeToExistingTracks(p, false) +} + +func (l *localParticipantListener) OnMigrateStateChange(_p types.LocalParticipant, _migrateState types.MigrateState) { +} + +func (l *localParticipantListener) OnDataMessage(p types.LocalParticipant, kind livekit.DataPacket_Kind, dp *livekit.DataPacket) { + l.room.onDataMessage(p, kind, dp) +} + +func (l *localParticipantListener) OnDataMessageUnlabeled(p types.LocalParticipant, data []byte) { + l.room.onDataMessageUnlabeled(p, data) +} + +func (l *localParticipantListener) OnSubscribeStatusChanged(p types.LocalParticipant, publisherID livekit.ParticipantID, subscribed bool) { + l.room.onSubscribeStatusChanged(p, publisherID, subscribed) +} + +func (l *localParticipantListener) OnUpdateSubscriptions( + p types.LocalParticipant, + trackIDs []livekit.TrackID, + participantTracks []*livekit.ParticipantTracks, + subscribe bool, +) { + l.room.onUpdateSubscriptions(p, trackIDs, participantTracks, subscribe) +} + +func (l *localParticipantListener) OnUpdateSubscriptionPermission(p types.LocalParticipant, subscriptionPermission *livekit.SubscriptionPermission) error { + return l.room.onUpdateSubscriptionPermission(p, subscriptionPermission) +} + +func (l *localParticipantListener) OnUpdateDataSubscriptions(p types.LocalParticipant, req *livekit.UpdateDataSubscription) { + l.room.onUpdateDataSubscriptions(p, req) +} + +func (l *localParticipantListener) OnSyncState(p types.LocalParticipant, state *livekit.SyncState) error { + return l.room.onSyncState(p, state) +} + +func (l *localParticipantListener) OnSimulateScenario(p types.LocalParticipant, simulateScenario *livekit.SimulateScenario) error { + return l.room.onSimulateScenario(p, simulateScenario) +} + +func (l *localParticipantListener) OnLeave(p types.LocalParticipant, closeReason types.ParticipantCloseReason) { + l.room.onLeave(p, closeReason) +} + +// ------------------------------------------------------------ + +var _ types.ParticipantTelemetryListener = (*participantTelemetryListener)(nil) + +type participantTelemetryListener struct { + room *Room +} + +func (l participantTelemetryListener) OnTrackPublishRequested(pID livekit.ParticipantID, identity livekit.ParticipantIdentity, ti *livekit.TrackInfo) { + l.room.telemetry.TrackPublishRequested(context.Background(), l.room.ID(), l.room.Name(), pID, identity, ti) +} + +func (l participantTelemetryListener) OnTrackPublished(pID livekit.ParticipantID, identity livekit.ParticipantIdentity, ti *livekit.TrackInfo, shouldSendEvent bool) { + l.room.telemetry.TrackPublished(context.Background(), l.room.ID(), l.room.Name(), pID, identity, ti, shouldSendEvent) +} + +func (l participantTelemetryListener) OnTrackUnpublished(pID livekit.ParticipantID, identity livekit.ParticipantIdentity, ti *livekit.TrackInfo, shouldSendEvent bool) { + l.room.telemetry.TrackUnpublished(context.Background(), l.room.ID(), l.room.Name(), pID, identity, ti, shouldSendEvent) +} + +func (l participantTelemetryListener) OnTrackSubscribeRequested(pID livekit.ParticipantID, ti *livekit.TrackInfo) { + l.room.telemetry.TrackSubscribeRequested(context.Background(), l.room.ID(), l.room.Name(), pID, ti) +} + +func (l participantTelemetryListener) OnTrackSubscribed(pID livekit.ParticipantID, ti *livekit.TrackInfo, publisherInfo *livekit.ParticipantInfo, shouldSendEvent bool) { + l.room.telemetry.TrackSubscribed(context.Background(), l.room.ID(), l.room.Name(), pID, ti, publisherInfo, shouldSendEvent) +} + +func (l participantTelemetryListener) OnTrackUnsubscribed(pID livekit.ParticipantID, ti *livekit.TrackInfo, shouldSendEvent bool) { + l.room.telemetry.TrackUnsubscribed(context.Background(), l.room.ID(), l.room.Name(), pID, ti, shouldSendEvent) +} + +func (l participantTelemetryListener) OnTrackSubscribeFailed(pID livekit.ParticipantID, trackID livekit.TrackID, err error, isUserError bool) { + l.room.telemetry.TrackSubscribeFailed(context.Background(), l.room.ID(), l.room.Name(), pID, trackID, err, isUserError) +} + +func (l participantTelemetryListener) OnTrackSubscribeStreamStarted(pID livekit.ParticipantID, ti *livekit.TrackInfo) { +} + +func (l participantTelemetryListener) OnTrackMuted(pID livekit.ParticipantID, ti *livekit.TrackInfo) { + l.room.telemetry.TrackMuted(context.Background(), l.room.ID(), l.room.Name(), pID, ti) +} + +func (l participantTelemetryListener) OnTrackUnmuted(pID livekit.ParticipantID, ti *livekit.TrackInfo) { + l.room.telemetry.TrackUnmuted(context.Background(), l.room.ID(), l.room.Name(), pID, ti) +} + +func (l participantTelemetryListener) OnTrackPublishedUpdate(pID livekit.ParticipantID, ti *livekit.TrackInfo) { + l.room.telemetry.TrackPublishedUpdate(context.Background(), l.room.ID(), l.room.Name(), pID, ti) +} + +func (l participantTelemetryListener) OnTrackMaxSubscribedVideoQuality(pID livekit.ParticipantID, ti *livekit.TrackInfo, mime mime.MimeType, maxQuality livekit.VideoQuality) { + l.room.telemetry.TrackMaxSubscribedVideoQuality(context.Background(), l.room.ID(), l.room.Name(), pID, ti, mime, maxQuality) +} + +func (l participantTelemetryListener) OnTrackPublishRTPStats(pID livekit.ParticipantID, trackID livekit.TrackID, mimeType mime.MimeType, layer int, stats *livekit.RTPStats) { + l.room.telemetry.TrackPublishRTPStats(context.Background(), l.room.ID(), l.room.Name(), pID, trackID, mimeType, layer, stats) +} + +func (l participantTelemetryListener) OnTrackSubscribeRTPStats(pID livekit.ParticipantID, trackID livekit.TrackID, mimeType mime.MimeType, stats *livekit.RTPStats) { + l.room.telemetry.TrackSubscribeRTPStats(context.Background(), l.room.ID(), l.room.Name(), pID, trackID, mimeType, stats) +} + +func (l participantTelemetryListener) OnTrackStats(key telemetry.StatsKey, stat *livekit.AnalyticsStat) { + roomID, roomName := l.room.ID(), l.room.Name() + stat.RoomId, stat.RoomName = string(roomID), string(roomName) + l.room.telemetry.TrackStats(roomID, roomName, key, stat) +} + +// ------------------------------------------------------------ + +func BroadcastDataPacketForRoom( + r types.Room, + source types.LocalParticipant, + kind livekit.DataPacket_Kind, + dp *livekit.DataPacket, + logger logger.Logger, +) { dp.Kind = kind // backward compatibility dest := dp.GetUser().GetDestinationSids() if u := dp.GetUser(); u != nil { @@ -1812,7 +2125,17 @@ func BroadcastDataPacketForRoom(r types.Room, source types.LocalParticipant, kin } utils.ParallelExec(destParticipants, dataForwardLoadBalanceThreshold, 1, func(op types.LocalParticipant) { - op.SendDataPacket(kind, dpData) + op.SendDataMessage(kind, dpData, livekit.ParticipantID(dp.GetParticipantSid()), dp.GetSequence()) + }) +} + +func BroadcastDataMessageForRoom(r types.Room, source types.LocalParticipant, data []byte, logger logger.Logger) { + utils.ParallelExec(r.GetLocalParticipants(), dataForwardLoadBalanceThreshold, 1, func(op types.LocalParticipant) { + if source != nil && op.ID() == source.ID() { + return + } + + op.SendDataMessageUnlabeled(data, false, source.Identity()) }) } @@ -1867,8 +2190,140 @@ func CompareParticipant(pi1 *livekit.ParticipantInfo, pi2 *livekit.ParticipantIn return 0 } -func connectionDetailsFields(infos []*types.ICEConnectionInfo) []interface{} { - var fields []interface{} +type ParticipantUpdate struct { + ParticipantInfo *livekit.ParticipantInfo + IsSynthesizedDisconnect bool + CloseReason types.ParticipantCloseReason +} + +// push a participant update for batched broadcast, optionally returning immediate updates to broadcast. +// it handles the following scenarios +// * subscriber-only updates will be queued for batch updates +// * publisher & immediate updates will be returned without queuing +// * when the SID changes, it will return both updates, with the earlier participant set to disconnected +func PushAndDequeueUpdates( + pi *livekit.ParticipantInfo, + closeReason types.ParticipantCloseReason, + isImmediate bool, + existingParticipant types.Participant, + cache map[livekit.ParticipantIdentity]*ParticipantUpdate, +) []*ParticipantUpdate { + var updates []*ParticipantUpdate + identity := livekit.ParticipantIdentity(pi.Identity) + existing := cache[identity] + shouldSend := isImmediate || pi.IsPublisher + + if existing != nil { + if pi.Sid == existing.ParticipantInfo.Sid { + // same participant session + if pi.Version < existing.ParticipantInfo.Version { + // out of order update + return nil + } + } else { + // different participant sessions + if CompareParticipant(existing.ParticipantInfo, pi) < 0 { + // existing is older, synthesize a DISCONNECT for older and + // send immediately along with newer session to signal switch + shouldSend = true + existing.ParticipantInfo.State = livekit.ParticipantInfo_DISCONNECTED + existing.IsSynthesizedDisconnect = true + updates = append(updates, existing) + } else { + // older session update, newer session has already become active, so nothing to do + return nil + } + } + } else { + if existingParticipant != nil { + epi := existingParticipant.ToProto() + if CompareParticipant(epi, pi) > 0 { + // older session update, newer session has already become active, so nothing to do + return nil + } + } + } + + if shouldSend { + // include any queued update, and return + delete(cache, identity) + updates = append(updates, &ParticipantUpdate{ParticipantInfo: pi, CloseReason: closeReason}) + } else { + // enqueue for batch + cache[identity] = &ParticipantUpdate{ParticipantInfo: pi, CloseReason: closeReason} + } + + return updates +} + +func SendParticipantUpdates(updates []*ParticipantUpdate, participants []types.LocalParticipant, batchTargetSize int) { + if len(updates) == 0 { + return + } + + // For filtered updates, skip + // 1. synthesized DISCONNECT - this happens on SID change + // 2. close reasons of DUPLICATE_IDENTITY/STALE - A newer session for that identity exists. + // + // Filtered updates are used with clients that can handle identity based reconnect and hence those + // conditions can be skipped. + var filteredUpdates []*livekit.ParticipantInfo + for _, update := range updates { + if update.IsSynthesizedDisconnect || IsCloseNotifySkippable(update.CloseReason) { + continue + } + filteredUpdates = append(filteredUpdates, update.ParticipantInfo) + } + + var fullUpdates []*livekit.ParticipantInfo + for _, update := range updates { + fullUpdates = append(fullUpdates, update.ParticipantInfo) + } + + filteredUpdateChunks := ChunkProtoBatch(filteredUpdates, batchTargetSize) + fullUpdateChunks := ChunkProtoBatch(fullUpdates, batchTargetSize) + + for _, op := range participants { + updateChunks := fullUpdateChunks + if op.ProtocolVersion().SupportsIdentityBasedReconnection() { + updateChunks = filteredUpdateChunks + } + for _, chunk := range updateChunks { + if err := op.SendParticipantUpdate(chunk); err != nil { + op.GetLogger().Errorw("could not send update to participant", err) + break + } + } + } +} + +// GetOtherParticipantInfo returns ParticipantInfo for everyone in the room except for the participant identified by lp.Identity() +func GetOtherParticipantInfo( + lp types.LocalParticipant, + isMigratingIn bool, + allParticipants []types.Participant, + skipSubscriberBroadcast bool, +) []*livekit.ParticipantInfo { + var lpIdentity livekit.ParticipantIdentity + if lp != nil { + lpIdentity = lp.Identity() + } + + pInfos := make([]*livekit.ParticipantInfo, 0, len(allParticipants)) + for _, op := range allParticipants { + if (!skipSubscriberBroadcast || !op.CanSkipBroadcast()) && + !op.Hidden() && + op.Identity() != lpIdentity && + !isMigratingIn { + pInfos = append(pInfos, op.ToProto()) + } + } + + return pInfos +} + +func connectionDetailsFields(infos []*types.ICEConnectionInfo) []any { + var fields []any connectionType := types.ICEConnectionTypeUnknown for _, info := range infos { candidates := make([]string, 0, len(info.Remote)+len(info.Local)) @@ -1882,7 +2337,7 @@ func connectionDetailsFields(infos []*types.ICEConnectionInfo) []interface{} { if c.Trickle { cStr += "[trickle]" } - cStr += " " + c.Local.String() + cStr += " " + c.Candidate.String() candidates = append(candidates, cStr) } for _, c := range info.Remote { @@ -1895,11 +2350,11 @@ func connectionDetailsFields(infos []*types.ICEConnectionInfo) []interface{} { if c.Trickle { cStr += "[trickle]" } - cStr += " " + fmt.Sprintf("%s %s %s:%d", c.Remote.NetworkType(), c.Remote.Type(), MaybeTruncateIP(c.Remote.Address()), c.Remote.Port()) - if relatedAddress := c.Remote.RelatedAddress(); relatedAddress != nil { - relatedAddr := MaybeTruncateIP(relatedAddress.Address) + cStr += " " + fmt.Sprintf("%s %s %s:%d", c.Candidate.Protocol.String(), c.Candidate.Typ.String(), MaybeTruncateIP(c.Candidate.Address), c.Candidate.Port) + if relatedAddress := c.Candidate.RelatedAddress; relatedAddress != "" { + relatedAddr := MaybeTruncateIP(relatedAddress) if relatedAddr != "" { - cStr += " " + fmt.Sprintf(" related %s:%d", relatedAddr, relatedAddress.Port) + cStr += " " + fmt.Sprintf(" related %s:%d", relatedAddr, c.Candidate.RelatedPort) } } candidates = append(candidates, cStr) @@ -1914,3 +2369,11 @@ func connectionDetailsFields(infos []*types.ICEConnectionInfo) []interface{} { fields = append(fields, "connectionType", connectionType) return fields } + +func toParticipants(lps []types.LocalParticipant) []types.Participant { + participants := make([]types.Participant, len(lps)) + for idx, lp := range lps { + participants[idx] = lp + } + return participants +} diff --git a/pkg/rtc/room_test.go b/pkg/rtc/room_test.go index 1ad8af2..0d69710 100644 --- a/pkg/rtc/room_test.go +++ b/pkg/rtc/room_test.go @@ -22,6 +22,7 @@ import ( "github.com/stretchr/testify/require" "google.golang.org/protobuf/proto" + "github.com/livekit/protocol/auth/authfakes" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/utils" "github.com/livekit/protocol/webhook" @@ -89,7 +90,7 @@ func TestJoinedState(t *testing.T) { func TestRoomJoin(t *testing.T) { t.Run("joining returns existing participant data", func(t *testing.T) { rm := newRoomWithParticipants(t, testRoomOpts{num: numParticipants}) - pNew := NewMockParticipant("new", types.CurrentProtocol, false, false) + pNew := NewMockParticipant("new", types.CurrentProtocol, false, false, rm.LocalParticipantListener()) _ = rm.Join(pNew, nil, nil, iceServersForRoom) @@ -104,14 +105,14 @@ func TestRoomJoin(t *testing.T) { t.Run("subscribe to existing channels upon join", func(t *testing.T) { numExisting := 3 rm := newRoomWithParticipants(t, testRoomOpts{num: numExisting}) - p := NewMockParticipant("new", types.CurrentProtocol, false, false) + lpl := rm.LocalParticipantListener() + p := NewMockParticipant("new", types.CurrentProtocol, false, false, lpl) err := rm.Join(p, nil, &ParticipantOptions{AutoSubscribe: true}, iceServersForRoom) require.NoError(t, err) - stateChangeCB := p.OnStateChangeArgsForCall(0) - require.NotNil(t, stateChangeCB) - stateChangeCB(p, livekit.ParticipantInfo_ACTIVE) + p.StateReturns(livekit.ParticipantInfo_ACTIVE) + lpl.OnStateChange(p) // it should become a subscriber when connectivity changes numTracks := 0 @@ -128,7 +129,7 @@ func TestRoomJoin(t *testing.T) { t.Run("participant state change is broadcasted to others", func(t *testing.T) { rm := newRoomWithParticipants(t, testRoomOpts{num: numParticipants}) var changedParticipant types.Participant - rm.OnParticipantChanged(func(participant types.LocalParticipant) { + rm.OnParticipantChanged(func(participant types.Participant) { changedParticipant = participant }) participants := rm.GetParticipants() @@ -159,7 +160,7 @@ func TestRoomJoin(t *testing.T) { rm.lock.Lock() rm.protoRoom.MaxParticipants = 1 rm.lock.Unlock() - p := NewMockParticipant("second", types.ProtocolVersion(0), false, false) + p := NewMockParticipant("second", types.ProtocolVersion(0), false, false, rm.LocalParticipantListener()) err := rm.Join(p, nil, nil, iceServersForRoom) require.Equal(t, ErrMaxParticipantsExceeded, err) @@ -177,7 +178,7 @@ func TestParticipantUpdate(t *testing.T) { "track mutes are sent to everyone", true, func(p types.LocalParticipant) { - p.SetTrackMuted("", true, false) + p.SetTrackMuted(&livekit.MuteTrackRequest{Muted: true}, false) }, }, { @@ -270,14 +271,14 @@ func TestPushAndDequeueUpdates(t *testing.T) { pi *livekit.ParticipantInfo closeReason types.ParticipantCloseReason immediate bool - existing *participantUpdate - expected []*participantUpdate - validate func(t *testing.T, rm *Room, updates []*participantUpdate) + existing *ParticipantUpdate + expected []*ParticipantUpdate + validate func(t *testing.T, rm *Room, updates []*ParticipantUpdate) }{ { name: "publisher updates are immediate", pi: publisher1v1, - expected: []*participantUpdate{{pi: publisher1v1}}, + expected: []*ParticipantUpdate{{ParticipantInfo: publisher1v1}}, }, { name: "subscriber updates are queued", @@ -286,20 +287,20 @@ func TestPushAndDequeueUpdates(t *testing.T) { { name: "last version is enqueued", pi: subscriber1v2, - existing: &participantUpdate{pi: utils.CloneProto(subscriber1v1)}, // clone the existing value since it can be modified when setting to disconnected - validate: func(t *testing.T, rm *Room, _ []*participantUpdate) { + existing: &ParticipantUpdate{ParticipantInfo: utils.CloneProto(subscriber1v1)}, // clone the existing value since it can be modified when setting to disconnected + validate: func(t *testing.T, rm *Room, _ []*ParticipantUpdate) { queued := rm.batchedUpdates[livekit.ParticipantIdentity(identity)] require.NotNil(t, queued) - requirePIEquals(t, subscriber1v2, queued.pi) + requirePIEquals(t, subscriber1v2, queued.ParticipantInfo) }, }, { name: "latest version when immediate", pi: subscriber1v2, - existing: &participantUpdate{pi: utils.CloneProto(subscriber1v1)}, + existing: &ParticipantUpdate{ParticipantInfo: utils.CloneProto(subscriber1v1)}, immediate: true, - expected: []*participantUpdate{{pi: subscriber1v2}}, - validate: func(t *testing.T, rm *Room, _ []*participantUpdate) { + expected: []*ParticipantUpdate{{ParticipantInfo: subscriber1v2}}, + validate: func(t *testing.T, rm *Room, _ []*ParticipantUpdate) { queued := rm.batchedUpdates[livekit.ParticipantIdentity(identity)] require.Nil(t, queued) }, @@ -307,37 +308,37 @@ func TestPushAndDequeueUpdates(t *testing.T) { { name: "out of order updates are rejected", pi: subscriber1v1, - existing: &participantUpdate{pi: utils.CloneProto(subscriber1v2)}, - validate: func(t *testing.T, rm *Room, updates []*participantUpdate) { + existing: &ParticipantUpdate{ParticipantInfo: utils.CloneProto(subscriber1v2)}, + validate: func(t *testing.T, rm *Room, updates []*ParticipantUpdate) { queued := rm.batchedUpdates[livekit.ParticipantIdentity(identity)] - requirePIEquals(t, subscriber1v2, queued.pi) + requirePIEquals(t, subscriber1v2, queued.ParticipantInfo) }, }, { name: "sid change is broadcasted immediately with synthsized disconnect", pi: publisher2, closeReason: types.ParticipantCloseReasonServiceRequestRemoveParticipant, // just to test if update contain the close reason - existing: &participantUpdate{pi: utils.CloneProto(subscriber1v2), closeReason: types.ParticipantCloseReasonStale}, - expected: []*participantUpdate{ + existing: &ParticipantUpdate{ParticipantInfo: utils.CloneProto(subscriber1v2), CloseReason: types.ParticipantCloseReasonStale}, + expected: []*ParticipantUpdate{ { - pi: &livekit.ParticipantInfo{ + ParticipantInfo: &livekit.ParticipantInfo{ Identity: identity, Sid: "1", Version: 2, State: livekit.ParticipantInfo_DISCONNECTED, }, - isSynthesizedDisconnect: true, - closeReason: types.ParticipantCloseReasonStale, + IsSynthesizedDisconnect: true, + CloseReason: types.ParticipantCloseReasonStale, }, - {pi: publisher2, closeReason: types.ParticipantCloseReasonServiceRequestRemoveParticipant}, + {ParticipantInfo: publisher2, CloseReason: types.ParticipantCloseReasonServiceRequestRemoveParticipant}, }, }, { name: "when switching to publisher, queue is cleared", pi: publisher1v2, - existing: &participantUpdate{pi: utils.CloneProto(subscriber1v1)}, - expected: []*participantUpdate{{pi: publisher1v2}}, - validate: func(t *testing.T, rm *Room, updates []*participantUpdate) { + existing: &ParticipantUpdate{ParticipantInfo: utils.CloneProto(subscriber1v1)}, + expected: []*ParticipantUpdate{{ParticipantInfo: publisher1v2}}, + validate: func(t *testing.T, rm *Room, updates []*ParticipantUpdate) { require.Empty(t, rm.batchedUpdates) }, }, @@ -347,14 +348,22 @@ func TestPushAndDequeueUpdates(t *testing.T) { t.Run(tc.name, func(t *testing.T) { rm := newRoomWithParticipants(t, testRoomOpts{num: 1}) if tc.existing != nil { - rm.batchedUpdates[livekit.ParticipantIdentity(tc.existing.pi.Identity)] = tc.existing + rm.batchedUpdates[livekit.ParticipantIdentity(tc.existing.ParticipantInfo.Identity)] = tc.existing } - updates := rm.pushAndDequeueUpdates(tc.pi, tc.closeReason, tc.immediate) + rm.batchedUpdatesMu.Lock() + updates := PushAndDequeueUpdates( + tc.pi, + tc.closeReason, + tc.immediate, + rm.GetParticipant(livekit.ParticipantIdentity(tc.pi.Identity)), + rm.batchedUpdates, + ) + rm.batchedUpdatesMu.Unlock() require.Equal(t, len(tc.expected), len(updates)) for i, item := range tc.expected { - requirePIEquals(t, item.pi, updates[i].pi) - require.Equal(t, item.isSynthesizedDisconnect, updates[i].isSynthesizedDisconnect) - require.Equal(t, item.closeReason, updates[i].closeReason) + requirePIEquals(t, item.ParticipantInfo, updates[i].ParticipantInfo) + require.Equal(t, item.IsSynthesizedDisconnect, updates[i].IsSynthesizedDisconnect) + require.Equal(t, item.CloseReason, updates[i].CloseReason) } if tc.validate != nil { @@ -417,6 +426,8 @@ func TestRoomClosure(t *testing.T) { func TestNewTrack(t *testing.T) { t.Run("new track should be added to ready participants", func(t *testing.T) { rm := newRoomWithParticipants(t, testRoomOpts{num: 3}) + lpl := rm.LocalParticipantListener() + participants := rm.GetParticipants() p0 := participants[0].(*typesfakes.FakeLocalParticipant) p0.StateReturns(livekit.ParticipantInfo_JOINED) @@ -427,9 +438,8 @@ func TestNewTrack(t *testing.T) { // pub adds track track := NewMockTrack(livekit.TrackType_VIDEO, "webcam") - trackCB := pub.OnTrackPublishedArgsForCall(0) - require.NotNil(t, trackCB) - trackCB(pub, track) + lpl.OnTrackPublished(pub, track) + // only p1 should've been subscribed to require.Equal(t, 0, p0.SubscribeToTrackCallCount()) require.Equal(t, 1, p1.SubscribeToTrackCallCount()) @@ -441,7 +451,7 @@ func TestActiveSpeakers(t *testing.T) { getActiveSpeakerUpdates := func(p *typesfakes.FakeLocalParticipant) [][]*livekit.SpeakerInfo { var updates [][]*livekit.SpeakerInfo numCalls := p.SendSpeakerUpdateCallCount() - for i := 0; i < numCalls; i++ { + for i := range numCalls { infos, _ := p.SendSpeakerUpdateArgsForCall(i) updates = append(updates, infos) } @@ -518,6 +528,7 @@ func TestActiveSpeakers(t *testing.T) { t.Run("audio level is smoothed", func(t *testing.T) { rm := newRoomWithParticipants(t, testRoomOpts{num: 2, protocol: 3, audioSmoothIntervals: 3}) defer rm.Close(types.ParticipantCloseReasonNone) + participants := rm.GetParticipants() p := participants[0].(*typesfakes.FakeLocalParticipant) op := participants[1].(*typesfakes.FakeLocalParticipant) @@ -611,6 +622,9 @@ func TestDataChannel(t *testing.T) { t.Run(modeNames[mode], func(t *testing.T) { rm := newRoomWithParticipants(t, testRoomOpts{num: 3}) defer rm.Close(types.ParticipantCloseReasonNone) + + lpl := rm.LocalParticipantListener() + participants := rm.GetParticipants() p := participants[0].(*typesfakes.FakeLocalParticipant) @@ -631,17 +645,17 @@ func TestDataChannel(t *testing.T) { } encoded, _ := proto.Marshal(packetExp) - p.OnDataPacketArgsForCall(0)(p, packet.Kind, packet) + lpl.OnDataMessage(p, packet.Kind, packet) // ensure everyone has received the packet for _, op := range participants { fp := op.(*typesfakes.FakeLocalParticipant) if fp == p { - require.Zero(t, fp.SendDataPacketCallCount()) + require.Zero(t, fp.SendDataMessageCallCount()) continue } - require.Equal(t, 1, fp.SendDataPacketCallCount()) - _, got := fp.SendDataPacketArgsForCall(0) + require.Equal(t, 1, fp.SendDataMessageCallCount()) + _, got, _, _ := fp.SendDataMessageArgsForCall(0) require.Equal(t, encoded, got) } }) @@ -654,6 +668,9 @@ func TestDataChannel(t *testing.T) { t.Run(modeNames[mode], func(t *testing.T) { rm := newRoomWithParticipants(t, testRoomOpts{num: 4}) defer rm.Close(types.ParticipantCloseReasonNone) + + lpl := rm.LocalParticipantListener() + participants := rm.GetParticipants() p := participants[0].(*typesfakes.FakeLocalParticipant) p1 := participants[1].(*typesfakes.FakeLocalParticipant) @@ -678,17 +695,17 @@ func TestDataChannel(t *testing.T) { } encoded, _ := proto.Marshal(packetExp) - p.OnDataPacketArgsForCall(0)(p, packet.Kind, packet) + lpl.OnDataMessage(p, packet.Kind, packet) // only p1 should receive the data for _, op := range participants { fp := op.(*typesfakes.FakeLocalParticipant) if fp != p1 { - require.Zero(t, fp.SendDataPacketCallCount()) + require.Zero(t, fp.SendDataMessageCallCount()) } } - require.Equal(t, 1, p1.SendDataPacketCallCount()) - _, got := p1.SendDataPacketArgsForCall(0) + require.Equal(t, 1, p1.SendDataMessageCallCount()) + _, got, _, _ := p1.SendDataMessageArgsForCall(0) require.Equal(t, encoded, got) }) } @@ -697,6 +714,7 @@ func TestDataChannel(t *testing.T) { t.Run("publishing disallowed", func(t *testing.T) { rm := newRoomWithParticipants(t, testRoomOpts{num: 2}) defer rm.Close(types.ParticipantCloseReasonNone) + participants := rm.GetParticipants() p := participants[0].(*typesfakes.FakeLocalParticipant) p.CanPublishDataReturns(false) @@ -710,13 +728,14 @@ func TestDataChannel(t *testing.T) { }, } if p.CanPublishData() { - p.OnDataPacketArgsForCall(0)(p, packet.Kind, &packet) + lpl := rm.LocalParticipantListener() + lpl.OnDataMessage(p, packet.Kind, &packet) } // no one should've been sent packet for _, op := range participants { fp := op.(*typesfakes.FakeLocalParticipant) - require.Zero(t, fp.SendDataPacketCallCount()) + require.Zero(t, fp.SendDataMessageCallCount()) } }) } @@ -726,7 +745,7 @@ func TestHiddenParticipants(t *testing.T) { rm := newRoomWithParticipants(t, testRoomOpts{num: 2, numHidden: 1}) defer rm.Close(types.ParticipantCloseReasonNone) - pNew := NewMockParticipant("new", types.CurrentProtocol, false, false) + pNew := NewMockParticipant("new", types.CurrentProtocol, false, false, rm.LocalParticipantListener()) rm.Join(pNew, nil, nil, iceServersForRoom) // expect new participant to get a JoinReply @@ -740,14 +759,14 @@ func TestHiddenParticipants(t *testing.T) { t.Run("hidden participant subscribes to tracks", func(t *testing.T) { rm := newRoomWithParticipants(t, testRoomOpts{num: 2}) - hidden := NewMockParticipant("hidden", types.CurrentProtocol, true, false) + lpl := rm.LocalParticipantListener() + hidden := NewMockParticipant("hidden", types.CurrentProtocol, true, false, lpl) err := rm.Join(hidden, nil, &ParticipantOptions{AutoSubscribe: true}, iceServersForRoom) require.NoError(t, err) - stateChangeCB := hidden.OnStateChangeArgsForCall(0) - require.NotNil(t, stateChangeCB) - stateChangeCB(hidden, livekit.ParticipantInfo_ACTIVE) + hidden.StateReturns(livekit.ParticipantInfo_ACTIVE) + lpl.OnStateChange(hidden) require.Eventually(t, func() bool { return hidden.SubscribeToTrackCallCount() == 2 }, 5*time.Second, 10*time.Millisecond) }) @@ -761,7 +780,7 @@ func TestRoomUpdate(t *testing.T) { p1 := rm.GetParticipants()[0].(*typesfakes.FakeLocalParticipant) require.Equal(t, 0, p1.SendRoomUpdateCallCount()) - p2 := NewMockParticipant("p2", types.CurrentProtocol, false, false) + p2 := NewMockParticipant("p2", types.CurrentProtocol, false, false, rm.LocalParticipantListener()) require.NoError(t, rm.Join(p2, nil, nil, iceServersForRoom)) // p1 should have received an update @@ -795,6 +814,12 @@ type testRoomOpts struct { } func newRoomWithParticipants(t *testing.T, opts testRoomOpts) *Room { + kp := &authfakes.FakeKeyProvider{} + kp.GetSecretReturns("testkey") + + n, err := webhook.NewDefaultNotifier(webhook.DefaultWebHookConfig, kp) + require.NoError(t, err) + rm := NewRoom( &livekit.Room{Name: "room"}, nil, @@ -816,12 +841,12 @@ func newRoomWithParticipants(t *testing.T, opts testRoomOpts) *Room { NodeId: "testnode", Region: "testregion", }, - telemetry.NewTelemetryService(webhook.NewDefaultNotifier("", "", nil), &telemetryfakes.FakeAnalyticsService{}), + telemetry.NewTelemetryService(n, &telemetryfakes.FakeAnalyticsService{}), nil, nil, nil, ) for i := 0; i < opts.num+opts.numHidden; i++ { identity := livekit.ParticipantIdentity(fmt.Sprintf("p%d", i)) - participant := NewMockParticipant(identity, opts.protocol, i >= opts.num, true) + participant := NewMockParticipant(identity, opts.protocol, i >= opts.num, true, rm.LocalParticipantListener()) err := rm.Join(participant, nil, &ParticipantOptions{AutoSubscribe: true}, iceServersForRoom) require.NoError(t, err) participant.StateReturns(livekit.ParticipantInfo_ACTIVE) diff --git a/pkg/rtc/roomtrackmanager.go b/pkg/rtc/roomtrackmanager.go index a2813a0..2a5c93f 100644 --- a/pkg/rtc/roomtrackmanager.go +++ b/pkg/rtc/roomtrackmanager.go @@ -22,14 +22,18 @@ import ( "github.com/livekit/livekit-server/pkg/rtc/types" "github.com/livekit/livekit-server/pkg/utils" "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/logger" ) // RoomTrackManager holds tracks that are published to the room type RoomTrackManager struct { + logger logger.Logger + lock sync.RWMutex changedNotifier *utils.ChangeNotifierManager removedNotifier *utils.ChangeNotifierManager - tracks map[livekit.TrackID]*TrackInfo + tracks map[livekit.TrackID][]*TrackInfo + dataTracks map[livekit.TrackID][]*DataTrackInfo } type TrackInfo struct { @@ -38,54 +42,101 @@ type TrackInfo struct { PublisherID livekit.ParticipantID } -func NewRoomTrackManager() *RoomTrackManager { +type DataTrackInfo struct { + DataTrack types.DataTrack + PublisherIdentity livekit.ParticipantIdentity + PublisherID livekit.ParticipantID +} + +func NewRoomTrackManager(logger logger.Logger) *RoomTrackManager { return &RoomTrackManager{ - tracks: make(map[livekit.TrackID]*TrackInfo), + logger: logger, + tracks: make(map[livekit.TrackID][]*TrackInfo), + dataTracks: make(map[livekit.TrackID][]*DataTrackInfo), changedNotifier: utils.NewChangeNotifierManager(), removedNotifier: utils.NewChangeNotifierManager(), } } func (r *RoomTrackManager) AddTrack(track types.MediaTrack, publisherIdentity livekit.ParticipantIdentity, publisherID livekit.ParticipantID) { + trackID := track.ID() r.lock.Lock() - r.tracks[track.ID()] = &TrackInfo{ + infos, ok := r.tracks[trackID] + if ok { + for _, info := range infos { + if info.Track == track { + r.lock.Unlock() + r.logger.Infow("not adding duplicate track", "trackID", trackID) + return + } + } + } + r.tracks[trackID] = append(r.tracks[trackID], &TrackInfo{ Track: track, PublisherIdentity: publisherIdentity, PublisherID: publisherID, - } + }) r.lock.Unlock() - r.NotifyTrackChanged(track.ID()) + r.NotifyTrackChanged(trackID) } func (r *RoomTrackManager) RemoveTrack(track types.MediaTrack) { + trackID := track.ID() r.lock.Lock() // ensure we are removing the same track as added - info, ok := r.tracks[track.ID()] - if !ok || info.Track != track { + infos, ok := r.tracks[trackID] + if !ok { r.lock.Unlock() return } - delete(r.tracks, track.ID()) - r.lock.Unlock() - n := r.removedNotifier.GetNotifier(string(track.ID())) + numRemoved := 0 + idx := 0 + for _, info := range infos { + if info.Track == track { + numRemoved++ + } else { + r.tracks[trackID][idx] = info + idx++ + } + } + for j := idx; j < len(infos); j++ { + r.tracks[trackID][j] = nil + } + r.tracks[trackID] = r.tracks[trackID][:idx] + if len(r.tracks[trackID]) == 0 { + delete(r.tracks, trackID) + } + r.lock.Unlock() + if numRemoved == 0 { + return + } + if numRemoved > 1 { + r.logger.Warnw("removed multiple tracks", nil, "trackID", trackID, "numRemoved", numRemoved) + } + + n := r.removedNotifier.GetNotifier(string(trackID)) if n != nil { n.NotifyChanged() } - r.changedNotifier.RemoveNotifier(string(track.ID()), true) - r.removedNotifier.RemoveNotifier(string(track.ID()), true) + r.changedNotifier.RemoveNotifier(string(trackID), true) + r.removedNotifier.RemoveNotifier(string(trackID), true) } func (r *RoomTrackManager) GetTrackInfo(trackID livekit.TrackID) *TrackInfo { r.lock.RLock() defer r.lock.RUnlock() - info := r.tracks[trackID] - if info == nil { + infos := r.tracks[trackID] + if len(infos) == 0 { return nil } + + // earliest added track is used till it is removed + info := infos[0] + // when track is about to close, do not resolve if info.Track != nil && !info.Track.IsOpen() { return nil @@ -123,3 +174,90 @@ func (r *RoomTrackManager) GetOrCreateTrackChangeNotifier(trackID livekit.TrackI func (r *RoomTrackManager) GetOrCreateTrackRemoveNotifier(trackID livekit.TrackID) *utils.ChangeNotifier { return r.removedNotifier.GetOrCreateNotifier(string(trackID)) } + +func (r *RoomTrackManager) AddDataTrack(dataTrack types.DataTrack, publisherIdentity livekit.ParticipantIdentity, publisherID livekit.ParticipantID) { + trackID := dataTrack.ID() + r.lock.Lock() + infos, ok := r.dataTracks[trackID] + if ok { + for _, info := range infos { + if info.DataTrack == dataTrack { + r.lock.Unlock() + r.logger.Infow("not adding duplicate data track", "trackID", trackID) + return + } + } + } + r.dataTracks[trackID] = append(r.dataTracks[trackID], &DataTrackInfo{ + DataTrack: dataTrack, + PublisherIdentity: publisherIdentity, + PublisherID: publisherID, + }) + r.lock.Unlock() + + r.NotifyTrackChanged(trackID) +} + +func (r *RoomTrackManager) RemoveDataTrack(dataTrack types.DataTrack) { + trackID := dataTrack.ID() + r.lock.Lock() + // ensure we are removing the same track as added + infos, ok := r.dataTracks[trackID] + if !ok { + r.lock.Unlock() + return + } + + numRemoved := 0 + idx := 0 + for _, info := range infos { + if info.DataTrack == dataTrack { + numRemoved++ + } else { + r.dataTracks[trackID][idx] = info + idx++ + } + } + for j := idx; j < len(infos); j++ { + r.dataTracks[trackID][j] = nil + } + r.dataTracks[trackID] = r.dataTracks[trackID][:idx] + if len(r.dataTracks[trackID]) == 0 { + delete(r.dataTracks, trackID) + } + r.lock.Unlock() + if numRemoved == 0 { + return + } + if numRemoved > 1 { + r.logger.Warnw("removed multiple data tracks", nil, "trackID", trackID, "numRemoved", numRemoved) + } + + n := r.removedNotifier.GetNotifier(string(trackID)) + if n != nil { + n.NotifyChanged() + } + + r.changedNotifier.RemoveNotifier(string(trackID), true) + r.removedNotifier.RemoveNotifier(string(trackID), true) +} + +func (r *RoomTrackManager) GetDataTrackInfo(trackID livekit.TrackID) *DataTrackInfo { + r.lock.RLock() + defer r.lock.RUnlock() + + infos := r.dataTracks[trackID] + if len(infos) == 0 { + return nil + } + + // earliest added data track is used till it is removed + return infos[0] +} + +func (r *RoomTrackManager) Report() (int, int) { + r.lock.RLock() + defer r.lock.RUnlock() + + return len(r.tracks), len(r.dataTracks) +} diff --git a/pkg/rtc/signalanddatastats.go b/pkg/rtc/signalanddatastats.go new file mode 100644 index 0000000..fe83bd9 --- /dev/null +++ b/pkg/rtc/signalanddatastats.go @@ -0,0 +1,307 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rtc + +import ( + "context" + "fmt" + "sync" + "time" + + "github.com/frostbyte73/core" + "go.uber.org/atomic" + + "github.com/livekit/livekit-server/pkg/rtc/types" + "github.com/livekit/livekit-server/pkg/telemetry" + "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/observability/roomobs" + "github.com/livekit/protocol/utils" +) + +type BytesTrackType string + +const ( + BytesTrackTypeData BytesTrackType = "DT" + BytesTrackTypeSignal BytesTrackType = "SG" +) + +// ------------------------------- + +type TrafficTotals struct { + At time.Time + SendBytes uint64 + SendMessages uint32 + RecvBytes uint64 + RecvMessages uint32 +} + +// -------------------------------- + +// stats for signal and data channel +type BytesTrackStats struct { + country string + pID livekit.ParticipantID + kind livekit.ParticipantInfo_Kind + kindDetails []livekit.ParticipantInfo_KindDetail + trackID livekit.TrackID + send, recv atomic.Uint64 + sendMessages, recvMessages atomic.Uint32 + totalSendBytes, totalRecvBytes atomic.Uint64 + totalSendMessages, totalRecvMessages atomic.Uint32 + telemetryListener types.ParticipantTelemetryListener + reporter roomobs.TrackReporter + done core.Fuse +} + +func NewBytesTrackStats( + country string, + trackID livekit.TrackID, + pID livekit.ParticipantID, + kind livekit.ParticipantInfo_Kind, + kindDetails []livekit.ParticipantInfo_KindDetail, + telemetryListener types.ParticipantTelemetryListener, + participantReporter roomobs.ParticipantSessionReporter, +) *BytesTrackStats { + s := &BytesTrackStats{ + country: country, + pID: pID, + kind: kind, + kindDetails: kindDetails, + trackID: trackID, + telemetryListener: telemetryListener, + reporter: participantReporter.WithTrack(trackID.String()), + } + go s.worker() + return s +} + +func (s *BytesTrackStats) AddBytes(bytes uint64, isSend bool) { + if isSend { + s.send.Add(bytes) + s.sendMessages.Inc() + s.totalSendBytes.Add(bytes) + s.totalSendMessages.Inc() + + s.reporter.Tx(func(tx roomobs.TrackTx) { + tx.ParticipantSession().ReportKindCode(roomobs.ParticipantKindCode(s.kind)) + tx.ParticipantSession().ReportKindDetailsCodes(roomobs.ParticipantKindDetailsCodes(s.kindDetails)) + tx.ReportType(roomobs.TrackTypeData) + tx.ReportSendBytes(uint32(bytes)) + tx.ReportSendPackets(1) + }) + } else { + s.recv.Add(bytes) + s.recvMessages.Inc() + s.totalRecvBytes.Add(bytes) + s.totalRecvMessages.Inc() + + s.reporter.Tx(func(tx roomobs.TrackTx) { + tx.ParticipantSession().ReportKindCode(roomobs.ParticipantKindCode(s.kind)) + tx.ParticipantSession().ReportKindDetailsCodes(roomobs.ParticipantKindDetailsCodes(s.kindDetails)) + tx.ReportType(roomobs.TrackTypeData) + tx.ReportRecvBytes(uint32(bytes)) + tx.ReportRecvPackets(1) + }) + } +} + +func (s *BytesTrackStats) GetTrafficTotals() *TrafficTotals { + return &TrafficTotals{ + At: time.Now(), + SendBytes: s.totalSendBytes.Load(), + SendMessages: s.totalSendMessages.Load(), + RecvBytes: s.totalRecvBytes.Load(), + RecvMessages: s.totalRecvMessages.Load(), + } +} + +func (s *BytesTrackStats) Stop() { + s.done.Break() +} + +func (s *BytesTrackStats) report() { + if recv := s.recv.Swap(0); recv > 0 { + packets := s.recvMessages.Swap(0) + s.telemetryListener.OnTrackStats( + telemetry.StatsKeyForData(s.country, livekit.StreamType_UPSTREAM, s.pID, s.trackID), + &livekit.AnalyticsStat{ + Streams: []*livekit.AnalyticsStream{ + { + PrimaryBytes: recv, + PrimaryPackets: packets, + }, + }, + }, + ) + } + + if send := s.send.Swap(0); send > 0 { + packets := s.sendMessages.Swap(0) + s.telemetryListener.OnTrackStats( + telemetry.StatsKeyForData(s.country, livekit.StreamType_DOWNSTREAM, s.pID, s.trackID), + &livekit.AnalyticsStat{ + Streams: []*livekit.AnalyticsStream{ + { + PrimaryBytes: send, + PrimaryPackets: packets, + }, + }, + }, + ) + } +} + +func (s *BytesTrackStats) worker() { + ticker := time.NewTicker(5 * time.Second) + defer func() { + ticker.Stop() + s.report() + }() + + for { + select { + case <-s.done.Watch(): + return + case <-ticker.C: + s.report() + } + } +} + +// ----------------------------------------------------------------------- + +var _ types.ParticipantTelemetryListener = (*BytesSignalStats)(nil) + +type BytesSignalStats struct { + BytesTrackStats + ctx context.Context + + telemetry telemetry.TelemetryService + guard *telemetry.ReferenceGuard + + participantResolver roomobs.ParticipantReporterResolver + trackResolver roomobs.KeyResolver + + mu sync.Mutex + ri *livekit.Room + pi *livekit.ParticipantInfo + stopped chan struct{} + + types.NullParticipantTelemetryListener +} + +func NewBytesSignalStats( + ctx context.Context, + telemetryService telemetry.TelemetryService, +) *BytesSignalStats { + projectReporter := telemetryService.RoomProjectReporter(ctx) + participantReporter, participantReporterResolver := roomobs.DeferredParticipantReporter(projectReporter) + trackReporter, trackReporterResolver := participantReporter.WithDeferredTrack() + b := &BytesSignalStats{ + ctx: ctx, + telemetry: telemetryService, + guard: &telemetry.ReferenceGuard{}, + participantResolver: participantReporterResolver, + trackResolver: trackReporterResolver, + } + b.BytesTrackStats = BytesTrackStats{ + telemetryListener: b, + reporter: trackReporter, + } + return b +} + +func (s *BytesSignalStats) ResolveRoom(ri *livekit.Room) { + s.mu.Lock() + defer s.mu.Unlock() + if s.ri == nil && ri.GetSid() != "" { + s.ri = &livekit.Room{ + Sid: ri.Sid, + Name: ri.Name, + } + s.maybeStart() + } +} + +func (s *BytesSignalStats) ResolveParticipant(pi *livekit.ParticipantInfo) { + s.mu.Lock() + defer s.mu.Unlock() + if s.pi == nil && pi != nil { + s.pi = &livekit.ParticipantInfo{ + Sid: pi.Sid, + Identity: pi.Identity, + Kind: pi.Kind, + KindDetails: pi.KindDetails, + } + s.kind = pi.Kind + s.kindDetails = pi.KindDetails + s.maybeStart() + } +} + +func (s *BytesSignalStats) Reset() { + s.mu.Lock() + defer s.mu.Unlock() + if s.stopped != nil { + s.done.Break() + <-s.stopped + s.stopped = nil + s.done = core.Fuse{} + s.guard = &telemetry.ReferenceGuard{} + } + s.ri = nil + s.pi = nil + + s.participantResolver.Reset() + s.trackResolver.Reset() +} + +func (s *BytesSignalStats) maybeStart() { + if s.ri == nil || s.pi == nil { + return + } + + s.pID = livekit.ParticipantID(s.pi.Sid) + s.trackID = BytesTrackIDForParticipantID(BytesTrackTypeSignal, s.pID) + + s.participantResolver.Resolve( + livekit.RoomName(s.ri.Name), + livekit.RoomID(s.ri.Sid), + livekit.ParticipantIdentity(s.pi.Identity), + livekit.ParticipantID(s.pi.Sid), + ) + s.trackResolver.Resolve(string(s.trackID)) + + s.telemetry.ParticipantJoined(s.ctx, s.ri, s.pi, nil, nil, false, s.guard) + s.stopped = make(chan struct{}) + go s.worker() +} + +func (s *BytesSignalStats) worker() { + s.BytesTrackStats.worker() + s.telemetry.ParticipantLeft(s.ctx, s.ri, s.pi, false, s.guard) + close(s.stopped) +} + +func (s *BytesSignalStats) OnTrackStats(key telemetry.StatsKey, stat *livekit.AnalyticsStat) { + stat.RoomId, stat.RoomName = s.ri.Sid, s.ri.Name + s.telemetry.TrackStats(livekit.RoomID(s.ri.Sid), livekit.RoomName(s.ri.Name), key, stat) +} + +// ----------------------------------------------------------------------- + +func BytesTrackIDForParticipantID(typ BytesTrackType, participantID livekit.ParticipantID) livekit.TrackID { + return livekit.TrackID(fmt.Sprintf("%s%s%s", utils.TrackPrefix, typ, participantID)) +} diff --git a/pkg/rtc/signalhandler.go b/pkg/rtc/signalhandler.go deleted file mode 100644 index 35ea703..0000000 --- a/pkg/rtc/signalhandler.go +++ /dev/null @@ -1,159 +0,0 @@ -// Copyright 2023 LiveKit, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package rtc - -import ( - "github.com/livekit/protocol/livekit" - "github.com/livekit/protocol/logger" - - "github.com/livekit/livekit-server/pkg/rtc/types" -) - -func HandleParticipantSignal(room types.Room, participant types.LocalParticipant, req *livekit.SignalRequest, pLogger logger.Logger) error { - participant.UpdateLastSeenSignal() - - switch msg := req.GetMessage().(type) { - case *livekit.SignalRequest_Offer: - participant.HandleOffer(FromProtoSessionDescription(msg.Offer)) - - case *livekit.SignalRequest_Answer: - participant.HandleAnswer(FromProtoSessionDescription(msg.Answer)) - - case *livekit.SignalRequest_Trickle: - candidateInit, err := FromProtoTrickle(msg.Trickle) - if err != nil { - pLogger.Warnw("could not decode trickle", err) - return nil - } - participant.AddICECandidate(candidateInit, msg.Trickle.Target) - - case *livekit.SignalRequest_AddTrack: - pLogger.Debugw("add track request", "trackID", msg.AddTrack.Cid) - participant.AddTrack(msg.AddTrack) - - case *livekit.SignalRequest_Mute: - participant.SetTrackMuted(livekit.TrackID(msg.Mute.Sid), msg.Mute.Muted, false) - - case *livekit.SignalRequest_Subscription: - // allow participant to indicate their interest in the subscription - // permission check happens later in SubscriptionManager - room.UpdateSubscriptions( - participant, - livekit.StringsAsIDs[livekit.TrackID](msg.Subscription.TrackSids), - msg.Subscription.ParticipantTracks, - msg.Subscription.Subscribe, - ) - - case *livekit.SignalRequest_TrackSetting: - for _, sid := range livekit.StringsAsIDs[livekit.TrackID](msg.TrackSetting.TrackSids) { - participant.UpdateSubscribedTrackSettings(sid, msg.TrackSetting) - } - - case *livekit.SignalRequest_Leave: - reason := types.ParticipantCloseReasonClientRequestLeave - switch msg.Leave.Reason { - case livekit.DisconnectReason_CLIENT_INITIATED: - reason = types.ParticipantCloseReasonClientRequestLeave - case livekit.DisconnectReason_USER_UNAVAILABLE: - reason = types.ParticipantCloseReasonUserUnavailable - case livekit.DisconnectReason_USER_REJECTED: - reason = types.ParticipantCloseReasonUserRejected - } - pLogger.Debugw("client leaving room", "reason", reason) - room.RemoveParticipant(participant.Identity(), participant.ID(), reason) - - case *livekit.SignalRequest_SubscriptionPermission: - err := room.UpdateSubscriptionPermission(participant, msg.SubscriptionPermission) - if err != nil { - pLogger.Warnw("could not update subscription permission", err, - "permissions", msg.SubscriptionPermission) - } - - case *livekit.SignalRequest_SyncState: - err := room.SyncState(participant, msg.SyncState) - if err != nil { - pLogger.Warnw("could not sync state", err, - "state", msg.SyncState) - } - - case *livekit.SignalRequest_Simulate: - err := room.SimulateScenario(participant, msg.Simulate) - if err != nil { - pLogger.Warnw("could not simulate scenario", err, - "simulate", msg.Simulate) - } - - case *livekit.SignalRequest_PingReq: - if msg.PingReq.Rtt > 0 { - participant.UpdateSignalingRTT(uint32(msg.PingReq.Rtt)) - } - - case *livekit.SignalRequest_UpdateMetadata: - requestResponse := &livekit.RequestResponse{ - RequestId: msg.UpdateMetadata.RequestId, - Reason: livekit.RequestResponse_OK, - } - if participant.ClaimGrants().Video.GetCanUpdateOwnMetadata() { - if err := participant.CheckMetadataLimits( - msg.UpdateMetadata.Name, - msg.UpdateMetadata.Metadata, - msg.UpdateMetadata.Attributes, - ); err == nil { - if msg.UpdateMetadata.Name != "" { - participant.SetName(msg.UpdateMetadata.Name) - } - if msg.UpdateMetadata.Metadata != "" { - participant.SetMetadata(msg.UpdateMetadata.Metadata) - } - if msg.UpdateMetadata.Attributes != nil { - participant.SetAttributes(msg.UpdateMetadata.Attributes) - } - } else { - pLogger.Warnw("could not update metadata", err) - - switch err { - case ErrNameExceedsLimits: - requestResponse.Reason = livekit.RequestResponse_LIMIT_EXCEEDED - requestResponse.Message = "exceeds name length limit" - - case ErrMetadataExceedsLimits: - requestResponse.Reason = livekit.RequestResponse_LIMIT_EXCEEDED - requestResponse.Message = "exceeds metadata size limit" - - case ErrAttributesExceedsLimits: - requestResponse.Reason = livekit.RequestResponse_LIMIT_EXCEEDED - requestResponse.Message = "exceeds attributes size limit" - } - - } - } else { - requestResponse.Reason = livekit.RequestResponse_NOT_ALLOWED - requestResponse.Message = "does not have permission to update own metadata" - } - participant.SendRequestResponse(requestResponse) - - case *livekit.SignalRequest_UpdateAudioTrack: - if err := participant.UpdateAudioTrack(msg.UpdateAudioTrack); err != nil { - pLogger.Warnw("could not update audio track", err, "update", msg.UpdateAudioTrack) - } - - case *livekit.SignalRequest_UpdateVideoTrack: - if err := participant.UpdateVideoTrack(msg.UpdateVideoTrack); err != nil { - pLogger.Warnw("could not update video track", err, "update", msg.UpdateVideoTrack) - } - } - - return nil -} diff --git a/pkg/rtc/signalling/errors.go b/pkg/rtc/signalling/errors.go new file mode 100644 index 0000000..c78a4c3 --- /dev/null +++ b/pkg/rtc/signalling/errors.go @@ -0,0 +1,27 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package signalling + +import ( + "errors" +) + +var ( + ErrInvalidMessageType = errors.New("invalid message type") + ErrNameExceedsLimits = errors.New("name length exceeds limits") + ErrMetadataExceedsLimits = errors.New("metadata size exceeds limits") + ErrAttributesExceedsLimits = errors.New("attributes size exceeds limits") + ErrUpdateOwnMetadataNotAllowed = errors.New("update own metadata not allowed") +) diff --git a/pkg/rtc/signalling/interfaces.go b/pkg/rtc/signalling/interfaces.go new file mode 100644 index 0000000..2470e9d --- /dev/null +++ b/pkg/rtc/signalling/interfaces.go @@ -0,0 +1,65 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package signalling + +import ( + "github.com/livekit/protocol/livekit" + + "github.com/livekit/livekit-server/pkg/routing" + "github.com/livekit/livekit-server/pkg/rtc/types" + + "google.golang.org/protobuf/proto" +) + +type ParticipantSignalHandler interface { + HandleMessage(msg proto.Message) error +} + +type ParticipantSignaller interface { + SwapResponseSink(sink routing.MessageSink, reason types.SignallingCloseReason) + GetResponseSink() routing.MessageSink + CloseSignalConnection(reason types.SignallingCloseReason) + + WriteMessage(msg proto.Message) error +} + +type ParticipantSignalling interface { + SignalJoinResponse(join *livekit.JoinResponse) proto.Message + SignalParticipantUpdate(participants []*livekit.ParticipantInfo) proto.Message + SignalSpeakerUpdate(speakers []*livekit.SpeakerInfo) proto.Message + SignalRoomUpdate(room *livekit.Room) proto.Message + SignalConnectionQualityUpdate(connectionQuality *livekit.ConnectionQualityUpdate) proto.Message + SignalRefreshToken(token string) proto.Message + SignalRequestResponse(requestResponse *livekit.RequestResponse) proto.Message + SignalRoomMovedResponse(roomMoved *livekit.RoomMovedResponse) proto.Message + SignalReconnectResponse(reconnect *livekit.ReconnectResponse) proto.Message + SignalICECandidate(trickle *livekit.TrickleRequest) proto.Message + SignalTrackMuted(mute *livekit.MuteTrackRequest) proto.Message + SignalTrackPublished(trackPublished *livekit.TrackPublishedResponse) proto.Message + SignalTrackUnpublished(trackUnpublished *livekit.TrackUnpublishedResponse) proto.Message + SignalTrackSubscribed(trackSubscribed *livekit.TrackSubscribed) proto.Message + SignalLeaveRequest(leave *livekit.LeaveRequest) proto.Message + SignalSdpAnswer(answer *livekit.SessionDescription) proto.Message + SignalSdpOffer(offer *livekit.SessionDescription) proto.Message + SignalStreamStateUpdate(streamStateUpdate *livekit.StreamStateUpdate) proto.Message + SignalSubscribedQualityUpdate(subscribedQualityUpdate *livekit.SubscribedQualityUpdate) proto.Message + SignalSubscriptionResponse(subscriptionResponse *livekit.SubscriptionResponse) proto.Message + SignalSubscriptionPermissionUpdate(subscriptionPermissionUpdate *livekit.SubscriptionPermissionUpdate) proto.Message + SignalMediaSectionsRequirement(mediaSectionsRequirement *livekit.MediaSectionsRequirement) proto.Message + SignalSubscribedAudioCodecUpdate(subscribedAudioCodecUpdate *livekit.SubscribedAudioCodecUpdate) proto.Message + SignalPublishDataTrackResponse(publishDataTrackResponse *livekit.PublishDataTrackResponse) proto.Message + SignalUnpublishDataTrackResponse(unpublishDataTrackResponse *livekit.UnpublishDataTrackResponse) proto.Message + SignalDataTrackSubscriberHandles(dataTrackSubscriberHandles *livekit.DataTrackSubscriberHandles) proto.Message +} diff --git a/pkg/rtc/signalling/signalhandler.go b/pkg/rtc/signalling/signalhandler.go new file mode 100644 index 0000000..7e69aee --- /dev/null +++ b/pkg/rtc/signalling/signalhandler.go @@ -0,0 +1,157 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package signalling + +import ( + "fmt" + + "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/logger" + "google.golang.org/protobuf/proto" + + "github.com/livekit/livekit-server/pkg/rtc/types" +) + +var _ ParticipantSignalHandler = (*signalhandler)(nil) + +type SignalHandlerParams struct { + Logger logger.Logger + Participant types.LocalParticipant +} + +type signalhandler struct { + params SignalHandlerParams +} + +func NewSignalHandler(params SignalHandlerParams) ParticipantSignalHandler { + return &signalhandler{ + params: params, + } +} + +func (s *signalhandler) HandleMessage(msg proto.Message) error { + req, ok := msg.(*livekit.SignalRequest) + if !ok { + s.params.Logger.Warnw( + "unknown message type", nil, + "messageType", fmt.Sprintf("%T", msg), + ) + return ErrInvalidMessageType + } + s.params.Participant.UpdateLastSeenSignal() + + s.params.Logger.Debugw("handling signal request", "request", logger.Proto(req)) + switch msg := req.GetMessage().(type) { + case *livekit.SignalRequest_Offer: + s.params.Participant.HandleOffer(msg.Offer) + + case *livekit.SignalRequest_Answer: + s.params.Participant.HandleAnswer(msg.Answer) + + case *livekit.SignalRequest_Trickle: + s.params.Participant.HandleICETrickle(msg.Trickle) + + case *livekit.SignalRequest_AddTrack: + s.params.Participant.AddTrack(msg.AddTrack) + + case *livekit.SignalRequest_Mute: + s.params.Participant.SetTrackMuted(msg.Mute, false) + + case *livekit.SignalRequest_Subscription: + // allow participant to indicate their interest in the subscription + // permission check happens later in SubscriptionManager + s.params.Participant.HandleUpdateSubscriptions( + livekit.StringsAsIDs[livekit.TrackID](msg.Subscription.TrackSids), + msg.Subscription.ParticipantTracks, + msg.Subscription.Subscribe, + ) + + case *livekit.SignalRequest_TrackSetting: + for _, sid := range livekit.StringsAsIDs[livekit.TrackID](msg.TrackSetting.TrackSids) { + s.params.Participant.UpdateSubscribedTrackSettings(sid, msg.TrackSetting) + } + + case *livekit.SignalRequest_Leave: + reason := types.ParticipantCloseReasonClientRequestLeave + switch msg.Leave.Reason { + case livekit.DisconnectReason_CLIENT_INITIATED: + reason = types.ParticipantCloseReasonClientRequestLeave + case livekit.DisconnectReason_USER_UNAVAILABLE: + reason = types.ParticipantCloseReasonUserUnavailable + case livekit.DisconnectReason_USER_REJECTED: + reason = types.ParticipantCloseReasonUserRejected + case livekit.DisconnectReason_AGENT_ERROR: + reason = types.ParticipantCloseReasonAgentError + } + s.params.Logger.Debugw("client leaving room", "reason", reason) + s.params.Participant.HandleLeaveRequest(reason) + + case *livekit.SignalRequest_SubscriptionPermission: + err := s.params.Participant.HandleUpdateSubscriptionPermission(msg.SubscriptionPermission) + if err != nil { + s.params.Logger.Warnw( + "could not update subscription permission", err, + "permissions", msg.SubscriptionPermission, + ) + } + + case *livekit.SignalRequest_SyncState: + err := s.params.Participant.HandleSyncState(msg.SyncState) + if err != nil { + s.params.Logger.Warnw( + "could not sync state", err, + "state", msg.SyncState, + ) + } + + case *livekit.SignalRequest_Simulate: + err := s.params.Participant.HandleSimulateScenario(msg.Simulate) + if err != nil { + s.params.Logger.Warnw( + "could not simulate scenario", err, + "simulate", msg.Simulate, + ) + } + + case *livekit.SignalRequest_PingReq: + if msg.PingReq.Rtt > 0 { + s.params.Participant.UpdateSignalingRTT(uint32(msg.PingReq.Rtt)) + } + + case *livekit.SignalRequest_UpdateMetadata: + s.params.Participant.UpdateMetadata(msg.UpdateMetadata, false) + + case *livekit.SignalRequest_UpdateAudioTrack: + if err := s.params.Participant.UpdateAudioTrack(msg.UpdateAudioTrack); err != nil { + s.params.Logger.Warnw("could not update audio track", err, "update", msg.UpdateAudioTrack) + } + + case *livekit.SignalRequest_UpdateVideoTrack: + if err := s.params.Participant.UpdateVideoTrack(msg.UpdateVideoTrack); err != nil { + s.params.Logger.Warnw("could not update video track", err, "update", msg.UpdateVideoTrack) + } + + case *livekit.SignalRequest_PublishDataTrackRequest: + s.params.Participant.HandlePublishDataTrackRequest(msg.PublishDataTrackRequest) + + case *livekit.SignalRequest_UnpublishDataTrackRequest: + s.params.Participant.HandleUnpublishDataTrackRequest(msg.UnpublishDataTrackRequest) + + case *livekit.SignalRequest_UpdateDataSubscription: + s.params.Participant.HandleUpdateDataSubscription(msg.UpdateDataSubscription) + } + + return nil +} diff --git a/pkg/rtc/signalling/signalhandlerunimplemented.go b/pkg/rtc/signalling/signalhandlerunimplemented.go new file mode 100644 index 0000000..9df37bd --- /dev/null +++ b/pkg/rtc/signalling/signalhandlerunimplemented.go @@ -0,0 +1,27 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package signalling + +import ( + "google.golang.org/protobuf/proto" +) + +var _ ParticipantSignalHandler = (*signalhandlerUnimplemented)(nil) + +type signalhandlerUnimplemented struct{} + +func (u *signalhandlerUnimplemented) HandleMessage(msg proto.Message) error { + return nil +} diff --git a/pkg/rtc/signalling/signallerasync.go b/pkg/rtc/signalling/signallerasync.go new file mode 100644 index 0000000..0dc0853 --- /dev/null +++ b/pkg/rtc/signalling/signallerasync.go @@ -0,0 +1,114 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package signalling + +import ( + "fmt" + + "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/logger" + "github.com/livekit/protocol/utils" + "github.com/livekit/psrpc" + + "github.com/livekit/livekit-server/pkg/routing" + "github.com/livekit/livekit-server/pkg/rtc/types" + + "google.golang.org/protobuf/proto" +) + +var _ ParticipantSignaller = (*signallerAsync)(nil) + +type SignallerAsyncParams struct { + Logger logger.Logger + Participant types.LocalParticipant +} + +type signallerAsync struct { + params SignallerAsyncParams + + *signallerAsyncBase +} + +func NewSignallerAsync(params SignallerAsyncParams) ParticipantSignaller { + return &signallerAsync{ + params: params, + signallerAsyncBase: newSignallerAsyncBase(signallerAsyncBaseParams{Logger: params.Logger}), + } +} + +func (s *signallerAsync) WriteMessage(msg proto.Message) error { + if msg == nil { + return nil + } + + getMessageType := func(msg proto.Message) string { + messageType := "unknown" + if typed, ok := msg.(*livekit.SignalResponse); ok { + messageType = fmt.Sprintf("%T", typed.Message) + } + return messageType + } + + if s.params.Participant.IsDisconnected() { + s.params.Logger.Debugw( + "could not send message to participant, participant disconnected", + "messageType", getMessageType(msg), + ) + return nil + } + + if !s.params.Participant.IsReady() { + if typed, ok := msg.(*livekit.SignalResponse); !ok { + s.params.Logger.Warnw( + "unknown message type", nil, + "messageType", fmt.Sprintf("%T", msg), + ) + } else { + if typed.GetJoin() == nil { + return nil + } + } + } + + sink := s.GetResponseSink() + if sink == nil { + s.params.Logger.Debugw( + "could not send message to participant, no sink", + "messageType", getMessageType(msg), + ) + return nil + } + + err := sink.WriteMessage(msg) + if err != nil { + if utils.ErrorIsOneOf(err, psrpc.Canceled, routing.ErrChannelClosed) { + s.params.Logger.Debugw( + "could not send message to participant", + "error", err, + "messageType", getMessageType(msg), + ) + return nil + } else { + s.params.Logger.Warnw( + "could not send message to participant", err, + "messageType", getMessageType(msg), + ) + return err + } + } else { + s.params.Logger.Debugw("sent signal response", "response", logger.Proto(msg)) + } + return nil +} diff --git a/pkg/rtc/signalling/signallerasyncbase.go b/pkg/rtc/signalling/signallerasyncbase.go new file mode 100644 index 0000000..0b4d2b6 --- /dev/null +++ b/pkg/rtc/signalling/signallerasyncbase.go @@ -0,0 +1,79 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package signalling + +import ( + "sync" + + "github.com/livekit/protocol/logger" + + "github.com/livekit/livekit-server/pkg/routing" + "github.com/livekit/livekit-server/pkg/rtc/types" +) + +type signallerAsyncBaseParams struct { + Logger logger.Logger +} + +type signallerAsyncBase struct { + signallerUnimplemented + + params signallerAsyncBaseParams + + resSinkMu sync.Mutex + resSink routing.MessageSink +} + +func newSignallerAsyncBase(params signallerAsyncBaseParams) *signallerAsyncBase { + return &signallerAsyncBase{ + params: params, + } +} + +func (s *signallerAsyncBase) SwapResponseSink(sink routing.MessageSink, reason types.SignallingCloseReason) { + s.resSinkMu.Lock() + oldSink := s.resSink + s.resSink = sink + s.resSinkMu.Unlock() + + if oldSink != nil { + if sink != nil { + s.params.Logger.Debugw( + "swapping signal connection", + "reason", reason, + "connID", oldSink.ConnectionID(), + "newConnID", sink.ConnectionID(), + ) + } else { + s.params.Logger.Debugw( + "closing signal connection", + "reason", reason, + "connID", oldSink.ConnectionID(), + ) + } + oldSink.Close() + } +} + +func (s *signallerAsyncBase) GetResponseSink() routing.MessageSink { + s.resSinkMu.Lock() + defer s.resSinkMu.Unlock() + return s.resSink +} + +// closes signal connection to notify client to resume/reconnect +func (s *signallerAsyncBase) CloseSignalConnection(reason types.SignallingCloseReason) { + s.SwapResponseSink(nil, reason) +} diff --git a/pkg/rtc/signalling/signallerunimplemented.go b/pkg/rtc/signalling/signallerunimplemented.go new file mode 100644 index 0000000..3592fd8 --- /dev/null +++ b/pkg/rtc/signalling/signallerunimplemented.go @@ -0,0 +1,39 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package signalling + +import ( + "github.com/livekit/livekit-server/pkg/routing" + "github.com/livekit/livekit-server/pkg/rtc/types" + + "google.golang.org/protobuf/proto" +) + +var _ ParticipantSignaller = (*signallerUnimplemented)(nil) + +type signallerUnimplemented struct{} + +func (u *signallerUnimplemented) SwapResponseSink(sink routing.MessageSink, reason types.SignallingCloseReason) { +} + +func (u *signallerUnimplemented) GetResponseSink() routing.MessageSink { + return nil +} + +func (u *signallerUnimplemented) CloseSignalConnection(reason types.SignallingCloseReason) {} + +func (u *signallerUnimplemented) WriteMessage(msg proto.Message) error { + return nil +} diff --git a/pkg/rtc/signalling/signalling.go b/pkg/rtc/signalling/signalling.go new file mode 100644 index 0000000..fffc2ba --- /dev/null +++ b/pkg/rtc/signalling/signalling.go @@ -0,0 +1,260 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package signalling + +import ( + "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/logger" + + "google.golang.org/protobuf/proto" +) + +var _ ParticipantSignalling = (*signalling)(nil) + +type SignallingParams struct { + Logger logger.Logger +} + +type signalling struct { + params SignallingParams +} + +func NewSignalling(params SignallingParams) ParticipantSignalling { + return &signalling{ + params: params, + } +} + +func (s *signalling) SignalJoinResponse(join *livekit.JoinResponse) proto.Message { + return &livekit.SignalResponse{ + Message: &livekit.SignalResponse_Join{ + Join: join, + }, + } +} + +func (s *signalling) SignalParticipantUpdate(participants []*livekit.ParticipantInfo) proto.Message { + if len(participants) == 0 { + return nil + } + + return &livekit.SignalResponse{ + Message: &livekit.SignalResponse_Update{ + Update: &livekit.ParticipantUpdate{ + Participants: participants, + }, + }, + } +} + +func (s *signalling) SignalSpeakerUpdate(speakers []*livekit.SpeakerInfo) proto.Message { + if len(speakers) == 0 { + return nil + } + + return &livekit.SignalResponse{ + Message: &livekit.SignalResponse_SpeakersChanged{ + SpeakersChanged: &livekit.SpeakersChanged{ + Speakers: speakers, + }, + }, + } +} + +func (s *signalling) SignalRoomUpdate(room *livekit.Room) proto.Message { + return &livekit.SignalResponse{ + Message: &livekit.SignalResponse_RoomUpdate{ + RoomUpdate: &livekit.RoomUpdate{ + Room: room, + }, + }, + } +} + +func (s *signalling) SignalConnectionQualityUpdate(connectionQuality *livekit.ConnectionQualityUpdate) proto.Message { + return &livekit.SignalResponse{ + Message: &livekit.SignalResponse_ConnectionQuality{ + ConnectionQuality: connectionQuality, + }, + } +} + +func (s *signalling) SignalRefreshToken(token string) proto.Message { + return &livekit.SignalResponse{ + Message: &livekit.SignalResponse_RefreshToken{ + RefreshToken: token, + }, + } +} + +func (s *signalling) SignalRequestResponse(requestResponse *livekit.RequestResponse) proto.Message { + return &livekit.SignalResponse{ + Message: &livekit.SignalResponse_RequestResponse{ + RequestResponse: requestResponse, + }, + } +} + +func (s *signalling) SignalRoomMovedResponse(roomMoved *livekit.RoomMovedResponse) proto.Message { + return &livekit.SignalResponse{ + Message: &livekit.SignalResponse_RoomMoved{ + RoomMoved: roomMoved, + }, + } +} + +func (s *signalling) SignalReconnectResponse(reconnect *livekit.ReconnectResponse) proto.Message { + return &livekit.SignalResponse{ + Message: &livekit.SignalResponse_Reconnect{ + Reconnect: reconnect, + }, + } +} + +func (s *signalling) SignalICECandidate(trickle *livekit.TrickleRequest) proto.Message { + return &livekit.SignalResponse{ + Message: &livekit.SignalResponse_Trickle{ + Trickle: trickle, + }, + } +} + +func (s *signalling) SignalTrackMuted(mute *livekit.MuteTrackRequest) proto.Message { + return &livekit.SignalResponse{ + Message: &livekit.SignalResponse_Mute{ + Mute: mute, + }, + } +} + +func (s *signalling) SignalTrackPublished(trackPublished *livekit.TrackPublishedResponse) proto.Message { + return &livekit.SignalResponse{ + Message: &livekit.SignalResponse_TrackPublished{ + TrackPublished: trackPublished, + }, + } +} + +func (s *signalling) SignalTrackUnpublished(trackUnpublished *livekit.TrackUnpublishedResponse) proto.Message { + return &livekit.SignalResponse{ + Message: &livekit.SignalResponse_TrackUnpublished{ + TrackUnpublished: trackUnpublished, + }, + } +} + +func (s *signalling) SignalTrackSubscribed(trackSubscribed *livekit.TrackSubscribed) proto.Message { + return &livekit.SignalResponse{ + Message: &livekit.SignalResponse_TrackSubscribed{ + TrackSubscribed: trackSubscribed, + }, + } +} + +func (s *signalling) SignalLeaveRequest(leave *livekit.LeaveRequest) proto.Message { + return &livekit.SignalResponse{ + Message: &livekit.SignalResponse_Leave{ + Leave: leave, + }, + } +} + +func (s *signalling) SignalSdpAnswer(answer *livekit.SessionDescription) proto.Message { + return &livekit.SignalResponse{ + Message: &livekit.SignalResponse_Answer{ + Answer: answer, + }, + } +} + +func (s *signalling) SignalSdpOffer(offer *livekit.SessionDescription) proto.Message { + return &livekit.SignalResponse{ + Message: &livekit.SignalResponse_Offer{ + Offer: offer, + }, + } +} + +func (s *signalling) SignalStreamStateUpdate(streamStateUpdate *livekit.StreamStateUpdate) proto.Message { + return &livekit.SignalResponse{ + Message: &livekit.SignalResponse_StreamStateUpdate{ + StreamStateUpdate: streamStateUpdate, + }, + } +} + +func (s *signalling) SignalSubscribedQualityUpdate(subscribedQualityUpdate *livekit.SubscribedQualityUpdate) proto.Message { + return &livekit.SignalResponse{ + Message: &livekit.SignalResponse_SubscribedQualityUpdate{ + SubscribedQualityUpdate: subscribedQualityUpdate, + }, + } +} + +func (s *signalling) SignalSubscriptionResponse(subscriptionResponse *livekit.SubscriptionResponse) proto.Message { + return &livekit.SignalResponse{ + Message: &livekit.SignalResponse_SubscriptionResponse{ + SubscriptionResponse: subscriptionResponse, + }, + } +} + +func (s *signalling) SignalSubscriptionPermissionUpdate(subscriptionPermissionUpdate *livekit.SubscriptionPermissionUpdate) proto.Message { + return &livekit.SignalResponse{ + Message: &livekit.SignalResponse_SubscriptionPermissionUpdate{ + SubscriptionPermissionUpdate: subscriptionPermissionUpdate, + }, + } +} + +func (s *signalling) SignalMediaSectionsRequirement(mediaSectionsRequirement *livekit.MediaSectionsRequirement) proto.Message { + return &livekit.SignalResponse{ + Message: &livekit.SignalResponse_MediaSectionsRequirement{ + MediaSectionsRequirement: mediaSectionsRequirement, + }, + } +} + +func (s *signalling) SignalSubscribedAudioCodecUpdate(subscribedAudioCodecUpdate *livekit.SubscribedAudioCodecUpdate) proto.Message { + return &livekit.SignalResponse{ + Message: &livekit.SignalResponse_SubscribedAudioCodecUpdate{ + SubscribedAudioCodecUpdate: subscribedAudioCodecUpdate, + }, + } +} + +func (s *signalling) SignalPublishDataTrackResponse(publishDataTrackResponse *livekit.PublishDataTrackResponse) proto.Message { + return &livekit.SignalResponse{ + Message: &livekit.SignalResponse_PublishDataTrackResponse{ + PublishDataTrackResponse: publishDataTrackResponse, + }, + } +} + +func (s *signalling) SignalUnpublishDataTrackResponse(unpublishDataTrackResponse *livekit.UnpublishDataTrackResponse) proto.Message { + return &livekit.SignalResponse{ + Message: &livekit.SignalResponse_UnpublishDataTrackResponse{ + UnpublishDataTrackResponse: unpublishDataTrackResponse, + }, + } +} + +func (s *signalling) SignalDataTrackSubscriberHandles(dataTrackSubscriberHandles *livekit.DataTrackSubscriberHandles) proto.Message { + return &livekit.SignalResponse{ + Message: &livekit.SignalResponse_DataTrackSubscriberHandles{ + DataTrackSubscriberHandles: dataTrackSubscriberHandles, + }, + } +} diff --git a/pkg/rtc/signalling/signallingunimplemented.go b/pkg/rtc/signalling/signallingunimplemented.go new file mode 100644 index 0000000..dca4877 --- /dev/null +++ b/pkg/rtc/signalling/signallingunimplemented.go @@ -0,0 +1,129 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package signalling + +import ( + "github.com/livekit/protocol/livekit" + + "google.golang.org/protobuf/proto" +) + +var _ ParticipantSignalling = (*signallingUnimplemented)(nil) + +type signallingUnimplemented struct{} + +func (u *signallingUnimplemented) SignalJoinResponse(join *livekit.JoinResponse) proto.Message { + return nil +} + +func (u *signallingUnimplemented) SignalParticipantUpdate(participants []*livekit.ParticipantInfo) proto.Message { + return nil +} + +func (u *signallingUnimplemented) SignalSpeakerUpdate(speakers []*livekit.SpeakerInfo) proto.Message { + return nil +} + +func (u *signallingUnimplemented) SignalRoomUpdate(room *livekit.Room) proto.Message { + return nil +} + +func (u *signallingUnimplemented) SignalConnectionQualityUpdate(connectionQuality *livekit.ConnectionQualityUpdate) proto.Message { + return nil +} + +func (u *signallingUnimplemented) SignalRefreshToken(token string) proto.Message { + return nil +} + +func (u *signallingUnimplemented) SignalRequestResponse(requestResponse *livekit.RequestResponse) proto.Message { + return nil +} + +func (u *signallingUnimplemented) SignalRoomMovedResponse(roomMoved *livekit.RoomMovedResponse) proto.Message { + return nil +} + +func (u *signallingUnimplemented) SignalReconnectResponse(reconnect *livekit.ReconnectResponse) proto.Message { + return nil +} + +func (u *signallingUnimplemented) SignalICECandidate(trickle *livekit.TrickleRequest) proto.Message { + return nil +} + +func (u *signallingUnimplemented) SignalTrackMuted(mute *livekit.MuteTrackRequest) proto.Message { + return nil +} + +func (u *signallingUnimplemented) SignalTrackPublished(trackPublished *livekit.TrackPublishedResponse) proto.Message { + return nil +} + +func (u *signallingUnimplemented) SignalTrackUnpublished(trackUnpublished *livekit.TrackUnpublishedResponse) proto.Message { + return nil +} + +func (u *signallingUnimplemented) SignalTrackSubscribed(trackSubscribed *livekit.TrackSubscribed) proto.Message { + return nil +} + +func (u *signallingUnimplemented) SignalLeaveRequest(leave *livekit.LeaveRequest) proto.Message { + return nil +} + +func (u *signallingUnimplemented) SignalSdpAnswer(answer *livekit.SessionDescription) proto.Message { + return nil +} + +func (u *signallingUnimplemented) SignalSdpOffer(offer *livekit.SessionDescription) proto.Message { + return nil +} + +func (u *signallingUnimplemented) SignalStreamStateUpdate(streamStateUpdate *livekit.StreamStateUpdate) proto.Message { + return nil +} + +func (u *signallingUnimplemented) SignalSubscribedQualityUpdate(subscribedQualityUpdate *livekit.SubscribedQualityUpdate) proto.Message { + return nil +} + +func (u *signallingUnimplemented) SignalSubscriptionResponse(subscriptionResponse *livekit.SubscriptionResponse) proto.Message { + return nil +} + +func (u *signallingUnimplemented) SignalSubscriptionPermissionUpdate(subscriptionPermissionUpdate *livekit.SubscriptionPermissionUpdate) proto.Message { + return nil +} + +func (u *signallingUnimplemented) SignalMediaSectionsRequirement(mediaSectionsRequirement *livekit.MediaSectionsRequirement) proto.Message { + return nil +} + +func (u *signallingUnimplemented) SignalSubscribedAudioCodecUpdate(subscribedAudioCodecUpdate *livekit.SubscribedAudioCodecUpdate) proto.Message { + return nil +} + +func (u *signallingUnimplemented) SignalPublishDataTrackResponse(publishDataTrackResponse *livekit.PublishDataTrackResponse) proto.Message { + return nil +} + +func (u *signallingUnimplemented) SignalUnpublishDataTrackResponse(unpublishDataTrackResponse *livekit.UnpublishDataTrackResponse) proto.Message { + return nil +} + +func (u *signallingUnimplemented) SignalDataTrackSubscriberHandles(dataTrackSubscriberHandles *livekit.DataTrackSubscriberHandles) proto.Message { + return nil +} diff --git a/pkg/rtc/subscribedtrack.go b/pkg/rtc/subscribedtrack.go index a45f4e1..3a8a061 100644 --- a/pkg/rtc/subscribedtrack.go +++ b/pkg/rtc/subscribedtrack.go @@ -23,9 +23,12 @@ import ( "go.uber.org/atomic" "google.golang.org/protobuf/proto" + "github.com/livekit/livekit-server/pkg/telemetry" sutils "github.com/livekit/livekit-server/pkg/utils" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" + "github.com/livekit/protocol/observability/roomobs" "github.com/livekit/protocol/utils" "github.com/livekit/livekit-server/pkg/rtc/types" @@ -37,19 +40,27 @@ const ( subscriptionDebounceInterval = 100 * time.Millisecond ) +var _ types.SubscribedTrack = (*SubscribedTrack)(nil) + type SubscribedTrackParams struct { - PublisherID livekit.ParticipantID - PublisherIdentity livekit.ParticipantIdentity - PublisherVersion uint32 - Subscriber types.LocalParticipant - MediaTrack types.MediaTrack - DownTrack *sfu.DownTrack - AdaptiveStream bool + ReceiverConfig ReceiverConfig + SubscriberConfig DirectionConfig + Subscriber types.LocalParticipant + MediaTrack types.MediaTrack + AdaptiveStream bool + TelemetryListener types.ParticipantTelemetryListener + WrappedReceiver *WrappedReceiver + IsRelayed bool + OnDownTrackCreated func(downTrack *sfu.DownTrack) + OnDownTrackClosed func(subscriberID livekit.ParticipantID) + OnSubscriberMaxQualityChange func(subscriberID livekit.ParticipantID, mime mime.MimeType, layer int32) + OnSubscriberAudioCodecChange func(subscriberID livekit.ParticipantID, mime mime.MimeType, enabled bool) } type SubscribedTrack struct { params SubscribedTrackParams logger logger.Logger + downTrack *sfu.DownTrack sender atomic.Pointer[webrtc.RTPSender] needsNegotiation atomic.Bool @@ -65,21 +76,98 @@ type SubscribedTrack struct { onClose atomic.Value // func(bool) debouncer func(func()) + + statsKey telemetry.StatsKey + reporter roomobs.TrackReporter } -func NewSubscribedTrack(params SubscribedTrackParams) *SubscribedTrack { +func NewSubscribedTrack(params SubscribedTrackParams) (*SubscribedTrack, error) { s := &SubscribedTrack{ params: params, logger: params.Subscriber.GetLogger().WithComponent(sutils.ComponentSub).WithValues( - "trackID", params.DownTrack.ID(), - "publisherID", params.PublisherID, - "publisher", params.PublisherIdentity, + "trackID", params.MediaTrack.ID(), + "publisherID", params.MediaTrack.PublisherID(), + "publisher", params.MediaTrack.PublisherIdentity(), ), versionGenerator: utils.NewDefaultTimedVersionGenerator(), debouncer: debounce.New(subscriptionDebounceInterval), + statsKey: telemetry.StatsKeyForTrack( + params.Subscriber.GetCountry(), + livekit.StreamType_DOWNSTREAM, + params.Subscriber.ID(), + params.MediaTrack.ID(), + params.MediaTrack.Source(), + params.MediaTrack.Kind(), + ), + reporter: params.Subscriber.GetReporter().WithTrack(params.MediaTrack.ID().String()), } - return s + var rtcpFeedback []webrtc.RTCPFeedback + var maxTrack int + switch params.MediaTrack.Kind() { + case livekit.TrackType_AUDIO: + rtcpFeedback = params.SubscriberConfig.RTCPFeedback.Audio + maxTrack = params.ReceiverConfig.PacketBufferSizeAudio + case livekit.TrackType_VIDEO: + rtcpFeedback = params.SubscriberConfig.RTCPFeedback.Video + maxTrack = params.ReceiverConfig.PacketBufferSizeVideo + default: + s.logger.Warnw("unexpected track type", nil, "kind", params.MediaTrack.Kind()) + } + codecs := params.WrappedReceiver.Codecs() + for _, c := range codecs { + c.RTCPFeedback = rtcpFeedback + } + + streamID := params.WrappedReceiver.StreamID() + if params.Subscriber.SupportsSyncStreamID() && params.MediaTrack.Stream() != "" { + streamID = PackSyncStreamID(params.MediaTrack.PublisherID(), params.MediaTrack.Stream()) + } + + isEncrypted := params.MediaTrack.IsEncrypted() + var trailer []byte + if isEncrypted { + trailer = params.Subscriber.GetTrailer() + } + subClientInfo := ClientInfo{ClientInfo: params.Subscriber.GetClientInfo()} + subSupportsPacketTrailer := subClientInfo.SupportsPacketTrailer() + // Strip packet trailer if track has packet trailer but subscriber does not have cap + stripPacketTrailer := params.MediaTrack.HasPacketTrailer() && !subSupportsPacketTrailer + downTrack, err := sfu.NewDownTrack(sfu.DownTrackParams{ + Codecs: codecs, + IsEncrypted: isEncrypted, + Source: params.MediaTrack.Source(), + Receiver: params.WrappedReceiver, + BufferFactory: params.Subscriber.GetBufferFactory(), + SubID: params.Subscriber.ID(), + StreamID: streamID, + MaxTrack: maxTrack, + PlayoutDelayLimit: params.Subscriber.GetPlayoutDelayConfig(), + Pacer: params.Subscriber.GetPacer(), + Trailer: trailer, + StripPacketTrailer: stripPacketTrailer, + Logger: LoggerWithTrack( + params.Subscriber.GetLogger().WithComponent(sutils.ComponentSub), + params.MediaTrack.ID(), + params.IsRelayed, + ), + RTCPWriter: params.Subscriber.WriteSubscriberRTCP, + DisableSenderReportPassThrough: params.Subscriber.GetDisableSenderReportPassThrough(), + SupportsCodecChange: params.Subscriber.SupportsCodecChange(), + Listener: s, + }) + if err != nil { + return nil, err + } + + if params.OnDownTrackCreated != nil { + params.OnDownTrackCreated(downTrack) + } + + downTrack.AddReceiverReportListener(params.Subscriber.HandleReceiverReport) + + s.downTrack = downTrack + return s, nil } func (t *SubscribedTrack) AddOnBind(f func(error)) { @@ -121,6 +209,7 @@ func (t *SubscribedTrack) Bound(err error) { if t.params.AdaptiveStream { // remove `disabled` flag to force a visibility update t.settings.Disabled = false + t.logger.Debugw("enabling subscriber track settings on bind", "settings", logger.Proto(t.settings)) } } else { if t.params.AdaptiveStream { @@ -128,6 +217,7 @@ func (t *SubscribedTrack) Bound(err error) { } else { t.settings = &livekit.UpdateTrackSettings{Quality: livekit.VideoQuality_HIGH} } + t.logger.Debugw("initializing subscriber track settings on bind", "settings", logger.Proto(t.settings)) } t.settingsLock.Unlock() t.applySettings() @@ -141,7 +231,7 @@ func (t *SubscribedTrack) Bound(err error) { // for DownTrack callback to notify us that it's closed func (t *SubscribedTrack) Close(isExpectedToResume bool) { if onClose := t.onClose.Load(); onClose != nil { - go onClose.(func(bool))(isExpectedToResume) + onClose.(func(bool))(isExpectedToResume) } } @@ -157,19 +247,19 @@ func (t *SubscribedTrack) IsBound() bool { } func (t *SubscribedTrack) ID() livekit.TrackID { - return livekit.TrackID(t.params.DownTrack.ID()) + return livekit.TrackID(t.downTrack.ID()) } func (t *SubscribedTrack) PublisherID() livekit.ParticipantID { - return t.params.PublisherID + return t.params.MediaTrack.PublisherID() } func (t *SubscribedTrack) PublisherIdentity() livekit.ParticipantIdentity { - return t.params.PublisherIdentity + return t.params.MediaTrack.PublisherIdentity() } func (t *SubscribedTrack) PublisherVersion() uint32 { - return t.params.PublisherVersion + return t.params.MediaTrack.PublisherVersion() } func (t *SubscribedTrack) SubscriberID() livekit.ParticipantID { @@ -185,7 +275,7 @@ func (t *SubscribedTrack) Subscriber() types.LocalParticipant { } func (t *SubscribedTrack) DownTrack() *sfu.DownTrack { - return t.params.DownTrack + return t.downTrack } func (t *SubscribedTrack) MediaTrack() types.MediaTrack { @@ -209,18 +299,20 @@ func (t *SubscribedTrack) isMutedLocked() bool { } func (t *SubscribedTrack) SetPublisherMuted(muted bool) { - t.DownTrack().PubMute(muted) + t.downTrack.PubMute(muted) } func (t *SubscribedTrack) UpdateSubscriberSettings(settings *livekit.UpdateTrackSettings, isImmediate bool) { t.settingsLock.Lock() if proto.Equal(t.settings, settings) { + t.logger.Debugw("skipping subscriber track settings", "settings", logger.Proto(t.settings)) t.settingsLock.Unlock() return } isImmediate = isImmediate || (!settings.Disabled && settings.Disabled != t.isMutedLocked()) t.settings = utils.CloneProto(settings) + t.logger.Debugw("saving subscriber track settings", "settings", logger.Proto(t.settings)) t.settingsLock.Unlock() if isImmediate { @@ -242,7 +334,6 @@ func (t *SubscribedTrack) applySettings() { return } - t.logger.Debugw("updating subscriber track settings", "settings", logger.Proto(t.settings)) t.settingsVersion = t.versionGenerator.Next() settingsVersion := t.settingsVersion t.settingsLock.Unlock() @@ -253,23 +344,25 @@ func (t *SubscribedTrack) applySettings() { if dt.Kind() == webrtc.RTPCodecTypeVideo { mt := t.MediaTrack() quality := t.settings.Quality + mimeType := dt.Mime() if t.settings.Width > 0 { - quality = mt.GetQualityForDimension(t.settings.Width, t.settings.Height) + quality = mt.GetQualityForDimension(mimeType, t.settings.Width, t.settings.Height) } - spatial = buffer.VideoQualityToSpatialLayer(quality, mt.ToProto()) + spatial = buffer.GetSpatialLayerForVideoQuality(mimeType, quality, mt.ToProto()) if t.settings.Fps > 0 { - temporal = mt.GetTemporalLayerForSpatialFps(spatial, t.settings.Fps, dt.Mime()) + temporal = mt.GetTemporalLayerForSpatialFps(mimeType, spatial, t.settings.Fps) } } t.settingsLock.Lock() if settingsVersion != t.settingsVersion { - // a newer settings has superceded this one + // a newer settings has superseded this one t.settingsLock.Unlock() return } + t.logger.Debugw("applying subscriber track settings", "settings", logger.Proto(t.settings)) if t.settings.Disabled { dt.Mute(true) t.settingsLock.Unlock() @@ -302,3 +395,96 @@ func (t *SubscribedTrack) RTPSender() *webrtc.RTPSender { func (t *SubscribedTrack) SetRTPSender(sender *webrtc.RTPSender) { t.sender.Store(sender) } + +// DownTrackListener implementation +var _ sfu.DownTrackListener = (*SubscribedTrack)(nil) + +func (t *SubscribedTrack) OnBindAndConnected() { + if t.params.Subscriber.Hidden() { + return + } + + t.params.MediaTrack.OnTrackSubscribed() +} + +func (t *SubscribedTrack) OnStatsUpdate(stat *livekit.AnalyticsStat) { + t.params.TelemetryListener.OnTrackStats(t.statsKey, stat) + + if cs, ok := telemetry.CondenseStat(stat); ok { + ti := t.params.WrappedReceiver.TrackInfo() + t.reporter.Tx(func(tx roomobs.TrackTx) { + tx.ParticipantSession().ReportKindCode(roomobs.ParticipantKindCode(t.params.Subscriber.Kind())) + tx.ParticipantSession().ReportKindDetailsCodes(roomobs.ParticipantKindDetailsCodes(t.params.Subscriber.KindDetails())) + tx.ReportName(ti.Name) + tx.ReportKind(roomobs.TrackKindSub) + tx.ReportType(roomobs.TrackTypeFromProto(ti.Type)) + tx.ReportSource(roomobs.TrackSourceFromProto(ti.Source)) + tx.ReportMime(mime.NormalizeMimeType(ti.MimeType).ReporterType()) + tx.ReportLayer(roomobs.PackTrackLayer(ti.Height, ti.Width)) + tx.ReportDuration(uint16(cs.EndTime.Sub(cs.StartTime).Milliseconds())) + tx.ReportFrames(uint16(cs.Frames)) + tx.ReportSendBytes(uint32(cs.Bytes)) + tx.ReportSendPackets(cs.Packets) + tx.ReportPacketsLost(cs.PacketsLost) + tx.ReportScore(stat.Score) + }) + } +} + +func (t *SubscribedTrack) OnMaxSubscribedLayerChanged(layer int32) { + if t.params.OnSubscriberMaxQualityChange != nil { + t.params.OnSubscriberMaxQualityChange(t.downTrack.SubscriberID(), t.downTrack.Mime(), layer) + } +} + +func (t *SubscribedTrack) OnRttUpdate(rtt uint32) { + go t.params.Subscriber.UpdateMediaRTT(rtt) +} + +func (t *SubscribedTrack) OnCodecNegotiated(codec webrtc.RTPCodecCapability) { + if isAvailable, needsPublish := t.params.WrappedReceiver.DetermineReceiver(codec); !isAvailable || !needsPublish { + return + } + + if t.params.OnSubscriberMaxQualityChange != nil || t.params.OnSubscriberAudioCodecChange != nil { + go func() { + mimeType := mime.NormalizeMimeType(codec.MimeType) + switch t.params.MediaTrack.Kind() { + case livekit.TrackType_VIDEO: + spatial := buffer.GetSpatialLayerForVideoQuality( + mimeType, + livekit.VideoQuality_HIGH, + t.params.MediaTrack.ToProto(), + ) + if t.params.OnSubscriberMaxQualityChange != nil { + t.params.OnSubscriberMaxQualityChange(t.downTrack.SubscriberID(), mimeType, spatial) + } + + case livekit.TrackType_AUDIO: + if t.params.OnSubscriberAudioCodecChange != nil { + t.params.OnSubscriberAudioCodecChange(t.downTrack.SubscriberID(), mimeType, true) + } + } + }() + } +} + +func (t *SubscribedTrack) OnDownTrackClose(isExpectedToResume bool) { + // Cache transceiver for potential re-use on resume. + // To ensure subscription manager does not re-subscribe before caching, + // delete the subscribed track only after caching. + if isExpectedToResume { + if tr := t.downTrack.GetTransceiver(); tr != nil { + t.params.Subscriber.CacheDownTrack(t.ID(), tr, t.downTrack.GetState()) + } + } + + if t.params.OnDownTrackClosed != nil { + t.params.OnDownTrackClosed(t.params.Subscriber.ID()) + } + t.Close(isExpectedToResume) +} + +func (t *SubscribedTrack) OnStreamStarted() { + t.params.TelemetryListener.OnTrackSubscribeStreamStarted(t.params.Subscriber.ID(), t.params.MediaTrack.ToProto()) +} diff --git a/pkg/rtc/subscriptionmanager.go b/pkg/rtc/subscriptionmanager.go index 2d41f37..640742b 100644 --- a/pkg/rtc/subscriptionmanager.go +++ b/pkg/rtc/subscriptionmanager.go @@ -19,16 +19,16 @@ package rtc import ( "context" "errors" + "maps" + "slices" "sync" "time" "github.com/pion/webrtc/v4/pkg/rtcerr" "go.uber.org/atomic" - "golang.org/x/exp/maps" "github.com/livekit/livekit-server/pkg/rtc/types" "github.com/livekit/livekit-server/pkg/sfu" - "github.com/livekit/livekit-server/pkg/telemetry" "github.com/livekit/livekit-server/pkg/telemetry/prometheus" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" @@ -42,9 +42,8 @@ var ( // ensuring this is longer than iceFailedTimeout so we are certain the participant won't return notFoundTimeout = time.Minute // amount of time to try otherwise before flagging subscription as failed - subscriptionTimeout = iceFailedTimeoutTotal - trackRemoveGracePeriod = time.Second - maxUnsubscribeWait = time.Second + subscriptionTimeout = iceFailedTimeoutTotal + maxUnsubscribeWait = time.Second ) const ( @@ -58,10 +57,12 @@ type SubscriptionManagerParams struct { OnTrackSubscribed func(subTrack types.SubscribedTrack) OnTrackUnsubscribed func(subTrack types.SubscribedTrack) OnSubscriptionError func(trackID livekit.TrackID, fatal bool, err error) - Telemetry telemetry.TelemetryService + TelemetryListener types.ParticipantTelemetryListener SubscriptionLimitVideo, SubscriptionLimitAudio int32 + DataTrackResolver types.DataTrackResolver + UseOneShotSignallingMode bool } @@ -69,27 +70,32 @@ type SubscriptionManagerParams struct { type SubscriptionManager struct { params SubscriptionManagerParams lock sync.RWMutex - subscriptions map[livekit.TrackID]*trackSubscription + subscriptions map[livekit.TrackID]*mediaTrackSubscription pendingUnsubscribes atomic.Int32 subscribedVideoCount, subscribedAudioCount atomic.Int32 - subscribedTo map[livekit.ParticipantID]map[livekit.TrackID]struct{} - reconcileCh chan livekit.TrackID - closeCh chan struct{} - doneCh chan struct{} + subscribedTo map[livekit.ParticipantID]map[livekit.TrackID]struct{} + reconcileCh chan livekit.TrackID + reconcileDataTrackCh chan livekit.TrackID + closeCh chan struct{} + doneCh chan struct{} onSubscribeStatusChanged func(publisherID livekit.ParticipantID, subscribed bool) + + dataTrackSubscriptions map[livekit.TrackID]*dataTrackSubscription } func NewSubscriptionManager(params SubscriptionManagerParams) *SubscriptionManager { m := &SubscriptionManager{ - params: params, - subscriptions: make(map[livekit.TrackID]*trackSubscription), - subscribedTo: make(map[livekit.ParticipantID]map[livekit.TrackID]struct{}), - reconcileCh: make(chan livekit.TrackID, 50), - closeCh: make(chan struct{}), - doneCh: make(chan struct{}), + params: params, + subscriptions: make(map[livekit.TrackID]*mediaTrackSubscription), + subscribedTo: make(map[livekit.ParticipantID]map[livekit.TrackID]struct{}), + reconcileCh: make(chan livekit.TrackID, 50), + reconcileDataTrackCh: make(chan livekit.TrackID, 5), + closeCh: make(chan struct{}), + doneCh: make(chan struct{}), + dataTrackSubscriptions: make(map[livekit.TrackID]*dataTrackSubscription), } go m.reconcileWorker() @@ -107,6 +113,23 @@ func (m *SubscriptionManager) Close(isExpectedToResume bool) { <-m.doneCh + prometheus.RecordTrackSubscribeCancels(int32(m.getNumCancellations())) + + // Remove observer closures from track change/remove notifiers to allow + // this participant and its transports to be garbage collected. + m.lock.Lock() + subs := maps.Clone(m.subscriptions) + dataTrackSubs := maps.Clone(m.dataTrackSubscriptions) + m.lock.Unlock() + for _, sub := range subs { + sub.setChangedNotifier(nil) + sub.setRemovedNotifier(nil) + } + for _, dataTrackSub := range dataTrackSubs { + dataTrackSub.setChangedNotifier(nil) + dataTrackSub.setRemovedNotifier(nil) + } + subTracks := m.GetSubscribedTracks() downTracksToClose := make([]*sfu.DownTrack, 0, len(subTracks)) for _, st := range subTracks { @@ -120,14 +143,32 @@ func (m *SubscriptionManager) Close(isExpectedToResume bool) { if isExpectedToResume { for _, dt := range downTracksToClose { - dt.CloseWithFlush(false) + dt.CloseWithFlush(false, true) } } else { // flush blocks, so execute in parallel for _, dt := range downTracksToClose { - go dt.CloseWithFlush(true) + go dt.CloseWithFlush(true, true) } } + + for trackID, sub := range dataTrackSubs { + m.setDataTrackDesired(trackID, false) + + dataDownTrack := sub.getDataDownTrack() + if dataDownTrack == nil { + // already unsubscribed + continue + } + + dataTrack := dataDownTrack.PublishDataTrack() + if dataTrack == nil { + continue + } + + dataTrack.RemoveSubscriber(sub.subscriberID) + } + m.notifyDataTrackSubscriberHandles() } func (m *SubscriptionManager) isClosed() bool { @@ -139,8 +180,12 @@ func (m *SubscriptionManager) isClosed() bool { } } -func (m *SubscriptionManager) SubscribeToTrack(trackID livekit.TrackID) { - if m.params.UseOneShotSignallingMode { +func (m *SubscriptionManager) SubscribeToTrack(trackID livekit.TrackID, isSync bool) { + if m.isClosed() { + return + } + + if m.params.UseOneShotSignallingMode || isSync { m.subscribeSynchronous(trackID) return } @@ -150,7 +195,7 @@ func (m *SubscriptionManager) SubscribeToTrack(trackID livekit.TrackID) { sLogger := m.params.Logger.WithValues( "trackID", trackID, ) - sub = newTrackSubscription(m.params.Participant.ID(), trackID, sLogger) + sub = newMediaTrackSubscription(m.params.Participant.ID(), trackID, sLogger) m.lock.Lock() m.subscriptions[trackID] = sub @@ -177,10 +222,78 @@ func (m *SubscriptionManager) UnsubscribeFromTrack(trackID livekit.TrackID) { return } + if sub.isCanceled() { + prometheus.RecordTrackSubscribeCancels(1) + } + sub.logger.Debugw("unsubscribing from track") m.queueReconcile(trackID) } +func (m *SubscriptionManager) SubscribeToDataTrack(trackID livekit.TrackID) { + if m.isClosed() { + return + } + + sub, desireChanged := m.setDataTrackDesired(trackID, true) + if sub == nil { + sLogger := m.params.Logger.WithValues( + "trackID", trackID, + ) + sub = newDataTrackSubscription(m.params.Participant.ID(), trackID, sLogger) + + m.lock.Lock() + m.dataTrackSubscriptions[trackID] = sub + m.lock.Unlock() + + sub, desireChanged = m.setDataTrackDesired(trackID, true) + } + if desireChanged { + sub.logger.Debugw("subscribing to data track") + } + + m.queueReconcileDataTrack(trackID) +} + +func (m *SubscriptionManager) UnsubscribeFromDataTrack(trackID livekit.TrackID) { + sub, desireChanged := m.setDataTrackDesired(trackID, false) + if sub == nil || !desireChanged { + return + } + + sub.logger.Debugw("unsubscribing from data track") + m.queueReconcileDataTrack(trackID) +} + +func (m *SubscriptionManager) ClearAllSubscriptions() { + m.params.Logger.Debugw("clearing all subscriptions") + + if m.params.UseOneShotSignallingMode { + for _, track := range m.GetSubscribedTracks() { + m.unsubscribeSynchronous(track.ID()) + } + + // no synchronous data tracks + } + + numCancellations := 0 + m.lock.RLock() + for _, sub := range m.subscriptions { + if sub.isCanceled() { + numCancellations++ + } + sub.setDesired(false) + } + + for _, sub := range m.dataTrackSubscriptions { + sub.setDesired(false) + } + m.lock.RUnlock() + prometheus.RecordTrackSubscribeCancels(int32(numCancellations)) + + m.ReconcileAll() +} + func (m *SubscriptionManager) GetSubscribedTracks() []types.SubscribedTrack { m.lock.RLock() defer m.lock.RUnlock() @@ -243,7 +356,7 @@ func (m *SubscriptionManager) GetSubscribedParticipants() []livekit.ParticipantI m.lock.RLock() defer m.lock.RUnlock() - return maps.Keys(m.subscribedTo) + return slices.Collect(maps.Keys(m.subscribedTo)) } func (m *SubscriptionManager) IsSubscribedTo(participantID livekit.ParticipantID) bool { @@ -261,7 +374,7 @@ func (m *SubscriptionManager) UpdateSubscribedTrackSettings(trackID livekit.Trac sLogger := m.params.Logger.WithValues( "trackID", trackID, ) - sub = newTrackSubscription(m.params.Participant.ID(), trackID, sLogger) + sub = newMediaTrackSubscription(m.params.Participant.ID(), trackID, sLogger) m.subscriptions[trackID] = sub } m.lock.Unlock() @@ -269,6 +382,21 @@ func (m *SubscriptionManager) UpdateSubscribedTrackSettings(trackID livekit.Trac sub.setSettings(settings) } +func (m *SubscriptionManager) UpdateDataTrackSubscriptionOptions(trackID livekit.TrackID, subscriptionOptions *livekit.DataTrackSubscriptionOptions) { + m.lock.Lock() + sub, ok := m.dataTrackSubscriptions[trackID] + if !ok { + sLogger := m.params.Logger.WithValues( + "trackID", trackID, + ) + sub = newDataTrackSubscription(m.params.Participant.ID(), trackID, sLogger) + m.dataTrackSubscriptions[trackID] = sub + } + m.lock.Unlock() + + sub.setSubscriptionOptions(subscriptionOptions) +} + // OnSubscribeStatusChanged callback will be notified when a participant subscribes or unsubscribes to another participant // it will only fire once per publisher. If current participant is subscribed to multiple tracks from another, this // callback will only fire once. @@ -301,9 +429,10 @@ func (m *SubscriptionManager) WaitUntilSubscribed(timeout time.Duration) error { func (m *SubscriptionManager) ReconcileAll() { m.queueReconcile(trackIDForReconcileSubscriptions) + m.queueReconcileDataTrack(trackIDForReconcileSubscriptions) } -func (m *SubscriptionManager) setDesired(trackID livekit.TrackID, desired bool) (*trackSubscription, bool) { +func (m *SubscriptionManager) setDesired(trackID livekit.TrackID, desired bool) (*mediaTrackSubscription, bool) { m.lock.RLock() defer m.lock.RUnlock() @@ -315,6 +444,18 @@ func (m *SubscriptionManager) setDesired(trackID livekit.TrackID, desired bool) return sub, sub.setDesired(desired) } +func (m *SubscriptionManager) setDataTrackDesired(trackID livekit.TrackID, desired bool) (*dataTrackSubscription, bool) { + m.lock.RLock() + defer m.lock.RUnlock() + + sub, ok := m.dataTrackSubscriptions[trackID] + if !ok { + return nil, false + } + + return sub, sub.setDesired(desired) +} + func (m *SubscriptionManager) canReconcile() bool { p := m.params.Participant if m.isClosed() || p.IsClosed() || p.IsDisconnected() { @@ -324,7 +465,7 @@ func (m *SubscriptionManager) canReconcile() bool { } func (m *SubscriptionManager) reconcileSubscriptions() { - var needsToReconcile []*trackSubscription + var needsToReconcile []*mediaTrackSubscription m.lock.RLock() for _, sub := range m.subscriptions { if sub.needsSubscribe() || sub.needsUnsubscribe() || sub.needsBind() || sub.needsCleanup() { @@ -338,7 +479,7 @@ func (m *SubscriptionManager) reconcileSubscriptions() { } } -func (m *SubscriptionManager) reconcileSubscription(s *trackSubscription) { +func (m *SubscriptionManager) reconcileSubscription(s *mediaTrackSubscription) { if !m.canReconcile() { return } @@ -354,9 +495,8 @@ func (m *SubscriptionManager) reconcileSubscription(s *trackSubscription) { numAttempts := s.getNumAttempts() if numAttempts == 0 { - m.params.Telemetry.TrackSubscribeRequested( - context.Background(), - m.params.Participant.ID(), + m.params.TelemetryListener.OnTrackSubscribeRequested( + s.subscriberID, &livekit.TrackInfo{ Sid: string(s.trackID), }, @@ -375,14 +515,14 @@ func (m *SubscriptionManager) reconcileSubscription(s *trackSubscription) { // - ErrSubscriptionLimitExceeded: the participant have reached the limit of subscriptions, wait for the other subscription to be unsubscribed // We'll still log an event to reflect this in telemetry since it's been too long if s.durationSinceStart() > subscriptionTimeout { - s.maybeRecordError(m.params.Telemetry, m.params.Participant.ID(), err, true) + s.maybeRecordError(m.params.TelemetryListener, err, true) } case ErrTrackNotFound: // source track was never published or closed // if after timeout we'd unsubscribe from it. // this is the *only* case we'd change desired state if s.durationSinceStart() > notFoundTimeout { - s.maybeRecordError(m.params.Telemetry, m.params.Participant.ID(), err, true) + s.maybeRecordError(m.params.TelemetryListener, err, true) s.logger.Infow("unsubscribing from track after notFoundTimeout", "error", err) s.setDesired(false) m.queueReconcile(s.trackID) @@ -391,15 +531,17 @@ func (m *SubscriptionManager) reconcileSubscription(s *trackSubscription) { default: // all other errors if s.durationSinceStart() > subscriptionTimeout { - s.logger.Warnw("failed to subscribe, triggering error handler", err, - "attempt", numAttempts, + s.logger.Warnw( + "failed to subscribe, triggering error handler", err, + "attempt", s.getNumAttempts(), ) - s.maybeRecordError(m.params.Telemetry, m.params.Participant.ID(), err, false) + s.maybeRecordError(m.params.TelemetryListener, err, false) m.params.OnSubscriptionError(s.trackID, true, err) } else { - s.logger.Debugw("failed to subscribe, retrying", + s.logger.Debugw( + "failed to subscribe, retrying", "error", err, - "attempt", numAttempts, + "attempt", s.getNumAttempts(), ) } } @@ -419,7 +561,7 @@ func (m *SubscriptionManager) reconcileSubscription(s *trackSubscription) { // and the clean up path to handle removing subscription from the subscription map. // It is possible that the track is re-published before subscribed track is closed. // That could create a new subscription and a duplicate entry in SDP. - // Waiting for susbcribed track close would ensure that the track is removed from + // Waiting for subscribed track close would ensure that the track is removed from // the peer connection before re-published track is re-subscribed and added back to the SDP. return } @@ -428,10 +570,13 @@ func (m *SubscriptionManager) reconcileSubscription(s *trackSubscription) { // check bound status, notify error callback if it's not bound // if a publisher leaves or closes the source track, SubscribedTrack will be closed as well and it will go // back to needsSubscribe state - if s.durationSinceStart() > subscriptionTimeout { - s.logger.Warnw("track not bound after timeout", nil) - s.maybeRecordError(m.params.Telemetry, m.params.Participant.ID(), ErrTrackNotBound, false) - m.params.OnSubscriptionError(s.trackID, true, ErrTrackNotBound) + if activeAt := m.params.Participant.ActiveAt(); !activeAt.IsZero() { + wait := min(time.Since(activeAt), s.durationSinceStart()) + if wait > subscriptionTimeout { + s.logger.Warnw("track not bound after timeout", nil) + s.maybeRecordError(m.params.TelemetryListener, ErrTrackNotBound, false) + m.params.OnSubscriptionError(s.trackID, true, ErrTrackNotBound) + } } } @@ -443,6 +588,86 @@ func (m *SubscriptionManager) reconcileSubscription(s *trackSubscription) { m.lock.Unlock() } +func (m *SubscriptionManager) reconcileDataTrackSubscriptions() { + var needsToReconcile []*dataTrackSubscription + m.lock.RLock() + for _, sub := range m.dataTrackSubscriptions { + if sub.needsSubscribe() || sub.needsUnsubscribe() { + needsToReconcile = append(needsToReconcile, sub) + } + } + m.lock.RUnlock() + + for _, s := range needsToReconcile { + m.reconcileDataTrackSubscription(s) + } +} + +func (m *SubscriptionManager) reconcileDataTrackSubscription(s *dataTrackSubscription) { + if !m.canReconcile() { + return + } + if s.needsSubscribe() { + if err := m.subscribeDataTrack(s); err != nil { + s.recordAttempt(false) + + switch err { + case ErrNoSubscribePermission: + // these are errors that are outside of our control, so we'll keep trying + // - ErrNoSubscribePermission: participant was not granted canSubscribe, may change any moment + case ErrTrackNotFound: + // source track was never published or closed + // if after timeout we'd unsubscribe from it. + // this is the *only* case we'd change desired state + if s.durationSinceStart() > notFoundTimeout { + s.logger.Infow("unsubscribing from data track after notFoundTimeout", "error", err) + s.setDesired(false) + m.queueReconcile(s.trackID) + } + default: + // all other errors + if s.durationSinceStart() > subscriptionTimeout { + s.logger.Warnw( + "failed to subscribe, triggering error handler", err, + "attempt", s.getNumAttempts(), + ) + } else { + s.logger.Debugw( + "failed to subscribe, retrying", + "error", err, + "attempt", s.getNumAttempts(), + ) + } + } + } else { + s.recordAttempt(true) + m.notifyDataTrackSubscriberHandles() + } + + return + } + + if s.needsUnsubscribe() { + if err := m.unsubscribeDataTrack(s); err != nil { + s.logger.Warnw("failed to unsubscribe", err) + } + + m.lock.Lock() + delete(m.dataTrackSubscriptions, s.trackID) + m.lock.Unlock() + m.notifyDataTrackSubscriberHandles() + return + } + + m.lock.Lock() + if s.needsCleanup() { + s.logger.Debugw("cleanup removing data track subscription") + delete(m.dataTrackSubscriptions, s.trackID) + m.notifyDataTrackSubscriberHandles() + } + m.lock.Unlock() +} + // trigger an immediate reconciliation, when trackID is empty, will reconcile all subscriptions func (m *SubscriptionManager) queueReconcile(trackID livekit.TrackID) { select { @@ -452,17 +677,29 @@ func (m *SubscriptionManager) queueReconcile(trackID livekit.TrackID) { } } +func (m *SubscriptionManager) queueReconcileDataTrack(trackID livekit.TrackID) { + select { + case m.reconcileDataTrackCh <- trackID: + default: + // queue is full, will reconcile based on timer + } +} + func (m *SubscriptionManager) reconcileWorker() { + defer close(m.doneCh) + reconcileTicker := time.NewTicker(reconcileInterval) defer reconcileTicker.Stop() - defer close(m.doneCh) for { select { case <-m.closeCh: return + case <-reconcileTicker.C: m.reconcileSubscriptions() + m.reconcileDataTrackSubscriptions() + case trackID := <-m.reconcileCh: m.lock.Lock() s := m.subscriptions[trackID] @@ -472,6 +709,16 @@ func (m *SubscriptionManager) reconcileWorker() { } else { m.reconcileSubscriptions() } + + case trackID := <-m.reconcileDataTrackCh: + m.lock.Lock() + s := m.dataTrackSubscriptions[trackID] + m.lock.Unlock() + if s != nil { + m.reconcileDataTrackSubscription(s) + } else { + m.reconcileDataTrackSubscriptions() + } } } } @@ -491,31 +738,31 @@ func (m *SubscriptionManager) hasCapacityForSubscription(kind livekit.TrackType) return true } -func (m *SubscriptionManager) subscribe(s *trackSubscription) error { - s.logger.Debugw("executing subscribe") +func (m *SubscriptionManager) subscribe(sub *mediaTrackSubscription) error { + sub.logger.Debugw("executing subscribe") if !m.params.Participant.CanSubscribe() { return ErrNoSubscribePermission } - if kind, ok := s.getKind(); ok && !m.hasCapacityForSubscription(kind) { + if kind, ok := sub.getKind(); ok && !m.hasCapacityForSubscription(kind) { return ErrSubscriptionLimitExceeded } - trackID := s.trackID + trackID := sub.trackID res := m.params.TrackResolver(m.params.Participant, trackID) - s.logger.Debugw("resolved track", "result", res) + sub.logger.Debugw("resolved track", "result", res) - if res.TrackChangedNotifier != nil && s.setChangedNotifier(res.TrackChangedNotifier) { + if res.TrackChangedNotifier != nil && sub.setChangedNotifier(res.TrackChangedNotifier) { // set callback only when we haven't done it before // we set the observer before checking for existence of track, so that we may get notified // when the track becomes available - res.TrackChangedNotifier.AddObserver(string(m.params.Participant.ID()), func() { + res.TrackChangedNotifier.AddObserver(string(sub.subscriberID), func() { m.queueReconcile(trackID) }) } - if res.TrackRemovedNotifier != nil && s.setRemovedNotifier(res.TrackRemovedNotifier) { - res.TrackRemovedNotifier.AddObserver(string(m.params.Participant.ID()), func() { + if res.TrackRemovedNotifier != nil && sub.setRemovedNotifier(res.TrackRemovedNotifier) { + res.TrackRemovedNotifier.AddObserver(string(sub.subscriberID), func() { // re-resolve the track in case the same track had been re-published res := m.params.TrackResolver(m.params.Participant, trackID) if res.Track != nil { @@ -530,93 +777,26 @@ func (m *SubscriptionManager) subscribe(s *trackSubscription) error { if track == nil { return ErrTrackNotFound } - s.trySetKind(track.Kind()) + sub.trySetKind(track.Kind()) if !m.hasCapacityForSubscription(track.Kind()) { return ErrSubscriptionLimitExceeded } - s.setPublisher(res.PublisherIdentity, res.PublisherID) + sub.setPublisher(res.PublisherIdentity, res.PublisherID) - permChanged := s.setHasPermission(res.HasPermission) + permChanged := sub.setHasPermission(res.HasPermission) if permChanged { - m.params.Participant.SubscriptionPermissionUpdate(s.getPublisherID(), trackID, res.HasPermission) + m.params.Participant.SendSubscriptionPermissionUpdate(sub.getPublisherID(), trackID, res.HasPermission) } if !res.HasPermission { return ErrNoTrackPermission } - subTrack, err := track.AddSubscriber(m.params.Participant) - if err != nil && !errors.Is(err, errAlreadySubscribed) { - // ignore error(s): already subscribed - if !utils.ErrorIsOneOf(err, ErrNoReceiver) { - // as track resolution could take some time, not logging errors due to waiting for track resolution - m.params.Logger.Warnw("add subscriber failed", err, "trackID", trackID) - } + if err := m.addSubscriber(sub, track, false); err != nil { return err } - if err == errAlreadySubscribed { - m.params.Logger.Debugw( - "already subscribed to track", - "trackID", trackID, - "subscribedAudioCount", m.subscribedAudioCount.Load(), - "subscribedVideoCount", m.subscribedVideoCount.Load(), - ) - } - if err == nil && subTrack != nil { // subTrack could be nil if already subscribed - subTrack.OnClose(func(isExpectedToResume bool) { - m.handleSubscribedTrackClose(s, isExpectedToResume) - }) - subTrack.AddOnBind(func(err error) { - if err != nil { - s.logger.Infow("failed to bind track", "err", err) - s.maybeRecordError(m.params.Telemetry, m.params.Participant.ID(), err, true) - m.UnsubscribeFromTrack(trackID) - m.params.OnSubscriptionError(trackID, false, err) - return - } - s.setBound() - s.maybeRecordSuccess(m.params.Telemetry, m.params.Participant.ID()) - }) - s.setSubscribedTrack(subTrack) - switch track.Kind() { - case livekit.TrackType_VIDEO: - m.subscribedVideoCount.Inc() - case livekit.TrackType_AUDIO: - m.subscribedAudioCount.Inc() - } - - if subTrack.NeedsNegotiation() { - m.params.Participant.Negotiate(false) - } - - go m.params.OnTrackSubscribed(subTrack) - - m.params.Logger.Debugw( - "subscribed to track", - "trackID", trackID, - "subscribedAudioCount", m.subscribedAudioCount.Load(), - "subscribedVideoCount", m.subscribedVideoCount.Load(), - ) - } - - // add mark the participant as someone we've subscribed to - firstSubscribe := false - publisherID := s.getPublisherID() - m.lock.Lock() - pTracks := m.subscribedTo[publisherID] - changedCB := m.onSubscribeStatusChanged - if pTracks == nil { - pTracks = make(map[livekit.TrackID]struct{}) - m.subscribedTo[publisherID] = pTracks - firstSubscribe = true - } - pTracks[trackID] = struct{}{} - m.lock.Unlock() - - if changedCB != nil && firstSubscribe { - changedCB(publisherID, true) - } + m.markSubscribedTo(sub.getPublisherID(), trackID) return nil } @@ -641,20 +821,29 @@ func (m *SubscriptionManager) subscribeSynchronous(trackID livekit.TrackID) erro sLogger := m.params.Logger.WithValues( "trackID", trackID, ) - sub = newTrackSubscription(m.params.Participant.ID(), trackID, sLogger) + sub = newMediaTrackSubscription(m.params.Participant.ID(), trackID, sLogger) m.subscriptions[trackID] = sub } m.lock.Unlock() sub.setDesired(true) + return m.addSubscriber(sub, track, true) +} + +func (m *SubscriptionManager) addSubscriber(sub *mediaTrackSubscription, track types.MediaTrack, isSync bool) error { + trackID := track.ID() subTrack, err := track.AddSubscriber(m.params.Participant) if err != nil && !errors.Is(err, errAlreadySubscribed) { + // ignore error(s): already subscribed + if !utils.ErrorIsOneOf(err, ErrNoReceiver) { + // as track resolution could take some time, not logging errors due to waiting for track resolution + m.params.Logger.Warnw("add subscriber failed", err, "trackID", trackID) + } return err } if err == errAlreadySubscribed { - m.params.Logger.Debugw( + sub.logger.Debugw( "already subscribed to track", - "trackID", trackID, "subscribedAudioCount", m.subscribedAudioCount.Load(), "subscribedVideoCount", m.subscribedVideoCount.Load(), ) @@ -663,20 +852,22 @@ func (m *SubscriptionManager) subscribeSynchronous(trackID livekit.TrackID) erro subTrack.OnClose(func(isExpectedToResume bool) { m.handleSubscribedTrackClose(sub, isExpectedToResume) - m.lock.Lock() - delete(m.subscriptions, trackID) - m.lock.Unlock() + if isSync { + m.lock.Lock() + delete(m.subscriptions, trackID) + m.lock.Unlock() + } }) subTrack.AddOnBind(func(err error) { if err != nil { sub.logger.Infow("failed to bind track", "err", err) - sub.maybeRecordError(m.params.Telemetry, m.params.Participant.ID(), err, true) + sub.maybeRecordError(m.params.TelemetryListener, err, true) m.UnsubscribeFromTrack(trackID) m.params.OnSubscriptionError(trackID, false, err) return } sub.setBound() - sub.maybeRecordSuccess(m.params.Telemetry, m.params.Participant.ID()) + sub.maybeRecordSuccess(m.params.TelemetryListener) }) sub.setSubscribedTrack(subTrack) @@ -687,11 +878,14 @@ func (m *SubscriptionManager) subscribeSynchronous(trackID livekit.TrackID) erro m.subscribedAudioCount.Inc() } + if !isSync && subTrack.NeedsNegotiation() { + m.params.Participant.Negotiate(false) + } + go m.params.OnTrackSubscribed(subTrack) - m.params.Logger.Debugw( + sub.logger.Debugw( "subscribed to track", - "trackID", trackID, "subscribedAudioCount", m.subscribedAudioCount.Load(), "subscribedVideoCount", m.subscribedVideoCount.Load(), ) @@ -699,7 +893,7 @@ func (m *SubscriptionManager) subscribeSynchronous(trackID livekit.TrackID) erro return nil } -func (m *SubscriptionManager) unsubscribe(s *trackSubscription) error { +func (m *SubscriptionManager) unsubscribe(s *mediaTrackSubscription) error { s.logger.Debugw("executing unsubscribe") subTrack := s.getSubscribedTrack() @@ -709,7 +903,7 @@ func (m *SubscriptionManager) unsubscribe(s *trackSubscription) error { } track := subTrack.MediaTrack() - pID := m.params.Participant.ID() + pID := s.subscriberID m.pendingUnsubscribes.Inc() go func() { defer m.pendingUnsubscribes.Dec() @@ -731,6 +925,10 @@ func (m *SubscriptionManager) unsubscribeSynchronous(trackID livekit.TrackID) er sub.logger.Debugw("executing unsubscribe synchronous") + if sub.isCanceled() { + prometheus.RecordTrackSubscribeCancels(1) + } + subTrack := sub.getSubscribedTrack() if subTrack == nil { // already unsubscribed @@ -738,7 +936,7 @@ func (m *SubscriptionManager) unsubscribeSynchronous(trackID livekit.TrackID) er } track := subTrack.MediaTrack() - track.RemoveSubscriber(m.params.Participant.ID(), false) + track.RemoveSubscriber(sub.subscriberID, false) return nil } @@ -748,6 +946,9 @@ func (m *SubscriptionManager) handleSourceTrackRemoved(trackID livekit.TrackID) m.lock.Unlock() if sub != nil { + if sub.isCanceled() { + prometheus.RecordTrackSubscribeCancels(1) + } sub.handleSourceTrackRemoved() } } @@ -757,7 +958,7 @@ func (m *SubscriptionManager) handleSourceTrackRemoved(trackID livekit.TrackID) // - subscriber-initiated unsubscribe // - UpTrack was closed // - publisher revoked permissions for the participant -func (m *SubscriptionManager) handleSubscribedTrackClose(s *trackSubscription, isExpectedToResume bool) { +func (m *SubscriptionManager) handleSubscribedTrackClose(s *mediaTrackSubscription, isExpectedToResume bool) { s.logger.Debugw( "subscribed track closed", "isExpectedToResume", isExpectedToResume, @@ -768,6 +969,8 @@ func (m *SubscriptionManager) handleSubscribedTrackClose(s *trackSubscription, i return } s.setSubscribedTrack(nil) + s.setChangedNotifier(nil) + s.setRemovedNotifier(nil) var relieveFromLimits bool switch subTrack.MediaTrack().Kind() { @@ -779,23 +982,7 @@ func (m *SubscriptionManager) handleSubscribedTrackClose(s *trackSubscription, i relieveFromLimits = m.params.SubscriptionLimitAudio > 0 && audioCount == m.params.SubscriptionLimitAudio-1 } - // remove from subscribedTo - publisherID := s.getPublisherID() - lastSubscription := false - m.lock.Lock() - changedCB := m.onSubscribeStatusChanged - pTracks := m.subscribedTo[publisherID] - if pTracks != nil { - delete(pTracks, s.trackID) - if len(pTracks) == 0 { - delete(m.subscribedTo, publisherID) - lastSubscription = true - } - } - m.lock.Unlock() - if changedCB != nil && lastSubscription { - go changedCB(publisherID, false) - } + m.unmarkSubscribedTo(s.getPublisherID(), s.trackID) go m.params.OnTrackUnsubscribed(subTrack) @@ -804,9 +991,8 @@ func (m *SubscriptionManager) handleSubscribedTrackClose(s *trackSubscription, i // * the participant isn't closing // * it's not a migration if wasBound { - m.params.Telemetry.TrackUnsubscribed( - context.Background(), - m.params.Participant.ID(), + m.params.TelemetryListener.OnTrackUnsubscribed( + s.subscriberID, &livekit.TrackInfo{Sid: string(s.trackID), Type: subTrack.MediaTrack().Kind()}, !isExpectedToResume, ) @@ -815,9 +1001,8 @@ func (m *SubscriptionManager) handleSubscribedTrackClose(s *trackSubscription, i if dt != nil { stats := dt.GetTrackStats() if stats != nil { - m.params.Telemetry.TrackSubscribeRTPStats( - context.Background(), - m.params.Participant.ID(), + m.params.TelemetryListener.OnTrackSubscribeRTPStats( + s.subscriberID, s.trackID, dt.Mime(), stats, @@ -860,6 +1045,162 @@ func (m *SubscriptionManager) handleSubscribedTrackClose(s *trackSubscription, i } } +func (m *SubscriptionManager) subscribeDataTrack(sub *dataTrackSubscription) error { + sub.logger.Debugw("executing subscribe") + + if !m.params.Participant.CanSubscribe() { + return ErrNoSubscribePermission + } + + trackID := sub.trackID + res := m.params.DataTrackResolver(m.params.Participant, trackID) + sub.logger.Debugw("resolved data track", "result", res) + + if res.TrackChangedNotifier != nil && sub.setChangedNotifier(res.TrackChangedNotifier) { + // set callback only when we haven't done it before + // we set the observer before checking for existence of track, so that we may get notified + // when the track becomes available + res.TrackChangedNotifier.AddObserver(string(sub.subscriberID), func() { + m.queueReconcileDataTrack(trackID) + }) + } + if res.TrackRemovedNotifier != nil && sub.setRemovedNotifier(res.TrackRemovedNotifier) { + res.TrackRemovedNotifier.AddObserver(string(sub.subscriberID), func() { + // re-resolve the track in case the same track had been re-published + res := m.params.DataTrackResolver(m.params.Participant, trackID) + if res.DataTrack != nil { + // do not unsubscribe, track is still available + return + } + m.handleSourceDataTrackRemoved(trackID) + }) + } + + dataTrack := res.DataTrack + if dataTrack == nil { + return ErrTrackNotFound + } + + sub.setPublisher(res.PublisherIdentity, res.PublisherID) + + dataDownTrack, err := dataTrack.AddSubscriber(m.params.Participant) + if err != nil && !errors.Is(err, errAlreadySubscribed) { + return err + } + if err == errAlreadySubscribed { + sub.logger.Debugw("already subscribed to data track") + } + if err == nil && dataDownTrack != nil { // subTrack could be nil if already subscribed + sub.setDataDownTrack(dataDownTrack) + sub.logger.Debugw("subscribed to data track") + } + + m.markSubscribedTo(sub.getPublisherID(), trackID) + return nil +} + +func (m *SubscriptionManager) unsubscribeDataTrack(s *dataTrackSubscription) error { + s.logger.Debugw("executing unsubscribe") + + dataDownTrack := s.getDataDownTrack() + if dataDownTrack == nil { + // already unsubscribed + return nil + } + + dataTrack := dataDownTrack.PublishDataTrack() + dataTrack.RemoveSubscriber(s.subscriberID) + + s.setChangedNotifier(nil) + s.setRemovedNotifier(nil) + + m.unmarkSubscribedTo(s.getPublisherID(), s.trackID) + return nil +} + +func (m *SubscriptionManager) notifyDataTrackSubscriberHandles() { + m.lock.Lock() + handles := make(map[uint32]*livekit.DataTrackSubscriberHandles_PublishedDataTrack, len(m.dataTrackSubscriptions)) + for _, sub := range m.dataTrackSubscriptions { + if !sub.isDesired() { + continue + } + dataDownTrack := sub.getDataDownTrack() + if dataDownTrack == nil { + continue + } + handles[uint32(dataDownTrack.Handle())] = &livekit.DataTrackSubscriberHandles_PublishedDataTrack{ + PublisherIdentity: string(sub.publisherIdentity), + PublisherSid: string(sub.publisherID), + TrackSid: string(sub.trackID), + } + } + m.lock.Unlock() + + m.params.Participant.SendDataTrackSubscriberHandles(handles) +} + +func (m *SubscriptionManager) handleSourceDataTrackRemoved(trackID livekit.TrackID) { + m.lock.Lock() + sub := m.dataTrackSubscriptions[trackID] + m.lock.Unlock() + + if sub != nil { + sub.handleSourceTrackRemoved() + } +} + +func (m *SubscriptionManager) markSubscribedTo(publisherID livekit.ParticipantID, trackID livekit.TrackID) { + // add mark the participant as someone we've subscribed to + firstSubscribe := false + m.lock.Lock() + pTracks := m.subscribedTo[publisherID] + changedCB := m.onSubscribeStatusChanged + if pTracks == nil { + pTracks = make(map[livekit.TrackID]struct{}) + m.subscribedTo[publisherID] = pTracks + firstSubscribe = true + } + pTracks[trackID] = struct{}{} + m.lock.Unlock() + + if changedCB != nil && firstSubscribe { + changedCB(publisherID, true) + } +} + +func (m *SubscriptionManager) unmarkSubscribedTo(publisherID livekit.ParticipantID, trackID livekit.TrackID) { + // remove from subscribedTo + lastSubscription := false + m.lock.Lock() + changedCB := m.onSubscribeStatusChanged + pTracks := m.subscribedTo[publisherID] + if pTracks != nil { + delete(pTracks, trackID) + if len(pTracks) == 0 { + delete(m.subscribedTo, publisherID) + lastSubscription = true + } + } + m.lock.Unlock() + if changedCB != nil && lastSubscription { + go changedCB(publisherID, false) + } +} + +func (m *SubscriptionManager) getNumCancellations() int { + m.lock.RLock() + defer m.lock.RUnlock() + + numCancellations := 0 + for _, sub := range m.subscriptions { + if sub.isCanceled() { + numCancellations++ + } + } + return numCancellations +} + // -------------------------------------------------------------------------------------- type trackSubscription struct { @@ -867,39 +1208,21 @@ type trackSubscription struct { trackID livekit.TrackID logger logger.Logger - lock sync.RWMutex - desired bool - publisherID livekit.ParticipantID - publisherIdentity livekit.ParticipantIdentity - settings *livekit.UpdateTrackSettings - changedNotifier types.ChangeNotifier - removedNotifier types.ChangeNotifier - hasPermissionInitialized bool - hasPermission bool - subscribedTrack types.SubscribedTrack - eventSent atomic.Bool - numAttempts atomic.Int32 - bound bool - kind atomic.Pointer[livekit.TrackType] + lock sync.RWMutex + desired bool + publisherID livekit.ParticipantID + publisherIdentity livekit.ParticipantIdentity + changedNotifier types.ChangeNotifier + removedNotifier types.ChangeNotifier + + numAttempts atomic.Int32 // the later of when subscription was requested OR when the first failure was encountered OR when permission is granted // this timestamp determines when failures are reported subStartedAt atomic.Pointer[time.Time] // the timestamp when the subscription was started, will be reset when downtrack is closed with expected resume - subscribeAt atomic.Pointer[time.Time] - succRecordCounter atomic.Int32 -} - -func newTrackSubscription(subscriberID livekit.ParticipantID, trackID livekit.TrackID, l logger.Logger) *trackSubscription { - s := &trackSubscription{ - subscriberID: subscriberID, - trackID: trackID, - logger: l, - } - t := time.Now() - s.subscribeAt.Store(&t) - return s + subscribeAt atomic.Pointer[time.Time] } func (s *trackSubscription) setPublisher(publisherIdentity livekit.ParticipantIdentity, publisherID livekit.ParticipantID) { @@ -944,70 +1267,35 @@ func (s *trackSubscription) setDesired(desired bool) bool { return true } -// set permission and return true if it has changed -func (s *trackSubscription) setHasPermission(perm bool) bool { - s.lock.Lock() - defer s.lock.Unlock() - if s.hasPermissionInitialized && s.hasPermission == perm { - return false - } - - s.hasPermissionInitialized = true - s.hasPermission = perm - if s.hasPermission { - // when permission is granted, reset the timer so it has sufficient time to reconcile - t := time.Now() - s.subStartedAt.Store(&t) - s.subscribeAt.Store(&t) - } - return true -} - -func (s *trackSubscription) getHasPermission() bool { - s.lock.RLock() - defer s.lock.RUnlock() - return s.hasPermission -} - func (s *trackSubscription) isDesired() bool { s.lock.RLock() defer s.lock.RUnlock() return s.desired } -func (s *trackSubscription) setSubscribedTrack(track types.SubscribedTrack) { - s.lock.Lock() - oldTrack := s.subscribedTrack - s.subscribedTrack = track - s.bound = false - settings := s.settings - s.lock.Unlock() - - if settings != nil && track != nil { - s.logger.Debugw("restoring subscriber settings", "settings", logger.Proto(settings)) - track.UpdateSubscriberSettings(settings, true) - } - if oldTrack != nil { - oldTrack.OnClose(nil) +func (s *trackSubscription) recordAttempt(success bool) { + if !success { + if s.numAttempts.Load() == 0 { + // on first failure, we'd want to start the timer + t := time.Now() + s.subStartedAt.Store(&t) + } + s.numAttempts.Add(1) + } else { + s.numAttempts.Store(0) } } -func (s *trackSubscription) trySetKind(kind livekit.TrackType) { - s.kind.CompareAndSwap(nil, &kind) +func (s *trackSubscription) getNumAttempts() int32 { + return s.numAttempts.Load() } -func (s *trackSubscription) getKind() (livekit.TrackType, bool) { - kind := s.kind.Load() - if kind == nil { - return livekit.TrackType_AUDIO, false +func (s *trackSubscription) durationSinceStart() time.Duration { + t := s.subStartedAt.Load() + if t == nil { + return 0 } - return *kind, true -} - -func (s *trackSubscription) getSubscribedTrack() types.SubscribedTrack { - s.lock.RLock() - defer s.lock.RUnlock() - return s.subscribedTrack + return time.Since(*t) } func (s *trackSubscription) setChangedNotifier(notifier types.ChangeNotifier) bool { @@ -1050,7 +1338,108 @@ func (s *trackSubscription) setRemovedNotifierLocked(notifier types.ChangeNotifi return true } -func (s *trackSubscription) setSettings(settings *livekit.UpdateTrackSettings) { +func (s *trackSubscription) handleSourceTrackRemoved() { + s.lock.Lock() + defer s.lock.Unlock() + + // source track removed, we would unsubscribe + s.logger.Debugw("unsubscribing from track since source track was removed") + s.desired = false + + s.setChangedNotifierLocked(nil) + s.setRemovedNotifierLocked(nil) +} + +// -------------------------------------------------------------------------------------- + +type mediaTrackSubscription struct { + trackSubscription + + settings *livekit.UpdateTrackSettings + hasPermissionInitialized bool + hasPermission bool + subscribedTrack types.SubscribedTrack + eventSent atomic.Bool + bound bool + kind atomic.Pointer[livekit.TrackType] + + succRecordCounter atomic.Int32 +} + +func newMediaTrackSubscription(subscriberID livekit.ParticipantID, trackID livekit.TrackID, l logger.Logger) *mediaTrackSubscription { + s := &mediaTrackSubscription{ + trackSubscription: trackSubscription{ + subscriberID: subscriberID, + trackID: trackID, + logger: l, + }, + } + t := time.Now() + s.subscribeAt.Store(&t) + return s +} + +// set permission and return true if it has changed +func (s *mediaTrackSubscription) setHasPermission(perm bool) bool { + s.lock.Lock() + defer s.lock.Unlock() + if s.hasPermissionInitialized && s.hasPermission == perm { + return false + } + + s.hasPermissionInitialized = true + s.hasPermission = perm + if s.hasPermission { + // when permission is granted, reset the timer so it has sufficient time to reconcile + t := time.Now() + s.subStartedAt.Store(&t) + s.subscribeAt.Store(&t) + } + return true +} + +func (s *mediaTrackSubscription) getHasPermission() bool { + s.lock.RLock() + defer s.lock.RUnlock() + return s.hasPermission +} + +func (s *mediaTrackSubscription) setSubscribedTrack(track types.SubscribedTrack) { + s.lock.Lock() + oldTrack := s.subscribedTrack + s.subscribedTrack = track + s.bound = false + settings := s.settings + s.lock.Unlock() + + if settings != nil && track != nil { + s.logger.Debugw("restoring subscriber settings", "settings", logger.Proto(settings)) + track.UpdateSubscriberSettings(settings, true) + } + if oldTrack != nil { + oldTrack.OnClose(nil) + } +} + +func (s *mediaTrackSubscription) getSubscribedTrack() types.SubscribedTrack { + s.lock.RLock() + defer s.lock.RUnlock() + return s.subscribedTrack +} + +func (s *mediaTrackSubscription) trySetKind(kind livekit.TrackType) { + s.kind.CompareAndSwap(nil, &kind) +} + +func (s *mediaTrackSubscription) getKind() (livekit.TrackType, bool) { + kind := s.kind.Load() + if kind == nil { + return livekit.TrackType_AUDIO, false + } + return *kind, true +} + +func (s *mediaTrackSubscription) setSettings(settings *livekit.UpdateTrackSettings) { s.lock.Lock() s.settings = settings subTrack := s.subscribedTrack @@ -1061,65 +1450,27 @@ func (s *trackSubscription) setSettings(settings *livekit.UpdateTrackSettings) { } // mark the subscription as bound - when we've received the client's answer -func (s *trackSubscription) setBound() { +func (s *mediaTrackSubscription) setBound() { s.lock.Lock() defer s.lock.Unlock() s.bound = true } -func (s *trackSubscription) isBound() bool { +func (s *mediaTrackSubscription) isBound() bool { s.lock.RLock() defer s.lock.RUnlock() return s.bound } -func (s *trackSubscription) recordAttempt(success bool) { - if !success { - if s.numAttempts.Load() == 0 { - // on first failure, we'd want to start the timer - t := time.Now() - s.subStartedAt.Store(&t) - } - s.numAttempts.Add(1) - } else { - s.numAttempts.Store(0) - } -} - -func (s *trackSubscription) getNumAttempts() int32 { - return s.numAttempts.Load() -} - -func (s *trackSubscription) handleSourceTrackRemoved() { - s.lock.Lock() - defer s.lock.Unlock() - startedAt := s.subStartedAt.Load() - if startedAt == nil || time.Since(*startedAt) < trackRemoveGracePeriod { - // to prevent race conditions, if we've recently been asked to subscribe to a track - // ignore when source was removed. reconciler will take care of it eventually - // this would address the case when a track was unpublished and republished immediately - // it's possible for another caller to call setDesired(true) for the republished track before - // handleSourceTrackRemoved is called on the previously unpublished track - return - } - - // source track removed, we would unsubscribe - s.logger.Debugw("unsubscribing from track since source track was removed") - s.desired = false - - s.setChangedNotifierLocked(nil) - s.setRemovedNotifierLocked(nil) -} - -func (s *trackSubscription) maybeRecordError(ts telemetry.TelemetryService, pID livekit.ParticipantID, err error, isUserError bool) { +func (s *mediaTrackSubscription) maybeRecordError(tl types.ParticipantTelemetryListener, err error, isUserError bool) { if s.eventSent.Swap(true) { return } - ts.TrackSubscribeFailed(context.Background(), pID, s.trackID, err, isUserError) + tl.OnTrackSubscribeFailed(s.subscriberID, s.trackID, err, isUserError) } -func (s *trackSubscription) maybeRecordSuccess(ts telemetry.TelemetryService, pID livekit.ParticipantID) { +func (s *mediaTrackSubscription) maybeRecordSuccess(tl types.ParticipantTelemetryListener) { subTrack := s.getSubscribedTrack() if subTrack == nil { return @@ -1132,7 +1483,15 @@ func (s *trackSubscription) maybeRecordSuccess(ts telemetry.TelemetryService, pI d := time.Since(*s.subscribeAt.Load()) s.logger.Debugw("track subscribed", "cost", d.Milliseconds()) subscriber := subTrack.Subscriber() - prometheus.RecordSubscribeTime(mediaTrack.Source(), mediaTrack.Kind(), d, subscriber.GetClientInfo().GetSdk(), subscriber.Kind(), int(s.succRecordCounter.Inc())) + prometheus.RecordSubscribeTime( + subscriber.GetCountry(), + mediaTrack.Source(), + mediaTrack.Kind(), + d, + subscriber.GetClientInfo().GetSdk(), + subscriber.Kind(), + int(s.succRecordCounter.Inc()), + ) eventSent := s.eventSent.Swap(true) @@ -1140,37 +1499,104 @@ func (s *trackSubscription) maybeRecordSuccess(ts telemetry.TelemetryService, pI Identity: string(subTrack.PublisherIdentity()), Sid: string(subTrack.PublisherID()), } - ts.TrackSubscribed(context.Background(), pID, mediaTrack.ToProto(), pi, !eventSent) + tl.OnTrackSubscribed(s.subscriberID, mediaTrack.ToProto(), pi, !eventSent) } -func (s *trackSubscription) durationSinceStart() time.Duration { - t := s.subStartedAt.Load() - if t == nil { - return 0 - } - return time.Since(*t) +func (s *mediaTrackSubscription) isCanceled() bool { + return !s.eventSent.Load() && s.isDesired() } -func (s *trackSubscription) needsSubscribe() bool { +func (s *mediaTrackSubscription) needsSubscribe() bool { s.lock.RLock() defer s.lock.RUnlock() return s.desired && s.subscribedTrack == nil } -func (s *trackSubscription) needsUnsubscribe() bool { +func (s *mediaTrackSubscription) needsUnsubscribe() bool { s.lock.RLock() defer s.lock.RUnlock() return !s.desired && s.subscribedTrack != nil } -func (s *trackSubscription) needsBind() bool { +func (s *mediaTrackSubscription) needsBind() bool { s.lock.RLock() defer s.lock.RUnlock() return s.desired && s.subscribedTrack != nil && !s.bound } -func (s *trackSubscription) needsCleanup() bool { +func (s *mediaTrackSubscription) needsCleanup() bool { s.lock.RLock() defer s.lock.RUnlock() return !s.desired && s.subscribedTrack == nil } + +// ----------------------------------------------------------------- + +type dataTrackSubscription struct { + trackSubscription + + subscriptionOptions *livekit.DataTrackSubscriptionOptions + + dataDownTrack types.DataDownTrack +} + +func newDataTrackSubscription(subscriberID livekit.ParticipantID, trackID livekit.TrackID, l logger.Logger) *dataTrackSubscription { + s := &dataTrackSubscription{ + trackSubscription: trackSubscription{ + subscriberID: subscriberID, + trackID: trackID, + logger: l, + }, + } + t := time.Now() + s.subscribeAt.Store(&t) + return s +} + +func (s *dataTrackSubscription) needsSubscribe() bool { + s.lock.RLock() + defer s.lock.RUnlock() + return s.desired && s.dataDownTrack == nil +} + +func (s *dataTrackSubscription) needsUnsubscribe() bool { + s.lock.RLock() + defer s.lock.RUnlock() + return !s.desired && s.dataDownTrack != nil +} + +func (s *dataTrackSubscription) needsCleanup() bool { + s.lock.RLock() + defer s.lock.RUnlock() + return !s.desired && s.dataDownTrack == nil +} + +func (s *dataTrackSubscription) setDataDownTrack(dataDownTrack types.DataDownTrack) { + s.lock.Lock() + s.dataDownTrack = dataDownTrack + subscriptionOptions := s.subscriptionOptions + s.lock.Unlock() + + if dataDownTrack != nil { + s.logger.Debugw("restoring data track subscription options", "subscriptionOptions", logger.Proto(subscriptionOptions)) + dataDownTrack.UpdateSubscriptionOptions(subscriptionOptions) + } + + // DT-TODO - DataTrack close callback on previous if not nil?, see setSubscribedTrack for example +} + +func (s *dataTrackSubscription) getDataDownTrack() types.DataDownTrack { + s.lock.RLock() + defer s.lock.RUnlock() + return s.dataDownTrack +} + +func (s *dataTrackSubscription) setSubscriptionOptions(subscriptionOptions *livekit.DataTrackSubscriptionOptions) { + s.lock.Lock() + s.subscriptionOptions = subscriptionOptions + dataDownTrack := s.dataDownTrack + s.lock.Unlock() + if dataDownTrack != nil { + dataDownTrack.UpdateSubscriptionOptions(subscriptionOptions) + } +} diff --git a/pkg/rtc/subscriptionmanager_test.go b/pkg/rtc/subscriptionmanager_test.go index a8dfe70..bfbb5bf 100644 --- a/pkg/rtc/subscriptionmanager_test.go +++ b/pkg/rtc/subscriptionmanager_test.go @@ -26,7 +26,6 @@ import ( "github.com/livekit/livekit-server/pkg/rtc/types" "github.com/livekit/livekit-server/pkg/rtc/types/typesfakes" - "github.com/livekit/livekit-server/pkg/telemetry/telemetryfakes" "github.com/livekit/livekit-server/pkg/utils" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" @@ -67,7 +66,7 @@ func TestSubscribe(t *testing.T) { } }) - sm.SubscribeToTrack("track") + sm.SubscribeToTrack("track", false) s := sm.subscriptions["track"] require.True(t, s.isDesired()) require.Eventually(t, func() bool { @@ -83,8 +82,8 @@ func TestSubscribe(t *testing.T) { require.Equal(t, "pubID", string(sm.GetSubscribedParticipants()[0])) // ensure telemetry events are sent - tm := sm.params.Telemetry.(*telemetryfakes.FakeTelemetryService) - require.Equal(t, 1, tm.TrackSubscribeRequestedCallCount()) + tl := sm.params.TelemetryListener.(*typesfakes.FakeParticipantTelemetryListener) + require.Equal(t, 1, tl.OnTrackSubscribeRequestedCallCount()) // ensure bound setTestSubscribedTrackBound(t, s.getSubscribedTrack()) @@ -93,7 +92,7 @@ func TestSubscribe(t *testing.T) { }, subSettleTimeout, subCheckInterval, "track was not bound") // telemetry event should have been sent - require.Equal(t, 1, tm.TrackSubscribedCallCount()) + require.Equal(t, 1, tl.OnTrackSubscribedCallCount()) time.Sleep(notFoundTimeout) require.False(t, failed.Load()) @@ -127,7 +126,7 @@ func TestSubscribe(t *testing.T) { failed.Store(true) } - sm.SubscribeToTrack("track") + sm.SubscribeToTrack("track", false) s := sm.subscriptions["track"] require.Eventually(t, func() bool { return !s.getHasPermission() @@ -142,9 +141,9 @@ func TestSubscribe(t *testing.T) { require.Len(t, sm.GetSubscribedTracks(), 0) // trackSubscribed telemetry not sent - tm := sm.params.Telemetry.(*telemetryfakes.FakeTelemetryService) - require.Equal(t, 1, tm.TrackSubscribeRequestedCallCount()) - require.Equal(t, 0, tm.TrackSubscribedCallCount()) + tl := sm.params.TelemetryListener.(*typesfakes.FakeParticipantTelemetryListener) + require.Equal(t, 1, tl.OnTrackSubscribeRequestedCallCount()) + require.Equal(t, 0, tl.OnTrackSubscribedCallCount()) // give permissions now resolver.lock.Lock() @@ -168,7 +167,7 @@ func TestSubscribe(t *testing.T) { failed.Store(true) } - sm.SubscribeToTrack("track") + sm.SubscribeToTrack("track", false) s := sm.subscriptions["track"] require.Eventually(t, func() bool { return !s.needsSubscribe() @@ -197,15 +196,17 @@ func TestUnsubscribe(t *testing.T) { resolver := newTestResolver(true, true, "pub", "pubID") - s := &trackSubscription{ - trackID: "track", - desired: true, - subscriberID: sm.params.Participant.ID(), - publisherID: "pubID", - publisherIdentity: "pub", - hasPermission: true, - bound: true, - logger: logger.GetLogger(), + s := &mediaTrackSubscription{ + trackSubscription: trackSubscription{ + trackID: "track", + desired: true, + subscriberID: sm.params.Participant.ID(), + publisherID: "pubID", + publisherIdentity: "pub", + logger: logger.GetLogger(), + }, + hasPermission: true, + bound: true, } // a bunch of unfortunate manual wiring res := resolver.Resolve(nil, s.trackID) @@ -242,18 +243,15 @@ func TestUnsubscribe(t *testing.T) { sm.lock.RLock() subLen := len(sm.subscriptions) sm.lock.RUnlock() - if subLen != 0 { - return false - } - return true + return subLen == 0 }, subSettleTimeout, subCheckInterval, "Track was not unsubscribed") // no traces should be left require.Len(t, sm.GetSubscribedTracks(), 0) require.False(t, res.TrackChangedNotifier.HasObservers()) - tm := sm.params.Telemetry.(*telemetryfakes.FakeTelemetryService) - require.Equal(t, 1, tm.TrackUnsubscribedCallCount()) + tl := sm.params.TelemetryListener.(*typesfakes.FakeParticipantTelemetryListener) + require.Equal(t, 1, tl.OnTrackUnsubscribedCallCount()) } func TestSubscribeStatusChanged(t *testing.T) { @@ -271,8 +269,8 @@ func TestSubscribeStatusChanged(t *testing.T) { } }) - sm.SubscribeToTrack("track1") - sm.SubscribeToTrack("track2") + sm.SubscribeToTrack("track1", false) + sm.SubscribeToTrack("track2", false) s1 := sm.subscriptions["track1"] s2 := sm.subscriptions["track2"] require.Eventually(t, func() bool { @@ -332,7 +330,7 @@ func TestUpdateSettingsBeforeSubscription(t *testing.T) { } sm.UpdateSubscribedTrackSettings("track", settings) - sm.SubscribeToTrack("track") + sm.SubscribeToTrack("track", false) s := sm.subscriptions["track"] require.Eventually(t, func() bool { @@ -376,7 +374,7 @@ func TestSubscriptionLimits(t *testing.T) { } }) - sm.SubscribeToTrack("track") + sm.SubscribeToTrack("track", false) s := sm.subscriptions["track"] require.True(t, s.isDesired()) require.Eventually(t, func() bool { @@ -392,8 +390,8 @@ func TestSubscriptionLimits(t *testing.T) { require.Equal(t, "pubID", string(sm.GetSubscribedParticipants()[0])) // ensure telemetry events are sent - tm := sm.params.Telemetry.(*telemetryfakes.FakeTelemetryService) - require.Equal(t, 1, tm.TrackSubscribeRequestedCallCount()) + tl := sm.params.TelemetryListener.(*typesfakes.FakeParticipantTelemetryListener) + require.Equal(t, 1, tl.OnTrackSubscribeRequestedCallCount()) // ensure bound setTestSubscribedTrackBound(t, s.getSubscribedTrack()) @@ -402,15 +400,15 @@ func TestSubscriptionLimits(t *testing.T) { }, subSettleTimeout, subCheckInterval, "track was not bound") // telemetry event should have been sent - require.Equal(t, 1, tm.TrackSubscribedCallCount()) + require.Equal(t, 1, tl.OnTrackSubscribedCallCount()) // reach subscription limit, subscribe pending - sm.SubscribeToTrack("track2") + sm.SubscribeToTrack("track2", false) s2 := sm.subscriptions["track2"] time.Sleep(subscriptionTimeout * 2) require.True(t, s2.needsSubscribe()) - require.Equal(t, 2, tm.TrackSubscribeRequestedCallCount()) - require.Equal(t, 1, tm.TrackSubscribeFailedCallCount()) + require.Equal(t, 2, tl.OnTrackSubscribeRequestedCallCount()) + require.Equal(t, 1, tl.OnTrackSubscribeFailedCallCount()) require.Len(t, sm.GetSubscribedTracks(), 1) // unsubscribe track1, then track2 should be subscribed @@ -427,7 +425,7 @@ func TestSubscriptionLimits(t *testing.T) { require.False(t, s2.needsSubscribe()) require.EqualValues(t, 2, subCount.Load()) require.NotNil(t, s2.getSubscribedTrack()) - require.Equal(t, 2, tm.TrackSubscribeRequestedCallCount()) + require.Equal(t, 2, tl.OnTrackSubscribeRequestedCallCount()) require.Len(t, sm.GetSubscribedTracks(), 1) // ensure bound @@ -437,13 +435,13 @@ func TestSubscriptionLimits(t *testing.T) { }, subSettleTimeout, subCheckInterval, "track was not bound") // subscribe to track1 again, which should pending - sm.SubscribeToTrack("track") + sm.SubscribeToTrack("track", false) s = sm.subscriptions["track"] require.True(t, s.isDesired()) time.Sleep(subscriptionTimeout * 2) require.True(t, s.needsSubscribe()) - require.Equal(t, 3, tm.TrackSubscribeRequestedCallCount()) - require.Equal(t, 2, tm.TrackSubscribeFailedCallCount()) + require.Equal(t, 3, tl.OnTrackSubscribeRequestedCallCount()) + require.Equal(t, 2, tl.OnTrackSubscribeFailedCallCount()) require.Len(t, sm.GetSubscribedTracks(), 1) } @@ -471,7 +469,7 @@ func newTestSubscriptionManagerWithParams(params testSubscriptionParams) *Subscr TrackResolver: func(sub types.LocalParticipant, trackID livekit.TrackID) types.MediaResolverResult { return types.MediaResolverResult{} }, - Telemetry: &telemetryfakes.FakeTelemetryService{}, + TelemetryListener: &typesfakes.FakeParticipantTelemetryListener{}, SubscriptionLimitAudio: params.SubscriptionLimitAudio, SubscriptionLimitVideo: params.SubscriptionLimitVideo, }) diff --git a/pkg/rtc/testutils.go b/pkg/rtc/testutils.go index 8231f02..0855821 100644 --- a/pkg/rtc/testutils.go +++ b/pkg/rtc/testutils.go @@ -17,6 +17,7 @@ package rtc import ( "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" + "github.com/livekit/protocol/observability/roomobs" "github.com/livekit/protocol/utils" "github.com/livekit/protocol/utils/guid" @@ -24,7 +25,13 @@ import ( "github.com/livekit/livekit-server/pkg/rtc/types/typesfakes" ) -func NewMockParticipant(identity livekit.ParticipantIdentity, protocol types.ProtocolVersion, hidden bool, publisher bool) *typesfakes.FakeLocalParticipant { +func NewMockParticipant( + identity livekit.ParticipantIdentity, + protocol types.ProtocolVersion, + hidden bool, + publisher bool, + participantListener types.LocalParticipantListener, +) *typesfakes.FakeLocalParticipant { p := &typesfakes.FakeLocalParticipant{} sid := guid.New(utils.ParticipantPrefix) p.IDReturns(livekit.ParticipantID(sid)) @@ -49,25 +56,13 @@ func NewMockParticipant(identity livekit.ParticipantIdentity, protocol types.Pro }, utils.TimedVersion(0)) p.SetMetadataCalls(func(m string) { - var f func(participant types.LocalParticipant) - if p.OnParticipantUpdateCallCount() > 0 { - f = p.OnParticipantUpdateArgsForCall(p.OnParticipantUpdateCallCount() - 1) - } - if f != nil { - f(p) - } + participantListener.OnParticipantUpdate(p) }) updateTrack := func() { - var f func(participant types.LocalParticipant, track types.MediaTrack) - if p.OnTrackUpdatedCallCount() > 0 { - f = p.OnTrackUpdatedArgsForCall(p.OnTrackUpdatedCallCount() - 1) - } - if f != nil { - f(p, NewMockTrack(livekit.TrackType_VIDEO, "testcam")) - } + participantListener.OnTrackUpdated(p, NewMockTrack(livekit.TrackType_VIDEO, "testcam")) } - p.SetTrackMutedCalls(func(sid livekit.TrackID, muted bool, fromServer bool) *livekit.TrackInfo { + p.SetTrackMutedCalls(func(mute *livekit.MuteTrackRequest, fromServer bool) *livekit.TrackInfo { updateTrack() return nil }) @@ -75,6 +70,7 @@ func NewMockParticipant(identity livekit.ParticipantIdentity, protocol types.Pro updateTrack() }) p.GetLoggerReturns(logger.GetLogger()) + p.GetReporterReturns(roomobs.NewNoopParticipantSessionReporter()) return p } diff --git a/pkg/rtc/transport.go b/pkg/rtc/transport.go index 89bac0e..8f6a5e5 100644 --- a/pkg/rtc/transport.go +++ b/pkg/rtc/transport.go @@ -15,9 +15,13 @@ package rtc import ( + "context" "fmt" "io" + "maps" + "math/rand" "net" + "slices" "strconv" "strings" "sync" @@ -35,16 +39,17 @@ import ( "github.com/pkg/errors" "go.uber.org/atomic" "go.uber.org/zap/zapcore" + "google.golang.org/protobuf/proto" "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/livekit-server/pkg/rtc/transport" "github.com/livekit/livekit-server/pkg/rtc/types" + "github.com/livekit/livekit-server/pkg/sfu/buffer" "github.com/livekit/livekit-server/pkg/sfu/bwe" "github.com/livekit/livekit-server/pkg/sfu/bwe/remotebwe" "github.com/livekit/livekit-server/pkg/sfu/bwe/sendsidebwe" "github.com/livekit/livekit-server/pkg/sfu/datachannel" sfuinterceptor "github.com/livekit/livekit-server/pkg/sfu/interceptor" - "github.com/livekit/livekit-server/pkg/sfu/mime" "github.com/livekit/livekit-server/pkg/sfu/pacer" pd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/playoutdelay" "github.com/livekit/livekit-server/pkg/sfu/streamallocator" @@ -52,21 +57,26 @@ import ( "github.com/livekit/livekit-server/pkg/telemetry/prometheus" "github.com/livekit/livekit-server/pkg/utils" lkinterceptor "github.com/livekit/mediatransportutil/pkg/interceptor" + "github.com/livekit/mediatransportutil/pkg/rtcconfig" lktwcc "github.com/livekit/mediatransportutil/pkg/twcc" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" "github.com/livekit/protocol/logger/pionlogger" lksdp "github.com/livekit/protocol/sdp" + "github.com/livekit/protocol/utils/mono" ) const ( - LossyDataChannel = "_lossy" - ReliableDataChannel = "_reliable" + LossyDataChannel = "_lossy" + ReliableDataChannel = "_reliable" + DataTrackDataChannel = "_data_track" fastNegotiationFrequency = 10 * time.Millisecond negotiationFrequency = 150 * time.Millisecond negotiationFailedTimeout = 15 * time.Second dtlsRetransmissionInterval = 100 * time.Millisecond + dtlsHandshakeTimeout = time.Minute iceDisconnectedTimeout = 10 * time.Second // compatible for ice-lite with firefox client iceFailedTimeout = 5 * time.Second // time between disconnected and failed @@ -81,18 +91,24 @@ const ( shortConnectionThreshold = 90 * time.Second - dataChannelBufferSize = 65535 + dataChannelBufferSize = 65535 + lossyDataChannelMinBufferedAmount = 8 * 1024 ) var ( - ErrNoICETransport = errors.New("no ICE transport") - ErrIceRestartWithoutLocalSDP = errors.New("ICE restart without local SDP settled") - ErrIceRestartOnClosedPeerConnection = errors.New("ICE restart on closed peer connection") - ErrNoTransceiver = errors.New("no transceiver") - ErrNoSender = errors.New("no sender") - ErrMidNotFound = errors.New("mid not found") - ErrNotSynchronousPeerConnectionMode = errors.New("not using synchronous peer connection mode") - ErrNoRemoteDescription = errors.New("no remote description") + ErrNoICETransport = errors.New("no ICE transport") + ErrIceRestartWithoutLocalSDP = errors.New("ICE restart without local SDP settled") + ErrIceRestartOnClosedPeerConnection = errors.New("ICE restart on closed peer connection") + ErrNoTransceiver = errors.New("no transceiver") + ErrNoSender = errors.New("no sender") + ErrMidNotFound = errors.New("mid not found") + ErrNotSynchronousLocalCandidatesMode = errors.New("not using synchronous local candidates mode") + ErrNoRemoteDescription = errors.New("no remote description") + ErrNoLocalDescription = errors.New("no local description") + ErrInvalidSDPFragment = errors.New("invalid sdp fragment") + ErrNoBundleMid = errors.New("could not get bundle mid") + ErrMidMismatch = errors.New("media mid does not match bundle mid") + ErrICECredentialMismatch = errors.New("ice credential mismatch") ) // ------------------------------------------------------------------------- @@ -132,7 +148,7 @@ func (s signal) String() string { type event struct { *PCTransport signal signal - data interface{} + data any } func (e event) String() string { @@ -153,16 +169,16 @@ func (w wrappedICECandidatePairLogger) MarshalLogObject(e zapcore.ObjectEncoder) if w.pair.Local != nil { e.AddString("localProtocol", w.pair.Local.Protocol.String()) e.AddString("localCandidateType", w.pair.Local.Typ.String()) - e.AddString("localAdddress", w.pair.Local.Address) + e.AddString("localAddress", w.pair.Local.Address) e.AddUint16("localPort", w.pair.Local.Port) } if w.pair.Remote != nil { e.AddString("remoteProtocol", w.pair.Remote.Protocol.String()) e.AddString("remoteCandidateType", w.pair.Remote.Typ.String()) - e.AddString("remoteAdddress", MaybeTruncateIP(w.pair.Remote.Address)) + e.AddString("remoteAddress", MaybeTruncateIP(w.pair.Remote.Address)) e.AddUint16("remotePort", w.pair.Remote.Port) if w.pair.Remote.RelatedAddress != "" { - e.AddString("relatedAdddress", MaybeTruncateIP(w.pair.Remote.RelatedAddress)) + e.AddString("relatedAddress", MaybeTruncateIP(w.pair.Remote.RelatedAddress)) e.AddUint16("relatedPort", w.pair.Remote.RelatedPort) } } @@ -171,16 +187,24 @@ func (w wrappedICECandidatePairLogger) MarshalLogObject(e zapcore.ObjectEncoder) // ------------------------------------------------------------------- -type SimulcastTrackInfo struct { - Mid string - Rid string -} - type trackDescription struct { mid string sender *webrtc.RTPSender } +func (t trackDescription) MarshalLogObject(e zapcore.ObjectEncoder) error { + e.AddString("mid", t.mid) + if t.sender != nil { + track := t.sender.Track() + if track != nil { + e.AddString("trackID", track.ID()) + } + } + return nil +} + +// ------------------------------------------------------------------- + // PCTransport is a wrapper around PeerConnection, with some helper methods type PCTransport struct { params TransportParams @@ -196,6 +220,8 @@ type PCTransport struct { reliableDCOpened bool lossyDC *datachannel.DataChannelWriter[*webrtc.DataChannel] lossyDCOpened bool + dataTrackDC *datachannel.DataChannelWriter[*webrtc.DataChannel] + unlabeledDataChannels []*datachannel.DataChannelWriter[*webrtc.DataChannel] iceStartedAt time.Time iceConnectedAt time.Time @@ -212,6 +238,8 @@ type PCTransport struct { onNegotiationStateChanged func(state transport.NegotiationState) + rtxInfoExtractorFactory *sfuinterceptor.RTXInfoExtractorFactory + // stream allocator for subscriber PC streamAllocator *streamallocator.StreamAllocator @@ -219,6 +247,18 @@ type PCTransport struct { bwe bwe.BWE pacer pacer.Pacer + // transceivers (senders) waiting for SetRemoteDescription (offer) to happen before + // SetCodecPreferences can be invoked on them. + // Pion adapts codecs/payload types from remote description. + // If SetCodecPreferences are done before the remote description is processed, + // it is possible that the transceiver gets payload types from media engine. + // Subssequently if the peer sends an offer with different payload type for the + // same codec, there could be two payload types for the same codec and the wrong + // one could be used in the forwarding path. So, wait for `SetRemoteDescription` + // to happen so that remote side payload types are adapted. + sendersPendingConfigMu sync.Mutex + sendersPendingConfig []configureSenderParams + previousAnswer *webrtc.SessionDescription // track id -> description map in previous offer sdp previousTrackDescription map[string]*trackDescription @@ -227,8 +267,26 @@ type PCTransport struct { preferTCP atomic.Bool isClosed atomic.Bool + // used to check for offer/answer pairing, + // i. e. every offer should have an answer before another offer can be sent + localOfferId atomic.Uint32 + remoteAnswerId atomic.Uint32 + + remoteOfferId atomic.Uint32 + localAnswerId atomic.Uint32 + eventsQueue *utils.TypedOpsQueue[event] + connectionDetails *types.ICEConnectionDetails + selectedPair atomic.Pointer[webrtc.ICECandidatePair] + mayFailedICEStats []iceCandidatePairStats + mayFailedICEStatsTimer *time.Timer + + numOutstandingAudios uint32 + numRequestSentAudios uint32 + numOutstandingVideos uint32 + numRequestSentVideos uint32 + // the following should be accessed only in event processing go routine cacheLocalCandidates bool cachedLocalCandidates []*webrtc.ICECandidate @@ -240,40 +298,40 @@ type PCTransport struct { signalStateCheckTimer *time.Timer currentOfferIceCredential string // ice user:pwd, for publish side ice restart checking pendingRestartIceOffer *webrtc.SessionDescription - - connectionDetails *types.ICEConnectionDetails - selectedPair atomic.Pointer[webrtc.ICECandidatePair] - mayFailedICEStats []iceCandidatePairStats - mayFailedICEStatsTimer *time.Timer } type TransportParams struct { - Handler transport.Handler - ParticipantID livekit.ParticipantID - ParticipantIdentity livekit.ParticipantIdentity - ProtocolVersion types.ProtocolVersion - Config *WebRTCConfig - Twcc *lktwcc.Responder - DirectionConfig DirectionConfig - CongestionControlConfig config.CongestionControlConfig - EnabledCodecs []*livekit.Codec - Logger logger.Logger - Transport livekit.SignalTarget - SimTracks map[uint32]SimulcastTrackInfo - ClientInfo ClientInfo - IsOfferer bool - IsSendSide bool - AllowPlayoutDelay bool - UseOneShotSignallingMode bool - FireOnTrackBySdp bool - DataChannelMaxBufferedAmount uint64 - DatachannelSlowThreshold int + Handler transport.Handler + ProtocolVersion types.ProtocolVersion + Config *WebRTCConfig + Twcc *lktwcc.Responder + DirectionConfig DirectionConfig + CongestionControlConfig config.CongestionControlConfig + EnabledPublishCodecs []*livekit.Codec + EnabledSubscribeCodecs []*livekit.Codec + Logger logger.Logger + Transport livekit.SignalTarget + SimTracks map[uint32]sfuinterceptor.SimulcastTrackInfo + ClientInfo ClientInfo + IsOfferer bool + IsSendSide bool + AllowPlayoutDelay bool + UseOneShotSignallingMode bool + FireOnTrackBySdp bool + DataChannelMaxBufferedAmount uint64 + DatachannelSlowThreshold int + DatachannelLossyTargetLatency time.Duration // for development test DatachannelMaxReceiverBufferSize int + + EnableDataTracks bool } -func newPeerConnection(params TransportParams, onBandwidthEstimator func(estimator cc.BandwidthEstimator)) (*webrtc.PeerConnection, *webrtc.MediaEngine, error) { +func newPeerConnection( + params TransportParams, + onBandwidthEstimator func(estimator cc.BandwidthEstimator), +) (*webrtc.PeerConnection, *webrtc.MediaEngine, *sfuinterceptor.RTXInfoExtractorFactory, error) { directionConfig := params.DirectionConfig if params.AllowPlayoutDelay { directionConfig.RTPHeaderExtension.Video = append(directionConfig.RTPHeaderExtension.Video, pd.PlayoutDelayURI) @@ -282,13 +340,20 @@ func newPeerConnection(params TransportParams, onBandwidthEstimator func(estimat // Some of the browser clients do not handle H.264 High Profile in signalling properly. // They still decode if the actual stream is H.264 High Profile, but do not handle it well in signalling. // So, disable H.264 High Profile for SUBSCRIBER peer connection to ensure it is not offered. - me, err := createMediaEngine(params.EnabledCodecs, directionConfig, params.IsOfferer) + // + // Single-PC mode registers the union of publish and subscribe codecs so subscriptions + // can negotiate subscribe-only codecs; per-direction filtering happens at the transceiver + // level (configureSenderCodecs, restrictReceiverCodecsToPublishList). + me, err := createMediaEngine(mergeCodecsByMime(params.EnabledPublishCodecs, params.EnabledSubscribeCodecs), directionConfig, params.IsOfferer) if err != nil { - return nil, nil, err + return nil, nil, nil, err } se := params.Config.SettingEngine se.DisableMediaEngineCopy(true) + // simulcast layer disable/enable signalled via signalling channel, + // so disable rid pause in SDP + se.SetIgnoreRidPauseForRecv(true) // Change elliptic curve to improve connectivity // https://github.com/pion/dtls/pull/474 @@ -309,7 +374,7 @@ func newPeerConnection(params TransportParams, onBandwidthEstimator func(estimat se.SetFireOnTrackBeforeFirstRTP(true) } - if params.ClientInfo.SupportSctpZeroChecksum() { + if params.ClientInfo.SupportsSctpZeroChecksum() { se.EnableSCTPZeroChecksum(true) } @@ -332,7 +397,7 @@ func newPeerConnection(params TransportParams, onBandwidthEstimator func(estimat // se.DisableSRTPReplayProtection(true) se.DisableSRTCPReplayProtection(true) - if !params.ProtocolVersion.SupportsICELite() || !params.ClientInfo.SupportPrflxOverRelay() { + if !params.ProtocolVersion.SupportsICELite() || !params.ClientInfo.SupportsPrflxOverRelay() { // if client don't support prflx over relay which is only Firefox, disable ICE Lite to ensure that // aggressive nomination is handled properly. Firefox does aggressive nomination even if peer is // ICE Lite (see comment as to historical reasons: https://github.com/pion/ice/pull/739#issuecomment-2452245066). @@ -343,10 +408,13 @@ func newPeerConnection(params TransportParams, onBandwidthEstimator func(estimat se.SetLite(false) } se.SetDTLSRetransmissionInterval(dtlsRetransmissionInterval) + se.SetDTLSConnectContextMaker(func() (context.Context, func()) { + return context.WithTimeout(context.Background(), dtlsHandshakeTimeout) + }) se.SetICETimeouts(iceDisconnectedTimeout, iceFailedTimeout, iceKeepaliveInterval) // if client don't support prflx over relay, we should not expose private address to it, use single external ip as host candidate - if !params.ClientInfo.SupportPrflxOverRelay() && len(params.Config.NAT1To1IPs) > 0 { + if !params.ClientInfo.SupportsPrflxOverRelay() && len(params.Config.NAT1To1IPs) > 0 { var nat1to1Ips []string var includeIps []string for _, mapping := range params.Config.NAT1To1IPs { @@ -359,26 +427,20 @@ func newPeerConnection(params TransportParams, onBandwidthEstimator func(estimat } if len(nat1to1Ips) > 0 { params.Logger.Infow("client doesn't support prflx over relay, use external ip only as host candidate", "ips", nat1to1Ips) - se.SetNAT1To1IPs(nat1to1Ips, webrtc.ICECandidateTypeHost) + if err := rtcconfig.SetNAT1To1AddressRewriteRules(&se, nat1to1Ips, false); err != nil { + params.Logger.Warnw("failed to set ICE address rewrite rules", err, "ips", nat1to1Ips) + } se.SetIPFilter(func(ip net.IP) bool { if ip.To4() == nil { return true } ipstr := ip.String() - for _, inc := range includeIps { - if inc == ipstr { - return true - } - } - return false + return slices.Contains(includeIps, ipstr) }) } } - lf := pionlogger.NewLoggerFactory(params.Logger) - if lf != nil { - se.LoggerFactory = lf - } + se.LoggerFactory = pionlogger.NewLoggerFactory(params.Logger) ir := &interceptor.Registry{} if params.IsSendSide { @@ -404,13 +466,14 @@ func newPeerConnection(params TransportParams, onBandwidthEstimator func(estimat } } } - } else { + } + if !params.IsOfferer { // sfu only use interceptor to send XR but don't read response from it (use buffer instead), // so use a empty callback here ir.Add(lkinterceptor.NewRTTFromXRFactory(func(rtt uint32) {})) } if len(params.SimTracks) > 0 { - f, err := NewUnhandleSimulcastInterceptorFactory(UnhandleSimulcastTracks(params.SimTracks)) + f, err := sfuinterceptor.NewUnhandleSimulcastInterceptorFactory(sfuinterceptor.UnhandleSimulcastTracks(params.Logger, params.SimTracks)) if err != nil { params.Logger.Warnw("NewUnhandleSimulcastInterceptorFactory failed", err) } else { @@ -439,25 +502,36 @@ func newPeerConnection(params TransportParams, onBandwidthEstimator func(estimat twccExtID := sfuutils.GetHeaderExtensionID(info.RTPHeaderExtensions, webrtc.RTPHeaderExtensionCapability{URI: sdp.TransportCCURI}) if twccExtID != 0 { if buffer := params.Config.BufferFactory.GetBuffer(info.SSRC); buffer != nil { - params.Logger.Debugw("set rtx twcc and ext id", "ssrc", info.SSRC, "twccExtID", twccExtID) + params.Logger.Debugw( + "set twcc and ext id", + "ssrc", info.SSRC, + "isRTX", mime.GetMimeTypeCodec(info.MimeType) == mime.MimeTypeCodecRTX, + "twccExtID", twccExtID, + ) buffer.SetTWCCAndExtID(params.Twcc, uint8(twccExtID)) } else { - params.Logger.Warnw("failed to get buffer for rtx stream", nil, "ssrc", info.SSRC) + params.Logger.Warnw("failed to get buffer for stream", nil, "ssrc", info.SSRC) } } } + rtxInfoExtractorFactory := sfuinterceptor.NewRTXInfoExtractorFactory( + setTWCCForVideo, + func(repair, base uint32, rsid string) { + params.Logger.Debugw("rtx pair found from extension", "repair", repair, "base", base, "rsid", rsid) + params.Config.BufferFactory.SetRTXPair(repair, base, rsid) + }, + params.Logger, + ) // put rtx interceptor behind unhandle simulcast interceptor so it can get the correct mid & rid - ir.Add(sfuinterceptor.NewRTXInfoExtractorFactory(setTWCCForVideo, func(repair, base uint32) { - params.Logger.Debugw("rtx pair found from extension", "repair", repair, "base", base) - params.Config.BufferFactory.SetRTXPair(repair, base) - }, params.Logger)) + ir.Add(rtxInfoExtractorFactory) + api := webrtc.NewAPI( webrtc.WithMediaEngine(me), webrtc.WithSettingEngine(se), webrtc.WithInterceptorRegistry(ir), ) pc, err := api.NewPeerConnection(params.Config.Configuration) - return pc, me, err + return pc, me, rtxInfoExtractorFactory, err } func NewPCTransport(params TransportParams) (*PCTransport, error) { @@ -478,6 +552,7 @@ func NewPCTransport(params TransportParams) (*PCTransport, error) { connectionDetails: types.NewICEConnectionDetails(params.Transport, params.Logger), lastNegotiate: time.Now(), } + t.localOfferId.Store(uint32(rand.Intn(1<<8) + 1)) bwe, err := t.createPeerConnection() if err != nil { @@ -486,12 +561,19 @@ func NewPCTransport(params TransportParams) (*PCTransport, error) { if params.IsSendSide { if params.CongestionControlConfig.UseSendSideBWE { - params.Logger.Infow("using send side BWE") + params.Logger.Infow("using send side BWE", "pacerBehavior", params.CongestionControlConfig.SendSideBWEPacer) t.bwe = sendsidebwe.NewSendSideBWE(sendsidebwe.SendSideBWEParams{ Config: params.CongestionControlConfig.SendSideBWE, Logger: params.Logger, }) - t.pacer = pacer.NewNoQueue(params.Logger, t.bwe) + switch pacer.PacerBehavior(params.CongestionControlConfig.SendSideBWEPacer) { + case pacer.PacerBehaviorPassThrough: + t.pacer = pacer.NewPassThrough(params.Logger, t.bwe) + case pacer.PacerBehaviorNoQueue: + t.pacer = pacer.NewNoQueue(params.Logger, t.bwe) + default: + t.pacer = pacer.NewNoQueue(params.Logger, t.bwe) + } } else { t.bwe = remotebwe.NewRemoteBWE(remotebwe.RemoteBWEParams{ Config: params.CongestionControlConfig.RemoteBWE, @@ -522,7 +604,7 @@ func NewPCTransport(params TransportParams) (*PCTransport, error) { func (t *PCTransport) createPeerConnection() (cc.BandwidthEstimator, error) { var bwe cc.BandwidthEstimator - pc, me, err := newPeerConnection(t.params, func(estimator cc.BandwidthEstimator) { + pc, me, rtxInfoExtractorFactory, err := newPeerConnection(t.params, func(estimator cc.BandwidthEstimator) { bwe = estimator }) if err != nil { @@ -559,9 +641,15 @@ func (t *PCTransport) createPeerConnection() (cc.BandwidthEstimator, error) { }) t.me = me + + t.rtxInfoExtractorFactory = rtxInfoExtractorFactory return bwe, nil } +func (t *PCTransport) RTPStreamPublished(ssrc uint32, mid, rid string) { + t.rtxInfoExtractorFactory.SetStreamInfo(ssrc, mid, rid, "") +} + func (t *PCTransport) GetPacer() pacer.Pacer { return t.pacer } @@ -613,13 +701,7 @@ func (t *PCTransport) setICEConnectedAt(at time.Time) { // set failure timer for dtls handshake iceDuration := at.Sub(t.iceStartedAt) - connTimeoutAfterICE := minConnectTimeoutAfterICE - if connTimeoutAfterICE < 3*iceDuration { - connTimeoutAfterICE = 3 * iceDuration - } - if connTimeoutAfterICE > maxConnectTimeoutAfterICE { - connTimeoutAfterICE = maxConnectTimeoutAfterICE - } + connTimeoutAfterICE := min(max(minConnectTimeoutAfterICE, 3*iceDuration), maxConnectTimeoutAfterICE) t.params.Logger.Debugw("setting connection timer after ICE connected", "timeout", connTimeoutAfterICE, "iceDuration", iceDuration) t.connectAfterICETimer = time.AfterFunc(connTimeoutAfterICE, func() { state := t.pc.ConnectionState() @@ -646,6 +728,10 @@ func (t *PCTransport) setICEConnectedAt(at time.Time) { } func (t *PCTransport) logMayFailedICEStats() { + if t.pc.ConnectionState() == webrtc.PeerConnectionStateClosed { + return + } + var candidatePairStats []webrtc.ICECandidatePairStats pairStats := t.pc.GetStats() candidateStats := make(map[string]webrtc.ICECandidateStats) @@ -714,7 +800,7 @@ func (t *PCTransport) setConnectedAt(at time.Time) bool { } t.firstConnectedAt = at - prometheus.ServiceOperationCounter.WithLabelValues("peer_connection", "success", "").Add(1) + prometheus.RecordServiceOperationSuccess("peer_connection") t.lock.Unlock() return true } @@ -782,6 +868,8 @@ func (t *PCTransport) onDataChannel(dc *webrtc.DataChannel) { dc.OnOpen(func() { t.params.Logger.Debugw(dc.Label() + " data channel open") var kind livekit.DataPacket_Kind + var isDataTrack bool + var isUnlabeled bool switch dc.Label() { case ReliableDataChannel: kind = livekit.DataPacket_RELIABLE @@ -789,9 +877,12 @@ func (t *PCTransport) onDataChannel(dc *webrtc.DataChannel) { case LossyDataChannel: kind = livekit.DataPacket_LOSSY + case DataTrackDataChannel: + isDataTrack = true + default: - t.params.Logger.Warnw("unsupported datachannel added", nil, "label", dc.Label()) - return + t.params.Logger.Infow("unlabeled datachannel added", "label", dc.Label()) + isUnlabeled = true } rawDC, err := dc.DetachWithDeadline() @@ -800,24 +891,45 @@ func (t *PCTransport) onDataChannel(dc *webrtc.DataChannel) { return } - switch kind { - case livekit.DataPacket_RELIABLE: - t.lock.Lock() + isHandled := true + t.lock.Lock() + switch { + case isUnlabeled: + t.unlabeledDataChannels = append( + t.unlabeledDataChannels, + datachannel.NewDataChannelWriterReliable(dc, rawDC, t.params.DatachannelSlowThreshold), + ) + + case isDataTrack: + if !t.params.EnableDataTracks { + t.params.Logger.Debugw("data tracks not enabled") + isHandled = false + } else { + if t.dataTrackDC != nil { + t.dataTrackDC.Close() + } + t.dataTrackDC = datachannel.NewDataChannelWriterUnreliable(dc, rawDC, 0, 0) + } + + case kind == livekit.DataPacket_RELIABLE: if t.reliableDC != nil { t.reliableDC.Close() } - t.reliableDC = datachannel.NewDataChannelWriter(dc, rawDC, t.params.DatachannelSlowThreshold) + t.reliableDC = datachannel.NewDataChannelWriterReliable(dc, rawDC, t.params.DatachannelSlowThreshold) t.reliableDCOpened = true - t.lock.Unlock() - case livekit.DataPacket_LOSSY: - t.lock.Lock() + case kind == livekit.DataPacket_LOSSY: if t.lossyDC != nil { t.lossyDC.Close() } - t.lossyDC = datachannel.NewDataChannelWriter(dc, rawDC, 0) + t.lossyDC = datachannel.NewDataChannelWriterUnreliable(dc, rawDC, t.params.DatachannelLossyTargetLatency, uint64(lossyDataChannelMinBufferedAmount)) t.lossyDCOpened = true - t.lock.Unlock() + } + t.lock.Unlock() + + if !isHandled { + rawDC.Close() + return } go func() { @@ -832,7 +944,16 @@ func (t *PCTransport) onDataChannel(dc *webrtc.DataChannel) { return } - t.params.Handler.OnDataPacket(kind, buffer[:n]) + switch { + case isUnlabeled: + t.params.Handler.OnDataMessageUnlabeled(buffer[:n]) + + case isDataTrack: + t.params.Handler.OnDataTrackMessage(buffer[:n], mono.UnixNano()) + + default: + t.params.Handler.OnDataMessage(kind, buffer[:n]) + } } }() @@ -850,7 +971,7 @@ func (t *PCTransport) isFullyEstablished() bool { t.lock.RLock() defer t.lock.RUnlock() - dataChannelReady := t.firstOfferNoDataChannel || (t.reliableDCOpened && t.lossyDCOpened) + dataChannelReady := t.params.UseOneShotSignallingMode || t.firstOfferNoDataChannel || (t.reliableDCOpened && t.lossyDCOpened) return dataChannelReady && !t.connectedAt.IsZero() } @@ -866,7 +987,60 @@ func (t *PCTransport) AddICECandidate(candidate webrtc.ICECandidateInit) { }) } -func (t *PCTransport) AddTrack(trackLocal webrtc.TrackLocal, params types.AddTrackParams) (sender *webrtc.RTPSender, transceiver *webrtc.RTPTransceiver, err error) { +func (t *PCTransport) queueOrConfigureSender( + transceiver *webrtc.RTPTransceiver, + enabledCodecs []*livekit.Codec, + rtcpFeedbackConfig RTCPFeedbackConfig, + enableAudioStereo bool, + enableAudioNACK bool, +) { + params := configureSenderParams{ + transceiver, + enabledCodecs, + rtcpFeedbackConfig, + !t.params.IsOfferer, + enableAudioStereo, + enableAudioNACK, + } + if !t.params.IsOfferer { + t.sendersPendingConfigMu.Lock() + t.sendersPendingConfig = append(t.sendersPendingConfig, params) + t.sendersPendingConfigMu.Unlock() + return + } + + configureSender(params) +} + +func (t *PCTransport) processSendersPendingConfig() { + t.sendersPendingConfigMu.Lock() + pending := t.sendersPendingConfig + t.sendersPendingConfig = nil + t.sendersPendingConfigMu.Unlock() + + var unprocessed []configureSenderParams + for _, p := range pending { + if p.transceiver.Mid() == "" { + unprocessed = append(unprocessed, p) + continue + } + + configureSender(p) + } + + if len(unprocessed) != 0 { + t.sendersPendingConfigMu.Lock() + t.sendersPendingConfig = append(t.sendersPendingConfig, unprocessed...) + t.sendersPendingConfigMu.Unlock() + } +} + +func (t *PCTransport) AddTrack( + trackLocal webrtc.TrackLocal, + params types.AddTrackParams, + enabledCodecs []*livekit.Codec, + rtcpFeedbackConfig RTCPFeedbackConfig, +) (sender *webrtc.RTPSender, transceiver *webrtc.RTPTransceiver, err error) { t.lock.Lock() canReuse := t.canReuseTransceiver td, ok := t.previousTrackDescription[trackLocal.ID()] @@ -886,7 +1060,7 @@ func (t *PCTransport) AddTrack(trackLocal webrtc.TrackLocal, params types.AddTra // if never negotiated with client, can't reuse transceiver for track not subscribed before migration if !canReuse { - return t.AddTransceiverFromTrack(trackLocal, params) + return t.AddTransceiverFromTrack(trackLocal, params, enabledCodecs, rtcpFeedbackConfig) } sender, err = t.pc.AddTrack(trackLocal) @@ -894,7 +1068,6 @@ func (t *PCTransport) AddTrack(trackLocal webrtc.TrackLocal, params types.AddTra return } - // as there is no way to get transceiver from sender, search for _, tr := range t.pc.GetTransceivers() { if tr.Sender() == sender { transceiver = tr @@ -907,13 +1080,24 @@ func (t *PCTransport) AddTrack(trackLocal webrtc.TrackLocal, params types.AddTra return } - if trackLocal.Kind() == webrtc.RTPCodecTypeAudio { - configureAudioTransceiver(transceiver, params.Stereo, !params.Red || !t.params.ClientInfo.SupportsAudioRED()) - } + t.queueOrConfigureSender( + transceiver, + enabledCodecs, + rtcpFeedbackConfig, + params.Stereo, + !params.Red || !t.params.ClientInfo.SupportsAudioRED(), + ) + + t.adjustNumOutstandingMedia(transceiver) return } -func (t *PCTransport) AddTransceiverFromTrack(trackLocal webrtc.TrackLocal, params types.AddTrackParams) (sender *webrtc.RTPSender, transceiver *webrtc.RTPTransceiver, err error) { +func (t *PCTransport) AddTransceiverFromTrack( + trackLocal webrtc.TrackLocal, + params types.AddTrackParams, + enabledCodecs []*livekit.Codec, + rtcpFeedbackConfig RTCPFeedbackConfig, +) (sender *webrtc.RTPSender, transceiver *webrtc.RTPTransceiver, err error) { transceiver, err = t.pc.AddTransceiverFromTrack(trackLocal) if err != nil { return @@ -925,27 +1109,78 @@ func (t *PCTransport) AddTransceiverFromTrack(trackLocal webrtc.TrackLocal, para return } - if trackLocal.Kind() == webrtc.RTPCodecTypeAudio { - configureAudioTransceiver(transceiver, params.Stereo, !params.Red || !t.params.ClientInfo.SupportsAudioRED()) - } + t.queueOrConfigureSender( + transceiver, + enabledCodecs, + rtcpFeedbackConfig, + params.Stereo, + !params.Red || !t.params.ClientInfo.SupportsAudioRED(), + ) + t.adjustNumOutstandingMedia(transceiver) return } +func (t *PCTransport) AddTransceiverFromKind( + kind webrtc.RTPCodecType, + init webrtc.RTPTransceiverInit, +) (*webrtc.RTPTransceiver, error) { + return t.pc.AddTransceiverFromKind(kind, init) +} + func (t *PCTransport) RemoveTrack(sender *webrtc.RTPSender) error { return t.pc.RemoveTrack(sender) } +func (t *PCTransport) CurrentLocalDescription() *webrtc.SessionDescription { + cld := t.pc.CurrentLocalDescription() + if cld == nil { + return nil + } + + ld := *cld + return &ld +} + +func (t *PCTransport) CurrentRemoteDescription() *webrtc.SessionDescription { + crd := t.pc.CurrentRemoteDescription() + if crd == nil { + return nil + } + + rd := *crd + return &rd +} + +func (t *PCTransport) PendingRemoteDescription() *webrtc.SessionDescription { + prd := t.pc.PendingRemoteDescription() + if prd == nil { + return nil + } + + rd := *prd + return &rd +} + func (t *PCTransport) GetMid(rtpReceiver *webrtc.RTPReceiver) string { - for _, tr := range t.pc.GetTransceivers() { - if tr.Receiver() == rtpReceiver { - return tr.Mid() - } + tr := rtpReceiver.RTPTransceiver() + if tr != nil { + return tr.Mid() } return "" } +func (t *PCTransport) GetRTPTransceiver(mid string) *webrtc.RTPTransceiver { + for _, tr := range t.pc.GetTransceivers() { + if tr.Mid() == mid { + return tr + } + } + + return nil +} + func (t *PCTransport) GetRTPReceiver(mid string) *webrtc.RTPReceiver { for _, tr := range t.pc.GetTransceivers() { if tr.Mid() == mid { @@ -956,27 +1191,65 @@ func (t *PCTransport) GetRTPReceiver(mid string) *webrtc.RTPReceiver { return nil } +func (t *PCTransport) getNumUnmatchedTransceivers() (uint32, uint32) { + if t.isClosed.Load() || t.pc.ConnectionState() == webrtc.PeerConnectionStateClosed { + return 0, 0 + } + + numAudios := uint32(0) + numVideos := uint32(0) + for _, tr := range t.pc.GetTransceivers() { + if tr.Mid() != "" { + continue + } + + switch tr.Kind() { + case webrtc.RTPCodecTypeAudio: + numAudios++ + + case webrtc.RTPCodecTypeVideo: + numVideos++ + } + } + + return numAudios, numVideos +} + func (t *PCTransport) CreateDataChannel(label string, dci *webrtc.DataChannelInit) error { + if label == DataTrackDataChannel && !t.params.EnableDataTracks { + t.params.Logger.Debugw("data tracks not enabled") + return nil + } + dc, err := t.pc.CreateDataChannel(label, dci) if err != nil { return err } var ( - dcPtr **datachannel.DataChannelWriter[*webrtc.DataChannel] - dcReady *bool + dcPtr **datachannel.DataChannelWriter[*webrtc.DataChannel] + dcReady *bool + isDataTrack bool + isUnlabeled bool + kind livekit.DataPacket_Kind ) switch dc.Label() { default: - // TODO: Appears that it's never called, so not sure what needs to be done here. We just keep the DC open? - // Maybe just add "reliable" parameter instead of checking the label. - t.params.Logger.Warnw("unknown data channel label", nil, "label", dc.Label()) - return nil + isUnlabeled = true + t.params.Logger.Infow("unlabeled datachannel added", "label", dc.Label()) + case ReliableDataChannel: dcPtr = &t.reliableDC dcReady = &t.reliableDCOpened + kind = livekit.DataPacket_RELIABLE + case LossyDataChannel: dcPtr = &t.lossyDC dcReady = &t.lossyDCOpened + kind = livekit.DataPacket_LOSSY + + case DataTrackDataChannel: + dcPtr = &t.dataTrackDC + isDataTrack = true } dc.OnOpen(func() { @@ -987,26 +1260,114 @@ func (t *PCTransport) CreateDataChannel(label string, dci *webrtc.DataChannelIni } var slowThreshold int - if dc.Label() == ReliableDataChannel { + if dc.Label() == ReliableDataChannel || isUnlabeled { slowThreshold = t.params.DatachannelSlowThreshold } t.lock.Lock() - if *dcPtr != nil { - (*dcPtr).Close() + if isUnlabeled { + t.unlabeledDataChannels = append( + t.unlabeledDataChannels, + datachannel.NewDataChannelWriterReliable(dc, rawDC, slowThreshold), + ) + } else { + if *dcPtr != nil { + (*dcPtr).Close() + } + switch { + case dcPtr == &t.reliableDC: + *dcPtr = datachannel.NewDataChannelWriterReliable(dc, rawDC, slowThreshold) + case dcPtr == &t.lossyDC: + *dcPtr = datachannel.NewDataChannelWriterUnreliable(dc, rawDC, t.params.DatachannelLossyTargetLatency, uint64(lossyDataChannelMinBufferedAmount)) + case dcPtr == &t.dataTrackDC: + *dcPtr = datachannel.NewDataChannelWriterUnreliable(dc, rawDC, 0, 0) + } + if dcReady != nil { + *dcReady = true + } } - *dcPtr = datachannel.NewDataChannelWriter(dc, rawDC, slowThreshold) - *dcReady = true t.lock.Unlock() t.params.Logger.Debugw(dc.Label() + " data channel open") + go func() { + defer rawDC.Close() + buffer := make([]byte, dataChannelBufferSize) + for { + n, _, err := rawDC.ReadDataChannel(buffer) + if err != nil { + if !errors.Is(err, io.EOF) && !strings.Contains(err.Error(), "state=Closed") { + t.params.Logger.Warnw("error reading data channel", err, "label", dc.Label()) + } + return + } + + switch { + case isUnlabeled: + t.params.Handler.OnDataMessageUnlabeled(buffer[:n]) + + case isDataTrack: + t.params.Handler.OnDataTrackMessage(buffer[:n], mono.UnixNano()) + + default: + t.params.Handler.OnDataMessage(kind, buffer[:n]) + } + } + }() + t.maybeNotifyFullyEstablished() }) return nil } +// for testing only +func (t *PCTransport) CreateReadableDataChannel(label string, dci *webrtc.DataChannelInit) error { + dc, err := t.pc.CreateDataChannel(label, dci) + if err != nil { + return err + } + + dc.OnOpen(func() { + t.params.Logger.Debugw(dc.Label() + " data channel open") + rawDC, err := dc.DetachWithDeadline() + if err != nil { + t.params.Logger.Errorw("failed to detach data channel", err, "label", dc.Label()) + return + } + + t.lock.Lock() + t.unlabeledDataChannels = append( + t.unlabeledDataChannels, + datachannel.NewDataChannelWriterReliable(dc, rawDC, t.params.DatachannelSlowThreshold), + ) + t.lock.Unlock() + + go func() { + defer rawDC.Close() + buffer := make([]byte, dataChannelBufferSize) + for { + n, _, err := rawDC.ReadDataChannel(buffer) + if err != nil { + if !errors.Is(err, io.EOF) && !strings.Contains(err.Error(), "state=Closed") { + t.params.Logger.Warnw("error reading data channel", err, "label", dc.Label()) + } + return + } + + t.params.Handler.OnDataMessageUnlabeled(buffer[:n]) + } + }() + }) + return nil +} + func (t *PCTransport) CreateDataChannelIfEmpty(dcLabel string, dci *webrtc.DataChannelInit) (label string, id uint16, existing bool, err error) { + if dcLabel == DataTrackDataChannel && !t.params.EnableDataTracks { + t.params.Logger.Debugw("data tracks not enabled") + err = errors.New("data tracks not enabled") + return + } + t.lock.RLock() var dcw *datachannel.DataChannelWriter[*webrtc.DataChannel] switch dcLabel { @@ -1014,6 +1375,8 @@ func (t *PCTransport) CreateDataChannelIfEmpty(dcLabel string, dci *webrtc.DataC dcw = t.reliableDC case LossyDataChannel: dcw = t.lossyDC + case DataTrackDataChannel: + dcw = t.dataTrackDC default: t.params.Logger.Warnw("unknown data channel label", nil, "label", label) err = errors.New("unknown data channel label") @@ -1046,7 +1409,6 @@ func (t *PCTransport) GetRTT() (float64, bool) { return scps.CurrentRoundTripTime, true } -// IsEstablished returns true if the PeerConnection has been established func (t *PCTransport) IsEstablished() bool { return t.pc.ConnectionState() != webrtc.PeerConnectionStateNew } @@ -1062,20 +1424,95 @@ func (t *PCTransport) GetICEConnectionInfo() *types.ICEConnectionInfo { return t.connectionDetails.GetInfo() } +func (t *PCTransport) GetICEConnectionType() types.ICEConnectionType { + return t.connectionDetails.GetConnectionType() +} + func (t *PCTransport) WriteRTCP(pkts []rtcp.Packet) error { return t.pc.WriteRTCP(pkts) } -func (t *PCTransport) SendDataPacket(kind livekit.DataPacket_Kind, encoded []byte) error { +func (t *PCTransport) SendDataMessage(kind livekit.DataPacket_Kind, data []byte) error { + convertFromUserPacket := false var dc *datachannel.DataChannelWriter[*webrtc.DataChannel] t.lock.RLock() - if kind == livekit.DataPacket_RELIABLE { - dc = t.reliableDC + if t.params.UseOneShotSignallingMode { + if len(t.unlabeledDataChannels) > 0 { + // use the first unlabeled to send + dc = t.unlabeledDataChannels[0] + } + convertFromUserPacket = true } else { - dc = t.lossyDC + if kind == livekit.DataPacket_RELIABLE { + dc = t.reliableDC + } else { + dc = t.lossyDC + } } t.lock.RUnlock() + if convertFromUserPacket { + dp := &livekit.DataPacket{} + if err := proto.Unmarshal(data, dp); err != nil { + return err + } + + switch payload := dp.Value.(type) { + case *livekit.DataPacket_User: + return t.sendDataMessage(dc, payload.User.Payload) + default: + return errors.New("cannot forward non user data packet") + } + } + + return t.sendDataMessage(dc, data) +} + +func (t *PCTransport) SendDataMessageUnlabeled(data []byte, useRaw bool, sender livekit.ParticipantIdentity) error { + convertToUserPacket := false + var dc *datachannel.DataChannelWriter[*webrtc.DataChannel] + t.lock.RLock() + if t.params.UseOneShotSignallingMode || useRaw { + if len(t.unlabeledDataChannels) > 0 { + // use the first unlabeled to send + dc = t.unlabeledDataChannels[0] + } + } else { + if t.reliableDC != nil { + dc = t.reliableDC + } else if t.lossyDC != nil { + dc = t.lossyDC + } + + convertToUserPacket = true + } + t.lock.RUnlock() + + if convertToUserPacket { + dpData, err := proto.Marshal(&livekit.DataPacket{ + ParticipantIdentity: string(sender), + Value: &livekit.DataPacket_User{ + User: &livekit.UserPacket{Payload: data}, + }, + }) + if err != nil { + return err + } + return t.sendDataMessage(dc, dpData) + } + + return t.sendDataMessage(dc, data) +} + +func (t *PCTransport) SendDataTrackMessage(data []byte) error { + t.lock.RLock() + dc := t.dataTrackDC + t.lock.RUnlock() + + return t.sendDataMessage(dc, data) +} + +func (t *PCTransport) sendDataMessage(dc *datachannel.DataChannelWriter[*webrtc.DataChannel], data []byte) error { if dc == nil { return ErrDataChannelUnavailable } @@ -1087,8 +1524,7 @@ func (t *PCTransport) SendDataPacket(kind livekit.DataPacket_Kind, encoded []byt if t.params.DatachannelSlowThreshold == 0 && t.params.DataChannelMaxBufferedAmount > 0 && dc.BufferedAmountGetter().BufferedAmount() > t.params.DataChannelMaxBufferedAmount { return ErrDataChannelBufferFull } - _, err := dc.Write(encoded) - + _, err := dc.Write(data) return err } @@ -1097,21 +1533,29 @@ func (t *PCTransport) Close() { return } + if err := t.pc.Close(); err != nil { + t.params.Logger.Warnw("unclean close of peer connection", err) + } + <-t.eventsQueue.Stop() t.clearSignalStateCheckTimer() if t.streamAllocator != nil { t.streamAllocator.Stop() } + if t.pacer != nil { t.pacer.Stop() } - _ = t.pc.Close() - t.clearConnTimer() t.lock.Lock() + if t.mayFailedICEStatsTimer != nil { + t.mayFailedICEStatsTimer.Stop() + t.mayFailedICEStatsTimer = nil + } + if t.reliableDC != nil { t.reliableDC.Close() t.reliableDC = nil @@ -1122,29 +1566,55 @@ func (t *PCTransport) Close() { t.lossyDC = nil } - if t.mayFailedICEStatsTimer != nil { - t.mayFailedICEStatsTimer.Stop() - t.mayFailedICEStatsTimer = nil + if t.dataTrackDC != nil { + t.dataTrackDC.Close() + t.dataTrackDC = nil } + + for _, dc := range t.unlabeledDataChannels { + dc.Close() + } + t.unlabeledDataChannels = nil t.lock.Unlock() + t.outputAndClearICEStats() } func (t *PCTransport) clearConnTimer() { t.lock.Lock() defer t.lock.Unlock() + if t.connectAfterICETimer != nil { t.connectAfterICETimer.Stop() t.connectAfterICETimer = nil } + if t.tcpICETimer != nil { t.tcpICETimer.Stop() t.tcpICETimer = nil } } -func (t *PCTransport) HandleRemoteDescription(sd webrtc.SessionDescription) error { +func (t *PCTransport) HandleRemoteDescription(sd webrtc.SessionDescription, remoteId uint32) error { if t.params.UseOneShotSignallingMode { + if sd.Type == webrtc.SDPTypeOffer { + remoteOfferId := t.remoteOfferId.Load() + if remoteOfferId != 0 && remoteOfferId != t.localAnswerId.Load() { + t.params.Logger.Warnw( + "sdp state: multiple offers without answer", nil, + "remoteOfferId", remoteOfferId, + "localAnswerId", t.localAnswerId.Load(), + "receivedRemoteOfferId", remoteId, + ) + } + t.remoteOfferId.Store(remoteId) + } else { + if remoteId != 0 && remoteId != t.localOfferId.Load() { + t.params.Logger.Warnw("sdp state: answer id mismatch", nil, "expected", t.localOfferId.Load(), "got", remoteId) + } + t.remoteAnswerId.Store(remoteId) + } + // add remote candidates to ICE connection details parsed, err := sd.Unmarshal() if err == nil { @@ -1169,34 +1639,46 @@ func (t *PCTransport) HandleRemoteDescription(sd webrtc.SessionDescription) erro err = t.pc.SetRemoteDescription(sd) if err != nil { t.params.Logger.Errorw("could not set remote description on synchronous mode peer connection", err) + return err } - return err + + rtxRepairs := nonSimulcastRTXRepairsFromSDP(parsed, t.params.Logger) + if len(rtxRepairs) > 0 { + t.params.Logger.Debugw("rtx pairs found from sdp", "ssrcs", rtxRepairs) + for repair, base := range rtxRepairs { + t.params.Config.BufferFactory.SetRTXPair(repair, base, "") + } + } + return nil } t.postEvent(event{ signal: signalRemoteDescriptionReceived, - data: &sd, + data: remoteDescriptionData{ + sessionDescription: &sd, + remoteId: remoteId, + }, }) return nil } -func (t *PCTransport) GetAnswer() (webrtc.SessionDescription, error) { +func (t *PCTransport) GetAnswer() (webrtc.SessionDescription, uint32, error) { if !t.params.UseOneShotSignallingMode { - return webrtc.SessionDescription{}, ErrNotSynchronousPeerConnectionMode + return webrtc.SessionDescription{}, 0, ErrNotSynchronousLocalCandidatesMode } prd := t.pc.PendingRemoteDescription() if prd == nil || prd.Type != webrtc.SDPTypeOffer { - return webrtc.SessionDescription{}, ErrNoRemoteDescription + return webrtc.SessionDescription{}, 0, ErrNoRemoteDescription } answer, err := t.pc.CreateAnswer(nil) if err != nil { - return webrtc.SessionDescription{}, err + return webrtc.SessionDescription{}, 0, err } if err = t.pc.SetLocalDescription(answer); err != nil { - return webrtc.SessionDescription{}, err + return webrtc.SessionDescription{}, 0, err } // wait for gathering to complete to include all candidates in the answer @@ -1225,7 +1707,225 @@ func (t *PCTransport) GetAnswer() (webrtc.SessionDescription, error) { } } - return *cld, nil + answerId := t.remoteOfferId.Load() + t.localAnswerId.Store(answerId) + + return *cld, answerId, nil +} + +func (t *PCTransport) GetICESessionUfrag() (string, error) { + cld := t.pc.CurrentLocalDescription() + if cld == nil { + return "", ErrNoLocalDescription + } + + parsed, err := cld.Unmarshal() + if err != nil { + return "", err + } + + ufrag, _, err := lksdp.ExtractICECredential(parsed) + if err != nil { + return "", err + } + + return ufrag, nil +} + +// Handles SDP Fragment for ICE Trickle in WHIP +func (t *PCTransport) HandleICETrickleSDPFragment(sdpFragment string) error { + if !t.params.UseOneShotSignallingMode { + return ErrNotSynchronousLocalCandidatesMode + } + + parsedFragment := &lksdp.SDPFragment{} + if err := parsedFragment.Unmarshal(sdpFragment); err != nil { + t.params.Logger.Warnw("could not parse SDP fragment", err, "sdpFragment", sdpFragment) + return ErrInvalidSDPFragment + } + + crd := t.pc.CurrentRemoteDescription() + if crd == nil { + t.params.Logger.Warnw("no remote description", nil) + return ErrNoRemoteDescription + } + + parsedRemote, err := crd.Unmarshal() + if err != nil { + t.params.Logger.Warnw("could not parse remote description", err, "offer", crd) + return err + } + + // check if BUNDLE mid matches the "mid" in the SDP fragment + bundleMid, found := lksdp.GetBundleMid(parsedRemote) + if !found { + return ErrNoBundleMid + } + + if parsedFragment.Mid() != bundleMid { + t.params.Logger.Warnw("incorrect mid", nil, "sdpFragment", sdpFragment) + return ErrMidMismatch + } + + fragmentICEUfrag, fragmentICEPwd, err := parsedFragment.ExtractICECredential() + if err != nil { + t.params.Logger.Warnw( + "could not get ICE crendential from fragment", err, + "sdpFragment", sdpFragment, + ) + return ErrInvalidSDPFragment + } + remoteICEUfrag, remoteICEPwd, err := lksdp.ExtractICECredential(parsedRemote) + if err != nil { + t.params.Logger.Warnw("could not get ICE crendential from remote description", err, "sdpFragment", sdpFragment, "remoteDescription", crd) + return err + } + if fragmentICEUfrag != "" && fragmentICEUfrag != remoteICEUfrag { + t.params.Logger.Warnw( + "ice ufrag mismatch", nil, + "remoteICEUfrag", remoteICEUfrag, + "fragmentICEUfrag", fragmentICEUfrag, + "sdpFragment", sdpFragment, + "remoteDescription", crd, + ) + return ErrICECredentialMismatch + } + if fragmentICEPwd != "" && fragmentICEPwd != remoteICEPwd { + t.params.Logger.Warnw( + "ice pwd mismatch", nil, + "remoteICEPwd", remoteICEPwd, + "fragmentICEPwd", fragmentICEPwd, + "sdpFragment", sdpFragment, + "remoteDescription", crd, + ) + return ErrICECredentialMismatch + } + + // add candidates from media description + for _, ic := range parsedFragment.Candidates() { + c, err := ice.UnmarshalCandidate(ic) + if err == nil { + t.connectionDetails.AddRemoteICECandidate(c, false, false, false) + } + + candidate := webrtc.ICECandidateInit{ + Candidate: ic, + } + if err := t.pc.AddICECandidate(candidate); err != nil { + t.params.Logger.Warnw("failed to add ICE candidate", err, "candidate", candidate) + } else { + t.params.Logger.Debugw("added ICE candidate", "candidate", candidate) + } + } + return nil +} + +// Handles SDP Fragment for ICE Restart in WHIP +func (t *PCTransport) HandleICERestartSDPFragment(sdpFragment string) (string, error) { + if !t.params.UseOneShotSignallingMode { + return "", ErrNotSynchronousLocalCandidatesMode + } + + parsedFragment := &lksdp.SDPFragment{} + if err := parsedFragment.Unmarshal(sdpFragment); err != nil { + t.params.Logger.Warnw("could not parse SDP fragment", err, "sdpFragment", sdpFragment) + return "", ErrInvalidSDPFragment + } + + crd := t.pc.CurrentRemoteDescription() + if crd == nil { + t.params.Logger.Warnw("no remote description", nil) + return "", ErrNoRemoteDescription + } + + parsedRemote, err := crd.Unmarshal() + if err != nil { + t.params.Logger.Warnw("could not parse remote description", err, "offer", crd) + return "", err + } + + if err := parsedFragment.PatchICECredentialAndCandidatesIntoSDP(parsedRemote); err != nil { + t.params.Logger.Warnw("could not patch SDP fragment into remote description", err, "offer", crd, "sdpFragment", sdpFragment) + return "", err + } + + bytes, err := parsedRemote.Marshal() + if err != nil { + t.params.Logger.Warnw("could not marshal SDP with patched remote", err) + return "", err + } + sd := webrtc.SessionDescription{ + SDP: string(bytes), + Type: webrtc.SDPTypeOffer, + } + if err := t.pc.SetRemoteDescription(sd); err != nil { + t.params.Logger.Warnw("could not set remote description", err) + return "", err + } + + // clear out connection details on ICE restart and re-populate + t.connectionDetails.Clear() + for _, candidate := range parsedFragment.Candidates() { + c, err := ice.UnmarshalCandidate(candidate) + if err != nil { + continue + } + t.connectionDetails.AddRemoteICECandidate(c, false, false, false) + } + + ans, err := t.pc.CreateAnswer(nil) + if err != nil { + t.params.Logger.Warnw("could not create answer", err) + return "", err + } + + if err = t.pc.SetLocalDescription(ans); err != nil { + t.params.Logger.Warnw("could not set local description", err) + return "", err + } + + // wait for gathering to complete to include all candidates in the answer + <-webrtc.GatheringCompletePromise(t.pc) + + cld := t.pc.CurrentLocalDescription() + + // add local candidates to ICE connection details + parsedAnswer, err := cld.Unmarshal() + if err != nil { + t.params.Logger.Warnw("could not parse local description", err) + return "", err + } + + addLocalICECandidates := func(attrs []sdp.Attribute) { + for _, a := range attrs { + if a.IsICECandidate() { + c, err := ice.UnmarshalCandidate(a.Value) + if err != nil { + continue + } + t.connectionDetails.AddLocalICECandidate(c, false, false) + } + } + } + + addLocalICECandidates(parsedAnswer.Attributes) + for _, m := range parsedAnswer.MediaDescriptions { + addLocalICECandidates(m.Attributes) + } + + parsedFragmentAnswer, err := lksdp.ExtractSDPFragment(parsedAnswer) + if err != nil { + t.params.Logger.Warnw("could not extract SDP fragment", err) + return "", err + } + + answerFragment, err := parsedFragmentAnswer.Marshal() + if err != nil { + t.params.Logger.Warnw("could not marshal answer SDP fragment", err) + return "", err + } + + return answerFragment, nil } func (t *PCTransport) OnNegotiationStateChanged(f func(state transport.NegotiationState)) { @@ -1253,7 +1953,7 @@ func (t *PCTransport) Negotiate(force bool) { // no op to cancel pending negotiation }) t.debouncePending = false - t.updateLastNeogitateLocked() + t.updateLastNegotiateLocked() postEvent = true } else { @@ -1267,7 +1967,7 @@ func (t *PCTransport) Negotiate(force bool) { t.debouncedNegotiate.Add(func() { t.lock.Lock() t.debouncePending = false - t.updateLastNeogitateLocked() + t.updateLastNegotiateLocked() t.lock.Unlock() t.postEvent(event{ @@ -1286,7 +1986,7 @@ func (t *PCTransport) Negotiate(force bool) { } } -func (t *PCTransport) updateLastNeogitateLocked() { +func (t *PCTransport) updateLastNegotiateLocked() { if now := time.Now(); now.After(t.lastNegotiate) { t.lastNegotiate = now } @@ -1313,10 +2013,14 @@ func (t *PCTransport) AddTrackToStreamAllocator(subTrack types.SubscribedTrack) return } + layers := buffer.GetVideoLayersForMimeType( + subTrack.DownTrack().Mime(), + subTrack.MediaTrack().ToProto(), + ) t.streamAllocator.AddTrack(subTrack.DownTrack(), streamallocator.AddTrackParams{ - Source: subTrack.MediaTrack().Source(), - IsSimulcast: subTrack.MediaTrack().IsSimulcast(), - PublisherID: subTrack.MediaTrack().PublisherID(), + Source: subTrack.MediaTrack().Source(), + IsMultiLayered: len(layers) > 1, + PublisherID: subTrack.MediaTrack().PublisherID(), }) } @@ -1372,6 +2076,7 @@ func (t *PCTransport) preparePC(previousAnswer webrtc.SessionDescription) error // se := webrtc.SettingEngine{} _ = se.SetAnsweringDTLSRole(lksdp.ExtractDTLSRole(parsed)) + se.SetIgnoreRidPauseForRecv(true) api := webrtc.NewAPI( webrtc.WithSettingEngine(se), webrtc.WithMediaEngine(t.me), @@ -1392,11 +2097,11 @@ func (t *PCTransport) preparePC(previousAnswer webrtc.SessionDescription) error return err } - // replace client's fingerprint into dump pc's answer, for pion's dtls process, it will + // replace client's fingerprint into dummy pc's answer, for pion's dtls process, it will // keep the fingerprint at first call of SetRemoteDescription, if dummy pc and client pc use // different fingerprint, that will cause pion denied dtls data after handshake with client // complete (can't pass fingerprint change). - // in this step, we don't established connection with dump pc(no candidate swap), just use + // in this step, we don't established connection with dummy pc(no candidate swap), just use // sdp negotiation to sticky data channel and keep client's fingerprint parsedAns, _ := ans.Unmarshal() fpLine := fpHahs + " " + fp @@ -1434,24 +2139,45 @@ func (t *PCTransport) initPCWithPreviousAnswer(previousAnswer webrtc.SessionDesc case "audio": codecType = webrtc.RTPCodecTypeAudio case "application": - // for pion generate unmatched sdp, it always appends data channel to last m-lines, - // that not consistent with our previous answer that data channel might at middle-line - // because sdp can negotiate multi times before migration.(it will sticky to the last m-line atfirst negotiate) - // so use a dummy pc to negotiate sdp to fixed the datachannel's mid at same position with previous answer - if err := t.preparePC(previousAnswer); err != nil { - t.params.Logger.Warnw("prepare pc for migration failed", err) - return senders, err + if t.params.IsOfferer { + // for pion generate unmatched sdp, it always appends data channel to last m-lines, + // that not consistent with our previous answer that data channel might at middle-line + // because sdp can negotiate multi times before migration.(it will sticky to the last m-line at first negotiate) + // so use a dummy pc to negotiate sdp to fixed the datachannel's mid at same position with previous answer + if err := t.preparePC(previousAnswer); err != nil { + t.params.Logger.Warnw("prepare pc for migration failed", err) + return senders, err + } } continue default: continue } - tr, err := t.pc.AddTransceiverFromKind(codecType, webrtc.RTPTransceiverInit{Direction: webrtc.RTPTransceiverDirectionSendonly}) + + if !t.params.IsOfferer { + // `sendrecv` or `sendonly` means this transceiver is used for sending + + // Note that a transceiver previously used to send could be `inactive`. + // Let those transceivers be created when remote description is set. + _, ok1 := m.Attribute(webrtc.RTPTransceiverDirectionSendrecv.String()) + _, ok2 := m.Attribute(webrtc.RTPTransceiverDirectionSendonly.String()) + if !ok1 && !ok2 { + continue + } + } + + tr, err := t.pc.AddTransceiverFromKind( + codecType, + webrtc.RTPTransceiverInit{ + Direction: webrtc.RTPTransceiverDirectionSendonly, + }, + ) if err != nil { return senders, err } mid := lksdp.GetMidValue(m) if mid == "" { + t.params.Logger.Warnw("cannot set up peer connection with previous answer, mid not found", nil, "senders", slices.Collect(maps.Keys(senders))) return senders, ErrMidNotFound } tr.SetMid(mid) @@ -1461,43 +2187,66 @@ func (t *PCTransport) initPCWithPreviousAnswer(previousAnswer webrtc.SessionDesc senders[mid] = sender // set transceiver to inactive - tr.SetSender(tr.Sender(), nil) + tr.SetSender(sender, nil) } + t.params.Logger.Debugw("set up peer connection with previous answer", "senders", slices.Collect(maps.Keys(senders))) return senders, nil } -func (t *PCTransport) SetPreviousSdp(offer, answer *webrtc.SessionDescription) { - // when there is no previous answer, cannot migrate, force a full reconnect - if answer == nil { - t.onNegotiationFailed(true, "no previous answer for previous sdp") +func (t *PCTransport) SetPreviousSdp(localDescription, remoteDescription *webrtc.SessionDescription) { + // when there is no answer, cannot migrate, force a full reconnect + if (t.params.IsOfferer && remoteDescription == nil) || (!t.params.IsOfferer && localDescription == nil) { + t.onNegotiationFailed(true, "no previous answer") return } t.lock.Lock() - if t.pc.RemoteDescription() == nil && t.previousAnswer == nil { - t.previousAnswer = answer - if senders, err := t.initPCWithPreviousAnswer(*t.previousAnswer); err != nil { - t.lock.Unlock() - - t.onNegotiationFailed(true, fmt.Sprintf("initPCWithPreviousAnswer failed, error: %s", err)) - return - } else if offer != nil { - // in migration case, can't reuse transceiver before negotiated except track subscribed at previous node - t.canReuseTransceiver = false - if err := t.parseTrackMid(*offer, senders); err != nil { - t.params.Logger.Warnw("parse previous offer failed", err, "offer", offer.SDP) - } + var ( + senders map[string]*webrtc.RTPSender + err error + parseMids bool + ) + if t.params.IsOfferer { + if t.pc.RemoteDescription() == nil && t.previousAnswer == nil { + t.previousAnswer = remoteDescription + senders, err = t.initPCWithPreviousAnswer(*remoteDescription) + parseMids = true + } + } else { + if t.pc.LocalDescription() == nil { + senders, err = t.initPCWithPreviousAnswer(*localDescription) + parseMids = true } } - // disable fast negotiation temporarily after migration to avoid sending offer - // contains part of subscribed tracks before migration, let the subscribed track - // resume at the same time. - t.lastNegotiate = time.Now().Add(iceFailedTimeoutTotal) + if err != nil { + t.lock.Unlock() + t.onNegotiationFailed(true, fmt.Sprintf("initPCWithPreviousAnswer failed, error: %s", err)) + return + } + + if localDescription != nil && parseMids { + // in migration case, can't reuse transceiver before negotiating expected tracks + // that were subscribed at previous node + t.canReuseTransceiver = false + if err := t.parseTrackMid(*localDescription, senders); err != nil { + t.params.Logger.Warnw( + "parse previous local description failed", err, + "localDescription", localDescription.SDP, + ) + } + } + + if t.params.IsOfferer { + // disable fast negotiation temporarily after migration to avoid sending offer + // contains part of subscribed tracks before migration, let the subscribed track + // resume at the same time. + t.lastNegotiate = time.Now().Add(iceFailedTimeoutTotal) + } t.lock.Unlock() } -func (t *PCTransport) parseTrackMid(offer webrtc.SessionDescription, senders map[string]*webrtc.RTPSender) error { - parsed, err := offer.Unmarshal() +func (t *PCTransport) parseTrackMid(sd webrtc.SessionDescription, senders map[string]*webrtc.RTPSender) error { + parsed, err := sd.Unmarshal() if err != nil { return err } @@ -1515,12 +2264,14 @@ func (t *PCTransport) parseTrackMid(offer webrtc.SessionDescription, senders map if mid == "" { return ErrMidNotFound } - t.previousTrackDescription[trackID] = &trackDescription{ - mid: mid, - sender: senders[mid], + if sender, ok := senders[mid]; ok { + t.previousTrackDescription[trackID] = &trackDescription{mid, sender} } } } + if len(t.previousTrackDescription) != 0 { + t.params.Logger.Debugw("previous track description", t.previousTrackDescription) + } return nil } @@ -1580,6 +2331,8 @@ func (t *PCTransport) handleICEGatheringCompleteAnswerer() error { if err := t.setRemoteDescription(offer); err != nil { return err } + t.params.Handler.OnSetRemoteDescriptionOffer() + t.processSendersPendingConfig() return t.createAndSendAnswer() } @@ -1663,8 +2416,11 @@ func (t *PCTransport) handleRemoteICECandidate(e event) error { } if err := t.pc.AddICECandidate(*c); err != nil { - t.params.Logger.Warnw("failed to add cached ICE candidate", err, "candidate", c) - return errors.Wrap(err, "add ice candidate failed") + t.params.Logger.Warnw("failed to add ICE candidate", err, "candidate", c) + // ignore ParseAddr error as it does not affect ICE connectivity + if !strings.Contains(err.Error(), "ParseAddr") { + return errors.Wrap(err, "add ice candidate failed") + } } else { t.params.Logger.Debugw("added ICE candidate", "candidate", c) } @@ -1755,6 +2511,40 @@ func (t *PCTransport) setupSignalStateCheckTimer() { }) } +func (t *PCTransport) adjustNumOutstandingMedia(transceiver *webrtc.RTPTransceiver) { + if transceiver.Mid() != "" { + return + } + + t.lock.Lock() + if transceiver.Kind() == webrtc.RTPCodecTypeAudio { + t.numOutstandingAudios++ + } else { + t.numOutstandingVideos++ + } + t.lock.Unlock() +} + +func (t *PCTransport) sendUnmatchedMediaRequirement(force bool) error { + // if there are unmatched media sections, notify remote peer to generate offer with + // enough media section in subsequent offers + t.lock.Lock() + numAudios := t.numOutstandingAudios - t.numRequestSentAudios + t.numRequestSentAudios += numAudios + + numVideos := t.numOutstandingVideos - t.numRequestSentVideos + t.numRequestSentVideos += numVideos + t.lock.Unlock() + + if force || (numAudios+numVideos) != 0 { + if err := t.params.Handler.OnUnmatchedMedia(numAudios, numVideos); err != nil { + return errors.Wrap(err, "could not send unmatched media requirements") + } + } + + return nil +} + func (t *PCTransport) createAndSendOffer(options *webrtc.OfferOptions) error { if t.pc.ConnectionState() == webrtc.PeerConnectionStateClosed { t.params.Logger.Warnw("trying to send offer on closed peer connection", nil) @@ -1762,11 +2552,12 @@ func (t *PCTransport) createAndSendOffer(options *webrtc.OfferOptions) error { } // when there's an ongoing negotiation, let it finish and not disrupt its state - if t.negotiationState == transport.NegotiationStateRemote { + switch t.negotiationState { + case transport.NegotiationStateRemote: t.params.Logger.Debugw("skipping negotiation, trying again later") t.setNegotiationState(transport.NegotiationStateRetry) return nil - } else if t.negotiationState == transport.NegotiationStateRetry { + case transport.NegotiationStateRetry: // already set to retry, we can safely skip this attempt return nil } @@ -1804,7 +2595,7 @@ func (t *PCTransport) createAndSendOffer(options *webrtc.OfferOptions) error { return nil } - prometheus.ServiceOperationCounter.WithLabelValues("offer", "error", "create").Add(1) + prometheus.RecordServiceOperationError("offer", "create") return errors.Wrap(err, "create offer failed") } @@ -1820,7 +2611,7 @@ func (t *PCTransport) createAndSendOffer(options *webrtc.OfferOptions) error { return nil } - prometheus.ServiceOperationCounter.WithLabelValues("offer", "error", "local_description").Add(1) + prometheus.RecordServiceOperationError("offer", "local_description") return errors.Wrap(err, "setting local description failed") } @@ -1840,25 +2631,45 @@ func (t *PCTransport) createAndSendOffer(options *webrtc.OfferOptions) error { t.setupSignalStateCheckTimer() - if err := t.params.Handler.OnOffer(offer); err != nil { - prometheus.ServiceOperationCounter.WithLabelValues("offer", "error", "write_message").Add(1) - return errors.Wrap(err, "could not send offer") + remoteAnswerId := t.remoteAnswerId.Load() + if remoteAnswerId != 0 && remoteAnswerId != t.localOfferId.Load() { + if options == nil || !options.ICERestart { + t.params.Logger.Warnw( + "sdp state: sending offer before receiving answer", nil, + "localOfferId", t.localOfferId.Load(), + "remoteAnswerId", remoteAnswerId, + ) + } } - prometheus.ServiceOperationCounter.WithLabelValues("offer", "success", "").Add(1) + if err := t.params.Handler.OnOffer(offer, t.localOfferId.Inc(), t.getMidToTrackIDMapping()); err != nil { + prometheus.RecordServiceOperationError("offer", "write_message") + return errors.Wrap(err, "could not send offer") + } + prometheus.RecordServiceOperationSuccess("offer") + return t.localDescriptionSent() } func (t *PCTransport) handleSendOffer(_ event) error { + if !t.params.IsOfferer { + return t.sendUnmatchedMediaRequirement(true) + } + return t.createAndSendOffer(nil) } +type remoteDescriptionData struct { + sessionDescription *webrtc.SessionDescription + remoteId uint32 +} + func (t *PCTransport) handleRemoteDescriptionReceived(e event) error { - sd := e.data.(*webrtc.SessionDescription) - if sd.Type == webrtc.SDPTypeOffer { - return t.handleRemoteOfferReceived(sd) + rdd := e.data.(remoteDescriptionData) + if rdd.sessionDescription.Type == webrtc.SDPTypeOffer { + return t.handleRemoteOfferReceived(rdd.sessionDescription, rdd.remoteId) } else { - return t.handleRemoteAnswerReceived(sd) + return t.handleRemoteAnswerReceived(rdd.sessionDescription, rdd.remoteId) } } @@ -1895,13 +2706,14 @@ func (t *PCTransport) setRemoteDescription(sd webrtc.SessionDescription) error { if sd.Type == webrtc.SDPTypeAnswer { sdpType = "answer" } - prometheus.ServiceOperationCounter.WithLabelValues(sdpType, "error", "remote_description").Add(1) + prometheus.RecordServiceOperationError(sdpType, "remote_description") return errors.Wrap(err, "setting remote description failed") } else if sd.Type == webrtc.SDPTypeAnswer { t.lock.Lock() if !t.canReuseTransceiver { t.canReuseTransceiver = true t.previousTrackDescription = make(map[string]*trackDescription) + t.params.Logger.Debugw("enabling transceiver reuse") } t.lock.Unlock() } @@ -1920,6 +2732,14 @@ func (t *PCTransport) setRemoteDescription(sd webrtc.SessionDescription) error { } func (t *PCTransport) createAndSendAnswer() error { + numOutstandingAudios, numOutstandingVideos := t.getNumUnmatchedTransceivers() + t.lock.Lock() + t.numOutstandingAudios, t.numOutstandingVideos = numOutstandingAudios, numOutstandingVideos + t.numRequestSentAudios, t.numRequestSentVideos = 0, 0 + t.lock.Unlock() + + t.restrictReceiverCodecsToPublishList() + answer, err := t.pc.CreateAnswer(nil) if err != nil { if errors.Is(err, webrtc.ErrConnectionClosed) { @@ -1927,7 +2747,7 @@ func (t *PCTransport) createAndSendAnswer() error { return nil } - prometheus.ServiceOperationCounter.WithLabelValues("answer", "error", "create").Add(1) + prometheus.RecordServiceOperationError("answer", "create") return errors.Wrap(err, "create answer failed") } @@ -1937,7 +2757,7 @@ func (t *PCTransport) createAndSendAnswer() error { } if err = t.pc.SetLocalDescription(answer); err != nil { - prometheus.ServiceOperationCounter.WithLabelValues("answer", "error", "local_description").Add(1) + prometheus.RecordServiceOperationError("answer", "local_description") return errors.Wrap(err, "setting local description failed") } @@ -1952,19 +2772,55 @@ func (t *PCTransport) createAndSendAnswer() error { t.params.Logger.Debugw("local answer (filtered)", "sdp", answer.SDP) } - if err := t.params.Handler.OnAnswer(answer); err != nil { - prometheus.ServiceOperationCounter.WithLabelValues("answer", "error", "write_message").Add(1) - return errors.Wrap(err, "could not send answer") + localAnswerId := t.localAnswerId.Load() + if localAnswerId != 0 && localAnswerId >= t.remoteOfferId.Load() { + t.params.Logger.Warnw( + "sdp state: duplicate answer", nil, + "localAnswerId", localAnswerId, + "remoteOfferId", t.remoteOfferId.Load(), + ) } - prometheus.ServiceOperationCounter.WithLabelValues("answer", "success", "").Add(1) + answerId := t.remoteOfferId.Load() + + if err := t.params.Handler.OnAnswer(answer, answerId, t.getMidToTrackIDMapping()); err != nil { + prometheus.RecordServiceOperationError("answer", "write_message") + return errors.Wrap(err, "could not send answer") + } + t.localAnswerId.Store(answerId) + prometheus.RecordServiceOperationSuccess("asnwer") + + if err := t.sendUnmatchedMediaRequirement(false); err != nil { + return err + } + + t.lock.Lock() + if !t.canReuseTransceiver { + t.canReuseTransceiver = true + t.previousTrackDescription = make(map[string]*trackDescription) + t.params.Logger.Debugw("enabling transceiver reuse") + } + t.lock.Unlock() + return t.localDescriptionSent() } -func (t *PCTransport) handleRemoteOfferReceived(sd *webrtc.SessionDescription) error { +func (t *PCTransport) handleRemoteOfferReceived(sd *webrtc.SessionDescription, offerId uint32) error { + t.params.Logger.Debugw("processing offer", "offerId", offerId) + remoteOfferId := t.remoteOfferId.Load() + if remoteOfferId != 0 && remoteOfferId != t.localAnswerId.Load() { + t.params.Logger.Warnw( + "sdp state: multiple offers without answer", nil, + "remoteOfferId", remoteOfferId, + "localAnswerId", t.localAnswerId.Load(), + "receivedRemoteOfferId", offerId, + ) + } + t.remoteOfferId.Store(offerId) + parsed, err := sd.Unmarshal() if err != nil { - return nil + return err } t.lock.Lock() @@ -2007,11 +2863,14 @@ func (t *PCTransport) handleRemoteOfferReceived(sd *webrtc.SessionDescription) e if err := t.setRemoteDescription(*sd); err != nil { return err } + t.params.Handler.OnSetRemoteDescriptionOffer() + t.processSendersPendingConfig() + rtxRepairs := nonSimulcastRTXRepairsFromSDP(parsed, t.params.Logger) if len(rtxRepairs) > 0 { t.params.Logger.Debugw("rtx pairs found from sdp", "ssrcs", rtxRepairs) for repair, base := range rtxRepairs { - t.params.Config.BufferFactory.SetRTXPair(repair, base) + t.params.Config.BufferFactory.SetRTXPair(repair, base, "") } } @@ -2022,7 +2881,17 @@ func (t *PCTransport) handleRemoteOfferReceived(sd *webrtc.SessionDescription) e return t.createAndSendAnswer() } -func (t *PCTransport) handleRemoteAnswerReceived(sd *webrtc.SessionDescription) error { +func (t *PCTransport) handleRemoteAnswerReceived(sd *webrtc.SessionDescription, answerId uint32) error { + t.params.Logger.Debugw("processing answer", "answerId", answerId) + if answerId != 0 && answerId != t.localOfferId.Load() { + t.params.Logger.Warnw( + "sdp state: answer id mismatch", nil, + "expected", t.localOfferId.Load(), + "got", answerId, + ) + } + t.remoteAnswerId.Store(answerId) + t.clearSignalStateCheckTimer() if err := t.setRemoteDescription(*sd); err != nil { @@ -2087,11 +2956,21 @@ func (t *PCTransport) doICERestart() error { t.params.Logger.Infow("deferring ice restart to next offer") t.setNegotiationState(transport.NegotiationStateRetry) t.restartAtNextOffer = true - err := t.params.Handler.OnOffer(*offer) + + remoteAnswerId := t.remoteAnswerId.Load() + if remoteAnswerId != 0 && remoteAnswerId != t.localOfferId.Load() { + t.params.Logger.Warnw( + "sdp state: answer not received in ICE restart", nil, + "localOfferId", t.localOfferId.Load(), + "remoteAnswerId", remoteAnswerId, + ) + } + + err := t.params.Handler.OnOffer(*offer, t.localOfferId.Inc(), t.getMidToTrackIDMapping()) if err != nil { - prometheus.ServiceOperationCounter.WithLabelValues("offer", "error", "write_message").Add(1) + prometheus.RecordServiceOperationError("offer", "write_message") } else { - prometheus.ServiceOperationCounter.WithLabelValues("offer", "success", "").Add(1) + prometheus.RecordServiceOperationSuccess("offer") } return err } @@ -2099,7 +2978,7 @@ func (t *PCTransport) doICERestart() error { // recover by re-applying the last answer t.params.Logger.Infow("recovering from client negotiation state on ICE restart") if err := t.pc.SetRemoteDescription(*currentRemoteDescription); err != nil { - prometheus.ServiceOperationCounter.WithLabelValues("offer", "error", "remote_description").Add(1) + prometheus.RecordServiceOperationError("offer", "remote_description") return errors.Wrap(err, "set remote description failed") } else { t.setNegotiationState(transport.NegotiationStateNone) @@ -2114,7 +2993,7 @@ func (t *PCTransport) handleICERestart(_ event) error { } func (t *PCTransport) onNegotiationFailed(warning bool, reason string) { - logFields := []interface{}{ + logFields := []any{ "reason", reason, "localCurrent", t.pc.CurrentLocalDescription(), "localPending", t.pc.PendingLocalDescription(), @@ -2144,16 +3023,54 @@ func (t *PCTransport) outputAndClearICEStats() { } } +func (t *PCTransport) getMidToTrackIDMapping() map[string]string { + transceivers := t.pc.GetTransceivers() + midToTrackID := make(map[string]string, len(transceivers)) + for _, tr := range transceivers { + if mid := tr.Mid(); mid != "" { + if sender := tr.Sender(); sender != nil { + if track := sender.Track(); track != nil { + midToTrackID[mid] = track.ID() + } + } + } + } + return midToTrackID +} + // ---------------------- +type configureSenderParams struct { + transceiver *webrtc.RTPTransceiver + enabledCodecs []*livekit.Codec + rtcpFeedbackConfig RTCPFeedbackConfig + filterOutH264HighProfile bool + enableAudioStereo bool + enableAudioNACK bool +} + +func configureSender(params configureSenderParams) { + configureSenderCodecs( + params.transceiver, + params.enabledCodecs, + params.rtcpFeedbackConfig, + params.filterOutH264HighProfile, + ) + + if params.transceiver.Kind() == webrtc.RTPCodecTypeAudio { + configureSenderAudio(params.transceiver, params.enableAudioStereo, params.enableAudioNACK) + } +} + // configure subscriber transceiver for audio stereo and nack // pion doesn't support per transciver codec configuration, so the nack of this session will be disabled // forever once it is first disabled by a transceiver. -func configureAudioTransceiver(tr *webrtc.RTPTransceiver, stereo bool, nack bool) { +func configureSenderAudio(tr *webrtc.RTPTransceiver, stereo bool, nack bool) { sender := tr.Sender() if sender == nil { return } + // enable stereo codecs := sender.GetParameters().Codecs configCodecs := make([]webrtc.RTPCodecParameters, 0, len(codecs)) @@ -2178,6 +3095,113 @@ func configureAudioTransceiver(tr *webrtc.RTPTransceiver, stereo bool, nack bool tr.SetCodecPreferences(configCodecs) } +// In single peer connection mode, set up enebled codecs for sender. +// The config provides config of direction. +// For publisher peer connection those are publish enabled codecs +// and for subscriber peer connection those are subscribe enabled codecs. +// +// But, in single peer connection mode, if setting up a transceiver where the media is +// flowing in the other direction, the other direction codec config needs to be set. +func configureSenderCodecs( + tr *webrtc.RTPTransceiver, + enabledCodecs []*livekit.Codec, + rtcpFeedbackConfig RTCPFeedbackConfig, + filterOutH264HighProfile bool, +) { + if len(enabledCodecs) == 0 { + return + } + + sender := tr.Sender() + if sender == nil { + return + } + + filteredCodecs := filterCodecs( + sender.GetParameters().Codecs, + enabledCodecs, + rtcpFeedbackConfig, + filterOutH264HighProfile, + ) + tr.SetCodecPreferences(filteredCodecs) +} + +// restrictReceiverCodecsToPublishList narrows recv-side transceiver codec +// preferences to the publish list, so the answer doesn't advertise +// subscribe-only codecs as receivable. No-op in dual-PC mode. +func (t *PCTransport) restrictReceiverCodecsToPublishList() { + for _, tr := range t.pc.GetTransceivers() { + if tr.Direction() != webrtc.RTPTransceiverDirectionRecvonly && + tr.Direction() != webrtc.RTPTransceiverDirectionSendrecv { + continue + } + receiver := tr.Receiver() + if receiver == nil { + continue + } + filtered := filterCodecs( + receiver.GetParameters().Codecs, + t.params.EnabledPublishCodecs, + t.params.DirectionConfig.RTCPFeedback, + false, + ) + if len(filtered) == 0 { + continue + } + if err := tr.SetCodecPreferences(filtered); err != nil { + t.params.Logger.Warnw("failed to set recv codec preferences", err, "mid", tr.Mid()) + } + } +} + +func configureReceiverCodecs( + tr *webrtc.RTPTransceiver, + preferredMimeType string, + compliesWithCodecOrderInSDPAnswer bool, +) { + receiver := tr.Receiver() + if receiver == nil { + return + } + + var preferredCodecs, leftCodecs []webrtc.RTPCodecParameters + for _, c := range receiver.GetParameters().Codecs { + if tr.Kind() == webrtc.RTPCodecTypeAudio { + nackFound := false + for _, fb := range c.RTCPFeedback { + if fb.Type == webrtc.TypeRTCPFBNACK { + nackFound = true + break + } + } + + if !nackFound { + c.RTCPFeedback = append(c.RTCPFeedback, webrtc.RTCPFeedback{Type: webrtc.TypeRTCPFBNACK}) + } + } + + if mime.GetMimeTypeCodec(preferredMimeType) == mime.GetMimeTypeCodec(c.RTPCodecCapability.MimeType) { + preferredCodecs = append(preferredCodecs, c) + } else { + leftCodecs = append(leftCodecs, c) + } + } + if len(preferredCodecs) == 0 { + return + } + + reorderedCodecs := append([]webrtc.RTPCodecParameters{}, preferredCodecs...) + if tr.Kind() == webrtc.RTPCodecTypeVideo { + // if the client don't comply with codec order in SDP answer, only keep preferred codecs to force client to use it + if compliesWithCodecOrderInSDPAnswer { + reorderedCodecs = append(reorderedCodecs, leftCodecs...) + } + } else { + reorderedCodecs = append(reorderedCodecs, leftCodecs...) + } + tr.SetCodecPreferences(reorderedCodecs) +} + func nonSimulcastRTXRepairsFromSDP(s *sdp.SessionDescription, logger logger.Logger) map[uint32]uint32 { rtxRepairFlows := map[uint32]uint32{} for _, media := range s.MediaDescriptions { @@ -2214,9 +3238,7 @@ func nonSimulcastRTXRepairsFromSDP(s *sdp.SessionDescription, logger logger.Logg } } if !ridFound { - for rtx, base := range rtxPairs { - rtxRepairFlows[rtx] = base - } + maps.Copy(rtxRepairFlows, rtxPairs) } } diff --git a/pkg/rtc/transport/handler.go b/pkg/rtc/transport/handler.go index 067fee8..5b6ea75 100644 --- a/pkg/rtc/transport/handler.go +++ b/pkg/rtc/transport/handler.go @@ -39,13 +39,17 @@ type Handler interface { OnFullyEstablished() OnFailed(isShortLived bool, iceConnectionInfo *types.ICEConnectionInfo) OnTrack(track *webrtc.TrackRemote, rtpReceiver *webrtc.RTPReceiver) - OnDataPacket(kind livekit.DataPacket_Kind, data []byte) + OnDataMessage(kind livekit.DataPacket_Kind, data []byte) + OnDataMessageUnlabeled(data []byte) + OnDataTrackMessage(data []byte, arrivalTime int64) OnDataSendError(err error) - OnOffer(sd webrtc.SessionDescription) error - OnAnswer(sd webrtc.SessionDescription) error + OnOffer(sd webrtc.SessionDescription, offerId uint32, midToTrackID map[string]string) error + OnSetRemoteDescriptionOffer() + OnAnswer(sd webrtc.SessionDescription, answerId uint32, midToTrackID map[string]string) error OnNegotiationStateChanged(state NegotiationState) OnNegotiationFailed() OnStreamStateChange(update *streamallocator.StreamStateUpdate) error + OnUnmatchedMedia(numAudios uint32, numVideos uint32) error } type UnimplementedHandler struct{} @@ -57,12 +61,15 @@ func (h UnimplementedHandler) OnInitialConnected() func (h UnimplementedHandler) OnFullyEstablished() {} func (h UnimplementedHandler) OnFailed(isShortLived bool) {} func (h UnimplementedHandler) OnTrack(track *webrtc.TrackRemote, rtpReceiver *webrtc.RTPReceiver) {} -func (h UnimplementedHandler) OnDataPacket(kind livekit.DataPacket_Kind, data []byte) {} +func (h UnimplementedHandler) OnDataMessage(kind livekit.DataPacket_Kind, data []byte) {} +func (h UnimplementedHandler) OnDataMessageUnlabeled(data []byte) {} +func (h UnimplementedHandler) OnDataTrackMessage(data []byte, arrivalTime int64) {} func (h UnimplementedHandler) OnDataSendError(err error) {} -func (h UnimplementedHandler) OnOffer(sd webrtc.SessionDescription) error { +func (h UnimplementedHandler) OnOffer(sd webrtc.SessionDescription, offerId uint32, midToTrackID map[string]string) error { return ErrNoOfferHandler } -func (h UnimplementedHandler) OnAnswer(sd webrtc.SessionDescription) error { +func (h UnimplementedHandler) OnSetRemoteDescriptionOffer() {} +func (h UnimplementedHandler) OnAnswer(sd webrtc.SessionDescription, answerId uint32, midToTrackID map[string]string) error { return ErrNoAnswerHandler } func (h UnimplementedHandler) OnNegotiationStateChanged(state NegotiationState) {} @@ -70,3 +77,6 @@ func (h UnimplementedHandler) OnNegotiationFailed() func (h UnimplementedHandler) OnStreamStateChange(update *streamallocator.StreamStateUpdate) error { return nil } +func (h UnimplementedHandler) OnUnmatchedMedia(numAudios uint32, numVideos uint32) error { + return nil +} diff --git a/pkg/rtc/transport/transportfakes/fake_handler.go b/pkg/rtc/transport/transportfakes/fake_handler.go index a3d243d..5c09af0 100644 --- a/pkg/rtc/transport/transportfakes/fake_handler.go +++ b/pkg/rtc/transport/transportfakes/fake_handler.go @@ -12,10 +12,12 @@ import ( ) type FakeHandler struct { - OnAnswerStub func(webrtc.SessionDescription) error + OnAnswerStub func(webrtc.SessionDescription, uint32, map[string]string) error onAnswerMutex sync.RWMutex onAnswerArgsForCall []struct { arg1 webrtc.SessionDescription + arg2 uint32 + arg3 map[string]string } onAnswerReturns struct { result1 error @@ -23,17 +25,28 @@ type FakeHandler struct { onAnswerReturnsOnCall map[int]struct { result1 error } - OnDataPacketStub func(livekit.DataPacket_Kind, []byte) - onDataPacketMutex sync.RWMutex - onDataPacketArgsForCall []struct { + OnDataMessageStub func(livekit.DataPacket_Kind, []byte) + onDataMessageMutex sync.RWMutex + onDataMessageArgsForCall []struct { arg1 livekit.DataPacket_Kind arg2 []byte } + OnDataMessageUnlabeledStub func([]byte) + onDataMessageUnlabeledMutex sync.RWMutex + onDataMessageUnlabeledArgsForCall []struct { + arg1 []byte + } OnDataSendErrorStub func(error) onDataSendErrorMutex sync.RWMutex onDataSendErrorArgsForCall []struct { arg1 error } + OnDataTrackMessageStub func([]byte, int64) + onDataTrackMessageMutex sync.RWMutex + onDataTrackMessageArgsForCall []struct { + arg1 []byte + arg2 int64 + } OnFailedStub func(bool, *types.ICEConnectionInfo) onFailedMutex sync.RWMutex onFailedArgsForCall []struct { @@ -69,10 +82,12 @@ type FakeHandler struct { onNegotiationStateChangedArgsForCall []struct { arg1 transport.NegotiationState } - OnOfferStub func(webrtc.SessionDescription) error + OnOfferStub func(webrtc.SessionDescription, uint32, map[string]string) error onOfferMutex sync.RWMutex onOfferArgsForCall []struct { arg1 webrtc.SessionDescription + arg2 uint32 + arg3 map[string]string } onOfferReturns struct { result1 error @@ -80,6 +95,10 @@ type FakeHandler struct { onOfferReturnsOnCall map[int]struct { result1 error } + OnSetRemoteDescriptionOfferStub func() + onSetRemoteDescriptionOfferMutex sync.RWMutex + onSetRemoteDescriptionOfferArgsForCall []struct { + } OnStreamStateChangeStub func(*streamallocator.StreamStateUpdate) error onStreamStateChangeMutex sync.RWMutex onStreamStateChangeArgsForCall []struct { @@ -97,22 +116,36 @@ type FakeHandler struct { arg1 *webrtc.TrackRemote arg2 *webrtc.RTPReceiver } + OnUnmatchedMediaStub func(uint32, uint32) error + onUnmatchedMediaMutex sync.RWMutex + onUnmatchedMediaArgsForCall []struct { + arg1 uint32 + arg2 uint32 + } + onUnmatchedMediaReturns struct { + result1 error + } + onUnmatchedMediaReturnsOnCall map[int]struct { + result1 error + } invocations map[string][][]interface{} invocationsMutex sync.RWMutex } -func (fake *FakeHandler) OnAnswer(arg1 webrtc.SessionDescription) error { +func (fake *FakeHandler) OnAnswer(arg1 webrtc.SessionDescription, arg2 uint32, arg3 map[string]string) error { fake.onAnswerMutex.Lock() ret, specificReturn := fake.onAnswerReturnsOnCall[len(fake.onAnswerArgsForCall)] fake.onAnswerArgsForCall = append(fake.onAnswerArgsForCall, struct { arg1 webrtc.SessionDescription - }{arg1}) + arg2 uint32 + arg3 map[string]string + }{arg1, arg2, arg3}) stub := fake.OnAnswerStub fakeReturns := fake.onAnswerReturns - fake.recordInvocation("OnAnswer", []interface{}{arg1}) + fake.recordInvocation("OnAnswer", []interface{}{arg1, arg2, arg3}) fake.onAnswerMutex.Unlock() if stub != nil { - return stub(arg1) + return stub(arg1, arg2, arg3) } if specificReturn { return ret.result1 @@ -126,17 +159,17 @@ func (fake *FakeHandler) OnAnswerCallCount() int { return len(fake.onAnswerArgsForCall) } -func (fake *FakeHandler) OnAnswerCalls(stub func(webrtc.SessionDescription) error) { +func (fake *FakeHandler) OnAnswerCalls(stub func(webrtc.SessionDescription, uint32, map[string]string) error) { fake.onAnswerMutex.Lock() defer fake.onAnswerMutex.Unlock() fake.OnAnswerStub = stub } -func (fake *FakeHandler) OnAnswerArgsForCall(i int) webrtc.SessionDescription { +func (fake *FakeHandler) OnAnswerArgsForCall(i int) (webrtc.SessionDescription, uint32, map[string]string) { fake.onAnswerMutex.RLock() defer fake.onAnswerMutex.RUnlock() argsForCall := fake.onAnswerArgsForCall[i] - return argsForCall.arg1 + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 } func (fake *FakeHandler) OnAnswerReturns(result1 error) { @@ -162,44 +195,81 @@ func (fake *FakeHandler) OnAnswerReturnsOnCall(i int, result1 error) { }{result1} } -func (fake *FakeHandler) OnDataPacket(arg1 livekit.DataPacket_Kind, arg2 []byte) { +func (fake *FakeHandler) OnDataMessage(arg1 livekit.DataPacket_Kind, arg2 []byte) { var arg2Copy []byte if arg2 != nil { arg2Copy = make([]byte, len(arg2)) copy(arg2Copy, arg2) } - fake.onDataPacketMutex.Lock() - fake.onDataPacketArgsForCall = append(fake.onDataPacketArgsForCall, struct { + fake.onDataMessageMutex.Lock() + fake.onDataMessageArgsForCall = append(fake.onDataMessageArgsForCall, struct { arg1 livekit.DataPacket_Kind arg2 []byte }{arg1, arg2Copy}) - stub := fake.OnDataPacketStub - fake.recordInvocation("OnDataPacket", []interface{}{arg1, arg2Copy}) - fake.onDataPacketMutex.Unlock() + stub := fake.OnDataMessageStub + fake.recordInvocation("OnDataMessage", []interface{}{arg1, arg2Copy}) + fake.onDataMessageMutex.Unlock() if stub != nil { - fake.OnDataPacketStub(arg1, arg2) + fake.OnDataMessageStub(arg1, arg2) } } -func (fake *FakeHandler) OnDataPacketCallCount() int { - fake.onDataPacketMutex.RLock() - defer fake.onDataPacketMutex.RUnlock() - return len(fake.onDataPacketArgsForCall) +func (fake *FakeHandler) OnDataMessageCallCount() int { + fake.onDataMessageMutex.RLock() + defer fake.onDataMessageMutex.RUnlock() + return len(fake.onDataMessageArgsForCall) } -func (fake *FakeHandler) OnDataPacketCalls(stub func(livekit.DataPacket_Kind, []byte)) { - fake.onDataPacketMutex.Lock() - defer fake.onDataPacketMutex.Unlock() - fake.OnDataPacketStub = stub +func (fake *FakeHandler) OnDataMessageCalls(stub func(livekit.DataPacket_Kind, []byte)) { + fake.onDataMessageMutex.Lock() + defer fake.onDataMessageMutex.Unlock() + fake.OnDataMessageStub = stub } -func (fake *FakeHandler) OnDataPacketArgsForCall(i int) (livekit.DataPacket_Kind, []byte) { - fake.onDataPacketMutex.RLock() - defer fake.onDataPacketMutex.RUnlock() - argsForCall := fake.onDataPacketArgsForCall[i] +func (fake *FakeHandler) OnDataMessageArgsForCall(i int) (livekit.DataPacket_Kind, []byte) { + fake.onDataMessageMutex.RLock() + defer fake.onDataMessageMutex.RUnlock() + argsForCall := fake.onDataMessageArgsForCall[i] return argsForCall.arg1, argsForCall.arg2 } +func (fake *FakeHandler) OnDataMessageUnlabeled(arg1 []byte) { + var arg1Copy []byte + if arg1 != nil { + arg1Copy = make([]byte, len(arg1)) + copy(arg1Copy, arg1) + } + fake.onDataMessageUnlabeledMutex.Lock() + fake.onDataMessageUnlabeledArgsForCall = append(fake.onDataMessageUnlabeledArgsForCall, struct { + arg1 []byte + }{arg1Copy}) + stub := fake.OnDataMessageUnlabeledStub + fake.recordInvocation("OnDataMessageUnlabeled", []interface{}{arg1Copy}) + fake.onDataMessageUnlabeledMutex.Unlock() + if stub != nil { + fake.OnDataMessageUnlabeledStub(arg1) + } +} + +func (fake *FakeHandler) OnDataMessageUnlabeledCallCount() int { + fake.onDataMessageUnlabeledMutex.RLock() + defer fake.onDataMessageUnlabeledMutex.RUnlock() + return len(fake.onDataMessageUnlabeledArgsForCall) +} + +func (fake *FakeHandler) OnDataMessageUnlabeledCalls(stub func([]byte)) { + fake.onDataMessageUnlabeledMutex.Lock() + defer fake.onDataMessageUnlabeledMutex.Unlock() + fake.OnDataMessageUnlabeledStub = stub +} + +func (fake *FakeHandler) OnDataMessageUnlabeledArgsForCall(i int) []byte { + fake.onDataMessageUnlabeledMutex.RLock() + defer fake.onDataMessageUnlabeledMutex.RUnlock() + argsForCall := fake.onDataMessageUnlabeledArgsForCall[i] + return argsForCall.arg1 +} + func (fake *FakeHandler) OnDataSendError(arg1 error) { fake.onDataSendErrorMutex.Lock() fake.onDataSendErrorArgsForCall = append(fake.onDataSendErrorArgsForCall, struct { @@ -232,6 +302,44 @@ func (fake *FakeHandler) OnDataSendErrorArgsForCall(i int) error { return argsForCall.arg1 } +func (fake *FakeHandler) OnDataTrackMessage(arg1 []byte, arg2 int64) { + var arg1Copy []byte + if arg1 != nil { + arg1Copy = make([]byte, len(arg1)) + copy(arg1Copy, arg1) + } + fake.onDataTrackMessageMutex.Lock() + fake.onDataTrackMessageArgsForCall = append(fake.onDataTrackMessageArgsForCall, struct { + arg1 []byte + arg2 int64 + }{arg1Copy, arg2}) + stub := fake.OnDataTrackMessageStub + fake.recordInvocation("OnDataTrackMessage", []interface{}{arg1Copy, arg2}) + fake.onDataTrackMessageMutex.Unlock() + if stub != nil { + fake.OnDataTrackMessageStub(arg1, arg2) + } +} + +func (fake *FakeHandler) OnDataTrackMessageCallCount() int { + fake.onDataTrackMessageMutex.RLock() + defer fake.onDataTrackMessageMutex.RUnlock() + return len(fake.onDataTrackMessageArgsForCall) +} + +func (fake *FakeHandler) OnDataTrackMessageCalls(stub func([]byte, int64)) { + fake.onDataTrackMessageMutex.Lock() + defer fake.onDataTrackMessageMutex.Unlock() + fake.OnDataTrackMessageStub = stub +} + +func (fake *FakeHandler) OnDataTrackMessageArgsForCall(i int) ([]byte, int64) { + fake.onDataTrackMessageMutex.RLock() + defer fake.onDataTrackMessageMutex.RUnlock() + argsForCall := fake.onDataTrackMessageArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + func (fake *FakeHandler) OnFailed(arg1 bool, arg2 *types.ICEConnectionInfo) { fake.onFailedMutex.Lock() fake.onFailedArgsForCall = append(fake.onFailedArgsForCall, struct { @@ -431,18 +539,20 @@ func (fake *FakeHandler) OnNegotiationStateChangedArgsForCall(i int) transport.N return argsForCall.arg1 } -func (fake *FakeHandler) OnOffer(arg1 webrtc.SessionDescription) error { +func (fake *FakeHandler) OnOffer(arg1 webrtc.SessionDescription, arg2 uint32, arg3 map[string]string) error { fake.onOfferMutex.Lock() ret, specificReturn := fake.onOfferReturnsOnCall[len(fake.onOfferArgsForCall)] fake.onOfferArgsForCall = append(fake.onOfferArgsForCall, struct { arg1 webrtc.SessionDescription - }{arg1}) + arg2 uint32 + arg3 map[string]string + }{arg1, arg2, arg3}) stub := fake.OnOfferStub fakeReturns := fake.onOfferReturns - fake.recordInvocation("OnOffer", []interface{}{arg1}) + fake.recordInvocation("OnOffer", []interface{}{arg1, arg2, arg3}) fake.onOfferMutex.Unlock() if stub != nil { - return stub(arg1) + return stub(arg1, arg2, arg3) } if specificReturn { return ret.result1 @@ -456,17 +566,17 @@ func (fake *FakeHandler) OnOfferCallCount() int { return len(fake.onOfferArgsForCall) } -func (fake *FakeHandler) OnOfferCalls(stub func(webrtc.SessionDescription) error) { +func (fake *FakeHandler) OnOfferCalls(stub func(webrtc.SessionDescription, uint32, map[string]string) error) { fake.onOfferMutex.Lock() defer fake.onOfferMutex.Unlock() fake.OnOfferStub = stub } -func (fake *FakeHandler) OnOfferArgsForCall(i int) webrtc.SessionDescription { +func (fake *FakeHandler) OnOfferArgsForCall(i int) (webrtc.SessionDescription, uint32, map[string]string) { fake.onOfferMutex.RLock() defer fake.onOfferMutex.RUnlock() argsForCall := fake.onOfferArgsForCall[i] - return argsForCall.arg1 + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 } func (fake *FakeHandler) OnOfferReturns(result1 error) { @@ -492,6 +602,30 @@ func (fake *FakeHandler) OnOfferReturnsOnCall(i int, result1 error) { }{result1} } +func (fake *FakeHandler) OnSetRemoteDescriptionOffer() { + fake.onSetRemoteDescriptionOfferMutex.Lock() + fake.onSetRemoteDescriptionOfferArgsForCall = append(fake.onSetRemoteDescriptionOfferArgsForCall, struct { + }{}) + stub := fake.OnSetRemoteDescriptionOfferStub + fake.recordInvocation("OnSetRemoteDescriptionOffer", []interface{}{}) + fake.onSetRemoteDescriptionOfferMutex.Unlock() + if stub != nil { + fake.OnSetRemoteDescriptionOfferStub() + } +} + +func (fake *FakeHandler) OnSetRemoteDescriptionOfferCallCount() int { + fake.onSetRemoteDescriptionOfferMutex.RLock() + defer fake.onSetRemoteDescriptionOfferMutex.RUnlock() + return len(fake.onSetRemoteDescriptionOfferArgsForCall) +} + +func (fake *FakeHandler) OnSetRemoteDescriptionOfferCalls(stub func()) { + fake.onSetRemoteDescriptionOfferMutex.Lock() + defer fake.onSetRemoteDescriptionOfferMutex.Unlock() + fake.OnSetRemoteDescriptionOfferStub = stub +} + func (fake *FakeHandler) OnStreamStateChange(arg1 *streamallocator.StreamStateUpdate) error { fake.onStreamStateChangeMutex.Lock() ret, specificReturn := fake.onStreamStateChangeReturnsOnCall[len(fake.onStreamStateChangeArgsForCall)] @@ -586,33 +720,71 @@ func (fake *FakeHandler) OnTrackArgsForCall(i int) (*webrtc.TrackRemote, *webrtc return argsForCall.arg1, argsForCall.arg2 } +func (fake *FakeHandler) OnUnmatchedMedia(arg1 uint32, arg2 uint32) error { + fake.onUnmatchedMediaMutex.Lock() + ret, specificReturn := fake.onUnmatchedMediaReturnsOnCall[len(fake.onUnmatchedMediaArgsForCall)] + fake.onUnmatchedMediaArgsForCall = append(fake.onUnmatchedMediaArgsForCall, struct { + arg1 uint32 + arg2 uint32 + }{arg1, arg2}) + stub := fake.OnUnmatchedMediaStub + fakeReturns := fake.onUnmatchedMediaReturns + fake.recordInvocation("OnUnmatchedMedia", []interface{}{arg1, arg2}) + fake.onUnmatchedMediaMutex.Unlock() + if stub != nil { + return stub(arg1, arg2) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeHandler) OnUnmatchedMediaCallCount() int { + fake.onUnmatchedMediaMutex.RLock() + defer fake.onUnmatchedMediaMutex.RUnlock() + return len(fake.onUnmatchedMediaArgsForCall) +} + +func (fake *FakeHandler) OnUnmatchedMediaCalls(stub func(uint32, uint32) error) { + fake.onUnmatchedMediaMutex.Lock() + defer fake.onUnmatchedMediaMutex.Unlock() + fake.OnUnmatchedMediaStub = stub +} + +func (fake *FakeHandler) OnUnmatchedMediaArgsForCall(i int) (uint32, uint32) { + fake.onUnmatchedMediaMutex.RLock() + defer fake.onUnmatchedMediaMutex.RUnlock() + argsForCall := fake.onUnmatchedMediaArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeHandler) OnUnmatchedMediaReturns(result1 error) { + fake.onUnmatchedMediaMutex.Lock() + defer fake.onUnmatchedMediaMutex.Unlock() + fake.OnUnmatchedMediaStub = nil + fake.onUnmatchedMediaReturns = struct { + result1 error + }{result1} +} + +func (fake *FakeHandler) OnUnmatchedMediaReturnsOnCall(i int, result1 error) { + fake.onUnmatchedMediaMutex.Lock() + defer fake.onUnmatchedMediaMutex.Unlock() + fake.OnUnmatchedMediaStub = nil + if fake.onUnmatchedMediaReturnsOnCall == nil { + fake.onUnmatchedMediaReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.onUnmatchedMediaReturnsOnCall[i] = struct { + result1 error + }{result1} +} + func (fake *FakeHandler) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() - fake.onAnswerMutex.RLock() - defer fake.onAnswerMutex.RUnlock() - fake.onDataPacketMutex.RLock() - defer fake.onDataPacketMutex.RUnlock() - fake.onDataSendErrorMutex.RLock() - defer fake.onDataSendErrorMutex.RUnlock() - fake.onFailedMutex.RLock() - defer fake.onFailedMutex.RUnlock() - fake.onFullyEstablishedMutex.RLock() - defer fake.onFullyEstablishedMutex.RUnlock() - fake.onICECandidateMutex.RLock() - defer fake.onICECandidateMutex.RUnlock() - fake.onInitialConnectedMutex.RLock() - defer fake.onInitialConnectedMutex.RUnlock() - fake.onNegotiationFailedMutex.RLock() - defer fake.onNegotiationFailedMutex.RUnlock() - fake.onNegotiationStateChangedMutex.RLock() - defer fake.onNegotiationStateChangedMutex.RUnlock() - fake.onOfferMutex.RLock() - defer fake.onOfferMutex.RUnlock() - fake.onStreamStateChangeMutex.RLock() - defer fake.onStreamStateChangeMutex.RUnlock() - fake.onTrackMutex.RLock() - defer fake.onTrackMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} for key, value := range fake.invocations { copiedInvocations[key] = value diff --git a/pkg/rtc/transport_test.go b/pkg/rtc/transport_test.go index ac0c1cc..ab328d7 100644 --- a/pkg/rtc/transport_test.go +++ b/pkg/rtc/transport_test.go @@ -28,17 +28,15 @@ import ( "github.com/livekit/livekit-server/pkg/rtc/transport" "github.com/livekit/livekit-server/pkg/rtc/transport/transportfakes" - "github.com/livekit/livekit-server/pkg/sfu/mime" "github.com/livekit/livekit-server/pkg/testutils" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" ) func TestMissingAnswerDuringICERestart(t *testing.T) { params := TransportParams{ - ParticipantID: "id", - ParticipantIdentity: "identity", - Config: &WebRTCConfig{}, - IsOfferer: true, + Config: &WebRTCConfig{}, + IsOfferer: true, } paramsA := params @@ -70,7 +68,7 @@ func TestMissingAnswerDuringICERestart(t *testing.T) { // offer again, but missed var offerReceived atomic.Bool - handlerA.OnOfferCalls(func(sd webrtc.SessionDescription) error { + handlerA.OnOfferCalls(func(sd webrtc.SessionDescription, _offerId uint32, _midToTrackID map[string]string) error { require.Equal(t, webrtc.SignalingStateHaveLocalOffer, transportA.pc.SignalingState()) require.Equal(t, transport.NegotiationStateRemote, negotiationState.Load().(transport.NegotiationState)) offerReceived.Store(true) @@ -91,10 +89,8 @@ func TestMissingAnswerDuringICERestart(t *testing.T) { func TestNegotiationTiming(t *testing.T) { params := TransportParams{ - ParticipantID: "id", - ParticipantIdentity: "identity", - Config: &WebRTCConfig{}, - IsOfferer: true, + Config: &WebRTCConfig{}, + IsOfferer: true, } paramsA := params @@ -116,9 +112,16 @@ func TestNegotiationTiming(t *testing.T) { require.False(t, transportB.IsEstablished()) handleICEExchange(t, transportA, transportB, handlerA, handlerB) - offer := atomic.Value{} - handlerA.OnOfferCalls(func(sd webrtc.SessionDescription) error { - offer.Store(&sd) + firstOffer := atomic.Value{} + firstOfferId := atomic.Uint32{} + secondOffer := atomic.Value{} + handlerA.OnOfferCalls(func(sd webrtc.SessionDescription, offerId uint32, _midToTrackID map[string]string) error { + if _, ok := firstOffer.Load().(*webrtc.SessionDescription); !ok { + firstOffer.Store(&sd) + firstOfferId.Store(offerId) + } else { + secondOffer.Store(&sd) + } return nil }) @@ -161,15 +164,22 @@ func TestNegotiationTiming(t *testing.T) { return state == transport.NegotiationStateRetry }, 10*time.Second, 10*time.Millisecond, "negotiation state does not match NegotiateStateRetry") - time.Sleep(5 * time.Millisecond) - actualOffer, ok := offer.Load().(*webrtc.SessionDescription) - require.True(t, ok) + require.Eventually(t, func() bool { + _, ok := firstOffer.Load().(*webrtc.SessionDescription) + if !ok { + return false + } + if firstOfferId.Load() == 0 { + return false + } + return true + }, 10*time.Second, 10*time.Millisecond, "first offer not received yet") - handlerB.OnAnswerCalls(func(answer webrtc.SessionDescription) error { - transportA.HandleRemoteDescription(answer) + handlerB.OnAnswerCalls(func(answer webrtc.SessionDescription, answerId uint32, _midToTrackID map[string]string) error { + transportA.HandleRemoteDescription(answer, answerId) return nil }) - transportB.HandleRemoteDescription(*actualOffer) + transportB.HandleRemoteDescription(*firstOffer.Load().(*webrtc.SessionDescription), firstOfferId.Load()) require.Eventually(t, func() bool { return transportA.IsEstablished() @@ -178,11 +188,18 @@ func TestNegotiationTiming(t *testing.T) { return transportB.IsEstablished() }, 10*time.Second, time.Millisecond*10, "transportB is not established") - // it should still be negotiating again - require.Equal(t, transport.NegotiationStateRemote, negotiationState.Load().(transport.NegotiationState)) - offer2, ok := offer.Load().(*webrtc.SessionDescription) + // offerer should send another offer after processing the answer + // as there were forced negotiations a couple of time above + require.Eventually(t, func() bool { + state, ok := negotiationState.Load().(transport.NegotiationState) + if !ok { + return false + } + + return state == transport.NegotiationStateRemote + }, 10*time.Second, 10*time.Millisecond, "negotiation state does not match NegotiateStateRemote") + _, ok := secondOffer.Load().(*webrtc.SessionDescription) require.True(t, ok) - require.False(t, offer2 == actualOffer) transportA.Close() transportB.Close() @@ -190,10 +207,8 @@ func TestNegotiationTiming(t *testing.T) { func TestFirstOfferMissedDuringICERestart(t *testing.T) { params := TransportParams{ - ParticipantID: "id", - ParticipantIdentity: "identity", - Config: &WebRTCConfig{}, - IsOfferer: true, + Config: &WebRTCConfig{}, + IsOfferer: true, } paramsA := params @@ -216,7 +231,7 @@ func TestFirstOfferMissedDuringICERestart(t *testing.T) { // first offer missed var firstOfferReceived atomic.Bool - handlerA.OnOfferCalls(func(sd webrtc.SessionDescription) error { + handlerA.OnOfferCalls(func(sd webrtc.SessionDescription, _offerId uint32, _midToTrackID map[string]string) error { firstOfferReceived.Store(true) return nil }) @@ -228,13 +243,13 @@ func TestFirstOfferMissedDuringICERestart(t *testing.T) { // set offer/answer with restart ICE, will negotiate twice, // first one is recover from missed offer // second one is restartICE - handlerB.OnAnswerCalls(func(answer webrtc.SessionDescription) error { - transportA.HandleRemoteDescription(answer) + handlerB.OnAnswerCalls(func(answer webrtc.SessionDescription, answerId uint32, _midToTrackID map[string]string) error { + transportA.HandleRemoteDescription(answer, answerId) return nil }) var offerCount atomic.Int32 - handlerA.OnOfferCalls(func(sd webrtc.SessionDescription) error { + handlerA.OnOfferCalls(func(sd webrtc.SessionDescription, offerId uint32, _midToTrackID map[string]string) error { offerCount.Inc() // the second offer is a ice restart offer, so we wait transportB complete the ice gathering @@ -244,7 +259,7 @@ func TestFirstOfferMissedDuringICERestart(t *testing.T) { }, 10*time.Second, time.Millisecond*10) } - transportB.HandleRemoteDescription(sd) + transportB.HandleRemoteDescription(sd, offerId) return nil }) @@ -264,10 +279,8 @@ func TestFirstOfferMissedDuringICERestart(t *testing.T) { func TestFirstAnswerMissedDuringICERestart(t *testing.T) { params := TransportParams{ - ParticipantID: "id", - ParticipantIdentity: "identity", - Config: &WebRTCConfig{}, - IsOfferer: true, + Config: &WebRTCConfig{}, + IsOfferer: true, } paramsA := params @@ -290,17 +303,17 @@ func TestFirstAnswerMissedDuringICERestart(t *testing.T) { // first answer missed var firstAnswerReceived atomic.Bool - handlerB.OnAnswerCalls(func(sd webrtc.SessionDescription) error { + handlerB.OnAnswerCalls(func(sd webrtc.SessionDescription, answerId uint32, _midToTrackID map[string]string) error { if firstAnswerReceived.Load() { - transportA.HandleRemoteDescription(sd) + transportA.HandleRemoteDescription(sd, answerId) } else { // do not send first answer so that remote misses the first answer firstAnswerReceived.Store(true) } return nil }) - handlerA.OnOfferCalls(func(sd webrtc.SessionDescription) error { - transportB.HandleRemoteDescription(sd) + handlerA.OnOfferCalls(func(sd webrtc.SessionDescription, offerId uint32, _midToTrackID map[string]string) error { + transportB.HandleRemoteDescription(sd, offerId) return nil }) @@ -313,7 +326,7 @@ func TestFirstAnswerMissedDuringICERestart(t *testing.T) { // first one is recover from missed offer // second one is restartICE var offerCount atomic.Int32 - handlerA.OnOfferCalls(func(sd webrtc.SessionDescription) error { + handlerA.OnOfferCalls(func(sd webrtc.SessionDescription, offerId uint32, _midToTrackID map[string]string) error { offerCount.Inc() // the second offer is a ice restart offer, so we wait for transportB to complete ICE gathering @@ -323,7 +336,7 @@ func TestFirstAnswerMissedDuringICERestart(t *testing.T) { }, 10*time.Second, time.Millisecond*10) } - transportB.HandleRemoteDescription(sd) + transportB.HandleRemoteDescription(sd, offerId) return nil }) @@ -343,10 +356,8 @@ func TestFirstAnswerMissedDuringICERestart(t *testing.T) { func TestNegotiationFailed(t *testing.T) { params := TransportParams{ - ParticipantID: "id", - ParticipantIdentity: "identity", - Config: &WebRTCConfig{}, - IsOfferer: true, + Config: &WebRTCConfig{}, + IsOfferer: true, } paramsA := params @@ -371,7 +382,9 @@ func TestNegotiationFailed(t *testing.T) { connectTransports(t, transportA, transportB, handlerA, handlerB, false, 1, 1) // reset OnOffer to force a negotiation failure - handlerA.OnOfferCalls(func(sd webrtc.SessionDescription) error { return nil }) + handlerA.OnOfferCalls(func(sd webrtc.SessionDescription, offerId uint32, _midToTrackID map[string]string) error { + return nil + }) var failed atomic.Int32 handlerA.OnNegotiationFailedCalls(func() { failed.Inc() @@ -386,10 +399,8 @@ func TestNegotiationFailed(t *testing.T) { func TestFilteringCandidates(t *testing.T) { params := TransportParams{ - ParticipantID: "id", - ParticipantIdentity: "identity", - Config: &WebRTCConfig{}, - EnabledCodecs: []*livekit.Codec{ + Config: &WebRTCConfig{}, + EnabledPublishCodecs: []*livekit.Codec{ {Mime: mime.MimeTypeOpus.String()}, {Mime: mime.MimeTypeVP8.String()}, {Mime: mime.MimeTypeH264.String()}, @@ -527,15 +538,15 @@ func handleICEExchange(t *testing.T, a, b *PCTransport, ah, bh *transportfakes.F func connectTransports(t *testing.T, offerer, answerer *PCTransport, offererHandler, answererHandler *transportfakes.FakeHandler, isICERestart bool, expectedOfferCount int32, expectedAnswerCount int32) { var offerCount atomic.Int32 var answerCount atomic.Int32 - answererHandler.OnAnswerCalls(func(answer webrtc.SessionDescription) error { + answererHandler.OnAnswerCalls(func(answer webrtc.SessionDescription, answerId uint32, _midToTrackID map[string]string) error { answerCount.Inc() - offerer.HandleRemoteDescription(answer) + offerer.HandleRemoteDescription(answer, answerId) return nil }) - offererHandler.OnOfferCalls(func(offer webrtc.SessionDescription) error { + offererHandler.OnOfferCalls(func(offer webrtc.SessionDescription, offerId uint32, _midToTrackID map[string]string) error { offerCount.Inc() - answerer.HandleRemoteDescription(offer) + answerer.HandleRemoteDescription(offer, offerId) return nil }) @@ -606,7 +617,7 @@ func TestConfigureAudioTransceiver(t *testing.T) { tr, err := pc.AddTransceiverFromKind(webrtc.RTPCodecTypeAudio, webrtc.RTPTransceiverInit{Direction: webrtc.RTPTransceiverDirectionSendonly}) require.NoError(t, err) - configureAudioTransceiver(tr, testcase.stereo, testcase.nack) + configureSenderAudio(tr, testcase.stereo, testcase.nack) codecs := tr.Sender().GetParameters().Codecs for _, codec := range codecs { if mime.IsMimeTypeStringOpus(codec.MimeType) { @@ -624,3 +635,145 @@ func TestConfigureAudioTransceiver(t *testing.T) { }) } } + +// In single-PC mode the publisher PC carries both publish and subscribe +// directions. If the MediaEngine were built only from the publish codec list, +// the SDP offer would not advertise some codecs in the m-section even though +// the subscribe direction is supposed to support it. This regression-tests +// the union behavior in newPeerConnection: build the MediaEngine from publish + +// subscribe codec lists. +func TestSinglePCMediaEngineUnionsCodecs(t *testing.T) { + videoMSectionCodecs := func(transport *PCTransport) []string { + _, err := transport.pc.AddTransceiverFromKind(webrtc.RTPCodecTypeVideo) + require.NoError(t, err) + offer, err := transport.pc.CreateOffer(nil) + require.NoError(t, err) + parsed, err := offer.Unmarshal() + require.NoError(t, err) + var rtpmaps []string + for _, m := range parsed.MediaDescriptions { + if m.MediaName.Media != "video" { + continue + } + for _, a := range m.Attributes { + if a.Key == "rtpmap" { + rtpmaps = append(rtpmaps, a.Value) + } + } + } + return rtpmaps + } + + sdpHasH264 := func(rtpmaps []string) bool { + for _, r := range rtpmaps { + if strings.Contains(r, "H264/") { + return true + } + } + return false + } + + publishOnly := []*livekit.Codec{ + {Mime: mime.MimeTypeOpus.String()}, + {Mime: mime.MimeTypeVP8.String()}, + } + subscribeOnly := []*livekit.Codec{ + {Mime: mime.MimeTypeOpus.String()}, + {Mime: mime.MimeTypeVP8.String()}, + {Mime: mime.MimeTypeH264.String()}, + } + + // Control: only publish codecs set (dual-PC publisher PC). H.264 absent. + dualPC, err := NewPCTransport(TransportParams{ + Config: &WebRTCConfig{}, + EnabledPublishCodecs: publishOnly, + Handler: &transportfakes.FakeHandler{}, + }) + require.NoError(t, err) + require.False(t, sdpHasH264(videoMSectionCodecs(dualPC)), + "dual-PC publisher must not advertise H.264 when it's stripped from the publish list") + + // Single-PC publisher PC: both lists set. H.264 must appear. + singlePC, err := NewPCTransport(TransportParams{ + Config: &WebRTCConfig{}, + EnabledPublishCodecs: publishOnly, + EnabledSubscribeCodecs: subscribeOnly, + IsSendSide: true, + Handler: &transportfakes.FakeHandler{}, + }) + require.NoError(t, err) + require.True(t, sdpHasH264(videoMSectionCodecs(singlePC)), + "single-PC publisher must advertise H.264 from the subscribe list even when it's stripped from the publish list") +} + +// Regression test for restrictReceiverCodecsToPublishList: subscribe-only +// codecs (e.g., H.264) registered for subscriptions must not leak into the +// recv-side m-section of an answer, or the peer could publish them. +func TestSinglePCAnswerStripsSubscribeOnlyCodecsFromRecvSide(t *testing.T) { + publishCodecs := []*livekit.Codec{ + {Mime: mime.MimeTypeOpus.String()}, + {Mime: mime.MimeTypeVP8.String()}, + } + subscribeCodecs := []*livekit.Codec{ + {Mime: mime.MimeTypeOpus.String()}, + {Mime: mime.MimeTypeVP8.String()}, + {Mime: mime.MimeTypeH264.String()}, + } + + handler := &transportfakes.FakeHandler{} + server, err := NewPCTransport(TransportParams{ + Config: &WebRTCConfig{}, + EnabledPublishCodecs: publishCodecs, + EnabledSubscribeCodecs: subscribeCodecs, + IsSendSide: true, + Handler: handler, + }) + require.NoError(t, err) + defer server.Close() + + var clientME webrtc.MediaEngine + require.NoError(t, registerCodecs(&clientME, subscribeCodecs, RTCPFeedbackConfig{}, false)) + client, err := webrtc.NewAPI(webrtc.WithMediaEngine(&clientME)).NewPeerConnection(webrtc.Configuration{}) + require.NoError(t, err) + defer client.Close() + + _, err = client.AddTransceiverFromKind(webrtc.RTPCodecTypeVideo, webrtc.RTPTransceiverInit{ + Direction: webrtc.RTPTransceiverDirectionSendonly, + }) + require.NoError(t, err) + offer, err := client.CreateOffer(nil) + require.NoError(t, err) + require.Contains(t, offer.SDP, "H264/", "offer must advertise H.264") + require.NoError(t, client.SetLocalDescription(offer)) + + var answer atomic.Pointer[webrtc.SessionDescription] + handler.OnAnswerCalls(func(sd webrtc.SessionDescription, _ uint32, _ map[string]string) error { + answer.Store(&sd) + return nil + }) + require.NoError(t, server.HandleRemoteDescription(*client.LocalDescription(), 1)) + + require.Eventually(t, func() bool { + return answer.Load() != nil + }, 5*time.Second, 10*time.Millisecond, "server did not produce answer") + + parsed, err := answer.Load().Unmarshal() + require.NoError(t, err) + + var videoSection *sdp.MediaDescription + for _, m := range parsed.MediaDescriptions { + if m.MediaName.Media == "video" { + videoSection = m + break + } + } + require.NotNil(t, videoSection, "answer missing video m-section") + + for _, a := range videoSection.Attributes { + if a.Key != "rtpmap" { + continue + } + require.NotContains(t, a.Value, "H264/", + "answer must not advertise H.264 in recv-side m-section: %s", a.Value) + } +} diff --git a/pkg/rtc/transportmanager.go b/pkg/rtc/transportmanager.go index 671171d..528cd72 100644 --- a/pkg/rtc/transportmanager.go +++ b/pkg/rtc/transportmanager.go @@ -39,8 +39,8 @@ import ( "github.com/livekit/livekit-server/pkg/rtc/types" "github.com/livekit/livekit-server/pkg/sfu" "github.com/livekit/livekit-server/pkg/sfu/datachannel" + "github.com/livekit/livekit-server/pkg/sfu/interceptor" "github.com/livekit/livekit-server/pkg/sfu/pacer" - "github.com/livekit/livekit-server/pkg/telemetry" ) const ( @@ -71,43 +71,33 @@ func (h TransportManagerTransportHandler) OnFailed(isShortLived bool, iceConnect // ------------------------------- -type TransportManagerPublisherTransportHandler struct { - TransportManagerTransportHandler -} - -func (h TransportManagerPublisherTransportHandler) OnAnswer(sd webrtc.SessionDescription) error { - h.t.lastPublisherAnswer.Store(sd) - return h.Handler.OnAnswer(sd) -} - -// ------------------------------- - type TransportManagerParams struct { - Identity livekit.ParticipantIdentity - SID livekit.ParticipantID - SubscriberAsPrimary bool - Config *WebRTCConfig - Twcc *twcc.Responder - ProtocolVersion types.ProtocolVersion - CongestionControlConfig config.CongestionControlConfig - EnabledSubscribeCodecs []*livekit.Codec - EnabledPublishCodecs []*livekit.Codec - SimTracks map[uint32]SimulcastTrackInfo - ClientInfo ClientInfo - Migration bool - AllowTCPFallback bool - TCPFallbackRTTThreshold int - AllowUDPUnstableFallback bool - TURNSEnabled bool - AllowPlayoutDelay bool - DataChannelMaxBufferedAmount uint64 - DatachannelSlowThreshold int - Logger logger.Logger - PublisherHandler transport.Handler - SubscriberHandler transport.Handler - DataChannelStats *telemetry.BytesTrackStats - UseOneShotSignallingMode bool - FireOnTrackBySdp bool + SubscriberAsPrimary bool + UseSinglePeerConnection bool + Config *WebRTCConfig + Twcc *twcc.Responder + ProtocolVersion types.ProtocolVersion + CongestionControlConfig config.CongestionControlConfig + EnabledSubscribeCodecs []*livekit.Codec + EnabledPublishCodecs []*livekit.Codec + SimTracks map[uint32]interceptor.SimulcastTrackInfo + ClientInfo ClientInfo + Migration bool + AllowTCPFallback bool + TCPFallbackRTTThreshold int + AllowUDPUnstableFallback bool + TURNSEnabled bool + AllowPlayoutDelay bool + DataChannelMaxBufferedAmount uint64 + DatachannelSlowThreshold int + DatachannelLossyTargetLatency time.Duration + Logger logger.Logger + PublisherHandler transport.Handler + SubscriberHandler transport.Handler + DataChannelStats *BytesTrackStats + UseOneShotSignallingMode bool + FireOnTrackBySdp bool + EnableDataTracks bool } type TransportManager struct { @@ -124,9 +114,8 @@ type TransportManager struct { signalSourceValid atomic.Bool pendingOfferPublisher *webrtc.SessionDescription + pendingOfferIdPublisher uint32 pendingDataChannelsPublisher []*livekit.DataChannelInfo - lastPublisherAnswer atomic.Value - lastPublisherOffer atomic.Value iceConfig *livekit.ICEConfig mediaLossProxy *MediaLossProxy @@ -135,7 +124,8 @@ type TransportManager struct { onICEConfigChanged func(iceConfig *livekit.ICEConfig) - droppedBySlowReaderCount atomic.Uint32 + dataChannelSendErrorDroppedBySlowReaderCount atomic.Uint32 + dataChannelSendErrorCount atomic.Uint32 } func NewTransportManager(params TransportManagerParams) (*TransportManager, error) { @@ -150,53 +140,67 @@ func NewTransportManager(params TransportManagerParams) (*TransportManager, erro t.mediaLossProxy.OnMediaLossUpdate(t.onMediaLossUpdate) lgr := LoggerWithPCTarget(params.Logger, livekit.SignalTarget_PUBLISHER) + // In single-PC mode the one PC carries both directions, so it needs both + // codec lists registered on its MediaEngine. In dual-PC mode the publisher + // PC is recvonly, so subscribe codecs are left nil. + var publisherSubscribeCodecs []*livekit.Codec + if params.UseSinglePeerConnection || params.UseOneShotSignallingMode { + publisherSubscribeCodecs = params.EnabledSubscribeCodecs + } publisher, err := NewPCTransport(TransportParams{ - ParticipantID: params.SID, - ParticipantIdentity: params.Identity, - ProtocolVersion: params.ProtocolVersion, - Config: params.Config, - Twcc: params.Twcc, - DirectionConfig: params.Config.Publisher, - CongestionControlConfig: params.CongestionControlConfig, - EnabledCodecs: params.EnabledPublishCodecs, - Logger: lgr, - SimTracks: params.SimTracks, - ClientInfo: params.ClientInfo, - Transport: livekit.SignalTarget_PUBLISHER, - Handler: TransportManagerPublisherTransportHandler{TransportManagerTransportHandler{params.PublisherHandler, t, lgr}}, - UseOneShotSignallingMode: params.UseOneShotSignallingMode, - DataChannelMaxBufferedAmount: params.DataChannelMaxBufferedAmount, - DatachannelSlowThreshold: params.DatachannelSlowThreshold, - FireOnTrackBySdp: params.FireOnTrackBySdp, + ProtocolVersion: params.ProtocolVersion, + Config: params.Config, + Twcc: params.Twcc, + DirectionConfig: params.Config.Publisher, + CongestionControlConfig: params.CongestionControlConfig, + EnabledPublishCodecs: params.EnabledPublishCodecs, + EnabledSubscribeCodecs: publisherSubscribeCodecs, + Logger: lgr, + SimTracks: params.SimTracks, + ClientInfo: params.ClientInfo, + IsSendSide: params.UseOneShotSignallingMode || params.UseSinglePeerConnection, + AllowPlayoutDelay: params.AllowPlayoutDelay, + Transport: livekit.SignalTarget_PUBLISHER, + Handler: TransportManagerTransportHandler{params.PublisherHandler, t, lgr}, + UseOneShotSignallingMode: params.UseOneShotSignallingMode, + DataChannelMaxBufferedAmount: params.DataChannelMaxBufferedAmount, + DatachannelSlowThreshold: params.DatachannelSlowThreshold, + DatachannelLossyTargetLatency: params.DatachannelLossyTargetLatency, + FireOnTrackBySdp: params.FireOnTrackBySdp, + EnableDataTracks: params.EnableDataTracks, }) if err != nil { return nil, err } t.publisher = publisher - lgr = LoggerWithPCTarget(params.Logger, livekit.SignalTarget_SUBSCRIBER) - subscriber, err := NewPCTransport(TransportParams{ - ParticipantID: params.SID, - ParticipantIdentity: params.Identity, - ProtocolVersion: params.ProtocolVersion, - Config: params.Config, - DirectionConfig: params.Config.Subscriber, - CongestionControlConfig: params.CongestionControlConfig, - EnabledCodecs: params.EnabledSubscribeCodecs, - Logger: lgr, - ClientInfo: params.ClientInfo, - IsOfferer: true, - IsSendSide: true, - AllowPlayoutDelay: params.AllowPlayoutDelay, - DatachannelSlowThreshold: params.DatachannelSlowThreshold, - Transport: livekit.SignalTarget_SUBSCRIBER, - Handler: TransportManagerTransportHandler{params.SubscriberHandler, t, lgr}, - }) - if err != nil { - return nil, err + if !t.params.UseOneShotSignallingMode && !t.params.UseSinglePeerConnection { + lgr := LoggerWithPCTarget(params.Logger, livekit.SignalTarget_SUBSCRIBER) + subscriber, err := NewPCTransport(TransportParams{ + ProtocolVersion: params.ProtocolVersion, + Config: params.Config, + DirectionConfig: params.Config.Subscriber, + CongestionControlConfig: params.CongestionControlConfig, + EnabledSubscribeCodecs: params.EnabledSubscribeCodecs, + Logger: lgr, + ClientInfo: params.ClientInfo, + IsOfferer: true, + IsSendSide: true, + AllowPlayoutDelay: params.AllowPlayoutDelay, + DataChannelMaxBufferedAmount: params.DataChannelMaxBufferedAmount, + DatachannelSlowThreshold: params.DatachannelSlowThreshold, + DatachannelLossyTargetLatency: params.DatachannelLossyTargetLatency, + Transport: livekit.SignalTarget_SUBSCRIBER, + Handler: TransportManagerTransportHandler{params.SubscriberHandler, t, lgr}, + FireOnTrackBySdp: params.FireOnTrackBySdp, + EnableDataTracks: params.EnableDataTracks, + }) + if err != nil { + return nil, err + } + t.subscriber = subscriber } - t.subscriber = subscriber - if !t.params.Migration { + if !t.params.Migration && t.params.SubscriberAsPrimary { if err := t.createDataChannelsForSubscriber(nil); err != nil { return nil, err } @@ -207,8 +211,12 @@ func NewTransportManager(params TransportManagerParams) (*TransportManager, erro } func (t *TransportManager) Close() { - t.publisher.Close() - t.subscriber.Close() + if t.publisher != nil { + t.publisher.Close() + } + if t.subscriber != nil { + t.subscriber.Close() + } } func (t *TransportManager) SubscriberClose() { @@ -231,6 +239,10 @@ func (t *TransportManager) GetPublisherMid(rtpReceiver *webrtc.RTPReceiver) stri return t.publisher.GetMid(rtpReceiver) } +func (t *TransportManager) GetPublisherRTPTransceiver(mid string) *webrtc.RTPTransceiver { + return t.publisher.GetRTPTransceiver(mid) +} + func (t *TransportManager) GetPublisherRTPReceiver(mid string) *webrtc.RTPReceiver { return t.publisher.GetRTPReceiver(mid) } @@ -240,37 +252,49 @@ func (t *TransportManager) WritePublisherRTCP(pkts []rtcp.Packet) error { } func (t *TransportManager) GetSubscriberRTT() (float64, bool) { - return t.subscriber.GetRTT() + if t.params.UseOneShotSignallingMode || t.params.UseSinglePeerConnection { + return t.publisher.GetRTT() + } else { + return t.subscriber.GetRTT() + } } func (t *TransportManager) HasSubscriberEverConnected() bool { - return t.subscriber.HasEverConnected() + if t.params.UseOneShotSignallingMode || t.params.UseSinglePeerConnection { + return t.publisher.HasEverConnected() + } else { + return t.subscriber.HasEverConnected() + } } func (t *TransportManager) AddTrackLocal( trackLocal webrtc.TrackLocal, params types.AddTrackParams, + enabledCodecs []*livekit.Codec, + rtcpFeedbackConfig RTCPFeedbackConfig, ) (*webrtc.RTPSender, *webrtc.RTPTransceiver, error) { - if t.params.UseOneShotSignallingMode { - return t.publisher.AddTrack(trackLocal, params) + if t.params.UseOneShotSignallingMode || t.params.UseSinglePeerConnection { + return t.publisher.AddTrack(trackLocal, params, enabledCodecs, rtcpFeedbackConfig) } else { - return t.subscriber.AddTrack(trackLocal, params) + return t.subscriber.AddTrack(trackLocal, params, enabledCodecs, rtcpFeedbackConfig) } } func (t *TransportManager) AddTransceiverFromTrackLocal( trackLocal webrtc.TrackLocal, params types.AddTrackParams, + enabledCodecs []*livekit.Codec, + rtcpFeedbackConfig RTCPFeedbackConfig, ) (*webrtc.RTPSender, *webrtc.RTPTransceiver, error) { - if t.params.UseOneShotSignallingMode { - return t.publisher.AddTransceiverFromTrack(trackLocal, params) + if t.params.UseOneShotSignallingMode || t.params.UseSinglePeerConnection { + return t.publisher.AddTransceiverFromTrack(trackLocal, params, enabledCodecs, rtcpFeedbackConfig) } else { - return t.subscriber.AddTransceiverFromTrack(trackLocal, params) + return t.subscriber.AddTransceiverFromTrack(trackLocal, params, enabledCodecs, rtcpFeedbackConfig) } } func (t *TransportManager) RemoveTrackLocal(sender *webrtc.RTPSender) error { - if t.params.UseOneShotSignallingMode { + if t.params.UseOneShotSignallingMode || t.params.UseSinglePeerConnection { return t.publisher.RemoveTrack(sender) } else { return t.subscriber.RemoveTrack(sender) @@ -278,7 +302,7 @@ func (t *TransportManager) RemoveTrackLocal(sender *webrtc.RTPSender) error { } func (t *TransportManager) WriteSubscriberRTCP(pkts []rtcp.Packet) error { - if t.params.UseOneShotSignallingMode { + if t.params.UseOneShotSignallingMode || t.params.UseSinglePeerConnection { return t.publisher.WriteRTCP(pkts) } else { return t.subscriber.WriteRTCP(pkts) @@ -286,36 +310,85 @@ func (t *TransportManager) WriteSubscriberRTCP(pkts []rtcp.Packet) error { } func (t *TransportManager) GetSubscriberPacer() pacer.Pacer { - return t.subscriber.GetPacer() + if t.params.UseOneShotSignallingMode || t.params.UseSinglePeerConnection { + return t.publisher.GetPacer() + } else { + return t.subscriber.GetPacer() + } } func (t *TransportManager) AddSubscribedTrack(subTrack types.SubscribedTrack) { - t.subscriber.AddTrackToStreamAllocator(subTrack) + if t.params.UseOneShotSignallingMode || t.params.UseSinglePeerConnection { + t.publisher.AddTrackToStreamAllocator(subTrack) + } else { + t.subscriber.AddTrackToStreamAllocator(subTrack) + } } func (t *TransportManager) RemoveSubscribedTrack(subTrack types.SubscribedTrack) { - t.subscriber.RemoveTrackFromStreamAllocator(subTrack) + if t.params.UseOneShotSignallingMode || t.params.UseSinglePeerConnection { + t.publisher.RemoveTrackFromStreamAllocator(subTrack) + } else { + t.subscriber.RemoveTrackFromStreamAllocator(subTrack) + } } -func (t *TransportManager) SendDataPacket(kind livekit.DataPacket_Kind, encoded []byte) error { +func (t *TransportManager) SendDataMessage(kind livekit.DataPacket_Kind, data []byte) error { // downstream data is sent via primary peer connection - err := t.getTransport(true).SendDataPacket(kind, encoded) + return t.handleSendDataResult(t.getTransport(true).SendDataMessage(kind, data), kind.String(), len(data)) +} + +func (t *TransportManager) SendDataMessageUnlabeled(data []byte, useRaw bool, sender livekit.ParticipantIdentity) error { + // downstream data is sent via primary peer connection + return t.handleSendDataResult( + t.getTransport(true).SendDataMessageUnlabeled(data, useRaw, sender), + "unlabeled", + len(data), + ) +} + +func (t *TransportManager) handleSendDataResult(err error, kind string, size int) error { if err != nil { - if !utils.ErrorIsOneOf(err, io.ErrClosedPipe, sctp.ErrStreamClosed, ErrTransportFailure, ErrDataChannelBufferFull, context.DeadlineExceeded) { + if !utils.ErrorIsOneOf( + err, + io.ErrClosedPipe, + sctp.ErrStreamClosed, + ErrTransportFailure, + ErrDataChannelUnavailable, + context.DeadlineExceeded, + datachannel.ErrDataDroppedByHighBufferedAmount, + ) { if errors.Is(err, datachannel.ErrDataDroppedBySlowReader) { - droppedBySlowReaderCount := t.droppedBySlowReaderCount.Inc() + droppedBySlowReaderCount := t.dataChannelSendErrorDroppedBySlowReaderCount.Inc() if (droppedBySlowReaderCount-1)%100 == 0 { - t.params.Logger.Infow("drop data packet by slow reader", "error", err, "kind", kind, "count", droppedBySlowReaderCount) + t.params.Logger.Infow( + "drop data message by slow reader", + "error", err, + "kind", kind, + "count", droppedBySlowReaderCount, + ) } } else { - t.params.Logger.Warnw("send data packet error", err) + count := t.dataChannelSendErrorCount.Inc() + if (count-1)%100 == 0 { + t.params.Logger.Infow( + "send data message error", + "error", err, + "kind", kind, + "count", count, + ) + } } } if utils.ErrorIsOneOf(err, sctp.ErrStreamClosed, io.ErrClosedPipe) { - t.params.SubscriberHandler.OnDataSendError(err) + if t.params.SubscriberAsPrimary { + t.params.SubscriberHandler.OnDataSendError(err) + } else { + t.params.PublisherHandler.OnDataSendError(err) + } } } else { - t.params.DataChannelStats.AddBytes(uint64(len(encoded)), true) + t.params.DataChannelStats.AddBytes(uint64(size), true) } return err @@ -323,8 +396,8 @@ func (t *TransportManager) SendDataPacket(kind livekit.DataPacket_Kind, encoded func (t *TransportManager) createDataChannelsForSubscriber(pendingDataChannels []*livekit.DataChannelInfo) error { var ( - reliableID, lossyID uint16 - reliableIDPtr, lossyIDPtr *uint16 + reliableID, lossyID, dataTrackID uint16 + reliableIDPtr, lossyIDPtr, dataTrackIDPtr *uint16 ) // @@ -335,13 +408,17 @@ func (t *TransportManager) createDataChannelsForSubscriber(pendingDataChannels [ // For new version migration clients, we create data channels with new ID and negotiate with client // for _, dc := range pendingDataChannels { - if dc.Label == ReliableDataChannel { - // pion use step 2 for auto generated ID, so we need to add 4 to avoid conflict - reliableID = uint16(dc.Id) + 4 + switch dc.Label { + case ReliableDataChannel: + // pion use step 2 for auto generated ID, so we need to add 6 to avoid conflict + reliableID = uint16(dc.Id) + 6 reliableIDPtr = &reliableID - } else if dc.Label == LossyDataChannel { - lossyID = uint16(dc.Id) + 4 + case LossyDataChannel: + lossyID = uint16(dc.Id) + 6 lossyIDPtr = &lossyID + case DataTrackDataChannel: + dataTrackID = uint16(dc.Id) + 6 + dataTrackIDPtr = &dataTrackID } } @@ -355,6 +432,7 @@ func (t *TransportManager) createDataChannelsForSubscriber(pendingDataChannels [ return err } + ordered = false retransmits := uint16(0) negotiated = t.params.Migration && lossyIDPtr == nil if err := t.subscriber.CreateDataChannel(LossyDataChannel, &webrtc.DataChannelInit{ @@ -365,26 +443,28 @@ func (t *TransportManager) createDataChannelsForSubscriber(pendingDataChannels [ }); err != nil { return err } + + negotiated = t.params.Migration && dataTrackIDPtr == nil + if err := t.subscriber.CreateDataChannel(DataTrackDataChannel, &webrtc.DataChannelInit{ + Ordered: &ordered, + MaxRetransmits: &retransmits, + ID: dataTrackIDPtr, + Negotiated: &negotiated, + }); err != nil { + return err + } + return nil } -func (t *TransportManager) GetUnmatchMediaForOffer(offer webrtc.SessionDescription, mediaType string) (parsed *sdp.SessionDescription, unmatched []*sdp.MediaDescription, err error) { - // prefer codec from offer for clients that don't support setCodecPreferences - parsed, err = offer.Unmarshal() - if err != nil { - t.params.Logger.Errorw("failed to parse offer for codec preference", err) - return - } - +func (t *TransportManager) GetUnmatchMediaForOffer(parsedOffer *sdp.SessionDescription, mediaType string) (unmatched []*sdp.MediaDescription, err error) { var lastMatchedMid string - lastAnswer := t.lastPublisherAnswer.Load() - if lastAnswer != nil { - answer := lastAnswer.(webrtc.SessionDescription) - parsedAnswer, err1 := answer.Unmarshal() + if lastAnswer := t.publisher.CurrentLocalDescription(); lastAnswer != nil { + parsedAnswer, err1 := lastAnswer.Unmarshal() if err1 != nil { // should not happen - t.params.Logger.Errorw("failed to parse last answer", err) - return + t.params.Logger.Errorw("failed to parse last answer", err1) + return unmatched, err1 } for i := len(parsedAnswer.MediaDescriptions) - 1; i >= 0; i-- { @@ -396,8 +476,8 @@ func (t *TransportManager) GetUnmatchMediaForOffer(offer webrtc.SessionDescripti } } - for i := len(parsed.MediaDescriptions) - 1; i >= 0; i-- { - media := parsed.MediaDescriptions[i] + for i := len(parsedOffer.MediaDescriptions) - 1; i >= 0; i-- { + media := parsedOffer.MediaDescriptions[i] if media.MediaName.Media == mediaType { mid, _ := media.Attribute(sdp.AttrKeyMID) if mid == lastMatchedMid { @@ -410,56 +490,76 @@ func (t *TransportManager) GetUnmatchMediaForOffer(offer webrtc.SessionDescripti return } -func (t *TransportManager) LastPublisherOffer() webrtc.SessionDescription { - if sd := t.lastPublisherOffer.Load(); sd != nil { - return sd.(webrtc.SessionDescription) - } - return webrtc.SessionDescription{} +func (t *TransportManager) LastPublisherOffer() *webrtc.SessionDescription { + return t.publisher.CurrentRemoteDescription() } -func (t *TransportManager) HandleOffer(offer webrtc.SessionDescription, shouldPend bool) error { +func (t *TransportManager) LastPublisherOfferPending() *webrtc.SessionDescription { + return t.publisher.PendingRemoteDescription() +} + +func (t *TransportManager) HandleOffer(offer webrtc.SessionDescription, offerId uint32, shouldPend bool) error { t.lock.Lock() if shouldPend { t.pendingOfferPublisher = &offer + t.pendingOfferIdPublisher = offerId t.lock.Unlock() return nil } t.lock.Unlock() - t.lastPublisherOffer.Store(offer) - return t.publisher.HandleRemoteDescription(offer) + return t.publisher.HandleRemoteDescription(offer, offerId) } -func (t *TransportManager) GetAnswer() (webrtc.SessionDescription, error) { - answer, err := t.publisher.GetAnswer() - if err == nil { - t.lastPublisherAnswer.Store(answer) - } - return answer, err +func (t *TransportManager) GetAnswer() (webrtc.SessionDescription, uint32, error) { + return t.publisher.GetAnswer() +} + +func (t *TransportManager) GetPublisherICESessionUfrag() (string, error) { + return t.publisher.GetICESessionUfrag() +} + +func (t *TransportManager) HandleICETrickleSDPFragment(sdpFragment string) error { + return t.publisher.HandleICETrickleSDPFragment(sdpFragment) +} + +func (t *TransportManager) HandleICERestartSDPFragment(sdpFragment string) (string, error) { + return t.publisher.HandleICERestartSDPFragment(sdpFragment) } func (t *TransportManager) ProcessPendingPublisherOffer() { t.lock.Lock() pendingOffer := t.pendingOfferPublisher t.pendingOfferPublisher = nil + + pendingOfferId := t.pendingOfferIdPublisher + t.pendingOfferIdPublisher = 0 t.lock.Unlock() if pendingOffer != nil { - t.HandleOffer(*pendingOffer, false) + t.HandleOffer(*pendingOffer, pendingOfferId, false) } } -func (t *TransportManager) HandleAnswer(answer webrtc.SessionDescription) { - t.subscriber.HandleRemoteDescription(answer) +func (t *TransportManager) HandleAnswer(answer webrtc.SessionDescription, answerId uint32) { + t.subscriber.HandleRemoteDescription(answer, answerId) } // AddICECandidate adds candidates for remote peer func (t *TransportManager) AddICECandidate(candidate webrtc.ICECandidateInit, target livekit.SignalTarget) { switch target { case livekit.SignalTarget_PUBLISHER: - t.publisher.AddICECandidate(candidate) + if t.publisher != nil { + t.publisher.AddICECandidate(candidate) + } else { + t.params.Logger.Warnw("ice candidate for publisher, but no peer connection", nil, "candidate", candidate) + } case livekit.SignalTarget_SUBSCRIBER: - t.subscriber.AddICECandidate(candidate) + if t.subscriber != nil { + t.subscriber.AddICECandidate(candidate) + } else { + t.params.Logger.Warnw("ice candidate for subscriber, but no peer connection", nil, "candidate", candidate) + } default: err := errors.New("unknown signal target") t.params.Logger.Errorw("ice candidate for unknown signal target", err, "target", target) @@ -467,7 +567,11 @@ func (t *TransportManager) AddICECandidate(candidate webrtc.ICECandidateInit, ta } func (t *TransportManager) NegotiateSubscriber(force bool) { - t.subscriber.Negotiate(force) + if t.subscriber != nil { + t.subscriber.Negotiate(force) + } else { + t.publisher.Negotiate(force) + } } func (t *TransportManager) HandleClientReconnect(reason livekit.ReconnectReason) { @@ -478,12 +582,16 @@ func (t *TransportManager) HandleClientReconnect(reason livekit.ReconnectReason) ) switch reason { case livekit.ReconnectReason_RR_PUBLISHER_FAILED: - resetShortConnection = true - isShort, duration = t.publisher.IsShortConnection(time.Now()) + if t.publisher != nil { + resetShortConnection = true + isShort, duration = t.publisher.IsShortConnection(time.Now()) + } case livekit.ReconnectReason_RR_SUBSCRIBER_FAILED: - resetShortConnection = true - isShort, duration = t.subscriber.IsShortConnection(time.Now()) + if t.subscriber != nil { + resetShortConnection = true + isShort, duration = t.subscriber.IsShortConnection(time.Now()) + } } if isShort { @@ -495,15 +603,23 @@ func (t *TransportManager) HandleClientReconnect(reason livekit.ReconnectReason) } if resetShortConnection { - t.publisher.ResetShortConnOnICERestart() - t.subscriber.ResetShortConnOnICERestart() + if t.publisher != nil { + t.publisher.ResetShortConnOnICERestart() + } + if t.subscriber != nil { + t.subscriber.ResetShortConnOnICERestart() + } } } func (t *TransportManager) ICERestart(iceConfig *livekit.ICEConfig) error { t.SetICEConfig(iceConfig) - return t.subscriber.ICERestart() + if t.subscriber != nil { + return t.subscriber.ICERestart() + } + + return nil } func (t *TransportManager) OnICEConfigChanged(f func(iceConfig *livekit.ICEConfig)) { @@ -555,8 +671,12 @@ func (t *TransportManager) configureICE(iceConfig *livekit.ICEConfig, reset bool t.mediaLossProxy.OnMediaLossUpdate(nil) } - t.publisher.SetPreferTCP(iceConfig.PreferencePublisher == livekit.ICECandidateType_ICT_TCP) - t.subscriber.SetPreferTCP(iceConfig.PreferenceSubscriber == livekit.ICECandidateType_ICT_TCP) + if t.publisher != nil { + t.publisher.SetPreferTCP(iceConfig.PreferencePublisher == livekit.ICECandidateType_ICT_TCP) + } + if t.subscriber != nil { + t.subscriber.SetPreferTCP(iceConfig.PreferenceSubscriber == livekit.ICECandidateType_ICT_TCP) + } if onICEConfigChanged != nil { onICEConfigChanged(iceConfig) @@ -570,6 +690,10 @@ func (t *TransportManager) SubscriberAsPrimary() bool { func (t *TransportManager) GetICEConnectionInfo() []*types.ICEConnectionInfo { infos := make([]*types.ICEConnectionInfo, 0, 2) for _, pc := range []*PCTransport{t.publisher, t.subscriber} { + if pc == nil { + continue + } + info := pc.GetICEConnectionInfo() if info.HasCandidates() { infos = append(infos, info) @@ -578,13 +702,43 @@ func (t *TransportManager) GetICEConnectionInfo() []*types.ICEConnectionInfo { return infos } -func (t *TransportManager) getTransport(isPrimary bool) *PCTransport { - pcTransport := t.publisher - if (isPrimary && t.params.SubscriberAsPrimary) || (!isPrimary && !t.params.SubscriberAsPrimary) { - pcTransport = t.subscriber - } +func (t *TransportManager) GetDataTrackTransport() types.DataTrackTransport { + return t.getTransport(true) +} - return pcTransport +func (t *TransportManager) getTransport(isPrimary bool) *PCTransport { + switch { + case t.publisher == nil: + return t.subscriber + + case t.subscriber == nil: + return t.publisher + + default: + pcTransport := t.publisher + if (isPrimary && t.params.SubscriberAsPrimary) || (!isPrimary && !t.params.SubscriberAsPrimary) { + pcTransport = t.subscriber + } + + return pcTransport + } +} + +func (t *TransportManager) getLowestPriorityConnectionType() types.ICEConnectionType { + switch { + case t.publisher == nil: + return t.subscriber.GetICEConnectionType() + + case t.subscriber == nil: + return t.publisher.GetICEConnectionType() + + default: + ctype := t.publisher.GetICEConnectionType() + if stype := t.subscriber.GetICEConnectionType(); stype > ctype { + ctype = stype + } + return ctype + } } func (t *TransportManager) handleConnectionFailed(isShortLived bool) { @@ -614,6 +768,8 @@ func (t *TransportManager) handleConnectionFailed(isShortLived bool) { return } + lowestPriorityConnectionType := t.getLowestPriorityConnectionType() + // // Checking only `PreferenceSubscriber` field although any connection failure (PUBLISHER OR SUBSCRIBER) will // flow through here. @@ -621,13 +777,36 @@ func (t *TransportManager) handleConnectionFailed(isShortLived bool) { // As both transports are switched to the same type on any failure, checking just subscriber should be fine. // getNext := func(ic *livekit.ICEConfig) livekit.ICECandidateType { - if ic.PreferenceSubscriber == livekit.ICECandidateType_ICT_NONE && t.params.ClientInfo.SupportsICETCP() && t.canUseICETCP() { - return livekit.ICECandidateType_ICT_TCP - } else if ic.PreferenceSubscriber != livekit.ICECandidateType_ICT_TLS && t.params.TURNSEnabled { - return livekit.ICECandidateType_ICT_TLS - } else { - return livekit.ICECandidateType_ICT_NONE + switch lowestPriorityConnectionType { + case types.ICEConnectionTypeUDP: + // try ICE/TCP if ICE/UDP failed + if ic.PreferenceSubscriber == livekit.ICECandidateType_ICT_NONE { + if t.params.ClientInfo.SupportsICETCP() && t.canUseICETCP() { + return livekit.ICECandidateType_ICT_TCP + } else if t.params.TURNSEnabled { + // fallback to TURN/TLS if TCP is not supported + return livekit.ICECandidateType_ICT_TLS + } + } + + case types.ICEConnectionTypeTCP: + // try TURN/TLS if ICE/TCP failed, + // the configuration could have been ICT_NONE or ICT_TCP, + // in either case, fallback to TURN/TLS + if t.params.TURNSEnabled { + return livekit.ICECandidateType_ICT_TLS + } else { + // keep the current config + return ic.PreferenceSubscriber + } + + case types.ICEConnectionTypeTURN: + // TURN/TLS is the most permissive option, if that fails there is nowhere to go to + // the configuration could have been ICT_NONE or ICT_TLS, + // keep the current config + return ic.PreferenceSubscriber } + return livekit.ICECandidateType_ICT_NONE } var preferNext livekit.ICECandidateType @@ -672,7 +851,11 @@ func (t *TransportManager) handleConnectionFailed(isShortLived bool) { }, false) } -func (t *TransportManager) SetMigrateInfo(previousOffer, previousAnswer *webrtc.SessionDescription, dataChannels []*livekit.DataChannelInfo) { +func (t *TransportManager) SetMigrateInfo( + previousOffer *webrtc.SessionDescription, + previousAnswer *webrtc.SessionDescription, + dataChannels []*livekit.DataChannelInfo, +) { t.lock.Lock() t.pendingDataChannelsPublisher = make([]*livekit.DataChannelInfo, 0, len(dataChannels)) pendingDataChannelsSubscriber := make([]*livekit.DataChannelInfo, 0, len(dataChannels)) @@ -691,7 +874,11 @@ func (t *TransportManager) SetMigrateInfo(previousOffer, previousAnswer *webrtc. } } - t.subscriber.SetPreviousSdp(previousOffer, previousAnswer) + if t.params.UseSinglePeerConnection { + t.publisher.SetPreviousSdp(previousAnswer, previousOffer) + } else { + t.subscriber.SetPreviousSdp(previousOffer, previousAnswer) + } } func (t *TransportManager) ProcessPendingPublisherDataChannels() { @@ -710,7 +897,16 @@ func (t *TransportManager) ProcessPendingPublisherDataChannels() { dcExisting bool err error ) - if ci.Label == LossyDataChannel { + switch ci.Label { + case ReliableDataChannel: + id := uint16(ci.GetId()) + dcLabel, dcID, dcExisting, err = t.publisher.CreateDataChannelIfEmpty(ReliableDataChannel, &webrtc.DataChannelInit{ + Ordered: &ordered, + Negotiated: &negotiated, + ID: &id, + }) + case LossyDataChannel: + ordered = false retransmits := uint16(0) id := uint16(ci.GetId()) dcLabel, dcID, dcExisting, err = t.publisher.CreateDataChannelIfEmpty(LossyDataChannel, &webrtc.DataChannelInit{ @@ -719,12 +915,15 @@ func (t *TransportManager) ProcessPendingPublisherDataChannels() { Negotiated: &negotiated, ID: &id, }) - } else if ci.Label == ReliableDataChannel { + case DataTrackDataChannel: + ordered = false + retransmits := uint16(0) id := uint16(ci.GetId()) - dcLabel, dcID, dcExisting, err = t.publisher.CreateDataChannelIfEmpty(ReliableDataChannel, &webrtc.DataChannelInit{ - Ordered: &ordered, - Negotiated: &negotiated, - ID: &id, + dcLabel, dcID, dcExisting, err = t.publisher.CreateDataChannelIfEmpty(DataTrackDataChannel, &webrtc.DataChannelInit{ + Ordered: &ordered, + MaxRetransmits: &retransmits, + Negotiated: &negotiated, + ID: &id, }) } if err != nil { @@ -755,7 +954,11 @@ func (t *TransportManager) onMediaLossUpdate(loss uint8) { t.lock.Unlock() t.params.Logger.Infow("udp connection unstable, switch to tcp", "signalingRTT", t.signalingRTT) - t.params.SubscriberHandler.OnFailed(true, t.subscriber.GetICEConnectionInfo()) + if t.params.UseSinglePeerConnection { + t.params.PublisherHandler.OnFailed(true, t.publisher.GetICEConnectionInfo()) + } else { + t.params.SubscriberHandler.OnFailed(true, t.subscriber.GetICEConnectionInfo()) + } return } } @@ -767,8 +970,12 @@ func (t *TransportManager) UpdateSignalingRTT(rtt uint32) { t.lock.Lock() t.signalingRTT = rtt t.lock.Unlock() - t.publisher.SetSignalingRTT(rtt) - t.subscriber.SetSignalingRTT(rtt) + if t.publisher != nil { + t.publisher.SetSignalingRTT(rtt) + } + if t.subscriber != nil { + t.subscriber.SetSignalingRTT(rtt) + } // TODO: considering using tcp rtt to calculate ice connection cost, if ice connection can't be established // within 5 * tcp rtt(at least 5s), means udp traffic might be block/dropped, switch to tcp. @@ -813,13 +1020,25 @@ func (t *TransportManager) SetSignalSourceValid(valid bool) { } func (t *TransportManager) SetSubscriberAllowPause(allowPause bool) { - t.subscriber.SetAllowPauseOfStreamAllocator(allowPause) + if t.params.UseOneShotSignallingMode || t.params.UseSinglePeerConnection { + t.publisher.SetAllowPauseOfStreamAllocator(allowPause) + } else { + t.subscriber.SetAllowPauseOfStreamAllocator(allowPause) + } } func (t *TransportManager) SetSubscriberChannelCapacity(channelCapacity int64) { - t.subscriber.SetChannelCapacityOfStreamAllocator(channelCapacity) + if t.params.UseOneShotSignallingMode || t.params.UseSinglePeerConnection { + t.publisher.SetChannelCapacityOfStreamAllocator(channelCapacity) + } else { + t.subscriber.SetChannelCapacityOfStreamAllocator(channelCapacity) + } } func (t *TransportManager) hasRecentSignalLocked() bool { return time.Since(t.lastSignalAt) < PingTimeoutSeconds*time.Second } + +func (t *TransportManager) RTPStreamPublished(ssrc uint32, mid, rid string) { + t.publisher.RTPStreamPublished(ssrc, mid, rid) +} diff --git a/pkg/rtc/types/ice.go b/pkg/rtc/types/ice.go index 3deede9..a78237d 100644 --- a/pkg/rtc/types/ice.go +++ b/pkg/rtc/types/ice.go @@ -18,30 +18,63 @@ package types import ( "fmt" + "slices" "strings" "sync" "github.com/pion/ice/v4" "github.com/pion/webrtc/v4" - "golang.org/x/exp/slices" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" + "github.com/livekit/protocol/observability/roomobs" ) -type ICEConnectionType string +type ICEConnectionType int const ( - ICEConnectionTypeUDP ICEConnectionType = "udp" - ICEConnectionTypeTCP ICEConnectionType = "tcp" - ICEConnectionTypeTURN ICEConnectionType = "turn" - ICEConnectionTypeUnknown ICEConnectionType = "unknown" + // this is in ICE priority highest -> lowest ordering + // WARNING: Keep this ordering as it is used to find lowest priority connection type. + ICEConnectionTypeUnknown ICEConnectionType = iota + ICEConnectionTypeUDP + ICEConnectionTypeTCP + ICEConnectionTypeTURN ) +func (i ICEConnectionType) String() string { + switch i { + case ICEConnectionTypeUnknown: + return "unknown" + case ICEConnectionTypeUDP: + return "udp" + case ICEConnectionTypeTCP: + return "tcp" + case ICEConnectionTypeTURN: + return "turn" + default: + return "unknown" + } +} + +func (i ICEConnectionType) ReporterType() roomobs.ConnectionType { + switch i { + case ICEConnectionTypeUnknown: + return roomobs.ConnectionTypeUndefined + case ICEConnectionTypeUDP: + return roomobs.ConnectionTypeUDP + case ICEConnectionTypeTCP: + return roomobs.ConnectionTypeTCP + case ICEConnectionTypeTURN: + return roomobs.ConnectionTypeTurn + default: + return roomobs.ConnectionTypeUndefined + } +} + +// -------------------------------------------- + type ICECandidateExtended struct { - // only one of local or remote is set. This is due to type foo in Pion - Local *webrtc.ICECandidate - Remote ice.Candidate + Candidate *webrtc.ICECandidate SelectedOrder int Filtered bool Trickle bool @@ -60,6 +93,15 @@ func (i *ICEConnectionInfo) HasCandidates() bool { return len(i.Local) > 0 || len(i.Remote) > 0 } +func ICEConnectionInfosType(infos []*ICEConnectionInfo) ICEConnectionType { + for _, info := range infos { + if info.Type != ICEConnectionTypeUnknown { + return info.Type + } + } + return ICEConnectionTypeUnknown +} + // -------------------------------------------- type ICEConnectionDetails struct { @@ -91,7 +133,7 @@ func (d *ICEConnectionDetails) GetInfo() *ICEConnectionInfo { } for _, c := range d.Local { info.Local = append(info.Local, &ICECandidateExtended{ - Local: c.Local, + Candidate: c.Candidate, Filtered: c.Filtered, SelectedOrder: c.SelectedOrder, Trickle: c.Trickle, @@ -99,7 +141,7 @@ func (d *ICEConnectionDetails) GetInfo() *ICEConnectionInfo { } for _, c := range d.Remote { info.Remote = append(info.Remote, &ICECandidateExtended{ - Remote: c.Remote, + Candidate: c.Candidate, Filtered: c.Filtered, SelectedOrder: c.SelectedOrder, Trickle: c.Trickle, @@ -108,19 +150,26 @@ func (d *ICEConnectionDetails) GetInfo() *ICEConnectionInfo { return info } +func (d *ICEConnectionDetails) GetConnectionType() ICEConnectionType { + d.lock.Lock() + defer d.lock.Unlock() + + return d.Type +} + func (d *ICEConnectionDetails) AddLocalCandidate(c *webrtc.ICECandidate, filtered, trickle bool) { d.lock.Lock() defer d.lock.Unlock() compFn := func(e *ICECandidateExtended) bool { - return isCandidateEqualTo(e.Local, c) + return isCandidateEqualTo(e.Candidate, c) } if slices.ContainsFunc(d.Local, compFn) { return } d.Local = append(d.Local, &ICECandidateExtended{ - Local: c, - Filtered: filtered, - Trickle: trickle, + Candidate: c, + Filtered: filtered, + Trickle: trickle, }) } @@ -135,24 +184,30 @@ func (d *ICEConnectionDetails) AddLocalICECandidate(c ice.Candidate, filtered, t } func (d *ICEConnectionDetails) AddRemoteCandidate(c webrtc.ICECandidateInit, filtered, trickle, canUpdate bool) { - candidate, err := unmarshalICECandidate(c) + iceCandidate, err := unmarshalICECandidate(c) if err != nil { d.logger.Errorw("could not unmarshal candidate", err, "candidate", c) return } - d.AddRemoteICECandidate(candidate, filtered, trickle, canUpdate) + d.AddRemoteICECandidate(iceCandidate, filtered, trickle, canUpdate) } -func (d *ICEConnectionDetails) AddRemoteICECandidate(candidate ice.Candidate, filtered, trickle, canUpdate bool) { - if candidate == nil { +func (d *ICEConnectionDetails) AddRemoteICECandidate(iceCandidate ice.Candidate, filtered, trickle, canUpdate bool) { + if iceCandidate == nil { // end-of-candidates candidate return } + candidate, err := unmarshalCandidate(iceCandidate) + if err != nil { + d.logger.Errorw("could not unmarshal ice candidate", err, "candidate", iceCandidate) + return + } + d.lock.Lock() defer d.lock.Unlock() indexFn := func(e *ICECandidateExtended) bool { - return isICECandidateEqualTo(e.Remote, candidate) + return isCandidateEqualTo(e.Candidate, candidate) } if idx := slices.IndexFunc(d.Remote, indexFn); idx != -1 { if canUpdate { @@ -162,10 +217,11 @@ func (d *ICEConnectionDetails) AddRemoteICECandidate(candidate ice.Candidate, fi return } d.Remote = append(d.Remote, &ICECandidateExtended{ - Remote: candidate, - Filtered: filtered, - Trickle: trickle, + Candidate: candidate, + Filtered: filtered, + Trickle: trickle, }) + d.updateConnectionTypeLocked() } func (d *ICEConnectionDetails) Clear() { @@ -183,56 +239,78 @@ func (d *ICEConnectionDetails) SetSelectedPair(pair *webrtc.ICECandidatePair) { d.selectedCount++ remoteIdx := slices.IndexFunc(d.Remote, func(e *ICECandidateExtended) bool { - return isICECandidateEqualToCandidate(e.Remote, pair.Remote) + return isCandidateEqualTo(e.Candidate, pair.Remote) }) if remoteIdx < 0 { // it's possible for prflx candidates to be generated by Pion, we'll add them - candidate, err := unmarshalICECandidate(pair.Remote.ToJSON()) - if err != nil { - d.logger.Errorw("could not unmarshal remote candidate", err, "candidate", pair.Remote) - return - } - if candidate == nil { - return - } d.Remote = append(d.Remote, &ICECandidateExtended{ - Remote: candidate, - Filtered: false, - Trickle: true, + Candidate: pair.Remote, + Filtered: false, + Trickle: false, }) remoteIdx = len(d.Remote) - 1 } - remote := d.Remote[remoteIdx] - remote.SelectedOrder = d.selectedCount + d.Remote[remoteIdx].SelectedOrder = d.selectedCount + d.updateConnectionTypeLocked() localIdx := slices.IndexFunc(d.Local, func(e *ICECandidateExtended) bool { - return isCandidateEqualTo(e.Local, pair.Local) + return isCandidateEqualTo(e.Candidate, pair.Local) }) if localIdx < 0 { d.logger.Errorw("could not match local candidate", nil, "local", pair.Local) // should not happen return } - local := d.Local[localIdx] - local.SelectedOrder = d.selectedCount + d.Local[localIdx].SelectedOrder = d.selectedCount +} - d.Type = ICEConnectionTypeUDP - if pair.Remote.Protocol == webrtc.ICEProtocolTCP { +func (d *ICEConnectionDetails) updateConnectionTypeLocked() { + highestSelectedOrder := -1 + var selectedRemoteCandidate *ICECandidateExtended + for _, remote := range d.Remote { + if remote.SelectedOrder == 0 { + continue + } + + if remote.SelectedOrder > highestSelectedOrder { + highestSelectedOrder = remote.SelectedOrder + selectedRemoteCandidate = remote + } + } + + if selectedRemoteCandidate == nil { + return + } + + remoteCandidate := selectedRemoteCandidate.Candidate + switch remoteCandidate.Protocol { + case webrtc.ICEProtocolUDP: + d.Type = ICEConnectionTypeUDP + + case webrtc.ICEProtocolTCP: d.Type = ICEConnectionTypeTCP } - if pair.Remote.Typ == webrtc.ICECandidateTypeRelay { + + switch remoteCandidate.Typ { + case webrtc.ICECandidateTypeRelay: d.Type = ICEConnectionTypeTURN - } else if pair.Remote.Typ == webrtc.ICECandidateTypePrflx { + + case webrtc.ICECandidateTypePrflx: // if the remote relay candidate pings us *before* we get a relay candidate, // Pion would have created a prflx candidate with the same address as the relay candidate. // to report an accurate connection type, we'll compare to see if existing relay candidates match for _, other := range d.Remote { - or := other.Remote - if or.Type() == ice.CandidateTypeRelay && - pair.Remote.Address == or.Address() && - pair.Remote.Port == uint16(or.Port()) && - pair.Remote.Protocol.String() == or.NetworkType().NetworkShort() { + or := other.Candidate + if or.Typ == webrtc.ICECandidateTypeRelay && + remoteCandidate.Address == or.Address && + // NOTE: port is not compared as relayed address reported by TURN ALLOCATE from + // pion/turn server -> client and later sent from client -> server via ICE Trickle does not + // match port of `prflx` candidate learnt via TURN path. TODO-INVESTIGATE: how and why doesn't + // port match? + // remoteCandidate.Port == or.Port && + remoteCandidate.Protocol == or.Protocol { d.Type = ICEConnectionTypeTURN + break } } } @@ -258,39 +336,6 @@ func isCandidateEqualTo(c1, c2 *webrtc.ICECandidate) bool { c1.TCPType == c2.TCPType } -func isICECandidateEqualTo(c1, c2 ice.Candidate) bool { - if c1 == nil && c2 == nil { - return true - } - if (c1 == nil && c2 != nil) || (c1 != nil && c2 == nil) { - return false - } - return c1.Type() == c2.Type() && - c1.NetworkType() == c2.NetworkType() && - c1.Address() == c2.Address() && - c1.Port() == c2.Port() && - c1.Foundation() == c2.Foundation() && - c1.Priority() == c2.Priority() && - c1.RelatedAddress().Equal(c2.RelatedAddress()) && - c1.TCPType() == c2.TCPType() -} - -func isICECandidateEqualToCandidate(c1 ice.Candidate, c2 *webrtc.ICECandidate) bool { - if c1 == nil && c2 == nil { - return true - } - if (c1 == nil && c2 != nil) || (c1 != nil && c2 == nil) { - return false - } - return c1.Type().String() == c2.Typ.String() && - c1.NetworkType().NetworkShort() == c2.Protocol.String() && - c1.Address() == c2.Address && - c1.Port() == int(c2.Port) && - c1.Foundation() == c2.Foundation && - c1.Priority() == c2.Priority && - c1.TCPType().String() == c2.TCPType -} - func unmarshalICECandidate(c webrtc.ICECandidateInit) (ice.Candidate, error) { candidateValue := strings.TrimPrefix(c.Candidate, "candidate:") if candidateValue == "" { @@ -306,28 +351,14 @@ func unmarshalICECandidate(c webrtc.ICECandidateInit) (ice.Candidate, error) { } func unmarshalCandidate(i ice.Candidate) (*webrtc.ICECandidate, error) { - var typ webrtc.ICECandidateType - switch i.Type() { - case ice.CandidateTypeHost: - typ = webrtc.ICECandidateTypeHost - case ice.CandidateTypeServerReflexive: - typ = webrtc.ICECandidateTypeSrflx - case ice.CandidateTypePeerReflexive: - typ = webrtc.ICECandidateTypePrflx - case ice.CandidateTypeRelay: - typ = webrtc.ICECandidateTypeRelay - default: - return nil, fmt.Errorf("unknown candidate type: %s", i.Type()) + typ, err := convertTypeFromICE(i.Type()) + if err != nil { + return nil, err } - var protocol webrtc.ICEProtocol - switch strings.ToLower(i.NetworkType().NetworkShort()) { - case "udp": - protocol = webrtc.ICEProtocolUDP - case "tcp": - protocol = webrtc.ICEProtocolTCP - default: - return nil, fmt.Errorf("unknown network type: %s", i.NetworkType()) + protocol, err := webrtc.NewICEProtocol(i.NetworkType().NetworkShort()) + if err != nil { + return nil, err } c := webrtc.ICECandidate{ @@ -349,6 +380,21 @@ func unmarshalCandidate(i ice.Candidate) (*webrtc.ICECandidate, error) { return &c, nil } +func convertTypeFromICE(t ice.CandidateType) (webrtc.ICECandidateType, error) { + switch t { + case ice.CandidateTypeHost: + return webrtc.ICECandidateTypeHost, nil + case ice.CandidateTypeServerReflexive: + return webrtc.ICECandidateTypeSrflx, nil + case ice.CandidateTypePeerReflexive: + return webrtc.ICECandidateTypePrflx, nil + case ice.CandidateTypeRelay: + return webrtc.ICECandidateTypeRelay, nil + default: + return webrtc.ICECandidateType(t), fmt.Errorf("unknown ice candidate type: %s", t) + } +} + func IsCandidateMDNS(candidate webrtc.ICECandidateInit) bool { c, err := unmarshalICECandidate(candidate) if err != nil { @@ -364,5 +410,5 @@ func IsICECandidateMDNS(candidate ice.Candidate) bool { return false } - return strings.HasSuffix(candidate.Address(), ".local") + return strings.HasSuffix(candidate.Address(), ".local") || strings.HasSuffix(candidate.Address(), ".invalid") } diff --git a/pkg/rtc/types/interfaces.go b/pkg/rtc/types/interfaces.go index d44bb32..e77393e 100644 --- a/pkg/rtc/types/interfaces.go +++ b/pkg/rtc/types/interfaces.go @@ -22,15 +22,20 @@ import ( "github.com/pion/webrtc/v4" "github.com/livekit/protocol/auth" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" + "github.com/livekit/protocol/observability/roomobs" "github.com/livekit/protocol/utils" "github.com/livekit/livekit-server/pkg/routing" + "github.com/livekit/livekit-server/pkg/rtc/datatrack" "github.com/livekit/livekit-server/pkg/sfu" "github.com/livekit/livekit-server/pkg/sfu/buffer" - "github.com/livekit/livekit-server/pkg/sfu/mime" "github.com/livekit/livekit-server/pkg/sfu/pacer" + "github.com/livekit/livekit-server/pkg/telemetry" + + "google.golang.org/protobuf/proto" ) //go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -generate @@ -111,6 +116,8 @@ const ( ParticipantCloseReasonRoomClosed ParticipantCloseReasonUserUnavailable ParticipantCloseReasonUserRejected + ParticipantCloseReasonMoveFailed + ParticipantCloseReasonAgentError ) func (p ParticipantCloseReason) String() string { @@ -169,6 +176,10 @@ func (p ParticipantCloseReason) String() string { return "USER_UNAVAILABLE" case ParticipantCloseReasonUserRejected: return "USER_REJECTED" + case ParticipantCloseReasonMoveFailed: + return "MOVE_FAILED" + case ParticipantCloseReasonAgentError: + return "AGENT_ERROR" default: return fmt.Sprintf("%d", int(p)) } @@ -184,7 +195,7 @@ func (p ParticipantCloseReason) ToDisconnectReason() livekit.DisconnectReason { // expected to be connected but is not return livekit.DisconnectReason_JOIN_FAILURE case ParticipantCloseReasonPeerConnectionDisconnected: - return livekit.DisconnectReason_STATE_MISMATCH + return livekit.DisconnectReason_CONNECTION_TIMEOUT case ParticipantCloseReasonDuplicateIdentity, ParticipantCloseReasonStale: return livekit.DisconnectReason_DUPLICATE_IDENTITY case ParticipantCloseReasonMigrationRequested, ParticipantCloseReasonMigrationComplete, ParticipantCloseReasonSimulateMigration: @@ -195,7 +206,8 @@ func (p ParticipantCloseReason) ToDisconnectReason() livekit.DisconnectReason { return livekit.DisconnectReason_ROOM_DELETED case ParticipantCloseReasonSimulateNodeFailure, ParticipantCloseReasonSimulateServerLeave: return livekit.DisconnectReason_SERVER_SHUTDOWN - case ParticipantCloseReasonNegotiateFailed, ParticipantCloseReasonPublicationError, ParticipantCloseReasonSubscriptionError, ParticipantCloseReasonDataChannelError, ParticipantCloseReasonMigrateCodecMismatch: + case ParticipantCloseReasonNegotiateFailed, ParticipantCloseReasonPublicationError, ParticipantCloseReasonSubscriptionError, + ParticipantCloseReasonDataChannelError, ParticipantCloseReasonMigrateCodecMismatch, ParticipantCloseReasonMoveFailed: return livekit.DisconnectReason_STATE_MISMATCH case ParticipantCloseReasonSignalSourceClose: return livekit.DisconnectReason_SIGNAL_CLOSE @@ -205,12 +217,31 @@ func (p ParticipantCloseReason) ToDisconnectReason() livekit.DisconnectReason { return livekit.DisconnectReason_USER_UNAVAILABLE case ParticipantCloseReasonUserRejected: return livekit.DisconnectReason_USER_REJECTED + case ParticipantCloseReasonAgentError: + return livekit.DisconnectReason_AGENT_ERROR default: // the other types will map to unknown reason return livekit.DisconnectReason_UNKNOWN_REASON } } +// IsIntentionalDisconnect reports whether a disconnect reason represents an +// intentional/expected closure (client leaving, admin action, room teardown, +// migration, etc.) as opposed to a connection failure. +func IsIntentionalDisconnect(reason livekit.DisconnectReason) bool { + switch reason { + case livekit.DisconnectReason_CLIENT_INITIATED, + livekit.DisconnectReason_SERVER_SHUTDOWN, + livekit.DisconnectReason_DUPLICATE_IDENTITY, + livekit.DisconnectReason_MIGRATION, + livekit.DisconnectReason_PARTICIPANT_REMOVED, + livekit.DisconnectReason_ROOM_DELETED, + livekit.DisconnectReason_ROOM_CLOSED: + return true + } + return false +} + // --------------------------------------------- type SignallingCloseReason int @@ -258,6 +289,12 @@ func (s SignallingCloseReason) String() string { } } +// --------------------------------------------- +const ( + ParticipantCloseKeyNormal = "normal" + ParticipantCloseKeyWHIP = "whip" +) + // --------------------------------------------- //counterfeiter:generate . Participant @@ -268,18 +305,26 @@ type Participant interface { ConnectedAt() time.Time CloseReason() ParticipantCloseReason Kind() livekit.ParticipantInfo_Kind + KindDetails() []livekit.ParticipantInfo_KindDetail IsRecorder() bool IsDependent() bool IsAgent() bool + GetLogger() logger.Logger + CanSkipBroadcast() bool Version() utils.TimedVersion ToProto() *livekit.ParticipantInfo + ToProtoWithVersion() (*livekit.ParticipantInfo, utils.TimedVersion) IsPublisher() bool GetPublishedTrack(trackID livekit.TrackID) MediaTrack GetPublishedTracks() []MediaTrack - RemovePublishedTrack(track MediaTrack, isExpectedToResume bool, shouldClose bool) + RemovePublishedTrack(track MediaTrack, isExpectedToResume bool) + + GetPublishedDataTracks() []DataTrack + GetPublishedDataTrack(handle uint16) DataTrack + RemovePublishedDataTrack(track DataTrack) GetAudioLevel() (smoothedLevel float64, active bool) @@ -290,7 +335,11 @@ type Participant interface { // permissions Hidden() bool + MigrateState() MigrateState + Close(sendLeave bool, reason ParticipantCloseReason, isExpectedToResume bool) error + IsClosed() bool + IsDisconnected() bool SubscriptionPermission() (*livekit.SubscriptionPermission, utils.TimedVersion) @@ -301,9 +350,11 @@ type Participant interface { resolverBySid func(participantID livekit.ParticipantID) LocalParticipant, ) error - DebugInfo() map[string]interface{} + DebugInfo() map[string]any - OnMetrics(callback func(Participant, *livekit.DataPacket)) + HandleReceivedDataTrackMessage([]byte, *datatrack.Packet, int64) + + GetParticipantListener() ParticipantListener } // ------------------------------------------------------- @@ -313,22 +364,51 @@ type AddTrackParams struct { Red bool } +type MoveToRoomParams struct { + RoomName livekit.RoomName + ParticipantID livekit.ParticipantID + Listener LocalParticipantListener + Helper LocalParticipantHelper +} + +type DataMessageCache struct { + Data []byte + SenderID livekit.ParticipantID + Seq uint32 + DestIdentities []livekit.ParticipantIdentity +} + +//counterfeiter:generate . LocalParticipantHelper +type LocalParticipantHelper interface { + ResolveMediaTrack(LocalParticipant, livekit.TrackID) MediaResolverResult + ResolveDataTrack(LocalParticipant, livekit.TrackID) DataResolverResult + GetParticipantInfo(pID livekit.ParticipantID) *livekit.ParticipantInfo + GetRegionSettings(ip string) *livekit.RegionSettings + GetSubscriberForwarderState(p LocalParticipant) (map[livekit.TrackID]*livekit.RTPForwarderState, error) + ShouldRegressCodec() bool + GetCachedReliableDataMessage(seqs map[livekit.ParticipantID]uint32) []*DataMessageCache +} + //counterfeiter:generate . LocalParticipant type LocalParticipant interface { Participant - ToProtoWithVersion() (*livekit.ParticipantInfo, utils.TimedVersion) + TelemetryGuard() *telemetry.ReferenceGuard + GetTelemetryListener() ParticipantTelemetryListener // getters + GetCountry() string GetTrailer() []byte - GetLogger() logger.Logger + GetLoggerResolver() logger.DeferredFieldResolver + GetReporter() roomobs.ParticipantSessionReporter + GetReporterResolver() roomobs.ParticipantReporterResolver GetAdaptiveStream() bool ProtocolVersion() ProtocolVersion SupportsSyncStreamID() bool - SupportsTransceiverReuse() bool - IsClosed() bool + SupportsTransceiverReuse(mt MediaTrack) bool + IsUsingSinglePeerConnection() bool IsReady() bool - IsDisconnected() bool + ActiveAt() time.Time Disconnected() <-chan struct{} IsIdle() bool SubscriberAsPrimary() bool @@ -340,15 +420,19 @@ type LocalParticipant interface { GetICEConnectionInfo() []*ICEConnectionInfo HasConnected() bool GetEnabledPublishCodecs() []*livekit.Codec + GetPublisherICESessionUfrag() (string, error) + SupportsMoving() error + GetLastReliableSequence(migrateOut bool) uint32 - SetResponseSink(sink routing.MessageSink) + SwapResponseSink(sink routing.MessageSink, reason SignallingCloseReason) + GetResponseSink() routing.MessageSink CloseSignalConnection(reason SignallingCloseReason) UpdateLastSeenSignal() SetSignalSourceValid(valid bool) HandleSignalSourceClose() // updates - CheckMetadataLimits(name string, metadata string, attributes map[string]string) error + UpdateMetadata(update *livekit.UpdateParticipantMetadata, fromAdmin bool) error SetName(name string) SetMetadata(metadata string) SetAttributes(attributes map[string]string) @@ -364,13 +448,15 @@ type LocalParticipant interface { CanPublishData() bool // PeerConnection - AddICECandidate(candidate webrtc.ICECandidateInit, target livekit.SignalTarget) - HandleOffer(sdp webrtc.SessionDescription) error - GetAnswer() (webrtc.SessionDescription, error) + HandleICETrickle(trickleRequest *livekit.TrickleRequest) + HandleOffer(sd *livekit.SessionDescription) error + GetAnswer() (webrtc.SessionDescription, uint32, error) + HandleICETrickleSDPFragment(sdpFragment string) error + HandleICERestartSDPFragment(sdpFragment string) (string, error) AddTrack(req *livekit.AddTrackRequest) - SetTrackMuted(trackID livekit.TrackID, muted bool, fromAdmin bool) *livekit.TrackInfo + SetTrackMuted(mute *livekit.MuteTrackRequest, fromAdmin bool) *livekit.TrackInfo - HandleAnswer(sdp webrtc.SessionDescription) + HandleAnswer(sd *livekit.SessionDescription) Negotiate(force bool) ICERestart(iceConfig *livekit.ICEConfig) AddTrackLocal(trackLocal webrtc.TrackLocal, params AddTrackParams) (*webrtc.RTPSender, *webrtc.RTPTransceiver, error) @@ -380,11 +466,14 @@ type LocalParticipant interface { WriteSubscriberRTCP(pkts []rtcp.Packet) error // subscriptions - SubscribeToTrack(trackID livekit.TrackID) + SubscribeToTrack(trackID livekit.TrackID, isSync bool) UnsubscribeFromTrack(trackID livekit.TrackID) UpdateSubscribedTrackSettings(trackID livekit.TrackID, settings *livekit.UpdateTrackSettings) GetSubscribedTracks() []SubscribedTrack IsTrackNameSubscribed(publisherIdentity livekit.ParticipantIdentity, trackName string) bool + SubscribeToDataTrack(trackID livekit.TrackID) + UnsubscribeFromDataTrack(trackID livekit.TrackID) + UpdateDataTrackSubscriptionOptions(trackID livekit.TrackID, subscriptionOptions *livekit.DataTrackSubscriptionOptions) Verify() bool VerifySubscribeParticipantInfo(pID livekit.ParticipantID, version uint32) // WaitUntilSubscribed waits until all subscriptions have been settled, or if the timeout @@ -403,29 +492,18 @@ type LocalParticipant interface { SendJoinResponse(joinResponse *livekit.JoinResponse) error SendParticipantUpdate(participants []*livekit.ParticipantInfo) error SendSpeakerUpdate(speakers []*livekit.SpeakerInfo, force bool) error - SendDataPacket(kind livekit.DataPacket_Kind, encoded []byte) error + SendDataMessage(kind livekit.DataPacket_Kind, data []byte, senderID livekit.ParticipantID, seq uint32) error + SendDataMessageUnlabeled(data []byte, useRaw bool, sender livekit.ParticipantIdentity) error SendRoomUpdate(room *livekit.Room) error SendConnectionQualityUpdate(update *livekit.ConnectionQualityUpdate) error - SubscriptionPermissionUpdate(publisherID livekit.ParticipantID, trackID livekit.TrackID, allowed bool) + SendSubscriptionPermissionUpdate(publisherID livekit.ParticipantID, trackID livekit.TrackID, allowed bool) error SendRefreshToken(token string) error - SendRequestResponse(requestResponse *livekit.RequestResponse) error HandleReconnectAndSendResponse(reconnectReason livekit.ReconnectReason, reconnectResponse *livekit.ReconnectResponse) error IssueFullReconnect(reason ParticipantCloseReason) + SendRoomMovedResponse(moved *livekit.RoomMovedResponse) error + SendDataTrackSubscriberHandles(handles map[uint32]*livekit.DataTrackSubscriberHandles_PublishedDataTrack) error - // callbacks - OnStateChange(func(p LocalParticipant, state livekit.ParticipantInfo_State)) - OnMigrateStateChange(func(p LocalParticipant, migrateState MigrateState)) - // OnTrackPublished - remote added a track - OnTrackPublished(func(LocalParticipant, MediaTrack)) - // OnTrackUpdated - one of its publishedTracks changed in status - OnTrackUpdated(callback func(LocalParticipant, MediaTrack)) - // OnTrackUnpublished - a track was unpublished - OnTrackUnpublished(callback func(LocalParticipant, MediaTrack)) - // OnParticipantUpdate - metadata or permission is updated - OnParticipantUpdate(callback func(LocalParticipant)) - OnDataPacket(callback func(LocalParticipant, livekit.DataPacket_Kind, *livekit.DataPacket)) - OnSubscribeStatusChanged(fn func(publisherID livekit.ParticipantID, subscribed bool)) - OnClose(callback func(LocalParticipant)) + AddOnClose(key string, callback func(LocalParticipant)) OnClaimsChanged(callback func(LocalParticipant)) HandleReceiverReport(dt *sfu.DownTrack, report *rtcp.ReceiverReport) @@ -434,13 +512,16 @@ type LocalParticipant interface { MaybeStartMigration(force bool, onStart func()) bool NotifyMigration() SetMigrateState(s MigrateState) - MigrateState() MigrateState SetMigrateInfo( - previousOffer, previousAnswer *webrtc.SessionDescription, + previousOffer *webrtc.SessionDescription, + previousAnswer *webrtc.SessionDescription, mediaTracks []*livekit.TrackPublishedResponse, dataChannels []*livekit.DataChannelInfo, + dataChannelReceiveState []*livekit.DataChannelReceiveState, + dataTracks []*livekit.PublishDataTrackResponse, ) IsReconnect() bool + MoveToRoom(params MoveToRoomParams) UpdateMediaRTT(rtt uint32) UpdateSignalingRTT(rtt uint32) @@ -454,6 +535,7 @@ type LocalParticipant interface { OnICEConfigChanged(callback func(participant LocalParticipant, iceConfig *livekit.ICEConfig)) UpdateSubscribedQuality(nodeID livekit.NodeID, trackID livekit.TrackID, maxQualities []SubscribedCodecQuality) error + UpdateSubscribedAudioCodecs(nodeID livekit.NodeID, trackID livekit.TrackID, codecs []*livekit.SubscribedAudioCodec) error UpdateMediaLoss(nodeID livekit.NodeID, trackID livekit.TrackID, fractionalLoss uint32) error // down stream bandwidth management @@ -465,8 +547,177 @@ type LocalParticipant interface { GetDisableSenderReportPassThrough() bool HandleMetrics(senderParticipantID livekit.ParticipantID, batch *livekit.MetricsBatch) error + HandleUpdateSubscriptions( + []livekit.TrackID, + []*livekit.ParticipantTracks, + bool, + ) + HandleUpdateSubscriptionPermission(*livekit.SubscriptionPermission) error + HandleSyncState(*livekit.SyncState) error + HandleSimulateScenario(*livekit.SimulateScenario) error + HandleLeaveRequest(reason ParticipantCloseReason) + + HandlePublishDataTrackRequest(*livekit.PublishDataTrackRequest) + HandleUnpublishDataTrackRequest(*livekit.UnpublishDataTrackRequest) + HandleUpdateDataSubscription(*livekit.UpdateDataSubscription) + + HandleSignalMessage(msg proto.Message) error + + PerformRpc(req *livekit.PerformRpcRequest, resultCh chan string, errorCh chan error) + + GetDataTrackTransport() DataTrackTransport + + ClearParticipantListener() + + GetNextSubscribedDataTrackHandle() uint16 } +// --------------------------------------------- + +//counterfeiter:generate . ParticipantListener +type ParticipantListener interface { + OnParticipantUpdate(Participant) + OnTrackPublished(Participant, MediaTrack) + OnTrackUpdated(Participant, MediaTrack) + OnTrackUnpublished(Participant, MediaTrack) + OnDataTrackPublished(Participant, DataTrack) + OnDataTrackUnpublished(Participant, DataTrack) + OnDataTrackMessage(Participant, []byte, *datatrack.Packet) + OnMetrics(Participant, *livekit.DataPacket) +} + +var _ ParticipantListener = (*NullParticipantListener)(nil) + +type NullParticipantListener struct{} + +func (*NullParticipantListener) OnParticipantUpdate(Participant) {} +func (*NullParticipantListener) OnTrackPublished(Participant, MediaTrack) {} +func (*NullParticipantListener) OnTrackUpdated(Participant, MediaTrack) {} +func (*NullParticipantListener) OnTrackUnpublished(Participant, MediaTrack) {} +func (*NullParticipantListener) OnDataTrackPublished(Participant, DataTrack) {} +func (*NullParticipantListener) OnDataTrackUnpublished(Participant, DataTrack) {} +func (*NullParticipantListener) OnDataTrackMessage(Participant, []byte, *datatrack.Packet) {} +func (*NullParticipantListener) OnMetrics(Participant, *livekit.DataPacket) {} + +// --------------------------------------------- + +//counterfeiter:generate . LocalParticipantListener +type LocalParticipantListener interface { + ParticipantListener + + OnStateChange(LocalParticipant) + OnSubscriberReady(LocalParticipant) + OnMigrateStateChange(LocalParticipant, MigrateState) + OnDataMessage(LocalParticipant, livekit.DataPacket_Kind, *livekit.DataPacket) + OnDataMessageUnlabeled(LocalParticipant, []byte) + OnSubscribeStatusChanged(LocalParticipant, livekit.ParticipantID, bool) + OnUpdateSubscriptions( + LocalParticipant, + []livekit.TrackID, + []*livekit.ParticipantTracks, + bool, + ) + OnUpdateSubscriptionPermission(LocalParticipant, *livekit.SubscriptionPermission) error + OnUpdateDataSubscriptions(LocalParticipant, *livekit.UpdateDataSubscription) + OnSyncState(LocalParticipant, *livekit.SyncState) error + OnSimulateScenario(LocalParticipant, *livekit.SimulateScenario) error + OnLeave(LocalParticipant, ParticipantCloseReason) +} + +var _ LocalParticipantListener = (*NullLocalParticipantListener)(nil) + +type NullLocalParticipantListener struct { + NullParticipantListener +} + +func (*NullLocalParticipantListener) OnStateChange(LocalParticipant) {} +func (*NullLocalParticipantListener) OnSubscriberReady(LocalParticipant) {} +func (*NullLocalParticipantListener) OnMigrateStateChange(LocalParticipant, MigrateState) {} +func (*NullLocalParticipantListener) OnDataMessage(LocalParticipant, livekit.DataPacket_Kind, *livekit.DataPacket) { +} +func (*NullLocalParticipantListener) OnDataMessageUnlabeled(LocalParticipant, []byte) {} +func (*NullLocalParticipantListener) OnSubscribeStatusChanged(LocalParticipant, livekit.ParticipantID, bool) { +} +func (*NullLocalParticipantListener) OnUpdateSubscriptions( + LocalParticipant, + []livekit.TrackID, + []*livekit.ParticipantTracks, + bool, +) { +} +func (*NullLocalParticipantListener) OnUpdateSubscriptionPermission(LocalParticipant, *livekit.SubscriptionPermission) error { + return nil +} +func (*NullLocalParticipantListener) OnUpdateDataSubscriptions(LocalParticipant, *livekit.UpdateDataSubscription) { +} +func (*NullLocalParticipantListener) OnSyncState(LocalParticipant, *livekit.SyncState) error { + return nil +} +func (*NullLocalParticipantListener) OnSimulateScenario(LocalParticipant, *livekit.SimulateScenario) error { + return nil +} +func (*NullLocalParticipantListener) OnLeave(LocalParticipant, ParticipantCloseReason) {} + +// --------------------------------------------- + +//counterfeiter:generate . ParticipantTelemetryListener +type ParticipantTelemetryListener interface { + OnTrackPublishRequested(pID livekit.ParticipantID, identity livekit.ParticipantIdentity, ti *livekit.TrackInfo) + OnTrackPublished(pID livekit.ParticipantID, identity livekit.ParticipantIdentity, ti *livekit.TrackInfo, shouldSendEvent bool) + OnTrackUnpublished(pID livekit.ParticipantID, identity livekit.ParticipantIdentity, ti *livekit.TrackInfo, shouldSendEvent bool) + OnTrackSubscribeRequested(pID livekit.ParticipantID, ti *livekit.TrackInfo) + OnTrackSubscribed(pID livekit.ParticipantID, ti *livekit.TrackInfo, publisherInfo *livekit.ParticipantInfo, shouldSendEvent bool) + OnTrackUnsubscribed(pID livekit.ParticipantID, ti *livekit.TrackInfo, shouldSendEvent bool) + OnTrackSubscribeFailed(pID livekit.ParticipantID, trackID livekit.TrackID, err error, isUserError bool) + OnTrackSubscribeStreamStarted(pID livekit.ParticipantID, ti *livekit.TrackInfo) + OnTrackMuted(pID livekit.ParticipantID, ti *livekit.TrackInfo) + OnTrackUnmuted(pID livekit.ParticipantID, ti *livekit.TrackInfo) + OnTrackPublishedUpdate(pID livekit.ParticipantID, ti *livekit.TrackInfo) + OnTrackMaxSubscribedVideoQuality(pID livekit.ParticipantID, ti *livekit.TrackInfo, mime mime.MimeType, maxQuality livekit.VideoQuality) + OnTrackPublishRTPStats(pID livekit.ParticipantID, trackID livekit.TrackID, mimeType mime.MimeType, layer int, stats *livekit.RTPStats) + OnTrackSubscribeRTPStats(pID livekit.ParticipantID, trackID livekit.TrackID, mimeType mime.MimeType, stats *livekit.RTPStats) + + OnTrackStats(key telemetry.StatsKey, stat *livekit.AnalyticsStat) +} + +var _ ParticipantTelemetryListener = (*NullParticipantTelemetryListener)(nil) + +type NullParticipantTelemetryListener struct{} + +func (NullParticipantTelemetryListener) OnTrackPublishRequested(pID livekit.ParticipantID, identity livekit.ParticipantIdentity, ti *livekit.TrackInfo) { +} +func (NullParticipantTelemetryListener) OnTrackPublished(pID livekit.ParticipantID, identity livekit.ParticipantIdentity, ti *livekit.TrackInfo, shouldSendEvent bool) { +} +func (NullParticipantTelemetryListener) OnTrackUnpublished(pID livekit.ParticipantID, identity livekit.ParticipantIdentity, ti *livekit.TrackInfo, shouldSendEvent bool) { +} +func (NullParticipantTelemetryListener) OnTrackSubscribeRequested(pID livekit.ParticipantID, ti *livekit.TrackInfo) { +} +func (NullParticipantTelemetryListener) OnTrackSubscribed(pID livekit.ParticipantID, ti *livekit.TrackInfo, publisherInfo *livekit.ParticipantInfo, shouldSendEvent bool) { +} +func (NullParticipantTelemetryListener) OnTrackUnsubscribed(pID livekit.ParticipantID, ti *livekit.TrackInfo, shouldSendEvent bool) { +} +func (NullParticipantTelemetryListener) OnTrackSubscribeFailed(pID livekit.ParticipantID, trackID livekit.TrackID, err error, isUserError bool) { +} +func (NullParticipantTelemetryListener) OnTrackSubscribeStreamStarted(pID livekit.ParticipantID, ti *livekit.TrackInfo) { +} +func (NullParticipantTelemetryListener) OnTrackMuted(pID livekit.ParticipantID, ti *livekit.TrackInfo) { +} +func (NullParticipantTelemetryListener) OnTrackUnmuted(pID livekit.ParticipantID, ti *livekit.TrackInfo) { +} +func (NullParticipantTelemetryListener) OnTrackPublishedUpdate(pID livekit.ParticipantID, ti *livekit.TrackInfo) { +} +func (NullParticipantTelemetryListener) OnTrackMaxSubscribedVideoQuality(pID livekit.ParticipantID, ti *livekit.TrackInfo, mime mime.MimeType, maxQuality livekit.VideoQuality) { +} +func (NullParticipantTelemetryListener) OnTrackPublishRTPStats(pID livekit.ParticipantID, trackID livekit.TrackID, mimeType mime.MimeType, layer int, stats *livekit.RTPStats) { +} +func (NullParticipantTelemetryListener) OnTrackSubscribeRTPStats(pID livekit.ParticipantID, trackID livekit.TrackID, mimeType mime.MimeType, stats *livekit.RTPStats) { +} + +func (NullParticipantTelemetryListener) OnTrackStats(key telemetry.StatsKey, stat *livekit.AnalyticsStat) { +} + +// --------------------------------------------- + // Room is a container of participants, and can provide room-level actions // //counterfeiter:generate . Room @@ -474,11 +725,14 @@ type Room interface { Name() livekit.RoomName ID() livekit.RoomID RemoveParticipant(identity livekit.ParticipantIdentity, pID livekit.ParticipantID, reason ParticipantCloseReason) - UpdateSubscriptions(participant LocalParticipant, trackIDs []livekit.TrackID, participantTracks []*livekit.ParticipantTracks, subscribe bool) - UpdateSubscriptionPermission(participant LocalParticipant, permissions *livekit.SubscriptionPermission) error - SyncState(participant LocalParticipant, state *livekit.SyncState) error - SimulateScenario(participant LocalParticipant, scenario *livekit.SimulateScenario) error + UpdateSubscriptions( + participant LocalParticipant, + trackIDs []livekit.TrackID, + participantTracks []*livekit.ParticipantTracks, + subscribe bool, + ) ResolveMediaTrackForSubscriber(sub LocalParticipant, trackID livekit.TrackID) MediaResolverResult + ResolveDataTrackForSubscriber(sub LocalParticipant, trackID livekit.TrackID) DataResolverResult GetLocalParticipants() []LocalParticipant IsDataMessageUserPacketDuplicate(ip *livekit.UserPacket) bool } @@ -501,12 +755,11 @@ type MediaTrack interface { PublisherID() livekit.ParticipantID PublisherIdentity() livekit.ParticipantIdentity PublisherVersion() uint32 + Logger() logger.Logger IsMuted() bool SetMuted(muted bool) - IsSimulcast() bool - GetAudioLevel() (level float64, active bool) Close(isExpectedToResume bool) @@ -525,15 +778,16 @@ type MediaTrack interface { OnTrackSubscribed() // returns quality information that's appropriate for width & height - GetQualityForDimension(width, height uint32) livekit.VideoQuality + GetQualityForDimension(mimeType mime.MimeType, width, height uint32) livekit.VideoQuality // returns temporal layer that's appropriate for fps - GetTemporalLayerForSpatialFps(spatial int32, fps uint32, mime mime.MimeType) int32 + GetTemporalLayerForSpatialFps(mimeType mime.MimeType, spatial int32, fps uint32) int32 Receivers() []sfu.TrackReceiver ClearAllReceivers(isExpectedToResume bool) IsEncrypted() bool + HasPacketTrailer() bool } //counterfeiter:generate . LocalMediaTrack @@ -542,7 +796,7 @@ type LocalMediaTrack interface { Restart() - SignalCid() string + HasSignalCid(cid string) bool HasSdpCid(cid string) bool GetConnectionScoreAndQuality() (float32, livekit.ConnectionQuality) @@ -551,9 +805,57 @@ type LocalMediaTrack interface { SetRTT(rtt uint32) NotifySubscriberNodeMaxQuality(nodeID livekit.NodeID, qualities []SubscribedCodecQuality) + NotifySubscriptionNode(nodeID livekit.NodeID, codecs []*livekit.SubscribedAudioCodec) + ClearSubscriberNodes() NotifySubscriberNodeMediaLoss(nodeID livekit.NodeID, fractionalLoss uint8) } +// DataTrack represents a data track +// +//counterfeiter:generate . DataTrack +type DataTrack interface { + ID() livekit.TrackID + PubHandle() uint16 + Name() string + ToProto() *livekit.DataTrackInfo + + PublisherID() livekit.ParticipantID + PublisherIdentity() livekit.ParticipantIdentity + + AddSubscriber(sub LocalParticipant) (DataDownTrack, error) + RemoveSubscriber(participantID livekit.ParticipantID) + IsSubscriber(subID livekit.ParticipantID) bool + + AddDataDownTrack(sender DataTrackSender) error + DeleteDataDownTrack(subscriberID livekit.ParticipantID) + + HandlePacket(data []byte, packet *datatrack.Packet, arrivalTime int64) + + Close() +} + +//counterfeiter:generate . DataDownTrack +type DataDownTrack interface { + Close() + + Handle() uint16 + PublishDataTrack() DataTrack + + UpdateSubscriptionOptions(subscriptionOptions *livekit.DataTrackSubscriptionOptions) +} + +//counterfeiter:generate . DataTrackSender +type DataTrackSender interface { + SubscriberID() livekit.ParticipantID + + WritePacket(data []byte, packet *datatrack.Packet, arrivalTime int64) +} + +//counterfeiter:generate . DataTrackTransport +type DataTrackTransport interface { + SendDataTrackMessage(data []byte) error +} + //counterfeiter:generate . SubscribedTrack type SubscribedTrack interface { AddOnBind(f func(error)) @@ -595,9 +897,20 @@ type MediaResolverResult struct { PublisherIdentity livekit.ParticipantIdentity } +type DataResolverResult struct { + TrackChangedNotifier ChangeNotifier + TrackRemovedNotifier ChangeNotifier + DataTrack DataTrack + PublisherID livekit.ParticipantID + PublisherIdentity livekit.ParticipantIdentity +} + // MediaTrackResolver locates a specific media track for a subscriber type MediaTrackResolver func(LocalParticipant, livekit.TrackID) MediaResolverResult +// DataTrackResolver locates a specific data track for a subscriber +type DataTrackResolver func(LocalParticipant, livekit.TrackID) DataResolverResult + // Supervisor/operation monitor related definitions type OperationMonitorEvent int @@ -626,7 +939,7 @@ func (o OperationMonitorEvent) String() string { } } -type OperationMonitorData interface{} +type OperationMonitorData any type OperationMonitor interface { PostEvent(ome OperationMonitorEvent, omd OperationMonitorData) diff --git a/pkg/rtc/types/protocol_version.go b/pkg/rtc/types/protocol_version.go index 79b2d71..9edab8b 100644 --- a/pkg/rtc/types/protocol_version.go +++ b/pkg/rtc/types/protocol_version.go @@ -16,7 +16,7 @@ package types type ProtocolVersion int -const CurrentProtocol = 15 +const CurrentProtocol = 17 func (v ProtocolVersion) SupportsPackedStreamId() bool { return v > 0 @@ -68,11 +68,11 @@ func (v ProtocolVersion) SupportFastStart() bool { return v > 7 } -func (v ProtocolVersion) SupportHandlesDisconnectedUpdate() bool { +func (v ProtocolVersion) SupportsDisconnectedUpdate() bool { return v > 8 } -func (v ProtocolVersion) SupportSyncStreamID() bool { +func (v ProtocolVersion) SupportsSyncStreamID() bool { return v > 9 } @@ -95,3 +95,11 @@ func (v ProtocolVersion) SupportsRegionsInLeaveRequest() bool { func (v ProtocolVersion) SupportsNonErrorSignalResponse() bool { return v > 14 } + +func (v ProtocolVersion) SupportsMoving() bool { + return v > 15 +} + +func (v ProtocolVersion) SupportsPacketTrailer() bool { + return v > 16 +} diff --git a/pkg/rtc/types/typesfakes/fake_data_down_track.go b/pkg/rtc/types/typesfakes/fake_data_down_track.go new file mode 100644 index 0000000..56d0d10 --- /dev/null +++ b/pkg/rtc/types/typesfakes/fake_data_down_track.go @@ -0,0 +1,229 @@ +// Code generated by counterfeiter. DO NOT EDIT. +package typesfakes + +import ( + "sync" + + "github.com/livekit/livekit-server/pkg/rtc/types" + "github.com/livekit/protocol/livekit" +) + +type FakeDataDownTrack struct { + CloseStub func() + closeMutex sync.RWMutex + closeArgsForCall []struct { + } + HandleStub func() uint16 + handleMutex sync.RWMutex + handleArgsForCall []struct { + } + handleReturns struct { + result1 uint16 + } + handleReturnsOnCall map[int]struct { + result1 uint16 + } + PublishDataTrackStub func() types.DataTrack + publishDataTrackMutex sync.RWMutex + publishDataTrackArgsForCall []struct { + } + publishDataTrackReturns struct { + result1 types.DataTrack + } + publishDataTrackReturnsOnCall map[int]struct { + result1 types.DataTrack + } + UpdateSubscriptionOptionsStub func(*livekit.DataTrackSubscriptionOptions) + updateSubscriptionOptionsMutex sync.RWMutex + updateSubscriptionOptionsArgsForCall []struct { + arg1 *livekit.DataTrackSubscriptionOptions + } + invocations map[string][][]interface{} + invocationsMutex sync.RWMutex +} + +func (fake *FakeDataDownTrack) Close() { + fake.closeMutex.Lock() + fake.closeArgsForCall = append(fake.closeArgsForCall, struct { + }{}) + stub := fake.CloseStub + fake.recordInvocation("Close", []interface{}{}) + fake.closeMutex.Unlock() + if stub != nil { + fake.CloseStub() + } +} + +func (fake *FakeDataDownTrack) CloseCallCount() int { + fake.closeMutex.RLock() + defer fake.closeMutex.RUnlock() + return len(fake.closeArgsForCall) +} + +func (fake *FakeDataDownTrack) CloseCalls(stub func()) { + fake.closeMutex.Lock() + defer fake.closeMutex.Unlock() + fake.CloseStub = stub +} + +func (fake *FakeDataDownTrack) Handle() uint16 { + fake.handleMutex.Lock() + ret, specificReturn := fake.handleReturnsOnCall[len(fake.handleArgsForCall)] + fake.handleArgsForCall = append(fake.handleArgsForCall, struct { + }{}) + stub := fake.HandleStub + fakeReturns := fake.handleReturns + fake.recordInvocation("Handle", []interface{}{}) + fake.handleMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeDataDownTrack) HandleCallCount() int { + fake.handleMutex.RLock() + defer fake.handleMutex.RUnlock() + return len(fake.handleArgsForCall) +} + +func (fake *FakeDataDownTrack) HandleCalls(stub func() uint16) { + fake.handleMutex.Lock() + defer fake.handleMutex.Unlock() + fake.HandleStub = stub +} + +func (fake *FakeDataDownTrack) HandleReturns(result1 uint16) { + fake.handleMutex.Lock() + defer fake.handleMutex.Unlock() + fake.HandleStub = nil + fake.handleReturns = struct { + result1 uint16 + }{result1} +} + +func (fake *FakeDataDownTrack) HandleReturnsOnCall(i int, result1 uint16) { + fake.handleMutex.Lock() + defer fake.handleMutex.Unlock() + fake.HandleStub = nil + if fake.handleReturnsOnCall == nil { + fake.handleReturnsOnCall = make(map[int]struct { + result1 uint16 + }) + } + fake.handleReturnsOnCall[i] = struct { + result1 uint16 + }{result1} +} + +func (fake *FakeDataDownTrack) PublishDataTrack() types.DataTrack { + fake.publishDataTrackMutex.Lock() + ret, specificReturn := fake.publishDataTrackReturnsOnCall[len(fake.publishDataTrackArgsForCall)] + fake.publishDataTrackArgsForCall = append(fake.publishDataTrackArgsForCall, struct { + }{}) + stub := fake.PublishDataTrackStub + fakeReturns := fake.publishDataTrackReturns + fake.recordInvocation("PublishDataTrack", []interface{}{}) + fake.publishDataTrackMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeDataDownTrack) PublishDataTrackCallCount() int { + fake.publishDataTrackMutex.RLock() + defer fake.publishDataTrackMutex.RUnlock() + return len(fake.publishDataTrackArgsForCall) +} + +func (fake *FakeDataDownTrack) PublishDataTrackCalls(stub func() types.DataTrack) { + fake.publishDataTrackMutex.Lock() + defer fake.publishDataTrackMutex.Unlock() + fake.PublishDataTrackStub = stub +} + +func (fake *FakeDataDownTrack) PublishDataTrackReturns(result1 types.DataTrack) { + fake.publishDataTrackMutex.Lock() + defer fake.publishDataTrackMutex.Unlock() + fake.PublishDataTrackStub = nil + fake.publishDataTrackReturns = struct { + result1 types.DataTrack + }{result1} +} + +func (fake *FakeDataDownTrack) PublishDataTrackReturnsOnCall(i int, result1 types.DataTrack) { + fake.publishDataTrackMutex.Lock() + defer fake.publishDataTrackMutex.Unlock() + fake.PublishDataTrackStub = nil + if fake.publishDataTrackReturnsOnCall == nil { + fake.publishDataTrackReturnsOnCall = make(map[int]struct { + result1 types.DataTrack + }) + } + fake.publishDataTrackReturnsOnCall[i] = struct { + result1 types.DataTrack + }{result1} +} + +func (fake *FakeDataDownTrack) UpdateSubscriptionOptions(arg1 *livekit.DataTrackSubscriptionOptions) { + fake.updateSubscriptionOptionsMutex.Lock() + fake.updateSubscriptionOptionsArgsForCall = append(fake.updateSubscriptionOptionsArgsForCall, struct { + arg1 *livekit.DataTrackSubscriptionOptions + }{arg1}) + stub := fake.UpdateSubscriptionOptionsStub + fake.recordInvocation("UpdateSubscriptionOptions", []interface{}{arg1}) + fake.updateSubscriptionOptionsMutex.Unlock() + if stub != nil { + fake.UpdateSubscriptionOptionsStub(arg1) + } +} + +func (fake *FakeDataDownTrack) UpdateSubscriptionOptionsCallCount() int { + fake.updateSubscriptionOptionsMutex.RLock() + defer fake.updateSubscriptionOptionsMutex.RUnlock() + return len(fake.updateSubscriptionOptionsArgsForCall) +} + +func (fake *FakeDataDownTrack) UpdateSubscriptionOptionsCalls(stub func(*livekit.DataTrackSubscriptionOptions)) { + fake.updateSubscriptionOptionsMutex.Lock() + defer fake.updateSubscriptionOptionsMutex.Unlock() + fake.UpdateSubscriptionOptionsStub = stub +} + +func (fake *FakeDataDownTrack) UpdateSubscriptionOptionsArgsForCall(i int) *livekit.DataTrackSubscriptionOptions { + fake.updateSubscriptionOptionsMutex.RLock() + defer fake.updateSubscriptionOptionsMutex.RUnlock() + argsForCall := fake.updateSubscriptionOptionsArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeDataDownTrack) Invocations() map[string][][]interface{} { + fake.invocationsMutex.RLock() + defer fake.invocationsMutex.RUnlock() + copiedInvocations := map[string][][]interface{}{} + for key, value := range fake.invocations { + copiedInvocations[key] = value + } + return copiedInvocations +} + +func (fake *FakeDataDownTrack) recordInvocation(key string, args []interface{}) { + fake.invocationsMutex.Lock() + defer fake.invocationsMutex.Unlock() + if fake.invocations == nil { + fake.invocations = map[string][][]interface{}{} + } + if fake.invocations[key] == nil { + fake.invocations[key] = [][]interface{}{} + } + fake.invocations[key] = append(fake.invocations[key], args) +} + +var _ types.DataDownTrack = new(FakeDataDownTrack) diff --git a/pkg/rtc/types/typesfakes/fake_data_track.go b/pkg/rtc/types/typesfakes/fake_data_track.go new file mode 100644 index 0000000..c54d77c --- /dev/null +++ b/pkg/rtc/types/typesfakes/fake_data_track.go @@ -0,0 +1,786 @@ +// Code generated by counterfeiter. DO NOT EDIT. +package typesfakes + +import ( + "sync" + + "github.com/livekit/livekit-server/pkg/rtc/datatrack" + "github.com/livekit/livekit-server/pkg/rtc/types" + "github.com/livekit/protocol/livekit" +) + +type FakeDataTrack struct { + AddDataDownTrackStub func(types.DataTrackSender) error + addDataDownTrackMutex sync.RWMutex + addDataDownTrackArgsForCall []struct { + arg1 types.DataTrackSender + } + addDataDownTrackReturns struct { + result1 error + } + addDataDownTrackReturnsOnCall map[int]struct { + result1 error + } + AddSubscriberStub func(types.LocalParticipant) (types.DataDownTrack, error) + addSubscriberMutex sync.RWMutex + addSubscriberArgsForCall []struct { + arg1 types.LocalParticipant + } + addSubscriberReturns struct { + result1 types.DataDownTrack + result2 error + } + addSubscriberReturnsOnCall map[int]struct { + result1 types.DataDownTrack + result2 error + } + CloseStub func() + closeMutex sync.RWMutex + closeArgsForCall []struct { + } + DeleteDataDownTrackStub func(livekit.ParticipantID) + deleteDataDownTrackMutex sync.RWMutex + deleteDataDownTrackArgsForCall []struct { + arg1 livekit.ParticipantID + } + HandlePacketStub func([]byte, *datatrack.Packet, int64) + handlePacketMutex sync.RWMutex + handlePacketArgsForCall []struct { + arg1 []byte + arg2 *datatrack.Packet + arg3 int64 + } + IDStub func() livekit.TrackID + iDMutex sync.RWMutex + iDArgsForCall []struct { + } + iDReturns struct { + result1 livekit.TrackID + } + iDReturnsOnCall map[int]struct { + result1 livekit.TrackID + } + IsSubscriberStub func(livekit.ParticipantID) bool + isSubscriberMutex sync.RWMutex + isSubscriberArgsForCall []struct { + arg1 livekit.ParticipantID + } + isSubscriberReturns struct { + result1 bool + } + isSubscriberReturnsOnCall map[int]struct { + result1 bool + } + NameStub func() string + nameMutex sync.RWMutex + nameArgsForCall []struct { + } + nameReturns struct { + result1 string + } + nameReturnsOnCall map[int]struct { + result1 string + } + PubHandleStub func() uint16 + pubHandleMutex sync.RWMutex + pubHandleArgsForCall []struct { + } + pubHandleReturns struct { + result1 uint16 + } + pubHandleReturnsOnCall map[int]struct { + result1 uint16 + } + PublisherIDStub func() livekit.ParticipantID + publisherIDMutex sync.RWMutex + publisherIDArgsForCall []struct { + } + publisherIDReturns struct { + result1 livekit.ParticipantID + } + publisherIDReturnsOnCall map[int]struct { + result1 livekit.ParticipantID + } + PublisherIdentityStub func() livekit.ParticipantIdentity + publisherIdentityMutex sync.RWMutex + publisherIdentityArgsForCall []struct { + } + publisherIdentityReturns struct { + result1 livekit.ParticipantIdentity + } + publisherIdentityReturnsOnCall map[int]struct { + result1 livekit.ParticipantIdentity + } + RemoveSubscriberStub func(livekit.ParticipantID) + removeSubscriberMutex sync.RWMutex + removeSubscriberArgsForCall []struct { + arg1 livekit.ParticipantID + } + ToProtoStub func() *livekit.DataTrackInfo + toProtoMutex sync.RWMutex + toProtoArgsForCall []struct { + } + toProtoReturns struct { + result1 *livekit.DataTrackInfo + } + toProtoReturnsOnCall map[int]struct { + result1 *livekit.DataTrackInfo + } + invocations map[string][][]interface{} + invocationsMutex sync.RWMutex +} + +func (fake *FakeDataTrack) AddDataDownTrack(arg1 types.DataTrackSender) error { + fake.addDataDownTrackMutex.Lock() + ret, specificReturn := fake.addDataDownTrackReturnsOnCall[len(fake.addDataDownTrackArgsForCall)] + fake.addDataDownTrackArgsForCall = append(fake.addDataDownTrackArgsForCall, struct { + arg1 types.DataTrackSender + }{arg1}) + stub := fake.AddDataDownTrackStub + fakeReturns := fake.addDataDownTrackReturns + fake.recordInvocation("AddDataDownTrack", []interface{}{arg1}) + fake.addDataDownTrackMutex.Unlock() + if stub != nil { + return stub(arg1) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeDataTrack) AddDataDownTrackCallCount() int { + fake.addDataDownTrackMutex.RLock() + defer fake.addDataDownTrackMutex.RUnlock() + return len(fake.addDataDownTrackArgsForCall) +} + +func (fake *FakeDataTrack) AddDataDownTrackCalls(stub func(types.DataTrackSender) error) { + fake.addDataDownTrackMutex.Lock() + defer fake.addDataDownTrackMutex.Unlock() + fake.AddDataDownTrackStub = stub +} + +func (fake *FakeDataTrack) AddDataDownTrackArgsForCall(i int) types.DataTrackSender { + fake.addDataDownTrackMutex.RLock() + defer fake.addDataDownTrackMutex.RUnlock() + argsForCall := fake.addDataDownTrackArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeDataTrack) AddDataDownTrackReturns(result1 error) { + fake.addDataDownTrackMutex.Lock() + defer fake.addDataDownTrackMutex.Unlock() + fake.AddDataDownTrackStub = nil + fake.addDataDownTrackReturns = struct { + result1 error + }{result1} +} + +func (fake *FakeDataTrack) AddDataDownTrackReturnsOnCall(i int, result1 error) { + fake.addDataDownTrackMutex.Lock() + defer fake.addDataDownTrackMutex.Unlock() + fake.AddDataDownTrackStub = nil + if fake.addDataDownTrackReturnsOnCall == nil { + fake.addDataDownTrackReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.addDataDownTrackReturnsOnCall[i] = struct { + result1 error + }{result1} +} + +func (fake *FakeDataTrack) AddSubscriber(arg1 types.LocalParticipant) (types.DataDownTrack, error) { + fake.addSubscriberMutex.Lock() + ret, specificReturn := fake.addSubscriberReturnsOnCall[len(fake.addSubscriberArgsForCall)] + fake.addSubscriberArgsForCall = append(fake.addSubscriberArgsForCall, struct { + arg1 types.LocalParticipant + }{arg1}) + stub := fake.AddSubscriberStub + fakeReturns := fake.addSubscriberReturns + fake.recordInvocation("AddSubscriber", []interface{}{arg1}) + fake.addSubscriberMutex.Unlock() + if stub != nil { + return stub(arg1) + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeDataTrack) AddSubscriberCallCount() int { + fake.addSubscriberMutex.RLock() + defer fake.addSubscriberMutex.RUnlock() + return len(fake.addSubscriberArgsForCall) +} + +func (fake *FakeDataTrack) AddSubscriberCalls(stub func(types.LocalParticipant) (types.DataDownTrack, error)) { + fake.addSubscriberMutex.Lock() + defer fake.addSubscriberMutex.Unlock() + fake.AddSubscriberStub = stub +} + +func (fake *FakeDataTrack) AddSubscriberArgsForCall(i int) types.LocalParticipant { + fake.addSubscriberMutex.RLock() + defer fake.addSubscriberMutex.RUnlock() + argsForCall := fake.addSubscriberArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeDataTrack) AddSubscriberReturns(result1 types.DataDownTrack, result2 error) { + fake.addSubscriberMutex.Lock() + defer fake.addSubscriberMutex.Unlock() + fake.AddSubscriberStub = nil + fake.addSubscriberReturns = struct { + result1 types.DataDownTrack + result2 error + }{result1, result2} +} + +func (fake *FakeDataTrack) AddSubscriberReturnsOnCall(i int, result1 types.DataDownTrack, result2 error) { + fake.addSubscriberMutex.Lock() + defer fake.addSubscriberMutex.Unlock() + fake.AddSubscriberStub = nil + if fake.addSubscriberReturnsOnCall == nil { + fake.addSubscriberReturnsOnCall = make(map[int]struct { + result1 types.DataDownTrack + result2 error + }) + } + fake.addSubscriberReturnsOnCall[i] = struct { + result1 types.DataDownTrack + result2 error + }{result1, result2} +} + +func (fake *FakeDataTrack) Close() { + fake.closeMutex.Lock() + fake.closeArgsForCall = append(fake.closeArgsForCall, struct { + }{}) + stub := fake.CloseStub + fake.recordInvocation("Close", []interface{}{}) + fake.closeMutex.Unlock() + if stub != nil { + fake.CloseStub() + } +} + +func (fake *FakeDataTrack) CloseCallCount() int { + fake.closeMutex.RLock() + defer fake.closeMutex.RUnlock() + return len(fake.closeArgsForCall) +} + +func (fake *FakeDataTrack) CloseCalls(stub func()) { + fake.closeMutex.Lock() + defer fake.closeMutex.Unlock() + fake.CloseStub = stub +} + +func (fake *FakeDataTrack) DeleteDataDownTrack(arg1 livekit.ParticipantID) { + fake.deleteDataDownTrackMutex.Lock() + fake.deleteDataDownTrackArgsForCall = append(fake.deleteDataDownTrackArgsForCall, struct { + arg1 livekit.ParticipantID + }{arg1}) + stub := fake.DeleteDataDownTrackStub + fake.recordInvocation("DeleteDataDownTrack", []interface{}{arg1}) + fake.deleteDataDownTrackMutex.Unlock() + if stub != nil { + fake.DeleteDataDownTrackStub(arg1) + } +} + +func (fake *FakeDataTrack) DeleteDataDownTrackCallCount() int { + fake.deleteDataDownTrackMutex.RLock() + defer fake.deleteDataDownTrackMutex.RUnlock() + return len(fake.deleteDataDownTrackArgsForCall) +} + +func (fake *FakeDataTrack) DeleteDataDownTrackCalls(stub func(livekit.ParticipantID)) { + fake.deleteDataDownTrackMutex.Lock() + defer fake.deleteDataDownTrackMutex.Unlock() + fake.DeleteDataDownTrackStub = stub +} + +func (fake *FakeDataTrack) DeleteDataDownTrackArgsForCall(i int) livekit.ParticipantID { + fake.deleteDataDownTrackMutex.RLock() + defer fake.deleteDataDownTrackMutex.RUnlock() + argsForCall := fake.deleteDataDownTrackArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeDataTrack) HandlePacket(arg1 []byte, arg2 *datatrack.Packet, arg3 int64) { + var arg1Copy []byte + if arg1 != nil { + arg1Copy = make([]byte, len(arg1)) + copy(arg1Copy, arg1) + } + fake.handlePacketMutex.Lock() + fake.handlePacketArgsForCall = append(fake.handlePacketArgsForCall, struct { + arg1 []byte + arg2 *datatrack.Packet + arg3 int64 + }{arg1Copy, arg2, arg3}) + stub := fake.HandlePacketStub + fake.recordInvocation("HandlePacket", []interface{}{arg1Copy, arg2, arg3}) + fake.handlePacketMutex.Unlock() + if stub != nil { + fake.HandlePacketStub(arg1, arg2, arg3) + } +} + +func (fake *FakeDataTrack) HandlePacketCallCount() int { + fake.handlePacketMutex.RLock() + defer fake.handlePacketMutex.RUnlock() + return len(fake.handlePacketArgsForCall) +} + +func (fake *FakeDataTrack) HandlePacketCalls(stub func([]byte, *datatrack.Packet, int64)) { + fake.handlePacketMutex.Lock() + defer fake.handlePacketMutex.Unlock() + fake.HandlePacketStub = stub +} + +func (fake *FakeDataTrack) HandlePacketArgsForCall(i int) ([]byte, *datatrack.Packet, int64) { + fake.handlePacketMutex.RLock() + defer fake.handlePacketMutex.RUnlock() + argsForCall := fake.handlePacketArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 +} + +func (fake *FakeDataTrack) ID() livekit.TrackID { + fake.iDMutex.Lock() + ret, specificReturn := fake.iDReturnsOnCall[len(fake.iDArgsForCall)] + fake.iDArgsForCall = append(fake.iDArgsForCall, struct { + }{}) + stub := fake.IDStub + fakeReturns := fake.iDReturns + fake.recordInvocation("ID", []interface{}{}) + fake.iDMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeDataTrack) IDCallCount() int { + fake.iDMutex.RLock() + defer fake.iDMutex.RUnlock() + return len(fake.iDArgsForCall) +} + +func (fake *FakeDataTrack) IDCalls(stub func() livekit.TrackID) { + fake.iDMutex.Lock() + defer fake.iDMutex.Unlock() + fake.IDStub = stub +} + +func (fake *FakeDataTrack) IDReturns(result1 livekit.TrackID) { + fake.iDMutex.Lock() + defer fake.iDMutex.Unlock() + fake.IDStub = nil + fake.iDReturns = struct { + result1 livekit.TrackID + }{result1} +} + +func (fake *FakeDataTrack) IDReturnsOnCall(i int, result1 livekit.TrackID) { + fake.iDMutex.Lock() + defer fake.iDMutex.Unlock() + fake.IDStub = nil + if fake.iDReturnsOnCall == nil { + fake.iDReturnsOnCall = make(map[int]struct { + result1 livekit.TrackID + }) + } + fake.iDReturnsOnCall[i] = struct { + result1 livekit.TrackID + }{result1} +} + +func (fake *FakeDataTrack) IsSubscriber(arg1 livekit.ParticipantID) bool { + fake.isSubscriberMutex.Lock() + ret, specificReturn := fake.isSubscriberReturnsOnCall[len(fake.isSubscriberArgsForCall)] + fake.isSubscriberArgsForCall = append(fake.isSubscriberArgsForCall, struct { + arg1 livekit.ParticipantID + }{arg1}) + stub := fake.IsSubscriberStub + fakeReturns := fake.isSubscriberReturns + fake.recordInvocation("IsSubscriber", []interface{}{arg1}) + fake.isSubscriberMutex.Unlock() + if stub != nil { + return stub(arg1) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeDataTrack) IsSubscriberCallCount() int { + fake.isSubscriberMutex.RLock() + defer fake.isSubscriberMutex.RUnlock() + return len(fake.isSubscriberArgsForCall) +} + +func (fake *FakeDataTrack) IsSubscriberCalls(stub func(livekit.ParticipantID) bool) { + fake.isSubscriberMutex.Lock() + defer fake.isSubscriberMutex.Unlock() + fake.IsSubscriberStub = stub +} + +func (fake *FakeDataTrack) IsSubscriberArgsForCall(i int) livekit.ParticipantID { + fake.isSubscriberMutex.RLock() + defer fake.isSubscriberMutex.RUnlock() + argsForCall := fake.isSubscriberArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeDataTrack) IsSubscriberReturns(result1 bool) { + fake.isSubscriberMutex.Lock() + defer fake.isSubscriberMutex.Unlock() + fake.IsSubscriberStub = nil + fake.isSubscriberReturns = struct { + result1 bool + }{result1} +} + +func (fake *FakeDataTrack) IsSubscriberReturnsOnCall(i int, result1 bool) { + fake.isSubscriberMutex.Lock() + defer fake.isSubscriberMutex.Unlock() + fake.IsSubscriberStub = nil + if fake.isSubscriberReturnsOnCall == nil { + fake.isSubscriberReturnsOnCall = make(map[int]struct { + result1 bool + }) + } + fake.isSubscriberReturnsOnCall[i] = struct { + result1 bool + }{result1} +} + +func (fake *FakeDataTrack) Name() string { + fake.nameMutex.Lock() + ret, specificReturn := fake.nameReturnsOnCall[len(fake.nameArgsForCall)] + fake.nameArgsForCall = append(fake.nameArgsForCall, struct { + }{}) + stub := fake.NameStub + fakeReturns := fake.nameReturns + fake.recordInvocation("Name", []interface{}{}) + fake.nameMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeDataTrack) NameCallCount() int { + fake.nameMutex.RLock() + defer fake.nameMutex.RUnlock() + return len(fake.nameArgsForCall) +} + +func (fake *FakeDataTrack) NameCalls(stub func() string) { + fake.nameMutex.Lock() + defer fake.nameMutex.Unlock() + fake.NameStub = stub +} + +func (fake *FakeDataTrack) NameReturns(result1 string) { + fake.nameMutex.Lock() + defer fake.nameMutex.Unlock() + fake.NameStub = nil + fake.nameReturns = struct { + result1 string + }{result1} +} + +func (fake *FakeDataTrack) NameReturnsOnCall(i int, result1 string) { + fake.nameMutex.Lock() + defer fake.nameMutex.Unlock() + fake.NameStub = nil + if fake.nameReturnsOnCall == nil { + fake.nameReturnsOnCall = make(map[int]struct { + result1 string + }) + } + fake.nameReturnsOnCall[i] = struct { + result1 string + }{result1} +} + +func (fake *FakeDataTrack) PubHandle() uint16 { + fake.pubHandleMutex.Lock() + ret, specificReturn := fake.pubHandleReturnsOnCall[len(fake.pubHandleArgsForCall)] + fake.pubHandleArgsForCall = append(fake.pubHandleArgsForCall, struct { + }{}) + stub := fake.PubHandleStub + fakeReturns := fake.pubHandleReturns + fake.recordInvocation("PubHandle", []interface{}{}) + fake.pubHandleMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeDataTrack) PubHandleCallCount() int { + fake.pubHandleMutex.RLock() + defer fake.pubHandleMutex.RUnlock() + return len(fake.pubHandleArgsForCall) +} + +func (fake *FakeDataTrack) PubHandleCalls(stub func() uint16) { + fake.pubHandleMutex.Lock() + defer fake.pubHandleMutex.Unlock() + fake.PubHandleStub = stub +} + +func (fake *FakeDataTrack) PubHandleReturns(result1 uint16) { + fake.pubHandleMutex.Lock() + defer fake.pubHandleMutex.Unlock() + fake.PubHandleStub = nil + fake.pubHandleReturns = struct { + result1 uint16 + }{result1} +} + +func (fake *FakeDataTrack) PubHandleReturnsOnCall(i int, result1 uint16) { + fake.pubHandleMutex.Lock() + defer fake.pubHandleMutex.Unlock() + fake.PubHandleStub = nil + if fake.pubHandleReturnsOnCall == nil { + fake.pubHandleReturnsOnCall = make(map[int]struct { + result1 uint16 + }) + } + fake.pubHandleReturnsOnCall[i] = struct { + result1 uint16 + }{result1} +} + +func (fake *FakeDataTrack) PublisherID() livekit.ParticipantID { + fake.publisherIDMutex.Lock() + ret, specificReturn := fake.publisherIDReturnsOnCall[len(fake.publisherIDArgsForCall)] + fake.publisherIDArgsForCall = append(fake.publisherIDArgsForCall, struct { + }{}) + stub := fake.PublisherIDStub + fakeReturns := fake.publisherIDReturns + fake.recordInvocation("PublisherID", []interface{}{}) + fake.publisherIDMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeDataTrack) PublisherIDCallCount() int { + fake.publisherIDMutex.RLock() + defer fake.publisherIDMutex.RUnlock() + return len(fake.publisherIDArgsForCall) +} + +func (fake *FakeDataTrack) PublisherIDCalls(stub func() livekit.ParticipantID) { + fake.publisherIDMutex.Lock() + defer fake.publisherIDMutex.Unlock() + fake.PublisherIDStub = stub +} + +func (fake *FakeDataTrack) PublisherIDReturns(result1 livekit.ParticipantID) { + fake.publisherIDMutex.Lock() + defer fake.publisherIDMutex.Unlock() + fake.PublisherIDStub = nil + fake.publisherIDReturns = struct { + result1 livekit.ParticipantID + }{result1} +} + +func (fake *FakeDataTrack) PublisherIDReturnsOnCall(i int, result1 livekit.ParticipantID) { + fake.publisherIDMutex.Lock() + defer fake.publisherIDMutex.Unlock() + fake.PublisherIDStub = nil + if fake.publisherIDReturnsOnCall == nil { + fake.publisherIDReturnsOnCall = make(map[int]struct { + result1 livekit.ParticipantID + }) + } + fake.publisherIDReturnsOnCall[i] = struct { + result1 livekit.ParticipantID + }{result1} +} + +func (fake *FakeDataTrack) PublisherIdentity() livekit.ParticipantIdentity { + fake.publisherIdentityMutex.Lock() + ret, specificReturn := fake.publisherIdentityReturnsOnCall[len(fake.publisherIdentityArgsForCall)] + fake.publisherIdentityArgsForCall = append(fake.publisherIdentityArgsForCall, struct { + }{}) + stub := fake.PublisherIdentityStub + fakeReturns := fake.publisherIdentityReturns + fake.recordInvocation("PublisherIdentity", []interface{}{}) + fake.publisherIdentityMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeDataTrack) PublisherIdentityCallCount() int { + fake.publisherIdentityMutex.RLock() + defer fake.publisherIdentityMutex.RUnlock() + return len(fake.publisherIdentityArgsForCall) +} + +func (fake *FakeDataTrack) PublisherIdentityCalls(stub func() livekit.ParticipantIdentity) { + fake.publisherIdentityMutex.Lock() + defer fake.publisherIdentityMutex.Unlock() + fake.PublisherIdentityStub = stub +} + +func (fake *FakeDataTrack) PublisherIdentityReturns(result1 livekit.ParticipantIdentity) { + fake.publisherIdentityMutex.Lock() + defer fake.publisherIdentityMutex.Unlock() + fake.PublisherIdentityStub = nil + fake.publisherIdentityReturns = struct { + result1 livekit.ParticipantIdentity + }{result1} +} + +func (fake *FakeDataTrack) PublisherIdentityReturnsOnCall(i int, result1 livekit.ParticipantIdentity) { + fake.publisherIdentityMutex.Lock() + defer fake.publisherIdentityMutex.Unlock() + fake.PublisherIdentityStub = nil + if fake.publisherIdentityReturnsOnCall == nil { + fake.publisherIdentityReturnsOnCall = make(map[int]struct { + result1 livekit.ParticipantIdentity + }) + } + fake.publisherIdentityReturnsOnCall[i] = struct { + result1 livekit.ParticipantIdentity + }{result1} +} + +func (fake *FakeDataTrack) RemoveSubscriber(arg1 livekit.ParticipantID) { + fake.removeSubscriberMutex.Lock() + fake.removeSubscriberArgsForCall = append(fake.removeSubscriberArgsForCall, struct { + arg1 livekit.ParticipantID + }{arg1}) + stub := fake.RemoveSubscriberStub + fake.recordInvocation("RemoveSubscriber", []interface{}{arg1}) + fake.removeSubscriberMutex.Unlock() + if stub != nil { + fake.RemoveSubscriberStub(arg1) + } +} + +func (fake *FakeDataTrack) RemoveSubscriberCallCount() int { + fake.removeSubscriberMutex.RLock() + defer fake.removeSubscriberMutex.RUnlock() + return len(fake.removeSubscriberArgsForCall) +} + +func (fake *FakeDataTrack) RemoveSubscriberCalls(stub func(livekit.ParticipantID)) { + fake.removeSubscriberMutex.Lock() + defer fake.removeSubscriberMutex.Unlock() + fake.RemoveSubscriberStub = stub +} + +func (fake *FakeDataTrack) RemoveSubscriberArgsForCall(i int) livekit.ParticipantID { + fake.removeSubscriberMutex.RLock() + defer fake.removeSubscriberMutex.RUnlock() + argsForCall := fake.removeSubscriberArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeDataTrack) ToProto() *livekit.DataTrackInfo { + fake.toProtoMutex.Lock() + ret, specificReturn := fake.toProtoReturnsOnCall[len(fake.toProtoArgsForCall)] + fake.toProtoArgsForCall = append(fake.toProtoArgsForCall, struct { + }{}) + stub := fake.ToProtoStub + fakeReturns := fake.toProtoReturns + fake.recordInvocation("ToProto", []interface{}{}) + fake.toProtoMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeDataTrack) ToProtoCallCount() int { + fake.toProtoMutex.RLock() + defer fake.toProtoMutex.RUnlock() + return len(fake.toProtoArgsForCall) +} + +func (fake *FakeDataTrack) ToProtoCalls(stub func() *livekit.DataTrackInfo) { + fake.toProtoMutex.Lock() + defer fake.toProtoMutex.Unlock() + fake.ToProtoStub = stub +} + +func (fake *FakeDataTrack) ToProtoReturns(result1 *livekit.DataTrackInfo) { + fake.toProtoMutex.Lock() + defer fake.toProtoMutex.Unlock() + fake.ToProtoStub = nil + fake.toProtoReturns = struct { + result1 *livekit.DataTrackInfo + }{result1} +} + +func (fake *FakeDataTrack) ToProtoReturnsOnCall(i int, result1 *livekit.DataTrackInfo) { + fake.toProtoMutex.Lock() + defer fake.toProtoMutex.Unlock() + fake.ToProtoStub = nil + if fake.toProtoReturnsOnCall == nil { + fake.toProtoReturnsOnCall = make(map[int]struct { + result1 *livekit.DataTrackInfo + }) + } + fake.toProtoReturnsOnCall[i] = struct { + result1 *livekit.DataTrackInfo + }{result1} +} + +func (fake *FakeDataTrack) Invocations() map[string][][]interface{} { + fake.invocationsMutex.RLock() + defer fake.invocationsMutex.RUnlock() + copiedInvocations := map[string][][]interface{}{} + for key, value := range fake.invocations { + copiedInvocations[key] = value + } + return copiedInvocations +} + +func (fake *FakeDataTrack) recordInvocation(key string, args []interface{}) { + fake.invocationsMutex.Lock() + defer fake.invocationsMutex.Unlock() + if fake.invocations == nil { + fake.invocations = map[string][][]interface{}{} + } + if fake.invocations[key] == nil { + fake.invocations[key] = [][]interface{}{} + } + fake.invocations[key] = append(fake.invocations[key], args) +} + +var _ types.DataTrack = new(FakeDataTrack) diff --git a/pkg/rtc/types/typesfakes/fake_data_track_sender.go b/pkg/rtc/types/typesfakes/fake_data_track_sender.go new file mode 100644 index 0000000..a9a6948 --- /dev/null +++ b/pkg/rtc/types/typesfakes/fake_data_track_sender.go @@ -0,0 +1,148 @@ +// Code generated by counterfeiter. DO NOT EDIT. +package typesfakes + +import ( + "sync" + + "github.com/livekit/livekit-server/pkg/rtc/datatrack" + "github.com/livekit/livekit-server/pkg/rtc/types" + "github.com/livekit/protocol/livekit" +) + +type FakeDataTrackSender struct { + SubscriberIDStub func() livekit.ParticipantID + subscriberIDMutex sync.RWMutex + subscriberIDArgsForCall []struct { + } + subscriberIDReturns struct { + result1 livekit.ParticipantID + } + subscriberIDReturnsOnCall map[int]struct { + result1 livekit.ParticipantID + } + WritePacketStub func([]byte, *datatrack.Packet, int64) + writePacketMutex sync.RWMutex + writePacketArgsForCall []struct { + arg1 []byte + arg2 *datatrack.Packet + arg3 int64 + } + invocations map[string][][]interface{} + invocationsMutex sync.RWMutex +} + +func (fake *FakeDataTrackSender) SubscriberID() livekit.ParticipantID { + fake.subscriberIDMutex.Lock() + ret, specificReturn := fake.subscriberIDReturnsOnCall[len(fake.subscriberIDArgsForCall)] + fake.subscriberIDArgsForCall = append(fake.subscriberIDArgsForCall, struct { + }{}) + stub := fake.SubscriberIDStub + fakeReturns := fake.subscriberIDReturns + fake.recordInvocation("SubscriberID", []interface{}{}) + fake.subscriberIDMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeDataTrackSender) SubscriberIDCallCount() int { + fake.subscriberIDMutex.RLock() + defer fake.subscriberIDMutex.RUnlock() + return len(fake.subscriberIDArgsForCall) +} + +func (fake *FakeDataTrackSender) SubscriberIDCalls(stub func() livekit.ParticipantID) { + fake.subscriberIDMutex.Lock() + defer fake.subscriberIDMutex.Unlock() + fake.SubscriberIDStub = stub +} + +func (fake *FakeDataTrackSender) SubscriberIDReturns(result1 livekit.ParticipantID) { + fake.subscriberIDMutex.Lock() + defer fake.subscriberIDMutex.Unlock() + fake.SubscriberIDStub = nil + fake.subscriberIDReturns = struct { + result1 livekit.ParticipantID + }{result1} +} + +func (fake *FakeDataTrackSender) SubscriberIDReturnsOnCall(i int, result1 livekit.ParticipantID) { + fake.subscriberIDMutex.Lock() + defer fake.subscriberIDMutex.Unlock() + fake.SubscriberIDStub = nil + if fake.subscriberIDReturnsOnCall == nil { + fake.subscriberIDReturnsOnCall = make(map[int]struct { + result1 livekit.ParticipantID + }) + } + fake.subscriberIDReturnsOnCall[i] = struct { + result1 livekit.ParticipantID + }{result1} +} + +func (fake *FakeDataTrackSender) WritePacket(arg1 []byte, arg2 *datatrack.Packet, arg3 int64) { + var arg1Copy []byte + if arg1 != nil { + arg1Copy = make([]byte, len(arg1)) + copy(arg1Copy, arg1) + } + fake.writePacketMutex.Lock() + fake.writePacketArgsForCall = append(fake.writePacketArgsForCall, struct { + arg1 []byte + arg2 *datatrack.Packet + arg3 int64 + }{arg1Copy, arg2, arg3}) + stub := fake.WritePacketStub + fake.recordInvocation("WritePacket", []interface{}{arg1Copy, arg2, arg3}) + fake.writePacketMutex.Unlock() + if stub != nil { + fake.WritePacketStub(arg1, arg2, arg3) + } +} + +func (fake *FakeDataTrackSender) WritePacketCallCount() int { + fake.writePacketMutex.RLock() + defer fake.writePacketMutex.RUnlock() + return len(fake.writePacketArgsForCall) +} + +func (fake *FakeDataTrackSender) WritePacketCalls(stub func([]byte, *datatrack.Packet, int64)) { + fake.writePacketMutex.Lock() + defer fake.writePacketMutex.Unlock() + fake.WritePacketStub = stub +} + +func (fake *FakeDataTrackSender) WritePacketArgsForCall(i int) ([]byte, *datatrack.Packet, int64) { + fake.writePacketMutex.RLock() + defer fake.writePacketMutex.RUnlock() + argsForCall := fake.writePacketArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 +} + +func (fake *FakeDataTrackSender) Invocations() map[string][][]interface{} { + fake.invocationsMutex.RLock() + defer fake.invocationsMutex.RUnlock() + copiedInvocations := map[string][][]interface{}{} + for key, value := range fake.invocations { + copiedInvocations[key] = value + } + return copiedInvocations +} + +func (fake *FakeDataTrackSender) recordInvocation(key string, args []interface{}) { + fake.invocationsMutex.Lock() + defer fake.invocationsMutex.Unlock() + if fake.invocations == nil { + fake.invocations = map[string][][]interface{}{} + } + if fake.invocations[key] == nil { + fake.invocations[key] = [][]interface{}{} + } + fake.invocations[key] = append(fake.invocations[key], args) +} + +var _ types.DataTrackSender = new(FakeDataTrackSender) diff --git a/pkg/rtc/types/typesfakes/fake_data_track_transport.go b/pkg/rtc/types/typesfakes/fake_data_track_transport.go new file mode 100644 index 0000000..d93c84f --- /dev/null +++ b/pkg/rtc/types/typesfakes/fake_data_track_transport.go @@ -0,0 +1,114 @@ +// Code generated by counterfeiter. DO NOT EDIT. +package typesfakes + +import ( + "sync" + + "github.com/livekit/livekit-server/pkg/rtc/types" +) + +type FakeDataTrackTransport struct { + SendDataTrackMessageStub func([]byte) error + sendDataTrackMessageMutex sync.RWMutex + sendDataTrackMessageArgsForCall []struct { + arg1 []byte + } + sendDataTrackMessageReturns struct { + result1 error + } + sendDataTrackMessageReturnsOnCall map[int]struct { + result1 error + } + invocations map[string][][]interface{} + invocationsMutex sync.RWMutex +} + +func (fake *FakeDataTrackTransport) SendDataTrackMessage(arg1 []byte) error { + var arg1Copy []byte + if arg1 != nil { + arg1Copy = make([]byte, len(arg1)) + copy(arg1Copy, arg1) + } + fake.sendDataTrackMessageMutex.Lock() + ret, specificReturn := fake.sendDataTrackMessageReturnsOnCall[len(fake.sendDataTrackMessageArgsForCall)] + fake.sendDataTrackMessageArgsForCall = append(fake.sendDataTrackMessageArgsForCall, struct { + arg1 []byte + }{arg1Copy}) + stub := fake.SendDataTrackMessageStub + fakeReturns := fake.sendDataTrackMessageReturns + fake.recordInvocation("SendDataTrackMessage", []interface{}{arg1Copy}) + fake.sendDataTrackMessageMutex.Unlock() + if stub != nil { + return stub(arg1) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeDataTrackTransport) SendDataTrackMessageCallCount() int { + fake.sendDataTrackMessageMutex.RLock() + defer fake.sendDataTrackMessageMutex.RUnlock() + return len(fake.sendDataTrackMessageArgsForCall) +} + +func (fake *FakeDataTrackTransport) SendDataTrackMessageCalls(stub func([]byte) error) { + fake.sendDataTrackMessageMutex.Lock() + defer fake.sendDataTrackMessageMutex.Unlock() + fake.SendDataTrackMessageStub = stub +} + +func (fake *FakeDataTrackTransport) SendDataTrackMessageArgsForCall(i int) []byte { + fake.sendDataTrackMessageMutex.RLock() + defer fake.sendDataTrackMessageMutex.RUnlock() + argsForCall := fake.sendDataTrackMessageArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeDataTrackTransport) SendDataTrackMessageReturns(result1 error) { + fake.sendDataTrackMessageMutex.Lock() + defer fake.sendDataTrackMessageMutex.Unlock() + fake.SendDataTrackMessageStub = nil + fake.sendDataTrackMessageReturns = struct { + result1 error + }{result1} +} + +func (fake *FakeDataTrackTransport) SendDataTrackMessageReturnsOnCall(i int, result1 error) { + fake.sendDataTrackMessageMutex.Lock() + defer fake.sendDataTrackMessageMutex.Unlock() + fake.SendDataTrackMessageStub = nil + if fake.sendDataTrackMessageReturnsOnCall == nil { + fake.sendDataTrackMessageReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.sendDataTrackMessageReturnsOnCall[i] = struct { + result1 error + }{result1} +} + +func (fake *FakeDataTrackTransport) Invocations() map[string][][]interface{} { + fake.invocationsMutex.RLock() + defer fake.invocationsMutex.RUnlock() + copiedInvocations := map[string][][]interface{}{} + for key, value := range fake.invocations { + copiedInvocations[key] = value + } + return copiedInvocations +} + +func (fake *FakeDataTrackTransport) recordInvocation(key string, args []interface{}) { + fake.invocationsMutex.Lock() + defer fake.invocationsMutex.Unlock() + if fake.invocations == nil { + fake.invocations = map[string][][]interface{}{} + } + if fake.invocations[key] == nil { + fake.invocations[key] = [][]interface{}{} + } + fake.invocations[key] = append(fake.invocations[key], args) +} + +var _ types.DataTrackTransport = new(FakeDataTrackTransport) diff --git a/pkg/rtc/types/typesfakes/fake_local_media_track.go b/pkg/rtc/types/typesfakes/fake_local_media_track.go index 877267f..d66f429 100644 --- a/pkg/rtc/types/typesfakes/fake_local_media_track.go +++ b/pkg/rtc/types/typesfakes/fake_local_media_track.go @@ -6,8 +6,9 @@ import ( "github.com/livekit/livekit-server/pkg/rtc/types" "github.com/livekit/livekit-server/pkg/sfu" - "github.com/livekit/livekit-server/pkg/sfu/mime" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/logger" ) type FakeLocalMediaTrack struct { @@ -34,6 +35,10 @@ type FakeLocalMediaTrack struct { clearAllReceiversArgsForCall []struct { arg1 bool } + ClearSubscriberNodesStub func() + clearSubscriberNodesMutex sync.RWMutex + clearSubscriberNodesArgsForCall []struct { + } CloseStub func(bool) closeMutex sync.RWMutex closeArgsForCall []struct { @@ -83,11 +88,12 @@ type FakeLocalMediaTrack struct { getNumSubscribersReturnsOnCall map[int]struct { result1 int } - GetQualityForDimensionStub func(uint32, uint32) livekit.VideoQuality + GetQualityForDimensionStub func(mime.MimeType, uint32, uint32) livekit.VideoQuality getQualityForDimensionMutex sync.RWMutex getQualityForDimensionArgsForCall []struct { - arg1 uint32 + arg1 mime.MimeType arg2 uint32 + arg3 uint32 } getQualityForDimensionReturns struct { result1 livekit.VideoQuality @@ -95,12 +101,12 @@ type FakeLocalMediaTrack struct { getQualityForDimensionReturnsOnCall map[int]struct { result1 livekit.VideoQuality } - GetTemporalLayerForSpatialFpsStub func(int32, uint32, mime.MimeType) int32 + GetTemporalLayerForSpatialFpsStub func(mime.MimeType, int32, uint32) int32 getTemporalLayerForSpatialFpsMutex sync.RWMutex getTemporalLayerForSpatialFpsArgsForCall []struct { - arg1 int32 - arg2 uint32 - arg3 mime.MimeType + arg1 mime.MimeType + arg2 int32 + arg3 uint32 } getTemporalLayerForSpatialFpsReturns struct { result1 int32 @@ -118,6 +124,16 @@ type FakeLocalMediaTrack struct { getTrackStatsReturnsOnCall map[int]struct { result1 *livekit.RTPStats } + HasPacketTrailerStub func() bool + hasPacketTrailerMutex sync.RWMutex + hasPacketTrailerArgsForCall []struct { + } + hasPacketTrailerReturns struct { + result1 bool + } + hasPacketTrailerReturnsOnCall map[int]struct { + result1 bool + } HasSdpCidStub func(string) bool hasSdpCidMutex sync.RWMutex hasSdpCidArgsForCall []struct { @@ -129,6 +145,17 @@ type FakeLocalMediaTrack struct { hasSdpCidReturnsOnCall map[int]struct { result1 bool } + HasSignalCidStub func(string) bool + hasSignalCidMutex sync.RWMutex + hasSignalCidArgsForCall []struct { + arg1 string + } + hasSignalCidReturns struct { + result1 bool + } + hasSignalCidReturnsOnCall map[int]struct { + result1 bool + } IDStub func() livekit.TrackID iDMutex sync.RWMutex iDArgsForCall []struct { @@ -169,16 +196,6 @@ type FakeLocalMediaTrack struct { isOpenReturnsOnCall map[int]struct { result1 bool } - IsSimulcastStub func() bool - isSimulcastMutex sync.RWMutex - isSimulcastArgsForCall []struct { - } - isSimulcastReturns struct { - result1 bool - } - isSimulcastReturnsOnCall map[int]struct { - result1 bool - } IsSubscriberStub func(livekit.ParticipantID) bool isSubscriberMutex sync.RWMutex isSubscriberArgsForCall []struct { @@ -200,6 +217,16 @@ type FakeLocalMediaTrack struct { kindReturnsOnCall map[int]struct { result1 livekit.TrackType } + LoggerStub func() logger.Logger + loggerMutex sync.RWMutex + loggerArgsForCall []struct { + } + loggerReturns struct { + result1 logger.Logger + } + loggerReturnsOnCall map[int]struct { + result1 logger.Logger + } NameStub func() string nameMutex sync.RWMutex nameArgsForCall []struct { @@ -222,6 +249,12 @@ type FakeLocalMediaTrack struct { arg1 livekit.NodeID arg2 uint8 } + NotifySubscriptionNodeStub func(livekit.NodeID, []*livekit.SubscribedAudioCodec) + notifySubscriptionNodeMutex sync.RWMutex + notifySubscriptionNodeArgsForCall []struct { + arg1 livekit.NodeID + arg2 []*livekit.SubscribedAudioCodec + } OnTrackSubscribedStub func() onTrackSubscribedMutex sync.RWMutex onTrackSubscribedArgsForCall []struct { @@ -297,16 +330,6 @@ type FakeLocalMediaTrack struct { setRTTArgsForCall []struct { arg1 uint32 } - SignalCidStub func() string - signalCidMutex sync.RWMutex - signalCidArgsForCall []struct { - } - signalCidReturns struct { - result1 string - } - signalCidReturnsOnCall map[int]struct { - result1 string - } SourceStub func() livekit.TrackSource sourceMutex sync.RWMutex sourceArgsForCall []struct { @@ -484,6 +507,30 @@ func (fake *FakeLocalMediaTrack) ClearAllReceiversArgsForCall(i int) bool { return argsForCall.arg1 } +func (fake *FakeLocalMediaTrack) ClearSubscriberNodes() { + fake.clearSubscriberNodesMutex.Lock() + fake.clearSubscriberNodesArgsForCall = append(fake.clearSubscriberNodesArgsForCall, struct { + }{}) + stub := fake.ClearSubscriberNodesStub + fake.recordInvocation("ClearSubscriberNodes", []interface{}{}) + fake.clearSubscriberNodesMutex.Unlock() + if stub != nil { + fake.ClearSubscriberNodesStub() + } +} + +func (fake *FakeLocalMediaTrack) ClearSubscriberNodesCallCount() int { + fake.clearSubscriberNodesMutex.RLock() + defer fake.clearSubscriberNodesMutex.RUnlock() + return len(fake.clearSubscriberNodesArgsForCall) +} + +func (fake *FakeLocalMediaTrack) ClearSubscriberNodesCalls(stub func()) { + fake.clearSubscriberNodesMutex.Lock() + defer fake.clearSubscriberNodesMutex.Unlock() + fake.ClearSubscriberNodesStub = stub +} + func (fake *FakeLocalMediaTrack) Close(arg1 bool) { fake.closeMutex.Lock() fake.closeArgsForCall = append(fake.closeArgsForCall, struct { @@ -734,19 +781,20 @@ func (fake *FakeLocalMediaTrack) GetNumSubscribersReturnsOnCall(i int, result1 i }{result1} } -func (fake *FakeLocalMediaTrack) GetQualityForDimension(arg1 uint32, arg2 uint32) livekit.VideoQuality { +func (fake *FakeLocalMediaTrack) GetQualityForDimension(arg1 mime.MimeType, arg2 uint32, arg3 uint32) livekit.VideoQuality { fake.getQualityForDimensionMutex.Lock() ret, specificReturn := fake.getQualityForDimensionReturnsOnCall[len(fake.getQualityForDimensionArgsForCall)] fake.getQualityForDimensionArgsForCall = append(fake.getQualityForDimensionArgsForCall, struct { - arg1 uint32 + arg1 mime.MimeType arg2 uint32 - }{arg1, arg2}) + arg3 uint32 + }{arg1, arg2, arg3}) stub := fake.GetQualityForDimensionStub fakeReturns := fake.getQualityForDimensionReturns - fake.recordInvocation("GetQualityForDimension", []interface{}{arg1, arg2}) + fake.recordInvocation("GetQualityForDimension", []interface{}{arg1, arg2, arg3}) fake.getQualityForDimensionMutex.Unlock() if stub != nil { - return stub(arg1, arg2) + return stub(arg1, arg2, arg3) } if specificReturn { return ret.result1 @@ -760,17 +808,17 @@ func (fake *FakeLocalMediaTrack) GetQualityForDimensionCallCount() int { return len(fake.getQualityForDimensionArgsForCall) } -func (fake *FakeLocalMediaTrack) GetQualityForDimensionCalls(stub func(uint32, uint32) livekit.VideoQuality) { +func (fake *FakeLocalMediaTrack) GetQualityForDimensionCalls(stub func(mime.MimeType, uint32, uint32) livekit.VideoQuality) { fake.getQualityForDimensionMutex.Lock() defer fake.getQualityForDimensionMutex.Unlock() fake.GetQualityForDimensionStub = stub } -func (fake *FakeLocalMediaTrack) GetQualityForDimensionArgsForCall(i int) (uint32, uint32) { +func (fake *FakeLocalMediaTrack) GetQualityForDimensionArgsForCall(i int) (mime.MimeType, uint32, uint32) { fake.getQualityForDimensionMutex.RLock() defer fake.getQualityForDimensionMutex.RUnlock() argsForCall := fake.getQualityForDimensionArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2 + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 } func (fake *FakeLocalMediaTrack) GetQualityForDimensionReturns(result1 livekit.VideoQuality) { @@ -796,13 +844,13 @@ func (fake *FakeLocalMediaTrack) GetQualityForDimensionReturnsOnCall(i int, resu }{result1} } -func (fake *FakeLocalMediaTrack) GetTemporalLayerForSpatialFps(arg1 int32, arg2 uint32, arg3 mime.MimeType) int32 { +func (fake *FakeLocalMediaTrack) GetTemporalLayerForSpatialFps(arg1 mime.MimeType, arg2 int32, arg3 uint32) int32 { fake.getTemporalLayerForSpatialFpsMutex.Lock() ret, specificReturn := fake.getTemporalLayerForSpatialFpsReturnsOnCall[len(fake.getTemporalLayerForSpatialFpsArgsForCall)] fake.getTemporalLayerForSpatialFpsArgsForCall = append(fake.getTemporalLayerForSpatialFpsArgsForCall, struct { - arg1 int32 - arg2 uint32 - arg3 mime.MimeType + arg1 mime.MimeType + arg2 int32 + arg3 uint32 }{arg1, arg2, arg3}) stub := fake.GetTemporalLayerForSpatialFpsStub fakeReturns := fake.getTemporalLayerForSpatialFpsReturns @@ -823,13 +871,13 @@ func (fake *FakeLocalMediaTrack) GetTemporalLayerForSpatialFpsCallCount() int { return len(fake.getTemporalLayerForSpatialFpsArgsForCall) } -func (fake *FakeLocalMediaTrack) GetTemporalLayerForSpatialFpsCalls(stub func(int32, uint32, mime.MimeType) int32) { +func (fake *FakeLocalMediaTrack) GetTemporalLayerForSpatialFpsCalls(stub func(mime.MimeType, int32, uint32) int32) { fake.getTemporalLayerForSpatialFpsMutex.Lock() defer fake.getTemporalLayerForSpatialFpsMutex.Unlock() fake.GetTemporalLayerForSpatialFpsStub = stub } -func (fake *FakeLocalMediaTrack) GetTemporalLayerForSpatialFpsArgsForCall(i int) (int32, uint32, mime.MimeType) { +func (fake *FakeLocalMediaTrack) GetTemporalLayerForSpatialFpsArgsForCall(i int) (mime.MimeType, int32, uint32) { fake.getTemporalLayerForSpatialFpsMutex.RLock() defer fake.getTemporalLayerForSpatialFpsMutex.RUnlock() argsForCall := fake.getTemporalLayerForSpatialFpsArgsForCall[i] @@ -912,6 +960,59 @@ func (fake *FakeLocalMediaTrack) GetTrackStatsReturnsOnCall(i int, result1 *live }{result1} } +func (fake *FakeLocalMediaTrack) HasPacketTrailer() bool { + fake.hasPacketTrailerMutex.Lock() + ret, specificReturn := fake.hasPacketTrailerReturnsOnCall[len(fake.hasPacketTrailerArgsForCall)] + fake.hasPacketTrailerArgsForCall = append(fake.hasPacketTrailerArgsForCall, struct { + }{}) + stub := fake.HasPacketTrailerStub + fakeReturns := fake.hasPacketTrailerReturns + fake.recordInvocation("HasPacketTrailer", []interface{}{}) + fake.hasPacketTrailerMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeLocalMediaTrack) HasPacketTrailerCallCount() int { + fake.hasPacketTrailerMutex.RLock() + defer fake.hasPacketTrailerMutex.RUnlock() + return len(fake.hasPacketTrailerArgsForCall) +} + +func (fake *FakeLocalMediaTrack) HasPacketTrailerCalls(stub func() bool) { + fake.hasPacketTrailerMutex.Lock() + defer fake.hasPacketTrailerMutex.Unlock() + fake.HasPacketTrailerStub = stub +} + +func (fake *FakeLocalMediaTrack) HasPacketTrailerReturns(result1 bool) { + fake.hasPacketTrailerMutex.Lock() + defer fake.hasPacketTrailerMutex.Unlock() + fake.HasPacketTrailerStub = nil + fake.hasPacketTrailerReturns = struct { + result1 bool + }{result1} +} + +func (fake *FakeLocalMediaTrack) HasPacketTrailerReturnsOnCall(i int, result1 bool) { + fake.hasPacketTrailerMutex.Lock() + defer fake.hasPacketTrailerMutex.Unlock() + fake.HasPacketTrailerStub = nil + if fake.hasPacketTrailerReturnsOnCall == nil { + fake.hasPacketTrailerReturnsOnCall = make(map[int]struct { + result1 bool + }) + } + fake.hasPacketTrailerReturnsOnCall[i] = struct { + result1 bool + }{result1} +} + func (fake *FakeLocalMediaTrack) HasSdpCid(arg1 string) bool { fake.hasSdpCidMutex.Lock() ret, specificReturn := fake.hasSdpCidReturnsOnCall[len(fake.hasSdpCidArgsForCall)] @@ -973,6 +1074,67 @@ func (fake *FakeLocalMediaTrack) HasSdpCidReturnsOnCall(i int, result1 bool) { }{result1} } +func (fake *FakeLocalMediaTrack) HasSignalCid(arg1 string) bool { + fake.hasSignalCidMutex.Lock() + ret, specificReturn := fake.hasSignalCidReturnsOnCall[len(fake.hasSignalCidArgsForCall)] + fake.hasSignalCidArgsForCall = append(fake.hasSignalCidArgsForCall, struct { + arg1 string + }{arg1}) + stub := fake.HasSignalCidStub + fakeReturns := fake.hasSignalCidReturns + fake.recordInvocation("HasSignalCid", []interface{}{arg1}) + fake.hasSignalCidMutex.Unlock() + if stub != nil { + return stub(arg1) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeLocalMediaTrack) HasSignalCidCallCount() int { + fake.hasSignalCidMutex.RLock() + defer fake.hasSignalCidMutex.RUnlock() + return len(fake.hasSignalCidArgsForCall) +} + +func (fake *FakeLocalMediaTrack) HasSignalCidCalls(stub func(string) bool) { + fake.hasSignalCidMutex.Lock() + defer fake.hasSignalCidMutex.Unlock() + fake.HasSignalCidStub = stub +} + +func (fake *FakeLocalMediaTrack) HasSignalCidArgsForCall(i int) string { + fake.hasSignalCidMutex.RLock() + defer fake.hasSignalCidMutex.RUnlock() + argsForCall := fake.hasSignalCidArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeLocalMediaTrack) HasSignalCidReturns(result1 bool) { + fake.hasSignalCidMutex.Lock() + defer fake.hasSignalCidMutex.Unlock() + fake.HasSignalCidStub = nil + fake.hasSignalCidReturns = struct { + result1 bool + }{result1} +} + +func (fake *FakeLocalMediaTrack) HasSignalCidReturnsOnCall(i int, result1 bool) { + fake.hasSignalCidMutex.Lock() + defer fake.hasSignalCidMutex.Unlock() + fake.HasSignalCidStub = nil + if fake.hasSignalCidReturnsOnCall == nil { + fake.hasSignalCidReturnsOnCall = make(map[int]struct { + result1 bool + }) + } + fake.hasSignalCidReturnsOnCall[i] = struct { + result1 bool + }{result1} +} + func (fake *FakeLocalMediaTrack) ID() livekit.TrackID { fake.iDMutex.Lock() ret, specificReturn := fake.iDReturnsOnCall[len(fake.iDArgsForCall)] @@ -1185,59 +1347,6 @@ func (fake *FakeLocalMediaTrack) IsOpenReturnsOnCall(i int, result1 bool) { }{result1} } -func (fake *FakeLocalMediaTrack) IsSimulcast() bool { - fake.isSimulcastMutex.Lock() - ret, specificReturn := fake.isSimulcastReturnsOnCall[len(fake.isSimulcastArgsForCall)] - fake.isSimulcastArgsForCall = append(fake.isSimulcastArgsForCall, struct { - }{}) - stub := fake.IsSimulcastStub - fakeReturns := fake.isSimulcastReturns - fake.recordInvocation("IsSimulcast", []interface{}{}) - fake.isSimulcastMutex.Unlock() - if stub != nil { - return stub() - } - if specificReturn { - return ret.result1 - } - return fakeReturns.result1 -} - -func (fake *FakeLocalMediaTrack) IsSimulcastCallCount() int { - fake.isSimulcastMutex.RLock() - defer fake.isSimulcastMutex.RUnlock() - return len(fake.isSimulcastArgsForCall) -} - -func (fake *FakeLocalMediaTrack) IsSimulcastCalls(stub func() bool) { - fake.isSimulcastMutex.Lock() - defer fake.isSimulcastMutex.Unlock() - fake.IsSimulcastStub = stub -} - -func (fake *FakeLocalMediaTrack) IsSimulcastReturns(result1 bool) { - fake.isSimulcastMutex.Lock() - defer fake.isSimulcastMutex.Unlock() - fake.IsSimulcastStub = nil - fake.isSimulcastReturns = struct { - result1 bool - }{result1} -} - -func (fake *FakeLocalMediaTrack) IsSimulcastReturnsOnCall(i int, result1 bool) { - fake.isSimulcastMutex.Lock() - defer fake.isSimulcastMutex.Unlock() - fake.IsSimulcastStub = nil - if fake.isSimulcastReturnsOnCall == nil { - fake.isSimulcastReturnsOnCall = make(map[int]struct { - result1 bool - }) - } - fake.isSimulcastReturnsOnCall[i] = struct { - result1 bool - }{result1} -} - func (fake *FakeLocalMediaTrack) IsSubscriber(arg1 livekit.ParticipantID) bool { fake.isSubscriberMutex.Lock() ret, specificReturn := fake.isSubscriberReturnsOnCall[len(fake.isSubscriberArgsForCall)] @@ -1352,6 +1461,59 @@ func (fake *FakeLocalMediaTrack) KindReturnsOnCall(i int, result1 livekit.TrackT }{result1} } +func (fake *FakeLocalMediaTrack) Logger() logger.Logger { + fake.loggerMutex.Lock() + ret, specificReturn := fake.loggerReturnsOnCall[len(fake.loggerArgsForCall)] + fake.loggerArgsForCall = append(fake.loggerArgsForCall, struct { + }{}) + stub := fake.LoggerStub + fakeReturns := fake.loggerReturns + fake.recordInvocation("Logger", []interface{}{}) + fake.loggerMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeLocalMediaTrack) LoggerCallCount() int { + fake.loggerMutex.RLock() + defer fake.loggerMutex.RUnlock() + return len(fake.loggerArgsForCall) +} + +func (fake *FakeLocalMediaTrack) LoggerCalls(stub func() logger.Logger) { + fake.loggerMutex.Lock() + defer fake.loggerMutex.Unlock() + fake.LoggerStub = stub +} + +func (fake *FakeLocalMediaTrack) LoggerReturns(result1 logger.Logger) { + fake.loggerMutex.Lock() + defer fake.loggerMutex.Unlock() + fake.LoggerStub = nil + fake.loggerReturns = struct { + result1 logger.Logger + }{result1} +} + +func (fake *FakeLocalMediaTrack) LoggerReturnsOnCall(i int, result1 logger.Logger) { + fake.loggerMutex.Lock() + defer fake.loggerMutex.Unlock() + fake.LoggerStub = nil + if fake.loggerReturnsOnCall == nil { + fake.loggerReturnsOnCall = make(map[int]struct { + result1 logger.Logger + }) + } + fake.loggerReturnsOnCall[i] = struct { + result1 logger.Logger + }{result1} +} + func (fake *FakeLocalMediaTrack) Name() string { fake.nameMutex.Lock() ret, specificReturn := fake.nameReturnsOnCall[len(fake.nameArgsForCall)] @@ -1476,6 +1638,44 @@ func (fake *FakeLocalMediaTrack) NotifySubscriberNodeMediaLossArgsForCall(i int) return argsForCall.arg1, argsForCall.arg2 } +func (fake *FakeLocalMediaTrack) NotifySubscriptionNode(arg1 livekit.NodeID, arg2 []*livekit.SubscribedAudioCodec) { + var arg2Copy []*livekit.SubscribedAudioCodec + if arg2 != nil { + arg2Copy = make([]*livekit.SubscribedAudioCodec, len(arg2)) + copy(arg2Copy, arg2) + } + fake.notifySubscriptionNodeMutex.Lock() + fake.notifySubscriptionNodeArgsForCall = append(fake.notifySubscriptionNodeArgsForCall, struct { + arg1 livekit.NodeID + arg2 []*livekit.SubscribedAudioCodec + }{arg1, arg2Copy}) + stub := fake.NotifySubscriptionNodeStub + fake.recordInvocation("NotifySubscriptionNode", []interface{}{arg1, arg2Copy}) + fake.notifySubscriptionNodeMutex.Unlock() + if stub != nil { + fake.NotifySubscriptionNodeStub(arg1, arg2) + } +} + +func (fake *FakeLocalMediaTrack) NotifySubscriptionNodeCallCount() int { + fake.notifySubscriptionNodeMutex.RLock() + defer fake.notifySubscriptionNodeMutex.RUnlock() + return len(fake.notifySubscriptionNodeArgsForCall) +} + +func (fake *FakeLocalMediaTrack) NotifySubscriptionNodeCalls(stub func(livekit.NodeID, []*livekit.SubscribedAudioCodec)) { + fake.notifySubscriptionNodeMutex.Lock() + defer fake.notifySubscriptionNodeMutex.Unlock() + fake.NotifySubscriptionNodeStub = stub +} + +func (fake *FakeLocalMediaTrack) NotifySubscriptionNodeArgsForCall(i int) (livekit.NodeID, []*livekit.SubscribedAudioCodec) { + fake.notifySubscriptionNodeMutex.RLock() + defer fake.notifySubscriptionNodeMutex.RUnlock() + argsForCall := fake.notifySubscriptionNodeArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + func (fake *FakeLocalMediaTrack) OnTrackSubscribed() { fake.onTrackSubscribedMutex.Lock() fake.onTrackSubscribedArgsForCall = append(fake.onTrackSubscribedArgsForCall, struct { @@ -1899,59 +2099,6 @@ func (fake *FakeLocalMediaTrack) SetRTTArgsForCall(i int) uint32 { return argsForCall.arg1 } -func (fake *FakeLocalMediaTrack) SignalCid() string { - fake.signalCidMutex.Lock() - ret, specificReturn := fake.signalCidReturnsOnCall[len(fake.signalCidArgsForCall)] - fake.signalCidArgsForCall = append(fake.signalCidArgsForCall, struct { - }{}) - stub := fake.SignalCidStub - fakeReturns := fake.signalCidReturns - fake.recordInvocation("SignalCid", []interface{}{}) - fake.signalCidMutex.Unlock() - if stub != nil { - return stub() - } - if specificReturn { - return ret.result1 - } - return fakeReturns.result1 -} - -func (fake *FakeLocalMediaTrack) SignalCidCallCount() int { - fake.signalCidMutex.RLock() - defer fake.signalCidMutex.RUnlock() - return len(fake.signalCidArgsForCall) -} - -func (fake *FakeLocalMediaTrack) SignalCidCalls(stub func() string) { - fake.signalCidMutex.Lock() - defer fake.signalCidMutex.Unlock() - fake.SignalCidStub = stub -} - -func (fake *FakeLocalMediaTrack) SignalCidReturns(result1 string) { - fake.signalCidMutex.Lock() - defer fake.signalCidMutex.Unlock() - fake.SignalCidStub = nil - fake.signalCidReturns = struct { - result1 string - }{result1} -} - -func (fake *FakeLocalMediaTrack) SignalCidReturnsOnCall(i int, result1 string) { - fake.signalCidMutex.Lock() - defer fake.signalCidMutex.Unlock() - fake.SignalCidStub = nil - if fake.signalCidReturnsOnCall == nil { - fake.signalCidReturnsOnCall = make(map[int]struct { - result1 string - }) - } - fake.signalCidReturnsOnCall[i] = struct { - result1 string - }{result1} -} - func (fake *FakeLocalMediaTrack) Source() livekit.TrackSource { fake.sourceMutex.Lock() ret, specificReturn := fake.sourceReturnsOnCall[len(fake.sourceArgsForCall)] @@ -2210,84 +2357,6 @@ func (fake *FakeLocalMediaTrack) UpdateVideoTrackArgsForCall(i int) *livekit.Upd func (fake *FakeLocalMediaTrack) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() - fake.addOnCloseMutex.RLock() - defer fake.addOnCloseMutex.RUnlock() - fake.addSubscriberMutex.RLock() - defer fake.addSubscriberMutex.RUnlock() - fake.clearAllReceiversMutex.RLock() - defer fake.clearAllReceiversMutex.RUnlock() - fake.closeMutex.RLock() - defer fake.closeMutex.RUnlock() - fake.getAllSubscribersMutex.RLock() - defer fake.getAllSubscribersMutex.RUnlock() - fake.getAudioLevelMutex.RLock() - defer fake.getAudioLevelMutex.RUnlock() - fake.getConnectionScoreAndQualityMutex.RLock() - defer fake.getConnectionScoreAndQualityMutex.RUnlock() - fake.getNumSubscribersMutex.RLock() - defer fake.getNumSubscribersMutex.RUnlock() - fake.getQualityForDimensionMutex.RLock() - defer fake.getQualityForDimensionMutex.RUnlock() - fake.getTemporalLayerForSpatialFpsMutex.RLock() - defer fake.getTemporalLayerForSpatialFpsMutex.RUnlock() - fake.getTrackStatsMutex.RLock() - defer fake.getTrackStatsMutex.RUnlock() - fake.hasSdpCidMutex.RLock() - defer fake.hasSdpCidMutex.RUnlock() - fake.iDMutex.RLock() - defer fake.iDMutex.RUnlock() - fake.isEncryptedMutex.RLock() - defer fake.isEncryptedMutex.RUnlock() - fake.isMutedMutex.RLock() - defer fake.isMutedMutex.RUnlock() - fake.isOpenMutex.RLock() - defer fake.isOpenMutex.RUnlock() - fake.isSimulcastMutex.RLock() - defer fake.isSimulcastMutex.RUnlock() - fake.isSubscriberMutex.RLock() - defer fake.isSubscriberMutex.RUnlock() - fake.kindMutex.RLock() - defer fake.kindMutex.RUnlock() - fake.nameMutex.RLock() - defer fake.nameMutex.RUnlock() - fake.notifySubscriberNodeMaxQualityMutex.RLock() - defer fake.notifySubscriberNodeMaxQualityMutex.RUnlock() - fake.notifySubscriberNodeMediaLossMutex.RLock() - defer fake.notifySubscriberNodeMediaLossMutex.RUnlock() - fake.onTrackSubscribedMutex.RLock() - defer fake.onTrackSubscribedMutex.RUnlock() - fake.publisherIDMutex.RLock() - defer fake.publisherIDMutex.RUnlock() - fake.publisherIdentityMutex.RLock() - defer fake.publisherIdentityMutex.RUnlock() - fake.publisherVersionMutex.RLock() - defer fake.publisherVersionMutex.RUnlock() - fake.receiversMutex.RLock() - defer fake.receiversMutex.RUnlock() - fake.removeSubscriberMutex.RLock() - defer fake.removeSubscriberMutex.RUnlock() - fake.restartMutex.RLock() - defer fake.restartMutex.RUnlock() - fake.revokeDisallowedSubscribersMutex.RLock() - defer fake.revokeDisallowedSubscribersMutex.RUnlock() - fake.setMutedMutex.RLock() - defer fake.setMutedMutex.RUnlock() - fake.setRTTMutex.RLock() - defer fake.setRTTMutex.RUnlock() - fake.signalCidMutex.RLock() - defer fake.signalCidMutex.RUnlock() - fake.sourceMutex.RLock() - defer fake.sourceMutex.RUnlock() - fake.streamMutex.RLock() - defer fake.streamMutex.RUnlock() - fake.toProtoMutex.RLock() - defer fake.toProtoMutex.RUnlock() - fake.updateAudioTrackMutex.RLock() - defer fake.updateAudioTrackMutex.RUnlock() - fake.updateTrackInfoMutex.RLock() - defer fake.updateTrackInfoMutex.RUnlock() - fake.updateVideoTrackMutex.RLock() - defer fake.updateVideoTrackMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} for key, value := range fake.invocations { copiedInvocations[key] = value diff --git a/pkg/rtc/types/typesfakes/fake_local_participant.go b/pkg/rtc/types/typesfakes/fake_local_participant.go index f71d519..96686a0 100644 --- a/pkg/rtc/types/typesfakes/fake_local_participant.go +++ b/pkg/rtc/types/typesfakes/fake_local_participant.go @@ -6,24 +6,38 @@ import ( "time" "github.com/livekit/livekit-server/pkg/routing" + "github.com/livekit/livekit-server/pkg/rtc/datatrack" "github.com/livekit/livekit-server/pkg/rtc/types" "github.com/livekit/livekit-server/pkg/sfu" "github.com/livekit/livekit-server/pkg/sfu/buffer" "github.com/livekit/livekit-server/pkg/sfu/pacer" + "github.com/livekit/livekit-server/pkg/telemetry" "github.com/livekit/protocol/auth" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" + "github.com/livekit/protocol/observability/roomobs" "github.com/livekit/protocol/utils" "github.com/pion/rtcp" webrtc "github.com/pion/webrtc/v4" + "google.golang.org/protobuf/proto" ) type FakeLocalParticipant struct { - AddICECandidateStub func(webrtc.ICECandidateInit, livekit.SignalTarget) - addICECandidateMutex sync.RWMutex - addICECandidateArgsForCall []struct { - arg1 webrtc.ICECandidateInit - arg2 livekit.SignalTarget + ActiveAtStub func() time.Time + activeAtMutex sync.RWMutex + activeAtArgsForCall []struct { + } + activeAtReturns struct { + result1 time.Time + } + activeAtReturnsOnCall map[int]struct { + result1 time.Time + } + AddOnCloseStub func(string, func(types.LocalParticipant)) + addOnCloseMutex sync.RWMutex + addOnCloseArgsForCall []struct { + arg1 string + arg2 func(types.LocalParticipant) } AddTrackStub func(*livekit.AddTrackRequest) addTrackMutex sync.RWMutex @@ -120,19 +134,6 @@ type FakeLocalParticipant struct { canSubscribeReturnsOnCall map[int]struct { result1 bool } - CheckMetadataLimitsStub func(string, string, map[string]string) error - checkMetadataLimitsMutex sync.RWMutex - checkMetadataLimitsArgsForCall []struct { - arg1 string - arg2 string - arg3 map[string]string - } - checkMetadataLimitsReturns struct { - result1 error - } - checkMetadataLimitsReturnsOnCall map[int]struct { - result1 error - } ClaimGrantsStub func() *auth.ClaimGrants claimGrantsMutex sync.RWMutex claimGrantsArgsForCall []struct { @@ -143,6 +144,10 @@ type FakeLocalParticipant struct { claimGrantsReturnsOnCall map[int]struct { result1 *auth.ClaimGrants } + ClearParticipantListenerStub func() + clearParticipantListenerMutex sync.RWMutex + clearParticipantListenerArgsForCall []struct { + } CloseStub func(bool, types.ParticipantCloseReason, bool) error closeMutex sync.RWMutex closeArgsForCall []struct { @@ -181,15 +186,15 @@ type FakeLocalParticipant struct { connectedAtReturnsOnCall map[int]struct { result1 time.Time } - DebugInfoStub func() map[string]interface{} + DebugInfoStub func() map[string]any debugInfoMutex sync.RWMutex debugInfoArgsForCall []struct { } debugInfoReturns struct { - result1 map[string]interface{} + result1 map[string]any } debugInfoReturnsOnCall map[int]struct { - result1 map[string]interface{} + result1 map[string]any } DisconnectedStub func() <-chan struct{} disconnectedMutex sync.RWMutex @@ -211,17 +216,19 @@ type FakeLocalParticipant struct { getAdaptiveStreamReturnsOnCall map[int]struct { result1 bool } - GetAnswerStub func() (webrtc.SessionDescription, error) + GetAnswerStub func() (webrtc.SessionDescription, uint32, error) getAnswerMutex sync.RWMutex getAnswerArgsForCall []struct { } getAnswerReturns struct { result1 webrtc.SessionDescription - result2 error + result2 uint32 + result3 error } getAnswerReturnsOnCall map[int]struct { result1 webrtc.SessionDescription - result2 error + result2 uint32 + result3 error } GetAudioLevelStub func() (float64, bool) getAudioLevelMutex sync.RWMutex @@ -288,6 +295,26 @@ type FakeLocalParticipant struct { getConnectionQualityReturnsOnCall map[int]struct { result1 *livekit.ConnectionQualityInfo } + GetCountryStub func() string + getCountryMutex sync.RWMutex + getCountryArgsForCall []struct { + } + getCountryReturns struct { + result1 string + } + getCountryReturnsOnCall map[int]struct { + result1 string + } + GetDataTrackTransportStub func() types.DataTrackTransport + getDataTrackTransportMutex sync.RWMutex + getDataTrackTransportArgsForCall []struct { + } + getDataTrackTransportReturns struct { + result1 types.DataTrackTransport + } + getDataTrackTransportReturnsOnCall map[int]struct { + result1 types.DataTrackTransport + } GetDisableSenderReportPassThroughStub func() bool getDisableSenderReportPassThroughMutex sync.RWMutex getDisableSenderReportPassThroughArgsForCall []struct { @@ -328,6 +355,17 @@ type FakeLocalParticipant struct { getICEConnectionInfoReturnsOnCall map[int]struct { result1 []*types.ICEConnectionInfo } + GetLastReliableSequenceStub func(bool) uint32 + getLastReliableSequenceMutex sync.RWMutex + getLastReliableSequenceArgsForCall []struct { + arg1 bool + } + getLastReliableSequenceReturns struct { + result1 uint32 + } + getLastReliableSequenceReturnsOnCall map[int]struct { + result1 uint32 + } GetLoggerStub func() logger.Logger getLoggerMutex sync.RWMutex getLoggerArgsForCall []struct { @@ -338,6 +376,26 @@ type FakeLocalParticipant struct { getLoggerReturnsOnCall map[int]struct { result1 logger.Logger } + GetLoggerResolverStub func() logger.DeferredFieldResolver + getLoggerResolverMutex sync.RWMutex + getLoggerResolverArgsForCall []struct { + } + getLoggerResolverReturns struct { + result1 logger.DeferredFieldResolver + } + getLoggerResolverReturnsOnCall map[int]struct { + result1 logger.DeferredFieldResolver + } + GetNextSubscribedDataTrackHandleStub func() uint16 + getNextSubscribedDataTrackHandleMutex sync.RWMutex + getNextSubscribedDataTrackHandleArgsForCall []struct { + } + getNextSubscribedDataTrackHandleReturns struct { + result1 uint16 + } + getNextSubscribedDataTrackHandleReturnsOnCall map[int]struct { + result1 uint16 + } GetPacerStub func() pacer.Pacer getPacerMutex sync.RWMutex getPacerArgsForCall []struct { @@ -348,6 +406,16 @@ type FakeLocalParticipant struct { getPacerReturnsOnCall map[int]struct { result1 pacer.Pacer } + GetParticipantListenerStub func() types.ParticipantListener + getParticipantListenerMutex sync.RWMutex + getParticipantListenerArgsForCall []struct { + } + getParticipantListenerReturns struct { + result1 types.ParticipantListener + } + getParticipantListenerReturnsOnCall map[int]struct { + result1 types.ParticipantListener + } GetPendingTrackStub func(livekit.TrackID) *livekit.TrackInfo getPendingTrackMutex sync.RWMutex getPendingTrackArgsForCall []struct { @@ -369,6 +437,27 @@ type FakeLocalParticipant struct { getPlayoutDelayConfigReturnsOnCall map[int]struct { result1 *livekit.PlayoutDelay } + GetPublishedDataTrackStub func(uint16) types.DataTrack + getPublishedDataTrackMutex sync.RWMutex + getPublishedDataTrackArgsForCall []struct { + arg1 uint16 + } + getPublishedDataTrackReturns struct { + result1 types.DataTrack + } + getPublishedDataTrackReturnsOnCall map[int]struct { + result1 types.DataTrack + } + GetPublishedDataTracksStub func() []types.DataTrack + getPublishedDataTracksMutex sync.RWMutex + getPublishedDataTracksArgsForCall []struct { + } + getPublishedDataTracksReturns struct { + result1 []types.DataTrack + } + getPublishedDataTracksReturnsOnCall map[int]struct { + result1 []types.DataTrack + } GetPublishedTrackStub func(livekit.TrackID) types.MediaTrack getPublishedTrackMutex sync.RWMutex getPublishedTrackArgsForCall []struct { @@ -390,6 +479,48 @@ type FakeLocalParticipant struct { getPublishedTracksReturnsOnCall map[int]struct { result1 []types.MediaTrack } + GetPublisherICESessionUfragStub func() (string, error) + getPublisherICESessionUfragMutex sync.RWMutex + getPublisherICESessionUfragArgsForCall []struct { + } + getPublisherICESessionUfragReturns struct { + result1 string + result2 error + } + getPublisherICESessionUfragReturnsOnCall map[int]struct { + result1 string + result2 error + } + GetReporterStub func() roomobs.ParticipantSessionReporter + getReporterMutex sync.RWMutex + getReporterArgsForCall []struct { + } + getReporterReturns struct { + result1 roomobs.ParticipantSessionReporter + } + getReporterReturnsOnCall map[int]struct { + result1 roomobs.ParticipantSessionReporter + } + GetReporterResolverStub func() roomobs.ParticipantReporterResolver + getReporterResolverMutex sync.RWMutex + getReporterResolverArgsForCall []struct { + } + getReporterResolverReturns struct { + result1 roomobs.ParticipantReporterResolver + } + getReporterResolverReturnsOnCall map[int]struct { + result1 roomobs.ParticipantReporterResolver + } + GetResponseSinkStub func() routing.MessageSink + getResponseSinkMutex sync.RWMutex + getResponseSinkArgsForCall []struct { + } + getResponseSinkReturns struct { + result1 routing.MessageSink + } + getResponseSinkReturnsOnCall map[int]struct { + result1 routing.MessageSink + } GetSubscribedParticipantsStub func() []livekit.ParticipantID getSubscribedParticipantsMutex sync.RWMutex getSubscribedParticipantsArgsForCall []struct { @@ -410,6 +541,16 @@ type FakeLocalParticipant struct { getSubscribedTracksReturnsOnCall map[int]struct { result1 []types.SubscribedTrack } + GetTelemetryListenerStub func() types.ParticipantTelemetryListener + getTelemetryListenerMutex sync.RWMutex + getTelemetryListenerArgsForCall []struct { + } + getTelemetryListenerReturns struct { + result1 types.ParticipantTelemetryListener + } + getTelemetryListenerReturnsOnCall map[int]struct { + result1 types.ParticipantTelemetryListener + } GetTrailerStub func() []byte getTrailerMutex sync.RWMutex getTrailerArgsForCall []struct { @@ -420,10 +561,44 @@ type FakeLocalParticipant struct { getTrailerReturnsOnCall map[int]struct { result1 []byte } - HandleAnswerStub func(webrtc.SessionDescription) + HandleAnswerStub func(*livekit.SessionDescription) handleAnswerMutex sync.RWMutex handleAnswerArgsForCall []struct { - arg1 webrtc.SessionDescription + arg1 *livekit.SessionDescription + } + HandleICERestartSDPFragmentStub func(string) (string, error) + handleICERestartSDPFragmentMutex sync.RWMutex + handleICERestartSDPFragmentArgsForCall []struct { + arg1 string + } + handleICERestartSDPFragmentReturns struct { + result1 string + result2 error + } + handleICERestartSDPFragmentReturnsOnCall map[int]struct { + result1 string + result2 error + } + HandleICETrickleStub func(*livekit.TrickleRequest) + handleICETrickleMutex sync.RWMutex + handleICETrickleArgsForCall []struct { + arg1 *livekit.TrickleRequest + } + HandleICETrickleSDPFragmentStub func(string) error + handleICETrickleSDPFragmentMutex sync.RWMutex + handleICETrickleSDPFragmentArgsForCall []struct { + arg1 string + } + handleICETrickleSDPFragmentReturns struct { + result1 error + } + handleICETrickleSDPFragmentReturnsOnCall map[int]struct { + result1 error + } + HandleLeaveRequestStub func(types.ParticipantCloseReason) + handleLeaveRequestMutex sync.RWMutex + handleLeaveRequestArgsForCall []struct { + arg1 types.ParticipantCloseReason } HandleMetricsStub func(livekit.ParticipantID, *livekit.MetricsBatch) error handleMetricsMutex sync.RWMutex @@ -437,10 +612,10 @@ type FakeLocalParticipant struct { handleMetricsReturnsOnCall map[int]struct { result1 error } - HandleOfferStub func(webrtc.SessionDescription) error + HandleOfferStub func(*livekit.SessionDescription) error handleOfferMutex sync.RWMutex handleOfferArgsForCall []struct { - arg1 webrtc.SessionDescription + arg1 *livekit.SessionDescription } handleOfferReturns struct { result1 error @@ -448,6 +623,18 @@ type FakeLocalParticipant struct { handleOfferReturnsOnCall map[int]struct { result1 error } + HandlePublishDataTrackRequestStub func(*livekit.PublishDataTrackRequest) + handlePublishDataTrackRequestMutex sync.RWMutex + handlePublishDataTrackRequestArgsForCall []struct { + arg1 *livekit.PublishDataTrackRequest + } + HandleReceivedDataTrackMessageStub func([]byte, *datatrack.Packet, int64) + handleReceivedDataTrackMessageMutex sync.RWMutex + handleReceivedDataTrackMessageArgsForCall []struct { + arg1 []byte + arg2 *datatrack.Packet + arg3 int64 + } HandleReceiverReportStub func(*sfu.DownTrack, *rtcp.ReceiverReport) handleReceiverReportMutex sync.RWMutex handleReceiverReportArgsForCall []struct { @@ -466,10 +653,71 @@ type FakeLocalParticipant struct { handleReconnectAndSendResponseReturnsOnCall map[int]struct { result1 error } + HandleSignalMessageStub func(proto.Message) error + handleSignalMessageMutex sync.RWMutex + handleSignalMessageArgsForCall []struct { + arg1 proto.Message + } + handleSignalMessageReturns struct { + result1 error + } + handleSignalMessageReturnsOnCall map[int]struct { + result1 error + } HandleSignalSourceCloseStub func() handleSignalSourceCloseMutex sync.RWMutex handleSignalSourceCloseArgsForCall []struct { } + HandleSimulateScenarioStub func(*livekit.SimulateScenario) error + handleSimulateScenarioMutex sync.RWMutex + handleSimulateScenarioArgsForCall []struct { + arg1 *livekit.SimulateScenario + } + handleSimulateScenarioReturns struct { + result1 error + } + handleSimulateScenarioReturnsOnCall map[int]struct { + result1 error + } + HandleSyncStateStub func(*livekit.SyncState) error + handleSyncStateMutex sync.RWMutex + handleSyncStateArgsForCall []struct { + arg1 *livekit.SyncState + } + handleSyncStateReturns struct { + result1 error + } + handleSyncStateReturnsOnCall map[int]struct { + result1 error + } + HandleUnpublishDataTrackRequestStub func(*livekit.UnpublishDataTrackRequest) + handleUnpublishDataTrackRequestMutex sync.RWMutex + handleUnpublishDataTrackRequestArgsForCall []struct { + arg1 *livekit.UnpublishDataTrackRequest + } + HandleUpdateDataSubscriptionStub func(*livekit.UpdateDataSubscription) + handleUpdateDataSubscriptionMutex sync.RWMutex + handleUpdateDataSubscriptionArgsForCall []struct { + arg1 *livekit.UpdateDataSubscription + } + HandleUpdateSubscriptionPermissionStub func(*livekit.SubscriptionPermission) error + handleUpdateSubscriptionPermissionMutex sync.RWMutex + handleUpdateSubscriptionPermissionArgsForCall []struct { + arg1 *livekit.SubscriptionPermission + } + handleUpdateSubscriptionPermissionReturns struct { + result1 error + } + handleUpdateSubscriptionPermissionReturnsOnCall map[int]struct { + result1 error + } + HandleUpdateSubscriptionsStub func([]livekit.TrackID, []*livekit.ParticipantTracks, bool) + handleUpdateSubscriptionsMutex sync.RWMutex + handleUpdateSubscriptionsArgsForCall []struct { + arg1 []livekit.TrackID + arg2 []*livekit.ParticipantTracks + arg3 bool + } HasConnectedStub func() bool hasConnectedMutex sync.RWMutex hasConnectedArgsForCall []struct { @@ -640,6 +888,16 @@ type FakeLocalParticipant struct { isTrackNameSubscribedReturnsOnCall map[int]struct { result1 bool } + IsUsingSinglePeerConnectionStub func() bool + isUsingSinglePeerConnectionMutex sync.RWMutex + isUsingSinglePeerConnectionArgsForCall []struct { + } + isUsingSinglePeerConnectionReturns struct { + result1 bool + } + isUsingSinglePeerConnectionReturnsOnCall map[int]struct { + result1 bool + } IssueFullReconnectStub func(types.ParticipantCloseReason) issueFullReconnectMutex sync.RWMutex issueFullReconnectArgsForCall []struct { @@ -655,6 +913,16 @@ type FakeLocalParticipant struct { kindReturnsOnCall map[int]struct { result1 livekit.ParticipantInfo_Kind } + KindDetailsStub func() []livekit.ParticipantInfo_KindDetail + kindDetailsMutex sync.RWMutex + kindDetailsArgsForCall []struct { + } + kindDetailsReturns struct { + result1 []livekit.ParticipantInfo_KindDetail + } + kindDetailsReturnsOnCall map[int]struct { + result1 []livekit.ParticipantInfo_KindDetail + } MaybeStartMigrationStub func(bool, func()) bool maybeStartMigrationMutex sync.RWMutex maybeStartMigrationArgsForCall []struct { @@ -677,6 +945,11 @@ type FakeLocalParticipant struct { migrateStateReturnsOnCall map[int]struct { result1 types.MigrateState } + MoveToRoomStub func(types.MoveToRoomParams) + moveToRoomMutex sync.RWMutex + moveToRoomArgsForCall []struct { + arg1 types.MoveToRoomParams + } NegotiateStub func(bool) negotiateMutex sync.RWMutex negotiateArgsForCall []struct { @@ -691,60 +964,17 @@ type FakeLocalParticipant struct { onClaimsChangedArgsForCall []struct { arg1 func(types.LocalParticipant) } - OnCloseStub func(func(types.LocalParticipant)) - onCloseMutex sync.RWMutex - onCloseArgsForCall []struct { - arg1 func(types.LocalParticipant) - } - OnDataPacketStub func(func(types.LocalParticipant, livekit.DataPacket_Kind, *livekit.DataPacket)) - onDataPacketMutex sync.RWMutex - onDataPacketArgsForCall []struct { - arg1 func(types.LocalParticipant, livekit.DataPacket_Kind, *livekit.DataPacket) - } OnICEConfigChangedStub func(func(participant types.LocalParticipant, iceConfig *livekit.ICEConfig)) onICEConfigChangedMutex sync.RWMutex onICEConfigChangedArgsForCall []struct { arg1 func(participant types.LocalParticipant, iceConfig *livekit.ICEConfig) } - OnMetricsStub func(func(types.Participant, *livekit.DataPacket)) - onMetricsMutex sync.RWMutex - onMetricsArgsForCall []struct { - arg1 func(types.Participant, *livekit.DataPacket) - } - OnMigrateStateChangeStub func(func(p types.LocalParticipant, migrateState types.MigrateState)) - onMigrateStateChangeMutex sync.RWMutex - onMigrateStateChangeArgsForCall []struct { - arg1 func(p types.LocalParticipant, migrateState types.MigrateState) - } - OnParticipantUpdateStub func(func(types.LocalParticipant)) - onParticipantUpdateMutex sync.RWMutex - onParticipantUpdateArgsForCall []struct { - arg1 func(types.LocalParticipant) - } - OnStateChangeStub func(func(p types.LocalParticipant, state livekit.ParticipantInfo_State)) - onStateChangeMutex sync.RWMutex - onStateChangeArgsForCall []struct { - arg1 func(p types.LocalParticipant, state livekit.ParticipantInfo_State) - } - OnSubscribeStatusChangedStub func(func(publisherID livekit.ParticipantID, subscribed bool)) - onSubscribeStatusChangedMutex sync.RWMutex - onSubscribeStatusChangedArgsForCall []struct { - arg1 func(publisherID livekit.ParticipantID, subscribed bool) - } - OnTrackPublishedStub func(func(types.LocalParticipant, types.MediaTrack)) - onTrackPublishedMutex sync.RWMutex - onTrackPublishedArgsForCall []struct { - arg1 func(types.LocalParticipant, types.MediaTrack) - } - OnTrackUnpublishedStub func(func(types.LocalParticipant, types.MediaTrack)) - onTrackUnpublishedMutex sync.RWMutex - onTrackUnpublishedArgsForCall []struct { - arg1 func(types.LocalParticipant, types.MediaTrack) - } - OnTrackUpdatedStub func(func(types.LocalParticipant, types.MediaTrack)) - onTrackUpdatedMutex sync.RWMutex - onTrackUpdatedArgsForCall []struct { - arg1 func(types.LocalParticipant, types.MediaTrack) + PerformRpcStub func(*livekit.PerformRpcRequest, chan string, chan error) + performRpcMutex sync.RWMutex + performRpcArgsForCall []struct { + arg1 *livekit.PerformRpcRequest + arg2 chan string + arg3 chan error } ProtocolVersionStub func() types.ProtocolVersion protocolVersionMutex sync.RWMutex @@ -756,12 +986,16 @@ type FakeLocalParticipant struct { protocolVersionReturnsOnCall map[int]struct { result1 types.ProtocolVersion } - RemovePublishedTrackStub func(types.MediaTrack, bool, bool) + RemovePublishedDataTrackStub func(types.DataTrack) + removePublishedDataTrackMutex sync.RWMutex + removePublishedDataTrackArgsForCall []struct { + arg1 types.DataTrack + } + RemovePublishedTrackStub func(types.MediaTrack, bool) removePublishedTrackMutex sync.RWMutex removePublishedTrackArgsForCall []struct { arg1 types.MediaTrack arg2 bool - arg3 bool } RemoveTrackLocalStub func(*webrtc.RTPSender) error removeTrackLocalMutex sync.RWMutex @@ -785,16 +1019,42 @@ type FakeLocalParticipant struct { sendConnectionQualityUpdateReturnsOnCall map[int]struct { result1 error } - SendDataPacketStub func(livekit.DataPacket_Kind, []byte) error - sendDataPacketMutex sync.RWMutex - sendDataPacketArgsForCall []struct { + SendDataMessageStub func(livekit.DataPacket_Kind, []byte, livekit.ParticipantID, uint32) error + sendDataMessageMutex sync.RWMutex + sendDataMessageArgsForCall []struct { arg1 livekit.DataPacket_Kind arg2 []byte + arg3 livekit.ParticipantID + arg4 uint32 } - sendDataPacketReturns struct { + sendDataMessageReturns struct { result1 error } - sendDataPacketReturnsOnCall map[int]struct { + sendDataMessageReturnsOnCall map[int]struct { + result1 error + } + SendDataMessageUnlabeledStub func([]byte, bool, livekit.ParticipantIdentity) error + sendDataMessageUnlabeledMutex sync.RWMutex + sendDataMessageUnlabeledArgsForCall []struct { + arg1 []byte + arg2 bool + arg3 livekit.ParticipantIdentity + } + sendDataMessageUnlabeledReturns struct { + result1 error + } + sendDataMessageUnlabeledReturnsOnCall map[int]struct { + result1 error + } + SendDataTrackSubscriberHandlesStub func(map[uint32]*livekit.DataTrackSubscriberHandles_PublishedDataTrack) error + sendDataTrackSubscriberHandlesMutex sync.RWMutex + sendDataTrackSubscriberHandlesArgsForCall []struct { + arg1 map[uint32]*livekit.DataTrackSubscriberHandles_PublishedDataTrack + } + sendDataTrackSubscriberHandlesReturns struct { + result1 error + } + sendDataTrackSubscriberHandlesReturnsOnCall map[int]struct { result1 error } SendJoinResponseStub func(*livekit.JoinResponse) error @@ -830,15 +1090,15 @@ type FakeLocalParticipant struct { sendRefreshTokenReturnsOnCall map[int]struct { result1 error } - SendRequestResponseStub func(*livekit.RequestResponse) error - sendRequestResponseMutex sync.RWMutex - sendRequestResponseArgsForCall []struct { - arg1 *livekit.RequestResponse + SendRoomMovedResponseStub func(*livekit.RoomMovedResponse) error + sendRoomMovedResponseMutex sync.RWMutex + sendRoomMovedResponseArgsForCall []struct { + arg1 *livekit.RoomMovedResponse } - sendRequestResponseReturns struct { + sendRoomMovedResponseReturns struct { result1 error } - sendRequestResponseReturnsOnCall map[int]struct { + sendRoomMovedResponseReturnsOnCall map[int]struct { result1 error } SendRoomUpdateStub func(*livekit.Room) error @@ -864,6 +1124,19 @@ type FakeLocalParticipant struct { sendSpeakerUpdateReturnsOnCall map[int]struct { result1 error } + SendSubscriptionPermissionUpdateStub func(livekit.ParticipantID, livekit.TrackID, bool) error + sendSubscriptionPermissionUpdateMutex sync.RWMutex + sendSubscriptionPermissionUpdateArgsForCall []struct { + arg1 livekit.ParticipantID + arg2 livekit.TrackID + arg3 bool + } + sendSubscriptionPermissionUpdateReturns struct { + result1 error + } + sendSubscriptionPermissionUpdateReturnsOnCall map[int]struct { + result1 error + } SetAttributesStub func(map[string]string) setAttributesMutex sync.RWMutex setAttributesArgsForCall []struct { @@ -879,13 +1152,15 @@ type FakeLocalParticipant struct { setMetadataArgsForCall []struct { arg1 string } - SetMigrateInfoStub func(*webrtc.SessionDescription, *webrtc.SessionDescription, []*livekit.TrackPublishedResponse, []*livekit.DataChannelInfo) + SetMigrateInfoStub func(*webrtc.SessionDescription, *webrtc.SessionDescription, []*livekit.TrackPublishedResponse, []*livekit.DataChannelInfo, []*livekit.DataChannelReceiveState, []*livekit.PublishDataTrackResponse) setMigrateInfoMutex sync.RWMutex setMigrateInfoArgsForCall []struct { arg1 *webrtc.SessionDescription arg2 *webrtc.SessionDescription arg3 []*livekit.TrackPublishedResponse arg4 []*livekit.DataChannelInfo + arg5 []*livekit.DataChannelReceiveState + arg6 []*livekit.PublishDataTrackResponse } SetMigrateStateStub func(types.MigrateState) setMigrateStateMutex sync.RWMutex @@ -908,11 +1183,6 @@ type FakeLocalParticipant struct { setPermissionReturnsOnCall map[int]struct { result1 bool } - SetResponseSinkStub func(routing.MessageSink) - setResponseSinkMutex sync.RWMutex - setResponseSinkArgsForCall []struct { - arg1 routing.MessageSink - } SetSignalSourceValidStub func(bool) setSignalSourceValidMutex sync.RWMutex setSignalSourceValidArgsForCall []struct { @@ -928,12 +1198,11 @@ type FakeLocalParticipant struct { setSubscriberChannelCapacityArgsForCall []struct { arg1 int64 } - SetTrackMutedStub func(livekit.TrackID, bool, bool) *livekit.TrackInfo + SetTrackMutedStub func(*livekit.MuteTrackRequest, bool) *livekit.TrackInfo setTrackMutedMutex sync.RWMutex setTrackMutedArgsForCall []struct { - arg1 livekit.TrackID + arg1 *livekit.MuteTrackRequest arg2 bool - arg3 bool } setTrackMutedReturns struct { result1 *livekit.TrackInfo @@ -961,10 +1230,16 @@ type FakeLocalParticipant struct { stopAndGetSubscribedTracksForwarderStateReturnsOnCall map[int]struct { result1 map[livekit.TrackID]*livekit.RTPForwarderState } - SubscribeToTrackStub func(livekit.TrackID) + SubscribeToDataTrackStub func(livekit.TrackID) + subscribeToDataTrackMutex sync.RWMutex + subscribeToDataTrackArgsForCall []struct { + arg1 livekit.TrackID + } + SubscribeToTrackStub func(livekit.TrackID, bool) subscribeToTrackMutex sync.RWMutex subscribeToTrackArgsForCall []struct { arg1 livekit.TrackID + arg2 bool } SubscriberAsPrimaryStub func() bool subscriberAsPrimaryMutex sync.RWMutex @@ -988,13 +1263,6 @@ type FakeLocalParticipant struct { result1 *livekit.SubscriptionPermission result2 utils.TimedVersion } - SubscriptionPermissionUpdateStub func(livekit.ParticipantID, livekit.TrackID, bool) - subscriptionPermissionUpdateMutex sync.RWMutex - subscriptionPermissionUpdateArgsForCall []struct { - arg1 livekit.ParticipantID - arg2 livekit.TrackID - arg3 bool - } SupportsCodecChangeStub func() bool supportsCodecChangeMutex sync.RWMutex supportsCodecChangeArgsForCall []struct { @@ -1005,6 +1273,16 @@ type FakeLocalParticipant struct { supportsCodecChangeReturnsOnCall map[int]struct { result1 bool } + SupportsMovingStub func() error + supportsMovingMutex sync.RWMutex + supportsMovingArgsForCall []struct { + } + supportsMovingReturns struct { + result1 error + } + supportsMovingReturnsOnCall map[int]struct { + result1 error + } SupportsSyncStreamIDStub func() bool supportsSyncStreamIDMutex sync.RWMutex supportsSyncStreamIDArgsForCall []struct { @@ -1015,9 +1293,10 @@ type FakeLocalParticipant struct { supportsSyncStreamIDReturnsOnCall map[int]struct { result1 bool } - SupportsTransceiverReuseStub func() bool + SupportsTransceiverReuseStub func(types.MediaTrack) bool supportsTransceiverReuseMutex sync.RWMutex supportsTransceiverReuseArgsForCall []struct { + arg1 types.MediaTrack } supportsTransceiverReuseReturns struct { result1 bool @@ -1025,6 +1304,22 @@ type FakeLocalParticipant struct { supportsTransceiverReuseReturnsOnCall map[int]struct { result1 bool } + SwapResponseSinkStub func(routing.MessageSink, types.SignallingCloseReason) + swapResponseSinkMutex sync.RWMutex + swapResponseSinkArgsForCall []struct { + arg1 routing.MessageSink + arg2 types.SignallingCloseReason + } + TelemetryGuardStub func() *telemetry.ReferenceGuard + telemetryGuardMutex sync.RWMutex + telemetryGuardArgsForCall []struct { + } + telemetryGuardReturns struct { + result1 *telemetry.ReferenceGuard + } + telemetryGuardReturnsOnCall map[int]struct { + result1 *telemetry.ReferenceGuard + } ToProtoStub func() *livekit.ParticipantInfo toProtoMutex sync.RWMutex toProtoArgsForCall []struct { @@ -1052,6 +1347,11 @@ type FakeLocalParticipant struct { uncacheDownTrackArgsForCall []struct { arg1 *webrtc.RTPTransceiver } + UnsubscribeFromDataTrackStub func(livekit.TrackID) + unsubscribeFromDataTrackMutex sync.RWMutex + unsubscribeFromDataTrackArgsForCall []struct { + arg1 livekit.TrackID + } UnsubscribeFromTrackStub func(livekit.TrackID) unsubscribeFromTrackMutex sync.RWMutex unsubscribeFromTrackArgsForCall []struct { @@ -1068,6 +1368,12 @@ type FakeLocalParticipant struct { updateAudioTrackReturnsOnCall map[int]struct { result1 error } + UpdateDataTrackSubscriptionOptionsStub func(livekit.TrackID, *livekit.DataTrackSubscriptionOptions) + updateDataTrackSubscriptionOptionsMutex sync.RWMutex + updateDataTrackSubscriptionOptionsArgsForCall []struct { + arg1 livekit.TrackID + arg2 *livekit.DataTrackSubscriptionOptions + } UpdateLastSeenSignalStub func() updateLastSeenSignalMutex sync.RWMutex updateLastSeenSignalArgsForCall []struct { @@ -1090,11 +1396,36 @@ type FakeLocalParticipant struct { updateMediaRTTArgsForCall []struct { arg1 uint32 } + UpdateMetadataStub func(*livekit.UpdateParticipantMetadata, bool) error + updateMetadataMutex sync.RWMutex + updateMetadataArgsForCall []struct { + arg1 *livekit.UpdateParticipantMetadata + arg2 bool + } + updateMetadataReturns struct { + result1 error + } + updateMetadataReturnsOnCall map[int]struct { + result1 error + } UpdateSignalingRTTStub func(uint32) updateSignalingRTTMutex sync.RWMutex updateSignalingRTTArgsForCall []struct { arg1 uint32 } + UpdateSubscribedAudioCodecsStub func(livekit.NodeID, livekit.TrackID, []*livekit.SubscribedAudioCodec) error + updateSubscribedAudioCodecsMutex sync.RWMutex + updateSubscribedAudioCodecsArgsForCall []struct { + arg1 livekit.NodeID + arg2 livekit.TrackID + arg3 []*livekit.SubscribedAudioCodec + } + updateSubscribedAudioCodecsReturns struct { + result1 error + } + updateSubscribedAudioCodecsReturnsOnCall map[int]struct { + result1 error + } UpdateSubscribedQualityStub func(livekit.NodeID, livekit.TrackID, []types.SubscribedCodecQuality) error updateSubscribedQualityMutex sync.RWMutex updateSubscribedQualityArgsForCall []struct { @@ -1190,36 +1521,89 @@ type FakeLocalParticipant struct { invocationsMutex sync.RWMutex } -func (fake *FakeLocalParticipant) AddICECandidate(arg1 webrtc.ICECandidateInit, arg2 livekit.SignalTarget) { - fake.addICECandidateMutex.Lock() - fake.addICECandidateArgsForCall = append(fake.addICECandidateArgsForCall, struct { - arg1 webrtc.ICECandidateInit - arg2 livekit.SignalTarget - }{arg1, arg2}) - stub := fake.AddICECandidateStub - fake.recordInvocation("AddICECandidate", []interface{}{arg1, arg2}) - fake.addICECandidateMutex.Unlock() +func (fake *FakeLocalParticipant) ActiveAt() time.Time { + fake.activeAtMutex.Lock() + ret, specificReturn := fake.activeAtReturnsOnCall[len(fake.activeAtArgsForCall)] + fake.activeAtArgsForCall = append(fake.activeAtArgsForCall, struct { + }{}) + stub := fake.ActiveAtStub + fakeReturns := fake.activeAtReturns + fake.recordInvocation("ActiveAt", []interface{}{}) + fake.activeAtMutex.Unlock() if stub != nil { - fake.AddICECandidateStub(arg1, arg2) + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeLocalParticipant) ActiveAtCallCount() int { + fake.activeAtMutex.RLock() + defer fake.activeAtMutex.RUnlock() + return len(fake.activeAtArgsForCall) +} + +func (fake *FakeLocalParticipant) ActiveAtCalls(stub func() time.Time) { + fake.activeAtMutex.Lock() + defer fake.activeAtMutex.Unlock() + fake.ActiveAtStub = stub +} + +func (fake *FakeLocalParticipant) ActiveAtReturns(result1 time.Time) { + fake.activeAtMutex.Lock() + defer fake.activeAtMutex.Unlock() + fake.ActiveAtStub = nil + fake.activeAtReturns = struct { + result1 time.Time + }{result1} +} + +func (fake *FakeLocalParticipant) ActiveAtReturnsOnCall(i int, result1 time.Time) { + fake.activeAtMutex.Lock() + defer fake.activeAtMutex.Unlock() + fake.ActiveAtStub = nil + if fake.activeAtReturnsOnCall == nil { + fake.activeAtReturnsOnCall = make(map[int]struct { + result1 time.Time + }) + } + fake.activeAtReturnsOnCall[i] = struct { + result1 time.Time + }{result1} +} + +func (fake *FakeLocalParticipant) AddOnClose(arg1 string, arg2 func(types.LocalParticipant)) { + fake.addOnCloseMutex.Lock() + fake.addOnCloseArgsForCall = append(fake.addOnCloseArgsForCall, struct { + arg1 string + arg2 func(types.LocalParticipant) + }{arg1, arg2}) + stub := fake.AddOnCloseStub + fake.recordInvocation("AddOnClose", []interface{}{arg1, arg2}) + fake.addOnCloseMutex.Unlock() + if stub != nil { + fake.AddOnCloseStub(arg1, arg2) } } -func (fake *FakeLocalParticipant) AddICECandidateCallCount() int { - fake.addICECandidateMutex.RLock() - defer fake.addICECandidateMutex.RUnlock() - return len(fake.addICECandidateArgsForCall) +func (fake *FakeLocalParticipant) AddOnCloseCallCount() int { + fake.addOnCloseMutex.RLock() + defer fake.addOnCloseMutex.RUnlock() + return len(fake.addOnCloseArgsForCall) } -func (fake *FakeLocalParticipant) AddICECandidateCalls(stub func(webrtc.ICECandidateInit, livekit.SignalTarget)) { - fake.addICECandidateMutex.Lock() - defer fake.addICECandidateMutex.Unlock() - fake.AddICECandidateStub = stub +func (fake *FakeLocalParticipant) AddOnCloseCalls(stub func(string, func(types.LocalParticipant))) { + fake.addOnCloseMutex.Lock() + defer fake.addOnCloseMutex.Unlock() + fake.AddOnCloseStub = stub } -func (fake *FakeLocalParticipant) AddICECandidateArgsForCall(i int) (webrtc.ICECandidateInit, livekit.SignalTarget) { - fake.addICECandidateMutex.RLock() - defer fake.addICECandidateMutex.RUnlock() - argsForCall := fake.addICECandidateArgsForCall[i] +func (fake *FakeLocalParticipant) AddOnCloseArgsForCall(i int) (string, func(types.LocalParticipant)) { + fake.addOnCloseMutex.RLock() + defer fake.addOnCloseMutex.RUnlock() + argsForCall := fake.addOnCloseArgsForCall[i] return argsForCall.arg1, argsForCall.arg2 } @@ -1698,69 +2082,6 @@ func (fake *FakeLocalParticipant) CanSubscribeReturnsOnCall(i int, result1 bool) }{result1} } -func (fake *FakeLocalParticipant) CheckMetadataLimits(arg1 string, arg2 string, arg3 map[string]string) error { - fake.checkMetadataLimitsMutex.Lock() - ret, specificReturn := fake.checkMetadataLimitsReturnsOnCall[len(fake.checkMetadataLimitsArgsForCall)] - fake.checkMetadataLimitsArgsForCall = append(fake.checkMetadataLimitsArgsForCall, struct { - arg1 string - arg2 string - arg3 map[string]string - }{arg1, arg2, arg3}) - stub := fake.CheckMetadataLimitsStub - fakeReturns := fake.checkMetadataLimitsReturns - fake.recordInvocation("CheckMetadataLimits", []interface{}{arg1, arg2, arg3}) - fake.checkMetadataLimitsMutex.Unlock() - if stub != nil { - return stub(arg1, arg2, arg3) - } - if specificReturn { - return ret.result1 - } - return fakeReturns.result1 -} - -func (fake *FakeLocalParticipant) CheckMetadataLimitsCallCount() int { - fake.checkMetadataLimitsMutex.RLock() - defer fake.checkMetadataLimitsMutex.RUnlock() - return len(fake.checkMetadataLimitsArgsForCall) -} - -func (fake *FakeLocalParticipant) CheckMetadataLimitsCalls(stub func(string, string, map[string]string) error) { - fake.checkMetadataLimitsMutex.Lock() - defer fake.checkMetadataLimitsMutex.Unlock() - fake.CheckMetadataLimitsStub = stub -} - -func (fake *FakeLocalParticipant) CheckMetadataLimitsArgsForCall(i int) (string, string, map[string]string) { - fake.checkMetadataLimitsMutex.RLock() - defer fake.checkMetadataLimitsMutex.RUnlock() - argsForCall := fake.checkMetadataLimitsArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 -} - -func (fake *FakeLocalParticipant) CheckMetadataLimitsReturns(result1 error) { - fake.checkMetadataLimitsMutex.Lock() - defer fake.checkMetadataLimitsMutex.Unlock() - fake.CheckMetadataLimitsStub = nil - fake.checkMetadataLimitsReturns = struct { - result1 error - }{result1} -} - -func (fake *FakeLocalParticipant) CheckMetadataLimitsReturnsOnCall(i int, result1 error) { - fake.checkMetadataLimitsMutex.Lock() - defer fake.checkMetadataLimitsMutex.Unlock() - fake.CheckMetadataLimitsStub = nil - if fake.checkMetadataLimitsReturnsOnCall == nil { - fake.checkMetadataLimitsReturnsOnCall = make(map[int]struct { - result1 error - }) - } - fake.checkMetadataLimitsReturnsOnCall[i] = struct { - result1 error - }{result1} -} - func (fake *FakeLocalParticipant) ClaimGrants() *auth.ClaimGrants { fake.claimGrantsMutex.Lock() ret, specificReturn := fake.claimGrantsReturnsOnCall[len(fake.claimGrantsArgsForCall)] @@ -1814,6 +2135,30 @@ func (fake *FakeLocalParticipant) ClaimGrantsReturnsOnCall(i int, result1 *auth. }{result1} } +func (fake *FakeLocalParticipant) ClearParticipantListener() { + fake.clearParticipantListenerMutex.Lock() + fake.clearParticipantListenerArgsForCall = append(fake.clearParticipantListenerArgsForCall, struct { + }{}) + stub := fake.ClearParticipantListenerStub + fake.recordInvocation("ClearParticipantListener", []interface{}{}) + fake.clearParticipantListenerMutex.Unlock() + if stub != nil { + fake.ClearParticipantListenerStub() + } +} + +func (fake *FakeLocalParticipant) ClearParticipantListenerCallCount() int { + fake.clearParticipantListenerMutex.RLock() + defer fake.clearParticipantListenerMutex.RUnlock() + return len(fake.clearParticipantListenerArgsForCall) +} + +func (fake *FakeLocalParticipant) ClearParticipantListenerCalls(stub func()) { + fake.clearParticipantListenerMutex.Lock() + defer fake.clearParticipantListenerMutex.Unlock() + fake.ClearParticipantListenerStub = stub +} + func (fake *FakeLocalParticipant) Close(arg1 bool, arg2 types.ParticipantCloseReason, arg3 bool) error { fake.closeMutex.Lock() ret, specificReturn := fake.closeReturnsOnCall[len(fake.closeArgsForCall)] @@ -2015,7 +2360,7 @@ func (fake *FakeLocalParticipant) ConnectedAtReturnsOnCall(i int, result1 time.T }{result1} } -func (fake *FakeLocalParticipant) DebugInfo() map[string]interface{} { +func (fake *FakeLocalParticipant) DebugInfo() map[string]any { fake.debugInfoMutex.Lock() ret, specificReturn := fake.debugInfoReturnsOnCall[len(fake.debugInfoArgsForCall)] fake.debugInfoArgsForCall = append(fake.debugInfoArgsForCall, struct { @@ -2039,32 +2384,32 @@ func (fake *FakeLocalParticipant) DebugInfoCallCount() int { return len(fake.debugInfoArgsForCall) } -func (fake *FakeLocalParticipant) DebugInfoCalls(stub func() map[string]interface{}) { +func (fake *FakeLocalParticipant) DebugInfoCalls(stub func() map[string]any) { fake.debugInfoMutex.Lock() defer fake.debugInfoMutex.Unlock() fake.DebugInfoStub = stub } -func (fake *FakeLocalParticipant) DebugInfoReturns(result1 map[string]interface{}) { +func (fake *FakeLocalParticipant) DebugInfoReturns(result1 map[string]any) { fake.debugInfoMutex.Lock() defer fake.debugInfoMutex.Unlock() fake.DebugInfoStub = nil fake.debugInfoReturns = struct { - result1 map[string]interface{} + result1 map[string]any }{result1} } -func (fake *FakeLocalParticipant) DebugInfoReturnsOnCall(i int, result1 map[string]interface{}) { +func (fake *FakeLocalParticipant) DebugInfoReturnsOnCall(i int, result1 map[string]any) { fake.debugInfoMutex.Lock() defer fake.debugInfoMutex.Unlock() fake.DebugInfoStub = nil if fake.debugInfoReturnsOnCall == nil { fake.debugInfoReturnsOnCall = make(map[int]struct { - result1 map[string]interface{} + result1 map[string]any }) } fake.debugInfoReturnsOnCall[i] = struct { - result1 map[string]interface{} + result1 map[string]any }{result1} } @@ -2174,7 +2519,7 @@ func (fake *FakeLocalParticipant) GetAdaptiveStreamReturnsOnCall(i int, result1 }{result1} } -func (fake *FakeLocalParticipant) GetAnswer() (webrtc.SessionDescription, error) { +func (fake *FakeLocalParticipant) GetAnswer() (webrtc.SessionDescription, uint32, error) { fake.getAnswerMutex.Lock() ret, specificReturn := fake.getAnswerReturnsOnCall[len(fake.getAnswerArgsForCall)] fake.getAnswerArgsForCall = append(fake.getAnswerArgsForCall, struct { @@ -2187,9 +2532,9 @@ func (fake *FakeLocalParticipant) GetAnswer() (webrtc.SessionDescription, error) return stub() } if specificReturn { - return ret.result1, ret.result2 + return ret.result1, ret.result2, ret.result3 } - return fakeReturns.result1, fakeReturns.result2 + return fakeReturns.result1, fakeReturns.result2, fakeReturns.result3 } func (fake *FakeLocalParticipant) GetAnswerCallCount() int { @@ -2198,36 +2543,39 @@ func (fake *FakeLocalParticipant) GetAnswerCallCount() int { return len(fake.getAnswerArgsForCall) } -func (fake *FakeLocalParticipant) GetAnswerCalls(stub func() (webrtc.SessionDescription, error)) { +func (fake *FakeLocalParticipant) GetAnswerCalls(stub func() (webrtc.SessionDescription, uint32, error)) { fake.getAnswerMutex.Lock() defer fake.getAnswerMutex.Unlock() fake.GetAnswerStub = stub } -func (fake *FakeLocalParticipant) GetAnswerReturns(result1 webrtc.SessionDescription, result2 error) { +func (fake *FakeLocalParticipant) GetAnswerReturns(result1 webrtc.SessionDescription, result2 uint32, result3 error) { fake.getAnswerMutex.Lock() defer fake.getAnswerMutex.Unlock() fake.GetAnswerStub = nil fake.getAnswerReturns = struct { result1 webrtc.SessionDescription - result2 error - }{result1, result2} + result2 uint32 + result3 error + }{result1, result2, result3} } -func (fake *FakeLocalParticipant) GetAnswerReturnsOnCall(i int, result1 webrtc.SessionDescription, result2 error) { +func (fake *FakeLocalParticipant) GetAnswerReturnsOnCall(i int, result1 webrtc.SessionDescription, result2 uint32, result3 error) { fake.getAnswerMutex.Lock() defer fake.getAnswerMutex.Unlock() fake.GetAnswerStub = nil if fake.getAnswerReturnsOnCall == nil { fake.getAnswerReturnsOnCall = make(map[int]struct { result1 webrtc.SessionDescription - result2 error + result2 uint32 + result3 error }) } fake.getAnswerReturnsOnCall[i] = struct { result1 webrtc.SessionDescription - result2 error - }{result1, result2} + result2 uint32 + result3 error + }{result1, result2, result3} } func (fake *FakeLocalParticipant) GetAudioLevel() (float64, bool) { @@ -2562,6 +2910,112 @@ func (fake *FakeLocalParticipant) GetConnectionQualityReturnsOnCall(i int, resul }{result1} } +func (fake *FakeLocalParticipant) GetCountry() string { + fake.getCountryMutex.Lock() + ret, specificReturn := fake.getCountryReturnsOnCall[len(fake.getCountryArgsForCall)] + fake.getCountryArgsForCall = append(fake.getCountryArgsForCall, struct { + }{}) + stub := fake.GetCountryStub + fakeReturns := fake.getCountryReturns + fake.recordInvocation("GetCountry", []interface{}{}) + fake.getCountryMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeLocalParticipant) GetCountryCallCount() int { + fake.getCountryMutex.RLock() + defer fake.getCountryMutex.RUnlock() + return len(fake.getCountryArgsForCall) +} + +func (fake *FakeLocalParticipant) GetCountryCalls(stub func() string) { + fake.getCountryMutex.Lock() + defer fake.getCountryMutex.Unlock() + fake.GetCountryStub = stub +} + +func (fake *FakeLocalParticipant) GetCountryReturns(result1 string) { + fake.getCountryMutex.Lock() + defer fake.getCountryMutex.Unlock() + fake.GetCountryStub = nil + fake.getCountryReturns = struct { + result1 string + }{result1} +} + +func (fake *FakeLocalParticipant) GetCountryReturnsOnCall(i int, result1 string) { + fake.getCountryMutex.Lock() + defer fake.getCountryMutex.Unlock() + fake.GetCountryStub = nil + if fake.getCountryReturnsOnCall == nil { + fake.getCountryReturnsOnCall = make(map[int]struct { + result1 string + }) + } + fake.getCountryReturnsOnCall[i] = struct { + result1 string + }{result1} +} + +func (fake *FakeLocalParticipant) GetDataTrackTransport() types.DataTrackTransport { + fake.getDataTrackTransportMutex.Lock() + ret, specificReturn := fake.getDataTrackTransportReturnsOnCall[len(fake.getDataTrackTransportArgsForCall)] + fake.getDataTrackTransportArgsForCall = append(fake.getDataTrackTransportArgsForCall, struct { + }{}) + stub := fake.GetDataTrackTransportStub + fakeReturns := fake.getDataTrackTransportReturns + fake.recordInvocation("GetDataTrackTransport", []interface{}{}) + fake.getDataTrackTransportMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeLocalParticipant) GetDataTrackTransportCallCount() int { + fake.getDataTrackTransportMutex.RLock() + defer fake.getDataTrackTransportMutex.RUnlock() + return len(fake.getDataTrackTransportArgsForCall) +} + +func (fake *FakeLocalParticipant) GetDataTrackTransportCalls(stub func() types.DataTrackTransport) { + fake.getDataTrackTransportMutex.Lock() + defer fake.getDataTrackTransportMutex.Unlock() + fake.GetDataTrackTransportStub = stub +} + +func (fake *FakeLocalParticipant) GetDataTrackTransportReturns(result1 types.DataTrackTransport) { + fake.getDataTrackTransportMutex.Lock() + defer fake.getDataTrackTransportMutex.Unlock() + fake.GetDataTrackTransportStub = nil + fake.getDataTrackTransportReturns = struct { + result1 types.DataTrackTransport + }{result1} +} + +func (fake *FakeLocalParticipant) GetDataTrackTransportReturnsOnCall(i int, result1 types.DataTrackTransport) { + fake.getDataTrackTransportMutex.Lock() + defer fake.getDataTrackTransportMutex.Unlock() + fake.GetDataTrackTransportStub = nil + if fake.getDataTrackTransportReturnsOnCall == nil { + fake.getDataTrackTransportReturnsOnCall = make(map[int]struct { + result1 types.DataTrackTransport + }) + } + fake.getDataTrackTransportReturnsOnCall[i] = struct { + result1 types.DataTrackTransport + }{result1} +} + func (fake *FakeLocalParticipant) GetDisableSenderReportPassThrough() bool { fake.getDisableSenderReportPassThroughMutex.Lock() ret, specificReturn := fake.getDisableSenderReportPassThroughReturnsOnCall[len(fake.getDisableSenderReportPassThroughArgsForCall)] @@ -2774,6 +3228,67 @@ func (fake *FakeLocalParticipant) GetICEConnectionInfoReturnsOnCall(i int, resul }{result1} } +func (fake *FakeLocalParticipant) GetLastReliableSequence(arg1 bool) uint32 { + fake.getLastReliableSequenceMutex.Lock() + ret, specificReturn := fake.getLastReliableSequenceReturnsOnCall[len(fake.getLastReliableSequenceArgsForCall)] + fake.getLastReliableSequenceArgsForCall = append(fake.getLastReliableSequenceArgsForCall, struct { + arg1 bool + }{arg1}) + stub := fake.GetLastReliableSequenceStub + fakeReturns := fake.getLastReliableSequenceReturns + fake.recordInvocation("GetLastReliableSequence", []interface{}{arg1}) + fake.getLastReliableSequenceMutex.Unlock() + if stub != nil { + return stub(arg1) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeLocalParticipant) GetLastReliableSequenceCallCount() int { + fake.getLastReliableSequenceMutex.RLock() + defer fake.getLastReliableSequenceMutex.RUnlock() + return len(fake.getLastReliableSequenceArgsForCall) +} + +func (fake *FakeLocalParticipant) GetLastReliableSequenceCalls(stub func(bool) uint32) { + fake.getLastReliableSequenceMutex.Lock() + defer fake.getLastReliableSequenceMutex.Unlock() + fake.GetLastReliableSequenceStub = stub +} + +func (fake *FakeLocalParticipant) GetLastReliableSequenceArgsForCall(i int) bool { + fake.getLastReliableSequenceMutex.RLock() + defer fake.getLastReliableSequenceMutex.RUnlock() + argsForCall := fake.getLastReliableSequenceArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeLocalParticipant) GetLastReliableSequenceReturns(result1 uint32) { + fake.getLastReliableSequenceMutex.Lock() + defer fake.getLastReliableSequenceMutex.Unlock() + fake.GetLastReliableSequenceStub = nil + fake.getLastReliableSequenceReturns = struct { + result1 uint32 + }{result1} +} + +func (fake *FakeLocalParticipant) GetLastReliableSequenceReturnsOnCall(i int, result1 uint32) { + fake.getLastReliableSequenceMutex.Lock() + defer fake.getLastReliableSequenceMutex.Unlock() + fake.GetLastReliableSequenceStub = nil + if fake.getLastReliableSequenceReturnsOnCall == nil { + fake.getLastReliableSequenceReturnsOnCall = make(map[int]struct { + result1 uint32 + }) + } + fake.getLastReliableSequenceReturnsOnCall[i] = struct { + result1 uint32 + }{result1} +} + func (fake *FakeLocalParticipant) GetLogger() logger.Logger { fake.getLoggerMutex.Lock() ret, specificReturn := fake.getLoggerReturnsOnCall[len(fake.getLoggerArgsForCall)] @@ -2827,6 +3342,112 @@ func (fake *FakeLocalParticipant) GetLoggerReturnsOnCall(i int, result1 logger.L }{result1} } +func (fake *FakeLocalParticipant) GetLoggerResolver() logger.DeferredFieldResolver { + fake.getLoggerResolverMutex.Lock() + ret, specificReturn := fake.getLoggerResolverReturnsOnCall[len(fake.getLoggerResolverArgsForCall)] + fake.getLoggerResolverArgsForCall = append(fake.getLoggerResolverArgsForCall, struct { + }{}) + stub := fake.GetLoggerResolverStub + fakeReturns := fake.getLoggerResolverReturns + fake.recordInvocation("GetLoggerResolver", []interface{}{}) + fake.getLoggerResolverMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeLocalParticipant) GetLoggerResolverCallCount() int { + fake.getLoggerResolverMutex.RLock() + defer fake.getLoggerResolverMutex.RUnlock() + return len(fake.getLoggerResolverArgsForCall) +} + +func (fake *FakeLocalParticipant) GetLoggerResolverCalls(stub func() logger.DeferredFieldResolver) { + fake.getLoggerResolverMutex.Lock() + defer fake.getLoggerResolverMutex.Unlock() + fake.GetLoggerResolverStub = stub +} + +func (fake *FakeLocalParticipant) GetLoggerResolverReturns(result1 logger.DeferredFieldResolver) { + fake.getLoggerResolverMutex.Lock() + defer fake.getLoggerResolverMutex.Unlock() + fake.GetLoggerResolverStub = nil + fake.getLoggerResolverReturns = struct { + result1 logger.DeferredFieldResolver + }{result1} +} + +func (fake *FakeLocalParticipant) GetLoggerResolverReturnsOnCall(i int, result1 logger.DeferredFieldResolver) { + fake.getLoggerResolverMutex.Lock() + defer fake.getLoggerResolverMutex.Unlock() + fake.GetLoggerResolverStub = nil + if fake.getLoggerResolverReturnsOnCall == nil { + fake.getLoggerResolverReturnsOnCall = make(map[int]struct { + result1 logger.DeferredFieldResolver + }) + } + fake.getLoggerResolverReturnsOnCall[i] = struct { + result1 logger.DeferredFieldResolver + }{result1} +} + +func (fake *FakeLocalParticipant) GetNextSubscribedDataTrackHandle() uint16 { + fake.getNextSubscribedDataTrackHandleMutex.Lock() + ret, specificReturn := fake.getNextSubscribedDataTrackHandleReturnsOnCall[len(fake.getNextSubscribedDataTrackHandleArgsForCall)] + fake.getNextSubscribedDataTrackHandleArgsForCall = append(fake.getNextSubscribedDataTrackHandleArgsForCall, struct { + }{}) + stub := fake.GetNextSubscribedDataTrackHandleStub + fakeReturns := fake.getNextSubscribedDataTrackHandleReturns + fake.recordInvocation("GetNextSubscribedDataTrackHandle", []interface{}{}) + fake.getNextSubscribedDataTrackHandleMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeLocalParticipant) GetNextSubscribedDataTrackHandleCallCount() int { + fake.getNextSubscribedDataTrackHandleMutex.RLock() + defer fake.getNextSubscribedDataTrackHandleMutex.RUnlock() + return len(fake.getNextSubscribedDataTrackHandleArgsForCall) +} + +func (fake *FakeLocalParticipant) GetNextSubscribedDataTrackHandleCalls(stub func() uint16) { + fake.getNextSubscribedDataTrackHandleMutex.Lock() + defer fake.getNextSubscribedDataTrackHandleMutex.Unlock() + fake.GetNextSubscribedDataTrackHandleStub = stub +} + +func (fake *FakeLocalParticipant) GetNextSubscribedDataTrackHandleReturns(result1 uint16) { + fake.getNextSubscribedDataTrackHandleMutex.Lock() + defer fake.getNextSubscribedDataTrackHandleMutex.Unlock() + fake.GetNextSubscribedDataTrackHandleStub = nil + fake.getNextSubscribedDataTrackHandleReturns = struct { + result1 uint16 + }{result1} +} + +func (fake *FakeLocalParticipant) GetNextSubscribedDataTrackHandleReturnsOnCall(i int, result1 uint16) { + fake.getNextSubscribedDataTrackHandleMutex.Lock() + defer fake.getNextSubscribedDataTrackHandleMutex.Unlock() + fake.GetNextSubscribedDataTrackHandleStub = nil + if fake.getNextSubscribedDataTrackHandleReturnsOnCall == nil { + fake.getNextSubscribedDataTrackHandleReturnsOnCall = make(map[int]struct { + result1 uint16 + }) + } + fake.getNextSubscribedDataTrackHandleReturnsOnCall[i] = struct { + result1 uint16 + }{result1} +} + func (fake *FakeLocalParticipant) GetPacer() pacer.Pacer { fake.getPacerMutex.Lock() ret, specificReturn := fake.getPacerReturnsOnCall[len(fake.getPacerArgsForCall)] @@ -2880,6 +3501,59 @@ func (fake *FakeLocalParticipant) GetPacerReturnsOnCall(i int, result1 pacer.Pac }{result1} } +func (fake *FakeLocalParticipant) GetParticipantListener() types.ParticipantListener { + fake.getParticipantListenerMutex.Lock() + ret, specificReturn := fake.getParticipantListenerReturnsOnCall[len(fake.getParticipantListenerArgsForCall)] + fake.getParticipantListenerArgsForCall = append(fake.getParticipantListenerArgsForCall, struct { + }{}) + stub := fake.GetParticipantListenerStub + fakeReturns := fake.getParticipantListenerReturns + fake.recordInvocation("GetParticipantListener", []interface{}{}) + fake.getParticipantListenerMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeLocalParticipant) GetParticipantListenerCallCount() int { + fake.getParticipantListenerMutex.RLock() + defer fake.getParticipantListenerMutex.RUnlock() + return len(fake.getParticipantListenerArgsForCall) +} + +func (fake *FakeLocalParticipant) GetParticipantListenerCalls(stub func() types.ParticipantListener) { + fake.getParticipantListenerMutex.Lock() + defer fake.getParticipantListenerMutex.Unlock() + fake.GetParticipantListenerStub = stub +} + +func (fake *FakeLocalParticipant) GetParticipantListenerReturns(result1 types.ParticipantListener) { + fake.getParticipantListenerMutex.Lock() + defer fake.getParticipantListenerMutex.Unlock() + fake.GetParticipantListenerStub = nil + fake.getParticipantListenerReturns = struct { + result1 types.ParticipantListener + }{result1} +} + +func (fake *FakeLocalParticipant) GetParticipantListenerReturnsOnCall(i int, result1 types.ParticipantListener) { + fake.getParticipantListenerMutex.Lock() + defer fake.getParticipantListenerMutex.Unlock() + fake.GetParticipantListenerStub = nil + if fake.getParticipantListenerReturnsOnCall == nil { + fake.getParticipantListenerReturnsOnCall = make(map[int]struct { + result1 types.ParticipantListener + }) + } + fake.getParticipantListenerReturnsOnCall[i] = struct { + result1 types.ParticipantListener + }{result1} +} + func (fake *FakeLocalParticipant) GetPendingTrack(arg1 livekit.TrackID) *livekit.TrackInfo { fake.getPendingTrackMutex.Lock() ret, specificReturn := fake.getPendingTrackReturnsOnCall[len(fake.getPendingTrackArgsForCall)] @@ -2994,6 +3668,120 @@ func (fake *FakeLocalParticipant) GetPlayoutDelayConfigReturnsOnCall(i int, resu }{result1} } +func (fake *FakeLocalParticipant) GetPublishedDataTrack(arg1 uint16) types.DataTrack { + fake.getPublishedDataTrackMutex.Lock() + ret, specificReturn := fake.getPublishedDataTrackReturnsOnCall[len(fake.getPublishedDataTrackArgsForCall)] + fake.getPublishedDataTrackArgsForCall = append(fake.getPublishedDataTrackArgsForCall, struct { + arg1 uint16 + }{arg1}) + stub := fake.GetPublishedDataTrackStub + fakeReturns := fake.getPublishedDataTrackReturns + fake.recordInvocation("GetPublishedDataTrack", []interface{}{arg1}) + fake.getPublishedDataTrackMutex.Unlock() + if stub != nil { + return stub(arg1) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeLocalParticipant) GetPublishedDataTrackCallCount() int { + fake.getPublishedDataTrackMutex.RLock() + defer fake.getPublishedDataTrackMutex.RUnlock() + return len(fake.getPublishedDataTrackArgsForCall) +} + +func (fake *FakeLocalParticipant) GetPublishedDataTrackCalls(stub func(uint16) types.DataTrack) { + fake.getPublishedDataTrackMutex.Lock() + defer fake.getPublishedDataTrackMutex.Unlock() + fake.GetPublishedDataTrackStub = stub +} + +func (fake *FakeLocalParticipant) GetPublishedDataTrackArgsForCall(i int) uint16 { + fake.getPublishedDataTrackMutex.RLock() + defer fake.getPublishedDataTrackMutex.RUnlock() + argsForCall := fake.getPublishedDataTrackArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeLocalParticipant) GetPublishedDataTrackReturns(result1 types.DataTrack) { + fake.getPublishedDataTrackMutex.Lock() + defer fake.getPublishedDataTrackMutex.Unlock() + fake.GetPublishedDataTrackStub = nil + fake.getPublishedDataTrackReturns = struct { + result1 types.DataTrack + }{result1} +} + +func (fake *FakeLocalParticipant) GetPublishedDataTrackReturnsOnCall(i int, result1 types.DataTrack) { + fake.getPublishedDataTrackMutex.Lock() + defer fake.getPublishedDataTrackMutex.Unlock() + fake.GetPublishedDataTrackStub = nil + if fake.getPublishedDataTrackReturnsOnCall == nil { + fake.getPublishedDataTrackReturnsOnCall = make(map[int]struct { + result1 types.DataTrack + }) + } + fake.getPublishedDataTrackReturnsOnCall[i] = struct { + result1 types.DataTrack + }{result1} +} + +func (fake *FakeLocalParticipant) GetPublishedDataTracks() []types.DataTrack { + fake.getPublishedDataTracksMutex.Lock() + ret, specificReturn := fake.getPublishedDataTracksReturnsOnCall[len(fake.getPublishedDataTracksArgsForCall)] + fake.getPublishedDataTracksArgsForCall = append(fake.getPublishedDataTracksArgsForCall, struct { + }{}) + stub := fake.GetPublishedDataTracksStub + fakeReturns := fake.getPublishedDataTracksReturns + fake.recordInvocation("GetPublishedDataTracks", []interface{}{}) + fake.getPublishedDataTracksMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeLocalParticipant) GetPublishedDataTracksCallCount() int { + fake.getPublishedDataTracksMutex.RLock() + defer fake.getPublishedDataTracksMutex.RUnlock() + return len(fake.getPublishedDataTracksArgsForCall) +} + +func (fake *FakeLocalParticipant) GetPublishedDataTracksCalls(stub func() []types.DataTrack) { + fake.getPublishedDataTracksMutex.Lock() + defer fake.getPublishedDataTracksMutex.Unlock() + fake.GetPublishedDataTracksStub = stub +} + +func (fake *FakeLocalParticipant) GetPublishedDataTracksReturns(result1 []types.DataTrack) { + fake.getPublishedDataTracksMutex.Lock() + defer fake.getPublishedDataTracksMutex.Unlock() + fake.GetPublishedDataTracksStub = nil + fake.getPublishedDataTracksReturns = struct { + result1 []types.DataTrack + }{result1} +} + +func (fake *FakeLocalParticipant) GetPublishedDataTracksReturnsOnCall(i int, result1 []types.DataTrack) { + fake.getPublishedDataTracksMutex.Lock() + defer fake.getPublishedDataTracksMutex.Unlock() + fake.GetPublishedDataTracksStub = nil + if fake.getPublishedDataTracksReturnsOnCall == nil { + fake.getPublishedDataTracksReturnsOnCall = make(map[int]struct { + result1 []types.DataTrack + }) + } + fake.getPublishedDataTracksReturnsOnCall[i] = struct { + result1 []types.DataTrack + }{result1} +} + func (fake *FakeLocalParticipant) GetPublishedTrack(arg1 livekit.TrackID) types.MediaTrack { fake.getPublishedTrackMutex.Lock() ret, specificReturn := fake.getPublishedTrackReturnsOnCall[len(fake.getPublishedTrackArgsForCall)] @@ -3108,6 +3896,221 @@ func (fake *FakeLocalParticipant) GetPublishedTracksReturnsOnCall(i int, result1 }{result1} } +func (fake *FakeLocalParticipant) GetPublisherICESessionUfrag() (string, error) { + fake.getPublisherICESessionUfragMutex.Lock() + ret, specificReturn := fake.getPublisherICESessionUfragReturnsOnCall[len(fake.getPublisherICESessionUfragArgsForCall)] + fake.getPublisherICESessionUfragArgsForCall = append(fake.getPublisherICESessionUfragArgsForCall, struct { + }{}) + stub := fake.GetPublisherICESessionUfragStub + fakeReturns := fake.getPublisherICESessionUfragReturns + fake.recordInvocation("GetPublisherICESessionUfrag", []interface{}{}) + fake.getPublisherICESessionUfragMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeLocalParticipant) GetPublisherICESessionUfragCallCount() int { + fake.getPublisherICESessionUfragMutex.RLock() + defer fake.getPublisherICESessionUfragMutex.RUnlock() + return len(fake.getPublisherICESessionUfragArgsForCall) +} + +func (fake *FakeLocalParticipant) GetPublisherICESessionUfragCalls(stub func() (string, error)) { + fake.getPublisherICESessionUfragMutex.Lock() + defer fake.getPublisherICESessionUfragMutex.Unlock() + fake.GetPublisherICESessionUfragStub = stub +} + +func (fake *FakeLocalParticipant) GetPublisherICESessionUfragReturns(result1 string, result2 error) { + fake.getPublisherICESessionUfragMutex.Lock() + defer fake.getPublisherICESessionUfragMutex.Unlock() + fake.GetPublisherICESessionUfragStub = nil + fake.getPublisherICESessionUfragReturns = struct { + result1 string + result2 error + }{result1, result2} +} + +func (fake *FakeLocalParticipant) GetPublisherICESessionUfragReturnsOnCall(i int, result1 string, result2 error) { + fake.getPublisherICESessionUfragMutex.Lock() + defer fake.getPublisherICESessionUfragMutex.Unlock() + fake.GetPublisherICESessionUfragStub = nil + if fake.getPublisherICESessionUfragReturnsOnCall == nil { + fake.getPublisherICESessionUfragReturnsOnCall = make(map[int]struct { + result1 string + result2 error + }) + } + fake.getPublisherICESessionUfragReturnsOnCall[i] = struct { + result1 string + result2 error + }{result1, result2} +} + +func (fake *FakeLocalParticipant) GetReporter() roomobs.ParticipantSessionReporter { + fake.getReporterMutex.Lock() + ret, specificReturn := fake.getReporterReturnsOnCall[len(fake.getReporterArgsForCall)] + fake.getReporterArgsForCall = append(fake.getReporterArgsForCall, struct { + }{}) + stub := fake.GetReporterStub + fakeReturns := fake.getReporterReturns + fake.recordInvocation("GetReporter", []interface{}{}) + fake.getReporterMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeLocalParticipant) GetReporterCallCount() int { + fake.getReporterMutex.RLock() + defer fake.getReporterMutex.RUnlock() + return len(fake.getReporterArgsForCall) +} + +func (fake *FakeLocalParticipant) GetReporterCalls(stub func() roomobs.ParticipantSessionReporter) { + fake.getReporterMutex.Lock() + defer fake.getReporterMutex.Unlock() + fake.GetReporterStub = stub +} + +func (fake *FakeLocalParticipant) GetReporterReturns(result1 roomobs.ParticipantSessionReporter) { + fake.getReporterMutex.Lock() + defer fake.getReporterMutex.Unlock() + fake.GetReporterStub = nil + fake.getReporterReturns = struct { + result1 roomobs.ParticipantSessionReporter + }{result1} +} + +func (fake *FakeLocalParticipant) GetReporterReturnsOnCall(i int, result1 roomobs.ParticipantSessionReporter) { + fake.getReporterMutex.Lock() + defer fake.getReporterMutex.Unlock() + fake.GetReporterStub = nil + if fake.getReporterReturnsOnCall == nil { + fake.getReporterReturnsOnCall = make(map[int]struct { + result1 roomobs.ParticipantSessionReporter + }) + } + fake.getReporterReturnsOnCall[i] = struct { + result1 roomobs.ParticipantSessionReporter + }{result1} +} + +func (fake *FakeLocalParticipant) GetReporterResolver() roomobs.ParticipantReporterResolver { + fake.getReporterResolverMutex.Lock() + ret, specificReturn := fake.getReporterResolverReturnsOnCall[len(fake.getReporterResolverArgsForCall)] + fake.getReporterResolverArgsForCall = append(fake.getReporterResolverArgsForCall, struct { + }{}) + stub := fake.GetReporterResolverStub + fakeReturns := fake.getReporterResolverReturns + fake.recordInvocation("GetReporterResolver", []interface{}{}) + fake.getReporterResolverMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeLocalParticipant) GetReporterResolverCallCount() int { + fake.getReporterResolverMutex.RLock() + defer fake.getReporterResolverMutex.RUnlock() + return len(fake.getReporterResolverArgsForCall) +} + +func (fake *FakeLocalParticipant) GetReporterResolverCalls(stub func() roomobs.ParticipantReporterResolver) { + fake.getReporterResolverMutex.Lock() + defer fake.getReporterResolverMutex.Unlock() + fake.GetReporterResolverStub = stub +} + +func (fake *FakeLocalParticipant) GetReporterResolverReturns(result1 roomobs.ParticipantReporterResolver) { + fake.getReporterResolverMutex.Lock() + defer fake.getReporterResolverMutex.Unlock() + fake.GetReporterResolverStub = nil + fake.getReporterResolverReturns = struct { + result1 roomobs.ParticipantReporterResolver + }{result1} +} + +func (fake *FakeLocalParticipant) GetReporterResolverReturnsOnCall(i int, result1 roomobs.ParticipantReporterResolver) { + fake.getReporterResolverMutex.Lock() + defer fake.getReporterResolverMutex.Unlock() + fake.GetReporterResolverStub = nil + if fake.getReporterResolverReturnsOnCall == nil { + fake.getReporterResolverReturnsOnCall = make(map[int]struct { + result1 roomobs.ParticipantReporterResolver + }) + } + fake.getReporterResolverReturnsOnCall[i] = struct { + result1 roomobs.ParticipantReporterResolver + }{result1} +} + +func (fake *FakeLocalParticipant) GetResponseSink() routing.MessageSink { + fake.getResponseSinkMutex.Lock() + ret, specificReturn := fake.getResponseSinkReturnsOnCall[len(fake.getResponseSinkArgsForCall)] + fake.getResponseSinkArgsForCall = append(fake.getResponseSinkArgsForCall, struct { + }{}) + stub := fake.GetResponseSinkStub + fakeReturns := fake.getResponseSinkReturns + fake.recordInvocation("GetResponseSink", []interface{}{}) + fake.getResponseSinkMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeLocalParticipant) GetResponseSinkCallCount() int { + fake.getResponseSinkMutex.RLock() + defer fake.getResponseSinkMutex.RUnlock() + return len(fake.getResponseSinkArgsForCall) +} + +func (fake *FakeLocalParticipant) GetResponseSinkCalls(stub func() routing.MessageSink) { + fake.getResponseSinkMutex.Lock() + defer fake.getResponseSinkMutex.Unlock() + fake.GetResponseSinkStub = stub +} + +func (fake *FakeLocalParticipant) GetResponseSinkReturns(result1 routing.MessageSink) { + fake.getResponseSinkMutex.Lock() + defer fake.getResponseSinkMutex.Unlock() + fake.GetResponseSinkStub = nil + fake.getResponseSinkReturns = struct { + result1 routing.MessageSink + }{result1} +} + +func (fake *FakeLocalParticipant) GetResponseSinkReturnsOnCall(i int, result1 routing.MessageSink) { + fake.getResponseSinkMutex.Lock() + defer fake.getResponseSinkMutex.Unlock() + fake.GetResponseSinkStub = nil + if fake.getResponseSinkReturnsOnCall == nil { + fake.getResponseSinkReturnsOnCall = make(map[int]struct { + result1 routing.MessageSink + }) + } + fake.getResponseSinkReturnsOnCall[i] = struct { + result1 routing.MessageSink + }{result1} +} + func (fake *FakeLocalParticipant) GetSubscribedParticipants() []livekit.ParticipantID { fake.getSubscribedParticipantsMutex.Lock() ret, specificReturn := fake.getSubscribedParticipantsReturnsOnCall[len(fake.getSubscribedParticipantsArgsForCall)] @@ -3214,6 +4217,59 @@ func (fake *FakeLocalParticipant) GetSubscribedTracksReturnsOnCall(i int, result }{result1} } +func (fake *FakeLocalParticipant) GetTelemetryListener() types.ParticipantTelemetryListener { + fake.getTelemetryListenerMutex.Lock() + ret, specificReturn := fake.getTelemetryListenerReturnsOnCall[len(fake.getTelemetryListenerArgsForCall)] + fake.getTelemetryListenerArgsForCall = append(fake.getTelemetryListenerArgsForCall, struct { + }{}) + stub := fake.GetTelemetryListenerStub + fakeReturns := fake.getTelemetryListenerReturns + fake.recordInvocation("GetTelemetryListener", []interface{}{}) + fake.getTelemetryListenerMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeLocalParticipant) GetTelemetryListenerCallCount() int { + fake.getTelemetryListenerMutex.RLock() + defer fake.getTelemetryListenerMutex.RUnlock() + return len(fake.getTelemetryListenerArgsForCall) +} + +func (fake *FakeLocalParticipant) GetTelemetryListenerCalls(stub func() types.ParticipantTelemetryListener) { + fake.getTelemetryListenerMutex.Lock() + defer fake.getTelemetryListenerMutex.Unlock() + fake.GetTelemetryListenerStub = stub +} + +func (fake *FakeLocalParticipant) GetTelemetryListenerReturns(result1 types.ParticipantTelemetryListener) { + fake.getTelemetryListenerMutex.Lock() + defer fake.getTelemetryListenerMutex.Unlock() + fake.GetTelemetryListenerStub = nil + fake.getTelemetryListenerReturns = struct { + result1 types.ParticipantTelemetryListener + }{result1} +} + +func (fake *FakeLocalParticipant) GetTelemetryListenerReturnsOnCall(i int, result1 types.ParticipantTelemetryListener) { + fake.getTelemetryListenerMutex.Lock() + defer fake.getTelemetryListenerMutex.Unlock() + fake.GetTelemetryListenerStub = nil + if fake.getTelemetryListenerReturnsOnCall == nil { + fake.getTelemetryListenerReturnsOnCall = make(map[int]struct { + result1 types.ParticipantTelemetryListener + }) + } + fake.getTelemetryListenerReturnsOnCall[i] = struct { + result1 types.ParticipantTelemetryListener + }{result1} +} + func (fake *FakeLocalParticipant) GetTrailer() []byte { fake.getTrailerMutex.Lock() ret, specificReturn := fake.getTrailerReturnsOnCall[len(fake.getTrailerArgsForCall)] @@ -3267,10 +4323,10 @@ func (fake *FakeLocalParticipant) GetTrailerReturnsOnCall(i int, result1 []byte) }{result1} } -func (fake *FakeLocalParticipant) HandleAnswer(arg1 webrtc.SessionDescription) { +func (fake *FakeLocalParticipant) HandleAnswer(arg1 *livekit.SessionDescription) { fake.handleAnswerMutex.Lock() fake.handleAnswerArgsForCall = append(fake.handleAnswerArgsForCall, struct { - arg1 webrtc.SessionDescription + arg1 *livekit.SessionDescription }{arg1}) stub := fake.HandleAnswerStub fake.recordInvocation("HandleAnswer", []interface{}{arg1}) @@ -3286,19 +4342,208 @@ func (fake *FakeLocalParticipant) HandleAnswerCallCount() int { return len(fake.handleAnswerArgsForCall) } -func (fake *FakeLocalParticipant) HandleAnswerCalls(stub func(webrtc.SessionDescription)) { +func (fake *FakeLocalParticipant) HandleAnswerCalls(stub func(*livekit.SessionDescription)) { fake.handleAnswerMutex.Lock() defer fake.handleAnswerMutex.Unlock() fake.HandleAnswerStub = stub } -func (fake *FakeLocalParticipant) HandleAnswerArgsForCall(i int) webrtc.SessionDescription { +func (fake *FakeLocalParticipant) HandleAnswerArgsForCall(i int) *livekit.SessionDescription { fake.handleAnswerMutex.RLock() defer fake.handleAnswerMutex.RUnlock() argsForCall := fake.handleAnswerArgsForCall[i] return argsForCall.arg1 } +func (fake *FakeLocalParticipant) HandleICERestartSDPFragment(arg1 string) (string, error) { + fake.handleICERestartSDPFragmentMutex.Lock() + ret, specificReturn := fake.handleICERestartSDPFragmentReturnsOnCall[len(fake.handleICERestartSDPFragmentArgsForCall)] + fake.handleICERestartSDPFragmentArgsForCall = append(fake.handleICERestartSDPFragmentArgsForCall, struct { + arg1 string + }{arg1}) + stub := fake.HandleICERestartSDPFragmentStub + fakeReturns := fake.handleICERestartSDPFragmentReturns + fake.recordInvocation("HandleICERestartSDPFragment", []interface{}{arg1}) + fake.handleICERestartSDPFragmentMutex.Unlock() + if stub != nil { + return stub(arg1) + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeLocalParticipant) HandleICERestartSDPFragmentCallCount() int { + fake.handleICERestartSDPFragmentMutex.RLock() + defer fake.handleICERestartSDPFragmentMutex.RUnlock() + return len(fake.handleICERestartSDPFragmentArgsForCall) +} + +func (fake *FakeLocalParticipant) HandleICERestartSDPFragmentCalls(stub func(string) (string, error)) { + fake.handleICERestartSDPFragmentMutex.Lock() + defer fake.handleICERestartSDPFragmentMutex.Unlock() + fake.HandleICERestartSDPFragmentStub = stub +} + +func (fake *FakeLocalParticipant) HandleICERestartSDPFragmentArgsForCall(i int) string { + fake.handleICERestartSDPFragmentMutex.RLock() + defer fake.handleICERestartSDPFragmentMutex.RUnlock() + argsForCall := fake.handleICERestartSDPFragmentArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeLocalParticipant) HandleICERestartSDPFragmentReturns(result1 string, result2 error) { + fake.handleICERestartSDPFragmentMutex.Lock() + defer fake.handleICERestartSDPFragmentMutex.Unlock() + fake.HandleICERestartSDPFragmentStub = nil + fake.handleICERestartSDPFragmentReturns = struct { + result1 string + result2 error + }{result1, result2} +} + +func (fake *FakeLocalParticipant) HandleICERestartSDPFragmentReturnsOnCall(i int, result1 string, result2 error) { + fake.handleICERestartSDPFragmentMutex.Lock() + defer fake.handleICERestartSDPFragmentMutex.Unlock() + fake.HandleICERestartSDPFragmentStub = nil + if fake.handleICERestartSDPFragmentReturnsOnCall == nil { + fake.handleICERestartSDPFragmentReturnsOnCall = make(map[int]struct { + result1 string + result2 error + }) + } + fake.handleICERestartSDPFragmentReturnsOnCall[i] = struct { + result1 string + result2 error + }{result1, result2} +} + +func (fake *FakeLocalParticipant) HandleICETrickle(arg1 *livekit.TrickleRequest) { + fake.handleICETrickleMutex.Lock() + fake.handleICETrickleArgsForCall = append(fake.handleICETrickleArgsForCall, struct { + arg1 *livekit.TrickleRequest + }{arg1}) + stub := fake.HandleICETrickleStub + fake.recordInvocation("HandleICETrickle", []interface{}{arg1}) + fake.handleICETrickleMutex.Unlock() + if stub != nil { + fake.HandleICETrickleStub(arg1) + } +} + +func (fake *FakeLocalParticipant) HandleICETrickleCallCount() int { + fake.handleICETrickleMutex.RLock() + defer fake.handleICETrickleMutex.RUnlock() + return len(fake.handleICETrickleArgsForCall) +} + +func (fake *FakeLocalParticipant) HandleICETrickleCalls(stub func(*livekit.TrickleRequest)) { + fake.handleICETrickleMutex.Lock() + defer fake.handleICETrickleMutex.Unlock() + fake.HandleICETrickleStub = stub +} + +func (fake *FakeLocalParticipant) HandleICETrickleArgsForCall(i int) *livekit.TrickleRequest { + fake.handleICETrickleMutex.RLock() + defer fake.handleICETrickleMutex.RUnlock() + argsForCall := fake.handleICETrickleArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeLocalParticipant) HandleICETrickleSDPFragment(arg1 string) error { + fake.handleICETrickleSDPFragmentMutex.Lock() + ret, specificReturn := fake.handleICETrickleSDPFragmentReturnsOnCall[len(fake.handleICETrickleSDPFragmentArgsForCall)] + fake.handleICETrickleSDPFragmentArgsForCall = append(fake.handleICETrickleSDPFragmentArgsForCall, struct { + arg1 string + }{arg1}) + stub := fake.HandleICETrickleSDPFragmentStub + fakeReturns := fake.handleICETrickleSDPFragmentReturns + fake.recordInvocation("HandleICETrickleSDPFragment", []interface{}{arg1}) + fake.handleICETrickleSDPFragmentMutex.Unlock() + if stub != nil { + return stub(arg1) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeLocalParticipant) HandleICETrickleSDPFragmentCallCount() int { + fake.handleICETrickleSDPFragmentMutex.RLock() + defer fake.handleICETrickleSDPFragmentMutex.RUnlock() + return len(fake.handleICETrickleSDPFragmentArgsForCall) +} + +func (fake *FakeLocalParticipant) HandleICETrickleSDPFragmentCalls(stub func(string) error) { + fake.handleICETrickleSDPFragmentMutex.Lock() + defer fake.handleICETrickleSDPFragmentMutex.Unlock() + fake.HandleICETrickleSDPFragmentStub = stub +} + +func (fake *FakeLocalParticipant) HandleICETrickleSDPFragmentArgsForCall(i int) string { + fake.handleICETrickleSDPFragmentMutex.RLock() + defer fake.handleICETrickleSDPFragmentMutex.RUnlock() + argsForCall := fake.handleICETrickleSDPFragmentArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeLocalParticipant) HandleICETrickleSDPFragmentReturns(result1 error) { + fake.handleICETrickleSDPFragmentMutex.Lock() + defer fake.handleICETrickleSDPFragmentMutex.Unlock() + fake.HandleICETrickleSDPFragmentStub = nil + fake.handleICETrickleSDPFragmentReturns = struct { + result1 error + }{result1} +} + +func (fake *FakeLocalParticipant) HandleICETrickleSDPFragmentReturnsOnCall(i int, result1 error) { + fake.handleICETrickleSDPFragmentMutex.Lock() + defer fake.handleICETrickleSDPFragmentMutex.Unlock() + fake.HandleICETrickleSDPFragmentStub = nil + if fake.handleICETrickleSDPFragmentReturnsOnCall == nil { + fake.handleICETrickleSDPFragmentReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.handleICETrickleSDPFragmentReturnsOnCall[i] = struct { + result1 error + }{result1} +} + +func (fake *FakeLocalParticipant) HandleLeaveRequest(arg1 types.ParticipantCloseReason) { + fake.handleLeaveRequestMutex.Lock() + fake.handleLeaveRequestArgsForCall = append(fake.handleLeaveRequestArgsForCall, struct { + arg1 types.ParticipantCloseReason + }{arg1}) + stub := fake.HandleLeaveRequestStub + fake.recordInvocation("HandleLeaveRequest", []interface{}{arg1}) + fake.handleLeaveRequestMutex.Unlock() + if stub != nil { + fake.HandleLeaveRequestStub(arg1) + } +} + +func (fake *FakeLocalParticipant) HandleLeaveRequestCallCount() int { + fake.handleLeaveRequestMutex.RLock() + defer fake.handleLeaveRequestMutex.RUnlock() + return len(fake.handleLeaveRequestArgsForCall) +} + +func (fake *FakeLocalParticipant) HandleLeaveRequestCalls(stub func(types.ParticipantCloseReason)) { + fake.handleLeaveRequestMutex.Lock() + defer fake.handleLeaveRequestMutex.Unlock() + fake.HandleLeaveRequestStub = stub +} + +func (fake *FakeLocalParticipant) HandleLeaveRequestArgsForCall(i int) types.ParticipantCloseReason { + fake.handleLeaveRequestMutex.RLock() + defer fake.handleLeaveRequestMutex.RUnlock() + argsForCall := fake.handleLeaveRequestArgsForCall[i] + return argsForCall.arg1 +} + func (fake *FakeLocalParticipant) HandleMetrics(arg1 livekit.ParticipantID, arg2 *livekit.MetricsBatch) error { fake.handleMetricsMutex.Lock() ret, specificReturn := fake.handleMetricsReturnsOnCall[len(fake.handleMetricsArgsForCall)] @@ -3361,11 +4606,11 @@ func (fake *FakeLocalParticipant) HandleMetricsReturnsOnCall(i int, result1 erro }{result1} } -func (fake *FakeLocalParticipant) HandleOffer(arg1 webrtc.SessionDescription) error { +func (fake *FakeLocalParticipant) HandleOffer(arg1 *livekit.SessionDescription) error { fake.handleOfferMutex.Lock() ret, specificReturn := fake.handleOfferReturnsOnCall[len(fake.handleOfferArgsForCall)] fake.handleOfferArgsForCall = append(fake.handleOfferArgsForCall, struct { - arg1 webrtc.SessionDescription + arg1 *livekit.SessionDescription }{arg1}) stub := fake.HandleOfferStub fakeReturns := fake.handleOfferReturns @@ -3386,13 +4631,13 @@ func (fake *FakeLocalParticipant) HandleOfferCallCount() int { return len(fake.handleOfferArgsForCall) } -func (fake *FakeLocalParticipant) HandleOfferCalls(stub func(webrtc.SessionDescription) error) { +func (fake *FakeLocalParticipant) HandleOfferCalls(stub func(*livekit.SessionDescription) error) { fake.handleOfferMutex.Lock() defer fake.handleOfferMutex.Unlock() fake.HandleOfferStub = stub } -func (fake *FakeLocalParticipant) HandleOfferArgsForCall(i int) webrtc.SessionDescription { +func (fake *FakeLocalParticipant) HandleOfferArgsForCall(i int) *livekit.SessionDescription { fake.handleOfferMutex.RLock() defer fake.handleOfferMutex.RUnlock() argsForCall := fake.handleOfferArgsForCall[i] @@ -3422,6 +4667,77 @@ func (fake *FakeLocalParticipant) HandleOfferReturnsOnCall(i int, result1 error) }{result1} } +func (fake *FakeLocalParticipant) HandlePublishDataTrackRequest(arg1 *livekit.PublishDataTrackRequest) { + fake.handlePublishDataTrackRequestMutex.Lock() + fake.handlePublishDataTrackRequestArgsForCall = append(fake.handlePublishDataTrackRequestArgsForCall, struct { + arg1 *livekit.PublishDataTrackRequest + }{arg1}) + stub := fake.HandlePublishDataTrackRequestStub + fake.recordInvocation("HandlePublishDataTrackRequest", []interface{}{arg1}) + fake.handlePublishDataTrackRequestMutex.Unlock() + if stub != nil { + fake.HandlePublishDataTrackRequestStub(arg1) + } +} + +func (fake *FakeLocalParticipant) HandlePublishDataTrackRequestCallCount() int { + fake.handlePublishDataTrackRequestMutex.RLock() + defer fake.handlePublishDataTrackRequestMutex.RUnlock() + return len(fake.handlePublishDataTrackRequestArgsForCall) +} + +func (fake *FakeLocalParticipant) HandlePublishDataTrackRequestCalls(stub func(*livekit.PublishDataTrackRequest)) { + fake.handlePublishDataTrackRequestMutex.Lock() + defer fake.handlePublishDataTrackRequestMutex.Unlock() + fake.HandlePublishDataTrackRequestStub = stub +} + +func (fake *FakeLocalParticipant) HandlePublishDataTrackRequestArgsForCall(i int) *livekit.PublishDataTrackRequest { + fake.handlePublishDataTrackRequestMutex.RLock() + defer fake.handlePublishDataTrackRequestMutex.RUnlock() + argsForCall := fake.handlePublishDataTrackRequestArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeLocalParticipant) HandleReceivedDataTrackMessage(arg1 []byte, arg2 *datatrack.Packet, arg3 int64) { + var arg1Copy []byte + if arg1 != nil { + arg1Copy = make([]byte, len(arg1)) + copy(arg1Copy, arg1) + } + fake.handleReceivedDataTrackMessageMutex.Lock() + fake.handleReceivedDataTrackMessageArgsForCall = append(fake.handleReceivedDataTrackMessageArgsForCall, struct { + arg1 []byte + arg2 *datatrack.Packet + arg3 int64 + }{arg1Copy, arg2, arg3}) + stub := fake.HandleReceivedDataTrackMessageStub + fake.recordInvocation("HandleReceivedDataTrackMessage", []interface{}{arg1Copy, arg2, arg3}) + fake.handleReceivedDataTrackMessageMutex.Unlock() + if stub != nil { + fake.HandleReceivedDataTrackMessageStub(arg1, arg2, arg3) + } +} + +func (fake *FakeLocalParticipant) HandleReceivedDataTrackMessageCallCount() int { + fake.handleReceivedDataTrackMessageMutex.RLock() + defer fake.handleReceivedDataTrackMessageMutex.RUnlock() + return len(fake.handleReceivedDataTrackMessageArgsForCall) +} + +func (fake *FakeLocalParticipant) HandleReceivedDataTrackMessageCalls(stub func([]byte, *datatrack.Packet, int64)) { + fake.handleReceivedDataTrackMessageMutex.Lock() + defer fake.handleReceivedDataTrackMessageMutex.Unlock() + fake.HandleReceivedDataTrackMessageStub = stub +} + +func (fake *FakeLocalParticipant) HandleReceivedDataTrackMessageArgsForCall(i int) ([]byte, *datatrack.Packet, int64) { + fake.handleReceivedDataTrackMessageMutex.RLock() + defer fake.handleReceivedDataTrackMessageMutex.RUnlock() + argsForCall := fake.handleReceivedDataTrackMessageArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 +} + func (fake *FakeLocalParticipant) HandleReceiverReport(arg1 *sfu.DownTrack, arg2 *rtcp.ReceiverReport) { fake.handleReceiverReportMutex.Lock() fake.handleReceiverReportArgsForCall = append(fake.handleReceiverReportArgsForCall, struct { @@ -3517,6 +4833,67 @@ func (fake *FakeLocalParticipant) HandleReconnectAndSendResponseReturnsOnCall(i }{result1} } +func (fake *FakeLocalParticipant) HandleSignalMessage(arg1 proto.Message) error { + fake.handleSignalMessageMutex.Lock() + ret, specificReturn := fake.handleSignalMessageReturnsOnCall[len(fake.handleSignalMessageArgsForCall)] + fake.handleSignalMessageArgsForCall = append(fake.handleSignalMessageArgsForCall, struct { + arg1 proto.Message + }{arg1}) + stub := fake.HandleSignalMessageStub + fakeReturns := fake.handleSignalMessageReturns + fake.recordInvocation("HandleSignalMessage", []interface{}{arg1}) + fake.handleSignalMessageMutex.Unlock() + if stub != nil { + return stub(arg1) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeLocalParticipant) HandleSignalMessageCallCount() int { + fake.handleSignalMessageMutex.RLock() + defer fake.handleSignalMessageMutex.RUnlock() + return len(fake.handleSignalMessageArgsForCall) +} + +func (fake *FakeLocalParticipant) HandleSignalMessageCalls(stub func(proto.Message) error) { + fake.handleSignalMessageMutex.Lock() + defer fake.handleSignalMessageMutex.Unlock() + fake.HandleSignalMessageStub = stub +} + +func (fake *FakeLocalParticipant) HandleSignalMessageArgsForCall(i int) proto.Message { + fake.handleSignalMessageMutex.RLock() + defer fake.handleSignalMessageMutex.RUnlock() + argsForCall := fake.handleSignalMessageArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeLocalParticipant) HandleSignalMessageReturns(result1 error) { + fake.handleSignalMessageMutex.Lock() + defer fake.handleSignalMessageMutex.Unlock() + fake.HandleSignalMessageStub = nil + fake.handleSignalMessageReturns = struct { + result1 error + }{result1} +} + +func (fake *FakeLocalParticipant) HandleSignalMessageReturnsOnCall(i int, result1 error) { + fake.handleSignalMessageMutex.Lock() + defer fake.handleSignalMessageMutex.Unlock() + fake.HandleSignalMessageStub = nil + if fake.handleSignalMessageReturnsOnCall == nil { + fake.handleSignalMessageReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.handleSignalMessageReturnsOnCall[i] = struct { + result1 error + }{result1} +} + func (fake *FakeLocalParticipant) HandleSignalSourceClose() { fake.handleSignalSourceCloseMutex.Lock() fake.handleSignalSourceCloseArgsForCall = append(fake.handleSignalSourceCloseArgsForCall, struct { @@ -3541,6 +4918,297 @@ func (fake *FakeLocalParticipant) HandleSignalSourceCloseCalls(stub func()) { fake.HandleSignalSourceCloseStub = stub } +func (fake *FakeLocalParticipant) HandleSimulateScenario(arg1 *livekit.SimulateScenario) error { + fake.handleSimulateScenarioMutex.Lock() + ret, specificReturn := fake.handleSimulateScenarioReturnsOnCall[len(fake.handleSimulateScenarioArgsForCall)] + fake.handleSimulateScenarioArgsForCall = append(fake.handleSimulateScenarioArgsForCall, struct { + arg1 *livekit.SimulateScenario + }{arg1}) + stub := fake.HandleSimulateScenarioStub + fakeReturns := fake.handleSimulateScenarioReturns + fake.recordInvocation("HandleSimulateScenario", []interface{}{arg1}) + fake.handleSimulateScenarioMutex.Unlock() + if stub != nil { + return stub(arg1) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeLocalParticipant) HandleSimulateScenarioCallCount() int { + fake.handleSimulateScenarioMutex.RLock() + defer fake.handleSimulateScenarioMutex.RUnlock() + return len(fake.handleSimulateScenarioArgsForCall) +} + +func (fake *FakeLocalParticipant) HandleSimulateScenarioCalls(stub func(*livekit.SimulateScenario) error) { + fake.handleSimulateScenarioMutex.Lock() + defer fake.handleSimulateScenarioMutex.Unlock() + fake.HandleSimulateScenarioStub = stub +} + +func (fake *FakeLocalParticipant) HandleSimulateScenarioArgsForCall(i int) *livekit.SimulateScenario { + fake.handleSimulateScenarioMutex.RLock() + defer fake.handleSimulateScenarioMutex.RUnlock() + argsForCall := fake.handleSimulateScenarioArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeLocalParticipant) HandleSimulateScenarioReturns(result1 error) { + fake.handleSimulateScenarioMutex.Lock() + defer fake.handleSimulateScenarioMutex.Unlock() + fake.HandleSimulateScenarioStub = nil + fake.handleSimulateScenarioReturns = struct { + result1 error + }{result1} +} + +func (fake *FakeLocalParticipant) HandleSimulateScenarioReturnsOnCall(i int, result1 error) { + fake.handleSimulateScenarioMutex.Lock() + defer fake.handleSimulateScenarioMutex.Unlock() + fake.HandleSimulateScenarioStub = nil + if fake.handleSimulateScenarioReturnsOnCall == nil { + fake.handleSimulateScenarioReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.handleSimulateScenarioReturnsOnCall[i] = struct { + result1 error + }{result1} +} + +func (fake *FakeLocalParticipant) HandleSyncState(arg1 *livekit.SyncState) error { + fake.handleSyncStateMutex.Lock() + ret, specificReturn := fake.handleSyncStateReturnsOnCall[len(fake.handleSyncStateArgsForCall)] + fake.handleSyncStateArgsForCall = append(fake.handleSyncStateArgsForCall, struct { + arg1 *livekit.SyncState + }{arg1}) + stub := fake.HandleSyncStateStub + fakeReturns := fake.handleSyncStateReturns + fake.recordInvocation("HandleSyncState", []interface{}{arg1}) + fake.handleSyncStateMutex.Unlock() + if stub != nil { + return stub(arg1) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeLocalParticipant) HandleSyncStateCallCount() int { + fake.handleSyncStateMutex.RLock() + defer fake.handleSyncStateMutex.RUnlock() + return len(fake.handleSyncStateArgsForCall) +} + +func (fake *FakeLocalParticipant) HandleSyncStateCalls(stub func(*livekit.SyncState) error) { + fake.handleSyncStateMutex.Lock() + defer fake.handleSyncStateMutex.Unlock() + fake.HandleSyncStateStub = stub +} + +func (fake *FakeLocalParticipant) HandleSyncStateArgsForCall(i int) *livekit.SyncState { + fake.handleSyncStateMutex.RLock() + defer fake.handleSyncStateMutex.RUnlock() + argsForCall := fake.handleSyncStateArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeLocalParticipant) HandleSyncStateReturns(result1 error) { + fake.handleSyncStateMutex.Lock() + defer fake.handleSyncStateMutex.Unlock() + fake.HandleSyncStateStub = nil + fake.handleSyncStateReturns = struct { + result1 error + }{result1} +} + +func (fake *FakeLocalParticipant) HandleSyncStateReturnsOnCall(i int, result1 error) { + fake.handleSyncStateMutex.Lock() + defer fake.handleSyncStateMutex.Unlock() + fake.HandleSyncStateStub = nil + if fake.handleSyncStateReturnsOnCall == nil { + fake.handleSyncStateReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.handleSyncStateReturnsOnCall[i] = struct { + result1 error + }{result1} +} + +func (fake *FakeLocalParticipant) HandleUnpublishDataTrackRequest(arg1 *livekit.UnpublishDataTrackRequest) { + fake.handleUnpublishDataTrackRequestMutex.Lock() + fake.handleUnpublishDataTrackRequestArgsForCall = append(fake.handleUnpublishDataTrackRequestArgsForCall, struct { + arg1 *livekit.UnpublishDataTrackRequest + }{arg1}) + stub := fake.HandleUnpublishDataTrackRequestStub + fake.recordInvocation("HandleUnpublishDataTrackRequest", []interface{}{arg1}) + fake.handleUnpublishDataTrackRequestMutex.Unlock() + if stub != nil { + fake.HandleUnpublishDataTrackRequestStub(arg1) + } +} + +func (fake *FakeLocalParticipant) HandleUnpublishDataTrackRequestCallCount() int { + fake.handleUnpublishDataTrackRequestMutex.RLock() + defer fake.handleUnpublishDataTrackRequestMutex.RUnlock() + return len(fake.handleUnpublishDataTrackRequestArgsForCall) +} + +func (fake *FakeLocalParticipant) HandleUnpublishDataTrackRequestCalls(stub func(*livekit.UnpublishDataTrackRequest)) { + fake.handleUnpublishDataTrackRequestMutex.Lock() + defer fake.handleUnpublishDataTrackRequestMutex.Unlock() + fake.HandleUnpublishDataTrackRequestStub = stub +} + +func (fake *FakeLocalParticipant) HandleUnpublishDataTrackRequestArgsForCall(i int) *livekit.UnpublishDataTrackRequest { + fake.handleUnpublishDataTrackRequestMutex.RLock() + defer fake.handleUnpublishDataTrackRequestMutex.RUnlock() + argsForCall := fake.handleUnpublishDataTrackRequestArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeLocalParticipant) HandleUpdateDataSubscription(arg1 *livekit.UpdateDataSubscription) { + fake.handleUpdateDataSubscriptionMutex.Lock() + fake.handleUpdateDataSubscriptionArgsForCall = append(fake.handleUpdateDataSubscriptionArgsForCall, struct { + arg1 *livekit.UpdateDataSubscription + }{arg1}) + stub := fake.HandleUpdateDataSubscriptionStub + fake.recordInvocation("HandleUpdateDataSubscription", []interface{}{arg1}) + fake.handleUpdateDataSubscriptionMutex.Unlock() + if stub != nil { + fake.HandleUpdateDataSubscriptionStub(arg1) + } +} + +func (fake *FakeLocalParticipant) HandleUpdateDataSubscriptionCallCount() int { + fake.handleUpdateDataSubscriptionMutex.RLock() + defer fake.handleUpdateDataSubscriptionMutex.RUnlock() + return len(fake.handleUpdateDataSubscriptionArgsForCall) +} + +func (fake *FakeLocalParticipant) HandleUpdateDataSubscriptionCalls(stub func(*livekit.UpdateDataSubscription)) { + fake.handleUpdateDataSubscriptionMutex.Lock() + defer fake.handleUpdateDataSubscriptionMutex.Unlock() + fake.HandleUpdateDataSubscriptionStub = stub +} + +func (fake *FakeLocalParticipant) HandleUpdateDataSubscriptionArgsForCall(i int) *livekit.UpdateDataSubscription { + fake.handleUpdateDataSubscriptionMutex.RLock() + defer fake.handleUpdateDataSubscriptionMutex.RUnlock() + argsForCall := fake.handleUpdateDataSubscriptionArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeLocalParticipant) HandleUpdateSubscriptionPermission(arg1 *livekit.SubscriptionPermission) error { + fake.handleUpdateSubscriptionPermissionMutex.Lock() + ret, specificReturn := fake.handleUpdateSubscriptionPermissionReturnsOnCall[len(fake.handleUpdateSubscriptionPermissionArgsForCall)] + fake.handleUpdateSubscriptionPermissionArgsForCall = append(fake.handleUpdateSubscriptionPermissionArgsForCall, struct { + arg1 *livekit.SubscriptionPermission + }{arg1}) + stub := fake.HandleUpdateSubscriptionPermissionStub + fakeReturns := fake.handleUpdateSubscriptionPermissionReturns + fake.recordInvocation("HandleUpdateSubscriptionPermission", []interface{}{arg1}) + fake.handleUpdateSubscriptionPermissionMutex.Unlock() + if stub != nil { + return stub(arg1) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeLocalParticipant) HandleUpdateSubscriptionPermissionCallCount() int { + fake.handleUpdateSubscriptionPermissionMutex.RLock() + defer fake.handleUpdateSubscriptionPermissionMutex.RUnlock() + return len(fake.handleUpdateSubscriptionPermissionArgsForCall) +} + +func (fake *FakeLocalParticipant) HandleUpdateSubscriptionPermissionCalls(stub func(*livekit.SubscriptionPermission) error) { + fake.handleUpdateSubscriptionPermissionMutex.Lock() + defer fake.handleUpdateSubscriptionPermissionMutex.Unlock() + fake.HandleUpdateSubscriptionPermissionStub = stub +} + +func (fake *FakeLocalParticipant) HandleUpdateSubscriptionPermissionArgsForCall(i int) *livekit.SubscriptionPermission { + fake.handleUpdateSubscriptionPermissionMutex.RLock() + defer fake.handleUpdateSubscriptionPermissionMutex.RUnlock() + argsForCall := fake.handleUpdateSubscriptionPermissionArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeLocalParticipant) HandleUpdateSubscriptionPermissionReturns(result1 error) { + fake.handleUpdateSubscriptionPermissionMutex.Lock() + defer fake.handleUpdateSubscriptionPermissionMutex.Unlock() + fake.HandleUpdateSubscriptionPermissionStub = nil + fake.handleUpdateSubscriptionPermissionReturns = struct { + result1 error + }{result1} +} + +func (fake *FakeLocalParticipant) HandleUpdateSubscriptionPermissionReturnsOnCall(i int, result1 error) { + fake.handleUpdateSubscriptionPermissionMutex.Lock() + defer fake.handleUpdateSubscriptionPermissionMutex.Unlock() + fake.HandleUpdateSubscriptionPermissionStub = nil + if fake.handleUpdateSubscriptionPermissionReturnsOnCall == nil { + fake.handleUpdateSubscriptionPermissionReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.handleUpdateSubscriptionPermissionReturnsOnCall[i] = struct { + result1 error + }{result1} +} + +func (fake *FakeLocalParticipant) HandleUpdateSubscriptions(arg1 []livekit.TrackID, arg2 []*livekit.ParticipantTracks, arg3 bool) { + var arg1Copy []livekit.TrackID + if arg1 != nil { + arg1Copy = make([]livekit.TrackID, len(arg1)) + copy(arg1Copy, arg1) + } + var arg2Copy []*livekit.ParticipantTracks + if arg2 != nil { + arg2Copy = make([]*livekit.ParticipantTracks, len(arg2)) + copy(arg2Copy, arg2) + } + fake.handleUpdateSubscriptionsMutex.Lock() + fake.handleUpdateSubscriptionsArgsForCall = append(fake.handleUpdateSubscriptionsArgsForCall, struct { + arg1 []livekit.TrackID + arg2 []*livekit.ParticipantTracks + arg3 bool + }{arg1Copy, arg2Copy, arg3}) + stub := fake.HandleUpdateSubscriptionsStub + fake.recordInvocation("HandleUpdateSubscriptions", []interface{}{arg1Copy, arg2Copy, arg3}) + fake.handleUpdateSubscriptionsMutex.Unlock() + if stub != nil { + fake.HandleUpdateSubscriptionsStub(arg1, arg2, arg3) + } +} + +func (fake *FakeLocalParticipant) HandleUpdateSubscriptionsCallCount() int { + fake.handleUpdateSubscriptionsMutex.RLock() + defer fake.handleUpdateSubscriptionsMutex.RUnlock() + return len(fake.handleUpdateSubscriptionsArgsForCall) +} + +func (fake *FakeLocalParticipant) HandleUpdateSubscriptionsCalls(stub func([]livekit.TrackID, []*livekit.ParticipantTracks, bool)) { + fake.handleUpdateSubscriptionsMutex.Lock() + defer fake.handleUpdateSubscriptionsMutex.Unlock() + fake.HandleUpdateSubscriptionsStub = stub +} + +func (fake *FakeLocalParticipant) HandleUpdateSubscriptionsArgsForCall(i int) ([]livekit.TrackID, []*livekit.ParticipantTracks, bool) { + fake.handleUpdateSubscriptionsMutex.RLock() + defer fake.handleUpdateSubscriptionsMutex.RUnlock() + argsForCall := fake.handleUpdateSubscriptionsArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 +} + func (fake *FakeLocalParticipant) HasConnected() bool { fake.hasConnectedMutex.Lock() ret, specificReturn := fake.hasConnectedReturnsOnCall[len(fake.hasConnectedArgsForCall)] @@ -4447,6 +6115,59 @@ func (fake *FakeLocalParticipant) IsTrackNameSubscribedReturnsOnCall(i int, resu }{result1} } +func (fake *FakeLocalParticipant) IsUsingSinglePeerConnection() bool { + fake.isUsingSinglePeerConnectionMutex.Lock() + ret, specificReturn := fake.isUsingSinglePeerConnectionReturnsOnCall[len(fake.isUsingSinglePeerConnectionArgsForCall)] + fake.isUsingSinglePeerConnectionArgsForCall = append(fake.isUsingSinglePeerConnectionArgsForCall, struct { + }{}) + stub := fake.IsUsingSinglePeerConnectionStub + fakeReturns := fake.isUsingSinglePeerConnectionReturns + fake.recordInvocation("IsUsingSinglePeerConnection", []interface{}{}) + fake.isUsingSinglePeerConnectionMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeLocalParticipant) IsUsingSinglePeerConnectionCallCount() int { + fake.isUsingSinglePeerConnectionMutex.RLock() + defer fake.isUsingSinglePeerConnectionMutex.RUnlock() + return len(fake.isUsingSinglePeerConnectionArgsForCall) +} + +func (fake *FakeLocalParticipant) IsUsingSinglePeerConnectionCalls(stub func() bool) { + fake.isUsingSinglePeerConnectionMutex.Lock() + defer fake.isUsingSinglePeerConnectionMutex.Unlock() + fake.IsUsingSinglePeerConnectionStub = stub +} + +func (fake *FakeLocalParticipant) IsUsingSinglePeerConnectionReturns(result1 bool) { + fake.isUsingSinglePeerConnectionMutex.Lock() + defer fake.isUsingSinglePeerConnectionMutex.Unlock() + fake.IsUsingSinglePeerConnectionStub = nil + fake.isUsingSinglePeerConnectionReturns = struct { + result1 bool + }{result1} +} + +func (fake *FakeLocalParticipant) IsUsingSinglePeerConnectionReturnsOnCall(i int, result1 bool) { + fake.isUsingSinglePeerConnectionMutex.Lock() + defer fake.isUsingSinglePeerConnectionMutex.Unlock() + fake.IsUsingSinglePeerConnectionStub = nil + if fake.isUsingSinglePeerConnectionReturnsOnCall == nil { + fake.isUsingSinglePeerConnectionReturnsOnCall = make(map[int]struct { + result1 bool + }) + } + fake.isUsingSinglePeerConnectionReturnsOnCall[i] = struct { + result1 bool + }{result1} +} + func (fake *FakeLocalParticipant) IssueFullReconnect(arg1 types.ParticipantCloseReason) { fake.issueFullReconnectMutex.Lock() fake.issueFullReconnectArgsForCall = append(fake.issueFullReconnectArgsForCall, struct { @@ -4532,6 +6253,59 @@ func (fake *FakeLocalParticipant) KindReturnsOnCall(i int, result1 livekit.Parti }{result1} } +func (fake *FakeLocalParticipant) KindDetails() []livekit.ParticipantInfo_KindDetail { + fake.kindDetailsMutex.Lock() + ret, specificReturn := fake.kindDetailsReturnsOnCall[len(fake.kindDetailsArgsForCall)] + fake.kindDetailsArgsForCall = append(fake.kindDetailsArgsForCall, struct { + }{}) + stub := fake.KindDetailsStub + fakeReturns := fake.kindDetailsReturns + fake.recordInvocation("KindDetails", []interface{}{}) + fake.kindDetailsMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeLocalParticipant) KindDetailsCallCount() int { + fake.kindDetailsMutex.RLock() + defer fake.kindDetailsMutex.RUnlock() + return len(fake.kindDetailsArgsForCall) +} + +func (fake *FakeLocalParticipant) KindDetailsCalls(stub func() []livekit.ParticipantInfo_KindDetail) { + fake.kindDetailsMutex.Lock() + defer fake.kindDetailsMutex.Unlock() + fake.KindDetailsStub = stub +} + +func (fake *FakeLocalParticipant) KindDetailsReturns(result1 []livekit.ParticipantInfo_KindDetail) { + fake.kindDetailsMutex.Lock() + defer fake.kindDetailsMutex.Unlock() + fake.KindDetailsStub = nil + fake.kindDetailsReturns = struct { + result1 []livekit.ParticipantInfo_KindDetail + }{result1} +} + +func (fake *FakeLocalParticipant) KindDetailsReturnsOnCall(i int, result1 []livekit.ParticipantInfo_KindDetail) { + fake.kindDetailsMutex.Lock() + defer fake.kindDetailsMutex.Unlock() + fake.KindDetailsStub = nil + if fake.kindDetailsReturnsOnCall == nil { + fake.kindDetailsReturnsOnCall = make(map[int]struct { + result1 []livekit.ParticipantInfo_KindDetail + }) + } + fake.kindDetailsReturnsOnCall[i] = struct { + result1 []livekit.ParticipantInfo_KindDetail + }{result1} +} + func (fake *FakeLocalParticipant) MaybeStartMigration(arg1 bool, arg2 func()) bool { fake.maybeStartMigrationMutex.Lock() ret, specificReturn := fake.maybeStartMigrationReturnsOnCall[len(fake.maybeStartMigrationArgsForCall)] @@ -4647,6 +6421,38 @@ func (fake *FakeLocalParticipant) MigrateStateReturnsOnCall(i int, result1 types }{result1} } +func (fake *FakeLocalParticipant) MoveToRoom(arg1 types.MoveToRoomParams) { + fake.moveToRoomMutex.Lock() + fake.moveToRoomArgsForCall = append(fake.moveToRoomArgsForCall, struct { + arg1 types.MoveToRoomParams + }{arg1}) + stub := fake.MoveToRoomStub + fake.recordInvocation("MoveToRoom", []interface{}{arg1}) + fake.moveToRoomMutex.Unlock() + if stub != nil { + fake.MoveToRoomStub(arg1) + } +} + +func (fake *FakeLocalParticipant) MoveToRoomCallCount() int { + fake.moveToRoomMutex.RLock() + defer fake.moveToRoomMutex.RUnlock() + return len(fake.moveToRoomArgsForCall) +} + +func (fake *FakeLocalParticipant) MoveToRoomCalls(stub func(types.MoveToRoomParams)) { + fake.moveToRoomMutex.Lock() + defer fake.moveToRoomMutex.Unlock() + fake.MoveToRoomStub = stub +} + +func (fake *FakeLocalParticipant) MoveToRoomArgsForCall(i int) types.MoveToRoomParams { + fake.moveToRoomMutex.RLock() + defer fake.moveToRoomMutex.RUnlock() + argsForCall := fake.moveToRoomArgsForCall[i] + return argsForCall.arg1 +} + func (fake *FakeLocalParticipant) Negotiate(arg1 bool) { fake.negotiateMutex.Lock() fake.negotiateArgsForCall = append(fake.negotiateArgsForCall, struct { @@ -4735,70 +6541,6 @@ func (fake *FakeLocalParticipant) OnClaimsChangedArgsForCall(i int) func(types.L return argsForCall.arg1 } -func (fake *FakeLocalParticipant) OnClose(arg1 func(types.LocalParticipant)) { - fake.onCloseMutex.Lock() - fake.onCloseArgsForCall = append(fake.onCloseArgsForCall, struct { - arg1 func(types.LocalParticipant) - }{arg1}) - stub := fake.OnCloseStub - fake.recordInvocation("OnClose", []interface{}{arg1}) - fake.onCloseMutex.Unlock() - if stub != nil { - fake.OnCloseStub(arg1) - } -} - -func (fake *FakeLocalParticipant) OnCloseCallCount() int { - fake.onCloseMutex.RLock() - defer fake.onCloseMutex.RUnlock() - return len(fake.onCloseArgsForCall) -} - -func (fake *FakeLocalParticipant) OnCloseCalls(stub func(func(types.LocalParticipant))) { - fake.onCloseMutex.Lock() - defer fake.onCloseMutex.Unlock() - fake.OnCloseStub = stub -} - -func (fake *FakeLocalParticipant) OnCloseArgsForCall(i int) func(types.LocalParticipant) { - fake.onCloseMutex.RLock() - defer fake.onCloseMutex.RUnlock() - argsForCall := fake.onCloseArgsForCall[i] - return argsForCall.arg1 -} - -func (fake *FakeLocalParticipant) OnDataPacket(arg1 func(types.LocalParticipant, livekit.DataPacket_Kind, *livekit.DataPacket)) { - fake.onDataPacketMutex.Lock() - fake.onDataPacketArgsForCall = append(fake.onDataPacketArgsForCall, struct { - arg1 func(types.LocalParticipant, livekit.DataPacket_Kind, *livekit.DataPacket) - }{arg1}) - stub := fake.OnDataPacketStub - fake.recordInvocation("OnDataPacket", []interface{}{arg1}) - fake.onDataPacketMutex.Unlock() - if stub != nil { - fake.OnDataPacketStub(arg1) - } -} - -func (fake *FakeLocalParticipant) OnDataPacketCallCount() int { - fake.onDataPacketMutex.RLock() - defer fake.onDataPacketMutex.RUnlock() - return len(fake.onDataPacketArgsForCall) -} - -func (fake *FakeLocalParticipant) OnDataPacketCalls(stub func(func(types.LocalParticipant, livekit.DataPacket_Kind, *livekit.DataPacket))) { - fake.onDataPacketMutex.Lock() - defer fake.onDataPacketMutex.Unlock() - fake.OnDataPacketStub = stub -} - -func (fake *FakeLocalParticipant) OnDataPacketArgsForCall(i int) func(types.LocalParticipant, livekit.DataPacket_Kind, *livekit.DataPacket) { - fake.onDataPacketMutex.RLock() - defer fake.onDataPacketMutex.RUnlock() - argsForCall := fake.onDataPacketArgsForCall[i] - return argsForCall.arg1 -} - func (fake *FakeLocalParticipant) OnICEConfigChanged(arg1 func(participant types.LocalParticipant, iceConfig *livekit.ICEConfig)) { fake.onICEConfigChangedMutex.Lock() fake.onICEConfigChangedArgsForCall = append(fake.onICEConfigChangedArgsForCall, struct { @@ -4831,260 +6573,38 @@ func (fake *FakeLocalParticipant) OnICEConfigChangedArgsForCall(i int) func(part return argsForCall.arg1 } -func (fake *FakeLocalParticipant) OnMetrics(arg1 func(types.Participant, *livekit.DataPacket)) { - fake.onMetricsMutex.Lock() - fake.onMetricsArgsForCall = append(fake.onMetricsArgsForCall, struct { - arg1 func(types.Participant, *livekit.DataPacket) - }{arg1}) - stub := fake.OnMetricsStub - fake.recordInvocation("OnMetrics", []interface{}{arg1}) - fake.onMetricsMutex.Unlock() +func (fake *FakeLocalParticipant) PerformRpc(arg1 *livekit.PerformRpcRequest, arg2 chan string, arg3 chan error) { + fake.performRpcMutex.Lock() + fake.performRpcArgsForCall = append(fake.performRpcArgsForCall, struct { + arg1 *livekit.PerformRpcRequest + arg2 chan string + arg3 chan error + }{arg1, arg2, arg3}) + stub := fake.PerformRpcStub + fake.recordInvocation("PerformRpc", []interface{}{arg1, arg2, arg3}) + fake.performRpcMutex.Unlock() if stub != nil { - fake.OnMetricsStub(arg1) + fake.PerformRpcStub(arg1, arg2, arg3) } } -func (fake *FakeLocalParticipant) OnMetricsCallCount() int { - fake.onMetricsMutex.RLock() - defer fake.onMetricsMutex.RUnlock() - return len(fake.onMetricsArgsForCall) +func (fake *FakeLocalParticipant) PerformRpcCallCount() int { + fake.performRpcMutex.RLock() + defer fake.performRpcMutex.RUnlock() + return len(fake.performRpcArgsForCall) } -func (fake *FakeLocalParticipant) OnMetricsCalls(stub func(func(types.Participant, *livekit.DataPacket))) { - fake.onMetricsMutex.Lock() - defer fake.onMetricsMutex.Unlock() - fake.OnMetricsStub = stub +func (fake *FakeLocalParticipant) PerformRpcCalls(stub func(*livekit.PerformRpcRequest, chan string, chan error)) { + fake.performRpcMutex.Lock() + defer fake.performRpcMutex.Unlock() + fake.PerformRpcStub = stub } -func (fake *FakeLocalParticipant) OnMetricsArgsForCall(i int) func(types.Participant, *livekit.DataPacket) { - fake.onMetricsMutex.RLock() - defer fake.onMetricsMutex.RUnlock() - argsForCall := fake.onMetricsArgsForCall[i] - return argsForCall.arg1 -} - -func (fake *FakeLocalParticipant) OnMigrateStateChange(arg1 func(p types.LocalParticipant, migrateState types.MigrateState)) { - fake.onMigrateStateChangeMutex.Lock() - fake.onMigrateStateChangeArgsForCall = append(fake.onMigrateStateChangeArgsForCall, struct { - arg1 func(p types.LocalParticipant, migrateState types.MigrateState) - }{arg1}) - stub := fake.OnMigrateStateChangeStub - fake.recordInvocation("OnMigrateStateChange", []interface{}{arg1}) - fake.onMigrateStateChangeMutex.Unlock() - if stub != nil { - fake.OnMigrateStateChangeStub(arg1) - } -} - -func (fake *FakeLocalParticipant) OnMigrateStateChangeCallCount() int { - fake.onMigrateStateChangeMutex.RLock() - defer fake.onMigrateStateChangeMutex.RUnlock() - return len(fake.onMigrateStateChangeArgsForCall) -} - -func (fake *FakeLocalParticipant) OnMigrateStateChangeCalls(stub func(func(p types.LocalParticipant, migrateState types.MigrateState))) { - fake.onMigrateStateChangeMutex.Lock() - defer fake.onMigrateStateChangeMutex.Unlock() - fake.OnMigrateStateChangeStub = stub -} - -func (fake *FakeLocalParticipant) OnMigrateStateChangeArgsForCall(i int) func(p types.LocalParticipant, migrateState types.MigrateState) { - fake.onMigrateStateChangeMutex.RLock() - defer fake.onMigrateStateChangeMutex.RUnlock() - argsForCall := fake.onMigrateStateChangeArgsForCall[i] - return argsForCall.arg1 -} - -func (fake *FakeLocalParticipant) OnParticipantUpdate(arg1 func(types.LocalParticipant)) { - fake.onParticipantUpdateMutex.Lock() - fake.onParticipantUpdateArgsForCall = append(fake.onParticipantUpdateArgsForCall, struct { - arg1 func(types.LocalParticipant) - }{arg1}) - stub := fake.OnParticipantUpdateStub - fake.recordInvocation("OnParticipantUpdate", []interface{}{arg1}) - fake.onParticipantUpdateMutex.Unlock() - if stub != nil { - fake.OnParticipantUpdateStub(arg1) - } -} - -func (fake *FakeLocalParticipant) OnParticipantUpdateCallCount() int { - fake.onParticipantUpdateMutex.RLock() - defer fake.onParticipantUpdateMutex.RUnlock() - return len(fake.onParticipantUpdateArgsForCall) -} - -func (fake *FakeLocalParticipant) OnParticipantUpdateCalls(stub func(func(types.LocalParticipant))) { - fake.onParticipantUpdateMutex.Lock() - defer fake.onParticipantUpdateMutex.Unlock() - fake.OnParticipantUpdateStub = stub -} - -func (fake *FakeLocalParticipant) OnParticipantUpdateArgsForCall(i int) func(types.LocalParticipant) { - fake.onParticipantUpdateMutex.RLock() - defer fake.onParticipantUpdateMutex.RUnlock() - argsForCall := fake.onParticipantUpdateArgsForCall[i] - return argsForCall.arg1 -} - -func (fake *FakeLocalParticipant) OnStateChange(arg1 func(p types.LocalParticipant, state livekit.ParticipantInfo_State)) { - fake.onStateChangeMutex.Lock() - fake.onStateChangeArgsForCall = append(fake.onStateChangeArgsForCall, struct { - arg1 func(p types.LocalParticipant, state livekit.ParticipantInfo_State) - }{arg1}) - stub := fake.OnStateChangeStub - fake.recordInvocation("OnStateChange", []interface{}{arg1}) - fake.onStateChangeMutex.Unlock() - if stub != nil { - fake.OnStateChangeStub(arg1) - } -} - -func (fake *FakeLocalParticipant) OnStateChangeCallCount() int { - fake.onStateChangeMutex.RLock() - defer fake.onStateChangeMutex.RUnlock() - return len(fake.onStateChangeArgsForCall) -} - -func (fake *FakeLocalParticipant) OnStateChangeCalls(stub func(func(p types.LocalParticipant, state livekit.ParticipantInfo_State))) { - fake.onStateChangeMutex.Lock() - defer fake.onStateChangeMutex.Unlock() - fake.OnStateChangeStub = stub -} - -func (fake *FakeLocalParticipant) OnStateChangeArgsForCall(i int) func(p types.LocalParticipant, state livekit.ParticipantInfo_State) { - fake.onStateChangeMutex.RLock() - defer fake.onStateChangeMutex.RUnlock() - argsForCall := fake.onStateChangeArgsForCall[i] - return argsForCall.arg1 -} - -func (fake *FakeLocalParticipant) OnSubscribeStatusChanged(arg1 func(publisherID livekit.ParticipantID, subscribed bool)) { - fake.onSubscribeStatusChangedMutex.Lock() - fake.onSubscribeStatusChangedArgsForCall = append(fake.onSubscribeStatusChangedArgsForCall, struct { - arg1 func(publisherID livekit.ParticipantID, subscribed bool) - }{arg1}) - stub := fake.OnSubscribeStatusChangedStub - fake.recordInvocation("OnSubscribeStatusChanged", []interface{}{arg1}) - fake.onSubscribeStatusChangedMutex.Unlock() - if stub != nil { - fake.OnSubscribeStatusChangedStub(arg1) - } -} - -func (fake *FakeLocalParticipant) OnSubscribeStatusChangedCallCount() int { - fake.onSubscribeStatusChangedMutex.RLock() - defer fake.onSubscribeStatusChangedMutex.RUnlock() - return len(fake.onSubscribeStatusChangedArgsForCall) -} - -func (fake *FakeLocalParticipant) OnSubscribeStatusChangedCalls(stub func(func(publisherID livekit.ParticipantID, subscribed bool))) { - fake.onSubscribeStatusChangedMutex.Lock() - defer fake.onSubscribeStatusChangedMutex.Unlock() - fake.OnSubscribeStatusChangedStub = stub -} - -func (fake *FakeLocalParticipant) OnSubscribeStatusChangedArgsForCall(i int) func(publisherID livekit.ParticipantID, subscribed bool) { - fake.onSubscribeStatusChangedMutex.RLock() - defer fake.onSubscribeStatusChangedMutex.RUnlock() - argsForCall := fake.onSubscribeStatusChangedArgsForCall[i] - return argsForCall.arg1 -} - -func (fake *FakeLocalParticipant) OnTrackPublished(arg1 func(types.LocalParticipant, types.MediaTrack)) { - fake.onTrackPublishedMutex.Lock() - fake.onTrackPublishedArgsForCall = append(fake.onTrackPublishedArgsForCall, struct { - arg1 func(types.LocalParticipant, types.MediaTrack) - }{arg1}) - stub := fake.OnTrackPublishedStub - fake.recordInvocation("OnTrackPublished", []interface{}{arg1}) - fake.onTrackPublishedMutex.Unlock() - if stub != nil { - fake.OnTrackPublishedStub(arg1) - } -} - -func (fake *FakeLocalParticipant) OnTrackPublishedCallCount() int { - fake.onTrackPublishedMutex.RLock() - defer fake.onTrackPublishedMutex.RUnlock() - return len(fake.onTrackPublishedArgsForCall) -} - -func (fake *FakeLocalParticipant) OnTrackPublishedCalls(stub func(func(types.LocalParticipant, types.MediaTrack))) { - fake.onTrackPublishedMutex.Lock() - defer fake.onTrackPublishedMutex.Unlock() - fake.OnTrackPublishedStub = stub -} - -func (fake *FakeLocalParticipant) OnTrackPublishedArgsForCall(i int) func(types.LocalParticipant, types.MediaTrack) { - fake.onTrackPublishedMutex.RLock() - defer fake.onTrackPublishedMutex.RUnlock() - argsForCall := fake.onTrackPublishedArgsForCall[i] - return argsForCall.arg1 -} - -func (fake *FakeLocalParticipant) OnTrackUnpublished(arg1 func(types.LocalParticipant, types.MediaTrack)) { - fake.onTrackUnpublishedMutex.Lock() - fake.onTrackUnpublishedArgsForCall = append(fake.onTrackUnpublishedArgsForCall, struct { - arg1 func(types.LocalParticipant, types.MediaTrack) - }{arg1}) - stub := fake.OnTrackUnpublishedStub - fake.recordInvocation("OnTrackUnpublished", []interface{}{arg1}) - fake.onTrackUnpublishedMutex.Unlock() - if stub != nil { - fake.OnTrackUnpublishedStub(arg1) - } -} - -func (fake *FakeLocalParticipant) OnTrackUnpublishedCallCount() int { - fake.onTrackUnpublishedMutex.RLock() - defer fake.onTrackUnpublishedMutex.RUnlock() - return len(fake.onTrackUnpublishedArgsForCall) -} - -func (fake *FakeLocalParticipant) OnTrackUnpublishedCalls(stub func(func(types.LocalParticipant, types.MediaTrack))) { - fake.onTrackUnpublishedMutex.Lock() - defer fake.onTrackUnpublishedMutex.Unlock() - fake.OnTrackUnpublishedStub = stub -} - -func (fake *FakeLocalParticipant) OnTrackUnpublishedArgsForCall(i int) func(types.LocalParticipant, types.MediaTrack) { - fake.onTrackUnpublishedMutex.RLock() - defer fake.onTrackUnpublishedMutex.RUnlock() - argsForCall := fake.onTrackUnpublishedArgsForCall[i] - return argsForCall.arg1 -} - -func (fake *FakeLocalParticipant) OnTrackUpdated(arg1 func(types.LocalParticipant, types.MediaTrack)) { - fake.onTrackUpdatedMutex.Lock() - fake.onTrackUpdatedArgsForCall = append(fake.onTrackUpdatedArgsForCall, struct { - arg1 func(types.LocalParticipant, types.MediaTrack) - }{arg1}) - stub := fake.OnTrackUpdatedStub - fake.recordInvocation("OnTrackUpdated", []interface{}{arg1}) - fake.onTrackUpdatedMutex.Unlock() - if stub != nil { - fake.OnTrackUpdatedStub(arg1) - } -} - -func (fake *FakeLocalParticipant) OnTrackUpdatedCallCount() int { - fake.onTrackUpdatedMutex.RLock() - defer fake.onTrackUpdatedMutex.RUnlock() - return len(fake.onTrackUpdatedArgsForCall) -} - -func (fake *FakeLocalParticipant) OnTrackUpdatedCalls(stub func(func(types.LocalParticipant, types.MediaTrack))) { - fake.onTrackUpdatedMutex.Lock() - defer fake.onTrackUpdatedMutex.Unlock() - fake.OnTrackUpdatedStub = stub -} - -func (fake *FakeLocalParticipant) OnTrackUpdatedArgsForCall(i int) func(types.LocalParticipant, types.MediaTrack) { - fake.onTrackUpdatedMutex.RLock() - defer fake.onTrackUpdatedMutex.RUnlock() - argsForCall := fake.onTrackUpdatedArgsForCall[i] - return argsForCall.arg1 +func (fake *FakeLocalParticipant) PerformRpcArgsForCall(i int) (*livekit.PerformRpcRequest, chan string, chan error) { + fake.performRpcMutex.RLock() + defer fake.performRpcMutex.RUnlock() + argsForCall := fake.performRpcArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 } func (fake *FakeLocalParticipant) ProtocolVersion() types.ProtocolVersion { @@ -5140,18 +6660,49 @@ func (fake *FakeLocalParticipant) ProtocolVersionReturnsOnCall(i int, result1 ty }{result1} } -func (fake *FakeLocalParticipant) RemovePublishedTrack(arg1 types.MediaTrack, arg2 bool, arg3 bool) { +func (fake *FakeLocalParticipant) RemovePublishedDataTrack(arg1 types.DataTrack) { + fake.removePublishedDataTrackMutex.Lock() + fake.removePublishedDataTrackArgsForCall = append(fake.removePublishedDataTrackArgsForCall, struct { + arg1 types.DataTrack + }{arg1}) + stub := fake.RemovePublishedDataTrackStub + fake.recordInvocation("RemovePublishedDataTrack", []interface{}{arg1}) + fake.removePublishedDataTrackMutex.Unlock() + if stub != nil { + fake.RemovePublishedDataTrackStub(arg1) + } +} + +func (fake *FakeLocalParticipant) RemovePublishedDataTrackCallCount() int { + fake.removePublishedDataTrackMutex.RLock() + defer fake.removePublishedDataTrackMutex.RUnlock() + return len(fake.removePublishedDataTrackArgsForCall) +} + +func (fake *FakeLocalParticipant) RemovePublishedDataTrackCalls(stub func(types.DataTrack)) { + fake.removePublishedDataTrackMutex.Lock() + defer fake.removePublishedDataTrackMutex.Unlock() + fake.RemovePublishedDataTrackStub = stub +} + +func (fake *FakeLocalParticipant) RemovePublishedDataTrackArgsForCall(i int) types.DataTrack { + fake.removePublishedDataTrackMutex.RLock() + defer fake.removePublishedDataTrackMutex.RUnlock() + argsForCall := fake.removePublishedDataTrackArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeLocalParticipant) RemovePublishedTrack(arg1 types.MediaTrack, arg2 bool) { fake.removePublishedTrackMutex.Lock() fake.removePublishedTrackArgsForCall = append(fake.removePublishedTrackArgsForCall, struct { arg1 types.MediaTrack arg2 bool - arg3 bool - }{arg1, arg2, arg3}) + }{arg1, arg2}) stub := fake.RemovePublishedTrackStub - fake.recordInvocation("RemovePublishedTrack", []interface{}{arg1, arg2, arg3}) + fake.recordInvocation("RemovePublishedTrack", []interface{}{arg1, arg2}) fake.removePublishedTrackMutex.Unlock() if stub != nil { - fake.RemovePublishedTrackStub(arg1, arg2, arg3) + fake.RemovePublishedTrackStub(arg1, arg2) } } @@ -5161,17 +6712,17 @@ func (fake *FakeLocalParticipant) RemovePublishedTrackCallCount() int { return len(fake.removePublishedTrackArgsForCall) } -func (fake *FakeLocalParticipant) RemovePublishedTrackCalls(stub func(types.MediaTrack, bool, bool)) { +func (fake *FakeLocalParticipant) RemovePublishedTrackCalls(stub func(types.MediaTrack, bool)) { fake.removePublishedTrackMutex.Lock() defer fake.removePublishedTrackMutex.Unlock() fake.RemovePublishedTrackStub = stub } -func (fake *FakeLocalParticipant) RemovePublishedTrackArgsForCall(i int) (types.MediaTrack, bool, bool) { +func (fake *FakeLocalParticipant) RemovePublishedTrackArgsForCall(i int) (types.MediaTrack, bool) { fake.removePublishedTrackMutex.RLock() defer fake.removePublishedTrackMutex.RUnlock() argsForCall := fake.removePublishedTrackArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 + return argsForCall.arg1, argsForCall.arg2 } func (fake *FakeLocalParticipant) RemoveTrackLocal(arg1 *webrtc.RTPSender) error { @@ -5296,24 +6847,26 @@ func (fake *FakeLocalParticipant) SendConnectionQualityUpdateReturnsOnCall(i int }{result1} } -func (fake *FakeLocalParticipant) SendDataPacket(arg1 livekit.DataPacket_Kind, arg2 []byte) error { +func (fake *FakeLocalParticipant) SendDataMessage(arg1 livekit.DataPacket_Kind, arg2 []byte, arg3 livekit.ParticipantID, arg4 uint32) error { var arg2Copy []byte if arg2 != nil { arg2Copy = make([]byte, len(arg2)) copy(arg2Copy, arg2) } - fake.sendDataPacketMutex.Lock() - ret, specificReturn := fake.sendDataPacketReturnsOnCall[len(fake.sendDataPacketArgsForCall)] - fake.sendDataPacketArgsForCall = append(fake.sendDataPacketArgsForCall, struct { + fake.sendDataMessageMutex.Lock() + ret, specificReturn := fake.sendDataMessageReturnsOnCall[len(fake.sendDataMessageArgsForCall)] + fake.sendDataMessageArgsForCall = append(fake.sendDataMessageArgsForCall, struct { arg1 livekit.DataPacket_Kind arg2 []byte - }{arg1, arg2Copy}) - stub := fake.SendDataPacketStub - fakeReturns := fake.sendDataPacketReturns - fake.recordInvocation("SendDataPacket", []interface{}{arg1, arg2Copy}) - fake.sendDataPacketMutex.Unlock() + arg3 livekit.ParticipantID + arg4 uint32 + }{arg1, arg2Copy, arg3, arg4}) + stub := fake.SendDataMessageStub + fakeReturns := fake.sendDataMessageReturns + fake.recordInvocation("SendDataMessage", []interface{}{arg1, arg2Copy, arg3, arg4}) + fake.sendDataMessageMutex.Unlock() if stub != nil { - return stub(arg1, arg2) + return stub(arg1, arg2, arg3, arg4) } if specificReturn { return ret.result1 @@ -5321,44 +6874,173 @@ func (fake *FakeLocalParticipant) SendDataPacket(arg1 livekit.DataPacket_Kind, a return fakeReturns.result1 } -func (fake *FakeLocalParticipant) SendDataPacketCallCount() int { - fake.sendDataPacketMutex.RLock() - defer fake.sendDataPacketMutex.RUnlock() - return len(fake.sendDataPacketArgsForCall) +func (fake *FakeLocalParticipant) SendDataMessageCallCount() int { + fake.sendDataMessageMutex.RLock() + defer fake.sendDataMessageMutex.RUnlock() + return len(fake.sendDataMessageArgsForCall) } -func (fake *FakeLocalParticipant) SendDataPacketCalls(stub func(livekit.DataPacket_Kind, []byte) error) { - fake.sendDataPacketMutex.Lock() - defer fake.sendDataPacketMutex.Unlock() - fake.SendDataPacketStub = stub +func (fake *FakeLocalParticipant) SendDataMessageCalls(stub func(livekit.DataPacket_Kind, []byte, livekit.ParticipantID, uint32) error) { + fake.sendDataMessageMutex.Lock() + defer fake.sendDataMessageMutex.Unlock() + fake.SendDataMessageStub = stub } -func (fake *FakeLocalParticipant) SendDataPacketArgsForCall(i int) (livekit.DataPacket_Kind, []byte) { - fake.sendDataPacketMutex.RLock() - defer fake.sendDataPacketMutex.RUnlock() - argsForCall := fake.sendDataPacketArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2 +func (fake *FakeLocalParticipant) SendDataMessageArgsForCall(i int) (livekit.DataPacket_Kind, []byte, livekit.ParticipantID, uint32) { + fake.sendDataMessageMutex.RLock() + defer fake.sendDataMessageMutex.RUnlock() + argsForCall := fake.sendDataMessageArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4 } -func (fake *FakeLocalParticipant) SendDataPacketReturns(result1 error) { - fake.sendDataPacketMutex.Lock() - defer fake.sendDataPacketMutex.Unlock() - fake.SendDataPacketStub = nil - fake.sendDataPacketReturns = struct { +func (fake *FakeLocalParticipant) SendDataMessageReturns(result1 error) { + fake.sendDataMessageMutex.Lock() + defer fake.sendDataMessageMutex.Unlock() + fake.SendDataMessageStub = nil + fake.sendDataMessageReturns = struct { result1 error }{result1} } -func (fake *FakeLocalParticipant) SendDataPacketReturnsOnCall(i int, result1 error) { - fake.sendDataPacketMutex.Lock() - defer fake.sendDataPacketMutex.Unlock() - fake.SendDataPacketStub = nil - if fake.sendDataPacketReturnsOnCall == nil { - fake.sendDataPacketReturnsOnCall = make(map[int]struct { +func (fake *FakeLocalParticipant) SendDataMessageReturnsOnCall(i int, result1 error) { + fake.sendDataMessageMutex.Lock() + defer fake.sendDataMessageMutex.Unlock() + fake.SendDataMessageStub = nil + if fake.sendDataMessageReturnsOnCall == nil { + fake.sendDataMessageReturnsOnCall = make(map[int]struct { result1 error }) } - fake.sendDataPacketReturnsOnCall[i] = struct { + fake.sendDataMessageReturnsOnCall[i] = struct { + result1 error + }{result1} +} + +func (fake *FakeLocalParticipant) SendDataMessageUnlabeled(arg1 []byte, arg2 bool, arg3 livekit.ParticipantIdentity) error { + var arg1Copy []byte + if arg1 != nil { + arg1Copy = make([]byte, len(arg1)) + copy(arg1Copy, arg1) + } + fake.sendDataMessageUnlabeledMutex.Lock() + ret, specificReturn := fake.sendDataMessageUnlabeledReturnsOnCall[len(fake.sendDataMessageUnlabeledArgsForCall)] + fake.sendDataMessageUnlabeledArgsForCall = append(fake.sendDataMessageUnlabeledArgsForCall, struct { + arg1 []byte + arg2 bool + arg3 livekit.ParticipantIdentity + }{arg1Copy, arg2, arg3}) + stub := fake.SendDataMessageUnlabeledStub + fakeReturns := fake.sendDataMessageUnlabeledReturns + fake.recordInvocation("SendDataMessageUnlabeled", []interface{}{arg1Copy, arg2, arg3}) + fake.sendDataMessageUnlabeledMutex.Unlock() + if stub != nil { + return stub(arg1, arg2, arg3) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeLocalParticipant) SendDataMessageUnlabeledCallCount() int { + fake.sendDataMessageUnlabeledMutex.RLock() + defer fake.sendDataMessageUnlabeledMutex.RUnlock() + return len(fake.sendDataMessageUnlabeledArgsForCall) +} + +func (fake *FakeLocalParticipant) SendDataMessageUnlabeledCalls(stub func([]byte, bool, livekit.ParticipantIdentity) error) { + fake.sendDataMessageUnlabeledMutex.Lock() + defer fake.sendDataMessageUnlabeledMutex.Unlock() + fake.SendDataMessageUnlabeledStub = stub +} + +func (fake *FakeLocalParticipant) SendDataMessageUnlabeledArgsForCall(i int) ([]byte, bool, livekit.ParticipantIdentity) { + fake.sendDataMessageUnlabeledMutex.RLock() + defer fake.sendDataMessageUnlabeledMutex.RUnlock() + argsForCall := fake.sendDataMessageUnlabeledArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 +} + +func (fake *FakeLocalParticipant) SendDataMessageUnlabeledReturns(result1 error) { + fake.sendDataMessageUnlabeledMutex.Lock() + defer fake.sendDataMessageUnlabeledMutex.Unlock() + fake.SendDataMessageUnlabeledStub = nil + fake.sendDataMessageUnlabeledReturns = struct { + result1 error + }{result1} +} + +func (fake *FakeLocalParticipant) SendDataMessageUnlabeledReturnsOnCall(i int, result1 error) { + fake.sendDataMessageUnlabeledMutex.Lock() + defer fake.sendDataMessageUnlabeledMutex.Unlock() + fake.SendDataMessageUnlabeledStub = nil + if fake.sendDataMessageUnlabeledReturnsOnCall == nil { + fake.sendDataMessageUnlabeledReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.sendDataMessageUnlabeledReturnsOnCall[i] = struct { + result1 error + }{result1} +} + +func (fake *FakeLocalParticipant) SendDataTrackSubscriberHandles(arg1 map[uint32]*livekit.DataTrackSubscriberHandles_PublishedDataTrack) error { + fake.sendDataTrackSubscriberHandlesMutex.Lock() + ret, specificReturn := fake.sendDataTrackSubscriberHandlesReturnsOnCall[len(fake.sendDataTrackSubscriberHandlesArgsForCall)] + fake.sendDataTrackSubscriberHandlesArgsForCall = append(fake.sendDataTrackSubscriberHandlesArgsForCall, struct { + arg1 map[uint32]*livekit.DataTrackSubscriberHandles_PublishedDataTrack + }{arg1}) + stub := fake.SendDataTrackSubscriberHandlesStub + fakeReturns := fake.sendDataTrackSubscriberHandlesReturns + fake.recordInvocation("SendDataTrackSubscriberHandles", []interface{}{arg1}) + fake.sendDataTrackSubscriberHandlesMutex.Unlock() + if stub != nil { + return stub(arg1) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeLocalParticipant) SendDataTrackSubscriberHandlesCallCount() int { + fake.sendDataTrackSubscriberHandlesMutex.RLock() + defer fake.sendDataTrackSubscriberHandlesMutex.RUnlock() + return len(fake.sendDataTrackSubscriberHandlesArgsForCall) +} + +func (fake *FakeLocalParticipant) SendDataTrackSubscriberHandlesCalls(stub func(map[uint32]*livekit.DataTrackSubscriberHandles_PublishedDataTrack) error) { + fake.sendDataTrackSubscriberHandlesMutex.Lock() + defer fake.sendDataTrackSubscriberHandlesMutex.Unlock() + fake.SendDataTrackSubscriberHandlesStub = stub +} + +func (fake *FakeLocalParticipant) SendDataTrackSubscriberHandlesArgsForCall(i int) map[uint32]*livekit.DataTrackSubscriberHandles_PublishedDataTrack { + fake.sendDataTrackSubscriberHandlesMutex.RLock() + defer fake.sendDataTrackSubscriberHandlesMutex.RUnlock() + argsForCall := fake.sendDataTrackSubscriberHandlesArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeLocalParticipant) SendDataTrackSubscriberHandlesReturns(result1 error) { + fake.sendDataTrackSubscriberHandlesMutex.Lock() + defer fake.sendDataTrackSubscriberHandlesMutex.Unlock() + fake.SendDataTrackSubscriberHandlesStub = nil + fake.sendDataTrackSubscriberHandlesReturns = struct { + result1 error + }{result1} +} + +func (fake *FakeLocalParticipant) SendDataTrackSubscriberHandlesReturnsOnCall(i int, result1 error) { + fake.sendDataTrackSubscriberHandlesMutex.Lock() + defer fake.sendDataTrackSubscriberHandlesMutex.Unlock() + fake.SendDataTrackSubscriberHandlesStub = nil + if fake.sendDataTrackSubscriberHandlesReturnsOnCall == nil { + fake.sendDataTrackSubscriberHandlesReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.sendDataTrackSubscriberHandlesReturnsOnCall[i] = struct { result1 error }{result1} } @@ -5551,16 +7233,16 @@ func (fake *FakeLocalParticipant) SendRefreshTokenReturnsOnCall(i int, result1 e }{result1} } -func (fake *FakeLocalParticipant) SendRequestResponse(arg1 *livekit.RequestResponse) error { - fake.sendRequestResponseMutex.Lock() - ret, specificReturn := fake.sendRequestResponseReturnsOnCall[len(fake.sendRequestResponseArgsForCall)] - fake.sendRequestResponseArgsForCall = append(fake.sendRequestResponseArgsForCall, struct { - arg1 *livekit.RequestResponse +func (fake *FakeLocalParticipant) SendRoomMovedResponse(arg1 *livekit.RoomMovedResponse) error { + fake.sendRoomMovedResponseMutex.Lock() + ret, specificReturn := fake.sendRoomMovedResponseReturnsOnCall[len(fake.sendRoomMovedResponseArgsForCall)] + fake.sendRoomMovedResponseArgsForCall = append(fake.sendRoomMovedResponseArgsForCall, struct { + arg1 *livekit.RoomMovedResponse }{arg1}) - stub := fake.SendRequestResponseStub - fakeReturns := fake.sendRequestResponseReturns - fake.recordInvocation("SendRequestResponse", []interface{}{arg1}) - fake.sendRequestResponseMutex.Unlock() + stub := fake.SendRoomMovedResponseStub + fakeReturns := fake.sendRoomMovedResponseReturns + fake.recordInvocation("SendRoomMovedResponse", []interface{}{arg1}) + fake.sendRoomMovedResponseMutex.Unlock() if stub != nil { return stub(arg1) } @@ -5570,44 +7252,44 @@ func (fake *FakeLocalParticipant) SendRequestResponse(arg1 *livekit.RequestRespo return fakeReturns.result1 } -func (fake *FakeLocalParticipant) SendRequestResponseCallCount() int { - fake.sendRequestResponseMutex.RLock() - defer fake.sendRequestResponseMutex.RUnlock() - return len(fake.sendRequestResponseArgsForCall) +func (fake *FakeLocalParticipant) SendRoomMovedResponseCallCount() int { + fake.sendRoomMovedResponseMutex.RLock() + defer fake.sendRoomMovedResponseMutex.RUnlock() + return len(fake.sendRoomMovedResponseArgsForCall) } -func (fake *FakeLocalParticipant) SendRequestResponseCalls(stub func(*livekit.RequestResponse) error) { - fake.sendRequestResponseMutex.Lock() - defer fake.sendRequestResponseMutex.Unlock() - fake.SendRequestResponseStub = stub +func (fake *FakeLocalParticipant) SendRoomMovedResponseCalls(stub func(*livekit.RoomMovedResponse) error) { + fake.sendRoomMovedResponseMutex.Lock() + defer fake.sendRoomMovedResponseMutex.Unlock() + fake.SendRoomMovedResponseStub = stub } -func (fake *FakeLocalParticipant) SendRequestResponseArgsForCall(i int) *livekit.RequestResponse { - fake.sendRequestResponseMutex.RLock() - defer fake.sendRequestResponseMutex.RUnlock() - argsForCall := fake.sendRequestResponseArgsForCall[i] +func (fake *FakeLocalParticipant) SendRoomMovedResponseArgsForCall(i int) *livekit.RoomMovedResponse { + fake.sendRoomMovedResponseMutex.RLock() + defer fake.sendRoomMovedResponseMutex.RUnlock() + argsForCall := fake.sendRoomMovedResponseArgsForCall[i] return argsForCall.arg1 } -func (fake *FakeLocalParticipant) SendRequestResponseReturns(result1 error) { - fake.sendRequestResponseMutex.Lock() - defer fake.sendRequestResponseMutex.Unlock() - fake.SendRequestResponseStub = nil - fake.sendRequestResponseReturns = struct { +func (fake *FakeLocalParticipant) SendRoomMovedResponseReturns(result1 error) { + fake.sendRoomMovedResponseMutex.Lock() + defer fake.sendRoomMovedResponseMutex.Unlock() + fake.SendRoomMovedResponseStub = nil + fake.sendRoomMovedResponseReturns = struct { result1 error }{result1} } -func (fake *FakeLocalParticipant) SendRequestResponseReturnsOnCall(i int, result1 error) { - fake.sendRequestResponseMutex.Lock() - defer fake.sendRequestResponseMutex.Unlock() - fake.SendRequestResponseStub = nil - if fake.sendRequestResponseReturnsOnCall == nil { - fake.sendRequestResponseReturnsOnCall = make(map[int]struct { +func (fake *FakeLocalParticipant) SendRoomMovedResponseReturnsOnCall(i int, result1 error) { + fake.sendRoomMovedResponseMutex.Lock() + defer fake.sendRoomMovedResponseMutex.Unlock() + fake.SendRoomMovedResponseStub = nil + if fake.sendRoomMovedResponseReturnsOnCall == nil { + fake.sendRoomMovedResponseReturnsOnCall = make(map[int]struct { result1 error }) } - fake.sendRequestResponseReturnsOnCall[i] = struct { + fake.sendRoomMovedResponseReturnsOnCall[i] = struct { result1 error }{result1} } @@ -5740,6 +7422,69 @@ func (fake *FakeLocalParticipant) SendSpeakerUpdateReturnsOnCall(i int, result1 }{result1} } +func (fake *FakeLocalParticipant) SendSubscriptionPermissionUpdate(arg1 livekit.ParticipantID, arg2 livekit.TrackID, arg3 bool) error { + fake.sendSubscriptionPermissionUpdateMutex.Lock() + ret, specificReturn := fake.sendSubscriptionPermissionUpdateReturnsOnCall[len(fake.sendSubscriptionPermissionUpdateArgsForCall)] + fake.sendSubscriptionPermissionUpdateArgsForCall = append(fake.sendSubscriptionPermissionUpdateArgsForCall, struct { + arg1 livekit.ParticipantID + arg2 livekit.TrackID + arg3 bool + }{arg1, arg2, arg3}) + stub := fake.SendSubscriptionPermissionUpdateStub + fakeReturns := fake.sendSubscriptionPermissionUpdateReturns + fake.recordInvocation("SendSubscriptionPermissionUpdate", []interface{}{arg1, arg2, arg3}) + fake.sendSubscriptionPermissionUpdateMutex.Unlock() + if stub != nil { + return stub(arg1, arg2, arg3) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeLocalParticipant) SendSubscriptionPermissionUpdateCallCount() int { + fake.sendSubscriptionPermissionUpdateMutex.RLock() + defer fake.sendSubscriptionPermissionUpdateMutex.RUnlock() + return len(fake.sendSubscriptionPermissionUpdateArgsForCall) +} + +func (fake *FakeLocalParticipant) SendSubscriptionPermissionUpdateCalls(stub func(livekit.ParticipantID, livekit.TrackID, bool) error) { + fake.sendSubscriptionPermissionUpdateMutex.Lock() + defer fake.sendSubscriptionPermissionUpdateMutex.Unlock() + fake.SendSubscriptionPermissionUpdateStub = stub +} + +func (fake *FakeLocalParticipant) SendSubscriptionPermissionUpdateArgsForCall(i int) (livekit.ParticipantID, livekit.TrackID, bool) { + fake.sendSubscriptionPermissionUpdateMutex.RLock() + defer fake.sendSubscriptionPermissionUpdateMutex.RUnlock() + argsForCall := fake.sendSubscriptionPermissionUpdateArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 +} + +func (fake *FakeLocalParticipant) SendSubscriptionPermissionUpdateReturns(result1 error) { + fake.sendSubscriptionPermissionUpdateMutex.Lock() + defer fake.sendSubscriptionPermissionUpdateMutex.Unlock() + fake.SendSubscriptionPermissionUpdateStub = nil + fake.sendSubscriptionPermissionUpdateReturns = struct { + result1 error + }{result1} +} + +func (fake *FakeLocalParticipant) SendSubscriptionPermissionUpdateReturnsOnCall(i int, result1 error) { + fake.sendSubscriptionPermissionUpdateMutex.Lock() + defer fake.sendSubscriptionPermissionUpdateMutex.Unlock() + fake.SendSubscriptionPermissionUpdateStub = nil + if fake.sendSubscriptionPermissionUpdateReturnsOnCall == nil { + fake.sendSubscriptionPermissionUpdateReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.sendSubscriptionPermissionUpdateReturnsOnCall[i] = struct { + result1 error + }{result1} +} + func (fake *FakeLocalParticipant) SetAttributes(arg1 map[string]string) { fake.setAttributesMutex.Lock() fake.setAttributesArgsForCall = append(fake.setAttributesArgsForCall, struct { @@ -5836,7 +7581,7 @@ func (fake *FakeLocalParticipant) SetMetadataArgsForCall(i int) string { return argsForCall.arg1 } -func (fake *FakeLocalParticipant) SetMigrateInfo(arg1 *webrtc.SessionDescription, arg2 *webrtc.SessionDescription, arg3 []*livekit.TrackPublishedResponse, arg4 []*livekit.DataChannelInfo) { +func (fake *FakeLocalParticipant) SetMigrateInfo(arg1 *webrtc.SessionDescription, arg2 *webrtc.SessionDescription, arg3 []*livekit.TrackPublishedResponse, arg4 []*livekit.DataChannelInfo, arg5 []*livekit.DataChannelReceiveState, arg6 []*livekit.PublishDataTrackResponse) { var arg3Copy []*livekit.TrackPublishedResponse if arg3 != nil { arg3Copy = make([]*livekit.TrackPublishedResponse, len(arg3)) @@ -5847,18 +7592,30 @@ func (fake *FakeLocalParticipant) SetMigrateInfo(arg1 *webrtc.SessionDescription arg4Copy = make([]*livekit.DataChannelInfo, len(arg4)) copy(arg4Copy, arg4) } + var arg5Copy []*livekit.DataChannelReceiveState + if arg5 != nil { + arg5Copy = make([]*livekit.DataChannelReceiveState, len(arg5)) + copy(arg5Copy, arg5) + } + var arg6Copy []*livekit.PublishDataTrackResponse + if arg6 != nil { + arg6Copy = make([]*livekit.PublishDataTrackResponse, len(arg6)) + copy(arg6Copy, arg6) + } fake.setMigrateInfoMutex.Lock() fake.setMigrateInfoArgsForCall = append(fake.setMigrateInfoArgsForCall, struct { arg1 *webrtc.SessionDescription arg2 *webrtc.SessionDescription arg3 []*livekit.TrackPublishedResponse arg4 []*livekit.DataChannelInfo - }{arg1, arg2, arg3Copy, arg4Copy}) + arg5 []*livekit.DataChannelReceiveState + arg6 []*livekit.PublishDataTrackResponse + }{arg1, arg2, arg3Copy, arg4Copy, arg5Copy, arg6Copy}) stub := fake.SetMigrateInfoStub - fake.recordInvocation("SetMigrateInfo", []interface{}{arg1, arg2, arg3Copy, arg4Copy}) + fake.recordInvocation("SetMigrateInfo", []interface{}{arg1, arg2, arg3Copy, arg4Copy, arg5Copy, arg6Copy}) fake.setMigrateInfoMutex.Unlock() if stub != nil { - fake.SetMigrateInfoStub(arg1, arg2, arg3, arg4) + fake.SetMigrateInfoStub(arg1, arg2, arg3, arg4, arg5, arg6) } } @@ -5868,17 +7625,17 @@ func (fake *FakeLocalParticipant) SetMigrateInfoCallCount() int { return len(fake.setMigrateInfoArgsForCall) } -func (fake *FakeLocalParticipant) SetMigrateInfoCalls(stub func(*webrtc.SessionDescription, *webrtc.SessionDescription, []*livekit.TrackPublishedResponse, []*livekit.DataChannelInfo)) { +func (fake *FakeLocalParticipant) SetMigrateInfoCalls(stub func(*webrtc.SessionDescription, *webrtc.SessionDescription, []*livekit.TrackPublishedResponse, []*livekit.DataChannelInfo, []*livekit.DataChannelReceiveState, []*livekit.PublishDataTrackResponse)) { fake.setMigrateInfoMutex.Lock() defer fake.setMigrateInfoMutex.Unlock() fake.SetMigrateInfoStub = stub } -func (fake *FakeLocalParticipant) SetMigrateInfoArgsForCall(i int) (*webrtc.SessionDescription, *webrtc.SessionDescription, []*livekit.TrackPublishedResponse, []*livekit.DataChannelInfo) { +func (fake *FakeLocalParticipant) SetMigrateInfoArgsForCall(i int) (*webrtc.SessionDescription, *webrtc.SessionDescription, []*livekit.TrackPublishedResponse, []*livekit.DataChannelInfo, []*livekit.DataChannelReceiveState, []*livekit.PublishDataTrackResponse) { fake.setMigrateInfoMutex.RLock() defer fake.setMigrateInfoMutex.RUnlock() argsForCall := fake.setMigrateInfoArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4 + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4, argsForCall.arg5, argsForCall.arg6 } func (fake *FakeLocalParticipant) SetMigrateState(arg1 types.MigrateState) { @@ -6006,38 +7763,6 @@ func (fake *FakeLocalParticipant) SetPermissionReturnsOnCall(i int, result1 bool }{result1} } -func (fake *FakeLocalParticipant) SetResponseSink(arg1 routing.MessageSink) { - fake.setResponseSinkMutex.Lock() - fake.setResponseSinkArgsForCall = append(fake.setResponseSinkArgsForCall, struct { - arg1 routing.MessageSink - }{arg1}) - stub := fake.SetResponseSinkStub - fake.recordInvocation("SetResponseSink", []interface{}{arg1}) - fake.setResponseSinkMutex.Unlock() - if stub != nil { - fake.SetResponseSinkStub(arg1) - } -} - -func (fake *FakeLocalParticipant) SetResponseSinkCallCount() int { - fake.setResponseSinkMutex.RLock() - defer fake.setResponseSinkMutex.RUnlock() - return len(fake.setResponseSinkArgsForCall) -} - -func (fake *FakeLocalParticipant) SetResponseSinkCalls(stub func(routing.MessageSink)) { - fake.setResponseSinkMutex.Lock() - defer fake.setResponseSinkMutex.Unlock() - fake.SetResponseSinkStub = stub -} - -func (fake *FakeLocalParticipant) SetResponseSinkArgsForCall(i int) routing.MessageSink { - fake.setResponseSinkMutex.RLock() - defer fake.setResponseSinkMutex.RUnlock() - argsForCall := fake.setResponseSinkArgsForCall[i] - return argsForCall.arg1 -} - func (fake *FakeLocalParticipant) SetSignalSourceValid(arg1 bool) { fake.setSignalSourceValidMutex.Lock() fake.setSignalSourceValidArgsForCall = append(fake.setSignalSourceValidArgsForCall, struct { @@ -6134,20 +7859,19 @@ func (fake *FakeLocalParticipant) SetSubscriberChannelCapacityArgsForCall(i int) return argsForCall.arg1 } -func (fake *FakeLocalParticipant) SetTrackMuted(arg1 livekit.TrackID, arg2 bool, arg3 bool) *livekit.TrackInfo { +func (fake *FakeLocalParticipant) SetTrackMuted(arg1 *livekit.MuteTrackRequest, arg2 bool) *livekit.TrackInfo { fake.setTrackMutedMutex.Lock() ret, specificReturn := fake.setTrackMutedReturnsOnCall[len(fake.setTrackMutedArgsForCall)] fake.setTrackMutedArgsForCall = append(fake.setTrackMutedArgsForCall, struct { - arg1 livekit.TrackID + arg1 *livekit.MuteTrackRequest arg2 bool - arg3 bool - }{arg1, arg2, arg3}) + }{arg1, arg2}) stub := fake.SetTrackMutedStub fakeReturns := fake.setTrackMutedReturns - fake.recordInvocation("SetTrackMuted", []interface{}{arg1, arg2, arg3}) + fake.recordInvocation("SetTrackMuted", []interface{}{arg1, arg2}) fake.setTrackMutedMutex.Unlock() if stub != nil { - return stub(arg1, arg2, arg3) + return stub(arg1, arg2) } if specificReturn { return ret.result1 @@ -6161,17 +7885,17 @@ func (fake *FakeLocalParticipant) SetTrackMutedCallCount() int { return len(fake.setTrackMutedArgsForCall) } -func (fake *FakeLocalParticipant) SetTrackMutedCalls(stub func(livekit.TrackID, bool, bool) *livekit.TrackInfo) { +func (fake *FakeLocalParticipant) SetTrackMutedCalls(stub func(*livekit.MuteTrackRequest, bool) *livekit.TrackInfo) { fake.setTrackMutedMutex.Lock() defer fake.setTrackMutedMutex.Unlock() fake.SetTrackMutedStub = stub } -func (fake *FakeLocalParticipant) SetTrackMutedArgsForCall(i int) (livekit.TrackID, bool, bool) { +func (fake *FakeLocalParticipant) SetTrackMutedArgsForCall(i int) (*livekit.MuteTrackRequest, bool) { fake.setTrackMutedMutex.RLock() defer fake.setTrackMutedMutex.RUnlock() argsForCall := fake.setTrackMutedArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 + return argsForCall.arg1, argsForCall.arg2 } func (fake *FakeLocalParticipant) SetTrackMutedReturns(result1 *livekit.TrackInfo) { @@ -6303,16 +8027,49 @@ func (fake *FakeLocalParticipant) StopAndGetSubscribedTracksForwarderStateReturn }{result1} } -func (fake *FakeLocalParticipant) SubscribeToTrack(arg1 livekit.TrackID) { +func (fake *FakeLocalParticipant) SubscribeToDataTrack(arg1 livekit.TrackID) { + fake.subscribeToDataTrackMutex.Lock() + fake.subscribeToDataTrackArgsForCall = append(fake.subscribeToDataTrackArgsForCall, struct { + arg1 livekit.TrackID + }{arg1}) + stub := fake.SubscribeToDataTrackStub + fake.recordInvocation("SubscribeToDataTrack", []interface{}{arg1}) + fake.subscribeToDataTrackMutex.Unlock() + if stub != nil { + fake.SubscribeToDataTrackStub(arg1) + } +} + +func (fake *FakeLocalParticipant) SubscribeToDataTrackCallCount() int { + fake.subscribeToDataTrackMutex.RLock() + defer fake.subscribeToDataTrackMutex.RUnlock() + return len(fake.subscribeToDataTrackArgsForCall) +} + +func (fake *FakeLocalParticipant) SubscribeToDataTrackCalls(stub func(livekit.TrackID)) { + fake.subscribeToDataTrackMutex.Lock() + defer fake.subscribeToDataTrackMutex.Unlock() + fake.SubscribeToDataTrackStub = stub +} + +func (fake *FakeLocalParticipant) SubscribeToDataTrackArgsForCall(i int) livekit.TrackID { + fake.subscribeToDataTrackMutex.RLock() + defer fake.subscribeToDataTrackMutex.RUnlock() + argsForCall := fake.subscribeToDataTrackArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeLocalParticipant) SubscribeToTrack(arg1 livekit.TrackID, arg2 bool) { fake.subscribeToTrackMutex.Lock() fake.subscribeToTrackArgsForCall = append(fake.subscribeToTrackArgsForCall, struct { arg1 livekit.TrackID - }{arg1}) + arg2 bool + }{arg1, arg2}) stub := fake.SubscribeToTrackStub - fake.recordInvocation("SubscribeToTrack", []interface{}{arg1}) + fake.recordInvocation("SubscribeToTrack", []interface{}{arg1, arg2}) fake.subscribeToTrackMutex.Unlock() if stub != nil { - fake.SubscribeToTrackStub(arg1) + fake.SubscribeToTrackStub(arg1, arg2) } } @@ -6322,17 +8079,17 @@ func (fake *FakeLocalParticipant) SubscribeToTrackCallCount() int { return len(fake.subscribeToTrackArgsForCall) } -func (fake *FakeLocalParticipant) SubscribeToTrackCalls(stub func(livekit.TrackID)) { +func (fake *FakeLocalParticipant) SubscribeToTrackCalls(stub func(livekit.TrackID, bool)) { fake.subscribeToTrackMutex.Lock() defer fake.subscribeToTrackMutex.Unlock() fake.SubscribeToTrackStub = stub } -func (fake *FakeLocalParticipant) SubscribeToTrackArgsForCall(i int) livekit.TrackID { +func (fake *FakeLocalParticipant) SubscribeToTrackArgsForCall(i int) (livekit.TrackID, bool) { fake.subscribeToTrackMutex.RLock() defer fake.subscribeToTrackMutex.RUnlock() argsForCall := fake.subscribeToTrackArgsForCall[i] - return argsForCall.arg1 + return argsForCall.arg1, argsForCall.arg2 } func (fake *FakeLocalParticipant) SubscriberAsPrimary() bool { @@ -6444,40 +8201,6 @@ func (fake *FakeLocalParticipant) SubscriptionPermissionReturnsOnCall(i int, res }{result1, result2} } -func (fake *FakeLocalParticipant) SubscriptionPermissionUpdate(arg1 livekit.ParticipantID, arg2 livekit.TrackID, arg3 bool) { - fake.subscriptionPermissionUpdateMutex.Lock() - fake.subscriptionPermissionUpdateArgsForCall = append(fake.subscriptionPermissionUpdateArgsForCall, struct { - arg1 livekit.ParticipantID - arg2 livekit.TrackID - arg3 bool - }{arg1, arg2, arg3}) - stub := fake.SubscriptionPermissionUpdateStub - fake.recordInvocation("SubscriptionPermissionUpdate", []interface{}{arg1, arg2, arg3}) - fake.subscriptionPermissionUpdateMutex.Unlock() - if stub != nil { - fake.SubscriptionPermissionUpdateStub(arg1, arg2, arg3) - } -} - -func (fake *FakeLocalParticipant) SubscriptionPermissionUpdateCallCount() int { - fake.subscriptionPermissionUpdateMutex.RLock() - defer fake.subscriptionPermissionUpdateMutex.RUnlock() - return len(fake.subscriptionPermissionUpdateArgsForCall) -} - -func (fake *FakeLocalParticipant) SubscriptionPermissionUpdateCalls(stub func(livekit.ParticipantID, livekit.TrackID, bool)) { - fake.subscriptionPermissionUpdateMutex.Lock() - defer fake.subscriptionPermissionUpdateMutex.Unlock() - fake.SubscriptionPermissionUpdateStub = stub -} - -func (fake *FakeLocalParticipant) SubscriptionPermissionUpdateArgsForCall(i int) (livekit.ParticipantID, livekit.TrackID, bool) { - fake.subscriptionPermissionUpdateMutex.RLock() - defer fake.subscriptionPermissionUpdateMutex.RUnlock() - argsForCall := fake.subscriptionPermissionUpdateArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 -} - func (fake *FakeLocalParticipant) SupportsCodecChange() bool { fake.supportsCodecChangeMutex.Lock() ret, specificReturn := fake.supportsCodecChangeReturnsOnCall[len(fake.supportsCodecChangeArgsForCall)] @@ -6531,6 +8254,59 @@ func (fake *FakeLocalParticipant) SupportsCodecChangeReturnsOnCall(i int, result }{result1} } +func (fake *FakeLocalParticipant) SupportsMoving() error { + fake.supportsMovingMutex.Lock() + ret, specificReturn := fake.supportsMovingReturnsOnCall[len(fake.supportsMovingArgsForCall)] + fake.supportsMovingArgsForCall = append(fake.supportsMovingArgsForCall, struct { + }{}) + stub := fake.SupportsMovingStub + fakeReturns := fake.supportsMovingReturns + fake.recordInvocation("SupportsMoving", []interface{}{}) + fake.supportsMovingMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeLocalParticipant) SupportsMovingCallCount() int { + fake.supportsMovingMutex.RLock() + defer fake.supportsMovingMutex.RUnlock() + return len(fake.supportsMovingArgsForCall) +} + +func (fake *FakeLocalParticipant) SupportsMovingCalls(stub func() error) { + fake.supportsMovingMutex.Lock() + defer fake.supportsMovingMutex.Unlock() + fake.SupportsMovingStub = stub +} + +func (fake *FakeLocalParticipant) SupportsMovingReturns(result1 error) { + fake.supportsMovingMutex.Lock() + defer fake.supportsMovingMutex.Unlock() + fake.SupportsMovingStub = nil + fake.supportsMovingReturns = struct { + result1 error + }{result1} +} + +func (fake *FakeLocalParticipant) SupportsMovingReturnsOnCall(i int, result1 error) { + fake.supportsMovingMutex.Lock() + defer fake.supportsMovingMutex.Unlock() + fake.SupportsMovingStub = nil + if fake.supportsMovingReturnsOnCall == nil { + fake.supportsMovingReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.supportsMovingReturnsOnCall[i] = struct { + result1 error + }{result1} +} + func (fake *FakeLocalParticipant) SupportsSyncStreamID() bool { fake.supportsSyncStreamIDMutex.Lock() ret, specificReturn := fake.supportsSyncStreamIDReturnsOnCall[len(fake.supportsSyncStreamIDArgsForCall)] @@ -6584,17 +8360,18 @@ func (fake *FakeLocalParticipant) SupportsSyncStreamIDReturnsOnCall(i int, resul }{result1} } -func (fake *FakeLocalParticipant) SupportsTransceiverReuse() bool { +func (fake *FakeLocalParticipant) SupportsTransceiverReuse(arg1 types.MediaTrack) bool { fake.supportsTransceiverReuseMutex.Lock() ret, specificReturn := fake.supportsTransceiverReuseReturnsOnCall[len(fake.supportsTransceiverReuseArgsForCall)] fake.supportsTransceiverReuseArgsForCall = append(fake.supportsTransceiverReuseArgsForCall, struct { - }{}) + arg1 types.MediaTrack + }{arg1}) stub := fake.SupportsTransceiverReuseStub fakeReturns := fake.supportsTransceiverReuseReturns - fake.recordInvocation("SupportsTransceiverReuse", []interface{}{}) + fake.recordInvocation("SupportsTransceiverReuse", []interface{}{arg1}) fake.supportsTransceiverReuseMutex.Unlock() if stub != nil { - return stub() + return stub(arg1) } if specificReturn { return ret.result1 @@ -6608,12 +8385,19 @@ func (fake *FakeLocalParticipant) SupportsTransceiverReuseCallCount() int { return len(fake.supportsTransceiverReuseArgsForCall) } -func (fake *FakeLocalParticipant) SupportsTransceiverReuseCalls(stub func() bool) { +func (fake *FakeLocalParticipant) SupportsTransceiverReuseCalls(stub func(types.MediaTrack) bool) { fake.supportsTransceiverReuseMutex.Lock() defer fake.supportsTransceiverReuseMutex.Unlock() fake.SupportsTransceiverReuseStub = stub } +func (fake *FakeLocalParticipant) SupportsTransceiverReuseArgsForCall(i int) types.MediaTrack { + fake.supportsTransceiverReuseMutex.RLock() + defer fake.supportsTransceiverReuseMutex.RUnlock() + argsForCall := fake.supportsTransceiverReuseArgsForCall[i] + return argsForCall.arg1 +} + func (fake *FakeLocalParticipant) SupportsTransceiverReuseReturns(result1 bool) { fake.supportsTransceiverReuseMutex.Lock() defer fake.supportsTransceiverReuseMutex.Unlock() @@ -6637,6 +8421,92 @@ func (fake *FakeLocalParticipant) SupportsTransceiverReuseReturnsOnCall(i int, r }{result1} } +func (fake *FakeLocalParticipant) SwapResponseSink(arg1 routing.MessageSink, arg2 types.SignallingCloseReason) { + fake.swapResponseSinkMutex.Lock() + fake.swapResponseSinkArgsForCall = append(fake.swapResponseSinkArgsForCall, struct { + arg1 routing.MessageSink + arg2 types.SignallingCloseReason + }{arg1, arg2}) + stub := fake.SwapResponseSinkStub + fake.recordInvocation("SwapResponseSink", []interface{}{arg1, arg2}) + fake.swapResponseSinkMutex.Unlock() + if stub != nil { + fake.SwapResponseSinkStub(arg1, arg2) + } +} + +func (fake *FakeLocalParticipant) SwapResponseSinkCallCount() int { + fake.swapResponseSinkMutex.RLock() + defer fake.swapResponseSinkMutex.RUnlock() + return len(fake.swapResponseSinkArgsForCall) +} + +func (fake *FakeLocalParticipant) SwapResponseSinkCalls(stub func(routing.MessageSink, types.SignallingCloseReason)) { + fake.swapResponseSinkMutex.Lock() + defer fake.swapResponseSinkMutex.Unlock() + fake.SwapResponseSinkStub = stub +} + +func (fake *FakeLocalParticipant) SwapResponseSinkArgsForCall(i int) (routing.MessageSink, types.SignallingCloseReason) { + fake.swapResponseSinkMutex.RLock() + defer fake.swapResponseSinkMutex.RUnlock() + argsForCall := fake.swapResponseSinkArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeLocalParticipant) TelemetryGuard() *telemetry.ReferenceGuard { + fake.telemetryGuardMutex.Lock() + ret, specificReturn := fake.telemetryGuardReturnsOnCall[len(fake.telemetryGuardArgsForCall)] + fake.telemetryGuardArgsForCall = append(fake.telemetryGuardArgsForCall, struct { + }{}) + stub := fake.TelemetryGuardStub + fakeReturns := fake.telemetryGuardReturns + fake.recordInvocation("TelemetryGuard", []interface{}{}) + fake.telemetryGuardMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeLocalParticipant) TelemetryGuardCallCount() int { + fake.telemetryGuardMutex.RLock() + defer fake.telemetryGuardMutex.RUnlock() + return len(fake.telemetryGuardArgsForCall) +} + +func (fake *FakeLocalParticipant) TelemetryGuardCalls(stub func() *telemetry.ReferenceGuard) { + fake.telemetryGuardMutex.Lock() + defer fake.telemetryGuardMutex.Unlock() + fake.TelemetryGuardStub = stub +} + +func (fake *FakeLocalParticipant) TelemetryGuardReturns(result1 *telemetry.ReferenceGuard) { + fake.telemetryGuardMutex.Lock() + defer fake.telemetryGuardMutex.Unlock() + fake.TelemetryGuardStub = nil + fake.telemetryGuardReturns = struct { + result1 *telemetry.ReferenceGuard + }{result1} +} + +func (fake *FakeLocalParticipant) TelemetryGuardReturnsOnCall(i int, result1 *telemetry.ReferenceGuard) { + fake.telemetryGuardMutex.Lock() + defer fake.telemetryGuardMutex.Unlock() + fake.TelemetryGuardStub = nil + if fake.telemetryGuardReturnsOnCall == nil { + fake.telemetryGuardReturnsOnCall = make(map[int]struct { + result1 *telemetry.ReferenceGuard + }) + } + fake.telemetryGuardReturnsOnCall[i] = struct { + result1 *telemetry.ReferenceGuard + }{result1} +} + func (fake *FakeLocalParticipant) ToProto() *livekit.ParticipantInfo { fake.toProtoMutex.Lock() ret, specificReturn := fake.toProtoReturnsOnCall[len(fake.toProtoArgsForCall)] @@ -6778,6 +8648,38 @@ func (fake *FakeLocalParticipant) UncacheDownTrackArgsForCall(i int) *webrtc.RTP return argsForCall.arg1 } +func (fake *FakeLocalParticipant) UnsubscribeFromDataTrack(arg1 livekit.TrackID) { + fake.unsubscribeFromDataTrackMutex.Lock() + fake.unsubscribeFromDataTrackArgsForCall = append(fake.unsubscribeFromDataTrackArgsForCall, struct { + arg1 livekit.TrackID + }{arg1}) + stub := fake.UnsubscribeFromDataTrackStub + fake.recordInvocation("UnsubscribeFromDataTrack", []interface{}{arg1}) + fake.unsubscribeFromDataTrackMutex.Unlock() + if stub != nil { + fake.UnsubscribeFromDataTrackStub(arg1) + } +} + +func (fake *FakeLocalParticipant) UnsubscribeFromDataTrackCallCount() int { + fake.unsubscribeFromDataTrackMutex.RLock() + defer fake.unsubscribeFromDataTrackMutex.RUnlock() + return len(fake.unsubscribeFromDataTrackArgsForCall) +} + +func (fake *FakeLocalParticipant) UnsubscribeFromDataTrackCalls(stub func(livekit.TrackID)) { + fake.unsubscribeFromDataTrackMutex.Lock() + defer fake.unsubscribeFromDataTrackMutex.Unlock() + fake.UnsubscribeFromDataTrackStub = stub +} + +func (fake *FakeLocalParticipant) UnsubscribeFromDataTrackArgsForCall(i int) livekit.TrackID { + fake.unsubscribeFromDataTrackMutex.RLock() + defer fake.unsubscribeFromDataTrackMutex.RUnlock() + argsForCall := fake.unsubscribeFromDataTrackArgsForCall[i] + return argsForCall.arg1 +} + func (fake *FakeLocalParticipant) UnsubscribeFromTrack(arg1 livekit.TrackID) { fake.unsubscribeFromTrackMutex.Lock() fake.unsubscribeFromTrackArgsForCall = append(fake.unsubscribeFromTrackArgsForCall, struct { @@ -6871,6 +8773,39 @@ func (fake *FakeLocalParticipant) UpdateAudioTrackReturnsOnCall(i int, result1 e }{result1} } +func (fake *FakeLocalParticipant) UpdateDataTrackSubscriptionOptions(arg1 livekit.TrackID, arg2 *livekit.DataTrackSubscriptionOptions) { + fake.updateDataTrackSubscriptionOptionsMutex.Lock() + fake.updateDataTrackSubscriptionOptionsArgsForCall = append(fake.updateDataTrackSubscriptionOptionsArgsForCall, struct { + arg1 livekit.TrackID + arg2 *livekit.DataTrackSubscriptionOptions + }{arg1, arg2}) + stub := fake.UpdateDataTrackSubscriptionOptionsStub + fake.recordInvocation("UpdateDataTrackSubscriptionOptions", []interface{}{arg1, arg2}) + fake.updateDataTrackSubscriptionOptionsMutex.Unlock() + if stub != nil { + fake.UpdateDataTrackSubscriptionOptionsStub(arg1, arg2) + } +} + +func (fake *FakeLocalParticipant) UpdateDataTrackSubscriptionOptionsCallCount() int { + fake.updateDataTrackSubscriptionOptionsMutex.RLock() + defer fake.updateDataTrackSubscriptionOptionsMutex.RUnlock() + return len(fake.updateDataTrackSubscriptionOptionsArgsForCall) +} + +func (fake *FakeLocalParticipant) UpdateDataTrackSubscriptionOptionsCalls(stub func(livekit.TrackID, *livekit.DataTrackSubscriptionOptions)) { + fake.updateDataTrackSubscriptionOptionsMutex.Lock() + defer fake.updateDataTrackSubscriptionOptionsMutex.Unlock() + fake.UpdateDataTrackSubscriptionOptionsStub = stub +} + +func (fake *FakeLocalParticipant) UpdateDataTrackSubscriptionOptionsArgsForCall(i int) (livekit.TrackID, *livekit.DataTrackSubscriptionOptions) { + fake.updateDataTrackSubscriptionOptionsMutex.RLock() + defer fake.updateDataTrackSubscriptionOptionsMutex.RUnlock() + argsForCall := fake.updateDataTrackSubscriptionOptionsArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + func (fake *FakeLocalParticipant) UpdateLastSeenSignal() { fake.updateLastSeenSignalMutex.Lock() fake.updateLastSeenSignalArgsForCall = append(fake.updateLastSeenSignalArgsForCall, struct { @@ -6990,6 +8925,68 @@ func (fake *FakeLocalParticipant) UpdateMediaRTTArgsForCall(i int) uint32 { return argsForCall.arg1 } +func (fake *FakeLocalParticipant) UpdateMetadata(arg1 *livekit.UpdateParticipantMetadata, arg2 bool) error { + fake.updateMetadataMutex.Lock() + ret, specificReturn := fake.updateMetadataReturnsOnCall[len(fake.updateMetadataArgsForCall)] + fake.updateMetadataArgsForCall = append(fake.updateMetadataArgsForCall, struct { + arg1 *livekit.UpdateParticipantMetadata + arg2 bool + }{arg1, arg2}) + stub := fake.UpdateMetadataStub + fakeReturns := fake.updateMetadataReturns + fake.recordInvocation("UpdateMetadata", []interface{}{arg1, arg2}) + fake.updateMetadataMutex.Unlock() + if stub != nil { + return stub(arg1, arg2) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeLocalParticipant) UpdateMetadataCallCount() int { + fake.updateMetadataMutex.RLock() + defer fake.updateMetadataMutex.RUnlock() + return len(fake.updateMetadataArgsForCall) +} + +func (fake *FakeLocalParticipant) UpdateMetadataCalls(stub func(*livekit.UpdateParticipantMetadata, bool) error) { + fake.updateMetadataMutex.Lock() + defer fake.updateMetadataMutex.Unlock() + fake.UpdateMetadataStub = stub +} + +func (fake *FakeLocalParticipant) UpdateMetadataArgsForCall(i int) (*livekit.UpdateParticipantMetadata, bool) { + fake.updateMetadataMutex.RLock() + defer fake.updateMetadataMutex.RUnlock() + argsForCall := fake.updateMetadataArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeLocalParticipant) UpdateMetadataReturns(result1 error) { + fake.updateMetadataMutex.Lock() + defer fake.updateMetadataMutex.Unlock() + fake.UpdateMetadataStub = nil + fake.updateMetadataReturns = struct { + result1 error + }{result1} +} + +func (fake *FakeLocalParticipant) UpdateMetadataReturnsOnCall(i int, result1 error) { + fake.updateMetadataMutex.Lock() + defer fake.updateMetadataMutex.Unlock() + fake.UpdateMetadataStub = nil + if fake.updateMetadataReturnsOnCall == nil { + fake.updateMetadataReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.updateMetadataReturnsOnCall[i] = struct { + result1 error + }{result1} +} + func (fake *FakeLocalParticipant) UpdateSignalingRTT(arg1 uint32) { fake.updateSignalingRTTMutex.Lock() fake.updateSignalingRTTArgsForCall = append(fake.updateSignalingRTTArgsForCall, struct { @@ -7022,6 +9019,74 @@ func (fake *FakeLocalParticipant) UpdateSignalingRTTArgsForCall(i int) uint32 { return argsForCall.arg1 } +func (fake *FakeLocalParticipant) UpdateSubscribedAudioCodecs(arg1 livekit.NodeID, arg2 livekit.TrackID, arg3 []*livekit.SubscribedAudioCodec) error { + var arg3Copy []*livekit.SubscribedAudioCodec + if arg3 != nil { + arg3Copy = make([]*livekit.SubscribedAudioCodec, len(arg3)) + copy(arg3Copy, arg3) + } + fake.updateSubscribedAudioCodecsMutex.Lock() + ret, specificReturn := fake.updateSubscribedAudioCodecsReturnsOnCall[len(fake.updateSubscribedAudioCodecsArgsForCall)] + fake.updateSubscribedAudioCodecsArgsForCall = append(fake.updateSubscribedAudioCodecsArgsForCall, struct { + arg1 livekit.NodeID + arg2 livekit.TrackID + arg3 []*livekit.SubscribedAudioCodec + }{arg1, arg2, arg3Copy}) + stub := fake.UpdateSubscribedAudioCodecsStub + fakeReturns := fake.updateSubscribedAudioCodecsReturns + fake.recordInvocation("UpdateSubscribedAudioCodecs", []interface{}{arg1, arg2, arg3Copy}) + fake.updateSubscribedAudioCodecsMutex.Unlock() + if stub != nil { + return stub(arg1, arg2, arg3) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeLocalParticipant) UpdateSubscribedAudioCodecsCallCount() int { + fake.updateSubscribedAudioCodecsMutex.RLock() + defer fake.updateSubscribedAudioCodecsMutex.RUnlock() + return len(fake.updateSubscribedAudioCodecsArgsForCall) +} + +func (fake *FakeLocalParticipant) UpdateSubscribedAudioCodecsCalls(stub func(livekit.NodeID, livekit.TrackID, []*livekit.SubscribedAudioCodec) error) { + fake.updateSubscribedAudioCodecsMutex.Lock() + defer fake.updateSubscribedAudioCodecsMutex.Unlock() + fake.UpdateSubscribedAudioCodecsStub = stub +} + +func (fake *FakeLocalParticipant) UpdateSubscribedAudioCodecsArgsForCall(i int) (livekit.NodeID, livekit.TrackID, []*livekit.SubscribedAudioCodec) { + fake.updateSubscribedAudioCodecsMutex.RLock() + defer fake.updateSubscribedAudioCodecsMutex.RUnlock() + argsForCall := fake.updateSubscribedAudioCodecsArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 +} + +func (fake *FakeLocalParticipant) UpdateSubscribedAudioCodecsReturns(result1 error) { + fake.updateSubscribedAudioCodecsMutex.Lock() + defer fake.updateSubscribedAudioCodecsMutex.Unlock() + fake.UpdateSubscribedAudioCodecsStub = nil + fake.updateSubscribedAudioCodecsReturns = struct { + result1 error + }{result1} +} + +func (fake *FakeLocalParticipant) UpdateSubscribedAudioCodecsReturnsOnCall(i int, result1 error) { + fake.updateSubscribedAudioCodecsMutex.Lock() + defer fake.updateSubscribedAudioCodecsMutex.Unlock() + fake.UpdateSubscribedAudioCodecsStub = nil + if fake.updateSubscribedAudioCodecsReturnsOnCall == nil { + fake.updateSubscribedAudioCodecsReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.updateSubscribedAudioCodecsReturnsOnCall[i] = struct { + result1 error + }{result1} +} + func (fake *FakeLocalParticipant) UpdateSubscribedQuality(arg1 livekit.NodeID, arg2 livekit.TrackID, arg3 []types.SubscribedCodecQuality) error { var arg3Copy []types.SubscribedCodecQuality if arg3 != nil { @@ -7516,266 +9581,6 @@ func (fake *FakeLocalParticipant) WriteSubscriberRTCPReturnsOnCall(i int, result func (fake *FakeLocalParticipant) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() - fake.addICECandidateMutex.RLock() - defer fake.addICECandidateMutex.RUnlock() - fake.addTrackMutex.RLock() - defer fake.addTrackMutex.RUnlock() - fake.addTrackLocalMutex.RLock() - defer fake.addTrackLocalMutex.RUnlock() - fake.addTransceiverFromTrackLocalMutex.RLock() - defer fake.addTransceiverFromTrackLocalMutex.RUnlock() - fake.cacheDownTrackMutex.RLock() - defer fake.cacheDownTrackMutex.RUnlock() - fake.canPublishMutex.RLock() - defer fake.canPublishMutex.RUnlock() - fake.canPublishDataMutex.RLock() - defer fake.canPublishDataMutex.RUnlock() - fake.canPublishSourceMutex.RLock() - defer fake.canPublishSourceMutex.RUnlock() - fake.canSkipBroadcastMutex.RLock() - defer fake.canSkipBroadcastMutex.RUnlock() - fake.canSubscribeMutex.RLock() - defer fake.canSubscribeMutex.RUnlock() - fake.checkMetadataLimitsMutex.RLock() - defer fake.checkMetadataLimitsMutex.RUnlock() - fake.claimGrantsMutex.RLock() - defer fake.claimGrantsMutex.RUnlock() - fake.closeMutex.RLock() - defer fake.closeMutex.RUnlock() - fake.closeReasonMutex.RLock() - defer fake.closeReasonMutex.RUnlock() - fake.closeSignalConnectionMutex.RLock() - defer fake.closeSignalConnectionMutex.RUnlock() - fake.connectedAtMutex.RLock() - defer fake.connectedAtMutex.RUnlock() - fake.debugInfoMutex.RLock() - defer fake.debugInfoMutex.RUnlock() - fake.disconnectedMutex.RLock() - defer fake.disconnectedMutex.RUnlock() - fake.getAdaptiveStreamMutex.RLock() - defer fake.getAdaptiveStreamMutex.RUnlock() - fake.getAnswerMutex.RLock() - defer fake.getAnswerMutex.RUnlock() - fake.getAudioLevelMutex.RLock() - defer fake.getAudioLevelMutex.RUnlock() - fake.getBufferFactoryMutex.RLock() - defer fake.getBufferFactoryMutex.RUnlock() - fake.getCachedDownTrackMutex.RLock() - defer fake.getCachedDownTrackMutex.RUnlock() - fake.getClientConfigurationMutex.RLock() - defer fake.getClientConfigurationMutex.RUnlock() - fake.getClientInfoMutex.RLock() - defer fake.getClientInfoMutex.RUnlock() - fake.getConnectionQualityMutex.RLock() - defer fake.getConnectionQualityMutex.RUnlock() - fake.getDisableSenderReportPassThroughMutex.RLock() - defer fake.getDisableSenderReportPassThroughMutex.RUnlock() - fake.getEnabledPublishCodecsMutex.RLock() - defer fake.getEnabledPublishCodecsMutex.RUnlock() - fake.getICEConfigMutex.RLock() - defer fake.getICEConfigMutex.RUnlock() - fake.getICEConnectionInfoMutex.RLock() - defer fake.getICEConnectionInfoMutex.RUnlock() - fake.getLoggerMutex.RLock() - defer fake.getLoggerMutex.RUnlock() - fake.getPacerMutex.RLock() - defer fake.getPacerMutex.RUnlock() - fake.getPendingTrackMutex.RLock() - defer fake.getPendingTrackMutex.RUnlock() - fake.getPlayoutDelayConfigMutex.RLock() - defer fake.getPlayoutDelayConfigMutex.RUnlock() - fake.getPublishedTrackMutex.RLock() - defer fake.getPublishedTrackMutex.RUnlock() - fake.getPublishedTracksMutex.RLock() - defer fake.getPublishedTracksMutex.RUnlock() - fake.getSubscribedParticipantsMutex.RLock() - defer fake.getSubscribedParticipantsMutex.RUnlock() - fake.getSubscribedTracksMutex.RLock() - defer fake.getSubscribedTracksMutex.RUnlock() - fake.getTrailerMutex.RLock() - defer fake.getTrailerMutex.RUnlock() - fake.handleAnswerMutex.RLock() - defer fake.handleAnswerMutex.RUnlock() - fake.handleMetricsMutex.RLock() - defer fake.handleMetricsMutex.RUnlock() - fake.handleOfferMutex.RLock() - defer fake.handleOfferMutex.RUnlock() - fake.handleReceiverReportMutex.RLock() - defer fake.handleReceiverReportMutex.RUnlock() - fake.handleReconnectAndSendResponseMutex.RLock() - defer fake.handleReconnectAndSendResponseMutex.RUnlock() - fake.handleSignalSourceCloseMutex.RLock() - defer fake.handleSignalSourceCloseMutex.RUnlock() - fake.hasConnectedMutex.RLock() - defer fake.hasConnectedMutex.RUnlock() - fake.hasPermissionMutex.RLock() - defer fake.hasPermissionMutex.RUnlock() - fake.hiddenMutex.RLock() - defer fake.hiddenMutex.RUnlock() - fake.iCERestartMutex.RLock() - defer fake.iCERestartMutex.RUnlock() - fake.iDMutex.RLock() - defer fake.iDMutex.RUnlock() - fake.identityMutex.RLock() - defer fake.identityMutex.RUnlock() - fake.isAgentMutex.RLock() - defer fake.isAgentMutex.RUnlock() - fake.isClosedMutex.RLock() - defer fake.isClosedMutex.RUnlock() - fake.isDependentMutex.RLock() - defer fake.isDependentMutex.RUnlock() - fake.isDisconnectedMutex.RLock() - defer fake.isDisconnectedMutex.RUnlock() - fake.isIdleMutex.RLock() - defer fake.isIdleMutex.RUnlock() - fake.isPublisherMutex.RLock() - defer fake.isPublisherMutex.RUnlock() - fake.isReadyMutex.RLock() - defer fake.isReadyMutex.RUnlock() - fake.isReconnectMutex.RLock() - defer fake.isReconnectMutex.RUnlock() - fake.isRecorderMutex.RLock() - defer fake.isRecorderMutex.RUnlock() - fake.isSubscribedToMutex.RLock() - defer fake.isSubscribedToMutex.RUnlock() - fake.isTrackNameSubscribedMutex.RLock() - defer fake.isTrackNameSubscribedMutex.RUnlock() - fake.issueFullReconnectMutex.RLock() - defer fake.issueFullReconnectMutex.RUnlock() - fake.kindMutex.RLock() - defer fake.kindMutex.RUnlock() - fake.maybeStartMigrationMutex.RLock() - defer fake.maybeStartMigrationMutex.RUnlock() - fake.migrateStateMutex.RLock() - defer fake.migrateStateMutex.RUnlock() - fake.negotiateMutex.RLock() - defer fake.negotiateMutex.RUnlock() - fake.notifyMigrationMutex.RLock() - defer fake.notifyMigrationMutex.RUnlock() - fake.onClaimsChangedMutex.RLock() - defer fake.onClaimsChangedMutex.RUnlock() - fake.onCloseMutex.RLock() - defer fake.onCloseMutex.RUnlock() - fake.onDataPacketMutex.RLock() - defer fake.onDataPacketMutex.RUnlock() - fake.onICEConfigChangedMutex.RLock() - defer fake.onICEConfigChangedMutex.RUnlock() - fake.onMetricsMutex.RLock() - defer fake.onMetricsMutex.RUnlock() - fake.onMigrateStateChangeMutex.RLock() - defer fake.onMigrateStateChangeMutex.RUnlock() - fake.onParticipantUpdateMutex.RLock() - defer fake.onParticipantUpdateMutex.RUnlock() - fake.onStateChangeMutex.RLock() - defer fake.onStateChangeMutex.RUnlock() - fake.onSubscribeStatusChangedMutex.RLock() - defer fake.onSubscribeStatusChangedMutex.RUnlock() - fake.onTrackPublishedMutex.RLock() - defer fake.onTrackPublishedMutex.RUnlock() - fake.onTrackUnpublishedMutex.RLock() - defer fake.onTrackUnpublishedMutex.RUnlock() - fake.onTrackUpdatedMutex.RLock() - defer fake.onTrackUpdatedMutex.RUnlock() - fake.protocolVersionMutex.RLock() - defer fake.protocolVersionMutex.RUnlock() - fake.removePublishedTrackMutex.RLock() - defer fake.removePublishedTrackMutex.RUnlock() - fake.removeTrackLocalMutex.RLock() - defer fake.removeTrackLocalMutex.RUnlock() - fake.sendConnectionQualityUpdateMutex.RLock() - defer fake.sendConnectionQualityUpdateMutex.RUnlock() - fake.sendDataPacketMutex.RLock() - defer fake.sendDataPacketMutex.RUnlock() - fake.sendJoinResponseMutex.RLock() - defer fake.sendJoinResponseMutex.RUnlock() - fake.sendParticipantUpdateMutex.RLock() - defer fake.sendParticipantUpdateMutex.RUnlock() - fake.sendRefreshTokenMutex.RLock() - defer fake.sendRefreshTokenMutex.RUnlock() - fake.sendRequestResponseMutex.RLock() - defer fake.sendRequestResponseMutex.RUnlock() - fake.sendRoomUpdateMutex.RLock() - defer fake.sendRoomUpdateMutex.RUnlock() - fake.sendSpeakerUpdateMutex.RLock() - defer fake.sendSpeakerUpdateMutex.RUnlock() - fake.setAttributesMutex.RLock() - defer fake.setAttributesMutex.RUnlock() - fake.setICEConfigMutex.RLock() - defer fake.setICEConfigMutex.RUnlock() - fake.setMetadataMutex.RLock() - defer fake.setMetadataMutex.RUnlock() - fake.setMigrateInfoMutex.RLock() - defer fake.setMigrateInfoMutex.RUnlock() - fake.setMigrateStateMutex.RLock() - defer fake.setMigrateStateMutex.RUnlock() - fake.setNameMutex.RLock() - defer fake.setNameMutex.RUnlock() - fake.setPermissionMutex.RLock() - defer fake.setPermissionMutex.RUnlock() - fake.setResponseSinkMutex.RLock() - defer fake.setResponseSinkMutex.RUnlock() - fake.setSignalSourceValidMutex.RLock() - defer fake.setSignalSourceValidMutex.RUnlock() - fake.setSubscriberAllowPauseMutex.RLock() - defer fake.setSubscriberAllowPauseMutex.RUnlock() - fake.setSubscriberChannelCapacityMutex.RLock() - defer fake.setSubscriberChannelCapacityMutex.RUnlock() - fake.setTrackMutedMutex.RLock() - defer fake.setTrackMutedMutex.RUnlock() - fake.stateMutex.RLock() - defer fake.stateMutex.RUnlock() - fake.stopAndGetSubscribedTracksForwarderStateMutex.RLock() - defer fake.stopAndGetSubscribedTracksForwarderStateMutex.RUnlock() - fake.subscribeToTrackMutex.RLock() - defer fake.subscribeToTrackMutex.RUnlock() - fake.subscriberAsPrimaryMutex.RLock() - defer fake.subscriberAsPrimaryMutex.RUnlock() - fake.subscriptionPermissionMutex.RLock() - defer fake.subscriptionPermissionMutex.RUnlock() - fake.subscriptionPermissionUpdateMutex.RLock() - defer fake.subscriptionPermissionUpdateMutex.RUnlock() - fake.supportsCodecChangeMutex.RLock() - defer fake.supportsCodecChangeMutex.RUnlock() - fake.supportsSyncStreamIDMutex.RLock() - defer fake.supportsSyncStreamIDMutex.RUnlock() - fake.supportsTransceiverReuseMutex.RLock() - defer fake.supportsTransceiverReuseMutex.RUnlock() - fake.toProtoMutex.RLock() - defer fake.toProtoMutex.RUnlock() - fake.toProtoWithVersionMutex.RLock() - defer fake.toProtoWithVersionMutex.RUnlock() - fake.uncacheDownTrackMutex.RLock() - defer fake.uncacheDownTrackMutex.RUnlock() - fake.unsubscribeFromTrackMutex.RLock() - defer fake.unsubscribeFromTrackMutex.RUnlock() - fake.updateAudioTrackMutex.RLock() - defer fake.updateAudioTrackMutex.RUnlock() - fake.updateLastSeenSignalMutex.RLock() - defer fake.updateLastSeenSignalMutex.RUnlock() - fake.updateMediaLossMutex.RLock() - defer fake.updateMediaLossMutex.RUnlock() - fake.updateMediaRTTMutex.RLock() - defer fake.updateMediaRTTMutex.RUnlock() - fake.updateSignalingRTTMutex.RLock() - defer fake.updateSignalingRTTMutex.RUnlock() - fake.updateSubscribedQualityMutex.RLock() - defer fake.updateSubscribedQualityMutex.RUnlock() - fake.updateSubscribedTrackSettingsMutex.RLock() - defer fake.updateSubscribedTrackSettingsMutex.RUnlock() - fake.updateSubscriptionPermissionMutex.RLock() - defer fake.updateSubscriptionPermissionMutex.RUnlock() - fake.updateVideoTrackMutex.RLock() - defer fake.updateVideoTrackMutex.RUnlock() - fake.verifyMutex.RLock() - defer fake.verifyMutex.RUnlock() - fake.verifySubscribeParticipantInfoMutex.RLock() - defer fake.verifySubscribeParticipantInfoMutex.RUnlock() - fake.versionMutex.RLock() - defer fake.versionMutex.RUnlock() - fake.waitUntilSubscribedMutex.RLock() - defer fake.waitUntilSubscribedMutex.RUnlock() - fake.writeSubscriberRTCPMutex.RLock() - defer fake.writeSubscriberRTCPMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} for key, value := range fake.invocations { copiedInvocations[key] = value diff --git a/pkg/rtc/types/typesfakes/fake_local_participant_helper.go b/pkg/rtc/types/typesfakes/fake_local_participant_helper.go new file mode 100644 index 0000000..4676dc6 --- /dev/null +++ b/pkg/rtc/types/typesfakes/fake_local_participant_helper.go @@ -0,0 +1,542 @@ +// Code generated by counterfeiter. DO NOT EDIT. +package typesfakes + +import ( + "sync" + + "github.com/livekit/livekit-server/pkg/rtc/types" + "github.com/livekit/protocol/livekit" +) + +type FakeLocalParticipantHelper struct { + GetCachedReliableDataMessageStub func(map[livekit.ParticipantID]uint32) []*types.DataMessageCache + getCachedReliableDataMessageMutex sync.RWMutex + getCachedReliableDataMessageArgsForCall []struct { + arg1 map[livekit.ParticipantID]uint32 + } + getCachedReliableDataMessageReturns struct { + result1 []*types.DataMessageCache + } + getCachedReliableDataMessageReturnsOnCall map[int]struct { + result1 []*types.DataMessageCache + } + GetParticipantInfoStub func(livekit.ParticipantID) *livekit.ParticipantInfo + getParticipantInfoMutex sync.RWMutex + getParticipantInfoArgsForCall []struct { + arg1 livekit.ParticipantID + } + getParticipantInfoReturns struct { + result1 *livekit.ParticipantInfo + } + getParticipantInfoReturnsOnCall map[int]struct { + result1 *livekit.ParticipantInfo + } + GetRegionSettingsStub func(string) *livekit.RegionSettings + getRegionSettingsMutex sync.RWMutex + getRegionSettingsArgsForCall []struct { + arg1 string + } + getRegionSettingsReturns struct { + result1 *livekit.RegionSettings + } + getRegionSettingsReturnsOnCall map[int]struct { + result1 *livekit.RegionSettings + } + GetSubscriberForwarderStateStub func(types.LocalParticipant) (map[livekit.TrackID]*livekit.RTPForwarderState, error) + getSubscriberForwarderStateMutex sync.RWMutex + getSubscriberForwarderStateArgsForCall []struct { + arg1 types.LocalParticipant + } + getSubscriberForwarderStateReturns struct { + result1 map[livekit.TrackID]*livekit.RTPForwarderState + result2 error + } + getSubscriberForwarderStateReturnsOnCall map[int]struct { + result1 map[livekit.TrackID]*livekit.RTPForwarderState + result2 error + } + ResolveDataTrackStub func(types.LocalParticipant, livekit.TrackID) types.DataResolverResult + resolveDataTrackMutex sync.RWMutex + resolveDataTrackArgsForCall []struct { + arg1 types.LocalParticipant + arg2 livekit.TrackID + } + resolveDataTrackReturns struct { + result1 types.DataResolverResult + } + resolveDataTrackReturnsOnCall map[int]struct { + result1 types.DataResolverResult + } + ResolveMediaTrackStub func(types.LocalParticipant, livekit.TrackID) types.MediaResolverResult + resolveMediaTrackMutex sync.RWMutex + resolveMediaTrackArgsForCall []struct { + arg1 types.LocalParticipant + arg2 livekit.TrackID + } + resolveMediaTrackReturns struct { + result1 types.MediaResolverResult + } + resolveMediaTrackReturnsOnCall map[int]struct { + result1 types.MediaResolverResult + } + ShouldRegressCodecStub func() bool + shouldRegressCodecMutex sync.RWMutex + shouldRegressCodecArgsForCall []struct { + } + shouldRegressCodecReturns struct { + result1 bool + } + shouldRegressCodecReturnsOnCall map[int]struct { + result1 bool + } + invocations map[string][][]interface{} + invocationsMutex sync.RWMutex +} + +func (fake *FakeLocalParticipantHelper) GetCachedReliableDataMessage(arg1 map[livekit.ParticipantID]uint32) []*types.DataMessageCache { + fake.getCachedReliableDataMessageMutex.Lock() + ret, specificReturn := fake.getCachedReliableDataMessageReturnsOnCall[len(fake.getCachedReliableDataMessageArgsForCall)] + fake.getCachedReliableDataMessageArgsForCall = append(fake.getCachedReliableDataMessageArgsForCall, struct { + arg1 map[livekit.ParticipantID]uint32 + }{arg1}) + stub := fake.GetCachedReliableDataMessageStub + fakeReturns := fake.getCachedReliableDataMessageReturns + fake.recordInvocation("GetCachedReliableDataMessage", []interface{}{arg1}) + fake.getCachedReliableDataMessageMutex.Unlock() + if stub != nil { + return stub(arg1) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeLocalParticipantHelper) GetCachedReliableDataMessageCallCount() int { + fake.getCachedReliableDataMessageMutex.RLock() + defer fake.getCachedReliableDataMessageMutex.RUnlock() + return len(fake.getCachedReliableDataMessageArgsForCall) +} + +func (fake *FakeLocalParticipantHelper) GetCachedReliableDataMessageCalls(stub func(map[livekit.ParticipantID]uint32) []*types.DataMessageCache) { + fake.getCachedReliableDataMessageMutex.Lock() + defer fake.getCachedReliableDataMessageMutex.Unlock() + fake.GetCachedReliableDataMessageStub = stub +} + +func (fake *FakeLocalParticipantHelper) GetCachedReliableDataMessageArgsForCall(i int) map[livekit.ParticipantID]uint32 { + fake.getCachedReliableDataMessageMutex.RLock() + defer fake.getCachedReliableDataMessageMutex.RUnlock() + argsForCall := fake.getCachedReliableDataMessageArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeLocalParticipantHelper) GetCachedReliableDataMessageReturns(result1 []*types.DataMessageCache) { + fake.getCachedReliableDataMessageMutex.Lock() + defer fake.getCachedReliableDataMessageMutex.Unlock() + fake.GetCachedReliableDataMessageStub = nil + fake.getCachedReliableDataMessageReturns = struct { + result1 []*types.DataMessageCache + }{result1} +} + +func (fake *FakeLocalParticipantHelper) GetCachedReliableDataMessageReturnsOnCall(i int, result1 []*types.DataMessageCache) { + fake.getCachedReliableDataMessageMutex.Lock() + defer fake.getCachedReliableDataMessageMutex.Unlock() + fake.GetCachedReliableDataMessageStub = nil + if fake.getCachedReliableDataMessageReturnsOnCall == nil { + fake.getCachedReliableDataMessageReturnsOnCall = make(map[int]struct { + result1 []*types.DataMessageCache + }) + } + fake.getCachedReliableDataMessageReturnsOnCall[i] = struct { + result1 []*types.DataMessageCache + }{result1} +} + +func (fake *FakeLocalParticipantHelper) GetParticipantInfo(arg1 livekit.ParticipantID) *livekit.ParticipantInfo { + fake.getParticipantInfoMutex.Lock() + ret, specificReturn := fake.getParticipantInfoReturnsOnCall[len(fake.getParticipantInfoArgsForCall)] + fake.getParticipantInfoArgsForCall = append(fake.getParticipantInfoArgsForCall, struct { + arg1 livekit.ParticipantID + }{arg1}) + stub := fake.GetParticipantInfoStub + fakeReturns := fake.getParticipantInfoReturns + fake.recordInvocation("GetParticipantInfo", []interface{}{arg1}) + fake.getParticipantInfoMutex.Unlock() + if stub != nil { + return stub(arg1) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeLocalParticipantHelper) GetParticipantInfoCallCount() int { + fake.getParticipantInfoMutex.RLock() + defer fake.getParticipantInfoMutex.RUnlock() + return len(fake.getParticipantInfoArgsForCall) +} + +func (fake *FakeLocalParticipantHelper) GetParticipantInfoCalls(stub func(livekit.ParticipantID) *livekit.ParticipantInfo) { + fake.getParticipantInfoMutex.Lock() + defer fake.getParticipantInfoMutex.Unlock() + fake.GetParticipantInfoStub = stub +} + +func (fake *FakeLocalParticipantHelper) GetParticipantInfoArgsForCall(i int) livekit.ParticipantID { + fake.getParticipantInfoMutex.RLock() + defer fake.getParticipantInfoMutex.RUnlock() + argsForCall := fake.getParticipantInfoArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeLocalParticipantHelper) GetParticipantInfoReturns(result1 *livekit.ParticipantInfo) { + fake.getParticipantInfoMutex.Lock() + defer fake.getParticipantInfoMutex.Unlock() + fake.GetParticipantInfoStub = nil + fake.getParticipantInfoReturns = struct { + result1 *livekit.ParticipantInfo + }{result1} +} + +func (fake *FakeLocalParticipantHelper) GetParticipantInfoReturnsOnCall(i int, result1 *livekit.ParticipantInfo) { + fake.getParticipantInfoMutex.Lock() + defer fake.getParticipantInfoMutex.Unlock() + fake.GetParticipantInfoStub = nil + if fake.getParticipantInfoReturnsOnCall == nil { + fake.getParticipantInfoReturnsOnCall = make(map[int]struct { + result1 *livekit.ParticipantInfo + }) + } + fake.getParticipantInfoReturnsOnCall[i] = struct { + result1 *livekit.ParticipantInfo + }{result1} +} + +func (fake *FakeLocalParticipantHelper) GetRegionSettings(arg1 string) *livekit.RegionSettings { + fake.getRegionSettingsMutex.Lock() + ret, specificReturn := fake.getRegionSettingsReturnsOnCall[len(fake.getRegionSettingsArgsForCall)] + fake.getRegionSettingsArgsForCall = append(fake.getRegionSettingsArgsForCall, struct { + arg1 string + }{arg1}) + stub := fake.GetRegionSettingsStub + fakeReturns := fake.getRegionSettingsReturns + fake.recordInvocation("GetRegionSettings", []interface{}{arg1}) + fake.getRegionSettingsMutex.Unlock() + if stub != nil { + return stub(arg1) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeLocalParticipantHelper) GetRegionSettingsCallCount() int { + fake.getRegionSettingsMutex.RLock() + defer fake.getRegionSettingsMutex.RUnlock() + return len(fake.getRegionSettingsArgsForCall) +} + +func (fake *FakeLocalParticipantHelper) GetRegionSettingsCalls(stub func(string) *livekit.RegionSettings) { + fake.getRegionSettingsMutex.Lock() + defer fake.getRegionSettingsMutex.Unlock() + fake.GetRegionSettingsStub = stub +} + +func (fake *FakeLocalParticipantHelper) GetRegionSettingsArgsForCall(i int) string { + fake.getRegionSettingsMutex.RLock() + defer fake.getRegionSettingsMutex.RUnlock() + argsForCall := fake.getRegionSettingsArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeLocalParticipantHelper) GetRegionSettingsReturns(result1 *livekit.RegionSettings) { + fake.getRegionSettingsMutex.Lock() + defer fake.getRegionSettingsMutex.Unlock() + fake.GetRegionSettingsStub = nil + fake.getRegionSettingsReturns = struct { + result1 *livekit.RegionSettings + }{result1} +} + +func (fake *FakeLocalParticipantHelper) GetRegionSettingsReturnsOnCall(i int, result1 *livekit.RegionSettings) { + fake.getRegionSettingsMutex.Lock() + defer fake.getRegionSettingsMutex.Unlock() + fake.GetRegionSettingsStub = nil + if fake.getRegionSettingsReturnsOnCall == nil { + fake.getRegionSettingsReturnsOnCall = make(map[int]struct { + result1 *livekit.RegionSettings + }) + } + fake.getRegionSettingsReturnsOnCall[i] = struct { + result1 *livekit.RegionSettings + }{result1} +} + +func (fake *FakeLocalParticipantHelper) GetSubscriberForwarderState(arg1 types.LocalParticipant) (map[livekit.TrackID]*livekit.RTPForwarderState, error) { + fake.getSubscriberForwarderStateMutex.Lock() + ret, specificReturn := fake.getSubscriberForwarderStateReturnsOnCall[len(fake.getSubscriberForwarderStateArgsForCall)] + fake.getSubscriberForwarderStateArgsForCall = append(fake.getSubscriberForwarderStateArgsForCall, struct { + arg1 types.LocalParticipant + }{arg1}) + stub := fake.GetSubscriberForwarderStateStub + fakeReturns := fake.getSubscriberForwarderStateReturns + fake.recordInvocation("GetSubscriberForwarderState", []interface{}{arg1}) + fake.getSubscriberForwarderStateMutex.Unlock() + if stub != nil { + return stub(arg1) + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeLocalParticipantHelper) GetSubscriberForwarderStateCallCount() int { + fake.getSubscriberForwarderStateMutex.RLock() + defer fake.getSubscriberForwarderStateMutex.RUnlock() + return len(fake.getSubscriberForwarderStateArgsForCall) +} + +func (fake *FakeLocalParticipantHelper) GetSubscriberForwarderStateCalls(stub func(types.LocalParticipant) (map[livekit.TrackID]*livekit.RTPForwarderState, error)) { + fake.getSubscriberForwarderStateMutex.Lock() + defer fake.getSubscriberForwarderStateMutex.Unlock() + fake.GetSubscriberForwarderStateStub = stub +} + +func (fake *FakeLocalParticipantHelper) GetSubscriberForwarderStateArgsForCall(i int) types.LocalParticipant { + fake.getSubscriberForwarderStateMutex.RLock() + defer fake.getSubscriberForwarderStateMutex.RUnlock() + argsForCall := fake.getSubscriberForwarderStateArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeLocalParticipantHelper) GetSubscriberForwarderStateReturns(result1 map[livekit.TrackID]*livekit.RTPForwarderState, result2 error) { + fake.getSubscriberForwarderStateMutex.Lock() + defer fake.getSubscriberForwarderStateMutex.Unlock() + fake.GetSubscriberForwarderStateStub = nil + fake.getSubscriberForwarderStateReturns = struct { + result1 map[livekit.TrackID]*livekit.RTPForwarderState + result2 error + }{result1, result2} +} + +func (fake *FakeLocalParticipantHelper) GetSubscriberForwarderStateReturnsOnCall(i int, result1 map[livekit.TrackID]*livekit.RTPForwarderState, result2 error) { + fake.getSubscriberForwarderStateMutex.Lock() + defer fake.getSubscriberForwarderStateMutex.Unlock() + fake.GetSubscriberForwarderStateStub = nil + if fake.getSubscriberForwarderStateReturnsOnCall == nil { + fake.getSubscriberForwarderStateReturnsOnCall = make(map[int]struct { + result1 map[livekit.TrackID]*livekit.RTPForwarderState + result2 error + }) + } + fake.getSubscriberForwarderStateReturnsOnCall[i] = struct { + result1 map[livekit.TrackID]*livekit.RTPForwarderState + result2 error + }{result1, result2} +} + +func (fake *FakeLocalParticipantHelper) ResolveDataTrack(arg1 types.LocalParticipant, arg2 livekit.TrackID) types.DataResolverResult { + fake.resolveDataTrackMutex.Lock() + ret, specificReturn := fake.resolveDataTrackReturnsOnCall[len(fake.resolveDataTrackArgsForCall)] + fake.resolveDataTrackArgsForCall = append(fake.resolveDataTrackArgsForCall, struct { + arg1 types.LocalParticipant + arg2 livekit.TrackID + }{arg1, arg2}) + stub := fake.ResolveDataTrackStub + fakeReturns := fake.resolveDataTrackReturns + fake.recordInvocation("ResolveDataTrack", []interface{}{arg1, arg2}) + fake.resolveDataTrackMutex.Unlock() + if stub != nil { + return stub(arg1, arg2) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeLocalParticipantHelper) ResolveDataTrackCallCount() int { + fake.resolveDataTrackMutex.RLock() + defer fake.resolveDataTrackMutex.RUnlock() + return len(fake.resolveDataTrackArgsForCall) +} + +func (fake *FakeLocalParticipantHelper) ResolveDataTrackCalls(stub func(types.LocalParticipant, livekit.TrackID) types.DataResolverResult) { + fake.resolveDataTrackMutex.Lock() + defer fake.resolveDataTrackMutex.Unlock() + fake.ResolveDataTrackStub = stub +} + +func (fake *FakeLocalParticipantHelper) ResolveDataTrackArgsForCall(i int) (types.LocalParticipant, livekit.TrackID) { + fake.resolveDataTrackMutex.RLock() + defer fake.resolveDataTrackMutex.RUnlock() + argsForCall := fake.resolveDataTrackArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeLocalParticipantHelper) ResolveDataTrackReturns(result1 types.DataResolverResult) { + fake.resolveDataTrackMutex.Lock() + defer fake.resolveDataTrackMutex.Unlock() + fake.ResolveDataTrackStub = nil + fake.resolveDataTrackReturns = struct { + result1 types.DataResolverResult + }{result1} +} + +func (fake *FakeLocalParticipantHelper) ResolveDataTrackReturnsOnCall(i int, result1 types.DataResolverResult) { + fake.resolveDataTrackMutex.Lock() + defer fake.resolveDataTrackMutex.Unlock() + fake.ResolveDataTrackStub = nil + if fake.resolveDataTrackReturnsOnCall == nil { + fake.resolveDataTrackReturnsOnCall = make(map[int]struct { + result1 types.DataResolverResult + }) + } + fake.resolveDataTrackReturnsOnCall[i] = struct { + result1 types.DataResolverResult + }{result1} +} + +func (fake *FakeLocalParticipantHelper) ResolveMediaTrack(arg1 types.LocalParticipant, arg2 livekit.TrackID) types.MediaResolverResult { + fake.resolveMediaTrackMutex.Lock() + ret, specificReturn := fake.resolveMediaTrackReturnsOnCall[len(fake.resolveMediaTrackArgsForCall)] + fake.resolveMediaTrackArgsForCall = append(fake.resolveMediaTrackArgsForCall, struct { + arg1 types.LocalParticipant + arg2 livekit.TrackID + }{arg1, arg2}) + stub := fake.ResolveMediaTrackStub + fakeReturns := fake.resolveMediaTrackReturns + fake.recordInvocation("ResolveMediaTrack", []interface{}{arg1, arg2}) + fake.resolveMediaTrackMutex.Unlock() + if stub != nil { + return stub(arg1, arg2) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeLocalParticipantHelper) ResolveMediaTrackCallCount() int { + fake.resolveMediaTrackMutex.RLock() + defer fake.resolveMediaTrackMutex.RUnlock() + return len(fake.resolveMediaTrackArgsForCall) +} + +func (fake *FakeLocalParticipantHelper) ResolveMediaTrackCalls(stub func(types.LocalParticipant, livekit.TrackID) types.MediaResolverResult) { + fake.resolveMediaTrackMutex.Lock() + defer fake.resolveMediaTrackMutex.Unlock() + fake.ResolveMediaTrackStub = stub +} + +func (fake *FakeLocalParticipantHelper) ResolveMediaTrackArgsForCall(i int) (types.LocalParticipant, livekit.TrackID) { + fake.resolveMediaTrackMutex.RLock() + defer fake.resolveMediaTrackMutex.RUnlock() + argsForCall := fake.resolveMediaTrackArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeLocalParticipantHelper) ResolveMediaTrackReturns(result1 types.MediaResolverResult) { + fake.resolveMediaTrackMutex.Lock() + defer fake.resolveMediaTrackMutex.Unlock() + fake.ResolveMediaTrackStub = nil + fake.resolveMediaTrackReturns = struct { + result1 types.MediaResolverResult + }{result1} +} + +func (fake *FakeLocalParticipantHelper) ResolveMediaTrackReturnsOnCall(i int, result1 types.MediaResolverResult) { + fake.resolveMediaTrackMutex.Lock() + defer fake.resolveMediaTrackMutex.Unlock() + fake.ResolveMediaTrackStub = nil + if fake.resolveMediaTrackReturnsOnCall == nil { + fake.resolveMediaTrackReturnsOnCall = make(map[int]struct { + result1 types.MediaResolverResult + }) + } + fake.resolveMediaTrackReturnsOnCall[i] = struct { + result1 types.MediaResolverResult + }{result1} +} + +func (fake *FakeLocalParticipantHelper) ShouldRegressCodec() bool { + fake.shouldRegressCodecMutex.Lock() + ret, specificReturn := fake.shouldRegressCodecReturnsOnCall[len(fake.shouldRegressCodecArgsForCall)] + fake.shouldRegressCodecArgsForCall = append(fake.shouldRegressCodecArgsForCall, struct { + }{}) + stub := fake.ShouldRegressCodecStub + fakeReturns := fake.shouldRegressCodecReturns + fake.recordInvocation("ShouldRegressCodec", []interface{}{}) + fake.shouldRegressCodecMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeLocalParticipantHelper) ShouldRegressCodecCallCount() int { + fake.shouldRegressCodecMutex.RLock() + defer fake.shouldRegressCodecMutex.RUnlock() + return len(fake.shouldRegressCodecArgsForCall) +} + +func (fake *FakeLocalParticipantHelper) ShouldRegressCodecCalls(stub func() bool) { + fake.shouldRegressCodecMutex.Lock() + defer fake.shouldRegressCodecMutex.Unlock() + fake.ShouldRegressCodecStub = stub +} + +func (fake *FakeLocalParticipantHelper) ShouldRegressCodecReturns(result1 bool) { + fake.shouldRegressCodecMutex.Lock() + defer fake.shouldRegressCodecMutex.Unlock() + fake.ShouldRegressCodecStub = nil + fake.shouldRegressCodecReturns = struct { + result1 bool + }{result1} +} + +func (fake *FakeLocalParticipantHelper) ShouldRegressCodecReturnsOnCall(i int, result1 bool) { + fake.shouldRegressCodecMutex.Lock() + defer fake.shouldRegressCodecMutex.Unlock() + fake.ShouldRegressCodecStub = nil + if fake.shouldRegressCodecReturnsOnCall == nil { + fake.shouldRegressCodecReturnsOnCall = make(map[int]struct { + result1 bool + }) + } + fake.shouldRegressCodecReturnsOnCall[i] = struct { + result1 bool + }{result1} +} + +func (fake *FakeLocalParticipantHelper) Invocations() map[string][][]interface{} { + fake.invocationsMutex.RLock() + defer fake.invocationsMutex.RUnlock() + copiedInvocations := map[string][][]interface{}{} + for key, value := range fake.invocations { + copiedInvocations[key] = value + } + return copiedInvocations +} + +func (fake *FakeLocalParticipantHelper) recordInvocation(key string, args []interface{}) { + fake.invocationsMutex.Lock() + defer fake.invocationsMutex.Unlock() + if fake.invocations == nil { + fake.invocations = map[string][][]interface{}{} + } + if fake.invocations[key] == nil { + fake.invocations[key] = [][]interface{}{} + } + fake.invocations[key] = append(fake.invocations[key], args) +} + +var _ types.LocalParticipantHelper = new(FakeLocalParticipantHelper) diff --git a/pkg/rtc/types/typesfakes/fake_local_participant_listener.go b/pkg/rtc/types/typesfakes/fake_local_participant_listener.go new file mode 100644 index 0000000..4cd0fef --- /dev/null +++ b/pkg/rtc/types/typesfakes/fake_local_participant_listener.go @@ -0,0 +1,948 @@ +// Code generated by counterfeiter. DO NOT EDIT. +package typesfakes + +import ( + "sync" + + "github.com/livekit/livekit-server/pkg/rtc/datatrack" + "github.com/livekit/livekit-server/pkg/rtc/types" + "github.com/livekit/protocol/livekit" +) + +type FakeLocalParticipantListener struct { + OnDataMessageStub func(types.LocalParticipant, livekit.DataPacket_Kind, *livekit.DataPacket) + onDataMessageMutex sync.RWMutex + onDataMessageArgsForCall []struct { + arg1 types.LocalParticipant + arg2 livekit.DataPacket_Kind + arg3 *livekit.DataPacket + } + OnDataMessageUnlabeledStub func(types.LocalParticipant, []byte) + onDataMessageUnlabeledMutex sync.RWMutex + onDataMessageUnlabeledArgsForCall []struct { + arg1 types.LocalParticipant + arg2 []byte + } + OnDataTrackMessageStub func(types.Participant, []byte, *datatrack.Packet) + onDataTrackMessageMutex sync.RWMutex + onDataTrackMessageArgsForCall []struct { + arg1 types.Participant + arg2 []byte + arg3 *datatrack.Packet + } + OnDataTrackPublishedStub func(types.Participant, types.DataTrack) + onDataTrackPublishedMutex sync.RWMutex + onDataTrackPublishedArgsForCall []struct { + arg1 types.Participant + arg2 types.DataTrack + } + OnDataTrackUnpublishedStub func(types.Participant, types.DataTrack) + onDataTrackUnpublishedMutex sync.RWMutex + onDataTrackUnpublishedArgsForCall []struct { + arg1 types.Participant + arg2 types.DataTrack + } + OnLeaveStub func(types.LocalParticipant, types.ParticipantCloseReason) + onLeaveMutex sync.RWMutex + onLeaveArgsForCall []struct { + arg1 types.LocalParticipant + arg2 types.ParticipantCloseReason + } + OnMetricsStub func(types.Participant, *livekit.DataPacket) + onMetricsMutex sync.RWMutex + onMetricsArgsForCall []struct { + arg1 types.Participant + arg2 *livekit.DataPacket + } + OnMigrateStateChangeStub func(types.LocalParticipant, types.MigrateState) + onMigrateStateChangeMutex sync.RWMutex + onMigrateStateChangeArgsForCall []struct { + arg1 types.LocalParticipant + arg2 types.MigrateState + } + OnParticipantUpdateStub func(types.Participant) + onParticipantUpdateMutex sync.RWMutex + onParticipantUpdateArgsForCall []struct { + arg1 types.Participant + } + OnSimulateScenarioStub func(types.LocalParticipant, *livekit.SimulateScenario) error + onSimulateScenarioMutex sync.RWMutex + onSimulateScenarioArgsForCall []struct { + arg1 types.LocalParticipant + arg2 *livekit.SimulateScenario + } + onSimulateScenarioReturns struct { + result1 error + } + onSimulateScenarioReturnsOnCall map[int]struct { + result1 error + } + OnStateChangeStub func(types.LocalParticipant) + onStateChangeMutex sync.RWMutex + onStateChangeArgsForCall []struct { + arg1 types.LocalParticipant + } + OnSubscribeStatusChangedStub func(types.LocalParticipant, livekit.ParticipantID, bool) + onSubscribeStatusChangedMutex sync.RWMutex + onSubscribeStatusChangedArgsForCall []struct { + arg1 types.LocalParticipant + arg2 livekit.ParticipantID + arg3 bool + } + OnSubscriberReadyStub func(types.LocalParticipant) + onSubscriberReadyMutex sync.RWMutex + onSubscriberReadyArgsForCall []struct { + arg1 types.LocalParticipant + } + OnSyncStateStub func(types.LocalParticipant, *livekit.SyncState) error + onSyncStateMutex sync.RWMutex + onSyncStateArgsForCall []struct { + arg1 types.LocalParticipant + arg2 *livekit.SyncState + } + onSyncStateReturns struct { + result1 error + } + onSyncStateReturnsOnCall map[int]struct { + result1 error + } + OnTrackPublishedStub func(types.Participant, types.MediaTrack) + onTrackPublishedMutex sync.RWMutex + onTrackPublishedArgsForCall []struct { + arg1 types.Participant + arg2 types.MediaTrack + } + OnTrackUnpublishedStub func(types.Participant, types.MediaTrack) + onTrackUnpublishedMutex sync.RWMutex + onTrackUnpublishedArgsForCall []struct { + arg1 types.Participant + arg2 types.MediaTrack + } + OnTrackUpdatedStub func(types.Participant, types.MediaTrack) + onTrackUpdatedMutex sync.RWMutex + onTrackUpdatedArgsForCall []struct { + arg1 types.Participant + arg2 types.MediaTrack + } + OnUpdateDataSubscriptionsStub func(types.LocalParticipant, *livekit.UpdateDataSubscription) + onUpdateDataSubscriptionsMutex sync.RWMutex + onUpdateDataSubscriptionsArgsForCall []struct { + arg1 types.LocalParticipant + arg2 *livekit.UpdateDataSubscription + } + OnUpdateSubscriptionPermissionStub func(types.LocalParticipant, *livekit.SubscriptionPermission) error + onUpdateSubscriptionPermissionMutex sync.RWMutex + onUpdateSubscriptionPermissionArgsForCall []struct { + arg1 types.LocalParticipant + arg2 *livekit.SubscriptionPermission + } + onUpdateSubscriptionPermissionReturns struct { + result1 error + } + onUpdateSubscriptionPermissionReturnsOnCall map[int]struct { + result1 error + } + OnUpdateSubscriptionsStub func(types.LocalParticipant, []livekit.TrackID, []*livekit.ParticipantTracks, bool) + onUpdateSubscriptionsMutex sync.RWMutex + onUpdateSubscriptionsArgsForCall []struct { + arg1 types.LocalParticipant + arg2 []livekit.TrackID + arg3 []*livekit.ParticipantTracks + arg4 bool + } + invocations map[string][][]interface{} + invocationsMutex sync.RWMutex +} + +func (fake *FakeLocalParticipantListener) OnDataMessage(arg1 types.LocalParticipant, arg2 livekit.DataPacket_Kind, arg3 *livekit.DataPacket) { + fake.onDataMessageMutex.Lock() + fake.onDataMessageArgsForCall = append(fake.onDataMessageArgsForCall, struct { + arg1 types.LocalParticipant + arg2 livekit.DataPacket_Kind + arg3 *livekit.DataPacket + }{arg1, arg2, arg3}) + stub := fake.OnDataMessageStub + fake.recordInvocation("OnDataMessage", []interface{}{arg1, arg2, arg3}) + fake.onDataMessageMutex.Unlock() + if stub != nil { + fake.OnDataMessageStub(arg1, arg2, arg3) + } +} + +func (fake *FakeLocalParticipantListener) OnDataMessageCallCount() int { + fake.onDataMessageMutex.RLock() + defer fake.onDataMessageMutex.RUnlock() + return len(fake.onDataMessageArgsForCall) +} + +func (fake *FakeLocalParticipantListener) OnDataMessageCalls(stub func(types.LocalParticipant, livekit.DataPacket_Kind, *livekit.DataPacket)) { + fake.onDataMessageMutex.Lock() + defer fake.onDataMessageMutex.Unlock() + fake.OnDataMessageStub = stub +} + +func (fake *FakeLocalParticipantListener) OnDataMessageArgsForCall(i int) (types.LocalParticipant, livekit.DataPacket_Kind, *livekit.DataPacket) { + fake.onDataMessageMutex.RLock() + defer fake.onDataMessageMutex.RUnlock() + argsForCall := fake.onDataMessageArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 +} + +func (fake *FakeLocalParticipantListener) OnDataMessageUnlabeled(arg1 types.LocalParticipant, arg2 []byte) { + var arg2Copy []byte + if arg2 != nil { + arg2Copy = make([]byte, len(arg2)) + copy(arg2Copy, arg2) + } + fake.onDataMessageUnlabeledMutex.Lock() + fake.onDataMessageUnlabeledArgsForCall = append(fake.onDataMessageUnlabeledArgsForCall, struct { + arg1 types.LocalParticipant + arg2 []byte + }{arg1, arg2Copy}) + stub := fake.OnDataMessageUnlabeledStub + fake.recordInvocation("OnDataMessageUnlabeled", []interface{}{arg1, arg2Copy}) + fake.onDataMessageUnlabeledMutex.Unlock() + if stub != nil { + fake.OnDataMessageUnlabeledStub(arg1, arg2) + } +} + +func (fake *FakeLocalParticipantListener) OnDataMessageUnlabeledCallCount() int { + fake.onDataMessageUnlabeledMutex.RLock() + defer fake.onDataMessageUnlabeledMutex.RUnlock() + return len(fake.onDataMessageUnlabeledArgsForCall) +} + +func (fake *FakeLocalParticipantListener) OnDataMessageUnlabeledCalls(stub func(types.LocalParticipant, []byte)) { + fake.onDataMessageUnlabeledMutex.Lock() + defer fake.onDataMessageUnlabeledMutex.Unlock() + fake.OnDataMessageUnlabeledStub = stub +} + +func (fake *FakeLocalParticipantListener) OnDataMessageUnlabeledArgsForCall(i int) (types.LocalParticipant, []byte) { + fake.onDataMessageUnlabeledMutex.RLock() + defer fake.onDataMessageUnlabeledMutex.RUnlock() + argsForCall := fake.onDataMessageUnlabeledArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeLocalParticipantListener) OnDataTrackMessage(arg1 types.Participant, arg2 []byte, arg3 *datatrack.Packet) { + var arg2Copy []byte + if arg2 != nil { + arg2Copy = make([]byte, len(arg2)) + copy(arg2Copy, arg2) + } + fake.onDataTrackMessageMutex.Lock() + fake.onDataTrackMessageArgsForCall = append(fake.onDataTrackMessageArgsForCall, struct { + arg1 types.Participant + arg2 []byte + arg3 *datatrack.Packet + }{arg1, arg2Copy, arg3}) + stub := fake.OnDataTrackMessageStub + fake.recordInvocation("OnDataTrackMessage", []interface{}{arg1, arg2Copy, arg3}) + fake.onDataTrackMessageMutex.Unlock() + if stub != nil { + fake.OnDataTrackMessageStub(arg1, arg2, arg3) + } +} + +func (fake *FakeLocalParticipantListener) OnDataTrackMessageCallCount() int { + fake.onDataTrackMessageMutex.RLock() + defer fake.onDataTrackMessageMutex.RUnlock() + return len(fake.onDataTrackMessageArgsForCall) +} + +func (fake *FakeLocalParticipantListener) OnDataTrackMessageCalls(stub func(types.Participant, []byte, *datatrack.Packet)) { + fake.onDataTrackMessageMutex.Lock() + defer fake.onDataTrackMessageMutex.Unlock() + fake.OnDataTrackMessageStub = stub +} + +func (fake *FakeLocalParticipantListener) OnDataTrackMessageArgsForCall(i int) (types.Participant, []byte, *datatrack.Packet) { + fake.onDataTrackMessageMutex.RLock() + defer fake.onDataTrackMessageMutex.RUnlock() + argsForCall := fake.onDataTrackMessageArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 +} + +func (fake *FakeLocalParticipantListener) OnDataTrackPublished(arg1 types.Participant, arg2 types.DataTrack) { + fake.onDataTrackPublishedMutex.Lock() + fake.onDataTrackPublishedArgsForCall = append(fake.onDataTrackPublishedArgsForCall, struct { + arg1 types.Participant + arg2 types.DataTrack + }{arg1, arg2}) + stub := fake.OnDataTrackPublishedStub + fake.recordInvocation("OnDataTrackPublished", []interface{}{arg1, arg2}) + fake.onDataTrackPublishedMutex.Unlock() + if stub != nil { + fake.OnDataTrackPublishedStub(arg1, arg2) + } +} + +func (fake *FakeLocalParticipantListener) OnDataTrackPublishedCallCount() int { + fake.onDataTrackPublishedMutex.RLock() + defer fake.onDataTrackPublishedMutex.RUnlock() + return len(fake.onDataTrackPublishedArgsForCall) +} + +func (fake *FakeLocalParticipantListener) OnDataTrackPublishedCalls(stub func(types.Participant, types.DataTrack)) { + fake.onDataTrackPublishedMutex.Lock() + defer fake.onDataTrackPublishedMutex.Unlock() + fake.OnDataTrackPublishedStub = stub +} + +func (fake *FakeLocalParticipantListener) OnDataTrackPublishedArgsForCall(i int) (types.Participant, types.DataTrack) { + fake.onDataTrackPublishedMutex.RLock() + defer fake.onDataTrackPublishedMutex.RUnlock() + argsForCall := fake.onDataTrackPublishedArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeLocalParticipantListener) OnDataTrackUnpublished(arg1 types.Participant, arg2 types.DataTrack) { + fake.onDataTrackUnpublishedMutex.Lock() + fake.onDataTrackUnpublishedArgsForCall = append(fake.onDataTrackUnpublishedArgsForCall, struct { + arg1 types.Participant + arg2 types.DataTrack + }{arg1, arg2}) + stub := fake.OnDataTrackUnpublishedStub + fake.recordInvocation("OnDataTrackUnpublished", []interface{}{arg1, arg2}) + fake.onDataTrackUnpublishedMutex.Unlock() + if stub != nil { + fake.OnDataTrackUnpublishedStub(arg1, arg2) + } +} + +func (fake *FakeLocalParticipantListener) OnDataTrackUnpublishedCallCount() int { + fake.onDataTrackUnpublishedMutex.RLock() + defer fake.onDataTrackUnpublishedMutex.RUnlock() + return len(fake.onDataTrackUnpublishedArgsForCall) +} + +func (fake *FakeLocalParticipantListener) OnDataTrackUnpublishedCalls(stub func(types.Participant, types.DataTrack)) { + fake.onDataTrackUnpublishedMutex.Lock() + defer fake.onDataTrackUnpublishedMutex.Unlock() + fake.OnDataTrackUnpublishedStub = stub +} + +func (fake *FakeLocalParticipantListener) OnDataTrackUnpublishedArgsForCall(i int) (types.Participant, types.DataTrack) { + fake.onDataTrackUnpublishedMutex.RLock() + defer fake.onDataTrackUnpublishedMutex.RUnlock() + argsForCall := fake.onDataTrackUnpublishedArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeLocalParticipantListener) OnLeave(arg1 types.LocalParticipant, arg2 types.ParticipantCloseReason) { + fake.onLeaveMutex.Lock() + fake.onLeaveArgsForCall = append(fake.onLeaveArgsForCall, struct { + arg1 types.LocalParticipant + arg2 types.ParticipantCloseReason + }{arg1, arg2}) + stub := fake.OnLeaveStub + fake.recordInvocation("OnLeave", []interface{}{arg1, arg2}) + fake.onLeaveMutex.Unlock() + if stub != nil { + fake.OnLeaveStub(arg1, arg2) + } +} + +func (fake *FakeLocalParticipantListener) OnLeaveCallCount() int { + fake.onLeaveMutex.RLock() + defer fake.onLeaveMutex.RUnlock() + return len(fake.onLeaveArgsForCall) +} + +func (fake *FakeLocalParticipantListener) OnLeaveCalls(stub func(types.LocalParticipant, types.ParticipantCloseReason)) { + fake.onLeaveMutex.Lock() + defer fake.onLeaveMutex.Unlock() + fake.OnLeaveStub = stub +} + +func (fake *FakeLocalParticipantListener) OnLeaveArgsForCall(i int) (types.LocalParticipant, types.ParticipantCloseReason) { + fake.onLeaveMutex.RLock() + defer fake.onLeaveMutex.RUnlock() + argsForCall := fake.onLeaveArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeLocalParticipantListener) OnMetrics(arg1 types.Participant, arg2 *livekit.DataPacket) { + fake.onMetricsMutex.Lock() + fake.onMetricsArgsForCall = append(fake.onMetricsArgsForCall, struct { + arg1 types.Participant + arg2 *livekit.DataPacket + }{arg1, arg2}) + stub := fake.OnMetricsStub + fake.recordInvocation("OnMetrics", []interface{}{arg1, arg2}) + fake.onMetricsMutex.Unlock() + if stub != nil { + fake.OnMetricsStub(arg1, arg2) + } +} + +func (fake *FakeLocalParticipantListener) OnMetricsCallCount() int { + fake.onMetricsMutex.RLock() + defer fake.onMetricsMutex.RUnlock() + return len(fake.onMetricsArgsForCall) +} + +func (fake *FakeLocalParticipantListener) OnMetricsCalls(stub func(types.Participant, *livekit.DataPacket)) { + fake.onMetricsMutex.Lock() + defer fake.onMetricsMutex.Unlock() + fake.OnMetricsStub = stub +} + +func (fake *FakeLocalParticipantListener) OnMetricsArgsForCall(i int) (types.Participant, *livekit.DataPacket) { + fake.onMetricsMutex.RLock() + defer fake.onMetricsMutex.RUnlock() + argsForCall := fake.onMetricsArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeLocalParticipantListener) OnMigrateStateChange(arg1 types.LocalParticipant, arg2 types.MigrateState) { + fake.onMigrateStateChangeMutex.Lock() + fake.onMigrateStateChangeArgsForCall = append(fake.onMigrateStateChangeArgsForCall, struct { + arg1 types.LocalParticipant + arg2 types.MigrateState + }{arg1, arg2}) + stub := fake.OnMigrateStateChangeStub + fake.recordInvocation("OnMigrateStateChange", []interface{}{arg1, arg2}) + fake.onMigrateStateChangeMutex.Unlock() + if stub != nil { + fake.OnMigrateStateChangeStub(arg1, arg2) + } +} + +func (fake *FakeLocalParticipantListener) OnMigrateStateChangeCallCount() int { + fake.onMigrateStateChangeMutex.RLock() + defer fake.onMigrateStateChangeMutex.RUnlock() + return len(fake.onMigrateStateChangeArgsForCall) +} + +func (fake *FakeLocalParticipantListener) OnMigrateStateChangeCalls(stub func(types.LocalParticipant, types.MigrateState)) { + fake.onMigrateStateChangeMutex.Lock() + defer fake.onMigrateStateChangeMutex.Unlock() + fake.OnMigrateStateChangeStub = stub +} + +func (fake *FakeLocalParticipantListener) OnMigrateStateChangeArgsForCall(i int) (types.LocalParticipant, types.MigrateState) { + fake.onMigrateStateChangeMutex.RLock() + defer fake.onMigrateStateChangeMutex.RUnlock() + argsForCall := fake.onMigrateStateChangeArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeLocalParticipantListener) OnParticipantUpdate(arg1 types.Participant) { + fake.onParticipantUpdateMutex.Lock() + fake.onParticipantUpdateArgsForCall = append(fake.onParticipantUpdateArgsForCall, struct { + arg1 types.Participant + }{arg1}) + stub := fake.OnParticipantUpdateStub + fake.recordInvocation("OnParticipantUpdate", []interface{}{arg1}) + fake.onParticipantUpdateMutex.Unlock() + if stub != nil { + fake.OnParticipantUpdateStub(arg1) + } +} + +func (fake *FakeLocalParticipantListener) OnParticipantUpdateCallCount() int { + fake.onParticipantUpdateMutex.RLock() + defer fake.onParticipantUpdateMutex.RUnlock() + return len(fake.onParticipantUpdateArgsForCall) +} + +func (fake *FakeLocalParticipantListener) OnParticipantUpdateCalls(stub func(types.Participant)) { + fake.onParticipantUpdateMutex.Lock() + defer fake.onParticipantUpdateMutex.Unlock() + fake.OnParticipantUpdateStub = stub +} + +func (fake *FakeLocalParticipantListener) OnParticipantUpdateArgsForCall(i int) types.Participant { + fake.onParticipantUpdateMutex.RLock() + defer fake.onParticipantUpdateMutex.RUnlock() + argsForCall := fake.onParticipantUpdateArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeLocalParticipantListener) OnSimulateScenario(arg1 types.LocalParticipant, arg2 *livekit.SimulateScenario) error { + fake.onSimulateScenarioMutex.Lock() + ret, specificReturn := fake.onSimulateScenarioReturnsOnCall[len(fake.onSimulateScenarioArgsForCall)] + fake.onSimulateScenarioArgsForCall = append(fake.onSimulateScenarioArgsForCall, struct { + arg1 types.LocalParticipant + arg2 *livekit.SimulateScenario + }{arg1, arg2}) + stub := fake.OnSimulateScenarioStub + fakeReturns := fake.onSimulateScenarioReturns + fake.recordInvocation("OnSimulateScenario", []interface{}{arg1, arg2}) + fake.onSimulateScenarioMutex.Unlock() + if stub != nil { + return stub(arg1, arg2) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeLocalParticipantListener) OnSimulateScenarioCallCount() int { + fake.onSimulateScenarioMutex.RLock() + defer fake.onSimulateScenarioMutex.RUnlock() + return len(fake.onSimulateScenarioArgsForCall) +} + +func (fake *FakeLocalParticipantListener) OnSimulateScenarioCalls(stub func(types.LocalParticipant, *livekit.SimulateScenario) error) { + fake.onSimulateScenarioMutex.Lock() + defer fake.onSimulateScenarioMutex.Unlock() + fake.OnSimulateScenarioStub = stub +} + +func (fake *FakeLocalParticipantListener) OnSimulateScenarioArgsForCall(i int) (types.LocalParticipant, *livekit.SimulateScenario) { + fake.onSimulateScenarioMutex.RLock() + defer fake.onSimulateScenarioMutex.RUnlock() + argsForCall := fake.onSimulateScenarioArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeLocalParticipantListener) OnSimulateScenarioReturns(result1 error) { + fake.onSimulateScenarioMutex.Lock() + defer fake.onSimulateScenarioMutex.Unlock() + fake.OnSimulateScenarioStub = nil + fake.onSimulateScenarioReturns = struct { + result1 error + }{result1} +} + +func (fake *FakeLocalParticipantListener) OnSimulateScenarioReturnsOnCall(i int, result1 error) { + fake.onSimulateScenarioMutex.Lock() + defer fake.onSimulateScenarioMutex.Unlock() + fake.OnSimulateScenarioStub = nil + if fake.onSimulateScenarioReturnsOnCall == nil { + fake.onSimulateScenarioReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.onSimulateScenarioReturnsOnCall[i] = struct { + result1 error + }{result1} +} + +func (fake *FakeLocalParticipantListener) OnStateChange(arg1 types.LocalParticipant) { + fake.onStateChangeMutex.Lock() + fake.onStateChangeArgsForCall = append(fake.onStateChangeArgsForCall, struct { + arg1 types.LocalParticipant + }{arg1}) + stub := fake.OnStateChangeStub + fake.recordInvocation("OnStateChange", []interface{}{arg1}) + fake.onStateChangeMutex.Unlock() + if stub != nil { + fake.OnStateChangeStub(arg1) + } +} + +func (fake *FakeLocalParticipantListener) OnStateChangeCallCount() int { + fake.onStateChangeMutex.RLock() + defer fake.onStateChangeMutex.RUnlock() + return len(fake.onStateChangeArgsForCall) +} + +func (fake *FakeLocalParticipantListener) OnStateChangeCalls(stub func(types.LocalParticipant)) { + fake.onStateChangeMutex.Lock() + defer fake.onStateChangeMutex.Unlock() + fake.OnStateChangeStub = stub +} + +func (fake *FakeLocalParticipantListener) OnStateChangeArgsForCall(i int) types.LocalParticipant { + fake.onStateChangeMutex.RLock() + defer fake.onStateChangeMutex.RUnlock() + argsForCall := fake.onStateChangeArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeLocalParticipantListener) OnSubscribeStatusChanged(arg1 types.LocalParticipant, arg2 livekit.ParticipantID, arg3 bool) { + fake.onSubscribeStatusChangedMutex.Lock() + fake.onSubscribeStatusChangedArgsForCall = append(fake.onSubscribeStatusChangedArgsForCall, struct { + arg1 types.LocalParticipant + arg2 livekit.ParticipantID + arg3 bool + }{arg1, arg2, arg3}) + stub := fake.OnSubscribeStatusChangedStub + fake.recordInvocation("OnSubscribeStatusChanged", []interface{}{arg1, arg2, arg3}) + fake.onSubscribeStatusChangedMutex.Unlock() + if stub != nil { + fake.OnSubscribeStatusChangedStub(arg1, arg2, arg3) + } +} + +func (fake *FakeLocalParticipantListener) OnSubscribeStatusChangedCallCount() int { + fake.onSubscribeStatusChangedMutex.RLock() + defer fake.onSubscribeStatusChangedMutex.RUnlock() + return len(fake.onSubscribeStatusChangedArgsForCall) +} + +func (fake *FakeLocalParticipantListener) OnSubscribeStatusChangedCalls(stub func(types.LocalParticipant, livekit.ParticipantID, bool)) { + fake.onSubscribeStatusChangedMutex.Lock() + defer fake.onSubscribeStatusChangedMutex.Unlock() + fake.OnSubscribeStatusChangedStub = stub +} + +func (fake *FakeLocalParticipantListener) OnSubscribeStatusChangedArgsForCall(i int) (types.LocalParticipant, livekit.ParticipantID, bool) { + fake.onSubscribeStatusChangedMutex.RLock() + defer fake.onSubscribeStatusChangedMutex.RUnlock() + argsForCall := fake.onSubscribeStatusChangedArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 +} + +func (fake *FakeLocalParticipantListener) OnSubscriberReady(arg1 types.LocalParticipant) { + fake.onSubscriberReadyMutex.Lock() + fake.onSubscriberReadyArgsForCall = append(fake.onSubscriberReadyArgsForCall, struct { + arg1 types.LocalParticipant + }{arg1}) + stub := fake.OnSubscriberReadyStub + fake.recordInvocation("OnSubscriberReady", []interface{}{arg1}) + fake.onSubscriberReadyMutex.Unlock() + if stub != nil { + fake.OnSubscriberReadyStub(arg1) + } +} + +func (fake *FakeLocalParticipantListener) OnSubscriberReadyCallCount() int { + fake.onSubscriberReadyMutex.RLock() + defer fake.onSubscriberReadyMutex.RUnlock() + return len(fake.onSubscriberReadyArgsForCall) +} + +func (fake *FakeLocalParticipantListener) OnSubscriberReadyCalls(stub func(types.LocalParticipant)) { + fake.onSubscriberReadyMutex.Lock() + defer fake.onSubscriberReadyMutex.Unlock() + fake.OnSubscriberReadyStub = stub +} + +func (fake *FakeLocalParticipantListener) OnSubscriberReadyArgsForCall(i int) types.LocalParticipant { + fake.onSubscriberReadyMutex.RLock() + defer fake.onSubscriberReadyMutex.RUnlock() + argsForCall := fake.onSubscriberReadyArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeLocalParticipantListener) OnSyncState(arg1 types.LocalParticipant, arg2 *livekit.SyncState) error { + fake.onSyncStateMutex.Lock() + ret, specificReturn := fake.onSyncStateReturnsOnCall[len(fake.onSyncStateArgsForCall)] + fake.onSyncStateArgsForCall = append(fake.onSyncStateArgsForCall, struct { + arg1 types.LocalParticipant + arg2 *livekit.SyncState + }{arg1, arg2}) + stub := fake.OnSyncStateStub + fakeReturns := fake.onSyncStateReturns + fake.recordInvocation("OnSyncState", []interface{}{arg1, arg2}) + fake.onSyncStateMutex.Unlock() + if stub != nil { + return stub(arg1, arg2) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeLocalParticipantListener) OnSyncStateCallCount() int { + fake.onSyncStateMutex.RLock() + defer fake.onSyncStateMutex.RUnlock() + return len(fake.onSyncStateArgsForCall) +} + +func (fake *FakeLocalParticipantListener) OnSyncStateCalls(stub func(types.LocalParticipant, *livekit.SyncState) error) { + fake.onSyncStateMutex.Lock() + defer fake.onSyncStateMutex.Unlock() + fake.OnSyncStateStub = stub +} + +func (fake *FakeLocalParticipantListener) OnSyncStateArgsForCall(i int) (types.LocalParticipant, *livekit.SyncState) { + fake.onSyncStateMutex.RLock() + defer fake.onSyncStateMutex.RUnlock() + argsForCall := fake.onSyncStateArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeLocalParticipantListener) OnSyncStateReturns(result1 error) { + fake.onSyncStateMutex.Lock() + defer fake.onSyncStateMutex.Unlock() + fake.OnSyncStateStub = nil + fake.onSyncStateReturns = struct { + result1 error + }{result1} +} + +func (fake *FakeLocalParticipantListener) OnSyncStateReturnsOnCall(i int, result1 error) { + fake.onSyncStateMutex.Lock() + defer fake.onSyncStateMutex.Unlock() + fake.OnSyncStateStub = nil + if fake.onSyncStateReturnsOnCall == nil { + fake.onSyncStateReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.onSyncStateReturnsOnCall[i] = struct { + result1 error + }{result1} +} + +func (fake *FakeLocalParticipantListener) OnTrackPublished(arg1 types.Participant, arg2 types.MediaTrack) { + fake.onTrackPublishedMutex.Lock() + fake.onTrackPublishedArgsForCall = append(fake.onTrackPublishedArgsForCall, struct { + arg1 types.Participant + arg2 types.MediaTrack + }{arg1, arg2}) + stub := fake.OnTrackPublishedStub + fake.recordInvocation("OnTrackPublished", []interface{}{arg1, arg2}) + fake.onTrackPublishedMutex.Unlock() + if stub != nil { + fake.OnTrackPublishedStub(arg1, arg2) + } +} + +func (fake *FakeLocalParticipantListener) OnTrackPublishedCallCount() int { + fake.onTrackPublishedMutex.RLock() + defer fake.onTrackPublishedMutex.RUnlock() + return len(fake.onTrackPublishedArgsForCall) +} + +func (fake *FakeLocalParticipantListener) OnTrackPublishedCalls(stub func(types.Participant, types.MediaTrack)) { + fake.onTrackPublishedMutex.Lock() + defer fake.onTrackPublishedMutex.Unlock() + fake.OnTrackPublishedStub = stub +} + +func (fake *FakeLocalParticipantListener) OnTrackPublishedArgsForCall(i int) (types.Participant, types.MediaTrack) { + fake.onTrackPublishedMutex.RLock() + defer fake.onTrackPublishedMutex.RUnlock() + argsForCall := fake.onTrackPublishedArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeLocalParticipantListener) OnTrackUnpublished(arg1 types.Participant, arg2 types.MediaTrack) { + fake.onTrackUnpublishedMutex.Lock() + fake.onTrackUnpublishedArgsForCall = append(fake.onTrackUnpublishedArgsForCall, struct { + arg1 types.Participant + arg2 types.MediaTrack + }{arg1, arg2}) + stub := fake.OnTrackUnpublishedStub + fake.recordInvocation("OnTrackUnpublished", []interface{}{arg1, arg2}) + fake.onTrackUnpublishedMutex.Unlock() + if stub != nil { + fake.OnTrackUnpublishedStub(arg1, arg2) + } +} + +func (fake *FakeLocalParticipantListener) OnTrackUnpublishedCallCount() int { + fake.onTrackUnpublishedMutex.RLock() + defer fake.onTrackUnpublishedMutex.RUnlock() + return len(fake.onTrackUnpublishedArgsForCall) +} + +func (fake *FakeLocalParticipantListener) OnTrackUnpublishedCalls(stub func(types.Participant, types.MediaTrack)) { + fake.onTrackUnpublishedMutex.Lock() + defer fake.onTrackUnpublishedMutex.Unlock() + fake.OnTrackUnpublishedStub = stub +} + +func (fake *FakeLocalParticipantListener) OnTrackUnpublishedArgsForCall(i int) (types.Participant, types.MediaTrack) { + fake.onTrackUnpublishedMutex.RLock() + defer fake.onTrackUnpublishedMutex.RUnlock() + argsForCall := fake.onTrackUnpublishedArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeLocalParticipantListener) OnTrackUpdated(arg1 types.Participant, arg2 types.MediaTrack) { + fake.onTrackUpdatedMutex.Lock() + fake.onTrackUpdatedArgsForCall = append(fake.onTrackUpdatedArgsForCall, struct { + arg1 types.Participant + arg2 types.MediaTrack + }{arg1, arg2}) + stub := fake.OnTrackUpdatedStub + fake.recordInvocation("OnTrackUpdated", []interface{}{arg1, arg2}) + fake.onTrackUpdatedMutex.Unlock() + if stub != nil { + fake.OnTrackUpdatedStub(arg1, arg2) + } +} + +func (fake *FakeLocalParticipantListener) OnTrackUpdatedCallCount() int { + fake.onTrackUpdatedMutex.RLock() + defer fake.onTrackUpdatedMutex.RUnlock() + return len(fake.onTrackUpdatedArgsForCall) +} + +func (fake *FakeLocalParticipantListener) OnTrackUpdatedCalls(stub func(types.Participant, types.MediaTrack)) { + fake.onTrackUpdatedMutex.Lock() + defer fake.onTrackUpdatedMutex.Unlock() + fake.OnTrackUpdatedStub = stub +} + +func (fake *FakeLocalParticipantListener) OnTrackUpdatedArgsForCall(i int) (types.Participant, types.MediaTrack) { + fake.onTrackUpdatedMutex.RLock() + defer fake.onTrackUpdatedMutex.RUnlock() + argsForCall := fake.onTrackUpdatedArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeLocalParticipantListener) OnUpdateDataSubscriptions(arg1 types.LocalParticipant, arg2 *livekit.UpdateDataSubscription) { + fake.onUpdateDataSubscriptionsMutex.Lock() + fake.onUpdateDataSubscriptionsArgsForCall = append(fake.onUpdateDataSubscriptionsArgsForCall, struct { + arg1 types.LocalParticipant + arg2 *livekit.UpdateDataSubscription + }{arg1, arg2}) + stub := fake.OnUpdateDataSubscriptionsStub + fake.recordInvocation("OnUpdateDataSubscriptions", []interface{}{arg1, arg2}) + fake.onUpdateDataSubscriptionsMutex.Unlock() + if stub != nil { + fake.OnUpdateDataSubscriptionsStub(arg1, arg2) + } +} + +func (fake *FakeLocalParticipantListener) OnUpdateDataSubscriptionsCallCount() int { + fake.onUpdateDataSubscriptionsMutex.RLock() + defer fake.onUpdateDataSubscriptionsMutex.RUnlock() + return len(fake.onUpdateDataSubscriptionsArgsForCall) +} + +func (fake *FakeLocalParticipantListener) OnUpdateDataSubscriptionsCalls(stub func(types.LocalParticipant, *livekit.UpdateDataSubscription)) { + fake.onUpdateDataSubscriptionsMutex.Lock() + defer fake.onUpdateDataSubscriptionsMutex.Unlock() + fake.OnUpdateDataSubscriptionsStub = stub +} + +func (fake *FakeLocalParticipantListener) OnUpdateDataSubscriptionsArgsForCall(i int) (types.LocalParticipant, *livekit.UpdateDataSubscription) { + fake.onUpdateDataSubscriptionsMutex.RLock() + defer fake.onUpdateDataSubscriptionsMutex.RUnlock() + argsForCall := fake.onUpdateDataSubscriptionsArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeLocalParticipantListener) OnUpdateSubscriptionPermission(arg1 types.LocalParticipant, arg2 *livekit.SubscriptionPermission) error { + fake.onUpdateSubscriptionPermissionMutex.Lock() + ret, specificReturn := fake.onUpdateSubscriptionPermissionReturnsOnCall[len(fake.onUpdateSubscriptionPermissionArgsForCall)] + fake.onUpdateSubscriptionPermissionArgsForCall = append(fake.onUpdateSubscriptionPermissionArgsForCall, struct { + arg1 types.LocalParticipant + arg2 *livekit.SubscriptionPermission + }{arg1, arg2}) + stub := fake.OnUpdateSubscriptionPermissionStub + fakeReturns := fake.onUpdateSubscriptionPermissionReturns + fake.recordInvocation("OnUpdateSubscriptionPermission", []interface{}{arg1, arg2}) + fake.onUpdateSubscriptionPermissionMutex.Unlock() + if stub != nil { + return stub(arg1, arg2) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeLocalParticipantListener) OnUpdateSubscriptionPermissionCallCount() int { + fake.onUpdateSubscriptionPermissionMutex.RLock() + defer fake.onUpdateSubscriptionPermissionMutex.RUnlock() + return len(fake.onUpdateSubscriptionPermissionArgsForCall) +} + +func (fake *FakeLocalParticipantListener) OnUpdateSubscriptionPermissionCalls(stub func(types.LocalParticipant, *livekit.SubscriptionPermission) error) { + fake.onUpdateSubscriptionPermissionMutex.Lock() + defer fake.onUpdateSubscriptionPermissionMutex.Unlock() + fake.OnUpdateSubscriptionPermissionStub = stub +} + +func (fake *FakeLocalParticipantListener) OnUpdateSubscriptionPermissionArgsForCall(i int) (types.LocalParticipant, *livekit.SubscriptionPermission) { + fake.onUpdateSubscriptionPermissionMutex.RLock() + defer fake.onUpdateSubscriptionPermissionMutex.RUnlock() + argsForCall := fake.onUpdateSubscriptionPermissionArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeLocalParticipantListener) OnUpdateSubscriptionPermissionReturns(result1 error) { + fake.onUpdateSubscriptionPermissionMutex.Lock() + defer fake.onUpdateSubscriptionPermissionMutex.Unlock() + fake.OnUpdateSubscriptionPermissionStub = nil + fake.onUpdateSubscriptionPermissionReturns = struct { + result1 error + }{result1} +} + +func (fake *FakeLocalParticipantListener) OnUpdateSubscriptionPermissionReturnsOnCall(i int, result1 error) { + fake.onUpdateSubscriptionPermissionMutex.Lock() + defer fake.onUpdateSubscriptionPermissionMutex.Unlock() + fake.OnUpdateSubscriptionPermissionStub = nil + if fake.onUpdateSubscriptionPermissionReturnsOnCall == nil { + fake.onUpdateSubscriptionPermissionReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.onUpdateSubscriptionPermissionReturnsOnCall[i] = struct { + result1 error + }{result1} +} + +func (fake *FakeLocalParticipantListener) OnUpdateSubscriptions(arg1 types.LocalParticipant, arg2 []livekit.TrackID, arg3 []*livekit.ParticipantTracks, arg4 bool) { + var arg2Copy []livekit.TrackID + if arg2 != nil { + arg2Copy = make([]livekit.TrackID, len(arg2)) + copy(arg2Copy, arg2) + } + var arg3Copy []*livekit.ParticipantTracks + if arg3 != nil { + arg3Copy = make([]*livekit.ParticipantTracks, len(arg3)) + copy(arg3Copy, arg3) + } + fake.onUpdateSubscriptionsMutex.Lock() + fake.onUpdateSubscriptionsArgsForCall = append(fake.onUpdateSubscriptionsArgsForCall, struct { + arg1 types.LocalParticipant + arg2 []livekit.TrackID + arg3 []*livekit.ParticipantTracks + arg4 bool + }{arg1, arg2Copy, arg3Copy, arg4}) + stub := fake.OnUpdateSubscriptionsStub + fake.recordInvocation("OnUpdateSubscriptions", []interface{}{arg1, arg2Copy, arg3Copy, arg4}) + fake.onUpdateSubscriptionsMutex.Unlock() + if stub != nil { + fake.OnUpdateSubscriptionsStub(arg1, arg2, arg3, arg4) + } +} + +func (fake *FakeLocalParticipantListener) OnUpdateSubscriptionsCallCount() int { + fake.onUpdateSubscriptionsMutex.RLock() + defer fake.onUpdateSubscriptionsMutex.RUnlock() + return len(fake.onUpdateSubscriptionsArgsForCall) +} + +func (fake *FakeLocalParticipantListener) OnUpdateSubscriptionsCalls(stub func(types.LocalParticipant, []livekit.TrackID, []*livekit.ParticipantTracks, bool)) { + fake.onUpdateSubscriptionsMutex.Lock() + defer fake.onUpdateSubscriptionsMutex.Unlock() + fake.OnUpdateSubscriptionsStub = stub +} + +func (fake *FakeLocalParticipantListener) OnUpdateSubscriptionsArgsForCall(i int) (types.LocalParticipant, []livekit.TrackID, []*livekit.ParticipantTracks, bool) { + fake.onUpdateSubscriptionsMutex.RLock() + defer fake.onUpdateSubscriptionsMutex.RUnlock() + argsForCall := fake.onUpdateSubscriptionsArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4 +} + +func (fake *FakeLocalParticipantListener) Invocations() map[string][][]interface{} { + fake.invocationsMutex.RLock() + defer fake.invocationsMutex.RUnlock() + copiedInvocations := map[string][][]interface{}{} + for key, value := range fake.invocations { + copiedInvocations[key] = value + } + return copiedInvocations +} + +func (fake *FakeLocalParticipantListener) recordInvocation(key string, args []interface{}) { + fake.invocationsMutex.Lock() + defer fake.invocationsMutex.Unlock() + if fake.invocations == nil { + fake.invocations = map[string][][]interface{}{} + } + if fake.invocations[key] == nil { + fake.invocations[key] = [][]interface{}{} + } + fake.invocations[key] = append(fake.invocations[key], args) +} + +var _ types.LocalParticipantListener = new(FakeLocalParticipantListener) diff --git a/pkg/rtc/types/typesfakes/fake_media_track.go b/pkg/rtc/types/typesfakes/fake_media_track.go index 6730471..797c812 100644 --- a/pkg/rtc/types/typesfakes/fake_media_track.go +++ b/pkg/rtc/types/typesfakes/fake_media_track.go @@ -6,8 +6,9 @@ import ( "github.com/livekit/livekit-server/pkg/rtc/types" "github.com/livekit/livekit-server/pkg/sfu" - "github.com/livekit/livekit-server/pkg/sfu/mime" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/logger" ) type FakeMediaTrack struct { @@ -71,11 +72,12 @@ type FakeMediaTrack struct { getNumSubscribersReturnsOnCall map[int]struct { result1 int } - GetQualityForDimensionStub func(uint32, uint32) livekit.VideoQuality + GetQualityForDimensionStub func(mime.MimeType, uint32, uint32) livekit.VideoQuality getQualityForDimensionMutex sync.RWMutex getQualityForDimensionArgsForCall []struct { - arg1 uint32 + arg1 mime.MimeType arg2 uint32 + arg3 uint32 } getQualityForDimensionReturns struct { result1 livekit.VideoQuality @@ -83,12 +85,12 @@ type FakeMediaTrack struct { getQualityForDimensionReturnsOnCall map[int]struct { result1 livekit.VideoQuality } - GetTemporalLayerForSpatialFpsStub func(int32, uint32, mime.MimeType) int32 + GetTemporalLayerForSpatialFpsStub func(mime.MimeType, int32, uint32) int32 getTemporalLayerForSpatialFpsMutex sync.RWMutex getTemporalLayerForSpatialFpsArgsForCall []struct { - arg1 int32 - arg2 uint32 - arg3 mime.MimeType + arg1 mime.MimeType + arg2 int32 + arg3 uint32 } getTemporalLayerForSpatialFpsReturns struct { result1 int32 @@ -96,6 +98,16 @@ type FakeMediaTrack struct { getTemporalLayerForSpatialFpsReturnsOnCall map[int]struct { result1 int32 } + HasPacketTrailerStub func() bool + hasPacketTrailerMutex sync.RWMutex + hasPacketTrailerArgsForCall []struct { + } + hasPacketTrailerReturns struct { + result1 bool + } + hasPacketTrailerReturnsOnCall map[int]struct { + result1 bool + } IDStub func() livekit.TrackID iDMutex sync.RWMutex iDArgsForCall []struct { @@ -136,16 +148,6 @@ type FakeMediaTrack struct { isOpenReturnsOnCall map[int]struct { result1 bool } - IsSimulcastStub func() bool - isSimulcastMutex sync.RWMutex - isSimulcastArgsForCall []struct { - } - isSimulcastReturns struct { - result1 bool - } - isSimulcastReturnsOnCall map[int]struct { - result1 bool - } IsSubscriberStub func(livekit.ParticipantID) bool isSubscriberMutex sync.RWMutex isSubscriberArgsForCall []struct { @@ -167,6 +169,16 @@ type FakeMediaTrack struct { kindReturnsOnCall map[int]struct { result1 livekit.TrackType } + LoggerStub func() logger.Logger + loggerMutex sync.RWMutex + loggerArgsForCall []struct { + } + loggerReturns struct { + result1 logger.Logger + } + loggerReturnsOnCall map[int]struct { + result1 logger.Logger + } NameStub func() string nameMutex sync.RWMutex nameArgsForCall []struct { @@ -614,19 +626,20 @@ func (fake *FakeMediaTrack) GetNumSubscribersReturnsOnCall(i int, result1 int) { }{result1} } -func (fake *FakeMediaTrack) GetQualityForDimension(arg1 uint32, arg2 uint32) livekit.VideoQuality { +func (fake *FakeMediaTrack) GetQualityForDimension(arg1 mime.MimeType, arg2 uint32, arg3 uint32) livekit.VideoQuality { fake.getQualityForDimensionMutex.Lock() ret, specificReturn := fake.getQualityForDimensionReturnsOnCall[len(fake.getQualityForDimensionArgsForCall)] fake.getQualityForDimensionArgsForCall = append(fake.getQualityForDimensionArgsForCall, struct { - arg1 uint32 + arg1 mime.MimeType arg2 uint32 - }{arg1, arg2}) + arg3 uint32 + }{arg1, arg2, arg3}) stub := fake.GetQualityForDimensionStub fakeReturns := fake.getQualityForDimensionReturns - fake.recordInvocation("GetQualityForDimension", []interface{}{arg1, arg2}) + fake.recordInvocation("GetQualityForDimension", []interface{}{arg1, arg2, arg3}) fake.getQualityForDimensionMutex.Unlock() if stub != nil { - return stub(arg1, arg2) + return stub(arg1, arg2, arg3) } if specificReturn { return ret.result1 @@ -640,17 +653,17 @@ func (fake *FakeMediaTrack) GetQualityForDimensionCallCount() int { return len(fake.getQualityForDimensionArgsForCall) } -func (fake *FakeMediaTrack) GetQualityForDimensionCalls(stub func(uint32, uint32) livekit.VideoQuality) { +func (fake *FakeMediaTrack) GetQualityForDimensionCalls(stub func(mime.MimeType, uint32, uint32) livekit.VideoQuality) { fake.getQualityForDimensionMutex.Lock() defer fake.getQualityForDimensionMutex.Unlock() fake.GetQualityForDimensionStub = stub } -func (fake *FakeMediaTrack) GetQualityForDimensionArgsForCall(i int) (uint32, uint32) { +func (fake *FakeMediaTrack) GetQualityForDimensionArgsForCall(i int) (mime.MimeType, uint32, uint32) { fake.getQualityForDimensionMutex.RLock() defer fake.getQualityForDimensionMutex.RUnlock() argsForCall := fake.getQualityForDimensionArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2 + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 } func (fake *FakeMediaTrack) GetQualityForDimensionReturns(result1 livekit.VideoQuality) { @@ -676,13 +689,13 @@ func (fake *FakeMediaTrack) GetQualityForDimensionReturnsOnCall(i int, result1 l }{result1} } -func (fake *FakeMediaTrack) GetTemporalLayerForSpatialFps(arg1 int32, arg2 uint32, arg3 mime.MimeType) int32 { +func (fake *FakeMediaTrack) GetTemporalLayerForSpatialFps(arg1 mime.MimeType, arg2 int32, arg3 uint32) int32 { fake.getTemporalLayerForSpatialFpsMutex.Lock() ret, specificReturn := fake.getTemporalLayerForSpatialFpsReturnsOnCall[len(fake.getTemporalLayerForSpatialFpsArgsForCall)] fake.getTemporalLayerForSpatialFpsArgsForCall = append(fake.getTemporalLayerForSpatialFpsArgsForCall, struct { - arg1 int32 - arg2 uint32 - arg3 mime.MimeType + arg1 mime.MimeType + arg2 int32 + arg3 uint32 }{arg1, arg2, arg3}) stub := fake.GetTemporalLayerForSpatialFpsStub fakeReturns := fake.getTemporalLayerForSpatialFpsReturns @@ -703,13 +716,13 @@ func (fake *FakeMediaTrack) GetTemporalLayerForSpatialFpsCallCount() int { return len(fake.getTemporalLayerForSpatialFpsArgsForCall) } -func (fake *FakeMediaTrack) GetTemporalLayerForSpatialFpsCalls(stub func(int32, uint32, mime.MimeType) int32) { +func (fake *FakeMediaTrack) GetTemporalLayerForSpatialFpsCalls(stub func(mime.MimeType, int32, uint32) int32) { fake.getTemporalLayerForSpatialFpsMutex.Lock() defer fake.getTemporalLayerForSpatialFpsMutex.Unlock() fake.GetTemporalLayerForSpatialFpsStub = stub } -func (fake *FakeMediaTrack) GetTemporalLayerForSpatialFpsArgsForCall(i int) (int32, uint32, mime.MimeType) { +func (fake *FakeMediaTrack) GetTemporalLayerForSpatialFpsArgsForCall(i int) (mime.MimeType, int32, uint32) { fake.getTemporalLayerForSpatialFpsMutex.RLock() defer fake.getTemporalLayerForSpatialFpsMutex.RUnlock() argsForCall := fake.getTemporalLayerForSpatialFpsArgsForCall[i] @@ -739,6 +752,59 @@ func (fake *FakeMediaTrack) GetTemporalLayerForSpatialFpsReturnsOnCall(i int, re }{result1} } +func (fake *FakeMediaTrack) HasPacketTrailer() bool { + fake.hasPacketTrailerMutex.Lock() + ret, specificReturn := fake.hasPacketTrailerReturnsOnCall[len(fake.hasPacketTrailerArgsForCall)] + fake.hasPacketTrailerArgsForCall = append(fake.hasPacketTrailerArgsForCall, struct { + }{}) + stub := fake.HasPacketTrailerStub + fakeReturns := fake.hasPacketTrailerReturns + fake.recordInvocation("HasPacketTrailer", []interface{}{}) + fake.hasPacketTrailerMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeMediaTrack) HasPacketTrailerCallCount() int { + fake.hasPacketTrailerMutex.RLock() + defer fake.hasPacketTrailerMutex.RUnlock() + return len(fake.hasPacketTrailerArgsForCall) +} + +func (fake *FakeMediaTrack) HasPacketTrailerCalls(stub func() bool) { + fake.hasPacketTrailerMutex.Lock() + defer fake.hasPacketTrailerMutex.Unlock() + fake.HasPacketTrailerStub = stub +} + +func (fake *FakeMediaTrack) HasPacketTrailerReturns(result1 bool) { + fake.hasPacketTrailerMutex.Lock() + defer fake.hasPacketTrailerMutex.Unlock() + fake.HasPacketTrailerStub = nil + fake.hasPacketTrailerReturns = struct { + result1 bool + }{result1} +} + +func (fake *FakeMediaTrack) HasPacketTrailerReturnsOnCall(i int, result1 bool) { + fake.hasPacketTrailerMutex.Lock() + defer fake.hasPacketTrailerMutex.Unlock() + fake.HasPacketTrailerStub = nil + if fake.hasPacketTrailerReturnsOnCall == nil { + fake.hasPacketTrailerReturnsOnCall = make(map[int]struct { + result1 bool + }) + } + fake.hasPacketTrailerReturnsOnCall[i] = struct { + result1 bool + }{result1} +} + func (fake *FakeMediaTrack) ID() livekit.TrackID { fake.iDMutex.Lock() ret, specificReturn := fake.iDReturnsOnCall[len(fake.iDArgsForCall)] @@ -951,59 +1017,6 @@ func (fake *FakeMediaTrack) IsOpenReturnsOnCall(i int, result1 bool) { }{result1} } -func (fake *FakeMediaTrack) IsSimulcast() bool { - fake.isSimulcastMutex.Lock() - ret, specificReturn := fake.isSimulcastReturnsOnCall[len(fake.isSimulcastArgsForCall)] - fake.isSimulcastArgsForCall = append(fake.isSimulcastArgsForCall, struct { - }{}) - stub := fake.IsSimulcastStub - fakeReturns := fake.isSimulcastReturns - fake.recordInvocation("IsSimulcast", []interface{}{}) - fake.isSimulcastMutex.Unlock() - if stub != nil { - return stub() - } - if specificReturn { - return ret.result1 - } - return fakeReturns.result1 -} - -func (fake *FakeMediaTrack) IsSimulcastCallCount() int { - fake.isSimulcastMutex.RLock() - defer fake.isSimulcastMutex.RUnlock() - return len(fake.isSimulcastArgsForCall) -} - -func (fake *FakeMediaTrack) IsSimulcastCalls(stub func() bool) { - fake.isSimulcastMutex.Lock() - defer fake.isSimulcastMutex.Unlock() - fake.IsSimulcastStub = stub -} - -func (fake *FakeMediaTrack) IsSimulcastReturns(result1 bool) { - fake.isSimulcastMutex.Lock() - defer fake.isSimulcastMutex.Unlock() - fake.IsSimulcastStub = nil - fake.isSimulcastReturns = struct { - result1 bool - }{result1} -} - -func (fake *FakeMediaTrack) IsSimulcastReturnsOnCall(i int, result1 bool) { - fake.isSimulcastMutex.Lock() - defer fake.isSimulcastMutex.Unlock() - fake.IsSimulcastStub = nil - if fake.isSimulcastReturnsOnCall == nil { - fake.isSimulcastReturnsOnCall = make(map[int]struct { - result1 bool - }) - } - fake.isSimulcastReturnsOnCall[i] = struct { - result1 bool - }{result1} -} - func (fake *FakeMediaTrack) IsSubscriber(arg1 livekit.ParticipantID) bool { fake.isSubscriberMutex.Lock() ret, specificReturn := fake.isSubscriberReturnsOnCall[len(fake.isSubscriberArgsForCall)] @@ -1118,6 +1131,59 @@ func (fake *FakeMediaTrack) KindReturnsOnCall(i int, result1 livekit.TrackType) }{result1} } +func (fake *FakeMediaTrack) Logger() logger.Logger { + fake.loggerMutex.Lock() + ret, specificReturn := fake.loggerReturnsOnCall[len(fake.loggerArgsForCall)] + fake.loggerArgsForCall = append(fake.loggerArgsForCall, struct { + }{}) + stub := fake.LoggerStub + fakeReturns := fake.loggerReturns + fake.recordInvocation("Logger", []interface{}{}) + fake.loggerMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeMediaTrack) LoggerCallCount() int { + fake.loggerMutex.RLock() + defer fake.loggerMutex.RUnlock() + return len(fake.loggerArgsForCall) +} + +func (fake *FakeMediaTrack) LoggerCalls(stub func() logger.Logger) { + fake.loggerMutex.Lock() + defer fake.loggerMutex.Unlock() + fake.LoggerStub = stub +} + +func (fake *FakeMediaTrack) LoggerReturns(result1 logger.Logger) { + fake.loggerMutex.Lock() + defer fake.loggerMutex.Unlock() + fake.LoggerStub = nil + fake.loggerReturns = struct { + result1 logger.Logger + }{result1} +} + +func (fake *FakeMediaTrack) LoggerReturnsOnCall(i int, result1 logger.Logger) { + fake.loggerMutex.Lock() + defer fake.loggerMutex.Unlock() + fake.LoggerStub = nil + if fake.loggerReturnsOnCall == nil { + fake.loggerReturnsOnCall = make(map[int]struct { + result1 logger.Logger + }) + } + fake.loggerReturnsOnCall[i] = struct { + result1 logger.Logger + }{result1} +} + func (fake *FakeMediaTrack) Name() string { fake.nameMutex.Lock() ret, specificReturn := fake.nameReturnsOnCall[len(fake.nameArgsForCall)] @@ -1796,68 +1862,6 @@ func (fake *FakeMediaTrack) UpdateVideoTrackArgsForCall(i int) *livekit.UpdateLo func (fake *FakeMediaTrack) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() - fake.addOnCloseMutex.RLock() - defer fake.addOnCloseMutex.RUnlock() - fake.addSubscriberMutex.RLock() - defer fake.addSubscriberMutex.RUnlock() - fake.clearAllReceiversMutex.RLock() - defer fake.clearAllReceiversMutex.RUnlock() - fake.closeMutex.RLock() - defer fake.closeMutex.RUnlock() - fake.getAllSubscribersMutex.RLock() - defer fake.getAllSubscribersMutex.RUnlock() - fake.getAudioLevelMutex.RLock() - defer fake.getAudioLevelMutex.RUnlock() - fake.getNumSubscribersMutex.RLock() - defer fake.getNumSubscribersMutex.RUnlock() - fake.getQualityForDimensionMutex.RLock() - defer fake.getQualityForDimensionMutex.RUnlock() - fake.getTemporalLayerForSpatialFpsMutex.RLock() - defer fake.getTemporalLayerForSpatialFpsMutex.RUnlock() - fake.iDMutex.RLock() - defer fake.iDMutex.RUnlock() - fake.isEncryptedMutex.RLock() - defer fake.isEncryptedMutex.RUnlock() - fake.isMutedMutex.RLock() - defer fake.isMutedMutex.RUnlock() - fake.isOpenMutex.RLock() - defer fake.isOpenMutex.RUnlock() - fake.isSimulcastMutex.RLock() - defer fake.isSimulcastMutex.RUnlock() - fake.isSubscriberMutex.RLock() - defer fake.isSubscriberMutex.RUnlock() - fake.kindMutex.RLock() - defer fake.kindMutex.RUnlock() - fake.nameMutex.RLock() - defer fake.nameMutex.RUnlock() - fake.onTrackSubscribedMutex.RLock() - defer fake.onTrackSubscribedMutex.RUnlock() - fake.publisherIDMutex.RLock() - defer fake.publisherIDMutex.RUnlock() - fake.publisherIdentityMutex.RLock() - defer fake.publisherIdentityMutex.RUnlock() - fake.publisherVersionMutex.RLock() - defer fake.publisherVersionMutex.RUnlock() - fake.receiversMutex.RLock() - defer fake.receiversMutex.RUnlock() - fake.removeSubscriberMutex.RLock() - defer fake.removeSubscriberMutex.RUnlock() - fake.revokeDisallowedSubscribersMutex.RLock() - defer fake.revokeDisallowedSubscribersMutex.RUnlock() - fake.setMutedMutex.RLock() - defer fake.setMutedMutex.RUnlock() - fake.sourceMutex.RLock() - defer fake.sourceMutex.RUnlock() - fake.streamMutex.RLock() - defer fake.streamMutex.RUnlock() - fake.toProtoMutex.RLock() - defer fake.toProtoMutex.RUnlock() - fake.updateAudioTrackMutex.RLock() - defer fake.updateAudioTrackMutex.RUnlock() - fake.updateTrackInfoMutex.RLock() - defer fake.updateTrackInfoMutex.RUnlock() - fake.updateVideoTrackMutex.RLock() - defer fake.updateVideoTrackMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} for key, value := range fake.invocations { copiedInvocations[key] = value diff --git a/pkg/rtc/types/typesfakes/fake_participant.go b/pkg/rtc/types/typesfakes/fake_participant.go index 9ec45f4..19498c9 100644 --- a/pkg/rtc/types/typesfakes/fake_participant.go +++ b/pkg/rtc/types/typesfakes/fake_participant.go @@ -5,8 +5,10 @@ import ( "sync" "time" + "github.com/livekit/livekit-server/pkg/rtc/datatrack" "github.com/livekit/livekit-server/pkg/rtc/types" "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/logger" "github.com/livekit/protocol/utils" ) @@ -54,15 +56,15 @@ type FakeParticipant struct { connectedAtReturnsOnCall map[int]struct { result1 time.Time } - DebugInfoStub func() map[string]interface{} + DebugInfoStub func() map[string]any debugInfoMutex sync.RWMutex debugInfoArgsForCall []struct { } debugInfoReturns struct { - result1 map[string]interface{} + result1 map[string]any } debugInfoReturnsOnCall map[int]struct { - result1 map[string]interface{} + result1 map[string]any } GetAudioLevelStub func() (float64, bool) getAudioLevelMutex sync.RWMutex @@ -76,6 +78,47 @@ type FakeParticipant struct { result1 float64 result2 bool } + GetLoggerStub func() logger.Logger + getLoggerMutex sync.RWMutex + getLoggerArgsForCall []struct { + } + getLoggerReturns struct { + result1 logger.Logger + } + getLoggerReturnsOnCall map[int]struct { + result1 logger.Logger + } + GetParticipantListenerStub func() types.ParticipantListener + getParticipantListenerMutex sync.RWMutex + getParticipantListenerArgsForCall []struct { + } + getParticipantListenerReturns struct { + result1 types.ParticipantListener + } + getParticipantListenerReturnsOnCall map[int]struct { + result1 types.ParticipantListener + } + GetPublishedDataTrackStub func(uint16) types.DataTrack + getPublishedDataTrackMutex sync.RWMutex + getPublishedDataTrackArgsForCall []struct { + arg1 uint16 + } + getPublishedDataTrackReturns struct { + result1 types.DataTrack + } + getPublishedDataTrackReturnsOnCall map[int]struct { + result1 types.DataTrack + } + GetPublishedDataTracksStub func() []types.DataTrack + getPublishedDataTracksMutex sync.RWMutex + getPublishedDataTracksArgsForCall []struct { + } + getPublishedDataTracksReturns struct { + result1 []types.DataTrack + } + getPublishedDataTracksReturnsOnCall map[int]struct { + result1 []types.DataTrack + } GetPublishedTrackStub func(livekit.TrackID) types.MediaTrack getPublishedTrackMutex sync.RWMutex getPublishedTrackArgsForCall []struct { @@ -97,6 +140,13 @@ type FakeParticipant struct { getPublishedTracksReturnsOnCall map[int]struct { result1 []types.MediaTrack } + HandleReceivedDataTrackMessageStub func([]byte, *datatrack.Packet, int64) + handleReceivedDataTrackMessageMutex sync.RWMutex + handleReceivedDataTrackMessageArgsForCall []struct { + arg1 []byte + arg2 *datatrack.Packet + arg3 int64 + } HasPermissionStub func(livekit.TrackID, livekit.ParticipantIdentity) bool hasPermissionMutex sync.RWMutex hasPermissionArgsForCall []struct { @@ -149,6 +199,16 @@ type FakeParticipant struct { isAgentReturnsOnCall map[int]struct { result1 bool } + IsClosedStub func() bool + isClosedMutex sync.RWMutex + isClosedArgsForCall []struct { + } + isClosedReturns struct { + result1 bool + } + isClosedReturnsOnCall map[int]struct { + result1 bool + } IsDependentStub func() bool isDependentMutex sync.RWMutex isDependentArgsForCall []struct { @@ -159,6 +219,16 @@ type FakeParticipant struct { isDependentReturnsOnCall map[int]struct { result1 bool } + IsDisconnectedStub func() bool + isDisconnectedMutex sync.RWMutex + isDisconnectedArgsForCall []struct { + } + isDisconnectedReturns struct { + result1 bool + } + isDisconnectedReturnsOnCall map[int]struct { + result1 bool + } IsPublisherStub func() bool isPublisherMutex sync.RWMutex isPublisherArgsForCall []struct { @@ -189,17 +259,36 @@ type FakeParticipant struct { kindReturnsOnCall map[int]struct { result1 livekit.ParticipantInfo_Kind } - OnMetricsStub func(func(types.Participant, *livekit.DataPacket)) - onMetricsMutex sync.RWMutex - onMetricsArgsForCall []struct { - arg1 func(types.Participant, *livekit.DataPacket) + KindDetailsStub func() []livekit.ParticipantInfo_KindDetail + kindDetailsMutex sync.RWMutex + kindDetailsArgsForCall []struct { } - RemovePublishedTrackStub func(types.MediaTrack, bool, bool) + kindDetailsReturns struct { + result1 []livekit.ParticipantInfo_KindDetail + } + kindDetailsReturnsOnCall map[int]struct { + result1 []livekit.ParticipantInfo_KindDetail + } + MigrateStateStub func() types.MigrateState + migrateStateMutex sync.RWMutex + migrateStateArgsForCall []struct { + } + migrateStateReturns struct { + result1 types.MigrateState + } + migrateStateReturnsOnCall map[int]struct { + result1 types.MigrateState + } + RemovePublishedDataTrackStub func(types.DataTrack) + removePublishedDataTrackMutex sync.RWMutex + removePublishedDataTrackArgsForCall []struct { + arg1 types.DataTrack + } + RemovePublishedTrackStub func(types.MediaTrack, bool) removePublishedTrackMutex sync.RWMutex removePublishedTrackArgsForCall []struct { arg1 types.MediaTrack arg2 bool - arg3 bool } StateStub func() livekit.ParticipantInfo_State stateMutex sync.RWMutex @@ -233,6 +322,18 @@ type FakeParticipant struct { toProtoReturnsOnCall map[int]struct { result1 *livekit.ParticipantInfo } + ToProtoWithVersionStub func() (*livekit.ParticipantInfo, utils.TimedVersion) + toProtoWithVersionMutex sync.RWMutex + toProtoWithVersionArgsForCall []struct { + } + toProtoWithVersionReturns struct { + result1 *livekit.ParticipantInfo + result2 utils.TimedVersion + } + toProtoWithVersionReturnsOnCall map[int]struct { + result1 *livekit.ParticipantInfo + result2 utils.TimedVersion + } UpdateSubscriptionPermissionStub func(*livekit.SubscriptionPermission, utils.TimedVersion, func(participantID livekit.ParticipantID) types.LocalParticipant) error updateSubscriptionPermissionMutex sync.RWMutex updateSubscriptionPermissionArgsForCall []struct { @@ -482,7 +583,7 @@ func (fake *FakeParticipant) ConnectedAtReturnsOnCall(i int, result1 time.Time) }{result1} } -func (fake *FakeParticipant) DebugInfo() map[string]interface{} { +func (fake *FakeParticipant) DebugInfo() map[string]any { fake.debugInfoMutex.Lock() ret, specificReturn := fake.debugInfoReturnsOnCall[len(fake.debugInfoArgsForCall)] fake.debugInfoArgsForCall = append(fake.debugInfoArgsForCall, struct { @@ -506,32 +607,32 @@ func (fake *FakeParticipant) DebugInfoCallCount() int { return len(fake.debugInfoArgsForCall) } -func (fake *FakeParticipant) DebugInfoCalls(stub func() map[string]interface{}) { +func (fake *FakeParticipant) DebugInfoCalls(stub func() map[string]any) { fake.debugInfoMutex.Lock() defer fake.debugInfoMutex.Unlock() fake.DebugInfoStub = stub } -func (fake *FakeParticipant) DebugInfoReturns(result1 map[string]interface{}) { +func (fake *FakeParticipant) DebugInfoReturns(result1 map[string]any) { fake.debugInfoMutex.Lock() defer fake.debugInfoMutex.Unlock() fake.DebugInfoStub = nil fake.debugInfoReturns = struct { - result1 map[string]interface{} + result1 map[string]any }{result1} } -func (fake *FakeParticipant) DebugInfoReturnsOnCall(i int, result1 map[string]interface{}) { +func (fake *FakeParticipant) DebugInfoReturnsOnCall(i int, result1 map[string]any) { fake.debugInfoMutex.Lock() defer fake.debugInfoMutex.Unlock() fake.DebugInfoStub = nil if fake.debugInfoReturnsOnCall == nil { fake.debugInfoReturnsOnCall = make(map[int]struct { - result1 map[string]interface{} + result1 map[string]any }) } fake.debugInfoReturnsOnCall[i] = struct { - result1 map[string]interface{} + result1 map[string]any }{result1} } @@ -591,6 +692,226 @@ func (fake *FakeParticipant) GetAudioLevelReturnsOnCall(i int, result1 float64, }{result1, result2} } +func (fake *FakeParticipant) GetLogger() logger.Logger { + fake.getLoggerMutex.Lock() + ret, specificReturn := fake.getLoggerReturnsOnCall[len(fake.getLoggerArgsForCall)] + fake.getLoggerArgsForCall = append(fake.getLoggerArgsForCall, struct { + }{}) + stub := fake.GetLoggerStub + fakeReturns := fake.getLoggerReturns + fake.recordInvocation("GetLogger", []interface{}{}) + fake.getLoggerMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeParticipant) GetLoggerCallCount() int { + fake.getLoggerMutex.RLock() + defer fake.getLoggerMutex.RUnlock() + return len(fake.getLoggerArgsForCall) +} + +func (fake *FakeParticipant) GetLoggerCalls(stub func() logger.Logger) { + fake.getLoggerMutex.Lock() + defer fake.getLoggerMutex.Unlock() + fake.GetLoggerStub = stub +} + +func (fake *FakeParticipant) GetLoggerReturns(result1 logger.Logger) { + fake.getLoggerMutex.Lock() + defer fake.getLoggerMutex.Unlock() + fake.GetLoggerStub = nil + fake.getLoggerReturns = struct { + result1 logger.Logger + }{result1} +} + +func (fake *FakeParticipant) GetLoggerReturnsOnCall(i int, result1 logger.Logger) { + fake.getLoggerMutex.Lock() + defer fake.getLoggerMutex.Unlock() + fake.GetLoggerStub = nil + if fake.getLoggerReturnsOnCall == nil { + fake.getLoggerReturnsOnCall = make(map[int]struct { + result1 logger.Logger + }) + } + fake.getLoggerReturnsOnCall[i] = struct { + result1 logger.Logger + }{result1} +} + +func (fake *FakeParticipant) GetParticipantListener() types.ParticipantListener { + fake.getParticipantListenerMutex.Lock() + ret, specificReturn := fake.getParticipantListenerReturnsOnCall[len(fake.getParticipantListenerArgsForCall)] + fake.getParticipantListenerArgsForCall = append(fake.getParticipantListenerArgsForCall, struct { + }{}) + stub := fake.GetParticipantListenerStub + fakeReturns := fake.getParticipantListenerReturns + fake.recordInvocation("GetParticipantListener", []interface{}{}) + fake.getParticipantListenerMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeParticipant) GetParticipantListenerCallCount() int { + fake.getParticipantListenerMutex.RLock() + defer fake.getParticipantListenerMutex.RUnlock() + return len(fake.getParticipantListenerArgsForCall) +} + +func (fake *FakeParticipant) GetParticipantListenerCalls(stub func() types.ParticipantListener) { + fake.getParticipantListenerMutex.Lock() + defer fake.getParticipantListenerMutex.Unlock() + fake.GetParticipantListenerStub = stub +} + +func (fake *FakeParticipant) GetParticipantListenerReturns(result1 types.ParticipantListener) { + fake.getParticipantListenerMutex.Lock() + defer fake.getParticipantListenerMutex.Unlock() + fake.GetParticipantListenerStub = nil + fake.getParticipantListenerReturns = struct { + result1 types.ParticipantListener + }{result1} +} + +func (fake *FakeParticipant) GetParticipantListenerReturnsOnCall(i int, result1 types.ParticipantListener) { + fake.getParticipantListenerMutex.Lock() + defer fake.getParticipantListenerMutex.Unlock() + fake.GetParticipantListenerStub = nil + if fake.getParticipantListenerReturnsOnCall == nil { + fake.getParticipantListenerReturnsOnCall = make(map[int]struct { + result1 types.ParticipantListener + }) + } + fake.getParticipantListenerReturnsOnCall[i] = struct { + result1 types.ParticipantListener + }{result1} +} + +func (fake *FakeParticipant) GetPublishedDataTrack(arg1 uint16) types.DataTrack { + fake.getPublishedDataTrackMutex.Lock() + ret, specificReturn := fake.getPublishedDataTrackReturnsOnCall[len(fake.getPublishedDataTrackArgsForCall)] + fake.getPublishedDataTrackArgsForCall = append(fake.getPublishedDataTrackArgsForCall, struct { + arg1 uint16 + }{arg1}) + stub := fake.GetPublishedDataTrackStub + fakeReturns := fake.getPublishedDataTrackReturns + fake.recordInvocation("GetPublishedDataTrack", []interface{}{arg1}) + fake.getPublishedDataTrackMutex.Unlock() + if stub != nil { + return stub(arg1) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeParticipant) GetPublishedDataTrackCallCount() int { + fake.getPublishedDataTrackMutex.RLock() + defer fake.getPublishedDataTrackMutex.RUnlock() + return len(fake.getPublishedDataTrackArgsForCall) +} + +func (fake *FakeParticipant) GetPublishedDataTrackCalls(stub func(uint16) types.DataTrack) { + fake.getPublishedDataTrackMutex.Lock() + defer fake.getPublishedDataTrackMutex.Unlock() + fake.GetPublishedDataTrackStub = stub +} + +func (fake *FakeParticipant) GetPublishedDataTrackArgsForCall(i int) uint16 { + fake.getPublishedDataTrackMutex.RLock() + defer fake.getPublishedDataTrackMutex.RUnlock() + argsForCall := fake.getPublishedDataTrackArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeParticipant) GetPublishedDataTrackReturns(result1 types.DataTrack) { + fake.getPublishedDataTrackMutex.Lock() + defer fake.getPublishedDataTrackMutex.Unlock() + fake.GetPublishedDataTrackStub = nil + fake.getPublishedDataTrackReturns = struct { + result1 types.DataTrack + }{result1} +} + +func (fake *FakeParticipant) GetPublishedDataTrackReturnsOnCall(i int, result1 types.DataTrack) { + fake.getPublishedDataTrackMutex.Lock() + defer fake.getPublishedDataTrackMutex.Unlock() + fake.GetPublishedDataTrackStub = nil + if fake.getPublishedDataTrackReturnsOnCall == nil { + fake.getPublishedDataTrackReturnsOnCall = make(map[int]struct { + result1 types.DataTrack + }) + } + fake.getPublishedDataTrackReturnsOnCall[i] = struct { + result1 types.DataTrack + }{result1} +} + +func (fake *FakeParticipant) GetPublishedDataTracks() []types.DataTrack { + fake.getPublishedDataTracksMutex.Lock() + ret, specificReturn := fake.getPublishedDataTracksReturnsOnCall[len(fake.getPublishedDataTracksArgsForCall)] + fake.getPublishedDataTracksArgsForCall = append(fake.getPublishedDataTracksArgsForCall, struct { + }{}) + stub := fake.GetPublishedDataTracksStub + fakeReturns := fake.getPublishedDataTracksReturns + fake.recordInvocation("GetPublishedDataTracks", []interface{}{}) + fake.getPublishedDataTracksMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeParticipant) GetPublishedDataTracksCallCount() int { + fake.getPublishedDataTracksMutex.RLock() + defer fake.getPublishedDataTracksMutex.RUnlock() + return len(fake.getPublishedDataTracksArgsForCall) +} + +func (fake *FakeParticipant) GetPublishedDataTracksCalls(stub func() []types.DataTrack) { + fake.getPublishedDataTracksMutex.Lock() + defer fake.getPublishedDataTracksMutex.Unlock() + fake.GetPublishedDataTracksStub = stub +} + +func (fake *FakeParticipant) GetPublishedDataTracksReturns(result1 []types.DataTrack) { + fake.getPublishedDataTracksMutex.Lock() + defer fake.getPublishedDataTracksMutex.Unlock() + fake.GetPublishedDataTracksStub = nil + fake.getPublishedDataTracksReturns = struct { + result1 []types.DataTrack + }{result1} +} + +func (fake *FakeParticipant) GetPublishedDataTracksReturnsOnCall(i int, result1 []types.DataTrack) { + fake.getPublishedDataTracksMutex.Lock() + defer fake.getPublishedDataTracksMutex.Unlock() + fake.GetPublishedDataTracksStub = nil + if fake.getPublishedDataTracksReturnsOnCall == nil { + fake.getPublishedDataTracksReturnsOnCall = make(map[int]struct { + result1 []types.DataTrack + }) + } + fake.getPublishedDataTracksReturnsOnCall[i] = struct { + result1 []types.DataTrack + }{result1} +} + func (fake *FakeParticipant) GetPublishedTrack(arg1 livekit.TrackID) types.MediaTrack { fake.getPublishedTrackMutex.Lock() ret, specificReturn := fake.getPublishedTrackReturnsOnCall[len(fake.getPublishedTrackArgsForCall)] @@ -705,6 +1026,45 @@ func (fake *FakeParticipant) GetPublishedTracksReturnsOnCall(i int, result1 []ty }{result1} } +func (fake *FakeParticipant) HandleReceivedDataTrackMessage(arg1 []byte, arg2 *datatrack.Packet, arg3 int64) { + var arg1Copy []byte + if arg1 != nil { + arg1Copy = make([]byte, len(arg1)) + copy(arg1Copy, arg1) + } + fake.handleReceivedDataTrackMessageMutex.Lock() + fake.handleReceivedDataTrackMessageArgsForCall = append(fake.handleReceivedDataTrackMessageArgsForCall, struct { + arg1 []byte + arg2 *datatrack.Packet + arg3 int64 + }{arg1Copy, arg2, arg3}) + stub := fake.HandleReceivedDataTrackMessageStub + fake.recordInvocation("HandleReceivedDataTrackMessage", []interface{}{arg1Copy, arg2, arg3}) + fake.handleReceivedDataTrackMessageMutex.Unlock() + if stub != nil { + fake.HandleReceivedDataTrackMessageStub(arg1, arg2, arg3) + } +} + +func (fake *FakeParticipant) HandleReceivedDataTrackMessageCallCount() int { + fake.handleReceivedDataTrackMessageMutex.RLock() + defer fake.handleReceivedDataTrackMessageMutex.RUnlock() + return len(fake.handleReceivedDataTrackMessageArgsForCall) +} + +func (fake *FakeParticipant) HandleReceivedDataTrackMessageCalls(stub func([]byte, *datatrack.Packet, int64)) { + fake.handleReceivedDataTrackMessageMutex.Lock() + defer fake.handleReceivedDataTrackMessageMutex.Unlock() + fake.HandleReceivedDataTrackMessageStub = stub +} + +func (fake *FakeParticipant) HandleReceivedDataTrackMessageArgsForCall(i int) ([]byte, *datatrack.Packet, int64) { + fake.handleReceivedDataTrackMessageMutex.RLock() + defer fake.handleReceivedDataTrackMessageMutex.RUnlock() + argsForCall := fake.handleReceivedDataTrackMessageArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 +} + func (fake *FakeParticipant) HasPermission(arg1 livekit.TrackID, arg2 livekit.ParticipantIdentity) bool { fake.hasPermissionMutex.Lock() ret, specificReturn := fake.hasPermissionReturnsOnCall[len(fake.hasPermissionArgsForCall)] @@ -979,6 +1339,59 @@ func (fake *FakeParticipant) IsAgentReturnsOnCall(i int, result1 bool) { }{result1} } +func (fake *FakeParticipant) IsClosed() bool { + fake.isClosedMutex.Lock() + ret, specificReturn := fake.isClosedReturnsOnCall[len(fake.isClosedArgsForCall)] + fake.isClosedArgsForCall = append(fake.isClosedArgsForCall, struct { + }{}) + stub := fake.IsClosedStub + fakeReturns := fake.isClosedReturns + fake.recordInvocation("IsClosed", []interface{}{}) + fake.isClosedMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeParticipant) IsClosedCallCount() int { + fake.isClosedMutex.RLock() + defer fake.isClosedMutex.RUnlock() + return len(fake.isClosedArgsForCall) +} + +func (fake *FakeParticipant) IsClosedCalls(stub func() bool) { + fake.isClosedMutex.Lock() + defer fake.isClosedMutex.Unlock() + fake.IsClosedStub = stub +} + +func (fake *FakeParticipant) IsClosedReturns(result1 bool) { + fake.isClosedMutex.Lock() + defer fake.isClosedMutex.Unlock() + fake.IsClosedStub = nil + fake.isClosedReturns = struct { + result1 bool + }{result1} +} + +func (fake *FakeParticipant) IsClosedReturnsOnCall(i int, result1 bool) { + fake.isClosedMutex.Lock() + defer fake.isClosedMutex.Unlock() + fake.IsClosedStub = nil + if fake.isClosedReturnsOnCall == nil { + fake.isClosedReturnsOnCall = make(map[int]struct { + result1 bool + }) + } + fake.isClosedReturnsOnCall[i] = struct { + result1 bool + }{result1} +} + func (fake *FakeParticipant) IsDependent() bool { fake.isDependentMutex.Lock() ret, specificReturn := fake.isDependentReturnsOnCall[len(fake.isDependentArgsForCall)] @@ -1032,6 +1445,59 @@ func (fake *FakeParticipant) IsDependentReturnsOnCall(i int, result1 bool) { }{result1} } +func (fake *FakeParticipant) IsDisconnected() bool { + fake.isDisconnectedMutex.Lock() + ret, specificReturn := fake.isDisconnectedReturnsOnCall[len(fake.isDisconnectedArgsForCall)] + fake.isDisconnectedArgsForCall = append(fake.isDisconnectedArgsForCall, struct { + }{}) + stub := fake.IsDisconnectedStub + fakeReturns := fake.isDisconnectedReturns + fake.recordInvocation("IsDisconnected", []interface{}{}) + fake.isDisconnectedMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeParticipant) IsDisconnectedCallCount() int { + fake.isDisconnectedMutex.RLock() + defer fake.isDisconnectedMutex.RUnlock() + return len(fake.isDisconnectedArgsForCall) +} + +func (fake *FakeParticipant) IsDisconnectedCalls(stub func() bool) { + fake.isDisconnectedMutex.Lock() + defer fake.isDisconnectedMutex.Unlock() + fake.IsDisconnectedStub = stub +} + +func (fake *FakeParticipant) IsDisconnectedReturns(result1 bool) { + fake.isDisconnectedMutex.Lock() + defer fake.isDisconnectedMutex.Unlock() + fake.IsDisconnectedStub = nil + fake.isDisconnectedReturns = struct { + result1 bool + }{result1} +} + +func (fake *FakeParticipant) IsDisconnectedReturnsOnCall(i int, result1 bool) { + fake.isDisconnectedMutex.Lock() + defer fake.isDisconnectedMutex.Unlock() + fake.IsDisconnectedStub = nil + if fake.isDisconnectedReturnsOnCall == nil { + fake.isDisconnectedReturnsOnCall = make(map[int]struct { + result1 bool + }) + } + fake.isDisconnectedReturnsOnCall[i] = struct { + result1 bool + }{result1} +} + func (fake *FakeParticipant) IsPublisher() bool { fake.isPublisherMutex.Lock() ret, specificReturn := fake.isPublisherReturnsOnCall[len(fake.isPublisherArgsForCall)] @@ -1191,50 +1657,155 @@ func (fake *FakeParticipant) KindReturnsOnCall(i int, result1 livekit.Participan }{result1} } -func (fake *FakeParticipant) OnMetrics(arg1 func(types.Participant, *livekit.DataPacket)) { - fake.onMetricsMutex.Lock() - fake.onMetricsArgsForCall = append(fake.onMetricsArgsForCall, struct { - arg1 func(types.Participant, *livekit.DataPacket) - }{arg1}) - stub := fake.OnMetricsStub - fake.recordInvocation("OnMetrics", []interface{}{arg1}) - fake.onMetricsMutex.Unlock() +func (fake *FakeParticipant) KindDetails() []livekit.ParticipantInfo_KindDetail { + fake.kindDetailsMutex.Lock() + ret, specificReturn := fake.kindDetailsReturnsOnCall[len(fake.kindDetailsArgsForCall)] + fake.kindDetailsArgsForCall = append(fake.kindDetailsArgsForCall, struct { + }{}) + stub := fake.KindDetailsStub + fakeReturns := fake.kindDetailsReturns + fake.recordInvocation("KindDetails", []interface{}{}) + fake.kindDetailsMutex.Unlock() if stub != nil { - fake.OnMetricsStub(arg1) + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeParticipant) KindDetailsCallCount() int { + fake.kindDetailsMutex.RLock() + defer fake.kindDetailsMutex.RUnlock() + return len(fake.kindDetailsArgsForCall) +} + +func (fake *FakeParticipant) KindDetailsCalls(stub func() []livekit.ParticipantInfo_KindDetail) { + fake.kindDetailsMutex.Lock() + defer fake.kindDetailsMutex.Unlock() + fake.KindDetailsStub = stub +} + +func (fake *FakeParticipant) KindDetailsReturns(result1 []livekit.ParticipantInfo_KindDetail) { + fake.kindDetailsMutex.Lock() + defer fake.kindDetailsMutex.Unlock() + fake.KindDetailsStub = nil + fake.kindDetailsReturns = struct { + result1 []livekit.ParticipantInfo_KindDetail + }{result1} +} + +func (fake *FakeParticipant) KindDetailsReturnsOnCall(i int, result1 []livekit.ParticipantInfo_KindDetail) { + fake.kindDetailsMutex.Lock() + defer fake.kindDetailsMutex.Unlock() + fake.KindDetailsStub = nil + if fake.kindDetailsReturnsOnCall == nil { + fake.kindDetailsReturnsOnCall = make(map[int]struct { + result1 []livekit.ParticipantInfo_KindDetail + }) + } + fake.kindDetailsReturnsOnCall[i] = struct { + result1 []livekit.ParticipantInfo_KindDetail + }{result1} +} + +func (fake *FakeParticipant) MigrateState() types.MigrateState { + fake.migrateStateMutex.Lock() + ret, specificReturn := fake.migrateStateReturnsOnCall[len(fake.migrateStateArgsForCall)] + fake.migrateStateArgsForCall = append(fake.migrateStateArgsForCall, struct { + }{}) + stub := fake.MigrateStateStub + fakeReturns := fake.migrateStateReturns + fake.recordInvocation("MigrateState", []interface{}{}) + fake.migrateStateMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeParticipant) MigrateStateCallCount() int { + fake.migrateStateMutex.RLock() + defer fake.migrateStateMutex.RUnlock() + return len(fake.migrateStateArgsForCall) +} + +func (fake *FakeParticipant) MigrateStateCalls(stub func() types.MigrateState) { + fake.migrateStateMutex.Lock() + defer fake.migrateStateMutex.Unlock() + fake.MigrateStateStub = stub +} + +func (fake *FakeParticipant) MigrateStateReturns(result1 types.MigrateState) { + fake.migrateStateMutex.Lock() + defer fake.migrateStateMutex.Unlock() + fake.MigrateStateStub = nil + fake.migrateStateReturns = struct { + result1 types.MigrateState + }{result1} +} + +func (fake *FakeParticipant) MigrateStateReturnsOnCall(i int, result1 types.MigrateState) { + fake.migrateStateMutex.Lock() + defer fake.migrateStateMutex.Unlock() + fake.MigrateStateStub = nil + if fake.migrateStateReturnsOnCall == nil { + fake.migrateStateReturnsOnCall = make(map[int]struct { + result1 types.MigrateState + }) + } + fake.migrateStateReturnsOnCall[i] = struct { + result1 types.MigrateState + }{result1} +} + +func (fake *FakeParticipant) RemovePublishedDataTrack(arg1 types.DataTrack) { + fake.removePublishedDataTrackMutex.Lock() + fake.removePublishedDataTrackArgsForCall = append(fake.removePublishedDataTrackArgsForCall, struct { + arg1 types.DataTrack + }{arg1}) + stub := fake.RemovePublishedDataTrackStub + fake.recordInvocation("RemovePublishedDataTrack", []interface{}{arg1}) + fake.removePublishedDataTrackMutex.Unlock() + if stub != nil { + fake.RemovePublishedDataTrackStub(arg1) } } -func (fake *FakeParticipant) OnMetricsCallCount() int { - fake.onMetricsMutex.RLock() - defer fake.onMetricsMutex.RUnlock() - return len(fake.onMetricsArgsForCall) +func (fake *FakeParticipant) RemovePublishedDataTrackCallCount() int { + fake.removePublishedDataTrackMutex.RLock() + defer fake.removePublishedDataTrackMutex.RUnlock() + return len(fake.removePublishedDataTrackArgsForCall) } -func (fake *FakeParticipant) OnMetricsCalls(stub func(func(types.Participant, *livekit.DataPacket))) { - fake.onMetricsMutex.Lock() - defer fake.onMetricsMutex.Unlock() - fake.OnMetricsStub = stub +func (fake *FakeParticipant) RemovePublishedDataTrackCalls(stub func(types.DataTrack)) { + fake.removePublishedDataTrackMutex.Lock() + defer fake.removePublishedDataTrackMutex.Unlock() + fake.RemovePublishedDataTrackStub = stub } -func (fake *FakeParticipant) OnMetricsArgsForCall(i int) func(types.Participant, *livekit.DataPacket) { - fake.onMetricsMutex.RLock() - defer fake.onMetricsMutex.RUnlock() - argsForCall := fake.onMetricsArgsForCall[i] +func (fake *FakeParticipant) RemovePublishedDataTrackArgsForCall(i int) types.DataTrack { + fake.removePublishedDataTrackMutex.RLock() + defer fake.removePublishedDataTrackMutex.RUnlock() + argsForCall := fake.removePublishedDataTrackArgsForCall[i] return argsForCall.arg1 } -func (fake *FakeParticipant) RemovePublishedTrack(arg1 types.MediaTrack, arg2 bool, arg3 bool) { +func (fake *FakeParticipant) RemovePublishedTrack(arg1 types.MediaTrack, arg2 bool) { fake.removePublishedTrackMutex.Lock() fake.removePublishedTrackArgsForCall = append(fake.removePublishedTrackArgsForCall, struct { arg1 types.MediaTrack arg2 bool - arg3 bool - }{arg1, arg2, arg3}) + }{arg1, arg2}) stub := fake.RemovePublishedTrackStub - fake.recordInvocation("RemovePublishedTrack", []interface{}{arg1, arg2, arg3}) + fake.recordInvocation("RemovePublishedTrack", []interface{}{arg1, arg2}) fake.removePublishedTrackMutex.Unlock() if stub != nil { - fake.RemovePublishedTrackStub(arg1, arg2, arg3) + fake.RemovePublishedTrackStub(arg1, arg2) } } @@ -1244,17 +1815,17 @@ func (fake *FakeParticipant) RemovePublishedTrackCallCount() int { return len(fake.removePublishedTrackArgsForCall) } -func (fake *FakeParticipant) RemovePublishedTrackCalls(stub func(types.MediaTrack, bool, bool)) { +func (fake *FakeParticipant) RemovePublishedTrackCalls(stub func(types.MediaTrack, bool)) { fake.removePublishedTrackMutex.Lock() defer fake.removePublishedTrackMutex.Unlock() fake.RemovePublishedTrackStub = stub } -func (fake *FakeParticipant) RemovePublishedTrackArgsForCall(i int) (types.MediaTrack, bool, bool) { +func (fake *FakeParticipant) RemovePublishedTrackArgsForCall(i int) (types.MediaTrack, bool) { fake.removePublishedTrackMutex.RLock() defer fake.removePublishedTrackMutex.RUnlock() argsForCall := fake.removePublishedTrackArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 + return argsForCall.arg1, argsForCall.arg2 } func (fake *FakeParticipant) State() livekit.ParticipantInfo_State { @@ -1419,6 +1990,62 @@ func (fake *FakeParticipant) ToProtoReturnsOnCall(i int, result1 *livekit.Partic }{result1} } +func (fake *FakeParticipant) ToProtoWithVersion() (*livekit.ParticipantInfo, utils.TimedVersion) { + fake.toProtoWithVersionMutex.Lock() + ret, specificReturn := fake.toProtoWithVersionReturnsOnCall[len(fake.toProtoWithVersionArgsForCall)] + fake.toProtoWithVersionArgsForCall = append(fake.toProtoWithVersionArgsForCall, struct { + }{}) + stub := fake.ToProtoWithVersionStub + fakeReturns := fake.toProtoWithVersionReturns + fake.recordInvocation("ToProtoWithVersion", []interface{}{}) + fake.toProtoWithVersionMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeParticipant) ToProtoWithVersionCallCount() int { + fake.toProtoWithVersionMutex.RLock() + defer fake.toProtoWithVersionMutex.RUnlock() + return len(fake.toProtoWithVersionArgsForCall) +} + +func (fake *FakeParticipant) ToProtoWithVersionCalls(stub func() (*livekit.ParticipantInfo, utils.TimedVersion)) { + fake.toProtoWithVersionMutex.Lock() + defer fake.toProtoWithVersionMutex.Unlock() + fake.ToProtoWithVersionStub = stub +} + +func (fake *FakeParticipant) ToProtoWithVersionReturns(result1 *livekit.ParticipantInfo, result2 utils.TimedVersion) { + fake.toProtoWithVersionMutex.Lock() + defer fake.toProtoWithVersionMutex.Unlock() + fake.ToProtoWithVersionStub = nil + fake.toProtoWithVersionReturns = struct { + result1 *livekit.ParticipantInfo + result2 utils.TimedVersion + }{result1, result2} +} + +func (fake *FakeParticipant) ToProtoWithVersionReturnsOnCall(i int, result1 *livekit.ParticipantInfo, result2 utils.TimedVersion) { + fake.toProtoWithVersionMutex.Lock() + defer fake.toProtoWithVersionMutex.Unlock() + fake.ToProtoWithVersionStub = nil + if fake.toProtoWithVersionReturnsOnCall == nil { + fake.toProtoWithVersionReturnsOnCall = make(map[int]struct { + result1 *livekit.ParticipantInfo + result2 utils.TimedVersion + }) + } + fake.toProtoWithVersionReturnsOnCall[i] = struct { + result1 *livekit.ParticipantInfo + result2 utils.TimedVersion + }{result1, result2} +} + func (fake *FakeParticipant) UpdateSubscriptionPermission(arg1 *livekit.SubscriptionPermission, arg2 utils.TimedVersion, arg3 func(participantID livekit.ParticipantID) types.LocalParticipant) error { fake.updateSubscriptionPermissionMutex.Lock() ret, specificReturn := fake.updateSubscriptionPermissionReturnsOnCall[len(fake.updateSubscriptionPermissionArgsForCall)] @@ -1538,54 +2165,6 @@ func (fake *FakeParticipant) VersionReturnsOnCall(i int, result1 utils.TimedVers func (fake *FakeParticipant) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() - fake.canSkipBroadcastMutex.RLock() - defer fake.canSkipBroadcastMutex.RUnlock() - fake.closeMutex.RLock() - defer fake.closeMutex.RUnlock() - fake.closeReasonMutex.RLock() - defer fake.closeReasonMutex.RUnlock() - fake.connectedAtMutex.RLock() - defer fake.connectedAtMutex.RUnlock() - fake.debugInfoMutex.RLock() - defer fake.debugInfoMutex.RUnlock() - fake.getAudioLevelMutex.RLock() - defer fake.getAudioLevelMutex.RUnlock() - fake.getPublishedTrackMutex.RLock() - defer fake.getPublishedTrackMutex.RUnlock() - fake.getPublishedTracksMutex.RLock() - defer fake.getPublishedTracksMutex.RUnlock() - fake.hasPermissionMutex.RLock() - defer fake.hasPermissionMutex.RUnlock() - fake.hiddenMutex.RLock() - defer fake.hiddenMutex.RUnlock() - fake.iDMutex.RLock() - defer fake.iDMutex.RUnlock() - fake.identityMutex.RLock() - defer fake.identityMutex.RUnlock() - fake.isAgentMutex.RLock() - defer fake.isAgentMutex.RUnlock() - fake.isDependentMutex.RLock() - defer fake.isDependentMutex.RUnlock() - fake.isPublisherMutex.RLock() - defer fake.isPublisherMutex.RUnlock() - fake.isRecorderMutex.RLock() - defer fake.isRecorderMutex.RUnlock() - fake.kindMutex.RLock() - defer fake.kindMutex.RUnlock() - fake.onMetricsMutex.RLock() - defer fake.onMetricsMutex.RUnlock() - fake.removePublishedTrackMutex.RLock() - defer fake.removePublishedTrackMutex.RUnlock() - fake.stateMutex.RLock() - defer fake.stateMutex.RUnlock() - fake.subscriptionPermissionMutex.RLock() - defer fake.subscriptionPermissionMutex.RUnlock() - fake.toProtoMutex.RLock() - defer fake.toProtoMutex.RUnlock() - fake.updateSubscriptionPermissionMutex.RLock() - defer fake.updateSubscriptionPermissionMutex.RUnlock() - fake.versionMutex.RLock() - defer fake.versionMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} for key, value := range fake.invocations { copiedInvocations[key] = value diff --git a/pkg/rtc/types/typesfakes/fake_participant_listener.go b/pkg/rtc/types/typesfakes/fake_participant_listener.go new file mode 100644 index 0000000..a05a3e0 --- /dev/null +++ b/pkg/rtc/types/typesfakes/fake_participant_listener.go @@ -0,0 +1,356 @@ +// Code generated by counterfeiter. DO NOT EDIT. +package typesfakes + +import ( + "sync" + + "github.com/livekit/livekit-server/pkg/rtc/datatrack" + "github.com/livekit/livekit-server/pkg/rtc/types" + "github.com/livekit/protocol/livekit" +) + +type FakeParticipantListener struct { + OnDataTrackMessageStub func(types.Participant, []byte, *datatrack.Packet) + onDataTrackMessageMutex sync.RWMutex + onDataTrackMessageArgsForCall []struct { + arg1 types.Participant + arg2 []byte + arg3 *datatrack.Packet + } + OnDataTrackPublishedStub func(types.Participant, types.DataTrack) + onDataTrackPublishedMutex sync.RWMutex + onDataTrackPublishedArgsForCall []struct { + arg1 types.Participant + arg2 types.DataTrack + } + OnDataTrackUnpublishedStub func(types.Participant, types.DataTrack) + onDataTrackUnpublishedMutex sync.RWMutex + onDataTrackUnpublishedArgsForCall []struct { + arg1 types.Participant + arg2 types.DataTrack + } + OnMetricsStub func(types.Participant, *livekit.DataPacket) + onMetricsMutex sync.RWMutex + onMetricsArgsForCall []struct { + arg1 types.Participant + arg2 *livekit.DataPacket + } + OnParticipantUpdateStub func(types.Participant) + onParticipantUpdateMutex sync.RWMutex + onParticipantUpdateArgsForCall []struct { + arg1 types.Participant + } + OnTrackPublishedStub func(types.Participant, types.MediaTrack) + onTrackPublishedMutex sync.RWMutex + onTrackPublishedArgsForCall []struct { + arg1 types.Participant + arg2 types.MediaTrack + } + OnTrackUnpublishedStub func(types.Participant, types.MediaTrack) + onTrackUnpublishedMutex sync.RWMutex + onTrackUnpublishedArgsForCall []struct { + arg1 types.Participant + arg2 types.MediaTrack + } + OnTrackUpdatedStub func(types.Participant, types.MediaTrack) + onTrackUpdatedMutex sync.RWMutex + onTrackUpdatedArgsForCall []struct { + arg1 types.Participant + arg2 types.MediaTrack + } + invocations map[string][][]interface{} + invocationsMutex sync.RWMutex +} + +func (fake *FakeParticipantListener) OnDataTrackMessage(arg1 types.Participant, arg2 []byte, arg3 *datatrack.Packet) { + var arg2Copy []byte + if arg2 != nil { + arg2Copy = make([]byte, len(arg2)) + copy(arg2Copy, arg2) + } + fake.onDataTrackMessageMutex.Lock() + fake.onDataTrackMessageArgsForCall = append(fake.onDataTrackMessageArgsForCall, struct { + arg1 types.Participant + arg2 []byte + arg3 *datatrack.Packet + }{arg1, arg2Copy, arg3}) + stub := fake.OnDataTrackMessageStub + fake.recordInvocation("OnDataTrackMessage", []interface{}{arg1, arg2Copy, arg3}) + fake.onDataTrackMessageMutex.Unlock() + if stub != nil { + fake.OnDataTrackMessageStub(arg1, arg2, arg3) + } +} + +func (fake *FakeParticipantListener) OnDataTrackMessageCallCount() int { + fake.onDataTrackMessageMutex.RLock() + defer fake.onDataTrackMessageMutex.RUnlock() + return len(fake.onDataTrackMessageArgsForCall) +} + +func (fake *FakeParticipantListener) OnDataTrackMessageCalls(stub func(types.Participant, []byte, *datatrack.Packet)) { + fake.onDataTrackMessageMutex.Lock() + defer fake.onDataTrackMessageMutex.Unlock() + fake.OnDataTrackMessageStub = stub +} + +func (fake *FakeParticipantListener) OnDataTrackMessageArgsForCall(i int) (types.Participant, []byte, *datatrack.Packet) { + fake.onDataTrackMessageMutex.RLock() + defer fake.onDataTrackMessageMutex.RUnlock() + argsForCall := fake.onDataTrackMessageArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 +} + +func (fake *FakeParticipantListener) OnDataTrackPublished(arg1 types.Participant, arg2 types.DataTrack) { + fake.onDataTrackPublishedMutex.Lock() + fake.onDataTrackPublishedArgsForCall = append(fake.onDataTrackPublishedArgsForCall, struct { + arg1 types.Participant + arg2 types.DataTrack + }{arg1, arg2}) + stub := fake.OnDataTrackPublishedStub + fake.recordInvocation("OnDataTrackPublished", []interface{}{arg1, arg2}) + fake.onDataTrackPublishedMutex.Unlock() + if stub != nil { + fake.OnDataTrackPublishedStub(arg1, arg2) + } +} + +func (fake *FakeParticipantListener) OnDataTrackPublishedCallCount() int { + fake.onDataTrackPublishedMutex.RLock() + defer fake.onDataTrackPublishedMutex.RUnlock() + return len(fake.onDataTrackPublishedArgsForCall) +} + +func (fake *FakeParticipantListener) OnDataTrackPublishedCalls(stub func(types.Participant, types.DataTrack)) { + fake.onDataTrackPublishedMutex.Lock() + defer fake.onDataTrackPublishedMutex.Unlock() + fake.OnDataTrackPublishedStub = stub +} + +func (fake *FakeParticipantListener) OnDataTrackPublishedArgsForCall(i int) (types.Participant, types.DataTrack) { + fake.onDataTrackPublishedMutex.RLock() + defer fake.onDataTrackPublishedMutex.RUnlock() + argsForCall := fake.onDataTrackPublishedArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeParticipantListener) OnDataTrackUnpublished(arg1 types.Participant, arg2 types.DataTrack) { + fake.onDataTrackUnpublishedMutex.Lock() + fake.onDataTrackUnpublishedArgsForCall = append(fake.onDataTrackUnpublishedArgsForCall, struct { + arg1 types.Participant + arg2 types.DataTrack + }{arg1, arg2}) + stub := fake.OnDataTrackUnpublishedStub + fake.recordInvocation("OnDataTrackUnpublished", []interface{}{arg1, arg2}) + fake.onDataTrackUnpublishedMutex.Unlock() + if stub != nil { + fake.OnDataTrackUnpublishedStub(arg1, arg2) + } +} + +func (fake *FakeParticipantListener) OnDataTrackUnpublishedCallCount() int { + fake.onDataTrackUnpublishedMutex.RLock() + defer fake.onDataTrackUnpublishedMutex.RUnlock() + return len(fake.onDataTrackUnpublishedArgsForCall) +} + +func (fake *FakeParticipantListener) OnDataTrackUnpublishedCalls(stub func(types.Participant, types.DataTrack)) { + fake.onDataTrackUnpublishedMutex.Lock() + defer fake.onDataTrackUnpublishedMutex.Unlock() + fake.OnDataTrackUnpublishedStub = stub +} + +func (fake *FakeParticipantListener) OnDataTrackUnpublishedArgsForCall(i int) (types.Participant, types.DataTrack) { + fake.onDataTrackUnpublishedMutex.RLock() + defer fake.onDataTrackUnpublishedMutex.RUnlock() + argsForCall := fake.onDataTrackUnpublishedArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeParticipantListener) OnMetrics(arg1 types.Participant, arg2 *livekit.DataPacket) { + fake.onMetricsMutex.Lock() + fake.onMetricsArgsForCall = append(fake.onMetricsArgsForCall, struct { + arg1 types.Participant + arg2 *livekit.DataPacket + }{arg1, arg2}) + stub := fake.OnMetricsStub + fake.recordInvocation("OnMetrics", []interface{}{arg1, arg2}) + fake.onMetricsMutex.Unlock() + if stub != nil { + fake.OnMetricsStub(arg1, arg2) + } +} + +func (fake *FakeParticipantListener) OnMetricsCallCount() int { + fake.onMetricsMutex.RLock() + defer fake.onMetricsMutex.RUnlock() + return len(fake.onMetricsArgsForCall) +} + +func (fake *FakeParticipantListener) OnMetricsCalls(stub func(types.Participant, *livekit.DataPacket)) { + fake.onMetricsMutex.Lock() + defer fake.onMetricsMutex.Unlock() + fake.OnMetricsStub = stub +} + +func (fake *FakeParticipantListener) OnMetricsArgsForCall(i int) (types.Participant, *livekit.DataPacket) { + fake.onMetricsMutex.RLock() + defer fake.onMetricsMutex.RUnlock() + argsForCall := fake.onMetricsArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeParticipantListener) OnParticipantUpdate(arg1 types.Participant) { + fake.onParticipantUpdateMutex.Lock() + fake.onParticipantUpdateArgsForCall = append(fake.onParticipantUpdateArgsForCall, struct { + arg1 types.Participant + }{arg1}) + stub := fake.OnParticipantUpdateStub + fake.recordInvocation("OnParticipantUpdate", []interface{}{arg1}) + fake.onParticipantUpdateMutex.Unlock() + if stub != nil { + fake.OnParticipantUpdateStub(arg1) + } +} + +func (fake *FakeParticipantListener) OnParticipantUpdateCallCount() int { + fake.onParticipantUpdateMutex.RLock() + defer fake.onParticipantUpdateMutex.RUnlock() + return len(fake.onParticipantUpdateArgsForCall) +} + +func (fake *FakeParticipantListener) OnParticipantUpdateCalls(stub func(types.Participant)) { + fake.onParticipantUpdateMutex.Lock() + defer fake.onParticipantUpdateMutex.Unlock() + fake.OnParticipantUpdateStub = stub +} + +func (fake *FakeParticipantListener) OnParticipantUpdateArgsForCall(i int) types.Participant { + fake.onParticipantUpdateMutex.RLock() + defer fake.onParticipantUpdateMutex.RUnlock() + argsForCall := fake.onParticipantUpdateArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeParticipantListener) OnTrackPublished(arg1 types.Participant, arg2 types.MediaTrack) { + fake.onTrackPublishedMutex.Lock() + fake.onTrackPublishedArgsForCall = append(fake.onTrackPublishedArgsForCall, struct { + arg1 types.Participant + arg2 types.MediaTrack + }{arg1, arg2}) + stub := fake.OnTrackPublishedStub + fake.recordInvocation("OnTrackPublished", []interface{}{arg1, arg2}) + fake.onTrackPublishedMutex.Unlock() + if stub != nil { + fake.OnTrackPublishedStub(arg1, arg2) + } +} + +func (fake *FakeParticipantListener) OnTrackPublishedCallCount() int { + fake.onTrackPublishedMutex.RLock() + defer fake.onTrackPublishedMutex.RUnlock() + return len(fake.onTrackPublishedArgsForCall) +} + +func (fake *FakeParticipantListener) OnTrackPublishedCalls(stub func(types.Participant, types.MediaTrack)) { + fake.onTrackPublishedMutex.Lock() + defer fake.onTrackPublishedMutex.Unlock() + fake.OnTrackPublishedStub = stub +} + +func (fake *FakeParticipantListener) OnTrackPublishedArgsForCall(i int) (types.Participant, types.MediaTrack) { + fake.onTrackPublishedMutex.RLock() + defer fake.onTrackPublishedMutex.RUnlock() + argsForCall := fake.onTrackPublishedArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeParticipantListener) OnTrackUnpublished(arg1 types.Participant, arg2 types.MediaTrack) { + fake.onTrackUnpublishedMutex.Lock() + fake.onTrackUnpublishedArgsForCall = append(fake.onTrackUnpublishedArgsForCall, struct { + arg1 types.Participant + arg2 types.MediaTrack + }{arg1, arg2}) + stub := fake.OnTrackUnpublishedStub + fake.recordInvocation("OnTrackUnpublished", []interface{}{arg1, arg2}) + fake.onTrackUnpublishedMutex.Unlock() + if stub != nil { + fake.OnTrackUnpublishedStub(arg1, arg2) + } +} + +func (fake *FakeParticipantListener) OnTrackUnpublishedCallCount() int { + fake.onTrackUnpublishedMutex.RLock() + defer fake.onTrackUnpublishedMutex.RUnlock() + return len(fake.onTrackUnpublishedArgsForCall) +} + +func (fake *FakeParticipantListener) OnTrackUnpublishedCalls(stub func(types.Participant, types.MediaTrack)) { + fake.onTrackUnpublishedMutex.Lock() + defer fake.onTrackUnpublishedMutex.Unlock() + fake.OnTrackUnpublishedStub = stub +} + +func (fake *FakeParticipantListener) OnTrackUnpublishedArgsForCall(i int) (types.Participant, types.MediaTrack) { + fake.onTrackUnpublishedMutex.RLock() + defer fake.onTrackUnpublishedMutex.RUnlock() + argsForCall := fake.onTrackUnpublishedArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeParticipantListener) OnTrackUpdated(arg1 types.Participant, arg2 types.MediaTrack) { + fake.onTrackUpdatedMutex.Lock() + fake.onTrackUpdatedArgsForCall = append(fake.onTrackUpdatedArgsForCall, struct { + arg1 types.Participant + arg2 types.MediaTrack + }{arg1, arg2}) + stub := fake.OnTrackUpdatedStub + fake.recordInvocation("OnTrackUpdated", []interface{}{arg1, arg2}) + fake.onTrackUpdatedMutex.Unlock() + if stub != nil { + fake.OnTrackUpdatedStub(arg1, arg2) + } +} + +func (fake *FakeParticipantListener) OnTrackUpdatedCallCount() int { + fake.onTrackUpdatedMutex.RLock() + defer fake.onTrackUpdatedMutex.RUnlock() + return len(fake.onTrackUpdatedArgsForCall) +} + +func (fake *FakeParticipantListener) OnTrackUpdatedCalls(stub func(types.Participant, types.MediaTrack)) { + fake.onTrackUpdatedMutex.Lock() + defer fake.onTrackUpdatedMutex.Unlock() + fake.OnTrackUpdatedStub = stub +} + +func (fake *FakeParticipantListener) OnTrackUpdatedArgsForCall(i int) (types.Participant, types.MediaTrack) { + fake.onTrackUpdatedMutex.RLock() + defer fake.onTrackUpdatedMutex.RUnlock() + argsForCall := fake.onTrackUpdatedArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeParticipantListener) Invocations() map[string][][]interface{} { + fake.invocationsMutex.RLock() + defer fake.invocationsMutex.RUnlock() + copiedInvocations := map[string][][]interface{}{} + for key, value := range fake.invocations { + copiedInvocations[key] = value + } + return copiedInvocations +} + +func (fake *FakeParticipantListener) recordInvocation(key string, args []interface{}) { + fake.invocationsMutex.Lock() + defer fake.invocationsMutex.Unlock() + if fake.invocations == nil { + fake.invocations = map[string][][]interface{}{} + } + if fake.invocations[key] == nil { + fake.invocations[key] = [][]interface{}{} + } + fake.invocations[key] = append(fake.invocations[key], args) +} + +var _ types.ParticipantListener = new(FakeParticipantListener) diff --git a/pkg/rtc/types/typesfakes/fake_participant_telemetry_listener.go b/pkg/rtc/types/typesfakes/fake_participant_telemetry_listener.go new file mode 100644 index 0000000..df4414e --- /dev/null +++ b/pkg/rtc/types/typesfakes/fake_participant_telemetry_listener.go @@ -0,0 +1,659 @@ +// Code generated by counterfeiter. DO NOT EDIT. +package typesfakes + +import ( + "sync" + + "github.com/livekit/livekit-server/pkg/rtc/types" + "github.com/livekit/livekit-server/pkg/telemetry" + "github.com/livekit/protocol/codecs/mime" + "github.com/livekit/protocol/livekit" +) + +type FakeParticipantTelemetryListener struct { + OnTrackMaxSubscribedVideoQualityStub func(livekit.ParticipantID, *livekit.TrackInfo, mime.MimeType, livekit.VideoQuality) + onTrackMaxSubscribedVideoQualityMutex sync.RWMutex + onTrackMaxSubscribedVideoQualityArgsForCall []struct { + arg1 livekit.ParticipantID + arg2 *livekit.TrackInfo + arg3 mime.MimeType + arg4 livekit.VideoQuality + } + OnTrackMutedStub func(livekit.ParticipantID, *livekit.TrackInfo) + onTrackMutedMutex sync.RWMutex + onTrackMutedArgsForCall []struct { + arg1 livekit.ParticipantID + arg2 *livekit.TrackInfo + } + OnTrackPublishRTPStatsStub func(livekit.ParticipantID, livekit.TrackID, mime.MimeType, int, *livekit.RTPStats) + onTrackPublishRTPStatsMutex sync.RWMutex + onTrackPublishRTPStatsArgsForCall []struct { + arg1 livekit.ParticipantID + arg2 livekit.TrackID + arg3 mime.MimeType + arg4 int + arg5 *livekit.RTPStats + } + OnTrackPublishRequestedStub func(livekit.ParticipantID, livekit.ParticipantIdentity, *livekit.TrackInfo) + onTrackPublishRequestedMutex sync.RWMutex + onTrackPublishRequestedArgsForCall []struct { + arg1 livekit.ParticipantID + arg2 livekit.ParticipantIdentity + arg3 *livekit.TrackInfo + } + OnTrackPublishedStub func(livekit.ParticipantID, livekit.ParticipantIdentity, *livekit.TrackInfo, bool) + onTrackPublishedMutex sync.RWMutex + onTrackPublishedArgsForCall []struct { + arg1 livekit.ParticipantID + arg2 livekit.ParticipantIdentity + arg3 *livekit.TrackInfo + arg4 bool + } + OnTrackPublishedUpdateStub func(livekit.ParticipantID, *livekit.TrackInfo) + onTrackPublishedUpdateMutex sync.RWMutex + onTrackPublishedUpdateArgsForCall []struct { + arg1 livekit.ParticipantID + arg2 *livekit.TrackInfo + } + OnTrackStatsStub func(telemetry.StatsKey, *livekit.AnalyticsStat) + onTrackStatsMutex sync.RWMutex + onTrackStatsArgsForCall []struct { + arg1 telemetry.StatsKey + arg2 *livekit.AnalyticsStat + } + OnTrackSubscribeFailedStub func(livekit.ParticipantID, livekit.TrackID, error, bool) + onTrackSubscribeFailedMutex sync.RWMutex + onTrackSubscribeFailedArgsForCall []struct { + arg1 livekit.ParticipantID + arg2 livekit.TrackID + arg3 error + arg4 bool + } + OnTrackSubscribeRTPStatsStub func(livekit.ParticipantID, livekit.TrackID, mime.MimeType, *livekit.RTPStats) + onTrackSubscribeRTPStatsMutex sync.RWMutex + onTrackSubscribeRTPStatsArgsForCall []struct { + arg1 livekit.ParticipantID + arg2 livekit.TrackID + arg3 mime.MimeType + arg4 *livekit.RTPStats + } + OnTrackSubscribeRequestedStub func(livekit.ParticipantID, *livekit.TrackInfo) + onTrackSubscribeRequestedMutex sync.RWMutex + onTrackSubscribeRequestedArgsForCall []struct { + arg1 livekit.ParticipantID + arg2 *livekit.TrackInfo + } + OnTrackSubscribeStreamStartedStub func(livekit.ParticipantID, *livekit.TrackInfo) + onTrackSubscribeStreamStartedMutex sync.RWMutex + onTrackSubscribeStreamStartedArgsForCall []struct { + arg1 livekit.ParticipantID + arg2 *livekit.TrackInfo + } + OnTrackSubscribedStub func(livekit.ParticipantID, *livekit.TrackInfo, *livekit.ParticipantInfo, bool) + onTrackSubscribedMutex sync.RWMutex + onTrackSubscribedArgsForCall []struct { + arg1 livekit.ParticipantID + arg2 *livekit.TrackInfo + arg3 *livekit.ParticipantInfo + arg4 bool + } + OnTrackUnmutedStub func(livekit.ParticipantID, *livekit.TrackInfo) + onTrackUnmutedMutex sync.RWMutex + onTrackUnmutedArgsForCall []struct { + arg1 livekit.ParticipantID + arg2 *livekit.TrackInfo + } + OnTrackUnpublishedStub func(livekit.ParticipantID, livekit.ParticipantIdentity, *livekit.TrackInfo, bool) + onTrackUnpublishedMutex sync.RWMutex + onTrackUnpublishedArgsForCall []struct { + arg1 livekit.ParticipantID + arg2 livekit.ParticipantIdentity + arg3 *livekit.TrackInfo + arg4 bool + } + OnTrackUnsubscribedStub func(livekit.ParticipantID, *livekit.TrackInfo, bool) + onTrackUnsubscribedMutex sync.RWMutex + onTrackUnsubscribedArgsForCall []struct { + arg1 livekit.ParticipantID + arg2 *livekit.TrackInfo + arg3 bool + } + invocations map[string][][]interface{} + invocationsMutex sync.RWMutex +} + +func (fake *FakeParticipantTelemetryListener) OnTrackMaxSubscribedVideoQuality(arg1 livekit.ParticipantID, arg2 *livekit.TrackInfo, arg3 mime.MimeType, arg4 livekit.VideoQuality) { + fake.onTrackMaxSubscribedVideoQualityMutex.Lock() + fake.onTrackMaxSubscribedVideoQualityArgsForCall = append(fake.onTrackMaxSubscribedVideoQualityArgsForCall, struct { + arg1 livekit.ParticipantID + arg2 *livekit.TrackInfo + arg3 mime.MimeType + arg4 livekit.VideoQuality + }{arg1, arg2, arg3, arg4}) + stub := fake.OnTrackMaxSubscribedVideoQualityStub + fake.recordInvocation("OnTrackMaxSubscribedVideoQuality", []interface{}{arg1, arg2, arg3, arg4}) + fake.onTrackMaxSubscribedVideoQualityMutex.Unlock() + if stub != nil { + fake.OnTrackMaxSubscribedVideoQualityStub(arg1, arg2, arg3, arg4) + } +} + +func (fake *FakeParticipantTelemetryListener) OnTrackMaxSubscribedVideoQualityCallCount() int { + fake.onTrackMaxSubscribedVideoQualityMutex.RLock() + defer fake.onTrackMaxSubscribedVideoQualityMutex.RUnlock() + return len(fake.onTrackMaxSubscribedVideoQualityArgsForCall) +} + +func (fake *FakeParticipantTelemetryListener) OnTrackMaxSubscribedVideoQualityCalls(stub func(livekit.ParticipantID, *livekit.TrackInfo, mime.MimeType, livekit.VideoQuality)) { + fake.onTrackMaxSubscribedVideoQualityMutex.Lock() + defer fake.onTrackMaxSubscribedVideoQualityMutex.Unlock() + fake.OnTrackMaxSubscribedVideoQualityStub = stub +} + +func (fake *FakeParticipantTelemetryListener) OnTrackMaxSubscribedVideoQualityArgsForCall(i int) (livekit.ParticipantID, *livekit.TrackInfo, mime.MimeType, livekit.VideoQuality) { + fake.onTrackMaxSubscribedVideoQualityMutex.RLock() + defer fake.onTrackMaxSubscribedVideoQualityMutex.RUnlock() + argsForCall := fake.onTrackMaxSubscribedVideoQualityArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4 +} + +func (fake *FakeParticipantTelemetryListener) OnTrackMuted(arg1 livekit.ParticipantID, arg2 *livekit.TrackInfo) { + fake.onTrackMutedMutex.Lock() + fake.onTrackMutedArgsForCall = append(fake.onTrackMutedArgsForCall, struct { + arg1 livekit.ParticipantID + arg2 *livekit.TrackInfo + }{arg1, arg2}) + stub := fake.OnTrackMutedStub + fake.recordInvocation("OnTrackMuted", []interface{}{arg1, arg2}) + fake.onTrackMutedMutex.Unlock() + if stub != nil { + fake.OnTrackMutedStub(arg1, arg2) + } +} + +func (fake *FakeParticipantTelemetryListener) OnTrackMutedCallCount() int { + fake.onTrackMutedMutex.RLock() + defer fake.onTrackMutedMutex.RUnlock() + return len(fake.onTrackMutedArgsForCall) +} + +func (fake *FakeParticipantTelemetryListener) OnTrackMutedCalls(stub func(livekit.ParticipantID, *livekit.TrackInfo)) { + fake.onTrackMutedMutex.Lock() + defer fake.onTrackMutedMutex.Unlock() + fake.OnTrackMutedStub = stub +} + +func (fake *FakeParticipantTelemetryListener) OnTrackMutedArgsForCall(i int) (livekit.ParticipantID, *livekit.TrackInfo) { + fake.onTrackMutedMutex.RLock() + defer fake.onTrackMutedMutex.RUnlock() + argsForCall := fake.onTrackMutedArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeParticipantTelemetryListener) OnTrackPublishRTPStats(arg1 livekit.ParticipantID, arg2 livekit.TrackID, arg3 mime.MimeType, arg4 int, arg5 *livekit.RTPStats) { + fake.onTrackPublishRTPStatsMutex.Lock() + fake.onTrackPublishRTPStatsArgsForCall = append(fake.onTrackPublishRTPStatsArgsForCall, struct { + arg1 livekit.ParticipantID + arg2 livekit.TrackID + arg3 mime.MimeType + arg4 int + arg5 *livekit.RTPStats + }{arg1, arg2, arg3, arg4, arg5}) + stub := fake.OnTrackPublishRTPStatsStub + fake.recordInvocation("OnTrackPublishRTPStats", []interface{}{arg1, arg2, arg3, arg4, arg5}) + fake.onTrackPublishRTPStatsMutex.Unlock() + if stub != nil { + fake.OnTrackPublishRTPStatsStub(arg1, arg2, arg3, arg4, arg5) + } +} + +func (fake *FakeParticipantTelemetryListener) OnTrackPublishRTPStatsCallCount() int { + fake.onTrackPublishRTPStatsMutex.RLock() + defer fake.onTrackPublishRTPStatsMutex.RUnlock() + return len(fake.onTrackPublishRTPStatsArgsForCall) +} + +func (fake *FakeParticipantTelemetryListener) OnTrackPublishRTPStatsCalls(stub func(livekit.ParticipantID, livekit.TrackID, mime.MimeType, int, *livekit.RTPStats)) { + fake.onTrackPublishRTPStatsMutex.Lock() + defer fake.onTrackPublishRTPStatsMutex.Unlock() + fake.OnTrackPublishRTPStatsStub = stub +} + +func (fake *FakeParticipantTelemetryListener) OnTrackPublishRTPStatsArgsForCall(i int) (livekit.ParticipantID, livekit.TrackID, mime.MimeType, int, *livekit.RTPStats) { + fake.onTrackPublishRTPStatsMutex.RLock() + defer fake.onTrackPublishRTPStatsMutex.RUnlock() + argsForCall := fake.onTrackPublishRTPStatsArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4, argsForCall.arg5 +} + +func (fake *FakeParticipantTelemetryListener) OnTrackPublishRequested(arg1 livekit.ParticipantID, arg2 livekit.ParticipantIdentity, arg3 *livekit.TrackInfo) { + fake.onTrackPublishRequestedMutex.Lock() + fake.onTrackPublishRequestedArgsForCall = append(fake.onTrackPublishRequestedArgsForCall, struct { + arg1 livekit.ParticipantID + arg2 livekit.ParticipantIdentity + arg3 *livekit.TrackInfo + }{arg1, arg2, arg3}) + stub := fake.OnTrackPublishRequestedStub + fake.recordInvocation("OnTrackPublishRequested", []interface{}{arg1, arg2, arg3}) + fake.onTrackPublishRequestedMutex.Unlock() + if stub != nil { + fake.OnTrackPublishRequestedStub(arg1, arg2, arg3) + } +} + +func (fake *FakeParticipantTelemetryListener) OnTrackPublishRequestedCallCount() int { + fake.onTrackPublishRequestedMutex.RLock() + defer fake.onTrackPublishRequestedMutex.RUnlock() + return len(fake.onTrackPublishRequestedArgsForCall) +} + +func (fake *FakeParticipantTelemetryListener) OnTrackPublishRequestedCalls(stub func(livekit.ParticipantID, livekit.ParticipantIdentity, *livekit.TrackInfo)) { + fake.onTrackPublishRequestedMutex.Lock() + defer fake.onTrackPublishRequestedMutex.Unlock() + fake.OnTrackPublishRequestedStub = stub +} + +func (fake *FakeParticipantTelemetryListener) OnTrackPublishRequestedArgsForCall(i int) (livekit.ParticipantID, livekit.ParticipantIdentity, *livekit.TrackInfo) { + fake.onTrackPublishRequestedMutex.RLock() + defer fake.onTrackPublishRequestedMutex.RUnlock() + argsForCall := fake.onTrackPublishRequestedArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 +} + +func (fake *FakeParticipantTelemetryListener) OnTrackPublished(arg1 livekit.ParticipantID, arg2 livekit.ParticipantIdentity, arg3 *livekit.TrackInfo, arg4 bool) { + fake.onTrackPublishedMutex.Lock() + fake.onTrackPublishedArgsForCall = append(fake.onTrackPublishedArgsForCall, struct { + arg1 livekit.ParticipantID + arg2 livekit.ParticipantIdentity + arg3 *livekit.TrackInfo + arg4 bool + }{arg1, arg2, arg3, arg4}) + stub := fake.OnTrackPublishedStub + fake.recordInvocation("OnTrackPublished", []interface{}{arg1, arg2, arg3, arg4}) + fake.onTrackPublishedMutex.Unlock() + if stub != nil { + fake.OnTrackPublishedStub(arg1, arg2, arg3, arg4) + } +} + +func (fake *FakeParticipantTelemetryListener) OnTrackPublishedCallCount() int { + fake.onTrackPublishedMutex.RLock() + defer fake.onTrackPublishedMutex.RUnlock() + return len(fake.onTrackPublishedArgsForCall) +} + +func (fake *FakeParticipantTelemetryListener) OnTrackPublishedCalls(stub func(livekit.ParticipantID, livekit.ParticipantIdentity, *livekit.TrackInfo, bool)) { + fake.onTrackPublishedMutex.Lock() + defer fake.onTrackPublishedMutex.Unlock() + fake.OnTrackPublishedStub = stub +} + +func (fake *FakeParticipantTelemetryListener) OnTrackPublishedArgsForCall(i int) (livekit.ParticipantID, livekit.ParticipantIdentity, *livekit.TrackInfo, bool) { + fake.onTrackPublishedMutex.RLock() + defer fake.onTrackPublishedMutex.RUnlock() + argsForCall := fake.onTrackPublishedArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4 +} + +func (fake *FakeParticipantTelemetryListener) OnTrackPublishedUpdate(arg1 livekit.ParticipantID, arg2 *livekit.TrackInfo) { + fake.onTrackPublishedUpdateMutex.Lock() + fake.onTrackPublishedUpdateArgsForCall = append(fake.onTrackPublishedUpdateArgsForCall, struct { + arg1 livekit.ParticipantID + arg2 *livekit.TrackInfo + }{arg1, arg2}) + stub := fake.OnTrackPublishedUpdateStub + fake.recordInvocation("OnTrackPublishedUpdate", []interface{}{arg1, arg2}) + fake.onTrackPublishedUpdateMutex.Unlock() + if stub != nil { + fake.OnTrackPublishedUpdateStub(arg1, arg2) + } +} + +func (fake *FakeParticipantTelemetryListener) OnTrackPublishedUpdateCallCount() int { + fake.onTrackPublishedUpdateMutex.RLock() + defer fake.onTrackPublishedUpdateMutex.RUnlock() + return len(fake.onTrackPublishedUpdateArgsForCall) +} + +func (fake *FakeParticipantTelemetryListener) OnTrackPublishedUpdateCalls(stub func(livekit.ParticipantID, *livekit.TrackInfo)) { + fake.onTrackPublishedUpdateMutex.Lock() + defer fake.onTrackPublishedUpdateMutex.Unlock() + fake.OnTrackPublishedUpdateStub = stub +} + +func (fake *FakeParticipantTelemetryListener) OnTrackPublishedUpdateArgsForCall(i int) (livekit.ParticipantID, *livekit.TrackInfo) { + fake.onTrackPublishedUpdateMutex.RLock() + defer fake.onTrackPublishedUpdateMutex.RUnlock() + argsForCall := fake.onTrackPublishedUpdateArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeParticipantTelemetryListener) OnTrackStats(arg1 telemetry.StatsKey, arg2 *livekit.AnalyticsStat) { + fake.onTrackStatsMutex.Lock() + fake.onTrackStatsArgsForCall = append(fake.onTrackStatsArgsForCall, struct { + arg1 telemetry.StatsKey + arg2 *livekit.AnalyticsStat + }{arg1, arg2}) + stub := fake.OnTrackStatsStub + fake.recordInvocation("OnTrackStats", []interface{}{arg1, arg2}) + fake.onTrackStatsMutex.Unlock() + if stub != nil { + fake.OnTrackStatsStub(arg1, arg2) + } +} + +func (fake *FakeParticipantTelemetryListener) OnTrackStatsCallCount() int { + fake.onTrackStatsMutex.RLock() + defer fake.onTrackStatsMutex.RUnlock() + return len(fake.onTrackStatsArgsForCall) +} + +func (fake *FakeParticipantTelemetryListener) OnTrackStatsCalls(stub func(telemetry.StatsKey, *livekit.AnalyticsStat)) { + fake.onTrackStatsMutex.Lock() + defer fake.onTrackStatsMutex.Unlock() + fake.OnTrackStatsStub = stub +} + +func (fake *FakeParticipantTelemetryListener) OnTrackStatsArgsForCall(i int) (telemetry.StatsKey, *livekit.AnalyticsStat) { + fake.onTrackStatsMutex.RLock() + defer fake.onTrackStatsMutex.RUnlock() + argsForCall := fake.onTrackStatsArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeParticipantTelemetryListener) OnTrackSubscribeFailed(arg1 livekit.ParticipantID, arg2 livekit.TrackID, arg3 error, arg4 bool) { + fake.onTrackSubscribeFailedMutex.Lock() + fake.onTrackSubscribeFailedArgsForCall = append(fake.onTrackSubscribeFailedArgsForCall, struct { + arg1 livekit.ParticipantID + arg2 livekit.TrackID + arg3 error + arg4 bool + }{arg1, arg2, arg3, arg4}) + stub := fake.OnTrackSubscribeFailedStub + fake.recordInvocation("OnTrackSubscribeFailed", []interface{}{arg1, arg2, arg3, arg4}) + fake.onTrackSubscribeFailedMutex.Unlock() + if stub != nil { + fake.OnTrackSubscribeFailedStub(arg1, arg2, arg3, arg4) + } +} + +func (fake *FakeParticipantTelemetryListener) OnTrackSubscribeFailedCallCount() int { + fake.onTrackSubscribeFailedMutex.RLock() + defer fake.onTrackSubscribeFailedMutex.RUnlock() + return len(fake.onTrackSubscribeFailedArgsForCall) +} + +func (fake *FakeParticipantTelemetryListener) OnTrackSubscribeFailedCalls(stub func(livekit.ParticipantID, livekit.TrackID, error, bool)) { + fake.onTrackSubscribeFailedMutex.Lock() + defer fake.onTrackSubscribeFailedMutex.Unlock() + fake.OnTrackSubscribeFailedStub = stub +} + +func (fake *FakeParticipantTelemetryListener) OnTrackSubscribeFailedArgsForCall(i int) (livekit.ParticipantID, livekit.TrackID, error, bool) { + fake.onTrackSubscribeFailedMutex.RLock() + defer fake.onTrackSubscribeFailedMutex.RUnlock() + argsForCall := fake.onTrackSubscribeFailedArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4 +} + +func (fake *FakeParticipantTelemetryListener) OnTrackSubscribeRTPStats(arg1 livekit.ParticipantID, arg2 livekit.TrackID, arg3 mime.MimeType, arg4 *livekit.RTPStats) { + fake.onTrackSubscribeRTPStatsMutex.Lock() + fake.onTrackSubscribeRTPStatsArgsForCall = append(fake.onTrackSubscribeRTPStatsArgsForCall, struct { + arg1 livekit.ParticipantID + arg2 livekit.TrackID + arg3 mime.MimeType + arg4 *livekit.RTPStats + }{arg1, arg2, arg3, arg4}) + stub := fake.OnTrackSubscribeRTPStatsStub + fake.recordInvocation("OnTrackSubscribeRTPStats", []interface{}{arg1, arg2, arg3, arg4}) + fake.onTrackSubscribeRTPStatsMutex.Unlock() + if stub != nil { + fake.OnTrackSubscribeRTPStatsStub(arg1, arg2, arg3, arg4) + } +} + +func (fake *FakeParticipantTelemetryListener) OnTrackSubscribeRTPStatsCallCount() int { + fake.onTrackSubscribeRTPStatsMutex.RLock() + defer fake.onTrackSubscribeRTPStatsMutex.RUnlock() + return len(fake.onTrackSubscribeRTPStatsArgsForCall) +} + +func (fake *FakeParticipantTelemetryListener) OnTrackSubscribeRTPStatsCalls(stub func(livekit.ParticipantID, livekit.TrackID, mime.MimeType, *livekit.RTPStats)) { + fake.onTrackSubscribeRTPStatsMutex.Lock() + defer fake.onTrackSubscribeRTPStatsMutex.Unlock() + fake.OnTrackSubscribeRTPStatsStub = stub +} + +func (fake *FakeParticipantTelemetryListener) OnTrackSubscribeRTPStatsArgsForCall(i int) (livekit.ParticipantID, livekit.TrackID, mime.MimeType, *livekit.RTPStats) { + fake.onTrackSubscribeRTPStatsMutex.RLock() + defer fake.onTrackSubscribeRTPStatsMutex.RUnlock() + argsForCall := fake.onTrackSubscribeRTPStatsArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4 +} + +func (fake *FakeParticipantTelemetryListener) OnTrackSubscribeRequested(arg1 livekit.ParticipantID, arg2 *livekit.TrackInfo) { + fake.onTrackSubscribeRequestedMutex.Lock() + fake.onTrackSubscribeRequestedArgsForCall = append(fake.onTrackSubscribeRequestedArgsForCall, struct { + arg1 livekit.ParticipantID + arg2 *livekit.TrackInfo + }{arg1, arg2}) + stub := fake.OnTrackSubscribeRequestedStub + fake.recordInvocation("OnTrackSubscribeRequested", []interface{}{arg1, arg2}) + fake.onTrackSubscribeRequestedMutex.Unlock() + if stub != nil { + fake.OnTrackSubscribeRequestedStub(arg1, arg2) + } +} + +func (fake *FakeParticipantTelemetryListener) OnTrackSubscribeRequestedCallCount() int { + fake.onTrackSubscribeRequestedMutex.RLock() + defer fake.onTrackSubscribeRequestedMutex.RUnlock() + return len(fake.onTrackSubscribeRequestedArgsForCall) +} + +func (fake *FakeParticipantTelemetryListener) OnTrackSubscribeRequestedCalls(stub func(livekit.ParticipantID, *livekit.TrackInfo)) { + fake.onTrackSubscribeRequestedMutex.Lock() + defer fake.onTrackSubscribeRequestedMutex.Unlock() + fake.OnTrackSubscribeRequestedStub = stub +} + +func (fake *FakeParticipantTelemetryListener) OnTrackSubscribeRequestedArgsForCall(i int) (livekit.ParticipantID, *livekit.TrackInfo) { + fake.onTrackSubscribeRequestedMutex.RLock() + defer fake.onTrackSubscribeRequestedMutex.RUnlock() + argsForCall := fake.onTrackSubscribeRequestedArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeParticipantTelemetryListener) OnTrackSubscribeStreamStarted(arg1 livekit.ParticipantID, arg2 *livekit.TrackInfo) { + fake.onTrackSubscribeStreamStartedMutex.Lock() + fake.onTrackSubscribeStreamStartedArgsForCall = append(fake.onTrackSubscribeStreamStartedArgsForCall, struct { + arg1 livekit.ParticipantID + arg2 *livekit.TrackInfo + }{arg1, arg2}) + stub := fake.OnTrackSubscribeStreamStartedStub + fake.recordInvocation("OnTrackSubscribeStreamStarted", []interface{}{arg1, arg2}) + fake.onTrackSubscribeStreamStartedMutex.Unlock() + if stub != nil { + fake.OnTrackSubscribeStreamStartedStub(arg1, arg2) + } +} + +func (fake *FakeParticipantTelemetryListener) OnTrackSubscribeStreamStartedCallCount() int { + fake.onTrackSubscribeStreamStartedMutex.RLock() + defer fake.onTrackSubscribeStreamStartedMutex.RUnlock() + return len(fake.onTrackSubscribeStreamStartedArgsForCall) +} + +func (fake *FakeParticipantTelemetryListener) OnTrackSubscribeStreamStartedCalls(stub func(livekit.ParticipantID, *livekit.TrackInfo)) { + fake.onTrackSubscribeStreamStartedMutex.Lock() + defer fake.onTrackSubscribeStreamStartedMutex.Unlock() + fake.OnTrackSubscribeStreamStartedStub = stub +} + +func (fake *FakeParticipantTelemetryListener) OnTrackSubscribeStreamStartedArgsForCall(i int) (livekit.ParticipantID, *livekit.TrackInfo) { + fake.onTrackSubscribeStreamStartedMutex.RLock() + defer fake.onTrackSubscribeStreamStartedMutex.RUnlock() + argsForCall := fake.onTrackSubscribeStreamStartedArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeParticipantTelemetryListener) OnTrackSubscribed(arg1 livekit.ParticipantID, arg2 *livekit.TrackInfo, arg3 *livekit.ParticipantInfo, arg4 bool) { + fake.onTrackSubscribedMutex.Lock() + fake.onTrackSubscribedArgsForCall = append(fake.onTrackSubscribedArgsForCall, struct { + arg1 livekit.ParticipantID + arg2 *livekit.TrackInfo + arg3 *livekit.ParticipantInfo + arg4 bool + }{arg1, arg2, arg3, arg4}) + stub := fake.OnTrackSubscribedStub + fake.recordInvocation("OnTrackSubscribed", []interface{}{arg1, arg2, arg3, arg4}) + fake.onTrackSubscribedMutex.Unlock() + if stub != nil { + fake.OnTrackSubscribedStub(arg1, arg2, arg3, arg4) + } +} + +func (fake *FakeParticipantTelemetryListener) OnTrackSubscribedCallCount() int { + fake.onTrackSubscribedMutex.RLock() + defer fake.onTrackSubscribedMutex.RUnlock() + return len(fake.onTrackSubscribedArgsForCall) +} + +func (fake *FakeParticipantTelemetryListener) OnTrackSubscribedCalls(stub func(livekit.ParticipantID, *livekit.TrackInfo, *livekit.ParticipantInfo, bool)) { + fake.onTrackSubscribedMutex.Lock() + defer fake.onTrackSubscribedMutex.Unlock() + fake.OnTrackSubscribedStub = stub +} + +func (fake *FakeParticipantTelemetryListener) OnTrackSubscribedArgsForCall(i int) (livekit.ParticipantID, *livekit.TrackInfo, *livekit.ParticipantInfo, bool) { + fake.onTrackSubscribedMutex.RLock() + defer fake.onTrackSubscribedMutex.RUnlock() + argsForCall := fake.onTrackSubscribedArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4 +} + +func (fake *FakeParticipantTelemetryListener) OnTrackUnmuted(arg1 livekit.ParticipantID, arg2 *livekit.TrackInfo) { + fake.onTrackUnmutedMutex.Lock() + fake.onTrackUnmutedArgsForCall = append(fake.onTrackUnmutedArgsForCall, struct { + arg1 livekit.ParticipantID + arg2 *livekit.TrackInfo + }{arg1, arg2}) + stub := fake.OnTrackUnmutedStub + fake.recordInvocation("OnTrackUnmuted", []interface{}{arg1, arg2}) + fake.onTrackUnmutedMutex.Unlock() + if stub != nil { + fake.OnTrackUnmutedStub(arg1, arg2) + } +} + +func (fake *FakeParticipantTelemetryListener) OnTrackUnmutedCallCount() int { + fake.onTrackUnmutedMutex.RLock() + defer fake.onTrackUnmutedMutex.RUnlock() + return len(fake.onTrackUnmutedArgsForCall) +} + +func (fake *FakeParticipantTelemetryListener) OnTrackUnmutedCalls(stub func(livekit.ParticipantID, *livekit.TrackInfo)) { + fake.onTrackUnmutedMutex.Lock() + defer fake.onTrackUnmutedMutex.Unlock() + fake.OnTrackUnmutedStub = stub +} + +func (fake *FakeParticipantTelemetryListener) OnTrackUnmutedArgsForCall(i int) (livekit.ParticipantID, *livekit.TrackInfo) { + fake.onTrackUnmutedMutex.RLock() + defer fake.onTrackUnmutedMutex.RUnlock() + argsForCall := fake.onTrackUnmutedArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeParticipantTelemetryListener) OnTrackUnpublished(arg1 livekit.ParticipantID, arg2 livekit.ParticipantIdentity, arg3 *livekit.TrackInfo, arg4 bool) { + fake.onTrackUnpublishedMutex.Lock() + fake.onTrackUnpublishedArgsForCall = append(fake.onTrackUnpublishedArgsForCall, struct { + arg1 livekit.ParticipantID + arg2 livekit.ParticipantIdentity + arg3 *livekit.TrackInfo + arg4 bool + }{arg1, arg2, arg3, arg4}) + stub := fake.OnTrackUnpublishedStub + fake.recordInvocation("OnTrackUnpublished", []interface{}{arg1, arg2, arg3, arg4}) + fake.onTrackUnpublishedMutex.Unlock() + if stub != nil { + fake.OnTrackUnpublishedStub(arg1, arg2, arg3, arg4) + } +} + +func (fake *FakeParticipantTelemetryListener) OnTrackUnpublishedCallCount() int { + fake.onTrackUnpublishedMutex.RLock() + defer fake.onTrackUnpublishedMutex.RUnlock() + return len(fake.onTrackUnpublishedArgsForCall) +} + +func (fake *FakeParticipantTelemetryListener) OnTrackUnpublishedCalls(stub func(livekit.ParticipantID, livekit.ParticipantIdentity, *livekit.TrackInfo, bool)) { + fake.onTrackUnpublishedMutex.Lock() + defer fake.onTrackUnpublishedMutex.Unlock() + fake.OnTrackUnpublishedStub = stub +} + +func (fake *FakeParticipantTelemetryListener) OnTrackUnpublishedArgsForCall(i int) (livekit.ParticipantID, livekit.ParticipantIdentity, *livekit.TrackInfo, bool) { + fake.onTrackUnpublishedMutex.RLock() + defer fake.onTrackUnpublishedMutex.RUnlock() + argsForCall := fake.onTrackUnpublishedArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4 +} + +func (fake *FakeParticipantTelemetryListener) OnTrackUnsubscribed(arg1 livekit.ParticipantID, arg2 *livekit.TrackInfo, arg3 bool) { + fake.onTrackUnsubscribedMutex.Lock() + fake.onTrackUnsubscribedArgsForCall = append(fake.onTrackUnsubscribedArgsForCall, struct { + arg1 livekit.ParticipantID + arg2 *livekit.TrackInfo + arg3 bool + }{arg1, arg2, arg3}) + stub := fake.OnTrackUnsubscribedStub + fake.recordInvocation("OnTrackUnsubscribed", []interface{}{arg1, arg2, arg3}) + fake.onTrackUnsubscribedMutex.Unlock() + if stub != nil { + fake.OnTrackUnsubscribedStub(arg1, arg2, arg3) + } +} + +func (fake *FakeParticipantTelemetryListener) OnTrackUnsubscribedCallCount() int { + fake.onTrackUnsubscribedMutex.RLock() + defer fake.onTrackUnsubscribedMutex.RUnlock() + return len(fake.onTrackUnsubscribedArgsForCall) +} + +func (fake *FakeParticipantTelemetryListener) OnTrackUnsubscribedCalls(stub func(livekit.ParticipantID, *livekit.TrackInfo, bool)) { + fake.onTrackUnsubscribedMutex.Lock() + defer fake.onTrackUnsubscribedMutex.Unlock() + fake.OnTrackUnsubscribedStub = stub +} + +func (fake *FakeParticipantTelemetryListener) OnTrackUnsubscribedArgsForCall(i int) (livekit.ParticipantID, *livekit.TrackInfo, bool) { + fake.onTrackUnsubscribedMutex.RLock() + defer fake.onTrackUnsubscribedMutex.RUnlock() + argsForCall := fake.onTrackUnsubscribedArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 +} + +func (fake *FakeParticipantTelemetryListener) Invocations() map[string][][]interface{} { + fake.invocationsMutex.RLock() + defer fake.invocationsMutex.RUnlock() + copiedInvocations := map[string][][]interface{}{} + for key, value := range fake.invocations { + copiedInvocations[key] = value + } + return copiedInvocations +} + +func (fake *FakeParticipantTelemetryListener) recordInvocation(key string, args []interface{}) { + fake.invocationsMutex.Lock() + defer fake.invocationsMutex.Unlock() + if fake.invocations == nil { + fake.invocations = map[string][][]interface{}{} + } + if fake.invocations[key] == nil { + fake.invocations[key] = [][]interface{}{} + } + fake.invocations[key] = append(fake.invocations[key], args) +} + +var _ types.ParticipantTelemetryListener = new(FakeParticipantTelemetryListener) diff --git a/pkg/rtc/types/typesfakes/fake_room.go b/pkg/rtc/types/typesfakes/fake_room.go index 70fbc32..292fad0 100644 --- a/pkg/rtc/types/typesfakes/fake_room.go +++ b/pkg/rtc/types/typesfakes/fake_room.go @@ -57,6 +57,18 @@ type FakeRoom struct { arg2 livekit.ParticipantID arg3 types.ParticipantCloseReason } + ResolveDataTrackForSubscriberStub func(types.LocalParticipant, livekit.TrackID) types.DataResolverResult + resolveDataTrackForSubscriberMutex sync.RWMutex + resolveDataTrackForSubscriberArgsForCall []struct { + arg1 types.LocalParticipant + arg2 livekit.TrackID + } + resolveDataTrackForSubscriberReturns struct { + result1 types.DataResolverResult + } + resolveDataTrackForSubscriberReturnsOnCall map[int]struct { + result1 types.DataResolverResult + } ResolveMediaTrackForSubscriberStub func(types.LocalParticipant, livekit.TrackID) types.MediaResolverResult resolveMediaTrackForSubscriberMutex sync.RWMutex resolveMediaTrackForSubscriberArgsForCall []struct { @@ -69,42 +81,6 @@ type FakeRoom struct { resolveMediaTrackForSubscriberReturnsOnCall map[int]struct { result1 types.MediaResolverResult } - SimulateScenarioStub func(types.LocalParticipant, *livekit.SimulateScenario) error - simulateScenarioMutex sync.RWMutex - simulateScenarioArgsForCall []struct { - arg1 types.LocalParticipant - arg2 *livekit.SimulateScenario - } - simulateScenarioReturns struct { - result1 error - } - simulateScenarioReturnsOnCall map[int]struct { - result1 error - } - SyncStateStub func(types.LocalParticipant, *livekit.SyncState) error - syncStateMutex sync.RWMutex - syncStateArgsForCall []struct { - arg1 types.LocalParticipant - arg2 *livekit.SyncState - } - syncStateReturns struct { - result1 error - } - syncStateReturnsOnCall map[int]struct { - result1 error - } - UpdateSubscriptionPermissionStub func(types.LocalParticipant, *livekit.SubscriptionPermission) error - updateSubscriptionPermissionMutex sync.RWMutex - updateSubscriptionPermissionArgsForCall []struct { - arg1 types.LocalParticipant - arg2 *livekit.SubscriptionPermission - } - updateSubscriptionPermissionReturns struct { - result1 error - } - updateSubscriptionPermissionReturnsOnCall map[int]struct { - result1 error - } UpdateSubscriptionsStub func(types.LocalParticipant, []livekit.TrackID, []*livekit.ParticipantTracks, bool) updateSubscriptionsMutex sync.RWMutex updateSubscriptionsArgsForCall []struct { @@ -371,6 +347,68 @@ func (fake *FakeRoom) RemoveParticipantArgsForCall(i int) (livekit.ParticipantId return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 } +func (fake *FakeRoom) ResolveDataTrackForSubscriber(arg1 types.LocalParticipant, arg2 livekit.TrackID) types.DataResolverResult { + fake.resolveDataTrackForSubscriberMutex.Lock() + ret, specificReturn := fake.resolveDataTrackForSubscriberReturnsOnCall[len(fake.resolveDataTrackForSubscriberArgsForCall)] + fake.resolveDataTrackForSubscriberArgsForCall = append(fake.resolveDataTrackForSubscriberArgsForCall, struct { + arg1 types.LocalParticipant + arg2 livekit.TrackID + }{arg1, arg2}) + stub := fake.ResolveDataTrackForSubscriberStub + fakeReturns := fake.resolveDataTrackForSubscriberReturns + fake.recordInvocation("ResolveDataTrackForSubscriber", []interface{}{arg1, arg2}) + fake.resolveDataTrackForSubscriberMutex.Unlock() + if stub != nil { + return stub(arg1, arg2) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeRoom) ResolveDataTrackForSubscriberCallCount() int { + fake.resolveDataTrackForSubscriberMutex.RLock() + defer fake.resolveDataTrackForSubscriberMutex.RUnlock() + return len(fake.resolveDataTrackForSubscriberArgsForCall) +} + +func (fake *FakeRoom) ResolveDataTrackForSubscriberCalls(stub func(types.LocalParticipant, livekit.TrackID) types.DataResolverResult) { + fake.resolveDataTrackForSubscriberMutex.Lock() + defer fake.resolveDataTrackForSubscriberMutex.Unlock() + fake.ResolveDataTrackForSubscriberStub = stub +} + +func (fake *FakeRoom) ResolveDataTrackForSubscriberArgsForCall(i int) (types.LocalParticipant, livekit.TrackID) { + fake.resolveDataTrackForSubscriberMutex.RLock() + defer fake.resolveDataTrackForSubscriberMutex.RUnlock() + argsForCall := fake.resolveDataTrackForSubscriberArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeRoom) ResolveDataTrackForSubscriberReturns(result1 types.DataResolverResult) { + fake.resolveDataTrackForSubscriberMutex.Lock() + defer fake.resolveDataTrackForSubscriberMutex.Unlock() + fake.ResolveDataTrackForSubscriberStub = nil + fake.resolveDataTrackForSubscriberReturns = struct { + result1 types.DataResolverResult + }{result1} +} + +func (fake *FakeRoom) ResolveDataTrackForSubscriberReturnsOnCall(i int, result1 types.DataResolverResult) { + fake.resolveDataTrackForSubscriberMutex.Lock() + defer fake.resolveDataTrackForSubscriberMutex.Unlock() + fake.ResolveDataTrackForSubscriberStub = nil + if fake.resolveDataTrackForSubscriberReturnsOnCall == nil { + fake.resolveDataTrackForSubscriberReturnsOnCall = make(map[int]struct { + result1 types.DataResolverResult + }) + } + fake.resolveDataTrackForSubscriberReturnsOnCall[i] = struct { + result1 types.DataResolverResult + }{result1} +} + func (fake *FakeRoom) ResolveMediaTrackForSubscriber(arg1 types.LocalParticipant, arg2 livekit.TrackID) types.MediaResolverResult { fake.resolveMediaTrackForSubscriberMutex.Lock() ret, specificReturn := fake.resolveMediaTrackForSubscriberReturnsOnCall[len(fake.resolveMediaTrackForSubscriberArgsForCall)] @@ -433,192 +471,6 @@ func (fake *FakeRoom) ResolveMediaTrackForSubscriberReturnsOnCall(i int, result1 }{result1} } -func (fake *FakeRoom) SimulateScenario(arg1 types.LocalParticipant, arg2 *livekit.SimulateScenario) error { - fake.simulateScenarioMutex.Lock() - ret, specificReturn := fake.simulateScenarioReturnsOnCall[len(fake.simulateScenarioArgsForCall)] - fake.simulateScenarioArgsForCall = append(fake.simulateScenarioArgsForCall, struct { - arg1 types.LocalParticipant - arg2 *livekit.SimulateScenario - }{arg1, arg2}) - stub := fake.SimulateScenarioStub - fakeReturns := fake.simulateScenarioReturns - fake.recordInvocation("SimulateScenario", []interface{}{arg1, arg2}) - fake.simulateScenarioMutex.Unlock() - if stub != nil { - return stub(arg1, arg2) - } - if specificReturn { - return ret.result1 - } - return fakeReturns.result1 -} - -func (fake *FakeRoom) SimulateScenarioCallCount() int { - fake.simulateScenarioMutex.RLock() - defer fake.simulateScenarioMutex.RUnlock() - return len(fake.simulateScenarioArgsForCall) -} - -func (fake *FakeRoom) SimulateScenarioCalls(stub func(types.LocalParticipant, *livekit.SimulateScenario) error) { - fake.simulateScenarioMutex.Lock() - defer fake.simulateScenarioMutex.Unlock() - fake.SimulateScenarioStub = stub -} - -func (fake *FakeRoom) SimulateScenarioArgsForCall(i int) (types.LocalParticipant, *livekit.SimulateScenario) { - fake.simulateScenarioMutex.RLock() - defer fake.simulateScenarioMutex.RUnlock() - argsForCall := fake.simulateScenarioArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2 -} - -func (fake *FakeRoom) SimulateScenarioReturns(result1 error) { - fake.simulateScenarioMutex.Lock() - defer fake.simulateScenarioMutex.Unlock() - fake.SimulateScenarioStub = nil - fake.simulateScenarioReturns = struct { - result1 error - }{result1} -} - -func (fake *FakeRoom) SimulateScenarioReturnsOnCall(i int, result1 error) { - fake.simulateScenarioMutex.Lock() - defer fake.simulateScenarioMutex.Unlock() - fake.SimulateScenarioStub = nil - if fake.simulateScenarioReturnsOnCall == nil { - fake.simulateScenarioReturnsOnCall = make(map[int]struct { - result1 error - }) - } - fake.simulateScenarioReturnsOnCall[i] = struct { - result1 error - }{result1} -} - -func (fake *FakeRoom) SyncState(arg1 types.LocalParticipant, arg2 *livekit.SyncState) error { - fake.syncStateMutex.Lock() - ret, specificReturn := fake.syncStateReturnsOnCall[len(fake.syncStateArgsForCall)] - fake.syncStateArgsForCall = append(fake.syncStateArgsForCall, struct { - arg1 types.LocalParticipant - arg2 *livekit.SyncState - }{arg1, arg2}) - stub := fake.SyncStateStub - fakeReturns := fake.syncStateReturns - fake.recordInvocation("SyncState", []interface{}{arg1, arg2}) - fake.syncStateMutex.Unlock() - if stub != nil { - return stub(arg1, arg2) - } - if specificReturn { - return ret.result1 - } - return fakeReturns.result1 -} - -func (fake *FakeRoom) SyncStateCallCount() int { - fake.syncStateMutex.RLock() - defer fake.syncStateMutex.RUnlock() - return len(fake.syncStateArgsForCall) -} - -func (fake *FakeRoom) SyncStateCalls(stub func(types.LocalParticipant, *livekit.SyncState) error) { - fake.syncStateMutex.Lock() - defer fake.syncStateMutex.Unlock() - fake.SyncStateStub = stub -} - -func (fake *FakeRoom) SyncStateArgsForCall(i int) (types.LocalParticipant, *livekit.SyncState) { - fake.syncStateMutex.RLock() - defer fake.syncStateMutex.RUnlock() - argsForCall := fake.syncStateArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2 -} - -func (fake *FakeRoom) SyncStateReturns(result1 error) { - fake.syncStateMutex.Lock() - defer fake.syncStateMutex.Unlock() - fake.SyncStateStub = nil - fake.syncStateReturns = struct { - result1 error - }{result1} -} - -func (fake *FakeRoom) SyncStateReturnsOnCall(i int, result1 error) { - fake.syncStateMutex.Lock() - defer fake.syncStateMutex.Unlock() - fake.SyncStateStub = nil - if fake.syncStateReturnsOnCall == nil { - fake.syncStateReturnsOnCall = make(map[int]struct { - result1 error - }) - } - fake.syncStateReturnsOnCall[i] = struct { - result1 error - }{result1} -} - -func (fake *FakeRoom) UpdateSubscriptionPermission(arg1 types.LocalParticipant, arg2 *livekit.SubscriptionPermission) error { - fake.updateSubscriptionPermissionMutex.Lock() - ret, specificReturn := fake.updateSubscriptionPermissionReturnsOnCall[len(fake.updateSubscriptionPermissionArgsForCall)] - fake.updateSubscriptionPermissionArgsForCall = append(fake.updateSubscriptionPermissionArgsForCall, struct { - arg1 types.LocalParticipant - arg2 *livekit.SubscriptionPermission - }{arg1, arg2}) - stub := fake.UpdateSubscriptionPermissionStub - fakeReturns := fake.updateSubscriptionPermissionReturns - fake.recordInvocation("UpdateSubscriptionPermission", []interface{}{arg1, arg2}) - fake.updateSubscriptionPermissionMutex.Unlock() - if stub != nil { - return stub(arg1, arg2) - } - if specificReturn { - return ret.result1 - } - return fakeReturns.result1 -} - -func (fake *FakeRoom) UpdateSubscriptionPermissionCallCount() int { - fake.updateSubscriptionPermissionMutex.RLock() - defer fake.updateSubscriptionPermissionMutex.RUnlock() - return len(fake.updateSubscriptionPermissionArgsForCall) -} - -func (fake *FakeRoom) UpdateSubscriptionPermissionCalls(stub func(types.LocalParticipant, *livekit.SubscriptionPermission) error) { - fake.updateSubscriptionPermissionMutex.Lock() - defer fake.updateSubscriptionPermissionMutex.Unlock() - fake.UpdateSubscriptionPermissionStub = stub -} - -func (fake *FakeRoom) UpdateSubscriptionPermissionArgsForCall(i int) (types.LocalParticipant, *livekit.SubscriptionPermission) { - fake.updateSubscriptionPermissionMutex.RLock() - defer fake.updateSubscriptionPermissionMutex.RUnlock() - argsForCall := fake.updateSubscriptionPermissionArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2 -} - -func (fake *FakeRoom) UpdateSubscriptionPermissionReturns(result1 error) { - fake.updateSubscriptionPermissionMutex.Lock() - defer fake.updateSubscriptionPermissionMutex.Unlock() - fake.UpdateSubscriptionPermissionStub = nil - fake.updateSubscriptionPermissionReturns = struct { - result1 error - }{result1} -} - -func (fake *FakeRoom) UpdateSubscriptionPermissionReturnsOnCall(i int, result1 error) { - fake.updateSubscriptionPermissionMutex.Lock() - defer fake.updateSubscriptionPermissionMutex.Unlock() - fake.UpdateSubscriptionPermissionStub = nil - if fake.updateSubscriptionPermissionReturnsOnCall == nil { - fake.updateSubscriptionPermissionReturnsOnCall = make(map[int]struct { - result1 error - }) - } - fake.updateSubscriptionPermissionReturnsOnCall[i] = struct { - result1 error - }{result1} -} - func (fake *FakeRoom) UpdateSubscriptions(arg1 types.LocalParticipant, arg2 []livekit.TrackID, arg3 []*livekit.ParticipantTracks, arg4 bool) { var arg2Copy []livekit.TrackID if arg2 != nil { @@ -667,26 +519,6 @@ func (fake *FakeRoom) UpdateSubscriptionsArgsForCall(i int) (types.LocalParticip func (fake *FakeRoom) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() - fake.getLocalParticipantsMutex.RLock() - defer fake.getLocalParticipantsMutex.RUnlock() - fake.iDMutex.RLock() - defer fake.iDMutex.RUnlock() - fake.isDataMessageUserPacketDuplicateMutex.RLock() - defer fake.isDataMessageUserPacketDuplicateMutex.RUnlock() - fake.nameMutex.RLock() - defer fake.nameMutex.RUnlock() - fake.removeParticipantMutex.RLock() - defer fake.removeParticipantMutex.RUnlock() - fake.resolveMediaTrackForSubscriberMutex.RLock() - defer fake.resolveMediaTrackForSubscriberMutex.RUnlock() - fake.simulateScenarioMutex.RLock() - defer fake.simulateScenarioMutex.RUnlock() - fake.syncStateMutex.RLock() - defer fake.syncStateMutex.RUnlock() - fake.updateSubscriptionPermissionMutex.RLock() - defer fake.updateSubscriptionPermissionMutex.RUnlock() - fake.updateSubscriptionsMutex.RLock() - defer fake.updateSubscriptionsMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} for key, value := range fake.invocations { copiedInvocations[key] = value diff --git a/pkg/rtc/types/typesfakes/fake_subscribed_track.go b/pkg/rtc/types/typesfakes/fake_subscribed_track.go index 6a836a2..2123548 100644 --- a/pkg/rtc/types/typesfakes/fake_subscribed_track.go +++ b/pkg/rtc/types/typesfakes/fake_subscribed_track.go @@ -1052,44 +1052,6 @@ func (fake *FakeSubscribedTrack) UpdateVideoLayerCalls(stub func()) { func (fake *FakeSubscribedTrack) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() - fake.addOnBindMutex.RLock() - defer fake.addOnBindMutex.RUnlock() - fake.closeMutex.RLock() - defer fake.closeMutex.RUnlock() - fake.downTrackMutex.RLock() - defer fake.downTrackMutex.RUnlock() - fake.iDMutex.RLock() - defer fake.iDMutex.RUnlock() - fake.isBoundMutex.RLock() - defer fake.isBoundMutex.RUnlock() - fake.isMutedMutex.RLock() - defer fake.isMutedMutex.RUnlock() - fake.mediaTrackMutex.RLock() - defer fake.mediaTrackMutex.RUnlock() - fake.needsNegotiationMutex.RLock() - defer fake.needsNegotiationMutex.RUnlock() - fake.onCloseMutex.RLock() - defer fake.onCloseMutex.RUnlock() - fake.publisherIDMutex.RLock() - defer fake.publisherIDMutex.RUnlock() - fake.publisherIdentityMutex.RLock() - defer fake.publisherIdentityMutex.RUnlock() - fake.publisherVersionMutex.RLock() - defer fake.publisherVersionMutex.RUnlock() - fake.rTPSenderMutex.RLock() - defer fake.rTPSenderMutex.RUnlock() - fake.setPublisherMutedMutex.RLock() - defer fake.setPublisherMutedMutex.RUnlock() - fake.subscriberMutex.RLock() - defer fake.subscriberMutex.RUnlock() - fake.subscriberIDMutex.RLock() - defer fake.subscriberIDMutex.RUnlock() - fake.subscriberIdentityMutex.RLock() - defer fake.subscriberIdentityMutex.RUnlock() - fake.updateSubscriberSettingsMutex.RLock() - defer fake.updateSubscriberSettingsMutex.RUnlock() - fake.updateVideoLayerMutex.RLock() - defer fake.updateVideoLayerMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} for key, value := range fake.invocations { copiedInvocations[key] = value diff --git a/pkg/rtc/types/typesfakes/fake_websocket_client.go b/pkg/rtc/types/typesfakes/fake_websocket_client.go index 93d0957..6d5b44c 100644 --- a/pkg/rtc/types/typesfakes/fake_websocket_client.go +++ b/pkg/rtc/types/typesfakes/fake_websocket_client.go @@ -384,16 +384,6 @@ func (fake *FakeWebsocketClient) WriteMessageReturnsOnCall(i int, result1 error) func (fake *FakeWebsocketClient) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() - fake.closeMutex.RLock() - defer fake.closeMutex.RUnlock() - fake.readMessageMutex.RLock() - defer fake.readMessageMutex.RUnlock() - fake.setReadDeadlineMutex.RLock() - defer fake.setReadDeadlineMutex.RUnlock() - fake.writeControlMutex.RLock() - defer fake.writeControlMutex.RUnlock() - fake.writeMessageMutex.RLock() - defer fake.writeMessageMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} for key, value := range fake.invocations { copiedInvocations[key] = value diff --git a/pkg/rtc/unhandlesimulcast.go b/pkg/rtc/unhandlesimulcast.go deleted file mode 100644 index 3146cba..0000000 --- a/pkg/rtc/unhandlesimulcast.go +++ /dev/null @@ -1,132 +0,0 @@ -// Copyright 2023 LiveKit, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package rtc - -import ( - "github.com/pion/interceptor" - "github.com/pion/rtp" - "github.com/pion/sdp/v3" - "github.com/pion/webrtc/v4" - - "github.com/livekit/livekit-server/pkg/sfu/utils" -) - -const ( - simulcastProbeCount = 10 -) - -type UnhandleSimulcastOption func(r *UnhandleSimulcastInterceptor) error - -func UnhandleSimulcastTracks(tracks map[uint32]SimulcastTrackInfo) UnhandleSimulcastOption { - return func(r *UnhandleSimulcastInterceptor) error { - r.simTracks = tracks - return nil - } -} - -type UnhandleSimulcastInterceptorFactory struct { - opts []UnhandleSimulcastOption -} - -func (f *UnhandleSimulcastInterceptorFactory) NewInterceptor(id string) (interceptor.Interceptor, error) { - i := &UnhandleSimulcastInterceptor{simTracks: map[uint32]SimulcastTrackInfo{}} - for _, o := range f.opts { - if err := o(i); err != nil { - return nil, err - } - } - return i, nil -} - -func NewUnhandleSimulcastInterceptorFactory(opts ...UnhandleSimulcastOption) (*UnhandleSimulcastInterceptorFactory, error) { - return &UnhandleSimulcastInterceptorFactory{opts: opts}, nil -} - -type unhandleSimulcastRTPReader struct { - SimulcastTrackInfo - tryTimes int - reader interceptor.RTPReader - midExtensionID uint8 - streamIDExtensionID uint8 -} - -func (r *unhandleSimulcastRTPReader) Read(b []byte, a interceptor.Attributes) (int, interceptor.Attributes, error) { - n, a, err := r.reader.Read(b, a) - if r.tryTimes < 0 || err != nil { - return n, a, err - } - - header := rtp.Header{} - hsize, err := header.Unmarshal(b[:n]) - if err != nil { - return n, a, nil - } - var mid, rid string - if payload := header.GetExtension(r.midExtensionID); payload != nil { - mid = string(payload) - } - - if payload := header.GetExtension(r.streamIDExtensionID); payload != nil { - rid = string(payload) - } - - if mid != "" && rid != "" { - r.tryTimes = -1 - return n, a, nil - } - - r.tryTimes-- - - if mid == "" { - header.SetExtension(r.midExtensionID, []byte(r.Mid)) - } - if rid == "" { - header.SetExtension(r.streamIDExtensionID, []byte(r.Rid)) - } - - hsize2 := header.MarshalSize() - - if hsize2-hsize+n > len(b) { // no enough buf to set extension - return n, a, nil - } - copy(b[hsize2:], b[hsize:n]) - header.MarshalTo(b) - return hsize2 - hsize + n, a, nil -} - -type UnhandleSimulcastInterceptor struct { - interceptor.NoOp - simTracks map[uint32]SimulcastTrackInfo -} - -func (u *UnhandleSimulcastInterceptor) BindRemoteStream(info *interceptor.StreamInfo, reader interceptor.RTPReader) interceptor.RTPReader { - if t, ok := u.simTracks[info.SSRC]; ok { - // if we support fec for simulcast streams at future, should get rsid extensions - midExtensionID := utils.GetHeaderExtensionID(info.RTPHeaderExtensions, webrtc.RTPHeaderExtensionCapability{URI: sdp.SDESMidURI}) - streamIDExtensionID := utils.GetHeaderExtensionID(info.RTPHeaderExtensions, webrtc.RTPHeaderExtensionCapability{URI: sdp.SDESRTPStreamIDURI}) - if midExtensionID == 0 || streamIDExtensionID == 0 { - return reader - } - - return &unhandleSimulcastRTPReader{ - SimulcastTrackInfo: t, - reader: reader, - tryTimes: simulcastProbeCount, - midExtensionID: uint8(midExtensionID), - streamIDExtensionID: uint8(streamIDExtensionID), - } - } - return reader -} diff --git a/pkg/rtc/updatatrackmanager.go b/pkg/rtc/updatatrackmanager.go new file mode 100644 index 0000000..c3af8dc --- /dev/null +++ b/pkg/rtc/updatatrackmanager.go @@ -0,0 +1,126 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rtc + +import ( + "maps" + "slices" + "sync" + + "github.com/livekit/livekit-server/pkg/rtc/datatrack" + "github.com/livekit/livekit-server/pkg/rtc/types" + "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/logger" +) + +type UpDataTrackManagerParams struct { + Logger logger.Logger + Participant types.Participant +} + +type UpDataTrackManager struct { + params UpDataTrackManagerParams + + lock sync.RWMutex + dataTracks map[uint16]types.DataTrack + closed bool +} + +func NewUpDataTrackManager(params UpDataTrackManagerParams) *UpDataTrackManager { + return &UpDataTrackManager{ + params: params, + dataTracks: make(map[uint16]types.DataTrack), + } +} + +func (u *UpDataTrackManager) Close() { + u.lock.Lock() + if u.closed { + u.lock.Unlock() + return + } + + u.closed = true + + dataTracks := u.dataTracks + u.dataTracks = make(map[uint16]types.DataTrack) + u.lock.Unlock() + + for _, t := range dataTracks { + t.Close() + } +} + +func (u *UpDataTrackManager) AddPublishedDataTrack(dt types.DataTrack) { + u.lock.Lock() + u.dataTracks[dt.PubHandle()] = dt + u.lock.Unlock() + + u.params.Participant.GetParticipantListener().OnDataTrackPublished(u.params.Participant, dt) +} + +func (u *UpDataTrackManager) RemovePublishedDataTrack(dt types.DataTrack) { + var found bool + pubHandle := dt.PubHandle() + u.lock.Lock() + if u.dataTracks[pubHandle] == dt { + delete(u.dataTracks, pubHandle) + found = true + } + u.lock.Unlock() + + if found { + dt.Close() + + u.params.Participant.GetParticipantListener().OnDataTrackUnpublished(u.params.Participant, dt) + } +} + +func (u *UpDataTrackManager) GetPublishedDataTracks() []types.DataTrack { + u.lock.RLock() + defer u.lock.RUnlock() + + return slices.Collect(maps.Values(u.dataTracks)) +} + +func (u *UpDataTrackManager) GetPublishedDataTrack(handle uint16) types.DataTrack { + u.lock.RLock() + defer u.lock.RUnlock() + + return u.dataTracks[handle] +} + +func (u *UpDataTrackManager) HandleReceivedDataTrackMessage(data []byte, packet *datatrack.Packet, arrivalTime int64) { + u.lock.RLock() + dt := u.dataTracks[packet.Handle] + u.lock.RUnlock() + if dt == nil { + return + } + + dt.HandlePacket(data, packet, arrivalTime) +} + +func (u *UpDataTrackManager) ToProto() []*livekit.DataTrackInfo { + u.lock.RLock() + defer u.lock.RUnlock() + + var dataTrackInfos []*livekit.DataTrackInfo + for _, dt := range u.dataTracks { + dataTrackInfos = append(dataTrackInfos, dt.ToProto()) + } + + return dataTrackInfos +} diff --git a/pkg/rtc/uptrackmanager.go b/pkg/rtc/uptrackmanager.go index 13cec38..f75f501 100644 --- a/pkg/rtc/uptrackmanager.go +++ b/pkg/rtc/uptrackmanager.go @@ -16,11 +16,12 @@ package rtc import ( "errors" + "maps" + "slices" "sync" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" - "golang.org/x/exp/maps" "github.com/livekit/livekit-server/pkg/rtc/types" "github.com/livekit/protocol/utils" @@ -116,13 +117,15 @@ func (u *UpTrackManager) OnPublishedTrackUpdated(f func(track types.MediaTrack)) u.onTrackUpdated = f } -func (u *UpTrackManager) SetPublishedTrackMuted(trackID livekit.TrackID, muted bool) types.MediaTrack { +func (u *UpTrackManager) SetPublishedTrackMuted(trackID livekit.TrackID, muted bool) (types.MediaTrack, bool) { + changed := false track := u.GetPublishedTrack(trackID) if track != nil { currentMuted := track.IsMuted() track.SetMuted(muted) if currentMuted != track.IsMuted() { + changed = true u.params.Logger.Debugw("publisher mute status changed", "trackID", trackID, "muted", track.IsMuted()) if u.onTrackUpdated != nil { u.onTrackUpdated(track) @@ -130,7 +133,7 @@ func (u *UpTrackManager) SetPublishedTrackMuted(trackID livekit.TrackID, muted b } } - return track + return track, changed } func (u *UpTrackManager) GetPublishedTrack(trackID livekit.TrackID) types.MediaTrack { @@ -144,7 +147,7 @@ func (u *UpTrackManager) GetPublishedTracks() []types.MediaTrack { u.lock.RLock() defer u.lock.RUnlock() - return maps.Values(u.publishedTracks) + return slices.Collect(maps.Values(u.publishedTracks)) } func (u *UpTrackManager) UpdateSubscriptionPermission( @@ -272,12 +275,9 @@ func (u *UpTrackManager) AddPublishedTrack(track types.MediaTrack) { }) } -func (u *UpTrackManager) RemovePublishedTrack(track types.MediaTrack, isExpectedToResume bool, shouldClose bool) { - if shouldClose { - track.Close(isExpectedToResume) - } else { - track.ClearAllReceivers(isExpectedToResume) - } +func (u *UpTrackManager) RemovePublishedTrack(track types.MediaTrack, isExpectedToResume bool) { + track.Close(isExpectedToResume) + u.lock.Lock() delete(u.publishedTracks, track.ID()) u.lock.Unlock() @@ -311,7 +311,11 @@ func (u *UpTrackManager) parseSubscriptionPermissionsLocked( sub := resolver(livekit.ParticipantID(trackPerms.ParticipantSid)) if sub == nil { - u.params.Logger.Warnw("could not find subscriber for permissions update", nil, "subscriberID", trackPerms.ParticipantSid) + u.params.Logger.Warnw( + "could not find subscriber for permissions update", nil, + "subscriberID", trackPerms.ParticipantSid, + "subscriptionPermission", logger.Proto(subscriptionPermission), + ) continue } @@ -320,10 +324,19 @@ func (u *UpTrackManager) parseSubscriptionPermissionsLocked( if trackPerms.ParticipantSid != "" { sub := resolver(livekit.ParticipantID(trackPerms.ParticipantSid)) if sub != nil && sub.Identity() != subscriberIdentity { - u.params.Logger.Errorw("participant identity mismatch", nil, "expected", subscriberIdentity, "got", sub.Identity()) + u.params.Logger.Errorw( + "participant identity mismatch", nil, + "expected", subscriberIdentity, + "got", sub.Identity(), + "subscriptionPermission", logger.Proto(subscriptionPermission), + ) } if sub == nil { - u.params.Logger.Warnw("could not find subscriber for permissions update", nil, "subscriberID", trackPerms.ParticipantSid) + u.params.Logger.Warnw( + "could not find subscriber for permissions update", nil, + "subscriberID", trackPerms.ParticipantSid, + "subscriptionPermission", logger.Proto(subscriptionPermission), + ) } } } @@ -399,16 +412,16 @@ func (u *UpTrackManager) maybeRevokeSubscriptions() { } } -func (u *UpTrackManager) DebugInfo() map[string]interface{} { - info := map[string]interface{}{} - publishedTrackInfo := make(map[livekit.TrackID]interface{}) +func (u *UpTrackManager) DebugInfo() map[string]any { + info := map[string]any{} + publishedTrackInfo := make(map[livekit.TrackID]any) u.lock.RLock() for trackID, track := range u.publishedTracks { if mt, ok := track.(*MediaTrack); ok { publishedTrackInfo[trackID] = mt.DebugInfo() } else { - publishedTrackInfo[trackID] = map[string]interface{}{ + publishedTrackInfo[trackID] = map[string]any{ "ID": track.ID(), "Kind": track.Kind().String(), "PubMuted": track.IsMuted(), diff --git a/pkg/rtc/utils.go b/pkg/rtc/utils.go index b6d0c29..83d9b78 100644 --- a/pkg/rtc/utils.go +++ b/pkg/rtc/utils.go @@ -15,15 +15,15 @@ package rtc import ( - "encoding/json" "errors" "io" "net" "strings" "github.com/pion/webrtc/v4" + "google.golang.org/protobuf/proto" - "github.com/livekit/livekit-server/pkg/sfu/mime" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" ) @@ -80,49 +80,6 @@ func UnpackDataTrackLabel(packed string) (participantID livekit.ParticipantID, t return } -func ToProtoSessionDescription(sd webrtc.SessionDescription) *livekit.SessionDescription { - return &livekit.SessionDescription{ - Type: sd.Type.String(), - Sdp: sd.SDP, - } -} - -func FromProtoSessionDescription(sd *livekit.SessionDescription) webrtc.SessionDescription { - var sdType webrtc.SDPType - switch sd.Type { - case webrtc.SDPTypeOffer.String(): - sdType = webrtc.SDPTypeOffer - case webrtc.SDPTypeAnswer.String(): - sdType = webrtc.SDPTypeAnswer - case webrtc.SDPTypePranswer.String(): - sdType = webrtc.SDPTypePranswer - case webrtc.SDPTypeRollback.String(): - sdType = webrtc.SDPTypeRollback - } - return webrtc.SessionDescription{ - Type: sdType, - SDP: sd.Sdp, - } -} - -func ToProtoTrickle(candidateInit webrtc.ICECandidateInit, target livekit.SignalTarget, final bool) *livekit.TrickleRequest { - data, _ := json.Marshal(candidateInit) - return &livekit.TrickleRequest{ - CandidateInit: string(data), - Target: target, - Final: final, - } -} - -func FromProtoTrickle(trickle *livekit.TrickleRequest) (webrtc.ICECandidateInit, error) { - ci := webrtc.ICECandidateInit{} - err := json.Unmarshal([]byte(trickle.CandidateInit), &ci) - if err != nil { - return webrtc.ICECandidateInit{}, err - } - return ci, nil -} - func ToProtoTrackKind(kind webrtc.RTPCodecType) livekit.TrackType { switch kind { case webrtc.RTPCodecTypeVideo: @@ -160,12 +117,12 @@ func Recover(l logger.Logger) any { // logger helpers func LoggerWithParticipant(l logger.Logger, identity livekit.ParticipantIdentity, sid livekit.ParticipantID, isRemote bool) logger.Logger { - values := make([]interface{}, 0, 4) + values := make([]any, 0, 4) if identity != "" { values = append(values, "participant", identity) } if sid != "" { - values = append(values, "pID", sid) + values = append(values, "participantID", sid) } values = append(values, "remote", isRemote) // enable sampling per participant @@ -173,7 +130,7 @@ func LoggerWithParticipant(l logger.Logger, identity livekit.ParticipantIdentity } func LoggerWithRoom(l logger.Logger, name livekit.RoomName, roomID livekit.RoomID) logger.Logger { - values := make([]interface{}, 0, 2) + values := make([]any, 0, 2) if name != "" { values = append(values, "room", name) } @@ -215,3 +172,31 @@ func MaybeTruncateIP(addr string) string { return addr[:len(addr)-3] + "..." } + +func ChunkProtoBatch[T proto.Message](batch []T, target int) [][]T { + var chunks [][]T + var start, size int + for i, m := range batch { + s := proto.Size(m) + if size+s > target { + if start < i { + chunks = append(chunks, batch[start:i]) + } + start = i + size = 0 + } + size += s + } + if start < len(batch) { + chunks = append(chunks, batch[start:]) + } + return chunks +} + +func IsRedEnabled(ti *livekit.TrackInfo) bool { + if len(ti.Codecs) != 0 && ti.Codecs[0].MimeType != "" { + return mime.IsMimeTypeStringRED(ti.Codecs[0].MimeType) + } + + return !ti.GetDisableRed() +} diff --git a/pkg/rtc/utils_test.go b/pkg/rtc/utils_test.go index 813f910..37a6f01 100644 --- a/pkg/rtc/utils_test.go +++ b/pkg/rtc/utils_test.go @@ -15,11 +15,16 @@ package rtc import ( + "math/rand/v2" + "strings" "testing" + "github.com/google/uuid" "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/utils/guid" ) func TestPackStreamId(t *testing.T) { @@ -43,3 +48,28 @@ func TestPackDataTrackLabel(t *testing.T) { require.Equal(t, trackID, tr) require.Equal(t, label, l) } + +func TestChunkProtoBatch(t *testing.T) { + rng := rand.New(rand.NewPCG(1, 2)) + var updates []*livekit.ParticipantInfo + for range 32 { + updates = append(updates, &livekit.ParticipantInfo{ + Sid: guid.New(guid.ParticipantPrefix), + Identity: uuid.NewString(), + Metadata: strings.Repeat("x", rng.IntN(128*1024)), + }) + } + + target := 64 * 1024 + batches := ChunkProtoBatch(updates, target) + var count int + for _, b := range batches { + var sum int + for _, m := range b { + sum += proto.Size(m) + count++ + } + require.True(t, sum < target || len(b) == 1, "batch size exceeds target") + } + require.Equal(t, len(updates), count) +} diff --git a/pkg/rtc/wrappedreceiver.go b/pkg/rtc/wrappedreceiver.go index a4d41a7..4ff4da7 100644 --- a/pkg/rtc/wrappedreceiver.go +++ b/pkg/rtc/wrappedreceiver.go @@ -16,18 +16,21 @@ package rtc import ( "errors" + "maps" + "slices" "sync" "github.com/pion/webrtc/v4" "go.uber.org/atomic" - "golang.org/x/exp/maps" - "golang.org/x/exp/slices" + "github.com/livekit/mediatransportutil/pkg/codec" + protoCodecs "github.com/livekit/protocol/codecs" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" "github.com/livekit/livekit-server/pkg/sfu" - "github.com/livekit/livekit-server/pkg/sfu/mime" + "github.com/livekit/livekit-server/pkg/sfu/buffer" ) // wrapper around WebRTC receiver, overriding its ID @@ -39,6 +42,7 @@ type WrappedReceiverParams struct { UpstreamCodecs []webrtc.RTPCodecParameters Logger logger.Logger DisableRed bool + IsEncrypted bool } type WrappedReceiver struct { @@ -62,16 +66,10 @@ func NewWrappedReceiver(params WrappedReceiverParams) *WrappedReceiver { normalizedMimeType := mime.NormalizeMimeType(codecs[0].MimeType) if normalizedMimeType == mime.MimeTypeRED { // if upstream is opus/red, then add opus to match clients that don't support red - codecs = append(codecs, webrtc.RTPCodecParameters{ - RTPCodecCapability: OpusCodecCapability, - PayloadType: 111, - }) + codecs = append(codecs, protoCodecs.OpusCodecParameters) } else if !params.DisableRed && normalizedMimeType == mime.MimeTypeOpus { // if upstream is opus only and red enabled, add red to match clients that support red - codecs = append(codecs, webrtc.RTPCodecParameters{ - RTPCodecCapability: RedCodecCapability, - PayloadType: 63, - }) + codecs = append(codecs, protoCodecs.RedCodecParameters) // prefer red codec codecs[0], codecs[1] = codecs[1], codecs[0] } @@ -92,54 +90,67 @@ func (r *WrappedReceiver) StreamID() string { return r.params.StreamId } -// DetermineReceiver determines the receiver of negotiated codec and return ready state of the receiver -func (r *WrappedReceiver) DetermineReceiver(codec webrtc.RTPCodecCapability) bool { +// DetermineReceiver determines the receiver of negotiated codec and returns +// +// isAvailable: returns true if given codec is a potential codec from publisher or if an existing published codec can be translated +// needsPublish: indicates if the codec is needed from publisher, some combinations can be achieved via codec translation internally, +// +// example: unencrypted opus -> RED translation and vice-versa can be done without the need for publisher to send the other codec. +func (r *WrappedReceiver) DetermineReceiver(codec webrtc.RTPCodecCapability) (isAvailable bool, needsPublish bool) { r.lock.Lock() + reason := "no matching receiver" codecMimeType := mime.NormalizeMimeType(codec.MimeType) var trackReceiver sfu.TrackReceiver for _, receiver := range r.receivers { receiverMimeType := receiver.Mime() if receiverMimeType == codecMimeType { trackReceiver = receiver - break - } else if receiverMimeType == mime.MimeTypeRED && codecMimeType == mime.MimeTypeOpus { - // audio opus/red can match opus only - trackReceiver = receiver.GetPrimaryReceiverForRed() - break - } else if receiverMimeType == mime.MimeTypeOpus && codecMimeType == mime.MimeTypeRED { - trackReceiver = receiver.GetRedReceiver() + isAvailable = true + needsPublish = true break } + + if receiverMimeType == mime.MimeTypeRED && codecMimeType == mime.MimeTypeOpus { + // audio opus/red can match opus only + if !r.params.IsEncrypted { // cannot match encrypted source + trackReceiver = receiver.GetPrimaryReceiverForRed() + isAvailable = true + break + } else { + reason = "encrypted source" + } + } else if receiverMimeType == mime.MimeTypeOpus && codecMimeType == mime.MimeTypeRED { + if !r.params.IsEncrypted { // cannot match encrypted source + trackReceiver = receiver.GetRedReceiver() + isAvailable = true + break + } else { + reason = "encrypted source" + } + } + } if trackReceiver == nil { - r.params.Logger.Errorw("can't determine receiver for codec", nil, "codec", codec.MimeType) - if len(r.receivers) > 0 { - trackReceiver = r.receivers[0] - } + r.lock.Unlock() + r.params.Logger.Warnw( + "can't determine receiver for codec", nil, + "codec", codec.MimeType, + "reason", reason, + ) + return } r.TrackReceiver = trackReceiver - var onReadyCallbacks []func() - if trackReceiver != nil { - onReadyCallbacks = r.onReadyCallbacks - r.onReadyCallbacks = nil - } + onReadyCallbacks := r.onReadyCallbacks + r.onReadyCallbacks = nil r.lock.Unlock() - if trackReceiver != nil { - for _, f := range onReadyCallbacks { - trackReceiver.AddOnReady(f) - } - - if s, ok := trackReceiver.(*simulcastReceiver); ok { - if d, ok := s.TrackReceiver.(*DummyReceiver); ok { - return d.IsReady() - } - } - return true + for _, f := range onReadyCallbacks { + trackReceiver.AddOnReady(f) } - return false + + return } func (r *WrappedReceiver) Codecs() []webrtc.RTPCodecParameters { @@ -186,8 +197,6 @@ type DummyReceiver struct { settingsLock sync.Mutex maxExpectedLayerValid bool maxExpectedLayer int32 - pausedValid bool - paused bool redReceiver, primaryReceiver *DummyRedReceiver } @@ -217,8 +226,10 @@ func (d *DummyReceiver) Upgrade(receiver sfu.TrackReceiver) { receiver.AddDownTrack(t) } d.downTracks = make(map[livekit.ParticipantID]sfu.TrackSender) + onReadyCallbacks := d.onReadyCallbacks d.onReadyCallbacks = nil + codecChange := d.onCodecStateChange d.onCodecStateChange = nil d.downTrackLock.Unlock() @@ -232,16 +243,15 @@ func (d *DummyReceiver) Upgrade(receiver sfu.TrackReceiver) { } d.settingsLock.Lock() - if d.maxExpectedLayerValid { + maxExpectedLayerValid := d.maxExpectedLayerValid + d.maxExpectedLayerValid = false + d.settingsLock.Unlock() + + if maxExpectedLayerValid { receiver.SetMaxExpectedSpatialLayer(d.maxExpectedLayer) } - d.maxExpectedLayerValid = false - - if d.pausedValid { - receiver.SetUpTrackPaused(d.paused) - } - d.pausedValid = false + d.settingsLock.Lock() if d.primaryReceiver != nil { d.primaryReceiver.upgrade(receiver) } @@ -260,83 +270,82 @@ func (d *DummyReceiver) StreamID() string { } func (d *DummyReceiver) Codec() webrtc.RTPCodecParameters { - if r, ok := d.receiver.Load().(sfu.TrackReceiver); ok { - return r.Codec() + if receiver := d.getReceiver(); receiver != nil { + return receiver.Codec() } return d.codec } func (d *DummyReceiver) Mime() mime.MimeType { - if r, ok := d.receiver.Load().(sfu.TrackReceiver); ok { - return r.Mime() + if receiver := d.getReceiver(); receiver != nil { + return receiver.Mime() } return mime.NormalizeMimeType(d.codec.MimeType) } +func (d *DummyReceiver) VideoLayerMode() livekit.VideoLayer_Mode { + if receiver := d.getReceiver(); receiver != nil { + return receiver.VideoLayerMode() + } + return buffer.GetVideoLayerModeForMimeType(d.Mime(), d.TrackInfo()) +} + func (d *DummyReceiver) HeaderExtensions() []webrtc.RTPHeaderExtensionParameter { - if r, ok := d.receiver.Load().(sfu.TrackReceiver); ok { - return r.HeaderExtensions() + if receiver := d.getReceiver(); receiver != nil { + return receiver.HeaderExtensions() } return d.headerExtensions } func (d *DummyReceiver) ReadRTP(buf []byte, layer uint8, esn uint64) (int, error) { - if r, ok := d.receiver.Load().(sfu.TrackReceiver); ok { - return r.ReadRTP(buf, layer, esn) + if receiver := d.getReceiver(); receiver != nil { + return receiver.ReadRTP(buf, layer, esn) } return 0, errors.New("no receiver") } func (d *DummyReceiver) GetLayeredBitrate() ([]int32, sfu.Bitrates) { - if r, ok := d.receiver.Load().(sfu.TrackReceiver); ok { - return r.GetLayeredBitrate() + if receiver := d.getReceiver(); receiver != nil { + return receiver.GetLayeredBitrate() } return nil, sfu.Bitrates{} } func (d *DummyReceiver) GetAudioLevel() (float64, bool) { - if r, ok := d.receiver.Load().(sfu.TrackReceiver); ok { - return r.GetAudioLevel() + if receiver := d.getReceiver(); receiver != nil { + return receiver.GetAudioLevel() } return 0, false } func (d *DummyReceiver) SendPLI(layer int32, force bool) { - if r, ok := d.receiver.Load().(sfu.TrackReceiver); ok { - r.SendPLI(layer, force) - } -} - -func (d *DummyReceiver) SetUpTrackPaused(paused bool) { - d.settingsLock.Lock() - defer d.settingsLock.Unlock() - if r, ok := d.receiver.Load().(sfu.TrackReceiver); ok { - d.pausedValid = false - r.SetUpTrackPaused(paused) - } else { - d.pausedValid = true - d.paused = paused + if receiver := d.getReceiver(); receiver != nil { + receiver.SendPLI(layer, force) } } func (d *DummyReceiver) SetMaxExpectedSpatialLayer(layer int32) { d.settingsLock.Lock() - defer d.settingsLock.Unlock() - if r, ok := d.receiver.Load().(sfu.TrackReceiver); ok { + receiver := d.getReceiver() + if receiver != nil { d.maxExpectedLayerValid = false - r.SetMaxExpectedSpatialLayer(layer) } else { d.maxExpectedLayerValid = true d.maxExpectedLayer = layer } + d.settingsLock.Unlock() + + if receiver != nil { + receiver.SetMaxExpectedSpatialLayer(layer) + } } func (d *DummyReceiver) AddDownTrack(track sfu.TrackSender) error { d.downTrackLock.Lock() defer d.downTrackLock.Unlock() - if r, ok := d.receiver.Load().(sfu.TrackReceiver); ok { - r.AddDownTrack(track) + if receiver := d.getReceiver(); receiver != nil { + receiver.AddDownTrack(track) } else { d.downTracks[track.SubscriberID()] = track } @@ -347,8 +356,8 @@ func (d *DummyReceiver) DeleteDownTrack(subscriberID livekit.ParticipantID) { d.downTrackLock.Lock() defer d.downTrackLock.Unlock() - if r, ok := d.receiver.Load().(sfu.TrackReceiver); ok { - r.DeleteDownTrack(subscriberID) + if receiver := d.getReceiver(); receiver != nil { + receiver.DeleteDownTrack(subscriberID) } else { delete(d.downTracks, subscriberID) } @@ -358,42 +367,42 @@ func (d *DummyReceiver) GetDownTracks() []sfu.TrackSender { d.downTrackLock.Lock() defer d.downTrackLock.Unlock() - if r, ok := d.receiver.Load().(sfu.TrackReceiver); ok { - return r.GetDownTracks() + if receiver := d.getReceiver(); receiver != nil { + return receiver.GetDownTracks() } - return maps.Values(d.downTracks) + return slices.Collect(maps.Values(d.downTracks)) } -func (d *DummyReceiver) DebugInfo() map[string]interface{} { - if r, ok := d.receiver.Load().(sfu.TrackReceiver); ok { - return r.DebugInfo() +func (d *DummyReceiver) DebugInfo() map[string]any { + if receiver := d.getReceiver(); receiver != nil { + return receiver.DebugInfo() } return nil } func (d *DummyReceiver) GetTemporalLayerFpsForSpatial(spatial int32) []float32 { - if r, ok := d.receiver.Load().(sfu.TrackReceiver); ok { - return r.GetTemporalLayerFpsForSpatial(spatial) + if receiver := d.getReceiver(); receiver != nil { + return receiver.GetTemporalLayerFpsForSpatial(spatial) } return nil } func (d *DummyReceiver) TrackInfo() *livekit.TrackInfo { - if r, ok := d.receiver.Load().(sfu.TrackReceiver); ok { - return r.TrackInfo() + if receiver := d.getReceiver(); receiver != nil { + return receiver.TrackInfo() } return nil } func (d *DummyReceiver) UpdateTrackInfo(ti *livekit.TrackInfo) { - if r, ok := d.receiver.Load().(sfu.TrackReceiver); ok { - r.UpdateTrackInfo(ti) + if receiver := d.getReceiver(); receiver != nil { + receiver.UpdateTrackInfo(ti) } } func (d *DummyReceiver) IsClosed() bool { - if r, ok := d.receiver.Load().(sfu.TrackReceiver); ok { - return r.IsClosed() + if receiver := d.getReceiver(); receiver != nil { + return receiver.IsClosed() } return false } @@ -424,18 +433,16 @@ func (d *DummyReceiver) GetRedReceiver() sfu.TrackReceiver { } func (d *DummyReceiver) GetTrackStats() *livekit.RTPStats { - if r, ok := d.receiver.Load().(sfu.TrackReceiver); ok { - return r.GetTrackStats() + if receiver := d.getReceiver(); receiver != nil { + return receiver.GetTrackStats() } return nil } func (d *DummyReceiver) AddOnReady(f func()) { - var receiver sfu.TrackReceiver d.downTrackLock.Lock() - if r, ok := d.receiver.Load().(sfu.TrackReceiver); ok { - receiver = r - } else { + receiver := d.getReceiver() + if receiver == nil { d.onReadyCallbacks = append(d.onReadyCallbacks, f) } d.downTrackLock.Unlock() @@ -444,16 +451,10 @@ func (d *DummyReceiver) AddOnReady(f func()) { } } -func (d *DummyReceiver) IsReady() bool { - return d.receiver.Load() != nil -} - func (d *DummyReceiver) AddOnCodecStateChange(f func(codec webrtc.RTPCodecParameters, state sfu.ReceiverCodecState)) { - var receiver sfu.TrackReceiver d.downTrackLock.Lock() - if r, ok := d.receiver.Load().(sfu.TrackReceiver); ok { - receiver = r - } else { + receiver := d.getReceiver() + if receiver == nil { d.onCodecStateChange = append(d.onCodecStateChange, f) } d.downTrackLock.Unlock() @@ -463,12 +464,34 @@ func (d *DummyReceiver) AddOnCodecStateChange(f func(codec webrtc.RTPCodecParame } func (d *DummyReceiver) CodecState() sfu.ReceiverCodecState { - if r, ok := d.receiver.Load().(sfu.TrackReceiver); ok { - return r.CodecState() + if receiver := d.getReceiver(); receiver != nil { + return receiver.CodecState() } return sfu.ReceiverCodecStateNormal } +func (d *DummyReceiver) VideoSizes() []codec.VideoSize { + if receiver := d.getReceiver(); receiver != nil { + return receiver.VideoSizes() + } + + return nil +} + +func (d *DummyReceiver) Restart(reason string) { + if receiver := d.getReceiver(); receiver != nil { + receiver.Restart(reason) + } +} + +func (d *DummyReceiver) getReceiver() sfu.TrackReceiver { + if receiver, ok := d.receiver.Load().(sfu.TrackReceiver); ok { + return receiver + } + + return nil +} + // -------------------------------------------- type DummyRedReceiver struct { @@ -520,7 +543,7 @@ func (d *DummyRedReceiver) GetDownTracks() []sfu.TrackSender { if r, ok := d.redReceiver.Load().(sfu.TrackReceiver); ok { return r.GetDownTracks() } - return maps.Values(d.downTracks) + return slices.Collect(maps.Values(d.downTracks)) } func (d *DummyRedReceiver) ReadRTP(buf []byte, layer uint8, esn uint64) (int, error) { diff --git a/pkg/service/agent_dispatch_service.go b/pkg/service/agent_dispatch_service.go index b2e8bb4..b261c2d 100644 --- a/pkg/service/agent_dispatch_service.go +++ b/pkg/service/agent_dispatch_service.go @@ -16,11 +16,16 @@ package service import ( "context" + "fmt" "github.com/livekit/livekit-server/pkg/routing" + "github.com/livekit/protocol/agent" "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/logger" "github.com/livekit/protocol/rpc" + "github.com/livekit/protocol/utils" "github.com/livekit/protocol/utils/guid" + "github.com/livekit/psrpc" ) type AgentDispatchService struct { @@ -45,11 +50,16 @@ func NewAgentDispatchService( } func (ag *AgentDispatchService) CreateDispatch(ctx context.Context, req *livekit.CreateAgentDispatchRequest) (*livekit.AgentDispatch, error) { + AppendLogFields(ctx, "room", req.Room, "request", logger.Proto(redactCreateAgentDispatchRequest(req))) err := EnsureAdminPermission(ctx, livekit.RoomName(req.Room)) if err != nil { return nil, twirpAuthError(err) } + if err := agent.ValidateDeployment(req.GetDeployment()); err != nil { + return nil, psrpc.NewError(psrpc.InvalidArgument, err) + } + if ag.roomAllocator.AutoCreateEnabled(ctx) { err := ag.roomAllocator.SelectRoomNode(ctx, livekit.RoomName(req.Room), "") if err != nil { @@ -63,15 +73,18 @@ func (ag *AgentDispatchService) CreateDispatch(ctx context.Context, req *livekit } dispatch := &livekit.AgentDispatch{ - Id: guid.New(guid.AgentDispatchPrefix), - AgentName: req.AgentName, - Room: req.Room, - Metadata: req.Metadata, + Id: guid.New(guid.AgentDispatchPrefix), + AgentName: req.AgentName, + Room: req.Room, + Metadata: req.Metadata, + RestartPolicy: req.RestartPolicy, + Deployment: req.Deployment, } return ag.agentDispatchClient.CreateDispatch(ctx, ag.topicFormatter.RoomTopic(ctx, livekit.RoomName(req.Room)), dispatch) } func (ag *AgentDispatchService) DeleteDispatch(ctx context.Context, req *livekit.DeleteAgentDispatchRequest) (*livekit.AgentDispatch, error) { + AppendLogFields(ctx, "room", req.Room, "request", logger.Proto(req)) err := EnsureAdminPermission(ctx, livekit.RoomName(req.Room)) if err != nil { return nil, twirpAuthError(err) @@ -81,6 +94,7 @@ func (ag *AgentDispatchService) DeleteDispatch(ctx context.Context, req *livekit } func (ag *AgentDispatchService) ListDispatch(ctx context.Context, req *livekit.ListAgentDispatchRequest) (*livekit.ListAgentDispatchResponse, error) { + AppendLogFields(ctx, "room", req.Room, "request", logger.Proto(req)) err := EnsureAdminPermission(ctx, livekit.RoomName(req.Room)) if err != nil { return nil, twirpAuthError(err) @@ -88,3 +102,18 @@ func (ag *AgentDispatchService) ListDispatch(ctx context.Context, req *livekit.L return ag.agentDispatchClient.ListDispatch(ctx, ag.topicFormatter.RoomTopic(ctx, livekit.RoomName(req.Room)), req) } + +func redactCreateAgentDispatchRequest(req *livekit.CreateAgentDispatchRequest) *livekit.CreateAgentDispatchRequest { + if req.Metadata == "" { + return req + } + + clone := utils.CloneProto(req) + + // replace with size of metadata to provide visibility on request size + if clone.Metadata != "" { + clone.Metadata = fmt.Sprintf("__size: %d", len(clone.Metadata)) + } + + return clone +} diff --git a/pkg/service/agentservice.go b/pkg/service/agentservice.go index 0ee3604..8481a1e 100644 --- a/pkg/service/agentservice.go +++ b/pkg/service/agentservice.go @@ -17,7 +17,6 @@ package service import ( "context" "errors" - "fmt" "math/rand" "net/http" "slices" @@ -43,13 +42,19 @@ import ( "github.com/livekit/psrpc" ) -const agentWorkerLoadTarget = 0.65 - type AgentSocketUpgrader struct { websocket.Upgrader } -func (u AgentSocketUpgrader) Upgrade(w http.ResponseWriter, r *http.Request, responseHeader http.Header) (*websocket.Conn, agent.WorkerProtocolVersion, bool) { +func (u AgentSocketUpgrader) Upgrade( + w http.ResponseWriter, + r *http.Request, + responseHeader http.Header, +) ( + conn *websocket.Conn, + registration agent.WorkerRegistration, + ok bool, +) { if u.CheckOrigin == nil { // allow connections from any origin, since script may be hosted anywhere // security is enforced by access tokens @@ -61,29 +66,31 @@ func (u AgentSocketUpgrader) Upgrade(w http.ResponseWriter, r *http.Request, res // reject non websocket requests if !websocket.IsWebSocketUpgrade(r) { w.WriteHeader(404) - return nil, 0, false + return } // require a claim claims := GetGrants(r.Context()) if claims == nil || claims.Video == nil || !claims.Video.Agent { - handleError(w, r, http.StatusUnauthorized, rtc.ErrPermissionDenied) - return nil, 0, false + HandleError(w, r, http.StatusUnauthorized, rtc.ErrPermissionDenied) + return } + registration = agent.MakeWorkerRegistration() + registration.ClientIP = GetClientIP(r) + // upgrade conn, err := u.Upgrader.Upgrade(w, r, responseHeader) if err != nil { - handleError(w, r, http.StatusInternalServerError, err) - return nil, 0, false + HandleError(w, r, http.StatusInternalServerError, err) + return } - var protocol agent.WorkerProtocolVersion = agent.CurrentProtocol if pv, err := strconv.Atoi(r.FormValue("protocol")); err == nil { - protocol = agent.WorkerProtocolVersion(pv) + registration.Protocol = agent.WorkerProtocolVersion(pv) } - return conn, protocol, true + return conn, registration, true } func DispatchAgentWorkerSignal(c agent.SignalConn, h agent.WorkerSignalHandler, l logger.Logger) bool { @@ -105,8 +112,8 @@ func DispatchAgentWorkerSignal(c agent.SignalConn, h agent.WorkerSignalHandler, return true } -func HandshakeAgentWorker(c agent.SignalConn, serverInfo *livekit.ServerInfo, protocol agent.WorkerProtocolVersion, l logger.Logger) (r agent.WorkerRegistration, ok bool) { - wr := agent.NewWorkerRegisterer(c, serverInfo, protocol) +func HandshakeAgentWorker(c agent.SignalConn, serverInfo *livekit.ServerInfo, registration agent.WorkerRegistration, l logger.Logger) (r agent.WorkerRegistration, ok bool) { + wr := agent.NewWorkerRegisterer(c, serverInfo, registration) if err := c.SetReadDeadline(wr.Deadline()); err != nil { return } @@ -136,6 +143,7 @@ type AgentHandler struct { workers map[string]*agent.Worker jobToWorker map[livekit.JobID]*agent.Worker keyProvider auth.KeyProvider + targetLoad float32 namespaceWorkers map[workerKey][]*agent.Worker roomKeyCount int @@ -150,12 +158,14 @@ type AgentHandler struct { } type workerKey struct { - agentName string - namespace string - jobType livekit.JobType + agentName string + namespace string + jobType livekit.JobType + deployment string } -func NewAgentService(conf *config.Config, +func NewAgentService( + conf *config.Config, currentNode routing.LocalNode, bus psrpc.MessageBus, keyProvider auth.KeyProvider, @@ -180,6 +190,7 @@ func NewAgentService(conf *config.Config, keyProvider, logger.GetLogger(), serverInfo, + conf.Agents.TargetLoad, agent.RoomAgentTopic, agent.PublisherAgentTopic, agent.ParticipantAgentTopic, @@ -187,9 +198,9 @@ func NewAgentService(conf *config.Config, return s, nil } -func (s *AgentService) ServeHTTP(writer http.ResponseWriter, r *http.Request) { - if conn, protocol, ok := s.upgrader.Upgrade(writer, r, nil); ok { - s.HandleConnection(r.Context(), NewWSSignalConnection(conn), protocol) +func (s *AgentService) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if conn, registration, ok := s.upgrader.Upgrade(w, r, nil); ok { + s.HandleConnection(r.Context(), NewWSSignalConnection(conn), registration) conn.Close() } } @@ -199,6 +210,7 @@ func NewAgentHandler( keyProvider auth.KeyProvider, logger logger.Logger, serverInfo *livekit.ServerInfo, + targetLoad float32, roomTopic string, publisherTopic string, participantTopic string, @@ -211,14 +223,15 @@ func NewAgentHandler( namespaceWorkers: make(map[workerKey][]*agent.Worker), serverInfo: serverInfo, keyProvider: keyProvider, + targetLoad: targetLoad, roomTopic: roomTopic, publisherTopic: publisherTopic, participantTopic: participantTopic, } } -func (h *AgentHandler) HandleConnection(ctx context.Context, conn agent.SignalConn, protocol agent.WorkerProtocolVersion) { - registration, ok := HandshakeAgentWorker(conn, h.serverInfo, protocol, h.logger) +func (h *AgentHandler) HandleConnection(ctx context.Context, conn agent.SignalConn, registration agent.WorkerRegistration) { + registration, ok := HandshakeAgentWorker(conn, h.serverInfo, registration, h.logger) if !ok { return } @@ -243,13 +256,13 @@ func (h *AgentHandler) registerWorker(w *agent.Worker) { h.workers[w.ID] = w - key := workerKey{w.AgentName, w.Namespace, w.JobType} + key := workerKey{w.AgentName, w.Namespace, w.JobType, w.Deployment} workers := h.namespaceWorkers[key] created := len(workers) == 0 if created { - nameTopic := agent.GetAgentTopic(w.AgentName, w.Namespace) + nameTopic := agent.GetAgentTopic(w.AgentName, w.Namespace, w.Deployment) var typeTopic string switch w.JobType { case livekit.JobType_JT_ROOM: @@ -260,8 +273,6 @@ func (h *AgentHandler) registerWorker(w *agent.Worker) { typeTopic = h.participantTopic } - fmt.Println(">>> register worker", typeTopic) - err := h.agentServer.RegisterJobRequestTopic(nameTopic, typeTopic) if err != nil { h.mu.Unlock() @@ -310,7 +321,7 @@ func (h *AgentHandler) deregisterWorker(w *agent.Worker) { delete(h.workers, w.ID) - key := workerKey{w.AgentName, w.Namespace, w.JobType} + key := workerKey{w.AgentName, w.Namespace, w.JobType, w.Deployment} workers, ok := h.namespaceWorkers[key] if !ok { @@ -332,7 +343,7 @@ func (h *AgentHandler) deregisterWorker(w *agent.Worker) { ) delete(h.namespaceWorkers, key) - topic := agent.GetAgentTopic(w.AgentName, w.Namespace) + topic := agent.GetAgentTopic(w.AgentName, w.Namespace, w.Deployment) switch w.JobType { case livekit.JobType_JT_ROOM: @@ -383,7 +394,7 @@ func (h *AgentHandler) JobRequest(ctx context.Context, job *livekit.Job) (*rpc.J logger = logger.WithValues("participant", job.Participant.Identity) } - key := workerKey{job.AgentName, job.Namespace, job.Type} + key := workerKey{job.AgentName, job.Namespace, job.Type, job.Deployment} attempted := make(map[*agent.Worker]struct{}) for { selected, err := h.selectWorkerWeightedByLoad(key, attempted) @@ -395,28 +406,30 @@ func (h *AgentHandler) JobRequest(ctx context.Context, job *livekit.Job) (*rpc.J logger := logger.WithValues("workerID", selected.ID) attempted[selected] = struct{}{} - state, err := selected.AssignJob(ctx, job) - if err != nil { + state, err := selected.AssignJob(ctx, job, nil) + switch state.GetStatus() { + case livekit.JobStatus_JS_RUNNING: + logger.Infow("assigned job to worker", "apiKey", selected.APIKey()) + h.mu.Lock() + h.jobToWorker[livekit.JobID(job.Id)] = selected + h.mu.Unlock() + + err = h.agentServer.RegisterJobTerminateTopic(job.Id) + if err != nil { + logger.Errorw("failed to register JobTerminate handler", err) + } + fallthrough + case livekit.JobStatus_JS_SUCCESS: + return &rpc.JobRequestResponse{ + State: state, + }, nil + default: retry := utils.ErrorIsOneOf(err, agent.ErrWorkerNotAvailable, agent.ErrWorkerClosed) logger.Warnw("failed to assign job to worker", err, "retry", retry) - if retry { - continue // Try another worker + if !retry { + return nil, err } - return nil, err } - logger.Infow("assigned job to worker") - h.mu.Lock() - h.jobToWorker[livekit.JobID(job.Id)] = selected - h.mu.Unlock() - - err = h.agentServer.RegisterJobTerminateTopic(job.Id) - if err != nil { - logger.Errorw("failed to register JobTerminate handler", err) - } - - return &rpc.JobRequestResponse{ - State: state, - }, nil } } @@ -426,12 +439,12 @@ func (h *AgentHandler) JobRequestAffinity(ctx context.Context, job *livekit.Job) var affinity float32 for _, w := range h.workers { - if w.AgentName != job.AgentName || w.Namespace != job.Namespace || w.JobType != job.Type { + if w.AgentName != job.AgentName || w.Namespace != job.Namespace || w.JobType != job.Type || w.Deployment != job.Deployment { continue } if w.Status() == livekit.WorkerStatus_WS_AVAILABLE { - affinity += max(0, agentWorkerLoadTarget-w.Load()) + affinity += max(0, h.targetLoad-w.Load()) } } diff --git a/pkg/service/auth.go b/pkg/service/auth.go index 8edcf91..f8e6442 100644 --- a/pkg/service/auth.go +++ b/pkg/service/auth.go @@ -58,7 +58,7 @@ func NewAPIKeyAuthMiddleware(provider auth.KeyProvider) *APIKeyAuthMiddleware { } func (m *APIKeyAuthMiddleware) ServeHTTP(w http.ResponseWriter, r *http.Request, next http.HandlerFunc) { - if r.URL != nil && r.URL.Path == "/rtc/validate" { + if r.URL != nil && (r.URL.Path == "/rtc/validate" || r.URL.Path == "/rtc/v1/validate") { w.Header().Set("Access-Control-Allow-Origin", "*") } @@ -67,7 +67,7 @@ func (m *APIKeyAuthMiddleware) ServeHTTP(w http.ResponseWriter, r *http.Request, if authHeader != "" { if !strings.HasPrefix(authHeader, bearerPrefix) { - handleError(w, r, http.StatusUnauthorized, ErrMissingAuthorization) + HandleError(w, r, http.StatusUnauthorized, ErrMissingAuthorization) return } @@ -80,19 +80,19 @@ func (m *APIKeyAuthMiddleware) ServeHTTP(w http.ResponseWriter, r *http.Request, if authToken != "" { v, err := auth.ParseAPIToken(authToken) if err != nil { - handleError(w, r, http.StatusUnauthorized, ErrInvalidAuthorizationToken) + HandleError(w, r, http.StatusUnauthorized, ErrInvalidAuthorizationToken) return } secret := m.provider.GetSecret(v.APIKey()) if secret == "" { - handleError(w, r, http.StatusUnauthorized, errors.New("invalid API key: "+v.APIKey())) + HandleError(w, r, http.StatusUnauthorized, errors.New("invalid API key: "+v.APIKey())) return } - grants, err := v.Verify(secret) + _, grants, err := v.Verify(secret) if err != nil { - handleError(w, r, http.StatusUnauthorized, errors.New("invalid token: "+authToken+", error: "+err.Error())) + HandleError(w, r, http.StatusUnauthorized, errors.New("invalid token: "+authToken+", error: "+err.Error())) return } @@ -219,6 +219,19 @@ func EnsureSIPCallPermission(ctx context.Context) error { return nil } +func EnsureDestRoomPermission(ctx context.Context, source livekit.RoomName, destination livekit.RoomName) error { + claims := GetGrants(ctx) + if claims == nil || claims.Video == nil { + return ErrPermissionDenied + } + + if !claims.Video.RoomAdmin || source != livekit.RoomName(claims.Video.Room) || destination != livekit.RoomName(claims.Video.DestinationRoom) { + return ErrPermissionDenied + } + + return nil +} + // wraps authentication errors around Twirp func twirpAuthError(err error) error { return twirp.NewError(twirp.Unauthenticated, err.Error()) diff --git a/pkg/service/clients.go b/pkg/service/clients.go index 8a0eb39..d8fb337 100644 --- a/pkg/service/clients.go +++ b/pkg/service/clients.go @@ -19,12 +19,8 @@ import ( "google.golang.org/protobuf/types/known/emptypb" - "github.com/livekit/livekit-server/pkg/rtc" "github.com/livekit/protocol/livekit" - "github.com/livekit/protocol/logger" "github.com/livekit/protocol/rpc" - "github.com/livekit/protocol/utils" - "github.com/livekit/protocol/utils/guid" ) //counterfeiter:generate . IOClient @@ -35,41 +31,3 @@ type IOClient interface { CreateIngress(ctx context.Context, req *livekit.IngressInfo) (*emptypb.Empty, error) UpdateIngressState(ctx context.Context, req *rpc.UpdateIngressStateRequest) (*emptypb.Empty, error) } - -type egressLauncher struct { - client rpc.EgressClient - io IOClient -} - -func NewEgressLauncher(client rpc.EgressClient, io IOClient) rtc.EgressLauncher { - if client == nil { - return nil - } - return &egressLauncher{ - client: client, - io: io, - } -} - -func (s *egressLauncher) StartEgress(ctx context.Context, req *rpc.StartEgressRequest) (*livekit.EgressInfo, error) { - if s.client == nil { - return nil, ErrEgressNotConnected - } - - // Ensure we have an Egress ID - if req.EgressId == "" { - req.EgressId = guid.New(utils.EgressPrefix) - } - - info, err := s.client.StartEgress(ctx, "", req) - if err != nil { - return nil, err - } - - _, err = s.io.CreateEgress(ctx, info) - if err != nil { - logger.Errorw("failed to create egress", err) - } - - return info, nil -} diff --git a/pkg/service/docker_test.go b/pkg/service/docker_test.go index a3da604..9fc5b90 100644 --- a/pkg/service/docker_test.go +++ b/pkg/service/docker_test.go @@ -15,27 +15,31 @@ package service_test import ( + "context" "fmt" "log" "net" "os" "testing" + "time" "go.uber.org/atomic" - "github.com/ory/dockertest/v3" + mobyclient "github.com/moby/moby/client" + "github.com/ory/dockertest/v4" ) -var Docker *dockertest.Pool +var Docker dockertest.ClosablePool func TestMain(m *testing.M) { - pool, err := dockertest.NewPool("") + ctx := context.Background() + pool, err := dockertest.NewPool(ctx, "") if err != nil { log.Fatalf("Could not construct pool: %s", err) } // uses pool to try to connect to Docker - err = pool.Client.Ping() + _, err = pool.Client().Ping(ctx, mobyclient.PingOptions{}) if err != nil { log.Fatalf("Could not connect to Docker: %s", err) } @@ -46,7 +50,7 @@ func TestMain(m *testing.M) { } func waitTCPPort(t testing.TB, addr string) { - if err := Docker.Retry(func() error { + if err := Docker.Retry(t.Context(), 30*time.Second, func() error { conn, err := net.Dial("tcp", addr) if err != nil { t.Log(err) @@ -62,15 +66,18 @@ func waitTCPPort(t testing.TB, addr string) { var redisLast atomic.Uint32 func runRedis(t testing.TB) string { - c, err := Docker.RunWithOptions(&dockertest.RunOptions{ - Name: fmt.Sprintf("lktest-redis-%d", redisLast.Inc()), - Repository: "redis", Tag: "latest", - }) + c, err := Docker.Run(t.Context(), + "redis", + dockertest.WithName(fmt.Sprintf("lktest-redis-%d", redisLast.Inc())), + dockertest.WithTag("latest"), + ) if err != nil { t.Fatal(err) } t.Cleanup(func() { - _ = Docker.Purge(c) + // t.Context() is canceled before cleanup funcs run, so use a + // non-canceled context to let the container stop/remove complete. + _ = c.Close(context.Background()) }) addr := c.GetHostPort("6379/tcp") waitTCPPort(t, addr) diff --git a/pkg/service/egress.go b/pkg/service/egress.go index a745caa..2e37ce4 100644 --- a/pkg/service/egress.go +++ b/pkg/service/egress.go @@ -17,15 +17,21 @@ package service import ( "context" "encoding/json" + "errors" "fmt" "reflect" "github.com/twitchtv/twirp" - "github.com/livekit/livekit-server/pkg/rtc" "github.com/livekit/protocol/egress" "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/logger" "github.com/livekit/protocol/rpc" + "github.com/livekit/protocol/utils" + "github.com/livekit/protocol/utils/guid" + "github.com/livekit/psrpc" + + "github.com/livekit/livekit-server/pkg/rtc" ) type EgressService struct { @@ -33,27 +39,41 @@ type EgressService struct { client rpc.EgressClient io IOClient roomService livekit.RoomService - store ServiceStore +} + +type egressLauncher struct { + client rpc.EgressClient + io IOClient + store ServiceStore } func NewEgressService( client rpc.EgressClient, launcher rtc.EgressLauncher, - store ServiceStore, io IOClient, rs livekit.RoomService, ) *EgressService { return &EgressService{ client: client, - store: store, io: io, roomService: rs, launcher: launcher, } } +func NewEgressLauncher(client rpc.EgressClient, io IOClient, store ServiceStore) rtc.EgressLauncher { + if client == nil { + return nil + } + return &egressLauncher{ + client: client, + io: io, + store: store, + } +} + func (s *EgressService) StartRoomCompositeEgress(ctx context.Context, req *livekit.RoomCompositeEgressRequest) (*livekit.EgressInfo, error) { - fields := []interface{}{ + fields := []any{ "room", req.RoomName, "baseUrl", req.CustomBaseUrl, "outputType", egress.GetOutputType(req), @@ -61,7 +81,10 @@ func (s *EgressService) StartRoomCompositeEgress(ctx context.Context, req *livek defer func() { AppendLogFields(ctx, fields...) }() - ei, err := s.startEgress(ctx, livekit.RoomName(req.RoomName), &rpc.StartEgressRequest{ + + egressID, idFromCtx := EgressID(ctx) + ei, err := s.startEgress(ctx, &rpc.StartEgressRequest{ + EgressId: egressID, Request: &rpc.StartEgressRequest_RoomComposite{ RoomComposite: req, }, @@ -69,19 +92,23 @@ func (s *EgressService) StartRoomCompositeEgress(ctx context.Context, req *livek if err != nil { return nil, err } - fields = append(fields, "egressID", ei.EgressId) + if !idFromCtx { + fields = append(fields, "egressID", ei.EgressId) + } return ei, err } func (s *EgressService) StartWebEgress(ctx context.Context, req *livekit.WebEgressRequest) (*livekit.EgressInfo, error) { - fields := []interface{}{ + fields := []any{ "url", req.Url, "outputType", egress.GetOutputType(req), } defer func() { AppendLogFields(ctx, fields...) }() - ei, err := s.startEgress(ctx, "", &rpc.StartEgressRequest{ + egressID, idFromCtx := EgressID(ctx) + ei, err := s.startEgress(ctx, &rpc.StartEgressRequest{ + EgressId: egressID, Request: &rpc.StartEgressRequest_Web{ Web: req, }, @@ -89,12 +116,14 @@ func (s *EgressService) StartWebEgress(ctx context.Context, req *livekit.WebEgre if err != nil { return nil, err } - fields = append(fields, "egressID", ei.EgressId) + if !idFromCtx { + fields = append(fields, "egressID", ei.EgressId) + } return ei, err } func (s *EgressService) StartParticipantEgress(ctx context.Context, req *livekit.ParticipantEgressRequest) (*livekit.EgressInfo, error) { - fields := []interface{}{ + fields := []any{ "room", req.RoomName, "identity", req.Identity, "outputType", egress.GetOutputType(req), @@ -102,7 +131,9 @@ func (s *EgressService) StartParticipantEgress(ctx context.Context, req *livekit defer func() { AppendLogFields(ctx, fields...) }() - ei, err := s.startEgress(ctx, livekit.RoomName(req.RoomName), &rpc.StartEgressRequest{ + egressID, idFromCtx := EgressID(ctx) + ei, err := s.startEgress(ctx, &rpc.StartEgressRequest{ + EgressId: egressID, Request: &rpc.StartEgressRequest_Participant{ Participant: req, }, @@ -110,12 +141,14 @@ func (s *EgressService) StartParticipantEgress(ctx context.Context, req *livekit if err != nil { return nil, err } - fields = append(fields, "egressID", ei.EgressId) + if !idFromCtx { + fields = append(fields, "egressID", ei.EgressId) + } return ei, err } func (s *EgressService) StartTrackCompositeEgress(ctx context.Context, req *livekit.TrackCompositeEgressRequest) (*livekit.EgressInfo, error) { - fields := []interface{}{ + fields := []any{ "room", req.RoomName, "audioTrackID", req.AudioTrackId, "videoTrackID", req.VideoTrackId, @@ -124,7 +157,9 @@ func (s *EgressService) StartTrackCompositeEgress(ctx context.Context, req *live defer func() { AppendLogFields(ctx, fields...) }() - ei, err := s.startEgress(ctx, livekit.RoomName(req.RoomName), &rpc.StartEgressRequest{ + egressID, idFromCtx := EgressID(ctx) + ei, err := s.startEgress(ctx, &rpc.StartEgressRequest{ + EgressId: egressID, Request: &rpc.StartEgressRequest_TrackComposite{ TrackComposite: req, }, @@ -132,19 +167,23 @@ func (s *EgressService) StartTrackCompositeEgress(ctx context.Context, req *live if err != nil { return nil, err } - fields = append(fields, "egressID", ei.EgressId) + if !idFromCtx { + fields = append(fields, "egressID", ei.EgressId) + } return ei, err } func (s *EgressService) StartTrackEgress(ctx context.Context, req *livekit.TrackEgressRequest) (*livekit.EgressInfo, error) { - fields := []interface{}{"room", req.RoomName, "trackID", req.TrackId} + fields := []any{"room", req.RoomName, "trackID", req.TrackId} if t := reflect.TypeOf(req.Output); t != nil { fields = append(fields, "outputType", t.String()) } defer func() { AppendLogFields(ctx, fields...) }() - ei, err := s.startEgress(ctx, livekit.RoomName(req.RoomName), &rpc.StartEgressRequest{ + egressID, idFromCtx := EgressID(ctx) + ei, err := s.startEgress(ctx, &rpc.StartEgressRequest{ + EgressId: egressID, Request: &rpc.StartEgressRequest_Track{ Track: req, }, @@ -152,26 +191,76 @@ func (s *EgressService) StartTrackEgress(ctx context.Context, req *livekit.Track if err != nil { return nil, err } - fields = append(fields, "egressID", ei.EgressId) + if !idFromCtx { + fields = append(fields, "egressID", ei.EgressId) + } return ei, err } -func (s *EgressService) startEgress(ctx context.Context, roomName livekit.RoomName, req *rpc.StartEgressRequest) (*livekit.EgressInfo, error) { +func (s *EgressService) startEgress(ctx context.Context, req *rpc.StartEgressRequest) (*livekit.EgressInfo, error) { if err := EnsureRecordPermission(ctx); err != nil { return nil, twirpAuthError(err) } else if s.launcher == nil { return nil, ErrEgressNotConnected } - if roomName != "" { - room, _, err := s.store.LoadRoom(ctx, roomName, false) - if err != nil { - return nil, err - } - req.RoomId = room.Sid - } + return s.launcher.StartEgress(ctx, req) } +func (s *egressLauncher) StartEgress(ctx context.Context, req *rpc.StartEgressRequest) (*livekit.EgressInfo, error) { + if s.client == nil { + return nil, ErrEgressNotConnected + } + + // Ensure we have an Egress ID + if req.EgressId == "" { + req.EgressId = guid.New(utils.EgressPrefix) + } + + if req.RoomId == "" { + var roomName string + switch v := req.Request.(type) { + case *rpc.StartEgressRequest_RoomComposite: + roomName = v.RoomComposite.RoomName + case *rpc.StartEgressRequest_Web: + // no room name + case *rpc.StartEgressRequest_Participant: + roomName = v.Participant.RoomName + case *rpc.StartEgressRequest_TrackComposite: + roomName = v.TrackComposite.RoomName + case *rpc.StartEgressRequest_Track: + roomName = v.Track.RoomName + } + + if roomName != "" { + room, _, err := s.store.LoadRoom(ctx, livekit.RoomName(roomName), false) + if err != nil { + return nil, err + } + req.RoomId = room.Sid + } + } + + info, err := s.client.StartEgress(ctx, "", req) + if err != nil { + return nil, err + } + + _, err = s.io.CreateEgress(ctx, info) + if err != nil { + logger.Errorw("failed to create egress", err) + } + + return info, nil +} + +func (s *egressLauncher) StopEgress(ctx context.Context, req *livekit.StopEgressRequest) (*livekit.EgressInfo, error) { + if s.client == nil { + return nil, ErrEgressNotConnected + } + return s.client.StopEgress(ctx, req.EgressId, req) +} + type LayoutMetadata struct { Layout string `json:"layout"` } @@ -249,17 +338,20 @@ func (s *EgressService) ListEgress(ctx context.Context, req *livekit.ListEgressR return s.io.ListEgress(ctx, req) } -func (s *EgressService) StopEgress(ctx context.Context, req *livekit.StopEgressRequest) (*livekit.EgressInfo, error) { +func (s *EgressService) StopEgress(ctx context.Context, req *livekit.StopEgressRequest) (info *livekit.EgressInfo, err error) { + defer func() { + if errors.Is(err, psrpc.ErrNoResponse) { + // Do not map cases where the context times out to 503 + err = psrpc.ErrRequestTimedOut + } + }() + AppendLogFields(ctx, "egressID", req.EgressId) if err := EnsureRecordPermission(ctx); err != nil { return nil, twirpAuthError(err) } - if s.client == nil { - return nil, ErrEgressNotConnected - } - - info, err := s.client.StopEgress(ctx, req.EgressId, req) + info, err = s.launcher.StopEgress(ctx, req) if err != nil { var loadErr error info, loadErr = s.io.GetEgress(ctx, &rpc.GetEgressRequest{EgressId: req.EgressId}) @@ -279,3 +371,7 @@ func (s *EgressService) StopEgress(ctx context.Context, req *livekit.StopEgressR return info, nil } + +func (s *EgressService) StartEgress(ctx context.Context, req *livekit.StartEgressRequest) (*livekit.EgressInfo, error) { + return nil, errors.New("not implemented") +} diff --git a/pkg/service/egressid.go b/pkg/service/egressid.go new file mode 100644 index 0000000..a06d8c5 --- /dev/null +++ b/pkg/service/egressid.go @@ -0,0 +1,44 @@ +package service + +import ( + "context" + "strings" + + "github.com/twitchtv/twirp" + + "github.com/livekit/protocol/utils/guid" +) + +type egressIDKey struct{} + +func TwirpEgressID() *twirp.ServerHooks { + return &twirp.ServerHooks{ + RequestRouted: func(ctx context.Context) (context.Context, error) { + // generate egressID for start egress methods for tracing egress failure before it reaches egress service. + if isStartEgressMethod(ctx) { + egressID := guid.New(guid.EgressPrefix) + ctx = WithEgressID(ctx, egressID) + AppendLogFields(ctx, "egressID", egressID) + } + return ctx, nil + }, + } +} + +func WithEgressID(ctx context.Context, egressID string) context.Context { + return context.WithValue(ctx, egressIDKey{}, egressID) +} + +func EgressID(ctx context.Context) (string, bool) { + if egressID, ok := ctx.Value(egressIDKey{}).(string); ok && egressID != "" { + return egressID, true + } + return "", false +} + +func isStartEgressMethod(ctx context.Context) bool { + if methodName, ok := twirp.MethodName(ctx); ok && strings.HasPrefix(methodName, "Start") && strings.HasSuffix(methodName, "Egress") { + return true + } + return false +} diff --git a/pkg/service/errors.go b/pkg/service/errors.go index d208e43..7d07390 100644 --- a/pkg/service/errors.go +++ b/pkg/service/errors.go @@ -22,14 +22,17 @@ var ( ErrEgressNotFound = psrpc.NewErrorf(psrpc.NotFound, "egress does not exist") ErrEgressNotConnected = psrpc.NewErrorf(psrpc.Internal, "egress not connected (redis required)") ErrIdentityEmpty = psrpc.NewErrorf(psrpc.InvalidArgument, "identity cannot be empty") + ErrParticipantSidEmpty = psrpc.NewErrorf(psrpc.InvalidArgument, "participant sid cannot be empty") ErrIngressNotConnected = psrpc.NewErrorf(psrpc.Internal, "ingress not connected (redis required)") ErrIngressNotFound = psrpc.NewErrorf(psrpc.NotFound, "ingress does not exist") ErrIngressNonReusable = psrpc.NewErrorf(psrpc.InvalidArgument, "ingress is not reusable and cannot be modified") ErrNameExceedsLimits = psrpc.NewErrorf(psrpc.InvalidArgument, "name length exceeds limits") ErrMetadataExceedsLimits = psrpc.NewErrorf(psrpc.InvalidArgument, "metadata size exceeds limits") ErrAttributeExceedsLimits = psrpc.NewErrorf(psrpc.InvalidArgument, "attribute size exceeds limits") + ErrNoRoomName = psrpc.NewErrorf(psrpc.InvalidArgument, "no room name") ErrRoomNameExceedsLimits = psrpc.NewErrorf(psrpc.InvalidArgument, "room name length exceeds limits") ErrParticipantIdentityExceedsLimits = psrpc.NewErrorf(psrpc.InvalidArgument, "participant identity length exceeds limits") + ErrDestinationSameAsSourceRoom = psrpc.NewErrorf(psrpc.InvalidArgument, "destination room cannot be the same as source room") ErrOperationFailed = psrpc.NewErrorf(psrpc.Internal, "operation cannot be completed") ErrParticipantNotFound = psrpc.NewErrorf(psrpc.NotFound, "participant does not exist") ErrRoomNotFound = psrpc.NewErrorf(psrpc.NotFound, "requested room does not exist") @@ -42,4 +45,8 @@ var ( ErrSIPTrunkNotFound = psrpc.NewErrorf(psrpc.NotFound, "requested sip trunk does not exist") ErrSIPDispatchRuleNotFound = psrpc.NewErrorf(psrpc.NotFound, "requested sip dispatch rule does not exist") ErrSIPParticipantNotFound = psrpc.NewErrorf(psrpc.NotFound, "requested sip participant does not exist") + ErrInvalidMessageType = psrpc.NewErrorf(psrpc.Internal, "invalid message type") + ErrNoConnectRequest = psrpc.NewErrorf(psrpc.InvalidArgument, "no connect request") + ErrNoConnectResponse = psrpc.NewErrorf(psrpc.InvalidArgument, "no connect response") + ErrDestinationIdentityRequired = psrpc.NewErrorf(psrpc.InvalidArgument, "destination identity is required") ) diff --git a/pkg/service/ingress.go b/pkg/service/ingress.go index 40f2f51..2b82453 100644 --- a/pkg/service/ingress.go +++ b/pkg/service/ingress.go @@ -85,7 +85,7 @@ func NewIngressService( } func (s *IngressService) CreateIngress(ctx context.Context, req *livekit.CreateIngressRequest) (*livekit.IngressInfo, error) { - fields := []interface{}{ + fields := []any{ "inputType", req.InputType, "name", req.Name, } @@ -190,20 +190,19 @@ func (s *IngressService) CreateIngressWithUrl(ctx context.Context, urlStr string if err != nil { return info, err } - } - - // TODO Remove this store Ingress call for URL pull as it is redundant since - // the ingress service sends a CreateIngress RPC - _, err = s.io.CreateIngress(ctx, info) - switch err { - case nil: - break - case ingress.ErrIngressOutOfDate: - // Error returned if the ingress was already created by the ingress service - err = nil - default: - logger.Errorw("could not create ingress object", err) - return nil, err + // The Ingress instance will create the ingress object when handling the URL pull ingress + } else { + _, err = s.io.CreateIngress(ctx, info) + switch err { + case nil: + break + case ingress.ErrIngressOutOfDate: + // Error returned if the ingress was already created by the ingress service + err = nil + default: + logger.Errorw("could not create ingress object", err) + return nil, err + } } return info, nil @@ -218,7 +217,7 @@ func (s *IngressService) LaunchPullIngress(ctx context.Context, info *livekit.In } func updateEnableTranscoding(info *livekit.IngressInfo) { - // Set BypassTranscoding as well for backward compatiblity + // Set BypassTranscoding as well for backward compatibility if info.EnableTranscoding != nil { info.BypassTranscoding = !*info.EnableTranscoding return @@ -276,7 +275,7 @@ func updateInfoUsingRequest(req *livekit.UpdateIngressRequest, info *livekit.Ing } func (s *IngressService) UpdateIngress(ctx context.Context, req *livekit.UpdateIngressRequest) (*livekit.IngressInfo, error) { - fields := []interface{}{ + fields := []any{ "ingress", req.IngressId, "name", req.Name, } diff --git a/pkg/service/interfaces.go b/pkg/service/interfaces.go index e2a6978..22a8fdd 100644 --- a/pkg/service/interfaces.go +++ b/pkg/service/interfaces.go @@ -28,6 +28,7 @@ import ( //counterfeiter:generate . ObjectStore type ObjectStore interface { ServiceStore + OSSServiceStore // enable locking on a specific room to prevent race // returns a (lock uuid, error) @@ -43,11 +44,16 @@ type ObjectStore interface { //counterfeiter:generate . ServiceStore type ServiceStore interface { LoadRoom(ctx context.Context, roomName livekit.RoomName, includeInternal bool) (*livekit.Room, *livekit.RoomInternal, error) - DeleteRoom(ctx context.Context, roomName livekit.RoomName) error + RoomExists(ctx context.Context, roomName livekit.RoomName) (bool, error) // ListRooms returns currently active rooms. if names is not nil, it'll filter and return // only rooms that match ListRooms(ctx context.Context, roomNames []livekit.RoomName) ([]*livekit.Room, error) +} + +type OSSServiceStore interface { + DeleteRoom(ctx context.Context, roomName livekit.RoomName) error + HasParticipant(context.Context, livekit.RoomName, livekit.ParticipantIdentity) (bool, error) LoadParticipant(ctx context.Context, roomName livekit.RoomName, identity livekit.ParticipantIdentity) (*livekit.ParticipantInfo, error) ListParticipants(ctx context.Context, roomName livekit.RoomName) ([]*livekit.ParticipantInfo, error) } diff --git a/pkg/service/ioservice.go b/pkg/service/ioservice.go index e0f4213..32d86d9 100644 --- a/pkg/service/ioservice.go +++ b/pkg/service/ioservice.go @@ -23,6 +23,7 @@ import ( "github.com/livekit/protocol/logger" "github.com/livekit/protocol/rpc" "github.com/livekit/psrpc" + "github.com/livekit/psrpc/pkg/middleware/otelpsrpc" "github.com/livekit/livekit-server/pkg/telemetry" ) @@ -54,7 +55,9 @@ func NewIOInfoService( } if bus != nil { - ioServer, err := rpc.NewIOInfoServer(s, bus) + ioServer, err := rpc.NewIOInfoServer(s, bus, + otelpsrpc.ServerOptions(otelpsrpc.Config{}), + ) if err != nil { return nil, err } @@ -86,6 +89,10 @@ func (s *IOInfoService) Stop() { } func (s *IOInfoService) CreateEgress(ctx context.Context, info *livekit.EgressInfo) (*emptypb.Empty, error) { + if s.es == nil { + return nil, ErrEgressNotConnected + } + // check if egress already exists to avoid duplicate EgressStarted event if _, err := s.es.LoadEgress(ctx, info.EgressId); err == nil { return &emptypb.Empty{}, nil @@ -103,6 +110,10 @@ func (s *IOInfoService) CreateEgress(ctx context.Context, info *livekit.EgressIn } func (s *IOInfoService) UpdateEgress(ctx context.Context, info *livekit.EgressInfo) (*emptypb.Empty, error) { + if s.es == nil { + return nil, ErrEgressNotConnected + } + err := s.es.UpdateEgress(ctx, info) switch info.Status { @@ -126,6 +137,10 @@ func (s *IOInfoService) UpdateEgress(ctx context.Context, info *livekit.EgressIn } func (s *IOInfoService) GetEgress(ctx context.Context, req *rpc.GetEgressRequest) (*livekit.EgressInfo, error) { + if s.es == nil { + return nil, ErrEgressNotConnected + } + info, err := s.es.LoadEgress(ctx, req.EgressId) if err != nil { logger.Errorw("failed to load egress", err) @@ -136,6 +151,10 @@ func (s *IOInfoService) GetEgress(ctx context.Context, req *rpc.GetEgressRequest } func (s *IOInfoService) ListEgress(ctx context.Context, req *livekit.ListEgressRequest) (*livekit.ListEgressResponse, error) { + if s.es == nil { + return nil, ErrEgressNotConnected + } + if req.EgressId != "" { info, err := s.es.LoadEgress(ctx, req.EgressId) if err != nil { @@ -168,3 +187,8 @@ func (s *IOInfoService) UpdateSIPCallState(ctx context.Context, req *rpc.UpdateS // TODO: placeholder return &emptypb.Empty{}, nil } + +func (s *IOInfoService) RecordCallContext(context.Context, *rpc.RecordCallContextRequest) (*emptypb.Empty, error) { + // TODO: placeholder + return &emptypb.Empty{}, nil +} diff --git a/pkg/service/ioservice_ingress.go b/pkg/service/ioservice_ingress.go index 9e84e7e..b802c88 100644 --- a/pkg/service/ioservice_ingress.go +++ b/pkg/service/ioservice_ingress.go @@ -25,6 +25,10 @@ import ( ) func (s *IOInfoService) CreateIngress(ctx context.Context, info *livekit.IngressInfo) (*emptypb.Empty, error) { + if s.is == nil { + return nil, ErrIngressNotConnected + } + err := s.is.StoreIngress(ctx, info) if err != nil { return nil, err @@ -45,6 +49,10 @@ func (s *IOInfoService) GetIngressInfo(ctx context.Context, req *rpc.GetIngressI } func (s *IOInfoService) loadIngressFromInfoRequest(req *rpc.GetIngressInfoRequest) (info *livekit.IngressInfo, err error) { + if s.is == nil { + return nil, ErrIngressNotConnected + } + if req.IngressId != "" { info, err = s.is.LoadIngress(context.Background(), req.IngressId) } else if req.StreamKey != "" { @@ -56,6 +64,10 @@ func (s *IOInfoService) loadIngressFromInfoRequest(req *rpc.GetIngressInfoReques } func (s *IOInfoService) UpdateIngressState(ctx context.Context, req *rpc.UpdateIngressStateRequest) (*emptypb.Empty, error) { + if s.is == nil { + return nil, ErrIngressNotConnected + } + info, err := s.is.LoadIngress(ctx, req.IngressId) if err != nil { return nil, err diff --git a/pkg/service/ioservice_sip.go b/pkg/service/ioservice_sip.go index cadb69a..386baef 100644 --- a/pkg/service/ioservice_sip.go +++ b/pkg/service/ioservice_sip.go @@ -30,21 +30,21 @@ import ( // matchSIPTrunk finds a SIP Trunk definition matching the request. // Returns nil if no rules matched or an error if there are conflicting definitions. -func (s *IOInfoService) matchSIPTrunk(ctx context.Context, trunkID, calling, called string, srcIP netip.Addr) (*livekit.SIPInboundTrunkInfo, error) { +func (s *IOInfoService) matchSIPTrunk(ctx context.Context, trunkID string, call *rpc.SIPCall) (*livekit.SIPInboundTrunkInfo, error) { if s.ss == nil { return nil, ErrSIPNotConnected } if trunkID != "" { // This is a best-effort optimization. Fallthrough to listing trunks if it doesn't work. if tr, err := s.ss.LoadSIPInboundTrunk(ctx, trunkID); err == nil { - tr, err = sip.MatchTrunkIter(iters.Slice([]*livekit.SIPInboundTrunkInfo{tr}), srcIP, calling, called) + tr, err = sip.MatchTrunkIter(iters.Slice([]*livekit.SIPInboundTrunkInfo{tr}), call) if err == nil { return tr, nil } } } - it := s.SelectSIPInboundTrunk(ctx, called) - return sip.MatchTrunkIter(it, srcIP, calling, called) + it := s.SelectSIPInboundTrunk(ctx, call.To.User) + return sip.MatchTrunkIter(it, call) } func (s *IOInfoService) SelectSIPInboundTrunk(ctx context.Context, called string) iters.Iter[*livekit.SIPInboundTrunkInfo] { @@ -82,18 +82,19 @@ func (s *IOInfoService) SelectSIPDispatchRule(ctx context.Context, trunkID strin } func (s *IOInfoService) EvaluateSIPDispatchRules(ctx context.Context, req *rpc.EvaluateSIPDispatchRulesRequest) (*rpc.EvaluateSIPDispatchRulesResponse, error) { + call := req.SIPCall() log := logger.GetLogger() - log = log.WithValues("toUser", req.CalledNumber, "fromUser", req.CallingNumber, "src", req.SrcAddress) - if req.SrcAddress == "" { + log = log.WithValues("toUser", call.To.User, "fromUser", call.From.User, "src", call.SourceIp) + if call.SourceIp == "" { log.Warnw("source address is not set", nil) // TODO: return error in the next release } - srcIP, err := netip.ParseAddr(req.SrcAddress) - if req.SrcAddress != "" && err != nil { + _, err := netip.ParseAddr(call.SourceIp) + if call.SourceIp != "" && err != nil { log.Errorw("cannot parse source IP", err) return nil, twirp.WrapError(twirp.NewError(twirp.InvalidArgument, err.Error()), err) } - trunk, err := s.matchSIPTrunk(ctx, req.SipTrunkId, req.CallingNumber, req.CalledNumber, srcIP) + trunk, err := s.matchSIPTrunk(ctx, req.SipTrunkId, call) if err != nil { return nil, err } @@ -122,23 +123,25 @@ func (s *IOInfoService) EvaluateSIPDispatchRules(ctx context.Context, req *rpc.E if err != nil { return nil, err } + resp.Upgrade() resp.SipTrunkId = trunkID return resp, err } func (s *IOInfoService) GetSIPTrunkAuthentication(ctx context.Context, req *rpc.GetSIPTrunkAuthenticationRequest) (*rpc.GetSIPTrunkAuthenticationResponse, error) { + call := req.SIPCall() log := logger.GetLogger() - log = log.WithValues("toUser", req.To, "fromUser", req.From, "src", req.SrcAddress) - if req.SrcAddress == "" { + log = log.WithValues("toUser", call.To.User, "fromUser", call.From.User, "src", call.SourceIp) + if call.SourceIp == "" { log.Warnw("source address is not set", nil) // TODO: return error in the next release } - srcIP, err := netip.ParseAddr(req.SrcAddress) - if req.SrcAddress != "" && err != nil { + _, err := netip.ParseAddr(call.SourceIp) + if call.SourceIp != "" && err != nil { log.Errorw("cannot parse source IP", err) return nil, twirp.WrapError(twirp.NewError(twirp.InvalidArgument, err.Error()), err) } - trunk, err := s.matchSIPTrunk(ctx, "", req.From, req.To, srcIP) + trunk, err := s.matchSIPTrunk(ctx, "", call) if err != nil { return nil, err } @@ -147,9 +150,5 @@ func (s *IOInfoService) GetSIPTrunkAuthentication(ctx context.Context, req *rpc. return &rpc.GetSIPTrunkAuthenticationResponse{}, nil } log.Debugw("SIP trunk matched for auth", "sipTrunk", trunk.SipTrunkId) - return &rpc.GetSIPTrunkAuthenticationResponse{ - SipTrunkId: trunk.SipTrunkId, - Username: trunk.AuthUsername, - Password: trunk.AuthPassword, - }, nil + return sip.InboundTrunkAuthPrompt(trunk) } diff --git a/pkg/service/localstore.go b/pkg/service/localstore.go index 005386c..1cf747f 100644 --- a/pkg/service/localstore.go +++ b/pkg/service/localstore.go @@ -25,6 +25,8 @@ import ( "github.com/livekit/protocol/utils" ) +var _ OSSServiceStore = (*LocalStore)(nil) + // encapsulates CRUD operations for room settings type LocalStore struct { // map of roomName => room @@ -84,6 +86,16 @@ func (s *LocalStore) LoadRoom(_ context.Context, roomName livekit.RoomName, incl return room, internal, nil } +func (s *LocalStore) RoomExists(ctx context.Context, roomName livekit.RoomName) (bool, error) { + _, _, err := s.LoadRoom(ctx, roomName, false) + if err == ErrRoomNotFound { + return false, nil + } else if err != nil { + return false, err + } + return true, nil +} + func (s *LocalStore) ListRooms(_ context.Context, roomNames []livekit.RoomName) ([]*livekit.Room, error) { s.lock.RLock() defer s.lock.RUnlock() @@ -153,6 +165,11 @@ func (s *LocalStore) LoadParticipant(_ context.Context, roomName livekit.RoomNam return participant, nil } +func (s *LocalStore) HasParticipant(ctx context.Context, roomName livekit.RoomName, identity livekit.ParticipantIdentity) (bool, error) { + p, err := s.LoadParticipant(ctx, roomName, identity) + return p != nil, utils.ScreenError(err, ErrParticipantNotFound) +} + func (s *LocalStore) ListParticipants(_ context.Context, roomName livekit.RoomName) ([]*livekit.ParticipantInfo, error) { s.lock.RLock() defer s.lock.RUnlock() @@ -224,10 +241,8 @@ func (s *LocalStore) ListAgentDispatches(ctx context.Context, roomName livekit.R agentJobs := s.agentJobs[roomName] var js []*livekit.Job - if agentJobs != nil { - for _, j := range agentJobs { - js = append(js, utils.CloneProto(j)) - } + for _, j := range agentJobs { + js = append(js, utils.CloneProto(j)) } var ds []*livekit.AgentDispatch diff --git a/pkg/service/redisstore.go b/pkg/service/redisstore.go index 45576d3..292ed95 100644 --- a/pkg/service/redisstore.go +++ b/pkg/service/redisstore.go @@ -68,6 +68,8 @@ const ( maxRetries = 5 ) +var _ OSSServiceStore = (*RedisStore)(nil) + type RedisStore struct { rc redis.UniversalClient unlockScript *redis.Script @@ -195,6 +197,16 @@ func (s *RedisStore) LoadRoom(_ context.Context, roomName livekit.RoomName, incl return room, internal, nil } +func (s *RedisStore) RoomExists(ctx context.Context, roomName livekit.RoomName) (bool, error) { + _, _, err := s.LoadRoom(ctx, roomName, false) + if err == ErrRoomNotFound { + return false, nil + } else if err != nil { + return false, err + } + return true, nil +} + func (s *RedisStore) ListRooms(_ context.Context, roomNames []livekit.RoomName) ([]*livekit.Room, error) { var items []string var err error @@ -205,7 +217,7 @@ func (s *RedisStore) ListRooms(_ context.Context, roomNames []livekit.RoomName) } } else { names := livekit.IDsAsStrings(roomNames) - var results []interface{} + var results []any results, err = s.rc.HMGet(s.ctx, RoomsKey, names...).Result() if err != nil && err != redis.Nil { return nil, errors.Wrap(err, "could not get rooms by names") @@ -314,6 +326,11 @@ func (s *RedisStore) LoadParticipant(_ context.Context, roomName livekit.RoomNam return &pi, nil } +func (s *RedisStore) HasParticipant(ctx context.Context, roomName livekit.RoomName, identity livekit.ParticipantIdentity) (bool, error) { + p, err := s.LoadParticipant(ctx, roomName, identity) + return p != nil, utils.ScreenError(err, ErrParticipantNotFound) +} + func (s *RedisStore) ListParticipants(_ context.Context, roomName livekit.RoomName) ([]*livekit.ParticipantInfo, error) { key := RoomParticipantsPrefix + string(roomName) items, err := s.rc.HVals(s.ctx, key).Result() @@ -585,7 +602,7 @@ func (s *RedisStore) storeIngress(_ context.Context, info *livekit.IngressInfo) } // Retry if the key has been changed. - for i := 0; i < maxRetries; i++ { + for range maxRetries { err := s.rc.Watch(s.ctx, txf, IngressKey) switch err { case redis.TxFailedErr: @@ -660,7 +677,7 @@ func (s *RedisStore) storeIngressState(_ context.Context, ingressId string, stat } // Retry if the key has been changed. - for i := 0; i < maxRetries; i++ { + for range maxRetries { err := s.rc.Watch(s.ctx, txf, IngressStatePrefix+ingressId) switch err { case redis.TxFailedErr: diff --git a/pkg/service/redisstore_sip.go b/pkg/service/redisstore_sip.go index e6dc108..ba2931a 100644 --- a/pkg/service/redisstore_sip.go +++ b/pkg/service/redisstore_sip.go @@ -106,12 +106,19 @@ func (s *RedisStore) LoadSIPOutboundTrunk(ctx context.Context, id string) (*live } func (s *RedisStore) DeleteSIPTrunk(ctx context.Context, id string) error { - tx := s.rc.TxPipeline() - tx.HDel(s.ctx, SIPTrunkKey, id) - tx.HDel(s.ctx, SIPInboundTrunkKey, id) - tx.HDel(s.ctx, SIPOutboundTrunkKey, id) - _, err := tx.Exec(ctx) - return err + err1 := s.rc.HDel(s.ctx, SIPTrunkKey, id).Err() + err2 := s.rc.HDel(s.ctx, SIPInboundTrunkKey, id).Err() + err3 := s.rc.HDel(s.ctx, SIPOutboundTrunkKey, id).Err() + if err1 != nil { + return err1 + } + if err2 != nil { + return err2 + } + if err3 != nil { + return err3 + } + return nil } func (s *RedisStore) listSIPLegacyTrunk(ctx context.Context, page *livekit.Pagination) ([]*livekit.SIPTrunkInfo, error) { diff --git a/pkg/service/redisstore_sip_test.go b/pkg/service/redisstore_sip_test.go index bfa029d..9ca1236 100644 --- a/pkg/service/redisstore_sip_test.go +++ b/pkg/service/redisstore_sip_test.go @@ -320,7 +320,7 @@ func testIter[T listItem]( ) { ctx := context.Background() var all []string - for i := 0; i < 250; i++ { + for i := range 250 { id := fmt.Sprintf("%05d", i) all = append(all, id) err := create(ctx, id) diff --git a/pkg/service/roomallocator.go b/pkg/service/roomallocator.go index 656edce..436a433 100644 --- a/pkg/service/roomallocator.go +++ b/pkg/service/roomallocator.go @@ -174,7 +174,7 @@ func (r *StandardRoomAllocator) SelectRoomNode(ctx context.Context, roomName liv func (r *StandardRoomAllocator) ValidateCreateRoom(ctx context.Context, roomName livekit.RoomName) error { // when auto create is disabled, we'll check to ensure it's already created - if !r.config.Room.AutoCreate { + if !r.config.Room.AutoCreate && EnsureCreatePermission(ctx) != nil { _, _, err := r.roomStore.LoadRoom(ctx, roomName, false) if err != nil { return err @@ -208,17 +208,16 @@ func (r *StandardRoomAllocator) applyNamedRoomConfiguration(req *livekit.CreateR conf, ok := r.config.Room.RoomConfigurations[req.RoomPreset] if !ok { - return req, psrpc.NewErrorf(psrpc.InvalidArgument, "unknown room confguration in create room request") + return req, psrpc.NewErrorf(psrpc.InvalidArgument, "unknown room configuration in create room request") } clone := utils.CloneProto(req) - // Request overwrites conf if clone.EmptyTimeout == 0 { clone.EmptyTimeout = conf.EmptyTimeout } if clone.DepartureTimeout == 0 { - clone.DepartureTimeout = req.DepartureTimeout + clone.DepartureTimeout = conf.DepartureTimeout } if clone.MaxParticipants == 0 { clone.MaxParticipants = conf.MaxParticipants @@ -241,6 +240,9 @@ func (r *StandardRoomAllocator) applyNamedRoomConfiguration(req *livekit.CreateR if !clone.SyncStreams { clone.SyncStreams = conf.SyncStreams } + if clone.Metadata == "" { + clone.Metadata = conf.Metadata + } return clone, nil } diff --git a/pkg/service/roommanager.go b/pkg/service/roommanager.go index 2417f1f..9419e28 100644 --- a/pkg/service/roommanager.go +++ b/pkg/service/roommanager.go @@ -16,21 +16,27 @@ package service import ( "context" + "crypto/hmac" + "crypto/sha1" + "encoding/base64" "fmt" + "maps" + "net" "os" + "slices" + "strconv" "sync" "time" "github.com/pkg/errors" - "golang.org/x/exp/maps" - "github.com/livekit/livekit-server/pkg/agent" - "github.com/livekit/livekit-server/pkg/sfu" - sutils "github.com/livekit/livekit-server/pkg/utils" "github.com/livekit/mediatransportutil/pkg/rtcconfig" "github.com/livekit/protocol/auth" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" + "github.com/livekit/protocol/observability" + "github.com/livekit/protocol/observability/roomobs" "github.com/livekit/protocol/rpc" "github.com/livekit/protocol/utils" "github.com/livekit/protocol/utils/guid" @@ -38,6 +44,10 @@ import ( "github.com/livekit/psrpc" "github.com/livekit/psrpc/pkg/middleware" + "github.com/livekit/livekit-server/pkg/agent" + "github.com/livekit/livekit-server/pkg/sfu" + sutils "github.com/livekit/livekit-server/pkg/utils" + "github.com/livekit/livekit-server/pkg/clientconfiguration" "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/livekit-server/pkg/routing" @@ -53,8 +63,6 @@ const ( tokenDefaultTTL = 10 * time.Minute ) -var affinityEpoch = time.Date(2000, 0, 0, 0, 0, 0, 0, time.UTC) - type iceConfigCacheKey struct { roomName livekit.RoomName participantIdentity livekit.ParticipantIdentity @@ -72,6 +80,7 @@ type RoomManager struct { router routing.Router roomAllocator RoomAllocator roomManagerServer rpc.TypedRoomManagerServer + whipServer rpc.WHIPServer[livekit.NodeID] roomStore ObjectStore telemetry telemetry.TelemetryService clientConfManager clientconfiguration.ClientConfigurationManager @@ -84,13 +93,19 @@ type RoomManager struct { rooms map[livekit.RoomName]*rtc.Room - roomServers utils.MultitonService[rpc.RoomTopic] - agentDispatchServers utils.MultitonService[rpc.RoomTopic] - participantServers utils.MultitonService[rpc.ParticipantTopic] + roomServers utils.MultitonService[rpc.RoomTopic] + agentDispatchServers utils.MultitonService[rpc.RoomTopic] + participantServers utils.MultitonService[rpc.ParticipantTopic] + httpSignalParticipantServers utils.MultitonService[rpc.ParticipantTopic] + whipParticipantServers utils.MultitonService[rpc.ParticipantTopic] iceConfigCache *sutils.IceConfigCache[iceConfigCacheKey] forwardStats *sfu.ForwardStats + + rpc.UnimplementedParticipantServer + rpc.UnimplementedRoomServer + rpc.UnimplementedRoomManagerServer } func NewLocalRoomManager( @@ -100,7 +115,6 @@ func NewLocalRoomManager( router routing.Router, roomAllocator RoomAllocator, telemetry telemetry.TelemetryService, - clientConfManager clientconfiguration.ClientConfigurationManager, agentClient agent.Client, agentStore AgentStore, egressLauncher rtc.EgressLauncher, @@ -122,7 +136,7 @@ func NewLocalRoomManager( roomAllocator: roomAllocator, roomStore: roomStore, telemetry: telemetry, - clientConfManager: clientConfManager, + clientConfManager: clientconfiguration.NewStaticClientConfigurationManager(clientconfiguration.StaticConfigurations), egressLauncher: egressLauncher, agentClient: agentClient, agentStore: agentStore, @@ -153,6 +167,18 @@ func NewLocalRoomManager( return nil, err } + whipService, err := newWhipService(r) + if err != nil { + return nil, err + } + r.whipServer, err = rpc.NewWHIPServer[livekit.NodeID](whipService, bus, rpc.WithDefaultServerOptions(conf.PSRPC, logger.GetLogger())) + if err != nil { + return nil, err + } + if err := r.whipServer.RegisterAllCommonTopics(currentNode.NodeID()); err != nil { + return nil, err + } + return r, nil } @@ -193,7 +219,7 @@ func (r *RoomManager) deleteRoom(ctx context.Context, roomName livekit.RoomName) func (r *RoomManager) CloseIdleRooms() { r.lock.RLock() - rooms := maps.Values(r.rooms) + rooms := slices.Collect(maps.Values(r.rooms)) r.lock.RUnlock() for _, room := range rooms { @@ -216,7 +242,7 @@ func (r *RoomManager) HasParticipants() bool { func (r *RoomManager) Stop() { // disconnect all clients r.lock.RLock() - rooms := maps.Values(r.rooms) + rooms := slices.Collect(maps.Values(r.rooms)) r.lock.RUnlock() for _, room := range rooms { @@ -224,9 +250,12 @@ func (r *RoomManager) Stop() { } r.roomManagerServer.Kill() + r.whipServer.Kill() r.roomServers.Kill() r.agentDispatchServers.Kill() r.participantServers.Kill() + r.httpSignalParticipantServers.Kill() + r.whipParticipantServers.Kill() if r.rtcConfig != nil { if r.rtcConfig.UDPMux != nil { @@ -318,10 +347,12 @@ func (r *RoomManager) StartSession( Leave: leave, }, }) + prometheus.IncrementParticipantRtcCanceled(1) return errors.New("could not restart closed participant") } - participant.GetLogger().Infow("resuming RTC session", + participant.GetLogger().Infow( + "resuming RTC session", "nodeID", r.currentNode.NodeID(), "participantInit", &pi, "numParticipants", room.GetParticipantCount(), @@ -343,6 +374,9 @@ func (r *RoomManager) StartSession( return err } r.telemetry.ParticipantResumed(ctx, room.ToProto(), participant.ToProto(), r.currentNode.NodeID(), pi.ReconnectReason) + + go room.HandleSyncState(participant, pi.SyncState) + go r.rtcSessionWorker(room, participant, requestSource) return nil } @@ -371,6 +405,7 @@ func (r *RoomManager) StartSession( Leave: leave, }, }) + prometheus.IncrementParticipantRtcCanceled(1) return errors.New("could not restart participant") } @@ -381,7 +416,8 @@ func (r *RoomManager) StartSession( sid, false, ) - pLogger.Infow("starting RTC session", + pLogger.Infow( + "starting RTC session", "room", room.Name(), "nodeID", r.currentNode.NodeID(), "numParticipants", room.GetParticipantCount(), @@ -396,78 +432,98 @@ func (r *RoomManager) StartSession( if pi.DisableICELite { rtcConf.SettingEngine.SetLite(false) } + rtcConf.UpdatePublisherConfig(pi.UseSinglePeerConnection) + // default allow forceTCP allowFallback := true if r.config.RTC.AllowTCPFallback != nil { allowFallback = *r.config.RTC.AllowTCPFallback } + // default do not force full reconnect on a publication error reconnectOnPublicationError := false if r.config.RTC.ReconnectOnPublicationError != nil { reconnectOnPublicationError = *r.config.RTC.ReconnectOnPublicationError } + // default do not force full reconnect on a subscription error reconnectOnSubscriptionError := false if r.config.RTC.ReconnectOnSubscriptionError != nil { reconnectOnSubscriptionError = *r.config.RTC.ReconnectOnSubscriptionError } + // default do not force full reconnect on a data channel error reconnectOnDataChannelError := false if r.config.RTC.ReconnectOnDataChannelError != nil { reconnectOnDataChannelError = *r.config.RTC.ReconnectOnDataChannelError } + subscriberAllowPause := r.config.RTC.CongestionControl.AllowPause if pi.SubscriberAllowPause != nil { subscriberAllowPause = *pi.SubscriberAllowPause } + + enabledCodecs := protoRoom.EnabledCodecs + if !slices.ContainsFunc(enabledCodecs, func(codec *livekit.Codec) bool { + return mime.IsMimeTypeStringRTX(codec.Mime) + }) { + enabledCodecs = append(enabledCodecs, &livekit.Codec{Mime: mime.MimeTypeRTX.String()}) + } + participant, err = rtc.NewParticipant(rtc.ParticipantParams{ - Identity: pi.Identity, - Name: pi.Name, - SID: sid, - Config: &rtcConf, - Sink: responseSink, - AudioConfig: r.config.Audio, - VideoConfig: r.config.Video, - LimitConfig: r.config.Limit, - ProtocolVersion: pv, - SessionStartTime: sessionStartTime, - Telemetry: r.telemetry, - Trailer: room.Trailer(), - PLIThrottleConfig: r.config.RTC.PLIThrottle, - CongestionControlConfig: r.config.RTC.CongestionControl, - PublishEnabledCodecs: protoRoom.EnabledCodecs, - SubscribeEnabledCodecs: protoRoom.EnabledCodecs, - Grants: pi.Grants, - Reconnect: pi.Reconnect, - Logger: pLogger, - ClientConf: clientConf, - ClientInfo: rtc.ClientInfo{ClientInfo: pi.Client}, - Region: pi.Region, - AdaptiveStream: pi.AdaptiveStream, - AllowTCPFallback: allowFallback, - TURNSEnabled: r.config.IsTURNSEnabled(), - GetParticipantInfo: func(pID livekit.ParticipantID) *livekit.ParticipantInfo { - if p := room.GetParticipantByID(pID); p != nil { - return p.ToProto() - } - return nil + Identity: pi.Identity, + Name: pi.Name, + SID: sid, + Config: &rtcConf, + Sink: responseSink, + AudioConfig: r.config.Audio, + VideoConfig: r.config.Video, + LimitConfig: r.config.Limit, + ProtocolVersion: pv, + SessionStartTime: sessionStartTime, + SessionTimer: observability.NewSessionTimer(sessionStartTime), + TelemetryListener: room.ParticipantTelemetryListener(), + Trailer: room.Trailer(), + PLIThrottleConfig: r.config.RTC.PLIThrottle, + CongestionControlConfig: r.config.RTC.CongestionControl, + PublishEnabledCodecs: enabledCodecs, + SubscribeEnabledCodecs: enabledCodecs, + Grants: pi.Grants, + Reconnect: pi.Reconnect, + Logger: pLogger, + Reporter: roomobs.NewNoopParticipantSessionReporter(), + ClientConf: clientConf, + ClientInfo: rtc.ClientInfo{ClientInfo: pi.Client}, + Region: pi.Region, + AdaptiveStream: pi.AdaptiveStream, + AllowTCPFallback: allowFallback, + TCPFallbackRTTThreshold: r.config.RTC.TCPFallbackRTTThreshold, + AllowUDPUnstableFallback: r.config.RTC.AllowUDPUnstableFallback, + TURNSEnabled: r.config.IsTURNSEnabled(), + ParticipantListener: room.LocalParticipantListener(), + ParticipantHelper: &roomManagerParticipantHelper{ + room: room, + codecRegressionThreshold: r.config.Video.CodecRegressionThreshold, }, - ReconnectOnPublicationError: reconnectOnPublicationError, - ReconnectOnSubscriptionError: reconnectOnSubscriptionError, - ReconnectOnDataChannelError: reconnectOnDataChannelError, - VersionGenerator: r.versionGenerator, - TrackResolver: room.ResolveMediaTrackForSubscriber, - SubscriberAllowPause: subscriberAllowPause, - SubscriptionLimitAudio: r.config.Limit.SubscriptionLimitAudio, - SubscriptionLimitVideo: r.config.Limit.SubscriptionLimitVideo, - PlayoutDelay: roomInternal.GetPlayoutDelay(), - SyncStreams: roomInternal.GetSyncStreams(), - ForwardStats: r.forwardStats, - MetricConfig: r.config.Metric, - UseOneShotSignallingMode: useOneShotSignallingMode, - DataChannelMaxBufferedAmount: r.config.RTC.DataChannelMaxBufferedAmount, - DatachannelSlowThreshold: r.config.RTC.DatachannelSlowThreshold, - FireOnTrackBySdp: true, + ReconnectOnPublicationError: reconnectOnPublicationError, + ReconnectOnSubscriptionError: reconnectOnSubscriptionError, + ReconnectOnDataChannelError: reconnectOnDataChannelError, + VersionGenerator: r.versionGenerator, + SubscriberAllowPause: subscriberAllowPause, + SubscriptionLimitAudio: r.config.Limit.SubscriptionLimitAudio, + SubscriptionLimitVideo: r.config.Limit.SubscriptionLimitVideo, + PlayoutDelay: roomInternal.GetPlayoutDelay(), + SyncStreams: roomInternal.GetSyncStreams(), + ForwardStats: r.forwardStats, + MetricConfig: r.config.Metric, + UseOneShotSignallingMode: useOneShotSignallingMode, + DataChannelMaxBufferedAmount: r.config.RTC.DataChannelMaxBufferedAmount, + DatachannelSlowThreshold: r.config.RTC.DatachannelSlowThreshold, + DatachannelLossyTargetLatency: r.config.RTC.DatachannelLossyTargetLatency, + FireOnTrackBySdp: true, + UseSinglePeerConnection: pi.UseSinglePeerConnection, + EnableDataTracks: r.config.EnableDataTracks, + EnableRTPStreamRestartDetection: r.config.RTC.EnableRTPStreamRestartDetection, }) if err != nil { return err @@ -478,6 +534,9 @@ func (r *RoomManager) StartSession( opts := rtc.ParticipantOptions{ AutoSubscribe: pi.AutoSubscribe, } + if pi.AutoSubscribeDataTrack != nil { + opts.AutoSubscribeDataTrack = *pi.AutoSubscribeDataTrack + } iceServers := r.iceServersForParticipant(apiKey, participant, iceConfig.PreferenceSubscriber == livekit.ICECandidateType_ICT_TLS) if err = room.Join(participant, requestSource, &opts, iceServers); err != nil { pLogger.Errorw("could not join room", err) @@ -485,16 +544,28 @@ func (r *RoomManager) StartSession( return err } + var participantServerClosers utils.Closers participantTopic := rpc.FormatParticipantTopic(room.Name(), participant.Identity()) participantServer := must.Get(rpc.NewTypedParticipantServer(r, r.bus)) - killParticipantServer := r.participantServers.Replace(participantTopic, participantServer) + participantServerClosers = append(participantServerClosers, utils.CloseFunc(r.participantServers.Replace(participantTopic, participantServer))) if err := participantServer.RegisterAllParticipantTopics(participantTopic); err != nil { - killParticipantServer() + participantServerClosers.Close() pLogger.Errorw("could not join register participant topic", err) _ = participant.Close(true, types.ParticipantCloseReasonMessageBusFailed, false) return err } + if useOneShotSignallingMode { + whipParticipantServer := must.Get(rpc.NewTypedWHIPParticipantServer(whipParticipantService{r}, r.bus)) + participantServerClosers = append(participantServerClosers, utils.CloseFunc(r.whipParticipantServers.Replace(participantTopic, whipParticipantServer))) + if err := whipParticipantServer.RegisterAllCommonTopics(participantTopic); err != nil { + participantServerClosers.Close() + pLogger.Errorw("could not join register participant topic for rtc rest participant server", err) + _ = participant.Close(true, types.ParticipantCloseReasonMessageBusFailed, false) + return err + } + } + if err = r.roomStore.StoreParticipant(ctx, room.Name(), participant.ToProto()); err != nil { pLogger.Errorw("could not store participant", err) } @@ -512,9 +583,9 @@ func (r *RoomManager) StartSession( persistRoomForParticipantCount(room.ToProto()) clientMeta := &livekit.AnalyticsClientMeta{Region: r.currentNode.Region(), Node: string(r.currentNode.NodeID())} - r.telemetry.ParticipantJoined(ctx, protoRoom, participant.ToProto(), pi.Client, clientMeta, true) - participant.OnClose(func(p types.LocalParticipant) { - killParticipantServer() + r.telemetry.ParticipantJoined(ctx, protoRoom, participant.ToProto(), pi.Client, clientMeta, true, participant.TelemetryGuard()) + participant.AddOnClose(types.ParticipantCloseKeyNormal, func(p types.LocalParticipant) { + participantServerClosers.Close() if err := r.roomStore.DeleteParticipant(ctx, room.Name(), p.Identity()); err != nil { pLogger.Errorw("could not delete participant", err) @@ -523,7 +594,7 @@ func (r *RoomManager) StartSession( // update room store with new numParticipants proto := room.ToProto() persistRoomForParticipantCount(proto) - r.telemetry.ParticipantLeft(ctx, proto, p.ToProto(), true) + r.telemetry.ParticipantLeft(ctx, proto, p.ToProto(), true, participant.TelemetryGuard()) }) participant.OnClaimsChanged(func(participant types.LocalParticipant) { pLogger.Debugw("refreshing client token after claims change") @@ -535,6 +606,13 @@ func (r *RoomManager) StartSession( r.iceConfigCache.Put(iceConfigCacheKey{room.Name(), participant.Identity()}, iceConfig) }) + for _, addTrackRequest := range pi.AddTrackRequests { + participant.AddTrack(addTrackRequest) + } + if pi.PublisherOffer != nil { + participant.HandleOffer(pi.PublisherOffer) + } + go r.rtcSessionWorker(room, participant, requestSource) return nil } @@ -599,22 +677,22 @@ func (r *RoomManager) getOrCreateRoom(ctx context.Context, createRoom *livekit.C r.telemetry.RoomEnded(ctx, roomInfo) prometheus.RoomEnded(time.Unix(roomInfo.CreationTime, 0)) if err := r.deleteRoom(ctx, roomName); err != nil { - newRoom.Logger.Errorw("could not delete room", err) + newRoom.Logger().Errorw("could not delete room", err) } - newRoom.Logger.Infow("room closed") + newRoom.Logger().Infow("room closed") }) newRoom.OnRoomUpdated(func() { if err := r.roomStore.StoreRoom(ctx, newRoom.ToProto(), newRoom.Internal()); err != nil { - newRoom.Logger.Errorw("could not handle metadata update", err) + newRoom.Logger().Errorw("could not handle metadata update", err) } }) - newRoom.OnParticipantChanged(func(p types.LocalParticipant) { + newRoom.OnParticipantChanged(func(p types.Participant) { if !p.IsDisconnected() { if err := r.roomStore.StoreParticipant(ctx, roomName, p.ToProto()); err != nil { - newRoom.Logger.Errorw("could not handle participant change", err) + newRoom.Logger().Errorw("could not handle participant change", err) } } }) @@ -648,12 +726,7 @@ func (r *RoomManager) getOrCreateRoom(ctx context.Context, createRoom *livekit.C // manages an RTC session for a participant, runs on the RTC node func (r *RoomManager) rtcSessionWorker(room *rtc.Room, participant types.LocalParticipant, requestSource routing.MessageSource) { - pLogger := rtc.LoggerWithParticipant( - rtc.LoggerWithRoom(logger.GetLogger(), room.Name(), room.ID()), - participant.Identity(), - participant.ID(), - false, - ) + pLogger := participant.GetLogger() defer func() { pLogger.Debugw("RTC session finishing", "connID", requestSource.ConnectionID()) requestSource.Close() @@ -673,11 +746,13 @@ func (r *RoomManager) rtcSessionWorker(room *rtc.Room, participant types.LocalPa select { case <-participant.Disconnected(): return + case <-tokenTicker.C: // refresh token with the first API Key/secret pair if err := r.refreshToken(participant); err != nil { pLogger.Errorw("could not refresh token", err, "connID", requestSource.ConnectionID()) } + case obj := <-requestSource.ReadChan(): if obj == nil { if room.GetParticipantRequestSource(participant.Identity()) == requestSource { @@ -686,8 +761,7 @@ func (r *RoomManager) rtcSessionWorker(room *rtc.Room, participant types.LocalPa return } - req := obj.(*livekit.SignalRequest) - if err := rtc.HandleParticipantSignal(room, participant, req, pLogger); err != nil { + if err := participant.HandleSignalMessage(obj); err != nil { // more specific errors are already logged // treat errors returned as fatal return @@ -715,6 +789,37 @@ func (r *RoomManager) roomAndParticipantForReq(ctx context.Context, req particip return room, participant, nil } +func (r *RoomManager) ListParticipants(ctx context.Context, req *livekit.ListParticipantsRequest) (*livekit.ListParticipantsResponse, error) { + room := r.GetRoom(ctx, livekit.RoomName(req.Room)) + if room == nil { + return nil, ErrRoomNotFound + } + + participants := room.GetParticipants() + items := make([]*livekit.ParticipantInfo, 0, len(participants)) + for _, p := range participants { + items = append(items, p.ToProto()) + } + + return &livekit.ListParticipantsResponse{ + Participants: items, + }, nil +} + +func (r *RoomManager) GetParticipant(ctx context.Context, req *livekit.RoomParticipantIdentity) (*livekit.ParticipantInfo, error) { + room := r.GetRoom(ctx, livekit.RoomName(req.Room)) + if room == nil { + return nil, ErrRoomNotFound + } + + participant := room.GetParticipant(livekit.ParticipantIdentity(req.Identity)) + if participant == nil { + return nil, ErrParticipantNotFound + } + + return participant.ToProto(), nil +} + func (r *RoomManager) RemoveParticipant(ctx context.Context, req *livekit.RoomParticipantIdentity) (*livekit.RemoveParticipantResponse, error) { room, participant, err := r.roomAndParticipantForReq(ctx, req) if err != nil { @@ -738,7 +843,10 @@ func (r *RoomManager) MutePublishedTrack(ctx context.Context, req *livekit.MuteR participant.GetLogger().Errorw("cannot unmute track, remote unmute is disabled", nil) return nil, ErrRemoteUnmuteNoteEnabled } - track := participant.SetTrackMuted(livekit.TrackID(req.TrackSid), req.Muted, true) + track := participant.SetTrackMuted(&livekit.MuteTrackRequest{ + Sid: req.TrackSid, + Muted: req.Muted, + }, true) return &livekit.MuteRoomTrackResponse{Track: track}, nil } @@ -748,31 +856,58 @@ func (r *RoomManager) UpdateParticipant(ctx context.Context, req *livekit.Update return nil, err } - participant.GetLogger().Debugw("updating participant", - "metadata", req.Metadata, - "permission", req.Permission, - "attributes", req.Attributes, - ) - if err = participant.CheckMetadataLimits(req.Name, req.Metadata, req.Attributes); err != nil { + if err = participant.UpdateMetadata(&livekit.UpdateParticipantMetadata{ + Name: req.Name, + Metadata: req.Metadata, + Attributes: req.Attributes, + }, true); err != nil { return nil, err } - if req.Name != "" { - participant.SetName(req.Name) - } - if req.Metadata != "" { - participant.SetMetadata(req.Metadata) - } - if req.Attributes != nil { - participant.SetAttributes(req.Attributes) - } - if req.Permission != nil { + participant.GetLogger().Debugw( + "updating participant permission", + "permission", req.Permission, + ) + participant.SetPermission(req.Permission) } + return participant.ToProto(), nil } +func (r *RoomManager) ForwardParticipant(ctx context.Context, req *livekit.ForwardParticipantRequest) (*livekit.ForwardParticipantResponse, error) { + return nil, errors.New("not implemented") +} + +func (r *RoomManager) MoveParticipant(ctx context.Context, req *livekit.MoveParticipantRequest) (*livekit.MoveParticipantResponse, error) { + return nil, errors.New("not implemented") +} + +func (r *RoomManager) PerformRpc(ctx context.Context, req *livekit.PerformRpcRequest) (*livekit.PerformRpcResponse, error) { + room := r.GetRoom(ctx, livekit.RoomName(req.GetRoom())) + if room == nil { + return nil, ErrRoomNotFound + } + + participant := room.GetParticipant(livekit.ParticipantIdentity(req.GetDestinationIdentity())) + if participant == nil { + return nil, ErrParticipantNotFound + } + + resultChan := make(chan string, 1) + errorChan := make(chan error, 1) + + participant.PerformRpc(req, resultChan, errorChan) + + select { + case result := <-resultChan: + return &livekit.PerformRpcResponse{Payload: result}, nil + case err := <-errorChan: + return nil, err + } +} + func (r *RoomManager) DeleteRoom(ctx context.Context, req *livekit.DeleteRoomRequest) (*livekit.DeleteRoomResponse, error) { room := r.GetRoom(ctx, livekit.RoomName(req.Room)) if room == nil { @@ -784,7 +919,7 @@ func (r *RoomManager) DeleteRoom(ctx context.Context, req *livekit.DeleteRoomReq return nil, err } } else { - room.Logger.Infow("deleting room") + room.Logger().Infow("deleting room") room.Close(types.ParticipantCloseReasonServiceRequestDeleteRoom) } return &livekit.DeleteRoomResponse{}, nil @@ -812,7 +947,7 @@ func (r *RoomManager) SendData(ctx context.Context, req *livekit.SendDataRequest return nil, ErrRoomNotFound } - room.Logger.Debugw("api send data", "size", len(req.Data)) + room.Logger().Debugw("api send data", "size", len(req.Data)) room.SendDataPacket(&livekit.DataPacket{ Kind: req.Kind, DestinationIdentities: req.DestinationIdentities, @@ -835,7 +970,7 @@ func (r *RoomManager) UpdateRoomMetadata(ctx context.Context, req *livekit.Updat return nil, ErrRoomNotFound } - room.Logger.Debugw("updating room") + room.Logger().Debugw("updating room") done := room.SetMetadata(req.Metadata) // wait till the update is applied <-done @@ -903,19 +1038,20 @@ func (r *RoomManager) iceServersForParticipant(apiKey string, participant types. if r.config.TURN.UDPPort > 0 && !tlsOnly { // UDP TURN is used as STUN hasSTUN = true - urls = append(urls, fmt.Sprintf("turn:%s:%d?transport=udp", r.config.RTC.NodeIP, r.config.TURN.UDPPort)) + for _, ip := range r.config.RTC.NodeIP.ToStringSlice() { + urls = append(urls, fmt.Sprintf("turn:%s?transport=udp", net.JoinHostPort(ip, strconv.Itoa(int(r.config.TURN.UDPPort))))) + } } if r.config.TURN.TLSPort > 0 { urls = append(urls, fmt.Sprintf("turns:%s:443?transport=tcp", r.config.TURN.Domain)) } if len(urls) > 0 { - username := r.turnAuthHandler.CreateUsername(apiKey, participant.ID()) - password, err := r.turnAuthHandler.CreatePassword(apiKey, participant.ID()) + username, expiry := r.turnAuthHandler.CreateUsername(apiKey, participant.ID(), r.config.TURN.TTLSeconds) + password, err := r.turnAuthHandler.CreatePassword(apiKey, participant.ID(), expiry) if err != nil { participant.GetLogger().Warnw("could not create turn password", err) hasSTUN = false } else { - logger.Infow("created TURN password", "username", username, "password", password) iceServers = append(iceServers, &livekit.ICEServer{ Urls: urls, Username: username, @@ -930,17 +1066,41 @@ func (r *RoomManager) iceServersForParticipant(apiKey string, participant types. for _, s := range r.config.RTC.TURNServers { scheme := "turn" transport := "tcp" - if s.Protocol == "tls" { + switch s.Protocol { + case "tls": scheme = "turns" - } else if s.Protocol == "udp" { + case "udp": transport = "udp" } + + var username, credential string + if s.Secret != "" { + // Generate dynamic credentials using TURN static auth secrets + ttl := s.TTL + if ttl == 0 { + ttl = 14400 // Default 4 hours + } + + expiry := time.Now().Add(time.Duration(ttl) * time.Second).Unix() + participantID := string(participant.ID()) + username = fmt.Sprintf("%d:%s", expiry, participantID) + + // HMAC-SHA1 signature + h := hmac.New(sha1.New, []byte(s.Secret)) + h.Write([]byte(username)) + credential = base64.StdEncoding.EncodeToString(h.Sum(nil)) + } else { + // Use static credentials + username = s.Username + credential = s.Credential + } + is := &livekit.ICEServer{ Urls: []string{ fmt.Sprintf("%s:%s:%d?transport=%s", scheme, s.Host, s.Port, transport), }, - Username: s.Username, - Credential: s.Credential, + Username: username, + Credential: credential, } iceServers = append(iceServers, is) } @@ -967,6 +1127,7 @@ func (r *RoomManager) refreshToken(participant types.LocalParticipant) error { token := auth.NewAccessToken(key, secret) token.SetName(grants.Name). SetIdentity(string(participant.Identity())). + SetKind(grants.GetParticipantKind()). SetValidFor(tokenDefaultTTL). SetMetadata(grants.Metadata). SetAttributes(grants.Attributes). @@ -1010,3 +1171,41 @@ func iceServerForStunServers(servers []string) *livekit.ICEServer { } return iceServer } + +// ------------------------------------ + +type roomManagerParticipantHelper struct { + room *rtc.Room + codecRegressionThreshold int +} + +func (h *roomManagerParticipantHelper) GetParticipantInfo(pID livekit.ParticipantID) *livekit.ParticipantInfo { + if p := h.room.GetParticipantByID(pID); p != nil { + return p.ToProto() + } + return nil +} + +func (h *roomManagerParticipantHelper) GetRegionSettings(ip string) *livekit.RegionSettings { + return nil +} + +func (h *roomManagerParticipantHelper) GetSubscriberForwarderState(lp types.LocalParticipant) (map[livekit.TrackID]*livekit.RTPForwarderState, error) { + return nil, nil +} + +func (h *roomManagerParticipantHelper) ResolveMediaTrack(lp types.LocalParticipant, trackID livekit.TrackID) types.MediaResolverResult { + return h.room.ResolveMediaTrackForSubscriber(lp, trackID) +} + +func (h *roomManagerParticipantHelper) ResolveDataTrack(lp types.LocalParticipant, trackID livekit.TrackID) types.DataResolverResult { + return h.room.ResolveDataTrackForSubscriber(lp, trackID) +} + +func (h *roomManagerParticipantHelper) ShouldRegressCodec() bool { + return h.codecRegressionThreshold == 0 || h.room.GetParticipantCount() < h.codecRegressionThreshold +} + +func (h *roomManagerParticipantHelper) GetCachedReliableDataMessage(seqs map[livekit.ParticipantID]uint32) []*types.DataMessageCache { + return h.room.GetCachedReliableDataMessage(seqs) +} diff --git a/pkg/service/roommanager_service.go b/pkg/service/roommanager_service.go new file mode 100644 index 0000000..0b5ce6d --- /dev/null +++ b/pkg/service/roommanager_service.go @@ -0,0 +1,337 @@ +package service + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/pion/webrtc/v4" + "golang.org/x/sync/errgroup" + "google.golang.org/protobuf/types/known/emptypb" + + "github.com/livekit/livekit-server/pkg/routing" + "github.com/livekit/livekit-server/pkg/rtc/types" + "github.com/livekit/livekit-server/pkg/telemetry/prometheus" + "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/logger" + "github.com/livekit/protocol/rpc" + "github.com/livekit/psrpc" +) + +const ( + whipSessionNotifyInterval = 10 * time.Second +) + +type whipService struct { + *RoomManager + + ingressRpcCli rpc.IngressHandlerClient + + rpc.UnimplementedWHIPServer +} + +func newWhipService(rm *RoomManager) (*whipService, error) { + cli, err := rpc.NewIngressHandlerClient(rm.bus, rpc.WithDefaultClientOptions(logger.GetLogger())) + if err != nil { + return nil, err + } + return &whipService{ + RoomManager: rm, + ingressRpcCli: cli, + }, nil +} + +func (s whipService) Create(ctx context.Context, req *rpc.WHIPCreateRequest) (*rpc.WHIPCreateResponse, error) { + pi, err := routing.ParticipantInitFromStartSession(req.StartSession, s.RoomManager.currentNode.Region()) + if err != nil { + logger.Errorw("whip service: could not create participant init", err) + return nil, err + } + + prometheus.IncrementParticipantRtcInit(1) + + if err = s.RoomManager.StartSession( + ctx, + *pi, + routing.NewNullMessageSource(livekit.ConnectionID(req.StartSession.ConnectionId)), // no requestSource + routing.NewNullMessageSink(livekit.ConnectionID(req.StartSession.ConnectionId)), // no responseSink + true, // useOneShotSignallingMode + ); err != nil { + logger.Errorw("whip service: could not start session", err) + return nil, err + } + + room := s.RoomManager.GetRoom(ctx, livekit.RoomName(req.StartSession.RoomName)) + if room == nil { + logger.Errorw("whip service: could not find room", nil, "room", req.StartSession.RoomName) + return nil, ErrRoomNotFound + } + + lp := room.GetParticipant(pi.Identity) + if lp == nil { + room.Logger().Errorw("whip service: could not find local participant", nil, "participant", pi.Identity) + return nil, ErrParticipantNotFound + } + + if err := lp.HandleOffer(&livekit.SessionDescription{ + Type: webrtc.SDPTypeOffer.String(), + Sdp: req.OfferSdp, + Id: 0, + }); err != nil { + lp.GetLogger().Errorw("whip service: could not handle offer", err) + return nil, err + } + + // wait for subscriptions to resolve + // NOTE: this is outside the WHIP spec, but added as a convenience for clients doing + // one-shot signalling (i. e. send an offer and get an answer once) to publish and subscribe to + // well-known tracks (i. e. remote participant identity and track names are well known) + eg, _ := errgroup.WithContext(ctx) + for publisherIdentity, trackList := range req.SubscribedParticipantTracks { + for _, trackName := range trackList.TrackNames { + eg.Go(func() error { + for { + if lp.IsTrackNameSubscribed(livekit.ParticipantIdentity(publisherIdentity), trackName) { + return nil + } + time.Sleep(50 * time.Millisecond) + } + }) + } + } + err = eg.Wait() + if err != nil { + lp.GetLogger().Errorw("whip service: could not subscribe to tracks", err) + return nil, err + } + + answer, _, err := lp.GetAnswer() + if err != nil { + lp.GetLogger().Errorw("whip service: could not get answer", err) + return nil, err + } + + iceSessionID, err := lp.GetPublisherICESessionUfrag() + if err != nil { + lp.GetLogger().Errorw("whip service: could not get ICE session ID", err) + return nil, err + } + + if req.FromIngress { + aliveCtx, cancel := context.WithCancel(context.Background()) + + lp.AddOnClose(types.ParticipantCloseKeyWHIP, func(lp types.LocalParticipant) { + cancel() + + go func() { + lp.GetLogger().Debugw("whip service: notify participant closed") + + video, audio := getMediaStateForParticipant(lp) + + _, err := s.ingressRpcCli.WHIPRTCConnectionNotify(context.Background(), string(lp.ID()), &rpc.WHIPRTCConnectionNotifyRequest{ + ParticipantId: string(lp.ID()), + Closed: true, + Audio: audio, + Video: video, + }, psrpc.WithRequestTimeout(rpc.DefaultPSRPCConfig.Timeout)) + if err != nil { + lp.GetLogger().Warnw("whip service: could not notify ingress of participant closed", err) + } + }() + }) + go func() { + if err := s.notifySession(aliveCtx, lp); err != nil { + cancel() + } + }() + } + + var iceServers []*livekit.ICEServer + apiKey, _, err := s.RoomManager.getFirstKeyPair() + if err == nil { + iceServers = s.RoomManager.iceServersForParticipant( + apiKey, + lp, + false, + ) + } + return &rpc.WHIPCreateResponse{ + AnswerSdp: answer.SDP, + ParticipantId: string(lp.ID()), + IceServers: iceServers, + IceSessionId: iceSessionID, + }, nil +} + +func (s whipService) notifySession(ctx context.Context, participant types.Participant) error { + ticker := time.NewTicker(whipSessionNotifyInterval) + defer ticker.Stop() + + err := s.sendConnectionNotify(ctx, participant) + if err != nil { + if errors.Is(err, context.Canceled) { + return nil + } + } + + for { + select { + case <-ticker.C: + err := s.sendConnectionNotify(ctx, participant) + if err != nil { + if errors.Is(err, context.Canceled) { + return nil + } + } + + case <-ctx.Done(): + return nil + } + } +} + +func (s whipService) sendConnectionNotify(ctx context.Context, participant types.Participant) error { + video, audio := getMediaStateForParticipant(participant) + + _, err := s.ingressRpcCli.WHIPRTCConnectionNotify(ctx, string(participant.ID()), &rpc.WHIPRTCConnectionNotifyRequest{ + ParticipantId: string(participant.ID()), + Video: video, + Audio: audio, + }, psrpc.WithRequestTimeout(rpc.DefaultPSRPCConfig.Timeout)) + + return err +} + +func getMediaStateForParticipant(participant types.Participant) (*livekit.InputVideoState, *livekit.InputAudioState) { + pParticipant := participant.ToProto() + + var video *livekit.InputVideoState + var audio *livekit.InputAudioState + + for _, v := range pParticipant.Tracks { + if v == nil { + continue + } + + if v.Type != livekit.TrackType_VIDEO { + continue + } + + video = &livekit.InputVideoState{} + + video.MimeType = v.MimeType + video.Height = v.Height + video.Width = v.Width + + break + } + + for _, a := range pParticipant.Tracks { + if a == nil { + continue + } + + if a.Type != livekit.TrackType_AUDIO { + continue + } + + audio = &livekit.InputAudioState{} + + audio.MimeType = a.MimeType + audio.Channels = 1 + if a.Stereo { + audio.Channels = 2 + } + + break + } + + return video, audio +} + +// ------------------------------------------- + +type whipParticipantService struct { + *RoomManager +} + +func (r whipParticipantService) ICETrickle(ctx context.Context, req *rpc.WHIPParticipantICETrickleRequest) (*emptypb.Empty, error) { + room := r.RoomManager.GetRoom(ctx, livekit.RoomName(req.Room)) + if room == nil { + return nil, ErrRoomNotFound + } + + lp := room.GetParticipantByID(livekit.ParticipantID(req.ParticipantId)) + if lp == nil { + return nil, ErrParticipantNotFound + } + + iceSessionID, err := lp.GetPublisherICESessionUfrag() + if err != nil { + lp.GetLogger().Warnw("whipParticipant service ice-trickle: could not get ICE session ufrag", err) + return nil, psrpc.NewError(psrpc.Internal, err) + } + + if req.IceSessionId != iceSessionID { + return nil, psrpc.NewError( + psrpc.FailedPrecondition, + fmt.Errorf("ice session mismatch, expected: %s, got: %s", iceSessionID, req.IceSessionId), + ) + } + + if err := lp.HandleICETrickleSDPFragment(req.SdpFragment); err != nil { + return nil, psrpc.NewError(psrpc.InvalidArgument, err) + } + + return &emptypb.Empty{}, nil +} + +func (r whipParticipantService) ICERestart(ctx context.Context, req *rpc.WHIPParticipantICERestartRequest) (*rpc.WHIPParticipantICERestartResponse, error) { + room := r.RoomManager.GetRoom(ctx, livekit.RoomName(req.Room)) + if room == nil { + return nil, ErrRoomNotFound + } + + lp := room.GetParticipantByID(livekit.ParticipantID(req.ParticipantId)) + if lp == nil { + return nil, ErrParticipantNotFound + } + + sdpFragment, err := lp.HandleICERestartSDPFragment(req.SdpFragment) + if err != nil { + return nil, psrpc.NewError(psrpc.InvalidArgument, err) + } + + iceSessionID, err := lp.GetPublisherICESessionUfrag() + if err != nil { + lp.GetLogger().Warnw("whipParticipant service ice-restart: could not get ICE session ufrag", err) + return nil, psrpc.NewError(psrpc.Internal, err) + } + + return &rpc.WHIPParticipantICERestartResponse{ + IceSessionId: iceSessionID, + SdpFragment: sdpFragment, + }, nil +} + +func (r whipParticipantService) DeleteSession(ctx context.Context, req *rpc.WHIPParticipantDeleteSessionRequest) (*emptypb.Empty, error) { + room := r.RoomManager.GetRoom(ctx, livekit.RoomName(req.Room)) + if room == nil { + return nil, ErrRoomNotFound + } + + lp := room.GetParticipantByID(livekit.ParticipantID(req.ParticipantId)) + if lp != nil { + lp.AddOnClose(types.ParticipantCloseKeyWHIP, nil) + room.RemoveParticipant( + lp.Identity(), + lp.ID(), + types.ParticipantCloseReasonClientRequestLeave, + ) + } + + return &emptypb.Empty{}, nil +} + +// -------------------------------- diff --git a/pkg/service/roomservice.go b/pkg/service/roomservice.go index ea72852..3579eec 100644 --- a/pkg/service/roomservice.go +++ b/pkg/service/roomservice.go @@ -24,11 +24,10 @@ import ( "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/livekit-server/pkg/routing" "github.com/livekit/livekit-server/pkg/rtc" - "github.com/livekit/protocol/egress" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" "github.com/livekit/protocol/rpc" - "github.com/livekit/protocol/utils" + "github.com/livekit/psrpc" ) type RoomService struct { @@ -41,6 +40,9 @@ type RoomService struct { topicFormatter rpc.TopicFormatter roomClient rpc.TypedRoomClient participantClient rpc.TypedParticipantClient + + rpc.UnimplementedRoomServer + rpc.UnimplementedParticipantServer } func NewRoomService( @@ -69,10 +71,9 @@ func NewRoomService( } func (s *RoomService) CreateRoom(ctx context.Context, req *livekit.CreateRoomRequest) (*livekit.Room, error) { - redactedReq := redactCreateRoomRequest(req) - RecordRequest(ctx, redactedReq) + RecordRequest(ctx, req) - AppendLogFields(ctx, "room", req.Name, "request", logger.Proto(redactedReq)) + AppendLogFields(ctx, "room", req.Name, "request", logger.Proto(req)) if err := EnsureCreatePermission(ctx); err != nil { return nil, twirpAuthError(err) } else if req.Egress != nil && s.egressLauncher == nil { @@ -127,13 +128,15 @@ func (s *RoomService) DeleteRoom(ctx context.Context, req *livekit.DeleteRoomReq return nil, twirpAuthError(err) } - _, _, err := s.roomStore.LoadRoom(ctx, livekit.RoomName(req.Room), false) + exists, err := s.roomStore.RoomExists(ctx, livekit.RoomName(req.Room)) if err != nil { return nil, err + } else if !exists { + return nil, ErrRoomNotFound } // ensure at least one node is available to handle the request - _, err = s.router.CreateRoom(ctx, &livekit.CreateRoomRequest{Name: req.Room}) + room, err := s.router.CreateRoom(ctx, &livekit.CreateRoomRequest{Name: req.Room}) if err != nil { return nil, err } @@ -143,13 +146,15 @@ func (s *RoomService) DeleteRoom(ctx context.Context, req *livekit.DeleteRoomReq return nil, err } - err = s.roomStore.DeleteRoom(ctx, livekit.RoomName(req.Room)) + if os, ok := s.roomStore.(OSSServiceStore); ok { + err = os.DeleteRoom(ctx, livekit.RoomName(req.Room)) + } res := &livekit.DeleteRoomResponse{} - RecordResponse(ctx, res) + RecordResponse(ctx, room) return res, err } -func (s *RoomService) ListParticipants(ctx context.Context, req *livekit.ListParticipantsRequest) (*livekit.ListParticipantsResponse, error) { +func (s *RoomService) ListParticipants(ctx context.Context, req *livekit.ListParticipantsRequest) (res *livekit.ListParticipantsResponse, err error) { RecordRequest(ctx, req) AppendLogFields(ctx, "room", req.Room) @@ -157,19 +162,29 @@ func (s *RoomService) ListParticipants(ctx context.Context, req *livekit.ListPar return nil, twirpAuthError(err) } - participants, err := s.roomStore.ListParticipants(ctx, livekit.RoomName(req.Room)) + if s.apiConf.EnablePsrpcForGetListParticpants { + res, err = s.roomClient.ListParticipants(ctx, s.topicFormatter.RoomTopic(ctx, livekit.RoomName(req.Room)), req) + } else if store, ok := s.roomStore.(OSSServiceStore); ok { + var participants []*livekit.ParticipantInfo + participants, err = store.ListParticipants(ctx, livekit.RoomName(req.Room)) + if err == nil { + res = &livekit.ListParticipantsResponse{ + Participants: participants, + } + } + } else { + err = psrpc.ErrUnimplemented + } + if err != nil { return nil, err } - res := &livekit.ListParticipantsResponse{ - Participants: participants, - } RecordResponse(ctx, res) return res, nil } -func (s *RoomService) GetParticipant(ctx context.Context, req *livekit.RoomParticipantIdentity) (*livekit.ParticipantInfo, error) { +func (s *RoomService) GetParticipant(ctx context.Context, req *livekit.RoomParticipantIdentity) (participant *livekit.ParticipantInfo, err error) { RecordRequest(ctx, req) AppendLogFields(ctx, "room", req.Room, "participant", req.Identity) @@ -177,7 +192,14 @@ func (s *RoomService) GetParticipant(ctx context.Context, req *livekit.RoomParti return nil, twirpAuthError(err) } - participant, err := s.roomStore.LoadParticipant(ctx, livekit.RoomName(req.Room), livekit.ParticipantIdentity(req.Identity)) + if s.apiConf.EnablePsrpcForGetListParticpants { + participant, err = s.roomClient.GetParticipant(ctx, s.topicFormatter.RoomTopic(ctx, livekit.RoomName(req.Room)), req) + } else if store, ok := s.roomStore.(OSSServiceStore); ok { + participant, err = store.LoadParticipant(ctx, livekit.RoomName(req.Room), livekit.ParticipantIdentity(req.Identity)) + } else { + err = psrpc.ErrUnimplemented + } + if err != nil { return nil, err } @@ -195,8 +217,13 @@ func (s *RoomService) RemoveParticipant(ctx context.Context, req *livekit.RoomPa return nil, twirpAuthError(err) } - if _, err := s.roomStore.LoadParticipant(ctx, livekit.RoomName(req.Room), livekit.ParticipantIdentity(req.Identity)); err == ErrParticipantNotFound { - return nil, twirp.NotFoundError("participant not found") + if os, ok := s.roomStore.(OSSServiceStore); ok { + found, err := os.HasParticipant(ctx, livekit.RoomName(req.Room), livekit.ParticipantIdentity(req.Identity)) + if err != nil { + return nil, err + } else if !found { + return nil, ErrParticipantNotFound + } } res, err := s.participantClient.RemoveParticipant(ctx, s.topicFormatter.ParticipantTopic(ctx, livekit.RoomName(req.Room), livekit.ParticipantIdentity(req.Identity)), req) @@ -218,7 +245,7 @@ func (s *RoomService) MutePublishedTrack(ctx context.Context, req *livekit.MuteR } func (s *RoomService) UpdateParticipant(ctx context.Context, req *livekit.UpdateParticipantRequest) (*livekit.ParticipantInfo, error) { - RecordRequest(ctx, redactUpdateParticipantRequest(req)) + RecordRequest(ctx, req) AppendLogFields(ctx, "room", req.Room, "participant", req.Identity) @@ -238,6 +265,15 @@ func (s *RoomService) UpdateParticipant(ctx context.Context, req *livekit.Update return nil, twirpAuthError(err) } + if os, ok := s.roomStore.(OSSServiceStore); ok { + found, err := os.HasParticipant(ctx, livekit.RoomName(req.Room), livekit.ParticipantIdentity(req.Identity)) + if err != nil { + return nil, err + } else if !found { + return nil, ErrParticipantNotFound + } + } + res, err := s.participantClient.UpdateParticipant(ctx, s.topicFormatter.ParticipantTopic(ctx, livekit.RoomName(req.Room), livekit.ParticipantIdentity(req.Identity)), req) RecordResponse(ctx, res) return res, err @@ -262,7 +298,7 @@ func (s *RoomService) UpdateSubscriptions(ctx context.Context, req *livekit.Upda } func (s *RoomService) SendData(ctx context.Context, req *livekit.SendDataRequest) (*livekit.SendDataResponse, error) { - RecordRequest(ctx, redactSendDataRequest(req)) + RecordRequest(ctx, req) roomName := livekit.RoomName(req.Room) AppendLogFields(ctx, "room", roomName, "size", len(req.Data)) @@ -281,7 +317,7 @@ func (s *RoomService) SendData(ctx context.Context, req *livekit.SendDataRequest } func (s *RoomService) UpdateRoomMetadata(ctx context.Context, req *livekit.UpdateRoomMetadataRequest) (*livekit.Room, error) { - RecordRequest(ctx, redactUpdateRoomMetadataRequest(req)) + RecordRequest(ctx, req) AppendLogFields(ctx, "room", req.Room, "size", len(req.Metadata)) maxMetadataSize := int(s.limitConf.MaxMetadataSize) @@ -293,9 +329,11 @@ func (s *RoomService) UpdateRoomMetadata(ctx context.Context, req *livekit.Updat return nil, twirpAuthError(err) } - _, _, err := s.roomStore.LoadRoom(ctx, livekit.RoomName(req.Room), false) + exists, err := s.roomStore.RoomExists(ctx, livekit.RoomName(req.Room)) if err != nil { return nil, err + } else if !exists { + return nil, ErrRoomNotFound } room, err := s.roomClient.UpdateRoomMetadata(ctx, s.topicFormatter.RoomTopic(ctx, livekit.RoomName(req.Room)), req) @@ -307,86 +345,56 @@ func (s *RoomService) UpdateRoomMetadata(ctx context.Context, req *livekit.Updat return room, nil } -func redactCreateRoomRequest(req *livekit.CreateRoomRequest) *livekit.CreateRoomRequest { - if req.Egress == nil && req.Metadata == "" { - // nothing to redact - return req +func (s *RoomService) ForwardParticipant(ctx context.Context, req *livekit.ForwardParticipantRequest) (*livekit.ForwardParticipantResponse, error) { + RecordRequest(ctx, req) + + roomName := livekit.RoomName(req.Room) + AppendLogFields(ctx, "room", roomName, "participant", req.Identity) + if err := EnsureDestRoomPermission(ctx, roomName, livekit.RoomName(req.DestinationRoom)); err != nil { + return nil, twirpAuthError(err) } - clone := utils.CloneProto(req) - - if clone.Egress != nil { - if clone.Egress.Room != nil { - egress.RedactEncodedOutputs(clone.Egress.Room) - } - if clone.Egress.Participant != nil { - egress.RedactAutoEncodedOutput(clone.Egress.Participant) - } - if clone.Egress.Tracks != nil { - egress.RedactUpload(clone.Egress.Tracks) - } + if req.Room == req.DestinationRoom { + return nil, twirp.InvalidArgumentError(ErrDestinationSameAsSourceRoom.Error(), "") } - // replace with size of metadata to provide visibility on request size - if clone.Metadata != "" { - clone.Metadata = fmt.Sprintf("__size: %d", len(clone.Metadata)) - } - - return clone + res, err := s.participantClient.ForwardParticipant(ctx, s.topicFormatter.ParticipantTopic(ctx, livekit.RoomName(req.Room), livekit.ParticipantIdentity(req.Identity)), req) + RecordResponse(ctx, res) + return res, err } -func redactUpdateParticipantRequest(req *livekit.UpdateParticipantRequest) *livekit.UpdateParticipantRequest { - if req.Metadata == "" && len(req.Attributes) == 0 { - return req +func (s *RoomService) MoveParticipant(ctx context.Context, req *livekit.MoveParticipantRequest) (*livekit.MoveParticipantResponse, error) { + RecordRequest(ctx, req) + + roomName := livekit.RoomName(req.Room) + AppendLogFields(ctx, "room", roomName, "participant", req.Identity) + if err := EnsureDestRoomPermission(ctx, roomName, livekit.RoomName(req.DestinationRoom)); err != nil { + return nil, twirpAuthError(err) } - clone := utils.CloneProto(req) - - // replace with size of metadata/attributes to provide visibility on request size - if clone.Metadata != "" { - clone.Metadata = fmt.Sprintf("__size: %d", len(clone.Metadata)) + if req.Room == req.DestinationRoom { + return nil, twirp.InvalidArgumentError(ErrDestinationSameAsSourceRoom.Error(), "") } - if len(clone.Attributes) != 0 { - keysSize := 0 - valuesSize := 0 - for k, v := range clone.Attributes { - keysSize += len(k) - valuesSize += len(v) - } - - clone.Attributes = map[string]string{ - "__num_elements": fmt.Sprintf("%d", len(clone.Attributes)), - "__keys_size": fmt.Sprintf("%d", keysSize), - "__values_size": fmt.Sprintf("%d", valuesSize), - } - } - - return clone + res, err := s.participantClient.MoveParticipant(ctx, s.topicFormatter.ParticipantTopic(ctx, livekit.RoomName(req.Room), livekit.ParticipantIdentity(req.Identity)), req) + RecordResponse(ctx, res) + return res, err } -func redactSendDataRequest(req *livekit.SendDataRequest) *livekit.SendDataRequest { - if len(req.Data) == 0 { - return req +func (s *RoomService) PerformRpc(ctx context.Context, req *livekit.PerformRpcRequest) (*livekit.PerformRpcResponse, error) { + RecordRequest(ctx, req) + + roomName := livekit.RoomName(req.Room) + AppendLogFields(ctx, "room", roomName, "participant", req.DestinationIdentity) + + if err := EnsureAdminPermission(ctx, roomName); err != nil { + return nil, twirpAuthError(err) + } + if req.DestinationIdentity == "" { + return nil, ErrDestinationIdentityRequired } - clone := utils.CloneProto(req) - - // replace with size of data to provide visibility on request size - clone.Data = []byte(fmt.Sprintf("__size: %d", len(clone.Data))) - - return clone -} - -func redactUpdateRoomMetadataRequest(req *livekit.UpdateRoomMetadataRequest) *livekit.UpdateRoomMetadataRequest { - if req.Metadata == "" { - return req - } - - clone := utils.CloneProto(req) - - // replace with size of metadata to provide visibility on request size - clone.Metadata = fmt.Sprintf("__size: %d", len(clone.Metadata)) - - return clone + res, err := s.participantClient.PerformRpc(ctx, s.topicFormatter.ParticipantTopic(ctx, roomName, livekit.ParticipantIdentity(req.DestinationIdentity)), req) + RecordResponse(ctx, res) + return res, err } diff --git a/pkg/service/rtcservice.go b/pkg/service/rtcservice.go index a47f670..1b61718 100644 --- a/pkg/service/rtcservice.go +++ b/pkg/service/rtcservice.go @@ -16,8 +16,11 @@ package service import ( "context" + "encoding/base64" + "encoding/json" "errors" "fmt" + "maps" "math/rand" "net/http" "os" @@ -26,31 +29,28 @@ import ( "time" "github.com/gorilla/websocket" - "github.com/ua-parser/uap-go/uaparser" "go.uber.org/atomic" - "golang.org/x/exp/maps" + "google.golang.org/protobuf/proto" + + "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/logger" + "github.com/livekit/psrpc" "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/livekit-server/pkg/routing" - "github.com/livekit/livekit-server/pkg/routing/selector" "github.com/livekit/livekit-server/pkg/rtc" "github.com/livekit/livekit-server/pkg/telemetry" "github.com/livekit/livekit-server/pkg/telemetry/prometheus" "github.com/livekit/livekit-server/pkg/utils" - "github.com/livekit/protocol/livekit" - "github.com/livekit/psrpc" ) type RTCService struct { router routing.MessageRouter roomAllocator RoomAllocator - store ServiceStore upgrader websocket.Upgrader - currentNode routing.LocalNode config *config.Config isDev bool limits config.LimitConfig - parser *uaparser.Parser telemetry telemetry.TelemetryService mu sync.Mutex @@ -60,20 +60,15 @@ type RTCService struct { func NewRTCService( conf *config.Config, ra RoomAllocator, - store ServiceStore, router routing.MessageRouter, - currentNode routing.LocalNode, telemetry telemetry.TelemetryService, ) *RTCService { s := &RTCService{ router: router, roomAllocator: ra, - store: store, - currentNode: currentNode, config: conf, isDev: conf.Development, limits: conf.Limit, - parser: uaparser.NewFromSaved(), telemetry: telemetry, connections: map[*websocket.Conn]struct{}{}, } @@ -92,145 +87,294 @@ func NewRTCService( } func (s *RTCService) SetupRoutes(mux *http.ServeMux) { - mux.HandleFunc("/rtc/validate", s.validate) + mux.HandleFunc("/rtc", s.v0) + mux.HandleFunc("/rtc/validate", s.v0Validate) + mux.HandleFunc("/rtc/v1", s.v1) + mux.HandleFunc("/rtc/v1/validate", s.v1Validate) } -func (s *RTCService) validate(w http.ResponseWriter, r *http.Request) { - _, _, code, err := s.validateInternal(r) +func (s *RTCService) v0Validate(w http.ResponseWriter, r *http.Request) { + lgr := utils.GetLogger(r.Context()) + _, _, code, err := s.validateInternal(lgr, r, false, true) if err != nil { - handleError(w, r, code, err) + HandleError(w, r, code, err) return } _, _ = w.Write([]byte("success")) } -func (s *RTCService) validateInternal(r *http.Request) (livekit.RoomName, routing.ParticipantInit, int, error) { - claims := GetGrants(r.Context()) - var pi routing.ParticipantInit - - // require a claim - if claims == nil || claims.Video == nil { - return "", pi, http.StatusUnauthorized, rtc.ErrPermissionDenied - } - - onlyName, err := EnsureJoinPermission(r.Context()) +func (s *RTCService) v1Validate(w http.ResponseWriter, r *http.Request) { + lgr := utils.GetLogger(r.Context()) + _, _, code, err := s.validateInternal(lgr, r, true, true) if err != nil { - return "", pi, http.StatusUnauthorized, err + HandleError(w, r, code, err) + return } - - if claims.Identity == "" { - return "", pi, http.StatusBadRequest, ErrIdentityEmpty - } - if limit := s.config.Limit.MaxParticipantIdentityLength; limit > 0 && len(claims.Identity) > limit { - return "", pi, http.StatusBadRequest, fmt.Errorf("%w: max length %d", ErrParticipantIdentityExceedsLimits, limit) - } - - roomName := livekit.RoomName(r.FormValue("room")) - reconnectParam := r.FormValue("reconnect") - reconnectReason, _ := strconv.Atoi(r.FormValue("reconnect_reason")) // 0 means unknown reason - autoSubParam := r.FormValue("auto_subscribe") - publishParam := r.FormValue("publish") - adaptiveStreamParam := r.FormValue("adaptive_stream") - participantID := r.FormValue("sid") - subscriberAllowPauseParam := r.FormValue("subscriber_allow_pause") - disableICELite := r.FormValue("disable_ice_lite") - - if onlyName != "" { - roomName = onlyName - } - if limit := s.config.Limit.MaxRoomNameLength; limit > 0 && len(roomName) > limit { - return "", pi, http.StatusBadRequest, fmt.Errorf("%w: max length %d", ErrRoomNameExceedsLimits, limit) - } - - // this is new connection for existing participant - with publish only permissions - if publishParam != "" { - // Make sure grant has GetCanPublish set, - if !claims.Video.GetCanPublish() { - return "", routing.ParticipantInit{}, http.StatusUnauthorized, rtc.ErrPermissionDenied - } - // Make sure by default subscribe is off - claims.Video.SetCanSubscribe(false) - claims.Identity += "#" + publishParam - } - - // room allocator validations - err = s.roomAllocator.ValidateCreateRoom(r.Context(), roomName) - if err != nil { - if errors.Is(err, ErrRoomNotFound) { - return "", pi, http.StatusNotFound, err - } else { - return "", pi, http.StatusInternalServerError, err - } - } - - region := "" - if router, ok := s.router.(routing.Router); ok { - region = router.GetRegion() - if foundNode, err := router.GetNodeForRoom(r.Context(), roomName); err == nil { - if selector.LimitsReached(s.limits, foundNode.Stats) { - return "", pi, http.StatusServiceUnavailable, rtc.ErrLimitExceeded - } - } - } - - createRequest := &livekit.CreateRoomRequest{ - Name: string(roomName), - RoomPreset: claims.RoomPreset, - } - SetRoomConfiguration(createRequest, claims.GetRoomConfiguration()) - - pi = routing.ParticipantInit{ - Reconnect: boolValue(reconnectParam), - ReconnectReason: livekit.ReconnectReason(reconnectReason), - Identity: livekit.ParticipantIdentity(claims.Identity), - Name: livekit.ParticipantName(claims.Name), - AutoSubscribe: true, - Client: s.ParseClientInfo(r), - Grants: claims, - Region: region, - CreateRoom: createRequest, - } - if pi.Reconnect { - pi.ID = livekit.ParticipantID(participantID) - } - - if autoSubParam != "" { - pi.AutoSubscribe = boolValue(autoSubParam) - } - if adaptiveStreamParam != "" { - pi.AdaptiveStream = boolValue(adaptiveStreamParam) - } - if subscriberAllowPauseParam != "" { - subscriberAllowPause := boolValue(subscriberAllowPauseParam) - pi.SubscriberAllowPause = &subscriberAllowPause - } - if disableICELite != "" { - pi.DisableICELite = boolValue(disableICELite) - } - - return roomName, pi, http.StatusOK, nil + _, _ = w.Write([]byte("success")) } -func (s *RTCService) ServeHTTP(w http.ResponseWriter, r *http.Request) { +func decodeAttributes(str string) (map[string]string, error) { + data, err := base64.URLEncoding.DecodeString(str) + if err != nil { + return nil, err + } + var attrs map[string]string + if err := json.Unmarshal(data, &attrs); err != nil { + return nil, err + } + return attrs, nil +} + +var errJoinRequestTooLarge = errors.New("join request too large") + +func (s *RTCService) validateInternal( + lgr logger.Logger, + r *http.Request, + needsJoinRequest bool, + strict bool, +) (livekit.RoomName, routing.ParticipantInit, int, error) { + if claims := GetGrants(r.Context()); claims == nil || claims.Video == nil { + return "", routing.ParticipantInit{}, http.StatusUnauthorized, rtc.ErrPermissionDenied + } + + var params ValidateConnectRequestParams + useSinglePeerConnection := false + joinRequest := &livekit.JoinRequest{} + + wrappedJoinRequestBase64 := r.FormValue("join_request") + if wrappedJoinRequestBase64 == "" { + if needsJoinRequest { + return "", routing.ParticipantInit{}, http.StatusBadRequest, errors.New("join_request is required") + } + + params.publish = r.FormValue("publish") + + attributesStrParam := r.FormValue("attributes") + if attributesStrParam != "" { + attrs, err := decodeAttributes(attributesStrParam) + if err != nil { + if strict { + return "", routing.ParticipantInit{}, http.StatusBadRequest, errors.New("cannot decode attributes") + } + lgr.Debugw("failed to decode attributes", "error", err) + // attrs will be empty here, so just proceed + } + params.attributes = attrs + } + } else { + useSinglePeerConnection = true + if wrappedProtoBytes, err := base64.URLEncoding.DecodeString(wrappedJoinRequestBase64); err != nil { + return "", routing.ParticipantInit{}, http.StatusBadRequest, errors.New("cannot base64 decode wrapped join request") + } else { + wrappedJoinRequest := &livekit.WrappedJoinRequest{} + if err := proto.Unmarshal(wrappedProtoBytes, wrappedJoinRequest); err != nil { + return "", routing.ParticipantInit{}, http.StatusBadRequest, errors.New("cannot unmarshal wrapped join request") + } + + switch wrappedJoinRequest.Compression { + case livekit.WrappedJoinRequest_NONE: + if len(wrappedJoinRequest.JoinRequest) > http.DefaultMaxHeaderBytes { + return "", routing.ParticipantInit{}, http.StatusBadRequest, errJoinRequestTooLarge + } + if err := proto.Unmarshal(wrappedJoinRequest.JoinRequest, joinRequest); err != nil { + return "", routing.ParticipantInit{}, http.StatusBadRequest, errors.New("cannot unmarshal join request") + } + + case livekit.WrappedJoinRequest_GZIP: + protoBytes, err := DecompressGzip(wrappedJoinRequest.JoinRequest) + if err != nil { + switch { + case errors.Is(err, ErrGzipTooLarge): + err = errJoinRequestTooLarge + case errors.Is(err, ErrGzipReadFailed): + err = errors.New("cannot read decompressed join request") + } + return "", routing.ParticipantInit{}, http.StatusBadRequest, err + } + + if err := proto.Unmarshal(protoBytes, joinRequest); err != nil { + return "", routing.ParticipantInit{}, http.StatusBadRequest, errors.New("cannot unmarshal join request") + } + } + + params.metadata = joinRequest.Metadata + params.attributes = joinRequest.ParticipantAttributes + } + } + + res, code, err := ValidateConnectRequest( + lgr, + r, + s.limits, + params, + s.router, + s.roomAllocator, + ) + if err != nil { + return res.roomName, routing.ParticipantInit{}, code, err + } + + pi := routing.ParticipantInit{ + Identity: livekit.ParticipantIdentity(res.grants.Identity), + Name: livekit.ParticipantName(res.grants.Name), + Grants: res.grants, + Region: res.region, + CreateRoom: res.createRoomRequest, + UseSinglePeerConnection: useSinglePeerConnection, + } + + if wrappedJoinRequestBase64 == "" { + pi.Reconnect = boolValue(r.FormValue("reconnect")) + pi.Client = ParseClientInfo(r) + + pi.AutoSubscribe = true + if autoSubscribeParam := r.FormValue("auto_subscribe"); autoSubscribeParam != "" { + pi.AutoSubscribe = boolValue(autoSubscribeParam) + } + + if autoSubscribeDataTrackParam := r.FormValue("auto_subscribe_data_track"); autoSubscribeDataTrackParam != "" { + autoSubscribeDataTrack := boolValue(autoSubscribeDataTrackParam) + pi.AutoSubscribeDataTrack = &autoSubscribeDataTrack + } + + pi.AdaptiveStream = boolValue(r.FormValue("adaptive_stream")) + pi.DisableICELite = boolValue(r.FormValue("disable_ice_lite")) + + reconnectReason, _ := strconv.Atoi(r.FormValue("reconnect_reason")) // 0 means unknown reason + pi.ReconnectReason = livekit.ReconnectReason(reconnectReason) + + if pi.Reconnect { + pi.ID = livekit.ParticipantID(r.FormValue("sid")) + } + + if subscriberAllowPauseParam := r.FormValue("subscriber_allow_pause"); subscriberAllowPauseParam != "" { + subscriberAllowPause := boolValue(subscriberAllowPauseParam) + pi.SubscriberAllowPause = &subscriberAllowPause + } + } else { + lgr.Debugw("processing join request", "joinRequest", logger.Proto(joinRequest)) + + if joinRequest.ClientInfo == nil { + joinRequest.ClientInfo = &livekit.ClientInfo{} + } + AugmentClientInfo(joinRequest.ClientInfo, r) + pi.Client = joinRequest.ClientInfo + + pi.AutoSubscribe = joinRequest.GetConnectionSettings().GetAutoSubscribe() + + autoSubscribeDataTrack := joinRequest.GetConnectionSettings().GetAutoSubscribeDataTrack() + pi.AutoSubscribeDataTrack = &autoSubscribeDataTrack + + pi.AdaptiveStream = joinRequest.GetConnectionSettings().GetAdaptiveStream() + pi.DisableICELite = joinRequest.GetConnectionSettings().GetDisableIceLite() + + subscriberAllowPause := joinRequest.GetConnectionSettings().GetSubscriberAllowPause() + pi.SubscriberAllowPause = &subscriberAllowPause + + pi.AddTrackRequests = joinRequest.AddTrackRequests + pi.PublisherOffer = joinRequest.PublisherOffer + + pi.Reconnect = joinRequest.Reconnect + pi.ReconnectReason = joinRequest.ReconnectReason + pi.ID = livekit.ParticipantID(joinRequest.ParticipantSid) + } + + return res.roomName, pi, code, err +} + +func (s *RTCService) v0(w http.ResponseWriter, r *http.Request) { + s.serve(w, r, false) +} + +func (s *RTCService) v1(w http.ResponseWriter, r *http.Request) { + s.serve(w, r, true) +} + +func (s *RTCService) serve(w http.ResponseWriter, r *http.Request, needsJoinRequest bool) { // reject non websocket requests if !websocket.IsWebSocketUpgrade(r) { w.WriteHeader(404) return } - roomName, pi, code, err := s.validateInternal(r) + startedAt := time.Now() + var ( + roomName livekit.RoomName + roomID livekit.RoomID + participantIdentity livekit.ParticipantIdentity + pID livekit.ParticipantID + joinDuration time.Duration + loggerResolved bool + + pi routing.ParticipantInit + code int + err error + ) + + pLogger, loggerResolver := utils.GetLogger(r.Context()).WithDeferredValues() + + getLoggerFields := func() []any { + return []any{ + "room", roomName, + "roomID", roomID, + "participant", participantIdentity, + "participantID", pID, + "joinDuration", joinDuration, + } + } + + resolveLogger := func(force bool) { + if loggerResolved { + return + } + + if force { + if roomName == "" { + roomName = "unresolved" + } + if roomID == "" { + roomID = "unresolved" + } + if participantIdentity == "" { + participantIdentity = "unresolved" + } + if pID == "" { + pID = "unresolved" + } + if joinDuration == 0 { + joinDuration = time.Since(startedAt) + } + } + + if roomName != "" && roomID != "" && participantIdentity != "" && pID != "" { + loggerResolved = true + loggerResolver.Resolve(getLoggerFields()...) + } + } + + resetLogger := func() { + loggerResolver.Reset() + + roomName = "" + roomID = "" + participantIdentity = "" + pID = "" + loggerResolved = false + } + + roomName, pi, code, err = s.validateInternal(pLogger, r, needsJoinRequest, false) if err != nil { - handleError(w, r, code, err) + prometheus.IncrementParticipantJoinValidationFail(1) + resolveLogger(true) + HandleError(w, r, code, err, getLoggerFields()...) return } - loggerFields := []any{ - "participant", pi.Identity, - "pID", pi.ID, - "room", roomName, - "remote", false, + participantIdentity = pi.Identity + if pi.ID != "" { + pID = pi.ID } - pLogger := utils.GetLogger(r.Context()).WithValues(loggerFields...) + pLogger.Debugw("join request validated", append(getLoggerFields(), "participantInit", &pi)...) // give it a few attempts to start session var cr connectionResult @@ -251,17 +395,28 @@ func (s *RTCService) ServeHTTP(w http.ResponseWriter, r *http.Request) { if errors.As(err, &psrpcErr) { status = psrpcErr.ToHttp() } - handleError(w, r, status, err, loggerFields...) + resolveLogger(true) + HandleError(w, r, status, err, getLoggerFields()...) return } prometheus.IncrementParticipantJoin(1) + joinDuration = time.Since(startedAt) + pLogger = pLogger.WithValues("connID", cr.ConnectionID) if !pi.Reconnect && initialResponse.GetJoin() != nil { + joinRoomID := livekit.RoomID(initialResponse.GetJoin().GetRoom().GetSid()) + if joinRoomID != "" { + roomID = joinRoomID + } + pi.ID = livekit.ParticipantID(initialResponse.GetJoin().GetParticipant().GetSid()) + pID = pi.ID + + resolveLogger(false) } - signalStats := telemetry.NewBytesSignalStats(r.Context(), s.telemetry) + signalStats := rtc.NewBytesSignalStats(r.Context(), s.telemetry) if join := initialResponse.GetJoin(); join != nil { signalStats.ResolveRoom(join.GetRoom()) signalStats.ResolveParticipant(join.GetParticipant()) @@ -277,10 +432,8 @@ func (s *RTCService) ServeHTTP(w http.ResponseWriter, r *http.Request) { done := make(chan struct{}) // function exits when websocket terminates, it'll close the event reading off of request sink and response source as well defer func() { - pLogger.Debugw("finishing WS connection", - "connID", cr.ConnectionID, - "closedByClient", closedByClient.Load(), - ) + resolveLogger(true) + pLogger.Debugw("finishing WS connection", "closedByClient", closedByClient.Load()) cr.ResponseSource.Close() cr.RequestSink.Close() close(done) @@ -291,7 +444,8 @@ func (s *RTCService) ServeHTTP(w http.ResponseWriter, r *http.Request) { // upgrade only once the basics are good to go conn, err := s.upgrader.Upgrade(w, r, nil) if err != nil { - handleError(w, r, http.StatusInternalServerError, err, loggerFields...) + resolveLogger(true) + HandleError(w, r, http.StatusInternalServerError, err, getLoggerFields()...) return } @@ -307,15 +461,17 @@ func (s *RTCService) ServeHTTP(w http.ResponseWriter, r *http.Request) { // websocket established sigConn := NewWSSignalConnection(conn) + pLogger.Debugw("sending initial response", "response", logger.Proto(initialResponse)) count, err := sigConn.WriteResponse(initialResponse) if err != nil { + resolveLogger(true) pLogger.Warnw("could not write initial response", err) return } signalStats.AddBytes(uint64(count), true) - pLogger.Debugw("new client WS connected", - "connID", cr.ConnectionID, + pLogger.Debugw( + "new client WS connected", "reconnect", pi.Reconnect, "reconnectReason", pi.ReconnectReason, "adaptiveStream", pi.AdaptiveStream, @@ -343,7 +499,8 @@ func (s *RTCService) ServeHTTP(w http.ResponseWriter, r *http.Request) { return case msg := <-cr.ResponseSource.ReadChan(): if msg == nil { - pLogger.Debugw("nothing to read from response source", "connID", cr.ConnectionID) + resolveLogger(true) + pLogger.Debugw("nothing to read from response source") return } res, ok := msg.(*livekit.SignalResponse) @@ -351,7 +508,6 @@ func (s *RTCService) ServeHTTP(w http.ResponseWriter, r *http.Request) { pLogger.Errorw( "unexpected message type", nil, "type", fmt.Sprintf("%T", msg), - "connID", cr.ConnectionID, ) continue } @@ -359,17 +515,46 @@ func (s *RTCService) ServeHTTP(w http.ResponseWriter, r *http.Request) { switch m := res.Message.(type) { case *livekit.SignalResponse_Offer: pLogger.Debugw("sending offer", "offer", m) + case *livekit.SignalResponse_Answer: pLogger.Debugw("sending answer", "answer", m) + case *livekit.SignalResponse_Join: pLogger.Debugw("sending join", "join", m) signalStats.ResolveRoom(m.Join.GetRoom()) signalStats.ResolveParticipant(m.Join.GetParticipant()) + case *livekit.SignalResponse_RoomUpdate: + updateRoomID := livekit.RoomID(m.RoomUpdate.GetRoom().GetSid()) + if updateRoomID != "" { + roomID = updateRoomID + resolveLogger(false) + } pLogger.Debugw("sending room update", "roomUpdate", m) signalStats.ResolveRoom(m.RoomUpdate.GetRoom()) + case *livekit.SignalResponse_Update: pLogger.Debugw("sending participant update", "participantUpdate", m) + + case *livekit.SignalResponse_RoomMoved: + resetLogger() + signalStats.Reset() + + roomName = livekit.RoomName(m.RoomMoved.GetRoom().GetName()) + moveRoomID := livekit.RoomID(m.RoomMoved.GetRoom().GetSid()) + if moveRoomID != "" { + roomID = moveRoomID + } + participantIdentity = livekit.ParticipantIdentity(m.RoomMoved.GetParticipant().GetIdentity()) + pID = livekit.ParticipantID(m.RoomMoved.GetParticipant().GetSid()) + resolveLogger(false) + + signalStats.ResolveRoom(m.RoomMoved.GetRoom()) + signalStats.ResolveParticipant(m.RoomMoved.GetParticipant()) + pLogger.Debugw("sending room moved", "roomMoved", m) + + default: + pLogger.Debugw("sending signal response", "response", m) } if count, err := sigConn.WriteResponse(res); err != nil { @@ -389,7 +574,7 @@ func (s *RTCService) ServeHTTP(w http.ResponseWriter, r *http.Request) { if IsWebSocketCloseError(err) { closedByClient.Store(true) } else { - pLogger.Errorw("error reading from websocket", err, "connID", cr.ConnectionID) + pLogger.Errorw("error reading from websocket", err) } return } @@ -428,86 +613,17 @@ func (s *RTCService) ServeHTTP(w http.ResponseWriter, r *http.Request) { pLogger.Debugw("received offer", "offer", m) case *livekit.SignalRequest_Answer: pLogger.Debugw("received answer", "answer", m) + default: + pLogger.Debugw("received signal request", "request", m) } if err := cr.RequestSink.WriteMessage(req); err != nil { - pLogger.Warnw("error writing to request sink", err, "connID", cr.ConnectionID) + pLogger.Warnw("error writing to request sink", err) return } } } -func (s *RTCService) ParseClientInfo(r *http.Request) *livekit.ClientInfo { - values := r.Form - ci := &livekit.ClientInfo{} - if pv, err := strconv.Atoi(values.Get("protocol")); err == nil { - ci.Protocol = int32(pv) - } - sdkString := values.Get("sdk") - switch sdkString { - case "js": - ci.Sdk = livekit.ClientInfo_JS - case "ios", "swift": - ci.Sdk = livekit.ClientInfo_SWIFT - case "android": - ci.Sdk = livekit.ClientInfo_ANDROID - case "flutter": - ci.Sdk = livekit.ClientInfo_FLUTTER - case "go": - ci.Sdk = livekit.ClientInfo_GO - case "unity": - ci.Sdk = livekit.ClientInfo_UNITY - case "reactnative": - ci.Sdk = livekit.ClientInfo_REACT_NATIVE - case "rust": - ci.Sdk = livekit.ClientInfo_RUST - case "python": - ci.Sdk = livekit.ClientInfo_PYTHON - case "cpp": - ci.Sdk = livekit.ClientInfo_CPP - case "unityweb": - ci.Sdk = livekit.ClientInfo_UNITY_WEB - case "node": - ci.Sdk = livekit.ClientInfo_NODE - } - - ci.Version = values.Get("version") - ci.Os = values.Get("os") - ci.OsVersion = values.Get("os_version") - ci.Browser = values.Get("browser") - ci.BrowserVersion = values.Get("browser_version") - ci.DeviceModel = values.Get("device_model") - ci.Network = values.Get("network") - // get real address (forwarded http header) - check Cloudflare headers first, fall back to X-Forwarded-For - ci.Address = GetClientIP(r) - - // attempt to parse types for SDKs that support browser as a platform - if ci.Sdk == livekit.ClientInfo_JS || - ci.Sdk == livekit.ClientInfo_REACT_NATIVE || - ci.Sdk == livekit.ClientInfo_FLUTTER || - ci.Sdk == livekit.ClientInfo_UNITY { - client := s.parser.Parse(r.UserAgent()) - if ci.Browser == "" { - ci.Browser = client.UserAgent.Family - ci.BrowserVersion = client.UserAgent.ToVersionString() - } - if ci.Os == "" { - ci.Os = client.Os.Family - ci.OsVersion = client.Os.ToVersionString() - } - if ci.DeviceModel == "" { - model := client.Device.Family - if model != "" && client.Device.Model != "" && model != client.Device.Model { - model += " " + client.Device.Model - } - - ci.DeviceModel = model - } - } - - return ci -} - func (s *RTCService) DrainConnections(interval time.Duration) { s.mu.Lock() conns := maps.Clone(s.connections) diff --git a/pkg/service/server.go b/pkg/service/server.go index d3f92d4..4ca8561 100644 --- a/pkg/service/server.go +++ b/pkg/service/server.go @@ -27,7 +27,7 @@ import ( "strconv" "time" - "github.com/pion/turn/v4" + "github.com/pion/turn/v5" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/rs/cors" "github.com/twitchtv/twirp" @@ -49,6 +49,7 @@ type LivekitServer struct { config *config.Config ioService *IOInfoService rtcService *RTCService + whipService *WHIPService agentService *AgentService httpServer *http.Server promServer *http.Server @@ -70,6 +71,7 @@ func NewLivekitServer(conf *config.Config, sipService *SIPService, ioService *IOInfoService, rtcService *RTCService, + whipService *WHIPService, agentService *AgentService, keyProvider auth.KeyProvider, router routing.Router, @@ -82,6 +84,7 @@ func NewLivekitServer(conf *config.Config, config: conf, ioService: ioService, rtcService: rtcService, + whipService: whipService, agentService: agentService, router: router, roomManager: roomManager, @@ -100,7 +103,9 @@ func NewLivekitServer(conf *config.Config, AllowOriginFunc: func(origin string) bool { return true }, + AllowedMethods: []string{"OPTIONS", "HEAD", "GET", "POST", "PATCH", "DELETE"}, AllowedHeaders: []string{"*"}, + ExposedHeaders: []string{"*"}, // allow preflight to be cached for a day MaxAge: 86400, }), @@ -110,9 +115,10 @@ func NewLivekitServer(conf *config.Config, middlewares = append(middlewares, NewAPIKeyAuthMiddleware(keyProvider)) } - serverOptions := []interface{}{ + serverOptions := []any{ twirp.WithServerHooks(twirp.ChainHooks( TwirpLogger(), + TwirpEgressID(), TwirpRequestStatusReporter(), )), } @@ -138,8 +144,8 @@ func NewLivekitServer(conf *config.Config, xtwirp.RegisterServer(mux, egressServer) xtwirp.RegisterServer(mux, ingressServer) xtwirp.RegisterServer(mux, sipServer) - mux.Handle("/rtc", rtcService) rtcService.SetupRoutes(mux) + whipService.SetupRoutes(mux) mux.Handle("/agent", agentService) mux.HandleFunc("/", s.defaultHandler) @@ -231,7 +237,7 @@ func (s *LivekitServer) Start() error { } } - values := []interface{}{ + values := []any{ "portHttp", s.config.Port, "nodeID", s.currentNode.NodeID(), "nodeIP", s.currentNode.NodeIP(), @@ -342,7 +348,7 @@ func (s *LivekitServer) debugGoroutines(w http.ResponseWriter, _ *http.Request) func (s *LivekitServer) debugInfo(w http.ResponseWriter, _ *http.Request) { s.roomManager.lock.RLock() - info := make([]map[string]interface{}, 0, len(s.roomManager.rooms)) + info := make([]map[string]any, 0, len(s.roomManager.rooms)) for _, room := range s.roomManager.rooms { info = append(info, room.DebugInfo()) } @@ -372,7 +378,7 @@ func (s *LivekitServer) healthCheck(w http.ResponseWriter, _ *http.Request) { } if time.Since(updatedAt) > 4*time.Second { w.WriteHeader(http.StatusNotAcceptable) - _, _ = w.Write([]byte(fmt.Sprintf("Not Ready\nNode Updated At %s", updatedAt))) + _, _ = fmt.Fprintf(w, "Not Ready\nNode Updated At %s", updatedAt) return } diff --git a/pkg/service/servicefakes/fake_agent_store.go b/pkg/service/servicefakes/fake_agent_store.go index 71d43f5..6727597 100644 --- a/pkg/service/servicefakes/fake_agent_store.go +++ b/pkg/service/servicefakes/fake_agent_store.go @@ -392,16 +392,6 @@ func (fake *FakeAgentStore) StoreAgentJobReturnsOnCall(i int, result1 error) { func (fake *FakeAgentStore) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() - fake.deleteAgentDispatchMutex.RLock() - defer fake.deleteAgentDispatchMutex.RUnlock() - fake.deleteAgentJobMutex.RLock() - defer fake.deleteAgentJobMutex.RUnlock() - fake.listAgentDispatchesMutex.RLock() - defer fake.listAgentDispatchesMutex.RUnlock() - fake.storeAgentDispatchMutex.RLock() - defer fake.storeAgentDispatchMutex.RUnlock() - fake.storeAgentJobMutex.RLock() - defer fake.storeAgentJobMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} for key, value := range fake.invocations { copiedInvocations[key] = value diff --git a/pkg/service/servicefakes/fake_egress_store.go b/pkg/service/servicefakes/fake_egress_store.go index 37f1367..b642eb9 100644 --- a/pkg/service/servicefakes/fake_egress_store.go +++ b/pkg/service/servicefakes/fake_egress_store.go @@ -325,14 +325,6 @@ func (fake *FakeEgressStore) UpdateEgressReturnsOnCall(i int, result1 error) { func (fake *FakeEgressStore) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() - fake.listEgressMutex.RLock() - defer fake.listEgressMutex.RUnlock() - fake.loadEgressMutex.RLock() - defer fake.loadEgressMutex.RUnlock() - fake.storeEgressMutex.RLock() - defer fake.storeEgressMutex.RUnlock() - fake.updateEgressMutex.RLock() - defer fake.updateEgressMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} for key, value := range fake.invocations { copiedInvocations[key] = value diff --git a/pkg/service/servicefakes/fake_ingress_store.go b/pkg/service/servicefakes/fake_ingress_store.go index 4265e1d..f959ad4 100644 --- a/pkg/service/servicefakes/fake_ingress_store.go +++ b/pkg/service/servicefakes/fake_ingress_store.go @@ -552,20 +552,6 @@ func (fake *FakeIngressStore) UpdateIngressStateReturnsOnCall(i int, result1 err func (fake *FakeIngressStore) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() - fake.deleteIngressMutex.RLock() - defer fake.deleteIngressMutex.RUnlock() - fake.listIngressMutex.RLock() - defer fake.listIngressMutex.RUnlock() - fake.loadIngressMutex.RLock() - defer fake.loadIngressMutex.RUnlock() - fake.loadIngressFromStreamKeyMutex.RLock() - defer fake.loadIngressFromStreamKeyMutex.RUnlock() - fake.storeIngressMutex.RLock() - defer fake.storeIngressMutex.RUnlock() - fake.updateIngressMutex.RLock() - defer fake.updateIngressMutex.RUnlock() - fake.updateIngressStateMutex.RLock() - defer fake.updateIngressStateMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} for key, value := range fake.invocations { copiedInvocations[key] = value diff --git a/pkg/service/servicefakes/fake_ioclient.go b/pkg/service/servicefakes/fake_ioclient.go index 0e1e22a..0cdda41 100644 --- a/pkg/service/servicefakes/fake_ioclient.go +++ b/pkg/service/servicefakes/fake_ioclient.go @@ -414,16 +414,6 @@ func (fake *FakeIOClient) UpdateIngressStateReturnsOnCall(i int, result1 *emptyp func (fake *FakeIOClient) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() - fake.createEgressMutex.RLock() - defer fake.createEgressMutex.RUnlock() - fake.createIngressMutex.RLock() - defer fake.createIngressMutex.RUnlock() - fake.getEgressMutex.RLock() - defer fake.getEgressMutex.RUnlock() - fake.listEgressMutex.RLock() - defer fake.listEgressMutex.RUnlock() - fake.updateIngressStateMutex.RLock() - defer fake.updateIngressStateMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} for key, value := range fake.invocations { copiedInvocations[key] = value diff --git a/pkg/service/servicefakes/fake_object_store.go b/pkg/service/servicefakes/fake_object_store.go index 1c5259b..24d4273 100644 --- a/pkg/service/servicefakes/fake_object_store.go +++ b/pkg/service/servicefakes/fake_object_store.go @@ -36,6 +36,21 @@ type FakeObjectStore struct { deleteRoomReturnsOnCall map[int]struct { result1 error } + HasParticipantStub func(context.Context, livekit.RoomName, livekit.ParticipantIdentity) (bool, error) + hasParticipantMutex sync.RWMutex + hasParticipantArgsForCall []struct { + arg1 context.Context + arg2 livekit.RoomName + arg3 livekit.ParticipantIdentity + } + hasParticipantReturns struct { + result1 bool + result2 error + } + hasParticipantReturnsOnCall map[int]struct { + result1 bool + result2 error + } ListParticipantsStub func(context.Context, livekit.RoomName) ([]*livekit.ParticipantInfo, error) listParticipantsMutex sync.RWMutex listParticipantsArgsForCall []struct { @@ -111,6 +126,20 @@ type FakeObjectStore struct { result1 string result2 error } + RoomExistsStub func(context.Context, livekit.RoomName) (bool, error) + roomExistsMutex sync.RWMutex + roomExistsArgsForCall []struct { + arg1 context.Context + arg2 livekit.RoomName + } + roomExistsReturns struct { + result1 bool + result2 error + } + roomExistsReturnsOnCall map[int]struct { + result1 bool + result2 error + } StoreParticipantStub func(context.Context, livekit.RoomName, *livekit.ParticipantInfo) error storeParticipantMutex sync.RWMutex storeParticipantArgsForCall []struct { @@ -279,6 +308,72 @@ func (fake *FakeObjectStore) DeleteRoomReturnsOnCall(i int, result1 error) { }{result1} } +func (fake *FakeObjectStore) HasParticipant(arg1 context.Context, arg2 livekit.RoomName, arg3 livekit.ParticipantIdentity) (bool, error) { + fake.hasParticipantMutex.Lock() + ret, specificReturn := fake.hasParticipantReturnsOnCall[len(fake.hasParticipantArgsForCall)] + fake.hasParticipantArgsForCall = append(fake.hasParticipantArgsForCall, struct { + arg1 context.Context + arg2 livekit.RoomName + arg3 livekit.ParticipantIdentity + }{arg1, arg2, arg3}) + stub := fake.HasParticipantStub + fakeReturns := fake.hasParticipantReturns + fake.recordInvocation("HasParticipant", []interface{}{arg1, arg2, arg3}) + fake.hasParticipantMutex.Unlock() + if stub != nil { + return stub(arg1, arg2, arg3) + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeObjectStore) HasParticipantCallCount() int { + fake.hasParticipantMutex.RLock() + defer fake.hasParticipantMutex.RUnlock() + return len(fake.hasParticipantArgsForCall) +} + +func (fake *FakeObjectStore) HasParticipantCalls(stub func(context.Context, livekit.RoomName, livekit.ParticipantIdentity) (bool, error)) { + fake.hasParticipantMutex.Lock() + defer fake.hasParticipantMutex.Unlock() + fake.HasParticipantStub = stub +} + +func (fake *FakeObjectStore) HasParticipantArgsForCall(i int) (context.Context, livekit.RoomName, livekit.ParticipantIdentity) { + fake.hasParticipantMutex.RLock() + defer fake.hasParticipantMutex.RUnlock() + argsForCall := fake.hasParticipantArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 +} + +func (fake *FakeObjectStore) HasParticipantReturns(result1 bool, result2 error) { + fake.hasParticipantMutex.Lock() + defer fake.hasParticipantMutex.Unlock() + fake.HasParticipantStub = nil + fake.hasParticipantReturns = struct { + result1 bool + result2 error + }{result1, result2} +} + +func (fake *FakeObjectStore) HasParticipantReturnsOnCall(i int, result1 bool, result2 error) { + fake.hasParticipantMutex.Lock() + defer fake.hasParticipantMutex.Unlock() + fake.HasParticipantStub = nil + if fake.hasParticipantReturnsOnCall == nil { + fake.hasParticipantReturnsOnCall = make(map[int]struct { + result1 bool + result2 error + }) + } + fake.hasParticipantReturnsOnCall[i] = struct { + result1 bool + result2 error + }{result1, result2} +} + func (fake *FakeObjectStore) ListParticipants(arg1 context.Context, arg2 livekit.RoomName) ([]*livekit.ParticipantInfo, error) { fake.listParticipantsMutex.Lock() ret, specificReturn := fake.listParticipantsReturnsOnCall[len(fake.listParticipantsArgsForCall)] @@ -615,6 +710,71 @@ func (fake *FakeObjectStore) LockRoomReturnsOnCall(i int, result1 string, result }{result1, result2} } +func (fake *FakeObjectStore) RoomExists(arg1 context.Context, arg2 livekit.RoomName) (bool, error) { + fake.roomExistsMutex.Lock() + ret, specificReturn := fake.roomExistsReturnsOnCall[len(fake.roomExistsArgsForCall)] + fake.roomExistsArgsForCall = append(fake.roomExistsArgsForCall, struct { + arg1 context.Context + arg2 livekit.RoomName + }{arg1, arg2}) + stub := fake.RoomExistsStub + fakeReturns := fake.roomExistsReturns + fake.recordInvocation("RoomExists", []interface{}{arg1, arg2}) + fake.roomExistsMutex.Unlock() + if stub != nil { + return stub(arg1, arg2) + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeObjectStore) RoomExistsCallCount() int { + fake.roomExistsMutex.RLock() + defer fake.roomExistsMutex.RUnlock() + return len(fake.roomExistsArgsForCall) +} + +func (fake *FakeObjectStore) RoomExistsCalls(stub func(context.Context, livekit.RoomName) (bool, error)) { + fake.roomExistsMutex.Lock() + defer fake.roomExistsMutex.Unlock() + fake.RoomExistsStub = stub +} + +func (fake *FakeObjectStore) RoomExistsArgsForCall(i int) (context.Context, livekit.RoomName) { + fake.roomExistsMutex.RLock() + defer fake.roomExistsMutex.RUnlock() + argsForCall := fake.roomExistsArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeObjectStore) RoomExistsReturns(result1 bool, result2 error) { + fake.roomExistsMutex.Lock() + defer fake.roomExistsMutex.Unlock() + fake.RoomExistsStub = nil + fake.roomExistsReturns = struct { + result1 bool + result2 error + }{result1, result2} +} + +func (fake *FakeObjectStore) RoomExistsReturnsOnCall(i int, result1 bool, result2 error) { + fake.roomExistsMutex.Lock() + defer fake.roomExistsMutex.Unlock() + fake.RoomExistsStub = nil + if fake.roomExistsReturnsOnCall == nil { + fake.roomExistsReturnsOnCall = make(map[int]struct { + result1 bool + result2 error + }) + } + fake.roomExistsReturnsOnCall[i] = struct { + result1 bool + result2 error + }{result1, result2} +} + func (fake *FakeObjectStore) StoreParticipant(arg1 context.Context, arg2 livekit.RoomName, arg3 *livekit.ParticipantInfo) error { fake.storeParticipantMutex.Lock() ret, specificReturn := fake.storeParticipantReturnsOnCall[len(fake.storeParticipantArgsForCall)] @@ -807,26 +967,6 @@ func (fake *FakeObjectStore) UnlockRoomReturnsOnCall(i int, result1 error) { func (fake *FakeObjectStore) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() - fake.deleteParticipantMutex.RLock() - defer fake.deleteParticipantMutex.RUnlock() - fake.deleteRoomMutex.RLock() - defer fake.deleteRoomMutex.RUnlock() - fake.listParticipantsMutex.RLock() - defer fake.listParticipantsMutex.RUnlock() - fake.listRoomsMutex.RLock() - defer fake.listRoomsMutex.RUnlock() - fake.loadParticipantMutex.RLock() - defer fake.loadParticipantMutex.RUnlock() - fake.loadRoomMutex.RLock() - defer fake.loadRoomMutex.RUnlock() - fake.lockRoomMutex.RLock() - defer fake.lockRoomMutex.RUnlock() - fake.storeParticipantMutex.RLock() - defer fake.storeParticipantMutex.RUnlock() - fake.storeRoomMutex.RLock() - defer fake.storeRoomMutex.RUnlock() - fake.unlockRoomMutex.RLock() - defer fake.unlockRoomMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} for key, value := range fake.invocations { copiedInvocations[key] = value diff --git a/pkg/service/servicefakes/fake_room_allocator.go b/pkg/service/servicefakes/fake_room_allocator.go index 8cc11f6..b15ae56 100644 --- a/pkg/service/servicefakes/fake_room_allocator.go +++ b/pkg/service/servicefakes/fake_room_allocator.go @@ -330,14 +330,6 @@ func (fake *FakeRoomAllocator) ValidateCreateRoomReturnsOnCall(i int, result1 er func (fake *FakeRoomAllocator) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() - fake.autoCreateEnabledMutex.RLock() - defer fake.autoCreateEnabledMutex.RUnlock() - fake.createRoomMutex.RLock() - defer fake.createRoomMutex.RUnlock() - fake.selectRoomNodeMutex.RLock() - defer fake.selectRoomNodeMutex.RUnlock() - fake.validateCreateRoomMutex.RLock() - defer fake.validateCreateRoomMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} for key, value := range fake.invocations { copiedInvocations[key] = value diff --git a/pkg/service/servicefakes/fake_service_store.go b/pkg/service/servicefakes/fake_service_store.go index 7ecf3e2..45a6ed5 100644 --- a/pkg/service/servicefakes/fake_service_store.go +++ b/pkg/service/servicefakes/fake_service_store.go @@ -10,32 +10,6 @@ import ( ) type FakeServiceStore struct { - DeleteRoomStub func(context.Context, livekit.RoomName) error - deleteRoomMutex sync.RWMutex - deleteRoomArgsForCall []struct { - arg1 context.Context - arg2 livekit.RoomName - } - deleteRoomReturns struct { - result1 error - } - deleteRoomReturnsOnCall map[int]struct { - result1 error - } - ListParticipantsStub func(context.Context, livekit.RoomName) ([]*livekit.ParticipantInfo, error) - listParticipantsMutex sync.RWMutex - listParticipantsArgsForCall []struct { - arg1 context.Context - arg2 livekit.RoomName - } - listParticipantsReturns struct { - result1 []*livekit.ParticipantInfo - result2 error - } - listParticipantsReturnsOnCall map[int]struct { - result1 []*livekit.ParticipantInfo - result2 error - } ListRoomsStub func(context.Context, []livekit.RoomName) ([]*livekit.Room, error) listRoomsMutex sync.RWMutex listRoomsArgsForCall []struct { @@ -50,21 +24,6 @@ type FakeServiceStore struct { result1 []*livekit.Room result2 error } - LoadParticipantStub func(context.Context, livekit.RoomName, livekit.ParticipantIdentity) (*livekit.ParticipantInfo, error) - loadParticipantMutex sync.RWMutex - loadParticipantArgsForCall []struct { - arg1 context.Context - arg2 livekit.RoomName - arg3 livekit.ParticipantIdentity - } - loadParticipantReturns struct { - result1 *livekit.ParticipantInfo - result2 error - } - loadParticipantReturnsOnCall map[int]struct { - result1 *livekit.ParticipantInfo - result2 error - } LoadRoomStub func(context.Context, livekit.RoomName, bool) (*livekit.Room, *livekit.RoomInternal, error) loadRoomMutex sync.RWMutex loadRoomArgsForCall []struct { @@ -82,137 +41,24 @@ type FakeServiceStore struct { result2 *livekit.RoomInternal result3 error } + RoomExistsStub func(context.Context, livekit.RoomName) (bool, error) + roomExistsMutex sync.RWMutex + roomExistsArgsForCall []struct { + arg1 context.Context + arg2 livekit.RoomName + } + roomExistsReturns struct { + result1 bool + result2 error + } + roomExistsReturnsOnCall map[int]struct { + result1 bool + result2 error + } invocations map[string][][]interface{} invocationsMutex sync.RWMutex } -func (fake *FakeServiceStore) DeleteRoom(arg1 context.Context, arg2 livekit.RoomName) error { - fake.deleteRoomMutex.Lock() - ret, specificReturn := fake.deleteRoomReturnsOnCall[len(fake.deleteRoomArgsForCall)] - fake.deleteRoomArgsForCall = append(fake.deleteRoomArgsForCall, struct { - arg1 context.Context - arg2 livekit.RoomName - }{arg1, arg2}) - stub := fake.DeleteRoomStub - fakeReturns := fake.deleteRoomReturns - fake.recordInvocation("DeleteRoom", []interface{}{arg1, arg2}) - fake.deleteRoomMutex.Unlock() - if stub != nil { - return stub(arg1, arg2) - } - if specificReturn { - return ret.result1 - } - return fakeReturns.result1 -} - -func (fake *FakeServiceStore) DeleteRoomCallCount() int { - fake.deleteRoomMutex.RLock() - defer fake.deleteRoomMutex.RUnlock() - return len(fake.deleteRoomArgsForCall) -} - -func (fake *FakeServiceStore) DeleteRoomCalls(stub func(context.Context, livekit.RoomName) error) { - fake.deleteRoomMutex.Lock() - defer fake.deleteRoomMutex.Unlock() - fake.DeleteRoomStub = stub -} - -func (fake *FakeServiceStore) DeleteRoomArgsForCall(i int) (context.Context, livekit.RoomName) { - fake.deleteRoomMutex.RLock() - defer fake.deleteRoomMutex.RUnlock() - argsForCall := fake.deleteRoomArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2 -} - -func (fake *FakeServiceStore) DeleteRoomReturns(result1 error) { - fake.deleteRoomMutex.Lock() - defer fake.deleteRoomMutex.Unlock() - fake.DeleteRoomStub = nil - fake.deleteRoomReturns = struct { - result1 error - }{result1} -} - -func (fake *FakeServiceStore) DeleteRoomReturnsOnCall(i int, result1 error) { - fake.deleteRoomMutex.Lock() - defer fake.deleteRoomMutex.Unlock() - fake.DeleteRoomStub = nil - if fake.deleteRoomReturnsOnCall == nil { - fake.deleteRoomReturnsOnCall = make(map[int]struct { - result1 error - }) - } - fake.deleteRoomReturnsOnCall[i] = struct { - result1 error - }{result1} -} - -func (fake *FakeServiceStore) ListParticipants(arg1 context.Context, arg2 livekit.RoomName) ([]*livekit.ParticipantInfo, error) { - fake.listParticipantsMutex.Lock() - ret, specificReturn := fake.listParticipantsReturnsOnCall[len(fake.listParticipantsArgsForCall)] - fake.listParticipantsArgsForCall = append(fake.listParticipantsArgsForCall, struct { - arg1 context.Context - arg2 livekit.RoomName - }{arg1, arg2}) - stub := fake.ListParticipantsStub - fakeReturns := fake.listParticipantsReturns - fake.recordInvocation("ListParticipants", []interface{}{arg1, arg2}) - fake.listParticipantsMutex.Unlock() - if stub != nil { - return stub(arg1, arg2) - } - if specificReturn { - return ret.result1, ret.result2 - } - return fakeReturns.result1, fakeReturns.result2 -} - -func (fake *FakeServiceStore) ListParticipantsCallCount() int { - fake.listParticipantsMutex.RLock() - defer fake.listParticipantsMutex.RUnlock() - return len(fake.listParticipantsArgsForCall) -} - -func (fake *FakeServiceStore) ListParticipantsCalls(stub func(context.Context, livekit.RoomName) ([]*livekit.ParticipantInfo, error)) { - fake.listParticipantsMutex.Lock() - defer fake.listParticipantsMutex.Unlock() - fake.ListParticipantsStub = stub -} - -func (fake *FakeServiceStore) ListParticipantsArgsForCall(i int) (context.Context, livekit.RoomName) { - fake.listParticipantsMutex.RLock() - defer fake.listParticipantsMutex.RUnlock() - argsForCall := fake.listParticipantsArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2 -} - -func (fake *FakeServiceStore) ListParticipantsReturns(result1 []*livekit.ParticipantInfo, result2 error) { - fake.listParticipantsMutex.Lock() - defer fake.listParticipantsMutex.Unlock() - fake.ListParticipantsStub = nil - fake.listParticipantsReturns = struct { - result1 []*livekit.ParticipantInfo - result2 error - }{result1, result2} -} - -func (fake *FakeServiceStore) ListParticipantsReturnsOnCall(i int, result1 []*livekit.ParticipantInfo, result2 error) { - fake.listParticipantsMutex.Lock() - defer fake.listParticipantsMutex.Unlock() - fake.ListParticipantsStub = nil - if fake.listParticipantsReturnsOnCall == nil { - fake.listParticipantsReturnsOnCall = make(map[int]struct { - result1 []*livekit.ParticipantInfo - result2 error - }) - } - fake.listParticipantsReturnsOnCall[i] = struct { - result1 []*livekit.ParticipantInfo - result2 error - }{result1, result2} -} - func (fake *FakeServiceStore) ListRooms(arg1 context.Context, arg2 []livekit.RoomName) ([]*livekit.Room, error) { var arg2Copy []livekit.RoomName if arg2 != nil { @@ -283,72 +129,6 @@ func (fake *FakeServiceStore) ListRoomsReturnsOnCall(i int, result1 []*livekit.R }{result1, result2} } -func (fake *FakeServiceStore) LoadParticipant(arg1 context.Context, arg2 livekit.RoomName, arg3 livekit.ParticipantIdentity) (*livekit.ParticipantInfo, error) { - fake.loadParticipantMutex.Lock() - ret, specificReturn := fake.loadParticipantReturnsOnCall[len(fake.loadParticipantArgsForCall)] - fake.loadParticipantArgsForCall = append(fake.loadParticipantArgsForCall, struct { - arg1 context.Context - arg2 livekit.RoomName - arg3 livekit.ParticipantIdentity - }{arg1, arg2, arg3}) - stub := fake.LoadParticipantStub - fakeReturns := fake.loadParticipantReturns - fake.recordInvocation("LoadParticipant", []interface{}{arg1, arg2, arg3}) - fake.loadParticipantMutex.Unlock() - if stub != nil { - return stub(arg1, arg2, arg3) - } - if specificReturn { - return ret.result1, ret.result2 - } - return fakeReturns.result1, fakeReturns.result2 -} - -func (fake *FakeServiceStore) LoadParticipantCallCount() int { - fake.loadParticipantMutex.RLock() - defer fake.loadParticipantMutex.RUnlock() - return len(fake.loadParticipantArgsForCall) -} - -func (fake *FakeServiceStore) LoadParticipantCalls(stub func(context.Context, livekit.RoomName, livekit.ParticipantIdentity) (*livekit.ParticipantInfo, error)) { - fake.loadParticipantMutex.Lock() - defer fake.loadParticipantMutex.Unlock() - fake.LoadParticipantStub = stub -} - -func (fake *FakeServiceStore) LoadParticipantArgsForCall(i int) (context.Context, livekit.RoomName, livekit.ParticipantIdentity) { - fake.loadParticipantMutex.RLock() - defer fake.loadParticipantMutex.RUnlock() - argsForCall := fake.loadParticipantArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 -} - -func (fake *FakeServiceStore) LoadParticipantReturns(result1 *livekit.ParticipantInfo, result2 error) { - fake.loadParticipantMutex.Lock() - defer fake.loadParticipantMutex.Unlock() - fake.LoadParticipantStub = nil - fake.loadParticipantReturns = struct { - result1 *livekit.ParticipantInfo - result2 error - }{result1, result2} -} - -func (fake *FakeServiceStore) LoadParticipantReturnsOnCall(i int, result1 *livekit.ParticipantInfo, result2 error) { - fake.loadParticipantMutex.Lock() - defer fake.loadParticipantMutex.Unlock() - fake.LoadParticipantStub = nil - if fake.loadParticipantReturnsOnCall == nil { - fake.loadParticipantReturnsOnCall = make(map[int]struct { - result1 *livekit.ParticipantInfo - result2 error - }) - } - fake.loadParticipantReturnsOnCall[i] = struct { - result1 *livekit.ParticipantInfo - result2 error - }{result1, result2} -} - func (fake *FakeServiceStore) LoadRoom(arg1 context.Context, arg2 livekit.RoomName, arg3 bool) (*livekit.Room, *livekit.RoomInternal, error) { fake.loadRoomMutex.Lock() ret, specificReturn := fake.loadRoomReturnsOnCall[len(fake.loadRoomArgsForCall)] @@ -418,19 +198,74 @@ func (fake *FakeServiceStore) LoadRoomReturnsOnCall(i int, result1 *livekit.Room }{result1, result2, result3} } +func (fake *FakeServiceStore) RoomExists(arg1 context.Context, arg2 livekit.RoomName) (bool, error) { + fake.roomExistsMutex.Lock() + ret, specificReturn := fake.roomExistsReturnsOnCall[len(fake.roomExistsArgsForCall)] + fake.roomExistsArgsForCall = append(fake.roomExistsArgsForCall, struct { + arg1 context.Context + arg2 livekit.RoomName + }{arg1, arg2}) + stub := fake.RoomExistsStub + fakeReturns := fake.roomExistsReturns + fake.recordInvocation("RoomExists", []interface{}{arg1, arg2}) + fake.roomExistsMutex.Unlock() + if stub != nil { + return stub(arg1, arg2) + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeServiceStore) RoomExistsCallCount() int { + fake.roomExistsMutex.RLock() + defer fake.roomExistsMutex.RUnlock() + return len(fake.roomExistsArgsForCall) +} + +func (fake *FakeServiceStore) RoomExistsCalls(stub func(context.Context, livekit.RoomName) (bool, error)) { + fake.roomExistsMutex.Lock() + defer fake.roomExistsMutex.Unlock() + fake.RoomExistsStub = stub +} + +func (fake *FakeServiceStore) RoomExistsArgsForCall(i int) (context.Context, livekit.RoomName) { + fake.roomExistsMutex.RLock() + defer fake.roomExistsMutex.RUnlock() + argsForCall := fake.roomExistsArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeServiceStore) RoomExistsReturns(result1 bool, result2 error) { + fake.roomExistsMutex.Lock() + defer fake.roomExistsMutex.Unlock() + fake.RoomExistsStub = nil + fake.roomExistsReturns = struct { + result1 bool + result2 error + }{result1, result2} +} + +func (fake *FakeServiceStore) RoomExistsReturnsOnCall(i int, result1 bool, result2 error) { + fake.roomExistsMutex.Lock() + defer fake.roomExistsMutex.Unlock() + fake.RoomExistsStub = nil + if fake.roomExistsReturnsOnCall == nil { + fake.roomExistsReturnsOnCall = make(map[int]struct { + result1 bool + result2 error + }) + } + fake.roomExistsReturnsOnCall[i] = struct { + result1 bool + result2 error + }{result1, result2} +} + func (fake *FakeServiceStore) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() - fake.deleteRoomMutex.RLock() - defer fake.deleteRoomMutex.RUnlock() - fake.listParticipantsMutex.RLock() - defer fake.listParticipantsMutex.RUnlock() - fake.listRoomsMutex.RLock() - defer fake.listRoomsMutex.RUnlock() - fake.loadParticipantMutex.RLock() - defer fake.loadParticipantMutex.RUnlock() - fake.loadRoomMutex.RLock() - defer fake.loadRoomMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} for key, value := range fake.invocations { copiedInvocations[key] = value diff --git a/pkg/service/servicefakes/fake_session_handler.go b/pkg/service/servicefakes/fake_session_handler.go index a31c69d..e112e67 100644 --- a/pkg/service/servicefakes/fake_session_handler.go +++ b/pkg/service/servicefakes/fake_session_handler.go @@ -171,10 +171,6 @@ func (fake *FakeSessionHandler) LoggerReturnsOnCall(i int, result1 logger.Logger func (fake *FakeSessionHandler) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() - fake.handleSessionMutex.RLock() - defer fake.handleSessionMutex.RUnlock() - fake.loggerMutex.RLock() - defer fake.loggerMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} for key, value := range fake.invocations { copiedInvocations[key] = value diff --git a/pkg/service/servicefakes/fake_sipstore.go b/pkg/service/servicefakes/fake_sipstore.go index 055018f..ba88c87 100644 --- a/pkg/service/servicefakes/fake_sipstore.go +++ b/pkg/service/servicefakes/fake_sipstore.go @@ -146,34 +146,6 @@ type FakeSIPStore struct { result1 *livekit.SIPTrunkInfo result2 error } - SelectSIPDispatchRuleStub func(context.Context, string) ([]*livekit.SIPDispatchRuleInfo, error) - selectSIPDispatchRuleMutex sync.RWMutex - selectSIPDispatchRuleArgsForCall []struct { - arg1 context.Context - arg2 string - } - selectSIPDispatchRuleReturns struct { - result1 []*livekit.SIPDispatchRuleInfo - result2 error - } - selectSIPDispatchRuleReturnsOnCall map[int]struct { - result1 []*livekit.SIPDispatchRuleInfo - result2 error - } - SelectSIPInboundTrunkStub func(context.Context, string) ([]*livekit.SIPInboundTrunkInfo, error) - selectSIPInboundTrunkMutex sync.RWMutex - selectSIPInboundTrunkArgsForCall []struct { - arg1 context.Context - arg2 string - } - selectSIPInboundTrunkReturns struct { - result1 []*livekit.SIPInboundTrunkInfo - result2 error - } - selectSIPInboundTrunkReturnsOnCall map[int]struct { - result1 []*livekit.SIPInboundTrunkInfo - result2 error - } StoreSIPDispatchRuleStub func(context.Context, *livekit.SIPDispatchRuleInfo) error storeSIPDispatchRuleMutex sync.RWMutex storeSIPDispatchRuleArgsForCall []struct { @@ -870,136 +842,6 @@ func (fake *FakeSIPStore) LoadSIPTrunkReturnsOnCall(i int, result1 *livekit.SIPT }{result1, result2} } -func (fake *FakeSIPStore) SelectSIPDispatchRule(arg1 context.Context, arg2 string) ([]*livekit.SIPDispatchRuleInfo, error) { - fake.selectSIPDispatchRuleMutex.Lock() - ret, specificReturn := fake.selectSIPDispatchRuleReturnsOnCall[len(fake.selectSIPDispatchRuleArgsForCall)] - fake.selectSIPDispatchRuleArgsForCall = append(fake.selectSIPDispatchRuleArgsForCall, struct { - arg1 context.Context - arg2 string - }{arg1, arg2}) - stub := fake.SelectSIPDispatchRuleStub - fakeReturns := fake.selectSIPDispatchRuleReturns - fake.recordInvocation("SelectSIPDispatchRule", []interface{}{arg1, arg2}) - fake.selectSIPDispatchRuleMutex.Unlock() - if stub != nil { - return stub(arg1, arg2) - } - if specificReturn { - return ret.result1, ret.result2 - } - return fakeReturns.result1, fakeReturns.result2 -} - -func (fake *FakeSIPStore) SelectSIPDispatchRuleCallCount() int { - fake.selectSIPDispatchRuleMutex.RLock() - defer fake.selectSIPDispatchRuleMutex.RUnlock() - return len(fake.selectSIPDispatchRuleArgsForCall) -} - -func (fake *FakeSIPStore) SelectSIPDispatchRuleCalls(stub func(context.Context, string) ([]*livekit.SIPDispatchRuleInfo, error)) { - fake.selectSIPDispatchRuleMutex.Lock() - defer fake.selectSIPDispatchRuleMutex.Unlock() - fake.SelectSIPDispatchRuleStub = stub -} - -func (fake *FakeSIPStore) SelectSIPDispatchRuleArgsForCall(i int) (context.Context, string) { - fake.selectSIPDispatchRuleMutex.RLock() - defer fake.selectSIPDispatchRuleMutex.RUnlock() - argsForCall := fake.selectSIPDispatchRuleArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2 -} - -func (fake *FakeSIPStore) SelectSIPDispatchRuleReturns(result1 []*livekit.SIPDispatchRuleInfo, result2 error) { - fake.selectSIPDispatchRuleMutex.Lock() - defer fake.selectSIPDispatchRuleMutex.Unlock() - fake.SelectSIPDispatchRuleStub = nil - fake.selectSIPDispatchRuleReturns = struct { - result1 []*livekit.SIPDispatchRuleInfo - result2 error - }{result1, result2} -} - -func (fake *FakeSIPStore) SelectSIPDispatchRuleReturnsOnCall(i int, result1 []*livekit.SIPDispatchRuleInfo, result2 error) { - fake.selectSIPDispatchRuleMutex.Lock() - defer fake.selectSIPDispatchRuleMutex.Unlock() - fake.SelectSIPDispatchRuleStub = nil - if fake.selectSIPDispatchRuleReturnsOnCall == nil { - fake.selectSIPDispatchRuleReturnsOnCall = make(map[int]struct { - result1 []*livekit.SIPDispatchRuleInfo - result2 error - }) - } - fake.selectSIPDispatchRuleReturnsOnCall[i] = struct { - result1 []*livekit.SIPDispatchRuleInfo - result2 error - }{result1, result2} -} - -func (fake *FakeSIPStore) SelectSIPInboundTrunk(arg1 context.Context, arg2 string) ([]*livekit.SIPInboundTrunkInfo, error) { - fake.selectSIPInboundTrunkMutex.Lock() - ret, specificReturn := fake.selectSIPInboundTrunkReturnsOnCall[len(fake.selectSIPInboundTrunkArgsForCall)] - fake.selectSIPInboundTrunkArgsForCall = append(fake.selectSIPInboundTrunkArgsForCall, struct { - arg1 context.Context - arg2 string - }{arg1, arg2}) - stub := fake.SelectSIPInboundTrunkStub - fakeReturns := fake.selectSIPInboundTrunkReturns - fake.recordInvocation("SelectSIPInboundTrunk", []interface{}{arg1, arg2}) - fake.selectSIPInboundTrunkMutex.Unlock() - if stub != nil { - return stub(arg1, arg2) - } - if specificReturn { - return ret.result1, ret.result2 - } - return fakeReturns.result1, fakeReturns.result2 -} - -func (fake *FakeSIPStore) SelectSIPInboundTrunkCallCount() int { - fake.selectSIPInboundTrunkMutex.RLock() - defer fake.selectSIPInboundTrunkMutex.RUnlock() - return len(fake.selectSIPInboundTrunkArgsForCall) -} - -func (fake *FakeSIPStore) SelectSIPInboundTrunkCalls(stub func(context.Context, string) ([]*livekit.SIPInboundTrunkInfo, error)) { - fake.selectSIPInboundTrunkMutex.Lock() - defer fake.selectSIPInboundTrunkMutex.Unlock() - fake.SelectSIPInboundTrunkStub = stub -} - -func (fake *FakeSIPStore) SelectSIPInboundTrunkArgsForCall(i int) (context.Context, string) { - fake.selectSIPInboundTrunkMutex.RLock() - defer fake.selectSIPInboundTrunkMutex.RUnlock() - argsForCall := fake.selectSIPInboundTrunkArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2 -} - -func (fake *FakeSIPStore) SelectSIPInboundTrunkReturns(result1 []*livekit.SIPInboundTrunkInfo, result2 error) { - fake.selectSIPInboundTrunkMutex.Lock() - defer fake.selectSIPInboundTrunkMutex.Unlock() - fake.SelectSIPInboundTrunkStub = nil - fake.selectSIPInboundTrunkReturns = struct { - result1 []*livekit.SIPInboundTrunkInfo - result2 error - }{result1, result2} -} - -func (fake *FakeSIPStore) SelectSIPInboundTrunkReturnsOnCall(i int, result1 []*livekit.SIPInboundTrunkInfo, result2 error) { - fake.selectSIPInboundTrunkMutex.Lock() - defer fake.selectSIPInboundTrunkMutex.Unlock() - fake.SelectSIPInboundTrunkStub = nil - if fake.selectSIPInboundTrunkReturnsOnCall == nil { - fake.selectSIPInboundTrunkReturnsOnCall = make(map[int]struct { - result1 []*livekit.SIPInboundTrunkInfo - result2 error - }) - } - fake.selectSIPInboundTrunkReturnsOnCall[i] = struct { - result1 []*livekit.SIPInboundTrunkInfo - result2 error - }{result1, result2} -} - func (fake *FakeSIPStore) StoreSIPDispatchRule(arg1 context.Context, arg2 *livekit.SIPDispatchRuleInfo) error { fake.storeSIPDispatchRuleMutex.Lock() ret, specificReturn := fake.storeSIPDispatchRuleReturnsOnCall[len(fake.storeSIPDispatchRuleArgsForCall)] @@ -1251,38 +1093,6 @@ func (fake *FakeSIPStore) StoreSIPTrunkReturnsOnCall(i int, result1 error) { func (fake *FakeSIPStore) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() - fake.deleteSIPDispatchRuleMutex.RLock() - defer fake.deleteSIPDispatchRuleMutex.RUnlock() - fake.deleteSIPTrunkMutex.RLock() - defer fake.deleteSIPTrunkMutex.RUnlock() - fake.listSIPDispatchRuleMutex.RLock() - defer fake.listSIPDispatchRuleMutex.RUnlock() - fake.listSIPInboundTrunkMutex.RLock() - defer fake.listSIPInboundTrunkMutex.RUnlock() - fake.listSIPOutboundTrunkMutex.RLock() - defer fake.listSIPOutboundTrunkMutex.RUnlock() - fake.listSIPTrunkMutex.RLock() - defer fake.listSIPTrunkMutex.RUnlock() - fake.loadSIPDispatchRuleMutex.RLock() - defer fake.loadSIPDispatchRuleMutex.RUnlock() - fake.loadSIPInboundTrunkMutex.RLock() - defer fake.loadSIPInboundTrunkMutex.RUnlock() - fake.loadSIPOutboundTrunkMutex.RLock() - defer fake.loadSIPOutboundTrunkMutex.RUnlock() - fake.loadSIPTrunkMutex.RLock() - defer fake.loadSIPTrunkMutex.RUnlock() - fake.selectSIPDispatchRuleMutex.RLock() - defer fake.selectSIPDispatchRuleMutex.RUnlock() - fake.selectSIPInboundTrunkMutex.RLock() - defer fake.selectSIPInboundTrunkMutex.RUnlock() - fake.storeSIPDispatchRuleMutex.RLock() - defer fake.storeSIPDispatchRuleMutex.RUnlock() - fake.storeSIPInboundTrunkMutex.RLock() - defer fake.storeSIPInboundTrunkMutex.RUnlock() - fake.storeSIPOutboundTrunkMutex.RLock() - defer fake.storeSIPOutboundTrunkMutex.RUnlock() - fake.storeSIPTrunkMutex.RLock() - defer fake.storeSIPTrunkMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} for key, value := range fake.invocations { copiedInvocations[key] = value diff --git a/pkg/service/signal.go b/pkg/service/signal.go index 75866fa..6e5a2d0 100644 --- a/pkg/service/signal.go +++ b/pkg/service/signal.go @@ -23,6 +23,7 @@ import ( "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/livekit-server/pkg/routing" "github.com/livekit/livekit-server/pkg/telemetry/prometheus" + "github.com/livekit/livekit-server/pkg/utils" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" "github.com/livekit/protocol/rpc" @@ -86,7 +87,7 @@ type defaultSessionHandler struct { } func (s *defaultSessionHandler) Logger(ctx context.Context) logger.Logger { - return logger.GetLogger() + return utils.GetLogger(ctx) } func (s *defaultSessionHandler) HandleSession( @@ -119,8 +120,8 @@ func (s *SignalServer) Start() error { return s.server.RegisterAllNodeTopics(s.nodeID) } -func (r *SignalServer) Stop() { - r.server.Kill() +func (s *SignalServer) Stop() { + s.server.Kill() } type signalService struct { @@ -167,6 +168,8 @@ func (r *signalService) RelaySignal(stream psrpc.ServerStream[*rpc.RelaySignalRe reqChan, signalRequestMessageReader{}, r.config, + prometheus.RecordSignalRequestSuccess, + prometheus.RecordSignalRequestFailure, ) l.Debugw("signal stream closed", "error", err) diff --git a/pkg/service/signal_test.go b/pkg/service/signal_test.go index 81755cd..cfc7130 100644 --- a/pkg/service/signal_test.go +++ b/pkg/service/signal_test.go @@ -21,7 +21,6 @@ import ( "time" "github.com/stretchr/testify/require" - "google.golang.org/protobuf/encoding/protojson" "google.golang.org/protobuf/proto" "github.com/livekit/livekit-server/pkg/config" @@ -31,6 +30,7 @@ import ( "github.com/livekit/livekit-server/pkg/telemetry/prometheus" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" + "github.com/livekit/protocol/utils/protojson" "github.com/livekit/psrpc" ) diff --git a/pkg/service/sip.go b/pkg/service/sip.go index 64f571c..0de7487 100644 --- a/pkg/service/sip.go +++ b/pkg/service/sip.go @@ -16,6 +16,7 @@ package service import ( "context" + "errors" "time" "github.com/dennwc/iters" @@ -86,6 +87,9 @@ func (s *SIPService) CreateSIPTrunk(ctx context.Context, req *livekit.CreateSIPT Name: req.Name, Metadata: req.Metadata, } + if err := info.Validate(); err != nil { + return nil, twirp.WrapError(twirp.NewError(twirp.InvalidArgument, err.Error()), err) + } // Validate all trunks including the new one first. it, err := ListSIPInboundTrunk(ctx, s.store, &livekit.ListSIPInboundTrunkRequest{}, info.AsInbound()) @@ -120,7 +124,7 @@ func (s *SIPService) CreateSIPInboundTrunk(ctx context.Context, req *livekit.Cre if info.SipTrunkId != "" { return nil, twirp.NewError(twirp.InvalidArgument, "trunk ID must be empty") } - AppendLogFields(ctx, "trunk", logger.Proto(req.Trunk)) + AppendLogFields(ctx, "trunk", logger.Proto(info)) // Keep ID empty still, so that validation can print "" instead of a non-existent ID in the error. @@ -159,7 +163,7 @@ func (s *SIPService) CreateSIPOutboundTrunk(ctx context.Context, req *livekit.Cr if info.SipTrunkId != "" { return nil, twirp.NewError(twirp.InvalidArgument, "trunk ID must be empty") } - AppendLogFields(ctx, "trunk", logger.Proto(req.Trunk)) + AppendLogFields(ctx, "trunk", logger.Proto(info)) // No additional validation needed for outbound. info.SipTrunkId = guid.New(utils.SIPTrunkPrefix) @@ -169,6 +173,100 @@ func (s *SIPService) CreateSIPOutboundTrunk(ctx context.Context, req *livekit.Cr return info, nil } +func (s *SIPService) UpdateSIPInboundTrunk(ctx context.Context, req *livekit.UpdateSIPInboundTrunkRequest) (*livekit.SIPInboundTrunkInfo, error) { + if err := EnsureSIPAdminPermission(ctx); err != nil { + return nil, twirpAuthError(err) + } + if s.store == nil { + return nil, ErrSIPNotConnected + } + if err := req.Validate(); err != nil { + return nil, twirp.WrapError(twirp.NewError(twirp.InvalidArgument, err.Error()), err) + } + + AppendLogFields(ctx, + "request", logger.Proto(req), + "trunkID", req.SipTrunkId, + ) + + // Validate all trunks including the new one first. + info, err := s.store.LoadSIPInboundTrunk(ctx, req.SipTrunkId) + if err != nil { + if errors.Is(err, ErrSIPTrunkNotFound) { + return nil, twirp.NewError(twirp.NotFound, err.Error()) + } + return nil, err + } + switch a := req.Action.(type) { + default: + return nil, errors.New("missing or unsupported action") + case livekit.UpdateSIPInboundTrunkRequestAction: + info, err = a.Apply(info) + if err != nil { + return nil, err + } + } + + it, err := ListSIPInboundTrunk(ctx, s.store, &livekit.ListSIPInboundTrunkRequest{ + Numbers: info.Numbers, + }) + if err != nil { + return nil, err + } + defer it.Close() + if err = sip.ValidateTrunksIter(it, sip.WithTrunkReplace(func(t *livekit.SIPInboundTrunkInfo) *livekit.SIPInboundTrunkInfo { + if req.SipTrunkId == t.SipTrunkId { + return info // updated one + } + return t + })); err != nil { + return nil, err + } + if err := s.store.StoreSIPInboundTrunk(ctx, info); err != nil { + return nil, err + } + return info, nil +} + +func (s *SIPService) UpdateSIPOutboundTrunk(ctx context.Context, req *livekit.UpdateSIPOutboundTrunkRequest) (*livekit.SIPOutboundTrunkInfo, error) { + if err := EnsureSIPAdminPermission(ctx); err != nil { + return nil, twirpAuthError(err) + } + if s.store == nil { + return nil, ErrSIPNotConnected + } + if err := req.Validate(); err != nil { + return nil, twirp.WrapError(twirp.NewError(twirp.InvalidArgument, err.Error()), err) + } + + AppendLogFields(ctx, + "request", logger.Proto(req), + "trunkID", req.SipTrunkId, + ) + + info, err := s.store.LoadSIPOutboundTrunk(ctx, req.SipTrunkId) + if err != nil { + if errors.Is(err, ErrSIPTrunkNotFound) { + return nil, twirp.NewError(twirp.NotFound, err.Error()) + } + return nil, err + } + switch a := req.Action.(type) { + default: + return nil, errors.New("missing or unsupported action") + case livekit.UpdateSIPOutboundTrunkRequestAction: + info, err = a.Apply(info) + if err != nil { + return nil, err + } + } + // No additional validation needed for outbound. + if err := s.store.StoreSIPOutboundTrunk(ctx, info); err != nil { + return nil, err + } + return info, nil +} + func (s *SIPService) GetSIPInboundTrunk(ctx context.Context, req *livekit.GetSIPInboundTrunkRequest) (*livekit.GetSIPInboundTrunkResponse, error) { if err := EnsureSIPAdminPermission(ctx); err != nil { return nil, twirpAuthError(err) @@ -183,6 +281,9 @@ func (s *SIPService) GetSIPInboundTrunk(ctx context.Context, req *livekit.GetSIP trunk, err := s.store.LoadSIPInboundTrunk(ctx, req.SipTrunkId) if err != nil { + if errors.Is(err, ErrSIPTrunkNotFound) { + return nil, twirp.NewError(twirp.NotFound, err.Error()) + } return nil, err } @@ -203,6 +304,9 @@ func (s *SIPService) GetSIPOutboundTrunk(ctx context.Context, req *livekit.GetSI trunk, err := s.store.LoadSIPOutboundTrunk(ctx, req.SipTrunkId) if err != nil { + if errors.Is(err, ErrSIPTrunkNotFound) { + return nil, twirp.NewError(twirp.NotFound, err.Error()) + } return nil, err } @@ -317,6 +421,7 @@ func (s *SIPService) CreateSIPDispatchRule(ctx context.Context, req *livekit.Cre if s.store == nil { return nil, ErrSIPNotConnected } + req.DispatchRule.Upgrade() if err := req.Validate(); err != nil { return nil, twirp.WrapError(twirp.NewError(twirp.InvalidArgument, err.Error()), err) } @@ -326,16 +431,8 @@ func (s *SIPService) CreateSIPDispatchRule(ctx context.Context, req *livekit.Cre "trunkID", req.TrunkIds, ) // Keep ID empty, so that validation can print "" instead of a non-existent ID in the error. - info := &livekit.SIPDispatchRuleInfo{ - Rule: req.Rule, - TrunkIds: req.TrunkIds, - InboundNumbers: req.InboundNumbers, - HidePhoneNumber: req.HidePhoneNumber, - Name: req.Name, - Metadata: req.Metadata, - Attributes: req.Attributes, - RoomConfig: req.RoomConfig, - } + info := req.DispatchRuleInfo() + info.SipDispatchRuleId = "" // Validate all rules including the new one first. it, err := ListSIPDispatchRule(ctx, s.store, &livekit.ListSIPDispatchRuleRequest{ @@ -357,6 +454,62 @@ func (s *SIPService) CreateSIPDispatchRule(ctx context.Context, req *livekit.Cre return info, nil } +func (s *SIPService) UpdateSIPDispatchRule(ctx context.Context, req *livekit.UpdateSIPDispatchRuleRequest) (*livekit.SIPDispatchRuleInfo, error) { + if err := EnsureSIPAdminPermission(ctx); err != nil { + return nil, twirpAuthError(err) + } + if s.store == nil { + return nil, ErrSIPNotConnected + } + if err := req.Validate(); err != nil { + return nil, twirp.WrapError(twirp.NewError(twirp.InvalidArgument, err.Error()), err) + } + + AppendLogFields(ctx, + "request", logger.Proto(req), + "ruleID", req.SipDispatchRuleId, + ) + + // Validate all trunks including the new one first. + info, err := s.store.LoadSIPDispatchRule(ctx, req.SipDispatchRuleId) + if err != nil { + if errors.Is(err, ErrSIPDispatchRuleNotFound) { + return nil, twirp.NewError(twirp.NotFound, err.Error()) + } + return nil, err + } + switch a := req.Action.(type) { + default: + return nil, errors.New("missing or unsupported action") + case livekit.UpdateSIPDispatchRuleRequestAction: + info, err = a.Apply(info) + if err != nil { + return nil, err + } + } + + it, err := ListSIPDispatchRule(ctx, s.store, &livekit.ListSIPDispatchRuleRequest{ + TrunkIds: info.TrunkIds, + }) + if err != nil { + return nil, err + } + defer it.Close() + if _, err = sip.ValidateDispatchRulesIter(it, sip.WithDispatchRuleReplace(func(t *livekit.SIPDispatchRuleInfo) *livekit.SIPDispatchRuleInfo { + if req.SipDispatchRuleId == t.SipDispatchRuleId { + return info // updated one + } + return t + })); err != nil { + return nil, err + } + + if err := s.store.StoreSIPDispatchRule(ctx, info); err != nil { + return nil, err + } + return info, nil +} + func ListSIPDispatchRule(ctx context.Context, s SIPStore, req *livekit.ListSIPDispatchRuleRequest, add ...*livekit.SIPDispatchRuleInfo) (iters.Iter[*livekit.SIPDispatchRuleInfo], error) { if s == nil { return nil, ErrSIPNotConnected @@ -402,6 +555,9 @@ func (s *SIPService) DeleteSIPDispatchRule(ctx context.Context, req *livekit.Del info, err := s.store.LoadSIPDispatchRule(ctx, req.SipDispatchRuleId) if err != nil { + if errors.Is(err, ErrSIPDispatchRuleNotFound) { + return nil, twirp.NewError(twirp.NotFound, err.Error()) + } return nil, err } @@ -428,7 +584,7 @@ func (s *SIPService) CreateSIPParticipant(ctx context.Context, req *livekit.Crea ireq, err := s.CreateSIPParticipantRequest(ctx, req, "", "", "", "") if err != nil { unlikelyLogger.Errorw("cannot create sip participant request", err) - return nil, err + return nil, wrapSIPContextError(err) } unlikelyLogger = unlikelyLogger.WithValues( "callID", ireq.SipCallId, @@ -457,7 +613,7 @@ func (s *SIPService) CreateSIPParticipant(ctx context.Context, req *livekit.Crea resp, err := s.psrpcClient.CreateSIPParticipant(ctx, "", ireq, psrpc.WithRequestTimeout(timeout)) if err != nil { unlikelyLogger.Errorw("cannot create sip participant", err) - return nil, err + return nil, wrapSIPContextError(err) } return &livekit.SIPParticipantInfo{ ParticipantId: resp.ParticipantId, @@ -474,6 +630,10 @@ func (s *SIPService) CreateSIPParticipantRequest(ctx context.Context, req *livek if s.store == nil { return nil, ErrSIPNotConnected } + req.Upgrade() + if err := req.Validate(); err != nil { + return nil, twirp.WrapError(twirp.NewError(twirp.InvalidArgument, err.Error()), err) + } callID := sip.NewCallID() log := logger.GetLogger().WithUnlikelyValues( "callID", callID, @@ -485,10 +645,22 @@ func (s *SIPService) CreateSIPParticipantRequest(ctx context.Context, req *livek log = log.WithValues("projectID", projectID) } - trunk, err := s.store.LoadSIPOutboundTrunk(ctx, req.SipTrunkId) - if err != nil { - log.Errorw("cannot get trunk to update sip participant", err) - return nil, err + var trunk *livekit.SIPOutboundTrunkInfo + if req.SipTrunkId != "" { + var err error + trunk, err = s.store.LoadSIPOutboundTrunk(ctx, req.SipTrunkId) + if err != nil { + log.Errorw("cannot get trunk to update sip participant", err) + if errors.Is(err, ErrSIPTrunkNotFound) { + return nil, twirp.NewError(twirp.NotFound, err.Error()) + } + return nil, err + } + } + if trunk != nil && trunk.FromHost != "" { + host = trunk.FromHost + } else if t := req.Trunk; t != nil && t.FromHost != "" { + host = t.FromHost } return rpc.NewCreateSIPParticipantRequest(projectID, callID, host, wsUrl, token, req, trunk) } @@ -510,12 +682,25 @@ func (s *SIPService) TransferSIPParticipant(ctx context.Context, req *livekit.Tr ireq, err := s.transferSIPParticipantRequest(ctx, req) if err != nil { log.Errorw("cannot create transfer sip participant request", err) - return nil, err + return nil, wrapSIPContextError(err) } + // by default we set the timeout to be 30 seconds. + // this timeout covers: + // - a network failure between this process and the LiveKit SIP bridge + // - the SIP transfer target not returning 200 OK fast enough. + // WARN: any timeout/cancellation of a SIP transfer risks leaving + // either the SIP bridge, or the SIP REFER exchange, in a "unknown" state. timeout := 30 * time.Second + if req.RingingTimeout != nil { + timeout = req.RingingTimeout.AsDuration() + } + + // it's also possible the ctx has a Deadline. + // in that case we want to use that deadline, + // or our timeout, whichover is soonest. if deadline, ok := ctx.Deadline(); ok { - timeout = time.Until(deadline) + timeout = min(timeout, time.Until(deadline)) } else { var cancel func() ctx, cancel = context.WithTimeout(ctx, timeout) @@ -524,7 +709,7 @@ func (s *SIPService) TransferSIPParticipant(ctx context.Context, req *livekit.Tr _, err = s.psrpcClient.TransferSIPParticipant(ctx, ireq.SipCallId, ireq, psrpc.WithRequestTimeout(timeout)) if err != nil { log.Errorw("cannot transfer sip participant", err) - return nil, err + return nil, wrapSIPContextError(err) } return &emptypb.Empty{}, nil @@ -545,6 +730,9 @@ func (s *SIPService) transferSIPParticipantRequest(ctx context.Context, req *liv if err := EnsureAdminPermission(ctx, livekit.RoomName(req.RoomName)); err != nil { return nil, twirpAuthError(err) } + if err := req.Validate(); err != nil { + return nil, twirp.WrapError(twirp.NewError(twirp.InvalidArgument, err.Error()), err) + } resp, err := s.roomService.GetParticipant(ctx, &livekit.RoomParticipantIdentity{ Room: req.RoomName, @@ -561,8 +749,31 @@ func (s *SIPService) transferSIPParticipantRequest(ctx context.Context, req *liv } return &rpc.InternalTransferSIPParticipantRequest{ - SipCallId: callID, - TransferTo: req.TransferTo, - PlayDialtone: req.PlayDialtone, + SipCallId: callID, + TransferTo: req.TransferTo, + PlayDialtone: req.PlayDialtone, + Headers: req.Headers, + RingingTimeout: req.RingingTimeout, }, nil } + +// wrapSIPContextError converts raw context.DeadlineExceeded / context.Canceled +// into psrpc-coded errors so they aren't surfaced as @code:unknown / HTTP 500 +// at the Twirp boundary. psrpc errors and any error that already carries a +// gRPC status are passed through unchanged. +func wrapSIPContextError(err error) error { + if err == nil { + return nil + } + var psErr psrpc.Error + if errors.As(err, &psErr) { + return err + } + switch { + case errors.Is(err, context.DeadlineExceeded): + return psrpc.NewError(psrpc.DeadlineExceeded, err) + case errors.Is(err, context.Canceled): + return psrpc.NewError(psrpc.Canceled, err) + } + return err +} diff --git a/pkg/service/turn.go b/pkg/service/turn.go index dc2d21b..47b82d0 100644 --- a/pkg/service/turn.go +++ b/pkg/service/turn.go @@ -21,9 +21,11 @@ import ( "net" "strconv" "strings" + "time" "github.com/jxskiss/base62" - "github.com/pion/turn/v4" + "github.com/pion/stun/v3" + "github.com/pion/turn/v5" "github.com/pkg/errors" "github.com/livekit/protocol/auth" @@ -40,10 +42,10 @@ const ( LivekitRealm = "livekit" allocateRetries = 50 - turnMinPort = 1024 - turnMaxPort = 30000 ) +var ErrExpired = errors.New("expired") + func NewTurnServer(conf *config.Config, authHandler turn.AuthHandler, standalone bool) (*turn.Server, error) { turnConf := conf.TURN if !turnConf.Enabled { @@ -52,29 +54,7 @@ func NewTurnServer(conf *config.Config, authHandler turn.AuthHandler, standalone if turnConf.TLSPort <= 0 && turnConf.UDPPort <= 0 { return nil, errors.New("invalid TURN ports") - } - - serverConfig := turn.ServerConfig{ - Realm: LivekitRealm, - AuthHandler: authHandler, - LoggerFactory: pionlogger.NewLoggerFactory(logger.GetLogger()), - } - var relayAddrGen turn.RelayAddressGenerator = &turn.RelayAddressGeneratorPortRange{ - RelayAddress: net.ParseIP(conf.RTC.NodeIP), - Address: "0.0.0.0", - MinPort: turnConf.RelayPortRangeStart, - MaxPort: turnConf.RelayPortRangeEnd, - MaxRetries: allocateRetries, - } - if standalone { - relayAddrGen = telemetry.NewRelayAddressGenerator(relayAddrGen) - } - var logValues []interface{} - - logValues = append(logValues, "turn.relay_range_start", turnConf.RelayPortRangeStart) - logValues = append(logValues, "turn.relay_range_end", turnConf.RelayPortRangeEnd) - - if turnConf.TLSPort > 0 { + } else if turnConf.TLSPort > 0 { if turnConf.Domain == "" { return nil, errors.New("TURN domain required") } @@ -82,64 +62,129 @@ func NewTurnServer(conf *config.Config, authHandler turn.AuthHandler, standalone if !IsValidDomain(turnConf.Domain) { return nil, errors.New("TURN domain is not correct") } - - if !turnConf.ExternalTLS { - cert, err := tls.LoadX509KeyPair(turnConf.CertFile, turnConf.KeyFile) - if err != nil { - return nil, errors.Wrap(err, "TURN tls cert required") - } - - tlsListener, err := tls.Listen("tcp4", "0.0.0.0:"+strconv.Itoa(turnConf.TLSPort), - &tls.Config{ - MinVersion: tls.VersionTLS12, - Certificates: []tls.Certificate{cert}, - }) - if err != nil { - return nil, errors.Wrap(err, "could not listen on TURN TCP port") - } - if standalone { - tlsListener = telemetry.NewListener(tlsListener) - } - - listenerConfig := turn.ListenerConfig{ - Listener: tlsListener, - RelayAddressGenerator: relayAddrGen, - } - serverConfig.ListenerConfigs = append(serverConfig.ListenerConfigs, listenerConfig) - } else { - tcpListener, err := net.Listen("tcp4", "0.0.0.0:"+strconv.Itoa(turnConf.TLSPort)) - if err != nil { - return nil, errors.Wrap(err, "could not listen on TURN TCP port") - } - if standalone { - tcpListener = telemetry.NewListener(tcpListener) - } - - listenerConfig := turn.ListenerConfig{ - Listener: tcpListener, - RelayAddressGenerator: relayAddrGen, - } - serverConfig.ListenerConfigs = append(serverConfig.ListenerConfigs, listenerConfig) - } - logValues = append(logValues, "turn.portTLS", turnConf.TLSPort, "turn.externalTLS", turnConf.ExternalTLS) } - if turnConf.UDPPort > 0 { - udpListener, err := net.ListenPacket("udp4", "0.0.0.0:"+strconv.Itoa(turnConf.UDPPort)) - if err != nil { - return nil, errors.Wrap(err, "could not listen on TURN UDP port") + serverConfig := turn.ServerConfig{ + Realm: LivekitRealm, + AuthHandler: authHandler, + LoggerFactory: pionlogger.NewLoggerFactory(logger.GetLogger()), + } + + var logValues []any + logValues = append(logValues, "turn.relay_range_start", turnConf.RelayPortRangeStart) + logValues = append(logValues, "turn.relay_range_end", turnConf.RelayPortRangeEnd) + + for _, addr := range turnConf.BindAddresses { + var nodeIP string + if net.ParseIP(addr).To4() != nil { + nodeIP = conf.RTC.NodeIP.V4 + } else { + nodeIP = conf.RTC.NodeIP.V6 + } + if nodeIP == "" { + return nil, errors.New("no matching node IP for relay") } + var relayAddrGen turn.RelayAddressGenerator = &turn.RelayAddressGeneratorPortRange{ + RelayAddress: net.ParseIP(nodeIP), + Address: addr, + MinPort: turnConf.RelayPortRangeStart, + MaxPort: turnConf.RelayPortRangeEnd, + MaxRetries: allocateRetries, + } if standalone { - udpListener = telemetry.NewPacketConn(udpListener, prometheus.Incoming) + relayAddrGen = telemetry.NewRelayAddressGenerator(relayAddrGen) } - packetConfig := turn.PacketConnConfig{ - PacketConn: udpListener, - RelayAddressGenerator: relayAddrGen, + permissionHandler := func(_clientAddr net.Addr, peerIP net.IP) bool { + // restricted peer IP is denied by default, unless allowed by the allow list, + if peerIP.IsLoopback() || + peerIP.IsLinkLocalUnicast() || + peerIP.IsLinkLocalMulticast() || + peerIP.IsMulticast() || + peerIP.IsPrivate() || + peerIP.IsUnspecified() { + allowed := false + for _, cidr := range turnConf.AllowRestrictedPeerCIDRs { + if _, ipnet, err := net.ParseCIDR(cidr); err == nil { + if ipnet.Contains(peerIP) { + allowed = true + break + } + } + } + if !allowed { + return false + } + + // if allowed, check deny list for overrides + } + + for _, cidr := range turnConf.DenyPeerCIDRs { + if _, ipnet, err := net.ParseCIDR(cidr); err == nil { + if ipnet.Contains(peerIP) { + return false + } + } + } + + return true + } + + if turnConf.TLSPort > 0 { + var listener net.Listener + var listenerErr error + + if turnConf.ExternalTLS { + listener, listenerErr = net.Listen("tcp", net.JoinHostPort(addr, strconv.Itoa(turnConf.TLSPort))) + } else { + cert, err := tls.LoadX509KeyPair(turnConf.CertFile, turnConf.KeyFile) + if err != nil { + return nil, errors.Wrap(err, "TURN tls cert required") + } + + listener, listenerErr = tls.Listen("tcp", net.JoinHostPort(addr, strconv.Itoa(turnConf.TLSPort)), + &tls.Config{ + MinVersion: tls.VersionTLS12, + Certificates: []tls.Certificate{cert}, + }) + } + + if listenerErr != nil { + return nil, errors.Wrap(listenerErr, "could not listen on TURN TCP port") + } + if standalone { + listener = telemetry.NewListener(listener) + } + + listenerConfig := turn.ListenerConfig{ + Listener: listener, + RelayAddressGenerator: relayAddrGen, + PermissionHandler: permissionHandler, + } + serverConfig.ListenerConfigs = append(serverConfig.ListenerConfigs, listenerConfig) + + logValues = append(logValues, "turn.portTLS", turnConf.TLSPort, "turn.externalTLS", turnConf.ExternalTLS) + } + + if turnConf.UDPPort > 0 { + udpListener, err := net.ListenPacket("udp", net.JoinHostPort(addr, strconv.Itoa(turnConf.UDPPort))) + if err != nil { + return nil, errors.Wrap(err, "could not listen on TURN UDP port") + } + + if standalone { + udpListener = telemetry.NewPacketConn(udpListener, prometheus.Incoming) + } + + packetConfig := turn.PacketConnConfig{ + PacketConn: udpListener, + RelayAddressGenerator: relayAddrGen, + PermissionHandler: permissionHandler, + } + serverConfig.PacketConnConfigs = append(serverConfig.PacketConnConfigs, packetConfig) + logValues = append(logValues, "turn.portUDP", turnConf.UDPPort) } - serverConfig.PacketConnConfigs = append(serverConfig.PacketConnConfigs, packetConfig) - logValues = append(logValues, "turn.portUDP", turnConf.UDPPort) } logger.Infow("Starting TURN server", logValues...) @@ -160,33 +205,82 @@ func NewTURNAuthHandler(keyProvider auth.KeyProvider) *TURNAuthHandler { } } -func (h *TURNAuthHandler) CreateUsername(apiKey string, pID livekit.ParticipantID) string { - return base62.EncodeToString([]byte(fmt.Sprintf("%s|%s", apiKey, pID))) +func (h *TURNAuthHandler) CreateUsername(apiKey string, pID livekit.ParticipantID, ttlSeconds int) (string, int64) { + expiry := time.Now().Add(time.Duration(ttlSeconds) * time.Second).Unix() + return base62.EncodeToString(fmt.Appendf(nil, "%s|%s|%d", apiKey, pID, expiry)), expiry } -func (h *TURNAuthHandler) CreatePassword(apiKey string, pID livekit.ParticipantID) (string, error) { +func (h *TURNAuthHandler) ParseUsername(username string) (string, livekit.ParticipantID, int64, error) { + decoded, err := base62.DecodeString(username) + if err != nil { + return "", "", 0, err + } + parts := strings.Split(string(decoded), "|") + if len(parts) != 3 { + return "", "", 0, errors.New("invalid username") + } + expiry, err := strconv.ParseInt(parts[2], 10, 64) + if err != nil { + return "", "", 0, err + } + if expiry == 0 { + return "", "", 0, ErrExpired + } + + return parts[0], livekit.ParticipantID(parts[1]), expiry, nil +} + +func (h *TURNAuthHandler) CreatePassword(apiKey string, pID livekit.ParticipantID, expiry int64) (string, error) { + if expiry == 0 || time.Now().After(time.Unix(expiry, 0)) { + return "", ErrExpired + } + return h.computePassword(apiKey, pID, expiry) +} + +func (h *TURNAuthHandler) computePassword(apiKey string, pID livekit.ParticipantID, expiry int64) (string, error) { secret := h.keyProvider.GetSecret(apiKey) if secret == "" { return "", ErrInvalidAPIKey } - keyInput := fmt.Sprintf("%s|%s", secret, pID) + + keyInput := fmt.Sprintf("%s|%s|%d", secret, pID, expiry) + sum := sha256.Sum256([]byte(keyInput)) return base62.EncodeToString(sum[:]), nil } -func (h *TURNAuthHandler) HandleAuth(username, realm string, srcAddr net.Addr) (key []byte, ok bool) { +func (h *TURNAuthHandler) HandleAuth(ra *turn.RequestAttributes) (userID string, key []byte, ok bool) { + username := ra.Username decoded, err := base62.DecodeString(username) if err != nil { - return nil, false + return "", nil, false } parts := strings.Split(string(decoded), "|") - if len(parts) != 2 { - return nil, false + if len(parts) != 3 { + return "", nil, false } - password, err := h.CreatePassword(parts[0], livekit.ParticipantID(parts[1])) + expiry, err := strconv.ParseInt(parts[2], 10, 64) if err != nil { - logger.Warnw("could not create TURN password", err, "username", username) - return nil, false + return "", nil, false } - return turn.GenerateAuthKey(username, LivekitRealm, password), true + if expiry == 0 { + return "", nil, false + } + expiryTime := time.Unix(expiry, 0) + if time.Now().After(expiryTime) { + // TTL only applies to initial allocation. Refresh / CreatePermission / + // ChannelBind / Send / Data requests are still authenticated against the + // username/password but skip the TTL check so long-running sessions can + // keep refreshing past the credential expiry. + if ra.Method == stun.MethodAllocate { + logger.Infow("TURN credential expired", "username", decoded, "participantID", parts[1], "expiry", expiryTime, "method", ra.Method) + return "", nil, false + } + } + password, err := h.computePassword(parts[0], livekit.ParticipantID(parts[1]), expiry) + if err != nil { + logger.Warnw("could not create TURN password", err, "username", decoded) + return "", nil, false + } + return parts[1], turn.GenerateAuthKey(username, LivekitRealm, password), true } diff --git a/pkg/service/turn_test.go b/pkg/service/turn_test.go new file mode 100644 index 0000000..32abd28 --- /dev/null +++ b/pkg/service/turn_test.go @@ -0,0 +1,208 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package service + +import ( + "fmt" + "net" + "testing" + + "github.com/jxskiss/base62" + "github.com/pion/stun/v3" + "github.com/pion/turn/v5" + "github.com/stretchr/testify/require" + + "github.com/livekit/protocol/auth" + "github.com/livekit/protocol/livekit" +) + +const ( + turnTestAPIKey = "APITestKey" + turnTestAPISecret = "TestSecret" +) + +func newTestTurnAuthHandler() *TURNAuthHandler { + return NewTURNAuthHandler(auth.NewSimpleKeyProvider(turnTestAPIKey, turnTestAPISecret)) +} + +func mustAuthCreds(t *testing.T, h *TURNAuthHandler, pID livekit.ParticipantID, ttlSeconds int) (username string, key []byte) { + t.Helper() + username, expiry := h.CreateUsername(turnTestAPIKey, pID, ttlSeconds) + password, err := h.CreatePassword(turnTestAPIKey, pID, expiry) + require.NoError(t, err) + return username, turn.GenerateAuthKey(username, LivekitRealm, password) +} + +func TestTURNAuthHandler_HandleAuth_ValidCredentials(t *testing.T) { + h := newTestTurnAuthHandler() + pID := livekit.ParticipantID("PA_valid") + username, expectedKey := mustAuthCreds(t, h, pID, 300) + + for _, method := range []stun.Method{ + stun.MethodAllocate, + stun.MethodRefresh, + stun.MethodCreatePermission, + stun.MethodChannelBind, + stun.MethodSend, + } { + t.Run(method.String(), func(t *testing.T) { + userID, key, ok := h.HandleAuth(&turn.RequestAttributes{ + Username: username, + Realm: LivekitRealm, + SrcAddr: &net.UDPAddr{IP: net.ParseIP("127.0.0.1"), Port: 1234}, + Method: method, + }) + require.True(t, ok) + require.Equal(t, string(pID), userID) + require.Equal(t, expectedKey, key) + }) + } +} + +func TestTURNAuthHandler_HandleAuth_ExpiredAllocateRejected(t *testing.T) { + h := newTestTurnAuthHandler() + pID := livekit.ParticipantID("PA_expired_alloc") + + username, _ := h.CreateUsername(turnTestAPIKey, pID, -60) + _, _, ok := h.HandleAuth(&turn.RequestAttributes{ + Username: username, + Realm: LivekitRealm, + SrcAddr: &net.UDPAddr{IP: net.ParseIP("127.0.0.1"), Port: 1234}, + Method: stun.MethodAllocate, + }) + require.False(t, ok, "Allocate request with expired credentials must be rejected") +} + +func TestTURNAuthHandler_HandleAuth_ExpiredNonAllocateAllowed(t *testing.T) { + h := newTestTurnAuthHandler() + pID := livekit.ParticipantID("PA_expired_refresh") + + username, expiry := h.CreateUsername(turnTestAPIKey, pID, -60) + + // CreatePassword still enforces ErrExpired on its own, but the server hands + // the same key it generated at allocation time — reproduce that by directly + // hashing without going through CreatePassword's expiry guard. + password, err := h.computePassword(turnTestAPIKey, pID, expiry) + require.NoError(t, err) + expectedKey := turn.GenerateAuthKey(username, LivekitRealm, password) + + for _, method := range []stun.Method{ + stun.MethodRefresh, + stun.MethodCreatePermission, + stun.MethodChannelBind, + stun.MethodSend, + } { + t.Run(method.String(), func(t *testing.T) { + userID, key, ok := h.HandleAuth(&turn.RequestAttributes{ + Username: username, + Realm: LivekitRealm, + SrcAddr: &net.UDPAddr{IP: net.ParseIP("127.0.0.1"), Port: 1234}, + Method: method, + }) + require.True(t, ok, "Non-allocate request with expired credentials must succeed") + require.Equal(t, string(pID), userID) + require.Equal(t, expectedKey, key) + }) + } +} + +func TestTURNAuthHandler_HandleAuth_WrongUsernameRejected(t *testing.T) { + h := newTestTurnAuthHandler() + _, _, ok := h.HandleAuth(&turn.RequestAttributes{ + Username: "not-base62!!!", + Realm: LivekitRealm, + SrcAddr: &net.UDPAddr{IP: net.ParseIP("127.0.0.1"), Port: 1234}, + Method: stun.MethodRefresh, + }) + require.False(t, ok) +} + +func TestTURNAuthHandler_HandleAuth_TwoPartUsernameRejected(t *testing.T) { + h := newTestTurnAuthHandler() + pID := livekit.ParticipantID("PA_two_part") + + username := base62.EncodeToString(fmt.Appendf(nil, "%s|%s", turnTestAPIKey, pID)) + + for _, method := range []stun.Method{ + stun.MethodAllocate, + stun.MethodRefresh, + stun.MethodCreatePermission, + stun.MethodChannelBind, + stun.MethodSend, + } { + t.Run(method.String(), func(t *testing.T) { + _, _, ok := h.HandleAuth(&turn.RequestAttributes{ + Username: username, + Realm: LivekitRealm, + SrcAddr: &net.UDPAddr{IP: net.ParseIP("127.0.0.1"), Port: 1234}, + Method: method, + }) + require.False(t, ok, "Two-part username must be rejected") + }) + } +} + +func TestTURNAuthHandler_HandleAuth_ZeroExpiryRejected(t *testing.T) { + h := newTestTurnAuthHandler() + pID := livekit.ParticipantID("PA_zero_expiry") + + username := base62.EncodeToString(fmt.Appendf(nil, "%s|%s|%d", turnTestAPIKey, pID, 0)) + + for _, method := range []stun.Method{ + stun.MethodAllocate, + stun.MethodRefresh, + stun.MethodCreatePermission, + stun.MethodChannelBind, + stun.MethodSend, + } { + t.Run(method.String(), func(t *testing.T) { + _, _, ok := h.HandleAuth(&turn.RequestAttributes{ + Username: username, + Realm: LivekitRealm, + SrcAddr: &net.UDPAddr{IP: net.ParseIP("127.0.0.1"), Port: 1234}, + Method: method, + }) + require.False(t, ok, "Username with expiry=0 must be rejected") + }) + } +} + +func TestTURNAuthHandler_ParseUsername_TwoPartRejected(t *testing.T) { + h := newTestTurnAuthHandler() + pID := livekit.ParticipantID("PA_parse_two_part") + + username := base62.EncodeToString(fmt.Appendf(nil, "%s|%s", turnTestAPIKey, pID)) + + _, _, _, err := h.ParseUsername(username) + require.Error(t, err) +} + +func TestTURNAuthHandler_ParseUsername_ZeroExpiryRejected(t *testing.T) { + h := newTestTurnAuthHandler() + pID := livekit.ParticipantID("PA_parse_zero_expiry") + + username := base62.EncodeToString(fmt.Appendf(nil, "%s|%s|%d", turnTestAPIKey, pID, 0)) + + _, _, _, err := h.ParseUsername(username) + require.ErrorIs(t, err, ErrExpired) +} + +func TestTURNAuthHandler_CreatePassword_ZeroExpiryRejected(t *testing.T) { + h := newTestTurnAuthHandler() + pID := livekit.ParticipantID("PA_password_zero_expiry") + + _, err := h.CreatePassword(turnTestAPIKey, pID, 0) + require.ErrorIs(t, err, ErrExpired) +} diff --git a/pkg/service/twirp.go b/pkg/service/twirp.go index 556bfa0..1dbc0bc 100644 --- a/pkg/service/twirp.go +++ b/pkg/service/twirp.go @@ -26,10 +26,11 @@ import ( "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/timestamppb" + "github.com/livekit/protocol/livekit" + "github.com/livekit/livekit-server/pkg/telemetry" "github.com/livekit/livekit-server/pkg/telemetry/prometheus" "github.com/livekit/livekit-server/pkg/utils" - "github.com/livekit/protocol/livekit" ) type twirpRequestFields struct { @@ -46,9 +47,9 @@ type twirpLoggerKey struct{} // License: Apache-2.0 func TwirpLogger() *twirp.ServerHooks { loggerPool := &sync.Pool{ - New: func() interface{} { + New: func() any { return &twirpLogger{ - fieldsOrig: make([]interface{}, 0, 30), + fieldsOrig: make([]any, 0, 30), } }, } @@ -67,12 +68,20 @@ func TwirpLogger() *twirp.ServerHooks { type twirpLogger struct { twirpRequestFields - fieldsOrig []interface{} - fields []interface{} + fieldsOrig []any + fields []any startedAt time.Time + deadline time.Time } -func AppendLogFields(ctx context.Context, fields ...interface{}) { +func (t *twirpLogger) reset() { + t.fields = t.fieldsOrig + t.error = nil + t.startedAt = time.Time{} + t.deadline = time.Time{} +} + +func AppendLogFields(ctx context.Context, fields ...any) { r, ok := ctx.Value(twirpLoggerKey{}).(*twirpLogger) if !ok || r == nil { return @@ -84,6 +93,9 @@ func AppendLogFields(ctx context.Context, fields ...interface{}) { func loggerRequestReceived(ctx context.Context, twirpLoggerPool *sync.Pool) (context.Context, error) { r := twirpLoggerPool.Get().(*twirpLogger) r.startedAt = time.Now() + if deadline, ok := ctx.Deadline(); ok { + r.deadline = deadline + } r.fields = r.fieldsOrig r.error = nil @@ -114,7 +126,14 @@ func loggerResponseSent(ctx context.Context, twirpLoggerPool *sync.Pool) { return } - r.fields = append(r.fields, "duration", time.Since(r.startedAt)) + duration := time.Since(r.startedAt) + r.fields = append(r.fields, "duration", duration) + if !r.deadline.IsZero() { + r.fields = append(r.fields, "requestedTimeout", r.deadline.Sub(r.startedAt)) + } + if deadline, ok := ctx.Deadline(); ok { + r.fields = append(r.fields, "modifiedTimeout", deadline.Sub(r.startedAt)) + } if status, ok := twirp.StatusCode(ctx); ok { r.fields = append(r.fields, "status", status) @@ -126,10 +145,10 @@ func loggerResponseSent(ctx context.Context, twirpLoggerPool *sync.Pool) { serviceMethod := "API " + r.service + "." + r.method utils.GetLogger(ctx).WithComponent(utils.ComponentAPI).Infow(serviceMethod, r.fields...) + prometheus.RecordTwirpRequestLatency(r.service, r.method, duration) - r.fields = r.fieldsOrig - r.error = nil - + // reset fields and return to pool + r.reset() twirpLoggerPool.Put(r) } @@ -203,7 +222,7 @@ func statusReporterResponseSent(ctx context.Context) { code = r.error.Code() } - prometheus.TwirpRequestStatusCounter.WithLabelValues(r.service, r.method, statusFamily, string(code)).Add(1) + prometheus.RecordTwirpRequestStatus(r.service, r.method, statusFamily, code) } func statusReporterErrorReceived(ctx context.Context, e twirp.Error) context.Context { @@ -231,6 +250,7 @@ func TwirpTelemetry( ResponseSent: func(ctx context.Context) { telemetryResponseSent(ctx, nodeID, getProjectID, telemetry) }, + RequestRouted: telemetryRequestRouted, } } @@ -390,7 +410,7 @@ func telemetryResponseSent( } a.NodeId = string(nodeID) if statusCode, ok := twirp.StatusCode(ctx); ok { - if status, err := strconv.Atoi(statusCode); err == nil { + if status, err := strconv.ParseInt(statusCode, 10, 32); err == nil { a.Status = int32(status) } } diff --git a/pkg/service/utils.go b/pkg/service/utils.go index bac2b39..e129e2f 100644 --- a/pkg/service/utils.go +++ b/pkg/service/utils.go @@ -15,29 +15,85 @@ package service import ( + "bytes" + "compress/gzip" "context" + "encoding/json" "errors" + "fmt" + "io" "net" "net/http" "regexp" + "strconv" "strings" + "sync" + "github.com/ua-parser/uap-go/uaparser" + "gopkg.in/yaml.v3" + + "github.com/livekit/livekit-server/pkg/config" + "github.com/livekit/livekit-server/pkg/routing" + "github.com/livekit/livekit-server/pkg/routing/selector" + "github.com/livekit/livekit-server/pkg/rtc" + "github.com/livekit/livekit-server/pkg/utils" + "github.com/livekit/protocol/auth" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" ) -func handleError(w http.ResponseWriter, r *http.Request, status int, err error, keysAndValues ...interface{}) { +var ( + ErrGzipReadFailed = errors.New("cannot read decompressed data") + ErrGzipTooLarge = errors.New("decompressed data too large") +) + +var gzipReaderPool = sync.Pool{ + New: func() any { return &gzip.Reader{} }, +} + +func DecompressGzip(compressed []byte) ([]byte, error) { + reader := gzipReaderPool.Get().(*gzip.Reader) + defer gzipReaderPool.Put(reader) + if err := reader.Reset(bytes.NewReader(compressed)); err != nil { + return nil, fmt.Errorf("%w: %w", ErrGzipReadFailed, err) + } + + out, err := io.ReadAll(io.LimitReader(reader, http.DefaultMaxHeaderBytes+1)) + if err != nil { + return nil, fmt.Errorf("%w: %w", ErrGzipReadFailed, err) + } + if len(out) > http.DefaultMaxHeaderBytes { + return nil, ErrGzipTooLarge + } + return out, nil +} + +func handleError(w http.ResponseWriter, r *http.Request, status int, err error, keysAndValues ...any) { keysAndValues = append(keysAndValues, "status", status) if r != nil && r.URL != nil { keysAndValues = append(keysAndValues, "method", r.Method, "path", r.URL.Path) } if !errors.Is(err, context.Canceled) && !errors.Is(r.Context().Err(), context.Canceled) { - logger.GetLogger().WithCallDepth(1).Warnw("error handling request", err, keysAndValues...) + utils.GetLogger(r.Context()).WithCallDepth(1).Warnw("error handling request", err, keysAndValues...) } w.WriteHeader(status) +} + +func HandleError(w http.ResponseWriter, r *http.Request, status int, err error, keysAndValues ...any) { + handleError(w, r, status, err, keysAndValues...) _, _ = w.Write([]byte(err.Error())) } +func HandleErrorJson(w http.ResponseWriter, r *http.Request, status int, err error, keysAndValues ...any) { + handleError(w, r, status, err, keysAndValues...) + json.NewEncoder(w).Encode(struct { + Error string `json:"error"` + }{ + Error: err.Error(), + }) + w.Header().Add("Content-type", "application/json") +} + func boolValue(s string) bool { return s == "1" || s == "true" } @@ -81,4 +137,286 @@ func SetRoomConfiguration(createRequest *livekit.CreateRoomRequest, conf *liveki createRequest.MinPlayoutDelay = conf.MinPlayoutDelay createRequest.MaxPlayoutDelay = conf.MaxPlayoutDelay createRequest.SyncStreams = conf.SyncStreams + createRequest.Metadata = conf.Metadata + createRequest.Tags = conf.Tags +} + +func ParseClientInfo(r *http.Request) *livekit.ClientInfo { + values := r.Form + ci := &livekit.ClientInfo{} + if pv, err := strconv.ParseInt(values.Get("protocol"), 10, 32); err == nil { + ci.Protocol = int32(pv) + } + if cp, err := strconv.ParseInt(values.Get("client_protocol"), 10, 32); err == nil { + ci.ClientProtocol = int32(cp) + } + sdkString := values.Get("sdk") + switch sdkString { + case "js": + ci.Sdk = livekit.ClientInfo_JS + case "ios", "swift": + ci.Sdk = livekit.ClientInfo_SWIFT + case "android": + ci.Sdk = livekit.ClientInfo_ANDROID + case "flutter": + ci.Sdk = livekit.ClientInfo_FLUTTER + case "go": + ci.Sdk = livekit.ClientInfo_GO + case "unity": + ci.Sdk = livekit.ClientInfo_UNITY + case "reactnative": + ci.Sdk = livekit.ClientInfo_REACT_NATIVE + case "rust": + ci.Sdk = livekit.ClientInfo_RUST + case "python": + ci.Sdk = livekit.ClientInfo_PYTHON + case "cpp": + ci.Sdk = livekit.ClientInfo_CPP + case "unityweb": + ci.Sdk = livekit.ClientInfo_UNITY_WEB + case "node": + ci.Sdk = livekit.ClientInfo_NODE + case "esp32": + ci.Sdk = livekit.ClientInfo_ESP32 + } + + ci.Version = values.Get("version") + ci.Os = values.Get("os") + ci.OsVersion = values.Get("os_version") + ci.Browser = values.Get("browser") + ci.BrowserVersion = values.Get("browser_version") + ci.DeviceModel = values.Get("device_model") + ci.Network = values.Get("network") + + if capStr := values.Get("capabilities"); capStr != "" { + for _, name := range strings.Split(capStr, ",") { + name = strings.TrimSpace(name) + if name == "" { + continue + } + if v, ok := livekit.ClientInfo_Capability_value[name]; ok { + ci.Capabilities = append(ci.Capabilities, livekit.ClientInfo_Capability(v)) + } + } + } + + AugmentClientInfo(ci, r) + + return ci +} + +var ( + userAgentParserCache *uaparser.Parser + userAgentParserInit sync.Once +) + +func createUserAgentParserWithCustomRules() (*uaparser.Parser, error) { + defaultYaml := uaparser.DefinitionYaml + + rules := make(map[string]any) + err := yaml.Unmarshal(defaultYaml, rules) + if err != nil { + return nil, err + } + + rules["user_agent_parsers"] = append(rules["user_agent_parsers"].([]any), map[string]any{ + "regex": "OBS-Studio\\/([0-9\\.]+)", + "family_replacement": "OBS Studio", + "v1_replacement": "$1", + }) + + customYaml, err := yaml.Marshal(rules) + if err != nil { + return nil, err + } + + return uaparser.NewFromBytes([]byte(customYaml)) +} + +func getUserAgentParser() *uaparser.Parser { + userAgentParserInit.Do(func() { + if parser, err := createUserAgentParserWithCustomRules(); err != nil { + logger.Warnw("could not create user agent parser with custom rules, using default", err) + userAgentParserCache = uaparser.NewFromSaved() + } else { + userAgentParserCache = parser + } + }) + return userAgentParserCache +} + +func AugmentClientInfo(ci *livekit.ClientInfo, req *http.Request) { + if ci == nil { + return + } + + // get real address (forwarded http header) - check Cloudflare headers first, fall back to X-Forwarded-For + ci.Address = GetClientIP(req) + + // attempt to parse types for SDKs that support browser as a platform + if ci.Sdk == livekit.ClientInfo_JS || + ci.Sdk == livekit.ClientInfo_REACT_NATIVE || + ci.Sdk == livekit.ClientInfo_FLUTTER || + ci.Sdk == livekit.ClientInfo_UNITY || + ci.Sdk == livekit.ClientInfo_UNKNOWN { + client := getUserAgentParser().Parse(req.UserAgent()) + if ci.Browser == "" { + ci.Browser = client.UserAgent.Family + ci.BrowserVersion = client.UserAgent.ToVersionString() + } + if ci.Os == "" { + ci.Os = client.Os.Family + ci.OsVersion = client.Os.ToVersionString() + } + if ci.DeviceModel == "" { + model := client.Device.Family + if model != "" && client.Device.Model != "" && model != client.Device.Model { + model += " " + client.Device.Model + } + + ci.DeviceModel = model + } + } +} + +type ValidateConnectRequestParams struct { + roomName livekit.RoomName + publish string + metadata string + attributes map[string]string +} + +type ValidateConnectRequestResult struct { + roomName livekit.RoomName + grants *auth.ClaimGrants + region string + createRoomRequest *livekit.CreateRoomRequest +} + +func ValidateConnectRequest( + lgr logger.Logger, + r *http.Request, + limitConfig config.LimitConfig, + params ValidateConnectRequestParams, + router routing.MessageRouter, + roomAllocator RoomAllocator, +) (ValidateConnectRequestResult, int, error) { + var res ValidateConnectRequestResult + + // require a claim + claims := GetGrants(r.Context()) + if claims == nil || claims.Video == nil { + return res, http.StatusUnauthorized, rtc.ErrPermissionDenied + } + + roomNameInToken, err := EnsureJoinPermission(r.Context()) + if err != nil { + return res, http.StatusUnauthorized, err + } + + if claims.Identity == "" { + return res, http.StatusBadRequest, ErrIdentityEmpty + } + if !limitConfig.CheckParticipantIdentityLength(claims.Identity) { + return res, http.StatusBadRequest, fmt.Errorf("%w: max length %d", ErrParticipantIdentityExceedsLimits, limitConfig.MaxParticipantIdentityLength) + } + + if claims.RoomConfig != nil { + if err := claims.RoomConfig.CheckCredentials(); err != nil { + lgr.Warnw("credentials found in token", nil) + // TODO(dz): in a future version, we'll reject these connections + } + } + + res.roomName = params.roomName + if roomNameInToken != "" { + res.roomName = roomNameInToken + } + if res.roomName == "" { + return res, http.StatusBadRequest, ErrNoRoomName + } + if !limitConfig.CheckRoomNameLength(string(res.roomName)) { + return res, http.StatusBadRequest, fmt.Errorf("%w: max length %d", ErrRoomNameExceedsLimits, limitConfig.MaxRoomNameLength) + } + + // this is new connection for existing participant - with publish only permissions + if params.publish != "" { + // Make sure grant has GetCanPublish set, + if !claims.Video.GetCanPublish() { + return res, http.StatusUnauthorized, rtc.ErrPermissionDenied + } + // Make sure by default subscribe is off + claims.Video.SetCanSubscribe(false) + claims.Identity += "#" + params.publish + } + + // room allocator validations + err = roomAllocator.ValidateCreateRoom(r.Context(), res.roomName) + if err != nil { + if errors.Is(err, ErrRoomNotFound) { + return res, http.StatusNotFound, err + } else { + return res, http.StatusInternalServerError, err + } + } + + if router, ok := router.(routing.Router); ok { + res.region = router.GetRegion() + if foundNode, err := router.GetNodeForRoom(r.Context(), res.roomName); err == nil { + if selector.LimitsReached(limitConfig, foundNode.Stats) { + return res, http.StatusServiceUnavailable, rtc.ErrLimitExceeded + } + } + } + + createRequest := &livekit.CreateRoomRequest{ + Name: string(res.roomName), + RoomPreset: claims.RoomPreset, + } + SetRoomConfiguration(createRequest, claims.GetRoomConfiguration()) + res.createRoomRequest = createRequest + + if len(params.metadata) != 0 { + // Make sure grant has GetCanUpdateOwnMetadata set + if !claims.Video.GetCanUpdateOwnMetadata() { + return res, http.StatusUnauthorized, rtc.ErrPermissionDenied + } + claims.Metadata = params.metadata + } + + // Add extra attributes to the participant + if len(params.attributes) != 0 { + // Make sure grant has GetCanUpdateOwnMetadata set + if !claims.Video.GetCanUpdateOwnMetadata() { + return res, http.StatusUnauthorized, rtc.ErrPermissionDenied + } + if claims.Attributes == nil { + claims.Attributes = make(map[string]string, len(params.attributes)) + } + for k, v := range params.attributes { + if v == "" { + continue // do not allow deleting existing attributes + } + claims.Attributes[k] = v + } + } + + res.grants = claims + return res, http.StatusOK, nil +} + +func IsRTCPath(path string) bool { + return path == "/rtc" || path == "/rtc/v1" +} + +func IsRTCValidatePath(path string) bool { + return path == "/rtc/validate" || path == "/rtc/v1/validate" +} + +func IsAgentWorkerPath(path string) bool { + return path == "/agent" +} + +func IsAgentPath(path string) bool { + return strings.HasPrefix(path, "/agent") } diff --git a/pkg/service/utils_test.go b/pkg/service/utils_test.go index 5d8cdca..1f675c6 100644 --- a/pkg/service/utils_test.go +++ b/pkg/service/utils_test.go @@ -15,7 +15,10 @@ package service_test import ( + "bytes" + "compress/gzip" "context" + "net/http" "testing" "time" @@ -75,3 +78,52 @@ func TestIsValidDomain(t *testing.T) { require.Equal(t, service.IsValidDomain(key), result) } } + +func compress(t *testing.T, payload []byte) []byte { + t.Helper() + var buf bytes.Buffer + gw, err := gzip.NewWriterLevel(&buf, gzip.BestCompression) + require.NoError(t, err) + _, err = gw.Write(payload) + require.NoError(t, err) + require.NoError(t, gw.Close()) + return buf.Bytes() +} + +func TestDecompressGzip(t *testing.T) { + t.Run("small payload", func(t *testing.T) { + out, err := service.DecompressGzip(compress(t, []byte("hello world"))) + require.NoError(t, err) + require.Equal(t, []byte("hello world"), out) + }) + + t.Run("payload exactly at cap", func(t *testing.T) { + raw := make([]byte, http.DefaultMaxHeaderBytes) + out, err := service.DecompressGzip(compress(t, raw)) + require.NoError(t, err) + require.Len(t, out, http.DefaultMaxHeaderBytes) + }) + + t.Run("payload one byte over capd", func(t *testing.T) { + raw := make([]byte, http.DefaultMaxHeaderBytes+1) + _, err := service.DecompressGzip(compress(t, raw)) + require.ErrorIs(t, err, service.ErrGzipTooLarge) + }) + + t.Run("gzip decompression bomb", func(t *testing.T) { + // 100 MB of zeros + raw := make([]byte, 100<<20) + compressed := compress(t, raw) + require.Less(t, len(compressed), 1<<20, + "sanity: bomb input should compress dramatically") + + _, err := service.DecompressGzip(compressed) + require.ErrorIs(t, err, service.ErrGzipTooLarge) + }) + + t.Run("malformed gzip compression", func(t *testing.T) { + _, err := service.DecompressGzip([]byte("not gzip data")) + require.Error(t, err) + require.Contains(t, err.Error(), "cannot read decompressed") + }) +} diff --git a/pkg/service/whipservice.go b/pkg/service/whipservice.go new file mode 100644 index 0000000..7f5f20f --- /dev/null +++ b/pkg/service/whipservice.go @@ -0,0 +1,544 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package service + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + + "github.com/pion/webrtc/v4" + "github.com/tomnomnom/linkheader" + + "github.com/livekit/livekit-server/pkg/config" + "github.com/livekit/livekit-server/pkg/routing" + "github.com/livekit/livekit-server/pkg/rtc" + "github.com/livekit/livekit-server/pkg/rtc/types" + sutils "github.com/livekit/livekit-server/pkg/utils" + "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/logger" + "github.com/livekit/protocol/rpc" + "github.com/livekit/protocol/utils/guid" + "github.com/livekit/psrpc" +) + +const ( + cParticipantPath = "/whip/v1" + cParticipantIDPath = "/whip/v1/{participant_id}" +) + +type WHIPService struct { + http.Handler + + config *config.Config + router routing.Router + roomAllocator RoomAllocator + client rpc.WHIPClient[livekit.NodeID] + topicFormatter rpc.TopicFormatter + participantClient rpc.TypedWHIPParticipantClient +} + +func NewWHIPService( + config *config.Config, + router routing.Router, + roomAllocator RoomAllocator, + clientParams rpc.ClientParams, + topicFormatter rpc.TopicFormatter, + participantClient rpc.TypedWHIPParticipantClient, +) (*WHIPService, error) { + client, err := rpc.NewWHIPClient[livekit.NodeID](clientParams.Args()) + if err != nil { + return nil, err + } + + return &WHIPService{ + config: config, + router: router, + roomAllocator: roomAllocator, + client: client, + topicFormatter: topicFormatter, + participantClient: participantClient, + }, nil +} + +func (s *WHIPService) SetupRoutes(mux *http.ServeMux) { + mux.HandleFunc("GET "+cParticipantPath, s.handleGet) + mux.HandleFunc("OPTIONS "+cParticipantPath, s.handleOptions) + mux.HandleFunc("POST "+cParticipantPath, s.handleCreate) + mux.HandleFunc("GET "+cParticipantIDPath, s.handleParticipantGet) + mux.HandleFunc("PATCH "+cParticipantIDPath, s.handleParticipantPatch) + mux.HandleFunc("DELETE "+cParticipantIDPath, s.handleParticipantDelete) +} + +func (s *WHIPService) handleGet(w http.ResponseWriter, r *http.Request) { + // https:/www.rfc-editor.org/rfc/rfc9725.html#name-http-usage + w.WriteHeader(http.StatusNoContent) +} + +func (s *WHIPService) handleOptions(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Access-Control-Allow-Origin", "*") + w.Header().Set("Access-Control-Allow-Headers", "*") + w.Header().Set("Access-Control-Allow-Methods", "PATCH, OPTIONS, GET, POST, DELETE") + w.Header().Set("Access-Control-Expose-Headers", "*") + + w.WriteHeader(http.StatusOK) + + // According to https://www.rfc-editor.org/rfc/rfc9725.html#name-stun-turn-server-configurat, + // ICE servers can be returned in OPTIONS response, but not recommended. + // + // Supporting that here is tricky. This would have to get region settings like the + // session CREATE POST request and send a request to get ICE servers from a + // region + media node that is selected. The issue is that a subsequent POST, + // although unlikely, may end up in a different region. Media node in one region and + // TURN in another region, although shuttling media across regions, should still work. + // But, as this is not a recommended way, not supporting it. +} + +type createRequest struct { + RoomName livekit.RoomName + ParticipantInit routing.ParticipantInit + ClientIP string + OfferSDP string + SubscribedParticipantTrackNames map[string][]string + FromIngress bool +} + +func (s *WHIPService) validateCreate(w http.ResponseWriter, r *http.Request) (*createRequest, int, error) { + claims := GetGrants(r.Context()) + if claims == nil || claims.Video == nil { + return nil, http.StatusUnauthorized, rtc.ErrPermissionDenied + } + + roomName, err := EnsureJoinPermission(r.Context()) + if err != nil { + return nil, http.StatusUnauthorized, err + } + if roomName == "" { + return nil, http.StatusUnauthorized, errors.New("room name cannot be empty") + } + if !s.config.Limit.CheckRoomNameLength(string(roomName)) { + return nil, http.StatusBadRequest, fmt.Errorf("%w: max length %d", ErrRoomNameExceedsLimits, s.config.Limit.MaxRoomNameLength) + } + + if claims.Identity == "" { + return nil, http.StatusBadRequest, ErrIdentityEmpty + } + if !s.config.Limit.CheckParticipantIdentityLength(claims.Identity) { + return nil, http.StatusBadRequest, fmt.Errorf("%w: max length %d", ErrParticipantIdentityExceedsLimits, s.config.Limit.MaxParticipantIdentityLength) + } + + var clientInfo struct { + ClientIP string `json:"clientIp"` + SubscribedParticipantTrackNames map[string][]string `json:"subscribedParticipantTrackNames"` + } + clientInfoHeader := r.Header.Get("X-LiveKit-ClientInfo") + if clientInfoHeader != "" { + if err := json.NewDecoder(strings.NewReader(clientInfoHeader)).Decode(&clientInfo); err != nil { + return nil, http.StatusBadRequest, fmt.Errorf("malformed json in client info header: %s", err) + } + } + + fromIngress := r.Header.Get("X-Livekit-Ingress") + + offerSDPBytes, err := io.ReadAll(http.MaxBytesReader(w, r.Body, http.DefaultMaxHeaderBytes)) + if err != nil { + var maxErr *http.MaxBytesError + if errors.As(err, &maxErr) { + return nil, http.StatusRequestEntityTooLarge, fmt.Errorf("request body exceeds %d bytes", maxErr.Limit) + } + return nil, http.StatusBadRequest, fmt.Errorf("body does not have SDP offer: %s", err) + } + if len(offerSDPBytes) == 0 { + return nil, http.StatusBadRequest, errors.New("body does not have SDP offer") + } + offerSDP := string(offerSDPBytes) + sd := &webrtc.SessionDescription{ + Type: webrtc.SDPTypeOffer, + SDP: offerSDP, + } + _, err = sd.Unmarshal() + if err != nil { + return nil, http.StatusBadRequest, fmt.Errorf("malformed SDP offer: %s", err) + } + + ci := ParseClientInfo(r) + if ci.Protocol == 0 { + // if no client info available (which will be mostly the case with WHIP clients), at least set protocol + ci.Protocol = types.CurrentProtocol + } + + pi := routing.ParticipantInit{ + Identity: livekit.ParticipantIdentity(claims.Identity), + Name: livekit.ParticipantName(claims.Name), + AutoSubscribe: true, + Client: ci, + Grants: claims, + CreateRoom: &livekit.CreateRoomRequest{ + Name: string(roomName), + RoomPreset: claims.RoomPreset, + }, + AdaptiveStream: false, + DisableICELite: true, + } + SetRoomConfiguration(pi.CreateRoom, claims.GetRoomConfiguration()) + + return &createRequest{ + roomName, + pi, + clientInfo.ClientIP, + offerSDP, + clientInfo.SubscribedParticipantTrackNames, + fromIngress != "", + }, http.StatusOK, nil +} + +func (s *WHIPService) handleCreate(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Content-type") != "application/sdp" { + s.handleError("Create", w, r, http.StatusBadRequest, fmt.Errorf("unsupported content-type: %s", r.Header.Get("Content-type"))) + return + } + + w.Header().Add("Content-type", "application/sdp") + + req, status, err := s.validateCreate(w, r) + if err != nil { + s.handleError("Create", w, r, status, err) + return + } + + if err := s.roomAllocator.SelectRoomNode(r.Context(), req.RoomName, ""); err != nil { + s.handleError("Create", w, r, http.StatusInternalServerError, err) + return + } + + rtcNode, err := s.router.GetNodeForRoom(r.Context(), req.RoomName) + if err != nil { + s.handleError("Create", w, r, http.StatusInternalServerError, err) + return + } + + connID := livekit.ConnectionID(guid.New("CO_")) + starSession, err := req.ParticipantInit.ToStartSession(req.RoomName, connID) + if err != nil { + s.handleError("Create", w, r, http.StatusInternalServerError, err) + return + } + + subscribedParticipantTracks := map[string]*rpc.WHIPCreateRequest_TrackList{} + for identity, trackNames := range req.SubscribedParticipantTrackNames { + subscribedParticipantTracks[identity] = &rpc.WHIPCreateRequest_TrackList{ + TrackNames: trackNames, + } + } + + res, err := s.client.Create(r.Context(), livekit.NodeID(rtcNode.Id), &rpc.WHIPCreateRequest{ + OfferSdp: req.OfferSDP, + StartSession: starSession, + SubscribedParticipantTracks: subscribedParticipantTracks, + FromIngress: req.FromIngress, + }) + if err != nil { + s.handleError("Create", w, r, http.StatusServiceUnavailable, err) + return + } + + // created resource sent in Location header: + // https://www.rfc-editor.org/rfc/rfc9725.html#name-ingest-session-setup + // using relative location + w.Header().Add("Location", fmt.Sprintf("%s/%s", cParticipantPath, res.ParticipantId)) + + // ICE servers as Link header(s): + // https://www.rfc-editor.org/rfc/rfc9725.html#name-stun-turn-server-configurat + var iceServerLinks []*linkheader.Link + for _, iceServer := range res.IceServers { + for _, iceURL := range iceServer.Urls { + iceServerLink := &linkheader.Link{ + URL: url.PathEscape(iceURL), + Rel: "ice-server", + Params: map[string]string{}, + } + if iceServer.Username != "" { + iceServerLink.Params["username"] = iceServer.Username + } + if iceServer.Credential != "" { + iceServerLink.Params["credential"] = iceServer.Credential + } + + iceServerLinks = append(iceServerLinks, iceServerLink) + } + } + for _, iceServerLink := range iceServerLinks { + w.Header().Add("Link", iceServerLink.String()) + } + + // To support ICE Trickle/Restart, HTTP PATCH should have an ETag + // send ICE session ID (ICE ufrag is used as ID) in ETag header + // https://www.rfc-editor.org/rfc/rfc9725.html#name-http-patch-request-usage + if res.IceSessionId != "" { + w.Header().Add("ETag", res.IceSessionId) + } + + // 201 Status Created + w.WriteHeader(http.StatusCreated) + + // SDP answer in the response body + w.Write([]byte(res.AnswerSdp)) + + sutils.GetLogger(r.Context()).Infow( + "API WHIP.Create", + "connID", connID, + "participant", req.ParticipantInit.Identity, + "room", req.RoomName, + "status", http.StatusCreated, + "response", logger.Proto(res), + ) +} + +func (s *WHIPService) handleParticipantGet(w http.ResponseWriter, r *http.Request) { + // https:/www.rfc-editor.org/rfc/rfc9725.html#name-http-usage + w.WriteHeader(http.StatusNoContent) +} + +func (s *WHIPService) iceTrickle( + w http.ResponseWriter, + r *http.Request, + roomName livekit.RoomName, + participantIdentity livekit.ParticipantIdentity, + pID livekit.ParticipantID, + iceSessionID string, + sdpFragment string, +) { + _, err := s.participantClient.ICETrickle( + r.Context(), + s.topicFormatter.ParticipantTopic(r.Context(), roomName, participantIdentity), + &rpc.WHIPParticipantICETrickleRequest{ + Room: string(roomName), + ParticipantIdentity: string(participantIdentity), + ParticipantId: string(pID), + IceSessionId: iceSessionID, + SdpFragment: sdpFragment, + }, + ) + if err != nil { + var pe psrpc.Error + if errors.As(err, &pe) { + switch pe.Code() { + case psrpc.NotFound: + s.handleError("Patch", w, r, http.StatusNotFound, errors.New(pe.Error())) + + case psrpc.InvalidArgument: + switch pe.Error() { + case rtc.ErrInvalidSDPFragment.Error(), rtc.ErrMidMismatch.Error(), rtc.ErrICECredentialMismatch.Error(): + s.handleError("Patch", w, r, http.StatusBadRequest, errors.New(pe.Error())) + default: + s.handleError("Patch", w, r, http.StatusInternalServerError, errors.New(pe.Error())) + } + default: + s.handleError("Patch", w, r, http.StatusInternalServerError, errors.New(pe.Error())) + } + } else { + s.handleError("Patch", w, r, http.StatusInternalServerError, nil) + } + return + } + sutils.GetLogger(r.Context()).Infow( + "API WHIP.Patch", + "method", "ice-trickle", + "room", roomName, + "participant", participantIdentity, + "participantID", pID, + "sdpFragment", sdpFragment, + "status", http.StatusNoContent, + ) + w.WriteHeader(http.StatusNoContent) +} + +func (s *WHIPService) iceRestart( + w http.ResponseWriter, + r *http.Request, + roomName livekit.RoomName, + participantIdentity livekit.ParticipantIdentity, + pID livekit.ParticipantID, + sdpFragment string, +) { + res, err := s.participantClient.ICERestart( + r.Context(), + s.topicFormatter.ParticipantTopic(r.Context(), roomName, participantIdentity), + &rpc.WHIPParticipantICERestartRequest{ + Room: string(roomName), + ParticipantIdentity: string(participantIdentity), + ParticipantId: string(pID), + SdpFragment: sdpFragment, + }, + ) + if err != nil { + var pe psrpc.Error + if errors.As(err, &pe) { + switch pe.Code() { + case psrpc.NotFound: + s.handleError("Patch", w, r, http.StatusNotFound, errors.New(pe.Error())) + + case psrpc.InvalidArgument: + switch pe.Error() { + case rtc.ErrInvalidSDPFragment.Error(): + s.handleError("Patch", w, r, http.StatusBadRequest, errors.New(pe.Error())) + default: + s.handleError("Patch", w, r, http.StatusInternalServerError, errors.New(pe.Error())) + } + default: + s.handleError("Patch", w, r, http.StatusInternalServerError, errors.New(pe.Error())) + } + } else { + s.handleError("Patch", w, r, http.StatusInternalServerError, nil) + } + return + } + sutils.GetLogger(r.Context()).Infow( + "API WHIP.Patch", + "method", "ice-restart", + "room", roomName, + "participant", participantIdentity, + "participantID", pID, + "sdpFragment", sdpFragment, + "status", http.StatusNoContent, + "res", logger.Proto(res), + ) + if res.IceSessionId != "" { + w.Header().Add("ETag", res.IceSessionId) + } + w.WriteHeader(http.StatusOK) + w.Write([]byte(res.SdpFragment)) +} + +func (s *WHIPService) handleParticipantPatch(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Content-type") != "application/trickle-ice-sdpfrag" { + s.handleError("Patch", w, r, http.StatusBadRequest, fmt.Errorf("unsupported content-type: %s", r.Header.Get("Content-type"))) + return + } + + w.Header().Add("Content-type", "application/trickle-ice-sdpfrag") + + // https://www.rfc-editor.org/rfc/rfc9725.html#name-http-patch-request-usage + ifMatch := r.Header.Get("If-Match") + if ifMatch == "" { + s.handleError("Patch", w, r, http.StatusPreconditionRequired, errors.New("missing entity tag")) + return + } + + claims := GetGrants(r.Context()) + if claims == nil || claims.Video == nil { + s.handleError("Patch", w, r, http.StatusUnauthorized, rtc.ErrPermissionDenied) + return + } + + roomName, err := EnsureJoinPermission(r.Context()) + if err != nil { + s.handleError("Patch", w, r, http.StatusUnauthorized, err) + return + } + if roomName == "" { + s.handleError("Patch", w, r, http.StatusUnauthorized, errors.New("room name cannot be empty")) + return + } + if claims.Identity == "" { + s.handleError("Patch", w, r, http.StatusUnauthorized, errors.New("participant identity cannot be empty")) + return + } + pID := livekit.ParticipantID(r.PathValue("participant_id")) + if pID == "" { + s.handleError("Patch", w, r, http.StatusBadRequest, errors.New("participant ID cannot be empty")) + return + } + + sdpFragmentBytes, err := io.ReadAll(http.MaxBytesReader(w, r.Body, http.DefaultMaxHeaderBytes)) + if err != nil { + var maxErr *http.MaxBytesError + if errors.As(err, &maxErr) { + s.handleError("Patch", w, r, http.StatusRequestEntityTooLarge, fmt.Errorf("request body exceeds %d bytes", maxErr.Limit)) + return + } + s.handleError("Patch", w, r, http.StatusBadRequest, fmt.Errorf("body does not have SDP fragment: %s", err)) + return + } + sdpFragment := string(sdpFragmentBytes) + + if ifMatch == "*" { + s.iceRestart(w, r, roomName, livekit.ParticipantIdentity(claims.Identity), pID, sdpFragment) + } else { + s.iceTrickle(w, r, roomName, livekit.ParticipantIdentity(claims.Identity), pID, ifMatch, sdpFragment) + } +} + +func (s *WHIPService) handleParticipantDelete(w http.ResponseWriter, r *http.Request) { + claims := GetGrants(r.Context()) + if claims == nil || claims.Video == nil { + s.handleError("Delete", w, r, http.StatusUnauthorized, rtc.ErrPermissionDenied) + return + } + + roomName, err := EnsureJoinPermission(r.Context()) + if err != nil { + s.handleError("Delete", w, r, http.StatusUnauthorized, err) + return + } + if roomName == "" { + s.handleError("Delete", w, r, http.StatusUnauthorized, errors.New("room name cannot be empty")) + return + } + if claims.Identity == "" { + s.handleError("Delete", w, r, http.StatusUnauthorized, errors.New("participant identity cannot be empty")) + return + } + + _, err = s.participantClient.DeleteSession( + r.Context(), + s.topicFormatter.ParticipantTopic(r.Context(), roomName, livekit.ParticipantIdentity(claims.Identity)), + &rpc.WHIPParticipantDeleteSessionRequest{ + Room: string(roomName), + ParticipantIdentity: claims.Identity, + ParticipantId: r.PathValue("participant_id"), + }, + ) + if err != nil { + s.handleError("Delete", w, r, http.StatusNotFound, err) + return + } + + sutils.GetLogger(r.Context()).Infow( + "API WHIP.Delete", + "participant", claims.Identity, + "participantID", r.PathValue("participant_id"), + "room", roomName, + "status", http.StatusOK, + ) + w.WriteHeader(http.StatusOK) +} + +func (s *WHIPService) handleError(method string, w http.ResponseWriter, r *http.Request, status int, err error) { + sutils.GetLogger(r.Context()).Warnw( + fmt.Sprintf("API WHIP.%s", method), err, + "status", status, + ) + w.WriteHeader(status) + json.NewEncoder(w).Encode(struct { + Error string `json:"error"` + }{ + Error: err.Error(), + }) +} diff --git a/pkg/service/wire.go b/pkg/service/wire.go index d4eda2d..a2d60fd 100644 --- a/pkg/service/wire.go +++ b/pkg/service/wire.go @@ -22,17 +22,11 @@ import ( "os" "github.com/google/wire" - "github.com/pion/turn/v4" + "github.com/pion/turn/v5" "github.com/pkg/errors" "github.com/redis/go-redis/v9" "gopkg.in/yaml.v3" - "github.com/livekit/livekit-server/pkg/agent" - "github.com/livekit/livekit-server/pkg/clientconfiguration" - "github.com/livekit/livekit-server/pkg/config" - "github.com/livekit/livekit-server/pkg/routing" - "github.com/livekit/livekit-server/pkg/sfu" - "github.com/livekit/livekit-server/pkg/telemetry" "github.com/livekit/protocol/auth" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" @@ -41,6 +35,13 @@ import ( "github.com/livekit/protocol/utils" "github.com/livekit/protocol/webhook" "github.com/livekit/psrpc" + "github.com/livekit/psrpc/pkg/middleware/otelpsrpc" + + "github.com/livekit/livekit-server/pkg/agent" + "github.com/livekit/livekit-server/pkg/config" + "github.com/livekit/livekit-server/pkg/routing" + "github.com/livekit/livekit-server/pkg/sfu" + "github.com/livekit/livekit-server/pkg/telemetry" ) func InitializeServer(conf *config.Config, currentNode routing.LocalNode) (*LivekitServer, error) { @@ -51,15 +52,15 @@ func InitializeServer(conf *config.Config, currentNode routing.LocalNode) (*Live wire.Bind(new(ServiceStore), new(ObjectStore)), createKeyProvider, createWebhookNotifier, - createClientConfiguration, createForwardStats, + getNodeStatsConfig, routing.CreateRouter, getLimitConf, - config.DefaultAPIConfig, + getAPIConf, wire.Bind(new(routing.MessageRouter), new(routing.Router)), wire.Bind(new(livekit.RoomService), new(*RoomService)), telemetry.NewAnalyticsService, - telemetry.NewTelemetryService, + createTelemetryService, getMessageBus, NewIOInfoService, wire.Bind(new(IOClient), new(*IOInfoService)), @@ -71,15 +72,17 @@ func InitializeServer(conf *config.Config, currentNode routing.LocalNode) (*Live getIngressStore, getIngressConfig, NewIngressService, - rpc.NewSIPClient, + newSIPClient, getSIPStore, getSIPConfig, NewSIPService, NewRoomAllocator, NewRoomService, NewRTCService, + NewWHIPService, NewAgentService, NewAgentDispatchService, + getAgentConfig, agent.NewAgentClient, getAgentStore, getSignalRelayConfig, @@ -93,6 +96,7 @@ func InitializeServer(conf *config.Config, currentNode routing.LocalNode) (*Live rpc.NewTopicFormatter, rpc.NewTypedRoomClient, rpc.NewTypedParticipantClient, + rpc.NewTypedWHIPParticipantClient, rpc.NewTypedAgentDispatchInternalClient, NewLocalRoomManager, NewTURNAuthHandler, @@ -116,6 +120,7 @@ func InitializeRouter(conf *config.Config, currentNode routing.LocalNode) (routi getRoomConfig, routing.NewRoomManagerClient, rpc.NewKeepalivePubSub, + getNodeStatsConfig, routing.CreateRouter, ) @@ -157,15 +162,21 @@ func createKeyProvider(conf *config.Config) (auth.KeyProvider, error) { func createWebhookNotifier(conf *config.Config, provider auth.KeyProvider) (webhook.QueuedNotifier, error) { wc := conf.WebHook - if len(wc.URLs) == 0 { - return nil, nil - } + secret := provider.GetSecret(wc.APIKey) - if secret == "" { + if secret == "" && len(wc.URLs) > 0 { return nil, ErrWebHookMissingAPIKey } - return webhook.NewDefaultNotifier(wc.APIKey, secret, wc.URLs), nil + return webhook.NewDefaultNotifier(wc, provider) +} + +func createTelemetryService(notifier webhook.QueuedNotifier, analytics telemetry.AnalyticsService) telemetry.TelemetryService { + svc := telemetry.NewTelemetryService(notifier, analytics) + if notifier != nil { + notifier.RegisterProcessedHook(svc.Webhook) + } + return svc } func createRedisClient(conf *config.Config) (redis.UniversalClient, error) { @@ -222,6 +233,19 @@ func getIngressConfig(conf *config.Config) *config.IngressConfig { return &conf.Ingress } +func newSIPClient(p rpc.ClientParams) (rpc.SIPClient, error) { + // Do not pass parameters directly, as they set timeout that is too short, + // and might set retry policy that is not acceptable for SIP methods. + // Instead, set relevant parameters manually. + return rpc.NewSIPClientWithParams(rpc.ClientParams{ + Bus: p.Bus, + ClientOptions: []psrpc.ClientOption{ + rpc.WithClientLogger(p.Logger), + otelpsrpc.ClientOptions(otelpsrpc.Config{}), + }, + }) +} + func getSIPStore(s ObjectStore) SIPStore { switch store := s.(type) { case *RedisStore: @@ -235,10 +259,6 @@ func getSIPConfig(conf *config.Config) *config.SIPConfig { return &conf.SIP } -func createClientConfiguration() clientconfiguration.ClientConfigurationManager { - return clientconfiguration.NewStaticClientConfigurationManager(clientconfiguration.StaticConfigurations) -} - func getLimitConf(config *config.Config) config.LimitConfig { return config.Limit } @@ -256,7 +276,9 @@ func getPSRPCConfig(config *config.Config) rpc.PSRPCConfig { } func getPSRPCClientParams(config rpc.PSRPCConfig, bus psrpc.MessageBus) rpc.ClientParams { - return rpc.NewClientParams(config, bus, logger.GetLogger(), rpc.PSRPCMetricsObserver{}) + return rpc.NewClientParams(config, bus, logger.GetLogger(), rpc.PSRPCMetricsObserver{}, + otelpsrpc.ClientOptions(otelpsrpc.Config{}), + ) } func createForwardStats(conf *config.Config) *sfu.ForwardStats { @@ -269,3 +291,15 @@ func createForwardStats(conf *config.Config) *sfu.ForwardStats { func newInProcessTurnServer(conf *config.Config, authHandler turn.AuthHandler) (*turn.Server, error) { return NewTurnServer(conf, authHandler, false) } + +func getNodeStatsConfig(config *config.Config) config.NodeStatsConfig { + return config.NodeStats +} + +func getAgentConfig(config *config.Config) agent.Config { + return config.Agents +} + +func getAPIConf(config *config.Config) config.APIConfig { + return config.API +} diff --git a/pkg/service/wire_gen.go b/pkg/service/wire_gen.go index 1ef7867..5e9eff9 100644 --- a/pkg/service/wire_gen.go +++ b/pkg/service/wire_gen.go @@ -9,7 +9,6 @@ package service import ( "fmt" "github.com/livekit/livekit-server/pkg/agent" - "github.com/livekit/livekit-server/pkg/clientconfiguration" "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/livekit-server/pkg/routing" "github.com/livekit/livekit-server/pkg/sfu" @@ -22,7 +21,8 @@ import ( "github.com/livekit/protocol/utils" "github.com/livekit/protocol/webhook" "github.com/livekit/psrpc" - "github.com/pion/turn/v4" + "github.com/livekit/psrpc/pkg/middleware/otelpsrpc" + "github.com/pion/turn/v5" "github.com/pkg/errors" "github.com/redis/go-redis/v9" "gopkg.in/yaml.v3" @@ -37,7 +37,7 @@ import ( func InitializeServer(conf *config.Config, currentNode routing.LocalNode) (*LivekitServer, error) { limitConfig := getLimitConf(conf) - apiConfig := config.DefaultAPIConfig() + apiConfig := getAPIConf(conf) universalClient, err := createRedisClient(conf) if err != nil { return nil, err @@ -60,7 +60,8 @@ func InitializeServer(conf *config.Config, currentNode routing.LocalNode) (*Live if err != nil { return nil, err } - router := routing.CreateRouter(universalClient, currentNode, signalClient, roomManagerClient, keepalivePubSub) + nodeStatsConfig := getNodeStatsConfig(conf) + router := routing.CreateRouter(universalClient, currentNode, signalClient, roomManagerClient, keepalivePubSub, nodeStatsConfig) objectStore := createStore(universalClient) roomAllocator, err := NewRoomAllocator(conf, router, objectStore) if err != nil { @@ -82,12 +83,12 @@ func InitializeServer(conf *config.Config, currentNode routing.LocalNode) (*Live return nil, err } analyticsService := telemetry.NewAnalyticsService(conf, currentNode) - telemetryService := telemetry.NewTelemetryService(queuedNotifier, analyticsService) + telemetryService := createTelemetryService(queuedNotifier, analyticsService) ioInfoService, err := NewIOInfoService(messageBus, egressStore, ingressStore, sipStore, telemetryService) if err != nil { return nil, err } - rtcEgressLauncher := NewEgressLauncher(egressClient, ioInfoService) + rtcEgressLauncher := NewEgressLauncher(egressClient, ioInfoService, objectStore) topicFormatter := rpc.NewTopicFormatter() roomClient, err := rpc.NewTypedRoomClient(clientParams) if err != nil { @@ -106,7 +107,7 @@ func InitializeServer(conf *config.Config, currentNode routing.LocalNode) (*Live return nil, err } agentDispatchService := NewAgentDispatchService(agentDispatchInternalClient, topicFormatter, roomAllocator, router) - egressService := NewEgressService(egressClient, rtcEgressLauncher, objectStore, ioInfoService, roomService) + egressService := NewEgressService(egressClient, rtcEgressLauncher, ioInfoService, roomService) ingressConfig := getIngressConfig(conf) ingressClient, err := rpc.NewIngressClient(clientParams) if err != nil { @@ -114,18 +115,26 @@ func InitializeServer(conf *config.Config, currentNode routing.LocalNode) (*Live } ingressService := NewIngressService(ingressConfig, nodeID, messageBus, ingressClient, ingressStore, ioInfoService, telemetryService) sipConfig := getSIPConfig(conf) - sipClient, err := rpc.NewSIPClient(messageBus) + sipClient, err := newSIPClient(clientParams) if err != nil { return nil, err } sipService := NewSIPService(sipConfig, nodeID, messageBus, sipClient, sipStore, roomService, telemetryService) - rtcService := NewRTCService(conf, roomAllocator, objectStore, router, currentNode, telemetryService) + rtcService := NewRTCService(conf, roomAllocator, router, telemetryService) + whipParticipantClient, err := rpc.NewTypedWHIPParticipantClient(clientParams) + if err != nil { + return nil, err + } + serviceWHIPService, err := NewWHIPService(conf, router, roomAllocator, clientParams, topicFormatter, whipParticipantClient) + if err != nil { + return nil, err + } agentService, err := NewAgentService(conf, currentNode, messageBus, keyProvider) if err != nil { return nil, err } - clientConfigurationManager := createClientConfiguration() - client, err := agent.NewAgentClient(messageBus) + agentConfig := getAgentConfig(conf) + client, err := agent.NewAgentClient(messageBus, agentConfig) if err != nil { return nil, err } @@ -133,7 +142,7 @@ func InitializeServer(conf *config.Config, currentNode routing.LocalNode) (*Live timedVersionGenerator := utils.NewDefaultTimedVersionGenerator() turnAuthHandler := NewTURNAuthHandler(keyProvider) forwardStats := createForwardStats(conf) - roomManager, err := NewLocalRoomManager(conf, objectStore, currentNode, router, roomAllocator, telemetryService, clientConfigurationManager, client, agentStore, rtcEgressLauncher, timedVersionGenerator, turnAuthHandler, messageBus, forwardStats) + roomManager, err := NewLocalRoomManager(conf, objectStore, currentNode, router, roomAllocator, telemetryService, client, agentStore, rtcEgressLauncher, timedVersionGenerator, turnAuthHandler, messageBus, forwardStats) if err != nil { return nil, err } @@ -146,7 +155,7 @@ func InitializeServer(conf *config.Config, currentNode routing.LocalNode) (*Live if err != nil { return nil, err } - livekitServer, err := NewLivekitServer(conf, roomService, agentDispatchService, egressService, ingressService, sipService, ioInfoService, rtcService, agentService, keyProvider, router, roomManager, signalServer, server, currentNode) + livekitServer, err := NewLivekitServer(conf, roomService, agentDispatchService, egressService, ingressService, sipService, ioInfoService, rtcService, serviceWHIPService, agentService, keyProvider, router, roomManager, signalServer, server, currentNode) if err != nil { return nil, err } @@ -176,7 +185,8 @@ func InitializeRouter(conf *config.Config, currentNode routing.LocalNode) (routi if err != nil { return nil, err } - router := routing.CreateRouter(universalClient, currentNode, signalClient, roomManagerClient, keepalivePubSub) + nodeStatsConfig := getNodeStatsConfig(conf) + router := routing.CreateRouter(universalClient, currentNode, signalClient, roomManagerClient, keepalivePubSub, nodeStatsConfig) return router, nil } @@ -217,15 +227,21 @@ func createKeyProvider(conf *config.Config) (auth.KeyProvider, error) { func createWebhookNotifier(conf *config.Config, provider auth.KeyProvider) (webhook.QueuedNotifier, error) { wc := conf.WebHook - if len(wc.URLs) == 0 { - return nil, nil - } + secret := provider.GetSecret(wc.APIKey) - if secret == "" { + if secret == "" && len(wc.URLs) > 0 { return nil, ErrWebHookMissingAPIKey } - return webhook.NewDefaultNotifier(wc.APIKey, secret, wc.URLs), nil + return webhook.NewDefaultNotifier(wc, provider) +} + +func createTelemetryService(notifier webhook.QueuedNotifier, analytics telemetry.AnalyticsService) telemetry.TelemetryService { + svc := telemetry.NewTelemetryService(notifier, analytics) + if notifier != nil { + notifier.RegisterProcessedHook(svc.Webhook) + } + return svc } func createRedisClient(conf *config.Config) (redis.UniversalClient, error) { @@ -282,6 +298,14 @@ func getIngressConfig(conf *config.Config) *config.IngressConfig { return &conf.Ingress } +func newSIPClient(p rpc.ClientParams) (rpc.SIPClient, error) { + + return rpc.NewSIPClientWithParams(rpc.ClientParams{ + Bus: p.Bus, + ClientOptions: []psrpc.ClientOption{rpc.WithClientLogger(p.Logger), otelpsrpc.ClientOptions(otelpsrpc.Config{})}, + }) +} + func getSIPStore(s ObjectStore) SIPStore { switch store := s.(type) { case *RedisStore: @@ -295,10 +319,6 @@ func getSIPConfig(conf *config.Config) *config.SIPConfig { return &conf.SIP } -func createClientConfiguration() clientconfiguration.ClientConfigurationManager { - return clientconfiguration.NewStaticClientConfigurationManager(clientconfiguration.StaticConfigurations) -} - func getLimitConf(config2 *config.Config) config.LimitConfig { return config2.Limit } @@ -316,7 +336,7 @@ func getPSRPCConfig(config2 *config.Config) rpc.PSRPCConfig { } func getPSRPCClientParams(config2 rpc.PSRPCConfig, bus psrpc.MessageBus) rpc.ClientParams { - return rpc.NewClientParams(config2, bus, logger.GetLogger(), rpc.PSRPCMetricsObserver{}) + return rpc.NewClientParams(config2, bus, logger.GetLogger(), rpc.PSRPCMetricsObserver{}, otelpsrpc.ClientOptions(otelpsrpc.Config{})) } func createForwardStats(conf *config.Config) *sfu.ForwardStats { @@ -329,3 +349,15 @@ func createForwardStats(conf *config.Config) *sfu.ForwardStats { func newInProcessTurnServer(conf *config.Config, authHandler turn.AuthHandler) (*turn.Server, error) { return NewTurnServer(conf, authHandler, false) } + +func getNodeStatsConfig(config2 *config.Config) config.NodeStatsConfig { + return config2.NodeStats +} + +func getAgentConfig(config2 *config.Config) agent.Config { + return config2.Agents +} + +func getAPIConf(config2 *config.Config) config.APIConfig { + return config2.API +} diff --git a/pkg/service/wsprotocol.go b/pkg/service/wsprotocol.go index 9f4b50e..6c676d0 100644 --- a/pkg/service/wsprotocol.go +++ b/pkg/service/wsprotocol.go @@ -22,18 +22,19 @@ import ( "time" "github.com/gorilla/websocket" - "google.golang.org/protobuf/encoding/protojson" "google.golang.org/protobuf/proto" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" + "github.com/livekit/protocol/utils/protojson" "github.com/livekit/livekit-server/pkg/rtc/types" ) const ( - pingFrequency = 10 * time.Second - pingTimeout = 2 * time.Second + pingFrequency = 10 * time.Second + pingTimeout = 2 * time.Second + closeWriteTimeout = 5 * time.Second ) type WSSignalConnection struct { @@ -56,75 +57,77 @@ func (c *WSSignalConnection) Close() error { return c.conn.Close() } +func (c *WSSignalConnection) CloseWithReason(reason string) error { + msg := websocket.FormatCloseMessage(websocket.CloseNormalClosure, reason) + _ = c.conn.WriteControl(websocket.CloseMessage, msg, time.Now().Add(closeWriteTimeout)) + return c.conn.Close() +} + func (c *WSSignalConnection) SetReadDeadline(deadline time.Time) error { return c.conn.SetReadDeadline(deadline) } func (c *WSSignalConnection) ReadRequest() (*livekit.SignalRequest, int, error) { - for { - // handle special messages and pass on the rest - messageType, payload, err := c.conn.ReadMessage() - if err != nil { - return nil, 0, err - } + // handle special messages and pass on the rest + messageType, payload, err := c.conn.ReadMessage() + if err != nil { + return nil, 0, err + } - msg := &livekit.SignalRequest{} - switch messageType { - case websocket.BinaryMessage: - if c.useJSON { - c.mu.Lock() - // switch to protobuf if client supports it - c.useJSON = false - c.mu.Unlock() - } - // protobuf encoded - err := proto.Unmarshal(payload, msg) - return msg, len(payload), err - case websocket.TextMessage: + msg := &livekit.SignalRequest{} + switch messageType { + case websocket.BinaryMessage: + if c.useJSON { c.mu.Lock() - // json encoded, also write back JSON - c.useJSON = true + // switch to protobuf if client supports it + c.useJSON = false c.mu.Unlock() - err := protojson.Unmarshal(payload, msg) - return msg, len(payload), err - default: - logger.Debugw("unsupported message", "message", messageType) - return nil, len(payload), nil } + // protobuf encoded + err := proto.Unmarshal(payload, msg) + return msg, len(payload), err + case websocket.TextMessage: + c.mu.Lock() + // json encoded, also write back JSON + c.useJSON = true + c.mu.Unlock() + err := protojson.Unmarshal(payload, msg) + return msg, len(payload), err + default: + logger.Debugw("unsupported message", "message", messageType) + return nil, len(payload), nil } } func (c *WSSignalConnection) ReadWorkerMessage() (*livekit.WorkerMessage, int, error) { - for { - // handle special messages and pass on the rest - messageType, payload, err := c.conn.ReadMessage() - if err != nil { - return nil, 0, err - } + // handle special messages and pass on the rest + messageType, payload, err := c.conn.ReadMessage() + if err != nil { + return nil, 0, err + } - msg := &livekit.WorkerMessage{} - switch messageType { - case websocket.BinaryMessage: - if c.useJSON { - c.mu.Lock() - // switch to protobuf if client supports it - c.useJSON = false - c.mu.Unlock() - } - // protobuf encoded - err := proto.Unmarshal(payload, msg) - return msg, len(payload), err - case websocket.TextMessage: + msg := &livekit.WorkerMessage{} + switch messageType { + case websocket.BinaryMessage: + if c.useJSON { c.mu.Lock() - // json encoded, also write back JSON - c.useJSON = true + // switch to protobuf if client supports it + c.useJSON = false c.mu.Unlock() - err := protojson.Unmarshal(payload, msg) - return msg, len(payload), err - default: - logger.Debugw("unsupported message", "message", messageType) - return nil, len(payload), nil } + // protobuf encoded + err := proto.Unmarshal(payload, msg) + return msg, len(payload), err + case websocket.TextMessage: + c.mu.Lock() + // json encoded, also write back JSON + c.useJSON = true + c.mu.Unlock() + err := protojson.Unmarshal(payload, msg) + return msg, len(payload), err + default: + logger.Debugw("unsupported message", "message", messageType) + return nil, len(payload), nil } } diff --git a/pkg/sfu/audio/audiolevel.go b/pkg/sfu/audio/audiolevel.go index 02c91c8..b97e0a4 100644 --- a/pkg/sfu/audio/audiolevel.go +++ b/pkg/sfu/audio/audiolevel.go @@ -51,12 +51,15 @@ var ( // -------------------------------------- type AudioLevelParams struct { - Config AudioLevelConfig + ClockRate uint32 } // keeps track of audio level for a participant type AudioLevel struct { params AudioLevelParams + + config AudioLevelConfig + // min duration within an observe duration window to be considered active minActiveDuration uint32 smoothFactor float64 @@ -69,23 +72,32 @@ type AudioLevel struct { activeDuration uint32 // ms observedDuration uint32 // ms lastObservedAt int64 + + highestRTPTimestamp uint32 + highestRTPTimestampInitialized bool } func NewAudioLevel(params AudioLevelParams) *AudioLevel { - l := &AudioLevel{ + return &AudioLevel{ params: params, - minActiveDuration: uint32(params.Config.MinPercentile) * params.Config.UpdateInterval / 100, smoothFactor: 1, - activeThreshold: ConvertAudioLevel(float64(params.Config.ActiveLevel)), loudestObservedLevel: silentAudioLevel, } +} - if l.params.Config.SmoothIntervals > 0 { +func (l *AudioLevel) SetConfig(config AudioLevelConfig) { + l.lock.Lock() + defer l.lock.Unlock() + + l.config = config + l.minActiveDuration = uint32(l.config.MinPercentile) * l.config.UpdateInterval / 100 + if l.config.SmoothIntervals > 0 { // exponential moving average (EMA), same center of mass with simple moving average (SMA) - l.smoothFactor = float64(2) / (float64(l.params.Config.SmoothIntervals + 1)) + l.smoothFactor = float64(2) / (float64(l.config.SmoothIntervals + 1)) + } else { + l.smoothFactor = 1 } - - return l + l.activeThreshold = ConvertAudioLevel(float64(l.config.ActiveLevel)) } // Observes a new frame @@ -93,18 +105,26 @@ func (l *AudioLevel) Observe(level uint8, durationMs uint32, arrivalTime int64) l.lock.Lock() defer l.lock.Unlock() + l.observeLocked(level, durationMs, arrivalTime) +} + +func (l *AudioLevel) observeLocked(level uint8, durationMs uint32, arrivalTime int64) { + if l.config.UpdateInterval == 0 { + return + } + l.lastObservedAt = arrivalTime l.observedDuration += durationMs - if level <= l.params.Config.ActiveLevel { + if level <= l.config.ActiveLevel { l.activeDuration += durationMs if l.loudestObservedLevel > level { l.loudestObservedLevel = level } } - if l.observedDuration >= l.params.Config.UpdateInterval { + if l.observedDuration >= l.config.UpdateInterval { smoothedLevel := float64(0.0) // compute and reset if l.activeDuration >= l.minActiveDuration { @@ -112,7 +132,7 @@ func (l *AudioLevel) Observe(level uint8, durationMs uint32, arrivalTime int64) // Weight will be 0 if active the entire duration // > 0 if active for longer than observe duration // < 0 if active for less than observe duration - activityWeight := 20 * math.Log10(float64(l.activeDuration)/float64(l.params.Config.UpdateInterval)) + activityWeight := 20 * math.Log10(float64(l.activeDuration)/float64(l.config.UpdateInterval)) adjustedLevel := float64(l.loudestObservedLevel) - activityWeight linearLevel := ConvertAudioLevel(adjustedLevel) @@ -123,18 +143,39 @@ func (l *AudioLevel) Observe(level uint8, durationMs uint32, arrivalTime int64) } } +func (l *AudioLevel) ObserveWithRTPTimestamp(level uint8, ts uint32, arrivalTime int64) { + l.lock.Lock() + defer l.lock.Unlock() + + if !l.highestRTPTimestampInitialized { + l.highestRTPTimestampInitialized = true + l.highestRTPTimestamp = ts + } + + if (ts - l.highestRTPTimestamp) < (1 << 31) { + durationMs := (ts - l.highestRTPTimestamp) * 1e3 / l.params.ClockRate + l.observeLocked(level, durationMs, arrivalTime) + + l.highestRTPTimestamp = ts + } +} + // returns current smoothed audio level func (l *AudioLevel) GetLevel(now int64) (float64, bool) { l.lock.Lock() defer l.lock.Unlock() + if l.config.UpdateInterval == 0 { + return 0.0, false + } + l.resetIfStaleLocked(now) return l.smoothedLevel, l.smoothedLevel >= l.activeThreshold } func (l *AudioLevel) resetIfStaleLocked(arrivalTime int64) { - if (arrivalTime-l.lastObservedAt)/1e6 < int64(2*l.params.Config.UpdateInterval) { + if (arrivalTime-l.lastObservedAt)/1e6 < int64(2*l.config.UpdateInterval) { return } diff --git a/pkg/sfu/audio/audiolevel_test.go b/pkg/sfu/audio/audiolevel_test.go index 61990df..7876b28 100644 --- a/pkg/sfu/audio/audiolevel_test.go +++ b/pkg/sfu/audio/audiolevel_test.go @@ -15,6 +15,7 @@ package audio import ( + "math/rand" "testing" "time" @@ -104,20 +105,54 @@ func TestAudioLevel(t *testing.T) { require.Equal(t, float64(0.0), level) require.False(t, noisy) }) -} -func createAudioLevel(activeLevel uint8, minPercentile uint8, observeDuration uint32) *AudioLevel { - return NewAudioLevel(AudioLevelParams{ - Config: AudioLevelConfig{ - ActiveLevel: activeLevel, - MinPercentile: minPercentile, - UpdateInterval: observeDuration, - }, + t.Run("not noisy when samples are stale - with RTP timestamp", func(t *testing.T) { + clock := time.Now() + a := createAudioLevel(defaultActiveLevel, defaultPercentile, defaultObserveDuration) + + observeSamplesWithRTPTimestamp(a, 25, 100, clock) + clock = clock.Add(100 * 20 * time.Millisecond) + level, noisy := a.GetLevel(clock.UnixNano()) + require.True(t, noisy) + require.Greater(t, level, ConvertAudioLevel(float64(defaultActiveLevel))) + require.Less(t, level, ConvertAudioLevel(float64(20))) + + // let enough time pass to make the samples stale + clock = clock.Add(1500 * time.Millisecond) + level, noisy = a.GetLevel(clock.UnixNano()) + require.Equal(t, float64(0.0), level) + require.False(t, noisy) }) } +func createAudioLevel(activeLevel uint8, minPercentile uint8, observeDuration uint32) *AudioLevel { + al := NewAudioLevel(AudioLevelParams{ + ClockRate: 48000, + }) + al.SetConfig(AudioLevelConfig{ + ActiveLevel: activeLevel, + MinPercentile: minPercentile, + UpdateInterval: observeDuration, + }) + return al +} + func observeSamples(a *AudioLevel, level uint8, count int, baseTime time.Time) { - for i := 0; i < count; i++ { - a.Observe(level, 20, baseTime.Add(+time.Duration(i*20)*time.Millisecond).UnixNano()) + for i := range count { + a.Observe(level, 20, baseTime.Add(time.Duration(i*20)*time.Millisecond).UnixNano()) + } +} + +func observeSamplesWithRTPTimestamp(a *AudioLevel, level uint8, count int, baseTime time.Time) { + sampleTS := uint32(rand.Intn(1 << 20)) + sampleTime := baseTime + for i := range count { + if (i % 5) == 0 { + // out-of-order sample + a.ObserveWithRTPTimestamp(level, sampleTS-1920, sampleTime.UnixNano()) + } + a.ObserveWithRTPTimestamp(level, sampleTS, sampleTime.UnixNano()) + sampleTS += 960 // 20 ms at 48 kHz + sampleTime = sampleTime.Add(20 * time.Millisecond) } } diff --git a/pkg/sfu/buffer/buffer.go b/pkg/sfu/buffer/buffer.go index 427ca1c..5b376a4 100644 --- a/pkg/sfu/buffer/buffer.go +++ b/pkg/sfu/buffer/buffer.go @@ -17,169 +17,76 @@ package buffer import ( "encoding/binary" "errors" - "fmt" "io" - "strings" - "sync" - "time" - "github.com/gammazero/deque" "github.com/pion/rtcp" "github.com/pion/rtp" - "github.com/pion/rtp/codecs" - "github.com/pion/sdp/v3" "github.com/pion/webrtc/v4" - "go.uber.org/atomic" - "github.com/livekit/livekit-server/pkg/sfu/audio" - "github.com/livekit/livekit-server/pkg/sfu/mime" - act "github.com/livekit/livekit-server/pkg/sfu/rtpextension/abscapturetime" - dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor" - "github.com/livekit/livekit-server/pkg/sfu/rtpstats" - "github.com/livekit/livekit-server/pkg/sfu/utils" sutils "github.com/livekit/livekit-server/pkg/utils" "github.com/livekit/mediatransportutil/pkg/bucket" - "github.com/livekit/mediatransportutil/pkg/nack" "github.com/livekit/mediatransportutil/pkg/twcc" "github.com/livekit/protocol/livekit" - "github.com/livekit/protocol/logger" "github.com/livekit/protocol/utils/mono" ) const ( - ReportDelta = 1e9 + rtcpReceiverReportDelta = 1e9 InitPacketBufferSizeVideo = 300 InitPacketBufferSizeAudio = 70 ) +var ( + errInvalidCodec = errors.New("invalid codec") +) + +var _ BufferProvider = (*Buffer)(nil) + type pendingPacket struct { arrivalTime int64 packet []byte } -type ExtPacket struct { - VideoLayer - Arrival int64 - ExtSequenceNumber uint64 - ExtTimestamp uint64 - Packet *rtp.Packet - Payload interface{} - KeyFrame bool - RawPacket []byte - DependencyDescriptor *ExtDependencyDescriptor - AbsCaptureTimeExt *act.AbsCaptureTime - IsOutOfOrder bool -} - // Buffer contains all packets type Buffer struct { - sync.RWMutex - readCond *sync.Cond - bucket *bucket.Bucket[uint64] - nacker *nack.NackQueue - maxVideoPkts int - maxAudioPkts int - codecType webrtc.RTPCodecType - extPackets deque.Deque[*ExtPacket] - pPackets []pendingPacket - closeOnce sync.Once - mediaSSRC uint32 - clockRate uint32 - lastReport int64 - twccExtID uint8 - audioLevelExtID uint8 - bound bool - closed atomic.Bool + *BufferBase - rtpParameters webrtc.RTPParameters - payloadType uint8 - rtxPayloadType uint8 - mime mime.MimeType + pPackets []pendingPacket + lastReportAt int64 + isBound bool - snRangeMap *utils.RangeMap[uint64, uint64] + twcc *twcc.Responder + twccExtID uint8 - latestTSForAudioLevelInitialized bool - latestTSForAudioLevel uint32 - - twcc *twcc.Responder - audioLevelParams audio.AudioLevelParams - audioLevel *audio.AudioLevel - enableAudioLossProxying bool + enableAudioLossProxying bool + lastFractionLostToReport uint8 // Last fraction lost from subscribers, should report to publisher; Audio only lastPacketRead int - pliThrottle int64 - - rtpStats *rtpstats.RTPStatsReceiver - rrSnapshotId uint32 - deltaStatsSnapshotId uint32 - ppsSnapshotId uint32 - - lastFractionLostToReport uint8 // Last fraction lost from subscribers, should report to publisher; Audio only - // callbacks - onClose func() - onRtcpFeedback func([]rtcp.Packet) - onRtcpSenderReport func() - onFpsChanged func() - onFinalRtpStats func(*livekit.RTPStats) - onCodecChange func(webrtc.RTPCodecParameters) - - // logger - logger logger.Logger - - // dependency descriptor - ddExtID uint8 - ddParser *DependencyDescriptorParser - - paused bool - frameRateCalculator [DefaultMaxLayerSpatial + 1]FrameRateCalculator - frameRateCalculated bool - - packetNotFoundCount atomic.Uint32 - packetTooOldCount atomic.Uint32 - extPacketTooMuchCount atomic.Uint32 + onClose func() + onRtcpFeedback func([]rtcp.Packet) + onFinalRtpStats func(*livekit.RTPStats) + onNotifyRTX func(uint32, uint32, string) primaryBufferForRTX *Buffer rtxPktBuf []byte - - absCaptureTimeExtID uint8 } -// NewBuffer constructs a new Buffer func NewBuffer(ssrc uint32, maxVideoPkts, maxAudioPkts int) *Buffer { - l := logger.GetLogger() // will be reset with correct context via SetLogger - b := &Buffer{ - mediaSSRC: ssrc, - maxVideoPkts: maxVideoPkts, - maxAudioPkts: maxAudioPkts, - snRangeMap: utils.NewRangeMap[uint64, uint64](100), - pliThrottle: int64(500 * time.Millisecond), - logger: l.WithComponent(sutils.ComponentPub).WithComponent(sutils.ComponentSFU), - } - b.readCond = sync.NewCond(&b.RWMutex) - b.extPackets.SetBaseCap(128) + b := &Buffer{} + b.BufferBase = NewBufferBase(BufferBaseParams{ + SSRC: ssrc, + MaxVideoPkts: maxVideoPkts, + MaxAudioPkts: maxAudioPkts, + LoggerComponents: []string{sutils.ComponentPub, sutils.ComponentSFU}, + SendPLI: b.sendPLI, + IsReportingEnabled: true, + }) return b } -func (b *Buffer) SetLogger(logger logger.Logger) { - b.Lock() - defer b.Unlock() - - b.logger = logger.WithComponent(sutils.ComponentPub).WithComponent(sutils.ComponentSFU).WithValues("ssrc", b.mediaSSRC) - if b.rtpStats != nil { - b.rtpStats.SetLogger(b.logger) - } -} - -func (b *Buffer) SetPaused(paused bool) { - b.Lock() - defer b.Unlock() - - b.paused = paused -} - func (b *Buffer) SetTWCCAndExtID(twcc *twcc.Responder, extID uint8) { b.Lock() defer b.Unlock() @@ -188,13 +95,6 @@ func (b *Buffer) SetTWCCAndExtID(twcc *twcc.Responder, extID uint8) { b.twccExtID = extID } -func (b *Buffer) SetAudioLevelParams(audioLevelParams audio.AudioLevelParams) { - b.Lock() - defer b.Unlock() - - b.audioLevelParams = audioLevelParams -} - func (b *Buffer) SetAudioLossProxying(enable bool) { b.Lock() defer b.Unlock() @@ -202,144 +102,39 @@ func (b *Buffer) SetAudioLossProxying(enable bool) { b.enableAudioLossProxying = enable } -func (b *Buffer) Bind(params webrtc.RTPParameters, codec webrtc.RTPCodecCapability, bitrates int) { +func (b *Buffer) Bind(params webrtc.RTPParameters, codec webrtc.RTPCodecCapability, bitrates int) error { b.Lock() - defer b.Unlock() - if b.bound { - return + if b.isBound { + b.Unlock() + return nil } - b.rtpStats = rtpstats.NewRTPStatsReceiver(rtpstats.RTPStatsParams{ - ClockRate: codec.ClockRate, - Logger: b.logger, - }) - b.rrSnapshotId = b.rtpStats.NewSnapshotId() - b.deltaStatsSnapshotId = b.rtpStats.NewSnapshotId() - b.ppsSnapshotId = b.rtpStats.NewSnapshotId() - - b.clockRate = codec.ClockRate - b.lastReport = mono.UnixNano() - b.mime = mime.NormalizeMimeType(codec.MimeType) - b.rtpParameters = params - for _, codecParameter := range params.Codecs { - if mime.IsMimeTypeStringEqual(codecParameter.MimeType, codec.MimeType) { - b.payloadType = uint8(codecParameter.PayloadType) - break - } + if err := b.BufferBase.BindLocked(params, codec, bitrates); err != nil { + b.Unlock() + return err } - if b.payloadType == 0 { - b.logger.Warnw("could not find payload type for codec", nil, "codec", codec.MimeType, "parameters", params) - b.payloadType = uint8(params.Codecs[0].PayloadType) + b.lastReportAt = mono.UnixNano() + + if len(b.pPackets) != 0 { + b.logger.Debugw("releasing queued packets on bind", "count", len(b.pPackets)) } - - // find RTX payload type - for _, codec := range params.Codecs { - if mime.IsMimeTypeStringRTX(codec.MimeType) && strings.Contains(codec.SDPFmtpLine, fmt.Sprintf("apt=%d", b.payloadType)) { - b.rtxPayloadType = uint8(codec.PayloadType) - break - } - } - - for _, ext := range params.HeaderExtensions { - switch ext.URI { - case dd.ExtensionURI: - if b.ddExtID != 0 { - b.logger.Warnw("multiple dependency descriptor extensions found", nil, "id", ext.ID, "previous", b.ddExtID) - continue - } - b.ddExtID = uint8(ext.ID) - b.createDDParserAndFrameRateCalculator() - - case sdp.AudioLevelURI: - b.audioLevelExtID = uint8(ext.ID) - b.audioLevel = audio.NewAudioLevel(b.audioLevelParams) - - case act.AbsCaptureTimeURI: - b.absCaptureTimeExtID = uint8(ext.ID) - } - } - - switch { - case mime.IsMimeTypeAudio(b.mime): - b.codecType = webrtc.RTPCodecTypeAudio - b.bucket = bucket.NewBucket[uint64](InitPacketBufferSizeAudio) - - case mime.IsMimeTypeVideo(b.mime): - b.codecType = webrtc.RTPCodecTypeVideo - b.bucket = bucket.NewBucket[uint64](InitPacketBufferSizeVideo) - if b.frameRateCalculator[0] == nil { - b.createFrameRateCalculator() - } - if bitrates > 0 { - pps := bitrates / 8 / 1200 - for pps > b.bucket.Capacity() { - if b.bucket.Grow() >= b.maxVideoPkts { - break - } - } - } - - default: - b.codecType = webrtc.RTPCodecType(0) - } - - for _, fb := range codec.RTCPFeedback { - switch fb.Type { - case webrtc.TypeRTCPFBGoogREMB: - b.logger.Debugw("Setting feedback", "type", webrtc.TypeRTCPFBGoogREMB) - b.logger.Debugw("REMB not supported, RTCP feedback will not be generated") - case webrtc.TypeRTCPFBNACK: - // pion use a single mediaengine to manage negotiated codecs of peerconnection, that means we can't have different - // codec settings at track level for same codec type, so enable nack for all audio receivers but don't create nack queue - // for red codec. - if b.mime == mime.MimeTypeRED { - break - } - b.logger.Debugw("Setting feedback", "type", webrtc.TypeRTCPFBNACK) - b.nacker = nack.NewNACKQueue(nack.NackQueueParamsDefault) - } - } - + var rtcpPackets []rtcp.Packet for _, pp := range b.pPackets { - b.calc(pp.packet, nil, pp.arrivalTime, false) + rtcpPackets = append(rtcpPackets, b.calc(pp.packet, nil, pp.arrivalTime, true, false)...) } b.pPackets = nil - b.bound = true -} -func (b *Buffer) OnCodecChange(fn func(webrtc.RTPCodecParameters)) { - b.Lock() - b.onCodecChange = fn + b.isBound = true b.Unlock() -} -func (b *Buffer) createDDParserAndFrameRateCalculator() { - if mime.IsMimeTypeSVC(b.mime) || b.mime == mime.MimeTypeVP8 { - frc := NewFrameRateCalculatorDD(b.clockRate, b.logger) - for i := range b.frameRateCalculator { - b.frameRateCalculator[i] = frc.GetFrameRateCalculatorForSpatial(int32(i)) + if len(rtcpPackets) != 0 { + if cb := b.getOnRtcpFeedback(); cb != nil { + cb(rtcpPackets) } - b.ddParser = NewDependencyDescriptorParser(b.ddExtID, b.logger, func(spatial, temporal int32) { - frc.SetMaxLayer(spatial, temporal) - }) } -} -func (b *Buffer) createFrameRateCalculator() { - switch b.mime { - case mime.MimeTypeVP8: - b.frameRateCalculator[0] = NewFrameRateCalculatorVP8(b.clockRate, b.logger) - - case mime.MimeTypeVP9: - frc := NewFrameRateCalculatorVP9(b.clockRate, b.logger) - for i := range b.frameRateCalculator { - b.frameRateCalculator[i] = frc.GetFrameRateCalculatorForSpatial(int32(i)) - } - - case mime.MimeTypeH265: - b.frameRateCalculator[0] = NewFrameRateCalculatorH26x(b.clockRate, b.logger) - } + return nil } // Write adds an RTP Packet, ordering is not guaranteed, newer packets may arrive later @@ -351,14 +146,14 @@ func (b *Buffer) Write(pkt []byte) (n int, err error) { } b.Lock() - if b.closed.Load() { + if b.BufferBase.IsClosed() { b.Unlock() err = io.EOF return } now := mono.UnixNano() - if b.twcc != nil && b.twccExtID != 0 && !b.closed.Load() { + if b.twcc != nil && b.twccExtID != 0 { if ext := rtpPacket.GetExtension(b.twccExtID); ext != nil { b.twcc.Push(rtpPacket.SSRC, binary.BigEndian.Uint16(ext[0:2]), now, rtpPacket.Marker) } @@ -384,21 +179,37 @@ func (b *Buffer) Write(pkt []byte) (n int, err error) { return } - if !b.bound { + if !b.isBound { packet := make([]byte, len(pkt)) copy(packet, pkt) - b.pPackets = append(b.pPackets, pendingPacket{ + + if len(b.pPackets) == 0 { + b.logger.Debugw("received first packet") + } + + startIdx := 0 + overflow := len(b.pPackets) - max(b.BufferBase.MaxVideoPkts(), b.BufferBase.MaxAudioPkts()) + if overflow > 0 { + startIdx = overflow + } + b.pPackets = append(b.pPackets[startIdx:], pendingPacket{ packet: packet, arrivalTime: now, }) + + b.BufferBase.NotifyRead() b.Unlock() - b.readCond.Broadcast() return } - b.calc(pkt, &rtpPacket, now, false) + rtcpPackets := b.calc(pkt, &rtpPacket, now, false, false) b.Unlock() - b.readCond.Broadcast() + + if len(rtcpPackets) != 0 { + if cb := b.getOnRtcpFeedback(); cb != nil { + cb(rtcpPackets) + } + } return } @@ -408,6 +219,7 @@ func (b *Buffer) SetPrimaryBufferForRTX(primaryBuffer *Buffer) { pkts := b.pPackets b.pPackets = nil b.Unlock() + for _, pp := range pkts { var rtpPacket rtp.Packet err := rtpPacket.Unmarshal(pp.packet) @@ -421,10 +233,16 @@ func (b *Buffer) SetPrimaryBufferForRTX(primaryBuffer *Buffer) { } } -func (b *Buffer) writeRTX(rtxPkt *rtp.Packet, arrivalTime int64) (n int, err error) { +func (b *Buffer) NotifyRTX(ssrc uint32, repairSSRC uint32, rsid string) { + if onNotifyRTX := b.getOnNotifyRTX(); onNotifyRTX != nil { + onNotifyRTX(ssrc, repairSSRC, rsid) + } +} + +func (b *Buffer) writeRTX(rtxPkt *rtp.Packet, arrivalTime int64) { b.Lock() defer b.Unlock() - if !b.bound { + if !b.isBound { return } @@ -434,31 +252,36 @@ func (b *Buffer) writeRTX(rtxPkt *rtp.Packet, arrivalTime int64) (n int, err err } if b.rtxPktBuf == nil { - b.rtxPktBuf = make([]byte, bucket.MaxPktSize) + b.rtxPktBuf = make([]byte, bucket.RTPMaxPktSize) + } + + if len(rtxPkt.Payload) < 2 { + b.logger.Warnw("rtx payload too short", nil, "size", len(rtxPkt.Payload)) + return } repairedPkt := *rtxPkt repairedPkt.PayloadType = b.payloadType repairedPkt.SequenceNumber = binary.BigEndian.Uint16(rtxPkt.Payload[:2]) - repairedPkt.SSRC = b.mediaSSRC + repairedPkt.SSRC = b.BufferBase.SSRC() repairedPkt.Payload = rtxPkt.Payload[2:] - n, err = repairedPkt.MarshalTo(b.rtxPktBuf) + n, err := repairedPkt.MarshalTo(b.rtxPktBuf) if err != nil { - b.logger.Errorw("could not marshal repaired packet", err, "ssrc", b.mediaSSRC, "sn", repairedPkt.SequenceNumber) + b.logger.Errorw("could not marshal repaired packet", err, "ssrc", b.BufferBase.SSRC(), "sn", repairedPkt.SequenceNumber) return } - b.calc(b.rtxPktBuf[:n], &repairedPkt, arrivalTime, true) - return + b.calc(b.rtxPktBuf[:n], &repairedPkt, arrivalTime, false, true) } func (b *Buffer) Read(buff []byte) (n int, err error) { b.Lock() for { - if b.closed.Load() { + if b.BufferBase.IsClosed() { b.Unlock() return 0, io.EOF } + if b.pPackets != nil && len(b.pPackets) > b.lastPacketRead { if len(buff) < len(b.pPackets[b.lastPacketRead].packet) { b.Unlock() @@ -470,51 +293,26 @@ func (b *Buffer) Read(buff []byte) (n int, err error) { b.Unlock() return } - b.readCond.Wait() - } -} - -func (b *Buffer) ReadExtended(buf []byte) (*ExtPacket, error) { - b.Lock() - for { - if b.closed.Load() { - b.Unlock() - return nil, io.EOF - } - if b.extPackets.Len() > 0 { - ep := b.extPackets.PopFront() - ep = b.patchExtPacket(ep, buf) - if ep == nil { - continue - } - - b.Unlock() - return ep, nil - } - b.readCond.Wait() + b.BufferBase.WaitRead() } } func (b *Buffer) Close() error { - b.closeOnce.Do(func() { - b.closed.Store(true) + stats, err := b.BufferBase.CloseWithReason("close") + if err != nil { + return err + } - if b.rtpStats != nil { - b.rtpStats.Stop() - b.logger.Debugw("rtp stats", - "direction", "upstream", - "stats", b.rtpStats, - ) - if cb := b.getOnFinalRtpStats(); cb != nil { - cb(b.rtpStats.ToProto()) - } + if stats != nil { + if cb := b.getOnFinalRtpStats(); cb != nil { + cb(stats) } + } + + if cb := b.getOnClose(); cb != nil { + cb() + } - b.readCond.Broadcast() - if cb := b.getOnClose(); cb != nil { - cb() - } - }) return nil } @@ -531,26 +329,18 @@ func (b *Buffer) getOnClose() func() { return b.onClose } -func (b *Buffer) SetPLIThrottle(duration int64) { - b.Lock() - defer b.Unlock() - - b.pliThrottle = duration -} - -func (b *Buffer) SendPLI(force bool) { - b.RLock() - rtpStats := b.rtpStats - pliThrottle := b.pliThrottle - b.RUnlock() - - if (rtpStats == nil && !force) || !rtpStats.CheckAndUpdatePli(pliThrottle, force) { +func (b *Buffer) sendPLI() { + ssrc := b.BufferBase.SSRC() + if ssrc == 0 { return } - b.logger.Debugw("send pli", "ssrc", b.mediaSSRC, "force", force) + b.logger.Debugw("send pli", "mediaSSRC", ssrc) pli := []rtcp.Packet{ - &rtcp.PictureLossIndication{SenderSSRC: b.mediaSSRC, MediaSSRC: b.mediaSSRC}, + &rtcp.PictureLossIndication{ + SenderSSRC: ssrc, + MediaSSRC: ssrc, + }, } if cb := b.getOnRtcpFeedback(); cb != nil { @@ -558,508 +348,61 @@ func (b *Buffer) SendPLI(force bool) { } } -func (b *Buffer) SetRTT(rtt uint32) { - b.Lock() - defer b.Unlock() - - if rtt == 0 { - return - } - - if b.nacker != nil { - b.nacker.SetRTT(rtt) - } - - if b.rtpStats != nil { - b.rtpStats.UpdateRtt(rtt) - } -} - -func (b *Buffer) calc(rawPkt []byte, rtpPacket *rtp.Packet, arrivalTime int64, isRTX bool) { - defer func() { - b.doNACKs() - - b.doReports(arrivalTime) - }() - - if rtpPacket == nil { - rtpPacket = &rtp.Packet{} - if err := rtpPacket.Unmarshal(rawPkt); err != nil { - b.logger.Errorw("could not unmarshal RTP packet", err) - return - } - } - - // process header extensions always as padding packets could be used for probing - b.processHeaderExtensions(rtpPacket, arrivalTime, isRTX) - - flowState := b.updateStreamState(rtpPacket, arrivalTime) - if flowState.IsNotHandled { - return - } - - if len(rtpPacket.Payload) == 0 && (!flowState.IsOutOfOrder || flowState.IsDuplicate) { - // drop padding only in-order or duplicate packet - if !flowState.IsOutOfOrder { - // in-order packet - increment sequence number offset for subsequent packets - // Example: - // 40 - regular packet - pass through as sequence number 40 - // 41 - missing packet - don't know what it is, could be padding or not - // 42 - padding only packet - in-order - drop - increment sequence number offset to 1 - - // range[0, 42] = 0 offset - // 41 - arrives out of order - get offset 0 from cache - passed through as sequence number 41 - // 43 - regular packet - offset = 1 (running offset) - passes through as sequence number 42 - // 44 - padding only - in order - drop - increment sequence number offset to 2 - // range[0, 42] = 0 offset, range[43, 44] = 1 offset - // 43 - regular packet - out of order + duplicate - offset = 1 from cache - - // adjusted sequence number is 42, will be dropped by RTX buffer AddPacket method as duplicate - // 45 - regular packet - offset = 2 (running offset) - passed through with adjusted sequence number as 43 - // 44 - padding only - out-of-order + duplicate - dropped as duplicate - // - if err := b.snRangeMap.ExcludeRange(flowState.ExtSequenceNumber, flowState.ExtSequenceNumber+1); err != nil { - b.logger.Errorw( - "could not exclude range", err, - "sn", rtpPacket.SequenceNumber, - "esn", flowState.ExtSequenceNumber, - "rtpStats", b.rtpStats, - "snRangeMap", b.snRangeMap, - ) - } - } - return - } - - if !flowState.IsOutOfOrder && rtpPacket.PayloadType != b.payloadType && b.codecType == webrtc.RTPCodecTypeVideo { - b.handleCodecChange(rtpPacket.PayloadType) - } - - // add to RTX buffer using sequence number after accounting for dropped padding only packets - snAdjustment, err := b.snRangeMap.GetValue(flowState.ExtSequenceNumber) - if err != nil { - b.logger.Errorw( - "could not get sequence number adjustment", err, - "sn", rtpPacket.SequenceNumber, - "esn", flowState.ExtSequenceNumber, - "payloadSize", len(rtpPacket.Payload), - "rtpStats", b.rtpStats, - "snRangeMap", b.snRangeMap, - ) - return - } - flowState.ExtSequenceNumber -= snAdjustment - rtpPacket.Header.SequenceNumber = uint16(flowState.ExtSequenceNumber) - _, err = b.bucket.AddPacketWithSequenceNumber(rawPkt, flowState.ExtSequenceNumber) - if err != nil { - if !flowState.IsDuplicate { - if errors.Is(err, bucket.ErrPacketTooOld) { - packetTooOldCount := b.packetTooOldCount.Inc() - if (packetTooOldCount-1)%100 == 0 { - b.logger.Warnw( - "could not add packet to bucket", err, - "count", packetTooOldCount, - "flowState", &flowState, - "snAdjustment", snAdjustment, - "incomingSequenceNumber", flowState.ExtSequenceNumber+snAdjustment, - "rtpStats", b.rtpStats, - "snRangeMap", b.snRangeMap, - ) - } - } else if err != bucket.ErrRTXPacket { - b.logger.Warnw( - "could not add packet to bucket", err, - "flowState", &flowState, - "snAdjustment", snAdjustment, - "incomingSequenceNumber", flowState.ExtSequenceNumber+snAdjustment, - "rtpStats", b.rtpStats, - "snRangeMap", b.snRangeMap, - ) - } - } - return - } - - ep := b.getExtPacket(rtpPacket, arrivalTime, flowState) - if ep == nil { - return - } - b.extPackets.PushBack(ep) - - if b.extPackets.Len() > b.bucket.Capacity() { - if (b.extPacketTooMuchCount.Inc()-1)%100 == 0 { - b.logger.Warnw("too much ext packets", nil, "count", b.extPackets.Len()) - } - } - - b.doFpsCalc(ep) -} - -func (b *Buffer) patchExtPacket(ep *ExtPacket, buf []byte) *ExtPacket { - n, err := b.getPacket(buf, ep.ExtSequenceNumber) - if err != nil { - packetNotFoundCount := b.packetNotFoundCount.Inc() - if (packetNotFoundCount-1)%20 == 0 { - b.logger.Warnw( - "could not get packet from bucket", err, - "sn", ep.Packet.SequenceNumber, - "headSN", b.bucket.HeadSequenceNumber(), - "count", packetNotFoundCount, - "rtpStats", b.rtpStats, - "snRangeMap", b.snRangeMap, - ) - } - return nil - } - ep.RawPacket = buf[:n] - - // patch RTP packet to point payload to new buffer - pkt := *ep.Packet - payloadStart := ep.Packet.Header.MarshalSize() - payloadEnd := payloadStart + len(ep.Packet.Payload) - if payloadEnd > n { - b.logger.Warnw("unexpected marshal size", nil, "max", n, "need", payloadEnd) - return nil - } - pkt.Payload = buf[payloadStart:payloadEnd] - ep.Packet = &pkt - - return ep -} - -func (b *Buffer) doFpsCalc(ep *ExtPacket) { - if b.paused || b.frameRateCalculated || len(ep.Packet.Payload) == 0 { - return - } - spatial := ep.Spatial - if spatial < 0 || int(spatial) >= len(b.frameRateCalculator) { - spatial = 0 - } - if fr := b.frameRateCalculator[spatial]; fr != nil { - if fr.RecvPacket(ep) { - complete := true - for _, fr2 := range b.frameRateCalculator { - if fr2 != nil && !fr2.Completed() { - complete = false - break - } - } - if complete { - b.frameRateCalculated = true - if f := b.onFpsChanged; f != nil { - go f() - } - } - } - } -} - -func (b *Buffer) handleCodecChange(newPT uint8) { - var ( - codecFound, rtxFound bool - rtxPt uint8 - newCodec webrtc.RTPCodecParameters - ) - for _, codec := range b.rtpParameters.Codecs { - if !codecFound && uint8(codec.PayloadType) == newPT { - newCodec = codec - codecFound = true - } - - if mime.IsMimeTypeStringRTX(codec.MimeType) && strings.Contains(codec.SDPFmtpLine, fmt.Sprintf("apt=%d", newPT)) { - rtxFound = true - rtxPt = uint8(codec.PayloadType) - } - - if codecFound && rtxFound { - break - } - } - if !codecFound { - b.logger.Errorw("could not find codec for new payload type", nil, "pt", newPT, "rtpParameters", b.rtpParameters) - return - } - b.logger.Infow( - "codec changed", - "oldPayload", b.payloadType, "newPayload", newPT, - "oldRtxPayload", b.rtxPayloadType, "newRtxPayload", rtxPt, - "oldMime", b.mime, "newMime", newCodec.MimeType) - b.payloadType = newPT - b.rtxPayloadType = rtxPt - b.mime = mime.NormalizeMimeType(newCodec.MimeType) - b.frameRateCalculated = false - - if b.ddExtID != 0 { - b.createDDParserAndFrameRateCalculator() - } - - if b.frameRateCalculator[0] == nil { - b.createFrameRateCalculator() - } - - b.bucket.ResyncOnNextPacket() - - if f := b.onCodecChange; f != nil { - go f(newCodec) - } -} - -func (b *Buffer) updateStreamState(p *rtp.Packet, arrivalTime int64) rtpstats.RTPFlowState { - flowState := b.rtpStats.Update( +func (b *Buffer) calc(rawPkt []byte, rtpPacket *rtp.Packet, arrivalTime int64, isBuffered bool, isRTX bool) []rtcp.Packet { + b.BufferBase.HandleIncomingPacketLocked( + rawPkt, + rtpPacket, arrivalTime, - p.Header.SequenceNumber, - p.Header.Timestamp, - p.Header.Marker, - p.Header.MarshalSize(), - len(p.Payload), - int(p.PaddingSize), + isBuffered, + isRTX, + nil, + 0, ) - if b.nacker != nil { - b.nacker.Remove(p.SequenceNumber) - - for lost := flowState.LossStartInclusive; lost != flowState.LossEndExclusive; lost++ { - b.nacker.Push(uint16(lost)) - } - } - - return flowState + return b.getRTCPPackets(arrivalTime) } -func (b *Buffer) processHeaderExtensions(p *rtp.Packet, arrivalTime int64, isRTX bool) { - if b.audioLevelExtID != 0 && !isRTX { - if !b.latestTSForAudioLevelInitialized { - b.latestTSForAudioLevelInitialized = true - b.latestTSForAudioLevel = p.Timestamp - } - if e := p.GetExtension(b.audioLevelExtID); e != nil { - ext := rtp.AudioLevelExtension{} - if err := ext.Unmarshal(e); err == nil { - if (p.Timestamp - b.latestTSForAudioLevel) < (1 << 31) { - duration := (int64(p.Timestamp) - int64(b.latestTSForAudioLevel)) * 1e3 / int64(b.clockRate) - if duration > 0 { - b.audioLevel.Observe(ext.Level, uint32(duration), arrivalTime) - } - - b.latestTSForAudioLevel = p.Timestamp - } - } - } +func (b *Buffer) getRTCPPackets(arrivalTime int64) []rtcp.Packet { + var pkts []rtcp.Packet + if nackPkt := b.getNACKPacket(); nackPkt != nil { + pkts = append(pkts, nackPkt) } + if receiverReport := b.getRTCPReceiverReport(arrivalTime); receiverReport != nil { + pkts = append(pkts, receiverReport) + } + + return pkts } -func (b *Buffer) getExtPacket(rtpPacket *rtp.Packet, arrivalTime int64, flowState rtpstats.RTPFlowState) *ExtPacket { - ep := &ExtPacket{ - Arrival: arrivalTime, - ExtSequenceNumber: flowState.ExtSequenceNumber, - ExtTimestamp: flowState.ExtTimestamp, - Packet: rtpPacket, - VideoLayer: VideoLayer{ - Spatial: InvalidLayerSpatial, - Temporal: InvalidLayerTemporal, - }, - IsOutOfOrder: flowState.IsOutOfOrder, - } - - if len(rtpPacket.Payload) == 0 { - // padding only packet, nothing else to do - return ep - } - - ep.Temporal = 0 - if b.ddParser != nil { - ddVal, videoLayer, err := b.ddParser.Parse(ep.Packet) - if err != nil { - return nil - } else if ddVal != nil { - ep.DependencyDescriptor = ddVal - ep.VideoLayer = videoLayer - // DD-TODO : notify active decode target change if changed. - } - } - - switch b.mime { - case mime.MimeTypeVP8: - vp8Packet := VP8{} - if err := vp8Packet.Unmarshal(rtpPacket.Payload); err != nil { - b.logger.Warnw("could not unmarshal VP8 packet", err) - return nil - } - ep.KeyFrame = vp8Packet.IsKeyFrame - if ep.DependencyDescriptor == nil { - ep.Temporal = int32(vp8Packet.TID) - } else { - // vp8 with DependencyDescriptor enabled, use the TID from the descriptor - vp8Packet.TID = uint8(ep.Temporal) - } - ep.Payload = vp8Packet - ep.Spatial = InvalidLayerSpatial // vp8 don't have spatial scalability, reset to invalid - - case mime.MimeTypeVP9: - if ep.DependencyDescriptor == nil { - var vp9Packet codecs.VP9Packet - _, err := vp9Packet.Unmarshal(rtpPacket.Payload) - if err != nil { - b.logger.Warnw("could not unmarshal VP9 packet", err) - return nil - } - ep.VideoLayer = VideoLayer{ - Spatial: int32(vp9Packet.SID), - Temporal: int32(vp9Packet.TID), - } - ep.Payload = vp9Packet - } - ep.KeyFrame = IsVP9KeyFrame(rtpPacket.Payload) - - case mime.MimeTypeH264: - ep.KeyFrame = IsH264KeyFrame(rtpPacket.Payload) - ep.Spatial = InvalidLayerSpatial // h.264 don't have spatial scalability, reset to invalid - - case mime.MimeTypeAV1: - ep.KeyFrame = IsAV1KeyFrame(rtpPacket.Payload) - - case mime.MimeTypeH265: - if ep.DependencyDescriptor == nil { - if len(rtpPacket.Payload) < 2 { - b.logger.Warnw("invalid H265 packet", nil) - return nil - } - ep.VideoLayer = VideoLayer{ - Temporal: int32(rtpPacket.Payload[1]&0x07) - 1, - } - ep.Spatial = InvalidLayerSpatial - } - ep.KeyFrame = IsH265KeyFrame(rtpPacket.Payload) - } - - if ep.KeyFrame { - if b.rtpStats != nil { - b.rtpStats.UpdateKeyFrame(1) - } - } - - if b.absCaptureTimeExtID != 0 { - extData := rtpPacket.GetExtension(b.absCaptureTimeExtID) - - var actExt act.AbsCaptureTime - if err := actExt.Unmarshal(extData); err == nil { - ep.AbsCaptureTimeExt = &actExt - } - } - - return ep -} - -func (b *Buffer) doNACKs() { - if b.nacker == nil { - return - } - - if r, numSeqNumsNacked := b.buildNACKPacket(); r != nil { - if cb := b.onRtcpFeedback; cb != nil { - cb(r) - } - if b.rtpStats != nil { - b.rtpStats.UpdateNack(uint32(numSeqNumsNacked)) - } - } -} - -func (b *Buffer) doReports(arrivalTime int64) { - if arrivalTime-b.lastReport < ReportDelta { - return - } - - b.lastReport = arrivalTime - - // RTCP reports - pkts := b.getRTCP() - if pkts != nil { - if cb := b.onRtcpFeedback; cb != nil { - cb(pkts) - } - } - - b.mayGrowBucket() -} - -func (b *Buffer) mayGrowBucket() { - cap := b.bucket.Capacity() - maxPkts := b.maxVideoPkts - if b.codecType == webrtc.RTPCodecTypeAudio { - maxPkts = b.maxAudioPkts - } - if cap >= maxPkts { - return - } - oldCap := cap - if deltaInfo := b.rtpStats.DeltaInfo(b.ppsSnapshotId); deltaInfo != nil { - duration := deltaInfo.EndTime.Sub(deltaInfo.StartTime) - if duration > 500*time.Millisecond { - pps := int(time.Duration(deltaInfo.Packets) * time.Second / duration) - for pps > cap && cap < maxPkts { - cap = b.bucket.Grow() - } - if cap > oldCap { - b.logger.Debugw("grow bucket", "from", oldCap, "to", cap, "pps", pps) - } - } - } -} - -func (b *Buffer) buildNACKPacket() ([]rtcp.Packet, int) { - if nacks, numSeqNumsNacked := b.nacker.Pairs(); len(nacks) > 0 { - pkts := []rtcp.Packet{&rtcp.TransportLayerNack{ - SenderSSRC: b.mediaSSRC, - MediaSSRC: b.mediaSSRC, +func (b *Buffer) getNACKPacket() *rtcp.TransportLayerNack { + if nacks := b.BufferBase.GetNACKPairsLocked(); len(nacks) > 0 { + ssrc := b.BufferBase.SSRC() + return &rtcp.TransportLayerNack{ + SenderSSRC: ssrc, + MediaSSRC: ssrc, Nacks: nacks, - }} - return pkts, numSeqNumsNacked + } } - return nil, 0 + return nil } -func (b *Buffer) buildReceptionReport() *rtcp.ReceptionReport { - if b.rtpStats == nil { +func (b *Buffer) getRTCPReceiverReport(arrivalTime int64) *rtcp.ReceiverReport { + if arrivalTime-b.lastReportAt < rtcpReceiverReportDelta { return nil } + b.lastReportAt = arrivalTime proxyLoss := b.lastFractionLostToReport if b.codecType == webrtc.RTPCodecTypeAudio && !b.enableAudioLossProxying { proxyLoss = 0 } - return b.rtpStats.GetRtcpReceptionReport(b.mediaSSRC, proxyLoss, b.rrSnapshotId) -} - -func (b *Buffer) SetSenderReportData(rtpTime uint32, ntpTime uint64, packets uint32, octets uint32) { - b.RLock() - srData := &livekit.RTCPSenderReportState{ - RtpTimestamp: rtpTime, - NtpTimestamp: ntpTime, - At: mono.UnixNano(), - Packets: packets, - Octets: uint64(octets), - } - - didSet := false - if b.rtpStats != nil { - didSet = b.rtpStats.SetRtcpSenderReportData(srData) - } - b.RUnlock() - - if didSet { - if cb := b.getOnRtcpSenderReport(); cb != nil { - cb() + if receptionReport := b.BufferBase.GetRtcpReceptionReportLocked(proxyLoss); receptionReport != nil { + return &rtcp.ReceiverReport{ + SSRC: b.BufferBase.SSRC(), + Reports: []rtcp.ReceptionReport{*receptionReport}, } } -} - -func (b *Buffer) GetSenderReportData() *livekit.RTCPSenderReportState { - b.RLock() - defer b.RUnlock() - - if b.rtpStats != nil { - return b.rtpStats.GetRtcpSenderReportData() - } return nil } @@ -1071,34 +414,6 @@ func (b *Buffer) SetLastFractionLostReport(lost uint8) { b.lastFractionLostToReport = lost } -func (b *Buffer) getRTCP() []rtcp.Packet { - var pkts []rtcp.Packet - - rr := b.buildReceptionReport() - if rr != nil { - pkts = append(pkts, &rtcp.ReceiverReport{ - SSRC: b.mediaSSRC, - Reports: []rtcp.ReceptionReport{*rr}, - }) - } - - return pkts -} - -func (b *Buffer) GetPacket(buff []byte, esn uint64) (int, error) { - b.Lock() - defer b.Unlock() - - return b.getPacket(buff, esn) -} - -func (b *Buffer) getPacket(buff []byte, esn uint64) (int, error) { - if b.closed.Load() { - return 0, io.EOF - } - return b.bucket.GetPacket(buff, esn) -} - func (b *Buffer) OnRtcpFeedback(fn func(fb []rtcp.Packet)) { b.Lock() b.onRtcpFeedback = fn @@ -1112,19 +427,6 @@ func (b *Buffer) getOnRtcpFeedback() func(fb []rtcp.Packet) { return b.onRtcpFeedback } -func (b *Buffer) OnRtcpSenderReport(fn func()) { - b.Lock() - b.onRtcpSenderReport = fn - b.Unlock() -} - -func (b *Buffer) getOnRtcpSenderReport() func() { - b.RLock() - defer b.RUnlock() - - return b.onRtcpSenderReport -} - func (b *Buffer) OnFinalRtpStats(fn func(*livekit.RTPStats)) { b.Lock() b.onFinalRtpStats = fn @@ -1138,85 +440,15 @@ func (b *Buffer) getOnFinalRtpStats() func(*livekit.RTPStats) { return b.onFinalRtpStats } -// GetMediaSSRC returns the associated SSRC of the RTP stream -func (b *Buffer) GetMediaSSRC() uint32 { - return b.mediaSSRC -} - -// GetClockRate returns the RTP clock rate -func (b *Buffer) GetClockRate() uint32 { - return b.clockRate -} - -func (b *Buffer) GetStats() *livekit.RTPStats { - b.RLock() - defer b.RUnlock() - - if b.rtpStats == nil { - return nil - } - - return b.rtpStats.ToProto() -} - -func (b *Buffer) GetDeltaStats() *StreamStatsWithLayers { - b.RLock() - defer b.RUnlock() - - if b.rtpStats == nil { - return nil - } - - deltaStats := b.rtpStats.DeltaInfo(b.deltaStatsSnapshotId) - if deltaStats == nil { - return nil - } - - return &StreamStatsWithLayers{ - RTPStats: deltaStats, - Layers: map[int32]*rtpstats.RTPDeltaInfo{ - 0: deltaStats, - }, - } -} - -func (b *Buffer) GetLastSenderReportTime() time.Time { - b.RLock() - defer b.RUnlock() - - if b.rtpStats == nil { - return time.Time{} - } - - return b.rtpStats.LastSenderReportTime() -} - -func (b *Buffer) GetAudioLevel() (float64, bool) { - b.RLock() - defer b.RUnlock() - - if b.audioLevel == nil { - return 0, false - } - - return b.audioLevel.GetLevel(mono.UnixNano()) -} - -func (b *Buffer) OnFpsChanged(f func()) { +func (b *Buffer) OnNotifyRTX(fn func(ssrc uint32, repairSSRC uint32, rsid string)) { b.Lock() - b.onFpsChanged = f + b.onNotifyRTX = fn b.Unlock() } -func (b *Buffer) GetTemporalLayerFpsForSpatial(layer int32) []float32 { - if int(layer) >= len(b.frameRateCalculator) { - return nil - } +func (b *Buffer) getOnNotifyRTX() func(ssrc uint32, repairSSRC uint32, rsid string) { + b.RLock() + defer b.RUnlock() - if fc := b.frameRateCalculator[layer]; fc != nil { - return fc.GetFrameRate() - } - return nil + return b.onNotifyRTX } - -// --------------------------------------------------------------- diff --git a/pkg/sfu/buffer/buffer_base.go b/pkg/sfu/buffer/buffer_base.go new file mode 100644 index 0000000..b94326c --- /dev/null +++ b/pkg/sfu/buffer/buffer_base.go @@ -0,0 +1,1597 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package buffer + +import ( + "errors" + "fmt" + "io" + "strings" + "sync" + "time" + + "github.com/gammazero/deque" + "github.com/pion/rtcp" + "github.com/pion/rtp" + "github.com/pion/rtp/codecs" + "github.com/pion/sdp/v3" + "github.com/pion/webrtc/v4" + "go.uber.org/atomic" + + "github.com/livekit/livekit-server/pkg/sfu/audio" + act "github.com/livekit/livekit-server/pkg/sfu/rtpextension/abscapturetime" + dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor" + "github.com/livekit/livekit-server/pkg/sfu/rtpstats" + "github.com/livekit/livekit-server/pkg/sfu/utils" + "github.com/livekit/mediatransportutil/pkg/bucket" + "github.com/livekit/mediatransportutil/pkg/codec" + "github.com/livekit/mediatransportutil/pkg/nack" + "github.com/livekit/protocol/codecs/mime" + "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/logger" + "github.com/livekit/protocol/utils/mono" +) + +var ( + ExtPacketFactory = &sync.Pool{ + New: func() any { + return &ExtPacket{} + }, + } +) + +func ReleaseExtPacket(extPkt *ExtPacket) { + if extPkt == nil { + return + } + + ReleaseExtDependencyDescriptor(extPkt.DependencyDescriptor) + + *extPkt = ExtPacket{} + ExtPacketFactory.Put(extPkt) +} + +// -------------------------------------- + +type ExtPacket struct { + VideoLayer + Arrival int64 + ExtSequenceNumber uint64 + ExtTimestamp uint64 + Packet *rtp.Packet + Payload any + IsKeyFrame bool + RawPacket []byte + DependencyDescriptor *ExtDependencyDescriptor + AbsCaptureTimeExt *act.AbsCaptureTime + IsOutOfOrder bool + IsBuffered bool +} + +type BufferProvider interface { + SetLogger(lgr logger.Logger) + SetAudioLevelConfig(audioLevelConfig audio.AudioLevelConfig) + SetStreamRestartDetection(enable bool) + SetPLIThrottle(duration int64) + SetRTT(rtt uint32) + SetPaused(paused bool) + + SendPLI(force bool) + + ReadExtended(buf []byte) (*ExtPacket, error) + GetPacket(buf []byte, esn uint64) (int, error) + + GetAudioLevel() (float64, bool) + GetTemporalLayerFpsForSpatial(layer int32) []float32 + GetStats() *livekit.RTPStats + GetDeltaStats() *StreamStatsWithLayers + GetDeltaStatsLite() *rtpstats.RTPDeltaInfoLite + GetLastSenderReportTime() time.Time + GetNACKPairs() []rtcp.NackPair + + SetSenderReportData(srData *livekit.RTCPSenderReportState) + GetSenderReportData() *livekit.RTCPSenderReportState + + OnRtcpSenderReport(fn func()) + OnFpsChanged(fn func()) + OnVideoSizeChanged(fn func([]codec.VideoSize)) + OnCodecChange(fn func(webrtc.RTPCodecParameters)) + OnStreamRestart(fn func(string)) + + StartKeyFrameSeeder() + StopKeyFrameSeeder() + + HandleIncomingPacket( + rawPkt []byte, + rtpPacket *rtp.Packet, + arrivalTime int64, + isBuffered bool, + isRTX bool, + skippedSeqs []uint16, + oobSequenceNumber uint16, + ) (uint64, error) + + MarkForRestartStream(reason string) + RestartStream(reason string) + + CloseWithReason(reason string) (*livekit.RTPStats, error) +} + +const ( + bucketCapCheckInterval = 1e9 +) + +type BufferBaseParams struct { + SSRC uint32 + MaxVideoPkts int + MaxAudioPkts int + LoggerComponents []string + SendPLI func() + IsReportingEnabled bool + IsOOBSequenceNumber bool + IsDDRestartEnabled bool +} + +type BufferBase struct { + sync.RWMutex + + params BufferBaseParams + + readCond *sync.Cond + + bucket *bucket.Bucket[uint64, uint16] + lastBucketCapCheckAt int64 + + nacker nack.NackQueueInterface + rtpStatsLite *rtpstats.RTPStatsReceiverLite + liteStatsSnapshotId uint32 + + extPackets deque.Deque[*ExtPacket] + + codecType webrtc.RTPCodecType + closeOnce sync.Once + clockRate uint32 + mime mime.MimeType + + rtpParameters webrtc.RTPParameters + payloadType uint8 + rtxPayloadType uint8 + + snRangeMap *utils.RangeMap[uint64, uint64] + + audioLevelConfig audio.AudioLevelConfig + audioLevel *audio.AudioLevel + audioLevelExtID uint8 + + enableStreamRestartDetection bool + + pliThrottle int64 + + rtpStats *rtpstats.RTPStatsReceiver + ppsSnapshotId uint32 + rrSnapshotId uint32 + deltaStatsSnapshotId uint32 + + // callbacks + onRtcpSenderReport func() + onFpsChanged func() + onVideoSizeChanged func([]codec.VideoSize) + onCodecChange func(webrtc.RTPCodecParameters) + onStreamRestart func(string) + + // video size tracking for multiple spatial layers + currentVideoSize [DefaultMaxLayerSpatial + 1]codec.VideoSize + + logger logger.Logger + + // dependency descriptor + ddExtID uint8 + ddParser *DependencyDescriptorParser + + isPaused bool + frameRateCalculator [DefaultMaxLayerSpatial + 1]FrameRateCalculator + frameRateCalculated bool + + packetNotFoundCount atomic.Uint32 + packetTooOldCount atomic.Uint32 + extPacketTooMuchCount atomic.Uint32 + + absCaptureTimeExtID uint8 + + keyFrameSeederGeneration atomic.Int32 + + isRestartPending bool + + isClosed atomic.Bool +} + +func NewBufferBase(params BufferBaseParams) *BufferBase { + l := logger.GetLogger() // will be reset with correct context via SetLogger + for _, component := range params.LoggerComponents { + l = l.WithComponent(component) + } + l = l.WithValues("ssrc", params.SSRC) + + b := &BufferBase{ + params: params, + lastBucketCapCheckAt: mono.UnixNano(), + snRangeMap: utils.NewRangeMap[uint64, uint64](100), + pliThrottle: int64(500 * time.Millisecond), + logger: l, + } + b.readCond = sync.NewCond(&b.RWMutex) + b.extPackets.SetBaseCap(128) + return b +} + +func (b *BufferBase) SSRC() uint32 { + return b.params.SSRC +} + +func (b *BufferBase) MaxVideoPkts() int { + return b.params.MaxVideoPkts +} + +func (b *BufferBase) MaxAudioPkts() int { + return b.params.MaxAudioPkts +} + +func (b *BufferBase) SetLogger(lgr logger.Logger) { + b.Lock() + defer b.Unlock() + + for _, component := range b.params.LoggerComponents { + lgr = lgr.WithComponent(component) + } + lgr = lgr.WithValues("ssrc", b.params.SSRC) + b.logger = lgr + + if b.rtpStats != nil { + b.rtpStats.SetLogger(b.logger) + } + + if b.rtpStatsLite != nil { + b.rtpStatsLite.SetLogger(b.logger) + } +} + +func (b *BufferBase) Bind(rtpParameters webrtc.RTPParameters, codec webrtc.RTPCodecCapability, bitrate int) error { + b.logger.Debugw("binding track") + + b.Lock() + defer b.Unlock() + + return b.BindLocked(rtpParameters, codec, bitrate) +} + +func (b *BufferBase) BindLocked(rtpParameters webrtc.RTPParameters, codec webrtc.RTPCodecCapability, bitrate int) error { + b.logger.Debugw("binding track") + if codec.ClockRate == 0 { + b.logger.Warnw("invalid codec", nil, "rtpParameters", rtpParameters, "codec", codec, "bitrate", bitrate) + return errInvalidCodec + } + + b.setupRTPStats(codec.ClockRate) + + b.clockRate = codec.ClockRate + b.mime = mime.NormalizeMimeType(codec.MimeType) + b.rtpParameters = rtpParameters + for _, codecParameter := range rtpParameters.Codecs { + if mime.IsMimeTypeStringEqual(codecParameter.MimeType, codec.MimeType) { + b.payloadType = uint8(codecParameter.PayloadType) + break + } + } + + if b.payloadType == 0 && !mime.IsMimeTypeStringEqual(codec.MimeType, webrtc.MimeTypePCMU) { + b.logger.Warnw( + "could not find payload type for codec", nil, + "codec", codec.MimeType, + "rtpParameters", rtpParameters, + ) + b.payloadType = uint8(rtpParameters.Codecs[0].PayloadType) + } + + // find RTX payload type + for _, codec := range rtpParameters.Codecs { + if mime.IsMimeTypeStringRTX(codec.MimeType) && strings.Contains(codec.SDPFmtpLine, fmt.Sprintf("apt=%d", b.payloadType)) { + b.rtxPayloadType = uint8(codec.PayloadType) + break + } + } + + for _, ext := range rtpParameters.HeaderExtensions { + switch ext.URI { + case dd.ExtensionURI: + if b.ddExtID != 0 { + b.logger.Warnw( + "multiple dependency descriptor extensions found", nil, + "id", ext.ID, + "previous", b.ddExtID, + ) + continue + } + b.ddExtID = uint8(ext.ID) + b.createDDParserAndFrameRateCalculator() + + case sdp.AudioLevelURI: + b.audioLevelExtID = uint8(ext.ID) + b.audioLevel = audio.NewAudioLevel(audio.AudioLevelParams{ + ClockRate: b.clockRate, + }) + b.audioLevel.SetConfig(b.audioLevelConfig) + + case act.AbsCaptureTimeURI: + b.absCaptureTimeExtID = uint8(ext.ID) + } + } + + switch { + case mime.IsMimeTypeAudio(b.mime): + b.codecType = webrtc.RTPCodecTypeAudio + b.bucket = bucket.NewBucket[uint64, uint16]( + InitPacketBufferSizeAudio, + bucket.RTPMaxPktSize, + bucket.RTPSeqNumOffset, + ) + + case mime.IsMimeTypeVideo(b.mime): + b.codecType = webrtc.RTPCodecTypeVideo + b.bucket = bucket.NewBucket[uint64, uint16]( + InitPacketBufferSizeVideo, + bucket.RTPMaxPktSize, + bucket.RTPSeqNumOffset, + ) + + if b.frameRateCalculator[0] == nil { + b.createFrameRateCalculator() + } + + if bitrate > 0 { + pps := bitrate / 8 / 1200 + for pps > b.bucket.Capacity() { + if b.bucket.Grow() >= b.params.MaxVideoPkts { + break + } + } + } + + default: + b.codecType = webrtc.RTPCodecType(0) + } + + for _, fb := range codec.RTCPFeedback { + switch fb.Type { + case webrtc.TypeRTCPFBGoogREMB: + b.logger.Debugw("Setting feedback", "type", webrtc.TypeRTCPFBGoogREMB) + b.logger.Debugw("REMB not supported, RTCP feedback will not be generated") + + case webrtc.TypeRTCPFBNACK: + // pion uses a single mediaengine to manage negotiated codecs of peerconnection, that means we can't have different + // codec settings at track level for same codec type, so enable nack for all audio receivers but don't create nack queue + // for red codec. + if b.mime == mime.MimeTypeRED { + break + } + + b.logger.Debugw("Setting feedback", "type", webrtc.TypeRTCPFBNACK) + if b.nacker == nil { + b.nacker = nack.NewNACKQueue(nack.NackQueueParamsDefault) + } + } + } + + if b.nacker == nil && b.params.IsOOBSequenceNumber { + b.nacker = nack.NewNACKQueue(nack.NackQueueParamsDefault) + } + + b.StartKeyFrameSeeder() + + return nil +} + +func (b *BufferBase) CloseWithReason(reason string) (stats *livekit.RTPStats, err error) { + b.closeOnce.Do(func() { + b.isClosed.Store(true) + + b.StopKeyFrameSeeder() + + b.Lock() + stats, _ = b.stopRTPStats(reason) + b.readCond.Broadcast() + b.Unlock() + + go b.flushExtPackets() + }) + return +} + +func (b *BufferBase) IsClosed() bool { + return b.isClosed.Load() +} + +func (b *BufferBase) SetPaused(paused bool) { + b.Lock() + defer b.Unlock() + + b.isPaused = paused +} + +func (b *BufferBase) SetAudioLevelConfig(audioLevelConfig audio.AudioLevelConfig) { + b.Lock() + defer b.Unlock() + + b.audioLevelConfig = audioLevelConfig + if b.audioLevel != nil { + b.audioLevel.SetConfig(b.audioLevelConfig) + } +} + +func (b *BufferBase) SetStreamRestartDetection(enable bool) { + b.Lock() + defer b.Unlock() + + b.enableStreamRestartDetection = enable +} + +func (b *BufferBase) setupRTPStats(clockRate uint32) { + b.rtpStats = rtpstats.NewRTPStatsReceiver(rtpstats.RTPStatsParams{}) + b.rtpStats.SetLogger(b.logger) + b.rtpStats.SetClockRate(clockRate) + + b.ppsSnapshotId = b.rtpStats.NewSnapshotId() + if b.params.IsReportingEnabled { + b.rrSnapshotId = b.rtpStats.NewSnapshotId() + b.deltaStatsSnapshotId = b.rtpStats.NewSnapshotId() + } + + b.setupRTPStatsLite(clockRate) +} + +func (b *BufferBase) setupRTPStatsLite(clockRate uint32) { + if b.params.IsOOBSequenceNumber { + b.rtpStatsLite = rtpstats.NewRTPStatsReceiverLite(rtpstats.RTPStatsParams{}) + b.rtpStatsLite.SetLogger(b.logger) + b.rtpStatsLite.SetClockRate(clockRate) + + b.liteStatsSnapshotId = b.rtpStatsLite.NewSnapshotLiteId() + } +} + +func (b *BufferBase) stopRTPStats(reason string) (stats *livekit.RTPStats, statsLite *livekit.RTPStats) { + if b.rtpStats != nil { + b.rtpStats.Stop() + stats = b.rtpStats.ToProto() + + b.logger.Debugw( + "rtp stats", + "direction", "upstream", + "stats", b.rtpStats, + "reason", reason, + ) + } + + statsLite = b.stopRTPStatsLite(reason) + return +} + +func (b *BufferBase) stopRTPStatsLite(reason string) (statsLite *livekit.RTPStats) { + if b.rtpStatsLite != nil { + b.rtpStatsLite.Stop() + statsLite = b.rtpStatsLite.ToProto() + + b.logger.Debugw( + "rtp stats lite", + "direction", "upstream", + "statsLite", b.rtpStatsLite, + "reason", reason, + ) + } + return +} + +func (b *BufferBase) RestartOOBSequenceNumber(reason string) { + b.Lock() + defer b.Unlock() + + b.stopRTPStatsLite(reason) + b.setupRTPStatsLite(b.clockRate) + + if b.nacker != nil { + b.nacker = nack.NewNACKQueue(nack.NackQueueParamsDefault) + } +} + +func (b *BufferBase) MarkForRestartStream(reason string) { + b.logger.Debugw("marking for stream restart", "reason", reason) + + b.Lock() + defer b.Unlock() + + b.isRestartPending = true + b.readCond.Broadcast() +} + +func (b *BufferBase) RestartStream(reason string) { + b.logger.Debugw("stream restart", "reason", reason) + + b.Lock() + defer b.Unlock() + + b.restartStreamLocked(reason, false) + b.readCond.Broadcast() +} + +func (b *BufferBase) restartStreamLocked(reason string, isDetected bool) { + b.logger.Infow("stream restart", "reason", reason) + + // stop + b.StopKeyFrameSeeder() + b.stopRTPStats("stream-restart") + b.flushExtPacketsLocked() + + // restart + b.snRangeMap = utils.NewRangeMap[uint64, uint64](100) + b.setupRTPStats(b.clockRate) + + b.bucket.ResyncOnNextPacket() + b.lastBucketCapCheckAt = mono.UnixNano() + + if b.nacker != nil { + b.nacker = nack.NewNACKQueue(nack.NackQueueParamsDefault) + } + + if b.audioLevel != nil { + b.audioLevel = audio.NewAudioLevel(audio.AudioLevelParams{ + ClockRate: b.clockRate, + }) + b.audioLevel.SetConfig(b.audioLevelConfig) + } + + if b.ddExtID != 0 { + b.createDDParserAndFrameRateCalculator() + } + + b.frameRateCalculated = false + if b.frameRateCalculator[0] == nil { + b.createFrameRateCalculator() + } + + b.StartKeyFrameSeeder() + + if isDetected { + b.isRestartPending = true + + if f := b.onStreamRestart; f != nil { + go f(reason) + } + } +} + +func (b *BufferBase) createDDParserAndFrameRateCalculator() { + if mime.IsMimeTypeSVCCapable(b.mime) || b.mime == mime.MimeTypeVP8 { + frc := NewFrameRateCalculatorDD(b.clockRate, b.logger) + for i := range b.frameRateCalculator { + b.frameRateCalculator[i] = frc.GetFrameRateCalculatorForSpatial(int32(i)) + } + b.ddParser = NewDependencyDescriptorParser( + b.ddExtID, + b.logger, + func(spatial, temporal int32) { + frc.SetMaxLayer(spatial, temporal) + }, + b.params.IsDDRestartEnabled, + ) + } +} + +func (b *BufferBase) createFrameRateCalculator() { + switch b.mime { + case mime.MimeTypeVP8: + b.frameRateCalculator[0] = NewFrameRateCalculatorVP8(b.clockRate, b.logger) + + case mime.MimeTypeVP9: + frc := NewFrameRateCalculatorVP9(b.clockRate, b.logger) + for i := range b.frameRateCalculator { + b.frameRateCalculator[i] = frc.GetFrameRateCalculatorForSpatial(int32(i)) + } + + case mime.MimeTypeH265: + b.frameRateCalculator[0] = NewFrameRateCalculatorH26x(b.clockRate, b.logger) + } +} + +func (b *BufferBase) ReadExtended(buf []byte) (*ExtPacket, error) { + b.Lock() + for { + if b.isClosed.Load() { + b.Unlock() + return nil, io.EOF + } + + if b.isRestartPending { + b.isRestartPending = false + b.Unlock() + return nil, nil + } + + if b.extPackets.Len() > 0 { + ep := b.extPackets.PopFront() + patched := b.patchExtPacket(ep, buf) + if patched == nil { + ReleaseExtPacket(ep) + continue + } + + b.Unlock() + return patched, nil + } + + b.readCond.Wait() + } +} + +func (b *BufferBase) SetPLIThrottle(duration int64) { + b.Lock() + defer b.Unlock() + + b.pliThrottle = duration +} + +func (b *BufferBase) SendPLI(force bool) { + b.RLock() + if b.codecType != webrtc.RTPCodecTypeVideo { + b.RUnlock() + return + } + + rtpStats := b.rtpStats + pliThrottle := b.pliThrottle + b.RUnlock() + + if rtpStats == nil || !rtpStats.CheckAndUpdatePli(pliThrottle, force) { + return + } + + if b.params.SendPLI != nil { + b.params.SendPLI() + } +} + +func (b *BufferBase) SetRTT(rtt uint32) { + b.Lock() + defer b.Unlock() + + if rtt == 0 { + return + } + + if b.nacker != nil { + b.nacker.SetRTT(rtt) + } + + if b.rtpStats != nil { + b.rtpStats.UpdateRtt(rtt) + } +} + +func (b *BufferBase) WaitRead() { + b.readCond.Wait() +} + +func (b *BufferBase) NotifyRead() { + b.readCond.Broadcast() +} + +func (b *BufferBase) HandleIncomingPacket( + rawPkt []byte, + rtpPacket *rtp.Packet, + arrivalTime int64, + isBuffered bool, + isRTX bool, + skippedSeqs []uint16, + oobSequenceNumber uint16, +) (uint64, error) { + b.Lock() + defer b.Unlock() + + if b.isClosed.Load() { + return 0, io.EOF + } + + return b.HandleIncomingPacketLocked( + rawPkt, + rtpPacket, + arrivalTime, + isBuffered, + isRTX, + skippedSeqs, + oobSequenceNumber, + ) +} + +func (b *BufferBase) HandleIncomingPacketLocked( + rawPkt []byte, + rtpPacket *rtp.Packet, + arrivalTime int64, + isBuffered bool, + isRTX bool, + skippedSeqs []uint16, + oobSequenceNumber uint16, +) (uint64, error) { + if rtpPacket == nil { + rtpPacket = &rtp.Packet{} + if err := rtpPacket.Unmarshal(rawPkt); err != nil { + b.logger.Errorw("could not unmarshal RTP packet", err) + return 0, err + } + } + + b.processAudioSsrcLevelHeaderExtension(rtpPacket, arrivalTime) + + if len(skippedSeqs) > 0 { + skippedRtpPkt := rtp.Packet{ + Header: rtpPacket.Header, + } + skippedRtpPkt.Marker = false + // Use the current highest timestamp to prevent the case of old sequence number and newer timestamp. + // It is possible that the skipped packet is older. An example sequence + // - Packet 10, skipped 6, 7, 9 -> Packet 8 is unknown at this point + // - Packet 11, skipped 8 -> this would cause sequence number be older, but using timestamp from Packet 11 will make time stamp diff +ve + skippedRtpPkt.Timestamp = b.rtpStats.HighestTimestamp() + for _, sn := range skippedSeqs { + skippedRtpPkt.SequenceNumber = sn + flowState := b.rtpStats.Update( + arrivalTime, + skippedRtpPkt.Header.SequenceNumber, + skippedRtpPkt.Header.Timestamp, + skippedRtpPkt.Header.Marker, + skippedRtpPkt.Header.MarshalSize(), + len(skippedRtpPkt.Payload), + int(skippedRtpPkt.PaddingSize), + ) + if flowState.UnhandledReason == rtpstats.RTPFlowUnhandledReasonNone && !flowState.IsOutOfOrder { + if err := b.snRangeMap.ExcludeRange(flowState.ExtSequenceNumber, flowState.ExtSequenceNumber+1); err != nil { + b.logger.Errorw( + "could not exclude range", err, + "sequenceNumber", sn, + "extSequenceNumber", flowState.ExtSequenceNumber, + "rtpStats", b.rtpStats, + "rtpStatsLite", b.rtpStatsLite, + "snRangeMap", b.snRangeMap, + "skipped", skippedSeqs, + ) + } + } + } + } + + // do not start on an RTX packet + if isRTX && !b.rtpStats.IsActive() { + return 0, errors.New("cannot start on rtx packet") + } + + flowState := b.rtpStats.Update( + arrivalTime, + rtpPacket.Header.SequenceNumber, + rtpPacket.Header.Timestamp, + rtpPacket.Header.Marker, + rtpPacket.Header.MarshalSize(), + len(rtpPacket.Payload), + int(rtpPacket.PaddingSize), + ) + switch flowState.UnhandledReason { + case rtpstats.RTPFlowUnhandledReasonNone: + case rtpstats.RTPFlowUnhandledReasonRestart: + if !b.enableStreamRestartDetection { + return 0, fmt.Errorf("unhandled reason: %s", flowState.UnhandledReason.String()) + } + + b.restartStreamLocked("discontinuity", true) + + flowState = b.rtpStats.Update( + arrivalTime, + rtpPacket.Header.SequenceNumber, + rtpPacket.Header.Timestamp, + rtpPacket.Header.Marker, + rtpPacket.Header.MarshalSize(), + len(rtpPacket.Payload), + int(rtpPacket.PaddingSize), + ) + default: + return 0, fmt.Errorf("unhandled reason: %s", flowState.UnhandledReason.String()) + } + + if b.params.IsOOBSequenceNumber { + b.updateOOBNACKState(oobSequenceNumber, arrivalTime, len(rawPkt)) + } else { + b.updateNACKState(rtpPacket.SequenceNumber, flowState) + } + + if len(rtpPacket.Payload) == 0 && (!flowState.IsOutOfOrder || flowState.IsDuplicate) { + // drop padding only in-order or duplicate packet + if !flowState.IsOutOfOrder { + // in-order packet - increment sequence number offset for subsequent packets + // Example: + // 40 - regular packet - pass through as sequence number 40 + // 41 - missing packet - don't know what it is, could be padding or not + // 42 - padding only packet - in-order - drop - increment sequence number offset to 1 - + // range[0, 42] = 0 offset + // 41 - arrives out of order - get offset 0 from cache - passed through as sequence number 41 + // 43 - regular packet - offset = 1 (running offset) - passes through as sequence number 42 + // 44 - padding only - in order - drop - increment sequence number offset to 2 + // range[0, 42] = 0 offset, range[43, 44] = 1 offset + // 43 - regular packet - out of order + duplicate - offset = 1 from cache - + // adjusted sequence number is 42, will be dropped by RTX buffer AddPacket method as duplicate + // 45 - regular packet - offset = 2 (running offset) - passed through with adjusted sequence number as 43 + // 44 - padding only - out-of-order + duplicate - dropped as duplicate + // + if err := b.snRangeMap.ExcludeRange(flowState.ExtSequenceNumber, flowState.ExtSequenceNumber+1); err != nil { + b.logger.Errorw( + "could not exclude range", err, + "sn", rtpPacket.SequenceNumber, + "esn", flowState.ExtSequenceNumber, + "rtpStats", b.rtpStats, + "snRangeMap", b.snRangeMap, + ) + } + } + return 0, errors.New("padding only packet") + } + + if !flowState.IsOutOfOrder && rtpPacket.PayloadType != b.payloadType && b.codecType == webrtc.RTPCodecTypeVideo { + b.logger.Infow("possible codec change", "oldPT", b.payloadType, "receivedPT", rtpPacket.PayloadType) + b.handleCodecChange(rtpPacket.PayloadType) + } + + // add to RTX buffer using sequence number after accounting for dropped padding only packets + snAdjustment, err := b.snRangeMap.GetValue(flowState.ExtSequenceNumber) + if err != nil { + b.logger.Errorw( + "could not get sequence number adjustment", err, + "sequenceNumber", rtpPacket.SequenceNumber, + "extSequenceNumber", flowState.ExtSequenceNumber, + "timestamp", rtpPacket.Timestamp, + "extTimestamp", flowState.ExtTimestamp, + "payloadSize", len(rtpPacket.Payload), + "paddingSize", rtpPacket.PaddingSize, + "rtpStats", b.rtpStats, + "rtpStatsLite", b.rtpStatsLite, + "snRangeMap", b.snRangeMap, + ) + return 0, err + } + + flowState.ExtSequenceNumber -= snAdjustment + rtpPacket.Header.SequenceNumber = uint16(flowState.ExtSequenceNumber) + if _, err = b.bucket.AddPacketWithSequenceNumber(rawPkt, flowState.ExtSequenceNumber); err != nil { + if !flowState.IsDuplicate { + if errors.Is(err, bucket.ErrPacketTooOld) { + packetTooOldCount := b.packetTooOldCount.Inc() + if (packetTooOldCount-1)%100 == 0 { + b.logger.Warnw( + "could not add packet to bucket", err, + "count", packetTooOldCount, + "flowState", &flowState, + "snAdjustment", snAdjustment, + "incomingSequenceNumber", flowState.ExtSequenceNumber+snAdjustment, + "rtpStats", b.rtpStats, + "rtpStatsLite", b.rtpStatsLite, + "snRangeMap", b.snRangeMap, + "skipped", skippedSeqs, + ) + } + } else if err != bucket.ErrRTXPacket { + b.logger.Warnw( + "could not add packet to bucket", err, + "flowState", &flowState, + "snAdjustment", snAdjustment, + "incomingSequenceNumber", flowState.ExtSequenceNumber+snAdjustment, + "rtpStats", b.rtpStats, + "rtpStatsLite", b.rtpStatsLite, + "snRangeMap", b.snRangeMap, + "skipped", skippedSeqs, + ) + } + } + return 0, err + } + + ep := b.getExtPacket(rtpPacket, arrivalTime, isBuffered, flowState) + if ep == nil { + return 0, errors.New("could not get ext packet") + } + b.extPackets.PushBack(ep) + b.readCond.Broadcast() + + if b.extPackets.Len() > b.bucket.Capacity() { + if (b.extPacketTooMuchCount.Inc()-1)%100 == 0 { + b.logger.Warnw("too much ext packets", nil, "count", b.extPackets.Len()) + } + } + + b.maybeGrowBucket(arrivalTime) + + return ep.ExtSequenceNumber, nil +} + +func (b *BufferBase) updateNACKState(sequenceNumber uint16, flowState rtpstats.RTPFlowState) { + if b.nacker == nil { + return + } + + b.nacker.Remove(sequenceNumber) + + for lost := flowState.LossStartInclusive; lost != flowState.LossEndExclusive; lost++ { + b.nacker.Push(uint16(lost)) + } +} + +func (b *BufferBase) updateOOBNACKState(sequenceNumber uint16, arrivalTime int64, size int) { + if b.nacker == nil || !b.params.IsOOBSequenceNumber { + return + } + + fsLite := b.rtpStatsLite.Update(arrivalTime, size, sequenceNumber) + if fsLite.IsNotHandled { + return + } + + b.nacker.Remove(sequenceNumber) + + for lost := fsLite.LossStartInclusive; lost != fsLite.LossEndExclusive; lost++ { + b.nacker.Push(uint16(lost)) + } +} + +func (b *BufferBase) processAudioSsrcLevelHeaderExtension(p *rtp.Packet, arrivalTime int64) { + if b.audioLevelExtID == 0 { + return + } + + if e := p.GetExtension(b.audioLevelExtID); e != nil { + ext := rtp.AudioLevelExtension{} + if err := ext.Unmarshal(e); err == nil { + b.audioLevel.ObserveWithRTPTimestamp(ext.Level, p.Timestamp, arrivalTime) + } + } +} + +func (b *BufferBase) handleCodecChange(newPT uint8) { + var ( + codecFound, rtxFound bool + rtxPt uint8 + newCodec webrtc.RTPCodecParameters + ) + for _, codec := range b.rtpParameters.Codecs { + if !codecFound && uint8(codec.PayloadType) == newPT { + newCodec = codec + codecFound = true + } + + if mime.IsMimeTypeStringRTX(codec.MimeType) && strings.Contains(codec.SDPFmtpLine, fmt.Sprintf("apt=%d", newPT)) { + rtxFound = true + rtxPt = uint8(codec.PayloadType) + } + + if codecFound && rtxFound { + break + } + } + if !codecFound { + b.logger.Errorw( + "could not find codec for new payload type", nil, + "pt", newPT, + "rtpParameters", b.rtpParameters, + ) + return + } + + b.logger.Infow( + "codec changed", + "oldPayload", b.payloadType, "newPayload", newPT, + "oldRtxPayload", b.rtxPayloadType, "newRtxPayload", rtxPt, + "oldMime", b.mime, "newMime", newCodec.MimeType, + ) + b.payloadType = newPT + b.rtxPayloadType = rtxPt + b.mime = mime.NormalizeMimeType(newCodec.MimeType) + + if f := b.onCodecChange; f != nil { + go f(newCodec) + } +} + +func (b *BufferBase) getExtPacket( + rtpPacket *rtp.Packet, + arrivalTime int64, + isBuffered bool, + flowState rtpstats.RTPFlowState, +) *ExtPacket { + ep := ExtPacketFactory.Get().(*ExtPacket) + *ep = ExtPacket{ + Arrival: arrivalTime, + ExtSequenceNumber: flowState.ExtSequenceNumber, + ExtTimestamp: flowState.ExtTimestamp, + Packet: rtpPacket, + VideoLayer: VideoLayer{ + Spatial: InvalidLayerSpatial, + Temporal: InvalidLayerTemporal, + }, + IsOutOfOrder: flowState.IsOutOfOrder, + IsBuffered: isBuffered, + } + + if len(ep.Packet.Payload) == 0 { + // padding only packet, nothing else to do + return ep + } + + if err := b.processVideoPacket(ep); err != nil { + ReleaseExtPacket(ep) + return nil + } + + if b.absCaptureTimeExtID != 0 { + extData := rtpPacket.GetExtension(b.absCaptureTimeExtID) + + var actExt act.AbsCaptureTime + if err := actExt.Unmarshal(extData); err == nil { + ep.AbsCaptureTimeExt = &actExt + } + } + + return ep +} + +func (b *BufferBase) processVideoPacket(ep *ExtPacket) error { + if b.codecType != webrtc.RTPCodecTypeVideo { + return nil + } + + ep.Temporal = 0 + var videoSize []codec.VideoSize + if b.ddParser != nil { + ddVal, videoLayer, err := b.ddParser.Parse(ep.Packet) + if err != nil { + if errors.Is(err, ErrDDExtentionNotFound) { + if b.mime == mime.MimeTypeVP8 || b.mime == mime.MimeTypeVP9 { + b.logger.Infow("dd extension not found, disable dd parser") + b.ddParser = nil + b.createFrameRateCalculator() + } + } else { + return err + } + } else if ddVal != nil { + ep.DependencyDescriptor = ddVal + ep.VideoLayer = videoLayer + videoSize = ExtractDependencyDescriptorVideoSize(ddVal.Descriptor) + // DD-TODO : notify active decode target change if changed. + } + } + + switch b.mime { + case mime.MimeTypeVP8: + vp8Packet := codec.VP8{} + if err := vp8Packet.Unmarshal(ep.Packet.Payload); err != nil { + b.logger.Warnw("could not unmarshal VP8 packet", err) + return err + } + ep.IsKeyFrame = vp8Packet.IsKeyFrame + if ep.DependencyDescriptor == nil { + ep.Temporal = int32(vp8Packet.TID) + + if ep.IsKeyFrame { + if sz := codec.ExtractVP8VideoSize(&vp8Packet, ep.Packet.Payload); sz.Width > 0 && sz.Height > 0 { + videoSize = append(videoSize, sz) + } + } + } else { + // vp8 with DependencyDescriptor enabled, use the TID from the descriptor + vp8Packet.TID = uint8(ep.Temporal) + } + ep.Payload = vp8Packet + ep.Spatial = InvalidLayerSpatial // vp8 don't have spatial scalability, reset to invalid + + case mime.MimeTypeVP9: + if ep.DependencyDescriptor == nil { + var vp9Packet codecs.VP9Packet + _, err := vp9Packet.Unmarshal(ep.Packet.Payload) + if err != nil { + b.logger.Warnw("could not unmarshal VP9 packet", err) + return err + } + ep.VideoLayer = VideoLayer{ + Spatial: int32(vp9Packet.SID), + Temporal: int32(vp9Packet.TID), + } + ep.Payload = vp9Packet + ep.IsKeyFrame = codec.IsVP9KeyFrame(&vp9Packet, ep.Packet.Payload) + + if ep.IsKeyFrame { + for i := 0; i < len(vp9Packet.Width); i++ { + videoSize = append(videoSize, codec.VideoSize{ + Width: uint32(vp9Packet.Width[i]), + Height: uint32(vp9Packet.Height[i]), + }) + } + } + } else { + ep.IsKeyFrame = codec.IsVP9KeyFrame(nil, ep.Packet.Payload) + } + + case mime.MimeTypeH264: + ep.IsKeyFrame = codec.IsH264KeyFrame(ep.Packet.Payload) + ep.Spatial = InvalidLayerSpatial // h.264 don't have spatial scalability, reset to invalid + + // Check H264 key frame video size + if ep.IsKeyFrame { + if sz := codec.ExtractH264VideoSize(ep.Packet.Payload); sz.Width > 0 && sz.Height > 0 { + videoSize = append(videoSize, sz) + } + } + + case mime.MimeTypeAV1: + ep.IsKeyFrame = codec.IsAV1KeyFrame(ep.Packet.Payload) + + case mime.MimeTypeH265: + ep.IsKeyFrame = codec.IsH265KeyFrame(ep.Packet.Payload) + if ep.DependencyDescriptor == nil { + if len(ep.Packet.Payload) < 2 { + b.logger.Warnw("invalid H265 packet", nil, "payloadLen", len(ep.Packet.Payload)) + return errors.New("invalid H265 packet") + } + ep.VideoLayer = VideoLayer{ + Temporal: int32(ep.Packet.Payload[1]&0x07) - 1, + } + ep.Spatial = InvalidLayerSpatial + + if ep.IsKeyFrame { + if sz := codec.ExtractH265VideoSize(ep.Packet.Payload); sz.Width > 0 && sz.Height > 0 { + videoSize = append(videoSize, sz) + } + } + } + } + + if ep.IsKeyFrame { + if b.rtpStats != nil { + b.rtpStats.UpdateKeyFrame(1) + } + } + + if len(videoSize) > 0 { + b.checkVideoSizeChange(videoSize) + } + + b.doFpsCalc(ep) + + return nil +} + +func (b *BufferBase) patchExtPacket(ep *ExtPacket, buf []byte) *ExtPacket { + n, err := b.getPacketLocked(buf, ep.ExtSequenceNumber) + if err != nil { + packetNotFoundCount := b.packetNotFoundCount.Inc() + if (packetNotFoundCount-1)%20 == 0 { + b.logger.Warnw( + "could not get packet from bucket", err, + "sn", ep.Packet.SequenceNumber, + "headSN", b.bucket.HeadSequenceNumber(), + "count", packetNotFoundCount, + "rtpStats", b.rtpStats, + "rtpStatsLite", b.rtpStatsLite, + "snRangeMap", b.snRangeMap, + ) + } + return nil + } + ep.RawPacket = buf[:n] + + // patch RTP packet to point payload to new buffer + pkt := *ep.Packet + payloadStart := ep.Packet.Header.MarshalSize() + payloadEnd := payloadStart + len(ep.Packet.Payload) + if payloadEnd > n { + b.logger.Warnw("unexpected marshal size", nil, "max", n, "need", payloadEnd) + return nil + } + pkt.Payload = buf[payloadStart:payloadEnd] + ep.Packet = &pkt + + return ep +} + +func (b *BufferBase) flushExtPackets() { + b.Lock() + defer b.Unlock() + b.flushExtPacketsLocked() +} + +func (b *BufferBase) flushExtPacketsLocked() { + for b.extPackets.Len() > 0 { + ep := b.extPackets.PopFront() + ReleaseExtPacket(ep) + } + b.extPackets.Clear() +} + +func (b *BufferBase) maybeGrowBucket(now int64) { + if now-b.lastBucketCapCheckAt < bucketCapCheckInterval { + return + } + + b.lastBucketCapCheckAt = now + + // check and allocate in a go routine, away from the forwarding path + go func() { + b.Lock() + defer b.Unlock() + + cap := b.bucket.Capacity() + maxPkts := b.params.MaxVideoPkts + if b.codecType == webrtc.RTPCodecTypeAudio { + maxPkts = b.params.MaxAudioPkts + } + if cap >= maxPkts { + return + } + + oldCap := cap + if deltaInfo := b.rtpStats.DeltaInfo(b.ppsSnapshotId); deltaInfo != nil { + duration := deltaInfo.EndTime.Sub(deltaInfo.StartTime) + if duration < 500*time.Millisecond { + return + } + + pps := int(time.Duration(deltaInfo.Packets) * time.Second / duration) + for pps > cap && cap < maxPkts { + cap = b.bucket.Grow() + } + if cap > oldCap { + b.logger.Infow( + "grow bucket", + "from", oldCap, + "to", cap, + "pps", pps, + "deltaInfo", deltaInfo, + "rtpStats", b.rtpStats, + ) + } + } + }() +} + +func (b *BufferBase) doFpsCalc(ep *ExtPacket) { + if b.isPaused || b.frameRateCalculated || len(ep.Packet.Payload) == 0 { + return + } + + spatial := ep.Spatial + if spatial < 0 || int(spatial) >= len(b.frameRateCalculator) { + spatial = 0 + } + if fr := b.frameRateCalculator[spatial]; fr != nil { + if fr.RecvPacket(ep) { + complete := true + for _, fr2 := range b.frameRateCalculator { + if fr2 != nil && !fr2.Completed() { + complete = false + break + } + } + if complete { + b.frameRateCalculated = true + if f := b.onFpsChanged; f != nil { + go f() + } + } + } + } +} + +func (b *BufferBase) SetSenderReportData(srData *livekit.RTCPSenderReportState) { + b.RLock() + didSet := false + if b.rtpStats != nil { + didSet = b.rtpStats.SetRtcpSenderReportData(srData) + } + b.RUnlock() + + if didSet { + if cb := b.getOnRtcpSenderReport(); cb != nil { + cb() + } + } +} + +func (b *BufferBase) GetSenderReportData() *livekit.RTCPSenderReportState { + b.RLock() + defer b.RUnlock() + + if b.rtpStats != nil { + return b.rtpStats.GetRtcpSenderReportData() + } + + return nil +} + +func (b *BufferBase) GetPacket(buff []byte, esn uint64) (int, error) { + b.Lock() + defer b.Unlock() + + return b.getPacketLocked(buff, esn) +} + +func (b *BufferBase) getPacketLocked(buff []byte, esn uint64) (int, error) { + if b.isClosed.Load() { + return 0, io.EOF + } + return b.bucket.GetPacket(buff, esn) +} + +func (b *BufferBase) GetStats() *livekit.RTPStats { + b.RLock() + defer b.RUnlock() + + if b.rtpStats == nil { + return nil + } + + return b.rtpStats.ToProto() +} + +func (b *BufferBase) GetDeltaStats() *StreamStatsWithLayers { + b.RLock() + defer b.RUnlock() + + if b.rtpStats == nil { + return nil + } + + deltaStats := b.rtpStats.DeltaInfo(b.deltaStatsSnapshotId) + if deltaStats == nil { + return nil + } + + return &StreamStatsWithLayers{ + RTPStats: deltaStats, + Layers: map[int32]*rtpstats.RTPDeltaInfo{ + 0: deltaStats, + }, + } +} + +func (b *BufferBase) GetDeltaStatsLite() *rtpstats.RTPDeltaInfoLite { + b.RLock() + defer b.RUnlock() + + if b.rtpStatsLite == nil { + return nil + } + + return b.rtpStatsLite.DeltaInfoLite(b.liteStatsSnapshotId) +} + +func (b *BufferBase) GetLastSenderReportTime() time.Time { + b.RLock() + defer b.RUnlock() + + if b.rtpStats == nil { + return time.Time{} + } + + return b.rtpStats.LastSenderReportTime() +} + +func (b *BufferBase) GetAudioLevel() (float64, bool) { + b.RLock() + defer b.RUnlock() + + if b.audioLevel == nil { + return 0, false + } + + return b.audioLevel.GetLevel(mono.UnixNano()) +} + +func (b *BufferBase) OnRtcpSenderReport(fn func()) { + b.Lock() + b.onRtcpSenderReport = fn + b.Unlock() +} + +func (b *BufferBase) getOnRtcpSenderReport() func() { + b.RLock() + defer b.RUnlock() + + return b.onRtcpSenderReport +} + +func (b *BufferBase) OnFpsChanged(f func()) { + b.Lock() + b.onFpsChanged = f + b.Unlock() +} + +func (b *BufferBase) OnVideoSizeChanged(fn func([]codec.VideoSize)) { + b.Lock() + b.onVideoSizeChanged = fn + b.Unlock() +} + +func (b *BufferBase) OnCodecChange(fn func(webrtc.RTPCodecParameters)) { + b.Lock() + b.onCodecChange = fn + b.Unlock() +} + +func (b *BufferBase) OnStreamRestart(fn func(string)) { + b.Lock() + b.onStreamRestart = fn + b.Unlock() +} + +// checkVideoSizeChange checks if video size has changed for a specific spatial layer and fires callback +func (b *BufferBase) checkVideoSizeChange(videoSizes []codec.VideoSize) { + if len(videoSizes) > len(b.currentVideoSize) { + b.logger.Warnw( + "video size index out of range", nil, + "newSize", videoSizes, + "currentVideoSize", b.currentVideoSize, + ) + return + } + + if len(videoSizes) < len(b.currentVideoSize) { + videoSizes = append(videoSizes, make([]codec.VideoSize, len(b.currentVideoSize)-len(videoSizes))...) + } + + changed := false + for i, sz := range videoSizes { + if b.currentVideoSize[i].Width != sz.Width || b.currentVideoSize[i].Height != sz.Height { + changed = true + break + } + } + + if changed { + b.logger.Debugw("video size changed", "from", b.currentVideoSize, "to", videoSizes) + copy(b.currentVideoSize[:], videoSizes[:]) + if b.onVideoSizeChanged != nil { + go b.onVideoSizeChanged(videoSizes) + } + } +} + +func (b *BufferBase) GetTemporalLayerFpsForSpatial(layer int32) []float32 { + b.RLock() + defer b.RUnlock() + + if int(layer) >= len(b.frameRateCalculator) { + return nil + } + + if fc := b.frameRateCalculator[layer]; fc != nil { + return fc.GetFrameRate() + } + return nil +} + +func (b *BufferBase) StartKeyFrameSeeder() { + if b.codecType == webrtc.RTPCodecTypeVideo { + go b.seedKeyFrame(b.keyFrameSeederGeneration.Inc()) + } +} + +func (b *BufferBase) StopKeyFrameSeeder() { + b.keyFrameSeederGeneration.Inc() +} + +func (b *BufferBase) seedKeyFrame(keyFrameSeederGeneration int32) { + // a key frame is needed especially when using Dependency Descriptor + // to get the DD structure which is used in parsing subsequent packets, + // till then packets are dropped which results in stream tracker not + // getting any data which means it does not declare layer start. + // + // send gratuitous PLIs for some time or until a key frame is seen to + // get the engine rolling + timer := time.NewTimer(30 * time.Second) + defer timer.Stop() + + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + + initialCount := uint32(0) + b.RLock() + rtpStats := b.rtpStats + lgr := b.logger + b.RUnlock() + lgr.Debugw("starting key frame seeder", "generation", keyFrameSeederGeneration) + if rtpStats == nil { + lgr.Debugw("cannot do key frame seeding without stats", "generation", keyFrameSeederGeneration) + return + } + initialCount, _ = rtpStats.KeyFrame() + + for { + if b.isClosed.Load() || b.keyFrameSeederGeneration.Load() != keyFrameSeederGeneration { + lgr.Debugw( + "stopping key frame seeder: stopped", + "generation", keyFrameSeederGeneration, + "currentGeneration", b.keyFrameSeederGeneration.Load(), + ) + return + } + + select { + case <-timer.C: + lgr.Debugw("stopping key frame seeder: timeout", "generation", keyFrameSeederGeneration) + return + + case <-ticker.C: + cnt, last := rtpStats.KeyFrame() + if cnt > initialCount { + lgr.Debugw( + "stopping key frame seeder: received key frame", + "generation", keyFrameSeederGeneration, + "keyFrameCountInitial", initialCount, + "keyFrameCount", cnt, + "lastKeyFrame", last, + ) + return + } + + b.SendPLI(false) + } + } +} + +func (b *BufferBase) GetNACKPairs() []rtcp.NackPair { + b.RLock() + defer b.RUnlock() + + return b.GetNACKPairsLocked() +} + +func (b *BufferBase) GetNACKPairsLocked() []rtcp.NackPair { + if b.nacker == nil { + return nil + } + + pairs, numSeqNumsNacked := b.nacker.Pairs() + if !b.params.IsOOBSequenceNumber { + if b.rtpStats != nil { + b.rtpStats.UpdateNack(uint32(numSeqNumsNacked)) + } + } else { + if b.rtpStatsLite != nil { + b.rtpStatsLite.UpdateNack(uint32(numSeqNumsNacked)) + } + } + + return pairs +} + +func (b *BufferBase) GetRtcpReceptionReportLocked(proxyLoss uint8) *rtcp.ReceptionReport { + if b.rtpStats == nil { + return nil + } + + return b.rtpStats.GetRtcpReceptionReport(b.params.SSRC, proxyLoss, b.rrSnapshotId) +} + +// --------------------------------------------------------------- diff --git a/pkg/sfu/buffer/buffer_test.go b/pkg/sfu/buffer/buffer_test.go index 55aaebb..f0f3076 100644 --- a/pkg/sfu/buffer/buffer_test.go +++ b/pkg/sfu/buffer/buffer_test.go @@ -15,6 +15,7 @@ package buffer import ( + "fmt" "math" "sync" "testing" @@ -82,7 +83,7 @@ func TestNack(t *testing.T) { }, vp8Codec.RTPCodecCapability, 0) rtt := uint32(20) buff.nacker.SetRTT(rtt) - for i := 0; i < 15; i++ { + for i := range 15 { if i == 1 { continue } @@ -147,7 +148,7 @@ func TestNack(t *testing.T) { }, vp8Codec.RTPCodecCapability, 0) rtt := uint32(30) buff.nacker.SetRTT(rtt) - for i := 0; i < 15; i++ { + for i := range 15 { if i > 0 && i < 5 { continue } @@ -265,7 +266,7 @@ func TestFractionLostReport(t *testing.T) { HeaderExtensions: nil, Codecs: []webrtc.RTPCodecParameters{opusCodec}, }, opusCodec.RTPCodecCapability, 0) - for i := 0; i < 15; i++ { + for i := range 15 { pkt := rtp.Packet{ Header: rtp.Header{ Version: 2, @@ -303,7 +304,7 @@ func TestFractionLostReport(t *testing.T) { HeaderExtensions: nil, Codecs: []webrtc.RTPCodecParameters{opusCodec}, }, opusCodec.RTPCodecCapability, 0) - for i := 0; i < 15; i++ { + for i := range 15 { pkt := rtp.Packet{ Header: rtp.Header{ Version: 2, @@ -337,6 +338,15 @@ func TestCodecChange(t *testing.T) { t.Fatalf("codec change not consumed") } }) + buff.OnStreamRestart(func(reason string) { + require.Equal(t, "codec-change", reason) + + // read once to clear pending restart + var buf [1500]byte + extPkt, err := buff.ReadExtended(buf[:]) + require.NoError(t, err) + require.Nil(t, extPkt) + }) h265Pkt := rtp.Packet{ Header: rtp.Header{ @@ -359,20 +369,27 @@ func TestCodecChange(t *testing.T) { case <-time.After(100 * time.Millisecond): } - buff.Bind(webrtc.RTPParameters{ - HeaderExtensions: nil, - Codecs: []webrtc.RTPCodecParameters{vp8Codec, h265Codec}, - }, vp8Codec.RTPCodecCapability, 0) + // Bind sets up VP8 as expected codec, + // packet written to the buffer above before Bind is H.265, + // that should trigger a codec change and a stream restart + // when the queued packets from Write before Bind get flushed + buff.Bind( + webrtc.RTPParameters{ + HeaderExtensions: nil, + Codecs: []webrtc.RTPCodecParameters{vp8Codec, h265Codec}, + }, + vp8Codec.RTPCodecCapability, + 0, + ) select { case c := <-changedCodec: require.Equal(t, h265Codec, c) - case <-time.After(1 * time.Second): t.Fatalf("expected codec change") } - // codec change after bind + // second codec change - writing VP8 packet after Bind should trigger another codec change vp8Pkt := rtp.Packet{ Header: rtp.Header{ Version: 2, @@ -391,12 +408,16 @@ func TestCodecChange(t *testing.T) { select { case c := <-changedCodec: require.Equal(t, vp8Codec, c) - case <-time.After(1 * time.Second): t.Fatalf("expected codec change") } + fmt.Printf("done second codec change\n") // REMOVE // out of order pkts can't cause codec change + // rewrite the VP8 packet to start the sequence after a stream restart + _, err = buff.Write(buf) + require.NoError(t, err) + h265Pkt.SequenceNumber = 2 h265Pkt.Timestamp = 2 buf, err = h265Pkt.Marshal() @@ -409,7 +430,7 @@ func TestCodecChange(t *testing.T) { case <-time.After(100 * time.Millisecond): } - // unknown codec should not cause change + // unknown codec should not cause change even if it is in order h265Pkt.SequenceNumber = 4 h265Pkt.Timestamp = 4 h265Pkt.PayloadType = 117 @@ -422,14 +443,37 @@ func TestCodecChange(t *testing.T) { t.Fatalf("unexpected codec change") case <-time.After(100 * time.Millisecond): } + + // an in-order packet should change codec again + h265Pkt.SequenceNumber = 5 + h265Pkt.Timestamp = 5 + h265Pkt.PayloadType = 116 + buf, err = h265Pkt.Marshal() + require.NoError(t, err) + _, err = buff.Write(buf) + require.NoError(t, err) + select { + case c := <-changedCodec: + require.Equal(t, h265Codec, c) + case <-time.After(1 * time.Second): + t.Fatalf("expected codec change") + } } func BenchmarkMemcpu(b *testing.B) { buf := make([]byte, 1500*1500*10) buf2 := make([]byte, 1500*1500*20) - b.ResetTimer() - for i := 0; i < b.N; i++ { + for b.Loop() { copy(buf2, buf) } } + +func BenchmarkExtPacketFactory(b *testing.B) { + + for b.Loop() { + extPkt := ExtPacketFactory.Get().(*ExtPacket) + *extPkt = ExtPacket{} + ExtPacketFactory.Put(extPkt) + } +} diff --git a/pkg/sfu/buffer/dependencydescriptorparser.go b/pkg/sfu/buffer/dependencydescriptorparser.go index 7bab83e..618200a 100644 --- a/pkg/sfu/buffer/dependencydescriptorparser.go +++ b/pkg/sfu/buffer/dependencydescriptorparser.go @@ -16,20 +16,41 @@ package buffer import ( "fmt" - "sort" + "slices" + "sync" + "time" "github.com/pion/rtp" "go.uber.org/atomic" dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor" - "github.com/livekit/livekit-server/pkg/sfu/utils" - + "github.com/livekit/mediatransportutil/pkg/codec" + "github.com/livekit/mediatransportutil/pkg/utils" "github.com/livekit/protocol/logger" ) +var ( + ExtDependencyDescriptorFactory = &sync.Pool{ + New: func() any { + return &ExtDependencyDescriptor{} + }, + } +) + +// -------------------------------------- + +const ( + ddRestartThreshold = 30 * time.Second + + // frame integrity check 2 seconds for L3T3 30fps video + integrityCheckFrame = 180 + integrityCheckPkt = 1024 +) + var ( ErrFrameEarlierThanKeyFrame = fmt.Errorf("frame is earlier than current keyframe") ErrDDStructureAttachedToNonFirstPacket = fmt.Errorf("dependency descriptor structure is attached to non-first packet of a frame") + ErrDDExtentionNotFound = fmt.Errorf("dependency descriptor extension not found") ) type DependencyDescriptorParser struct { @@ -47,16 +68,22 @@ type DependencyDescriptorParser struct { frameChecker *FrameIntegrityChecker ddNotFoundCount atomic.Uint32 + + // restart detection + restartGeneration int + enableRestart bool + lastPacketAt time.Time } -func NewDependencyDescriptorParser(ddExtID uint8, logger logger.Logger, onMaxLayerChanged func(int32, int32)) *DependencyDescriptorParser { +func NewDependencyDescriptorParser(ddExtID uint8, logger logger.Logger, onMaxLayerChanged func(int32, int32), enableRestart bool) *DependencyDescriptorParser { return &DependencyDescriptorParser{ ddExtID: ddExtID, logger: logger, onMaxLayerChanged: onMaxLayerChanged, seqWrapAround: utils.NewWrapAround[uint16, uint64](utils.WrapAroundParams{IsRestartAllowed: false}), frameWrapAround: utils.NewWrapAround[uint16, uint64](utils.WrapAroundParams{IsRestartAllowed: false}), - frameChecker: NewFrameIntegrityChecker(180, 1024), // 2seconds for L3T3 30fps video + frameChecker: NewFrameIntegrityChecker(integrityCheckFrame, integrityCheckPkt), + enableRestart: enableRestart, } } @@ -70,6 +97,10 @@ type ExtDependencyDescriptor struct { ExtFrameNum uint64 // the frame number of the keyframe which the current frame depends on ExtKeyFrameNum uint64 + + // increase when the stream restarts, clear and reinitialize all dd state includes + // attached structure, frame chain, decode target. + RestartGeneration int } func (r *DependencyDescriptorParser) Parse(pkt *rtp.Packet) (*ExtDependencyDescriptor, VideoLayer, error) { @@ -80,7 +111,7 @@ func (r *DependencyDescriptorParser) Parse(pkt *rtp.Packet) (*ExtDependencyDescr if ddNotFoundCount%100 == 0 { r.logger.Warnw("dependency descriptor extension is not present", nil, "seq", pkt.SequenceNumber, "count", ddNotFoundCount) } - return nil, videoLayer, nil + return nil, videoLayer, ErrDDExtentionNotFound } var ddVal dd.DependencyDescriptor @@ -96,41 +127,92 @@ func (r *DependencyDescriptorParser) Parse(pkt *rtp.Packet) (*ExtDependencyDescr return nil, videoLayer, err } + var restart bool + if r.enableRestart { + if !r.lastPacketAt.IsZero() && time.Since(r.lastPacketAt) > ddRestartThreshold { + r.restart() + restart = true + r.logger.Debugw( + "dependency descriptor parser restart stream", + "generation", r.restartGeneration, + "lastPacketAt", r.lastPacketAt, + "sinceLast", time.Since(r.lastPacketAt), + "frameWrapAround", r.frameWrapAround, + ) + } + r.lastPacketAt = time.Now() + } + extSeq := r.seqWrapAround.Update(pkt.SequenceNumber).ExtendedVal if ddVal.FrameDependencies != nil { videoLayer.Spatial, videoLayer.Temporal = int32(ddVal.FrameDependencies.SpatialId), int32(ddVal.FrameDependencies.TemporalId) } - unwrapped := r.frameWrapAround.Update(ddVal.FrameNumber) + // assume the packet is in-order when stream restarting + unwrapped := r.frameWrapAround.UpdateWithOrderKnown(ddVal.FrameNumber, restart) extFN := unwrapped.ExtendedVal if extFN < r.structureExtFrameNum { - r.logger.Debugw("drop frame which is earlier than current structure", "frameNum", extFN, "structureFrameNum", r.structureExtFrameNum) + r.logger.Debugw( + "drop frame which is earlier than current structure", + "fn", ddVal.FrameNumber, + "extFN", extFN, + "structureExtFrameNum", r.structureExtFrameNum, + "unwrappedFN", unwrapped, + "frameWrapAround", r.frameWrapAround, + ) return nil, videoLayer, ErrFrameEarlierThanKeyFrame } r.frameChecker.AddPacket(extSeq, extFN, &ddVal) - extDD := &ExtDependencyDescriptor{ - Descriptor: &ddVal, - ExtFrameNum: extFN, - Integrity: r.frameChecker.FrameIntegrity(extFN), + extDD := ExtDependencyDescriptorFactory.Get().(*ExtDependencyDescriptor) + *extDD = ExtDependencyDescriptor{ + Descriptor: &ddVal, + ExtFrameNum: extFN, + Integrity: r.frameChecker.FrameIntegrity(extFN), + RestartGeneration: r.restartGeneration, } if ddVal.AttachedStructure != nil { if !ddVal.FirstPacketInFrame { - r.logger.Warnw("attached structure is not the first packet in frame", nil, "extSeq", extSeq, "extFN", extFN) + r.logger.Warnw( + "attached structure is not the first packet in frame", nil, + "sn", pkt.SequenceNumber, + "extSeq", extSeq, + "fn", ddVal.FrameNumber, + "extFN", extFN, + ) + ReleaseExtDependencyDescriptor(extDD) return nil, videoLayer, ErrDDStructureAttachedToNonFirstPacket } if r.structure == nil || ddVal.AttachedStructure.StructureId != r.structure.StructureId { - r.logger.Debugw("structure updated", "structureID", ddVal.AttachedStructure.StructureId, "extSeq", extSeq, "extFN", extFN, "descriptor", ddVal.String()) + r.logger.Debugw( + "structure updated", + "structureID", ddVal.AttachedStructure.StructureId, + "sn", pkt.SequenceNumber, + "extSeq", extSeq, + "fn", ddVal.FrameNumber, + "extFN", extFN, + "descriptor", ddVal.String(), + "unwrappedFN", unwrapped, + "frameWrapAround", r.frameWrapAround, + ) } r.structure = ddVal.AttachedStructure r.decodeTargets = ProcessFrameDependencyStructure(ddVal.AttachedStructure) if extFN > unwrapped.PreExtendedHighest && extFN-unwrapped.PreExtendedHighest > 1000 { - r.logger.Debugw("large frame number jump on structure updating", "extFN", extFN, "preExtendedHighest", unwrapped.PreExtendedHighest, "structureExtFrameNum", r.structureExtFrameNum) + r.logger.Debugw( + "large frame number jump on structure updating", + "fn", ddVal.FrameNumber, + "extFN", extFN, + "preExtendedHighest", unwrapped.PreExtendedHighest, + "structureExtFrameNum", r.structureExtFrameNum, + "unwrappedFN", unwrapped, + "frameWrapAround", r.frameWrapAround, + ) } r.structureExtFrameNum = extFN extDD.StructureUpdated = true @@ -166,6 +248,16 @@ func (r *DependencyDescriptorParser) Parse(pkt *rtp.Packet) (*ExtDependencyDescr return extDD, videoLayer, nil } +func (r *DependencyDescriptorParser) restart() { + r.frameChecker = NewFrameIntegrityChecker(integrityCheckFrame, integrityCheckPkt) + r.structure = nil + r.structureExtFrameNum = 0 + r.activeDecodeTargetsExtSeq = 0 + r.activeDecodeTargetsMask = 0 + r.decodeTargets = r.decodeTargets[:0] + r.restartGeneration++ +} + // ------------------------------------------------------------------------------ type DependencyDescriptorDecodeTarget struct { @@ -197,8 +289,14 @@ func ProcessFrameDependencyStructure(structure *dd.FrameDependencyStructure) []D } // sort decode target layer by spatial and temporal from high to low - sort.Slice(decodeTargets, func(i, j int) bool { - return decodeTargets[i].Layer.GreaterThan(decodeTargets[j].Layer) + slices.SortFunc(decodeTargets, func(a, b DependencyDescriptorDecodeTarget) int { + if a.Layer.GreaterThan(b.Layer) { + return -1 + } + if b.Layer.GreaterThan(a.Layer) { + return 1 + } + return 0 }) return decodeTargets @@ -216,3 +314,27 @@ func GetActiveDecodeTargetBitmask(layer VideoLayer, decodeTargets []DependencyDe } // ------------------------------------------------------------------------------ + +func ExtractDependencyDescriptorVideoSize(dd *dd.DependencyDescriptor) []codec.VideoSize { + if dd.AttachedStructure == nil { + return nil + } + + videoSizes := make([]codec.VideoSize, 0, len(dd.AttachedStructure.Resolutions)) + for _, res := range dd.AttachedStructure.Resolutions { + videoSizes = append(videoSizes, codec.VideoSize{Width: uint32(res.Width), Height: uint32(res.Height)}) + } + + return videoSizes +} + +// ------------------------------------------------------------------------------ + +func ReleaseExtDependencyDescriptor(extDD *ExtDependencyDescriptor) { + if extDD == nil { + return + } + + *extDD = ExtDependencyDescriptor{} + ExtDependencyDescriptorFactory.Put(extDD) +} diff --git a/pkg/sfu/buffer/factory.go b/pkg/sfu/buffer/factory.go index ae0bb07..4a73ab0 100644 --- a/pkg/sfu/buffer/factory.go +++ b/pkg/sfu/buffer/factory.go @@ -18,7 +18,7 @@ import ( "io" "sync" - "github.com/pion/transport/v3/packetio" + "github.com/pion/transport/v4/packetio" ) type FactoryOfBufferFactory struct { @@ -118,7 +118,7 @@ func (f *Factory) GetRTCPReader(ssrc uint32) *RTCPReader { return f.rtcpReaders[ssrc] } -func (f *Factory) SetRTXPair(repair, base uint32) { +func (f *Factory) SetRTXPair(repair, base uint32, rsid string) { f.Lock() repairBuffer, baseBuffer := f.rtpBuffers[repair], f.rtpBuffers[base] if repairBuffer == nil || baseBuffer == nil { @@ -127,5 +127,8 @@ func (f *Factory) SetRTXPair(repair, base uint32) { f.Unlock() if repairBuffer != nil && baseBuffer != nil { repairBuffer.SetPrimaryBufferForRTX(baseBuffer) + if rsid != "" { + baseBuffer.NotifyRTX(base, repair, rsid) + } } } diff --git a/pkg/sfu/buffer/fps.go b/pkg/sfu/buffer/fps.go index 5622198..023e40c 100644 --- a/pkg/sfu/buffer/fps.go +++ b/pkg/sfu/buffer/fps.go @@ -19,6 +19,7 @@ import ( "github.com/pion/rtp/codecs" + "github.com/livekit/mediatransportutil/pkg/codec" "github.com/livekit/protocol/logger" ) @@ -209,7 +210,7 @@ func (f *FrameRateCalculatorVP8) RecvPacket(ep *ExtPacket) bool { return true } - vp8, ok := ep.Payload.(VP8) + vp8, ok := ep.Payload.(codec.VP8) if !ok { f.logger.Debugw("no vp8 payload", "sn", ep.Packet.SequenceNumber) return false diff --git a/pkg/sfu/buffer/fps_test.go b/pkg/sfu/buffer/fps_test.go index f560d8d..9c32cc5 100644 --- a/pkg/sfu/buffer/fps_test.go +++ b/pkg/sfu/buffer/fps_test.go @@ -21,6 +21,7 @@ import ( "github.com/stretchr/testify/require" dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor" + "github.com/livekit/mediatransportutil/pkg/codec" "github.com/livekit/protocol/logger" ) @@ -35,7 +36,7 @@ type testFrameInfo struct { func (f *testFrameInfo) toVP8() *ExtPacket { return &ExtPacket{ Packet: &rtp.Packet{Header: f.header}, - Payload: VP8{ + Payload: codec.VP8{ PictureID: f.framenumber, }, VideoLayer: VideoLayer{Spatial: InvalidLayerSpatial, Temporal: int32(f.temporal)}, @@ -75,7 +76,7 @@ func createFrames(startFrameNumber uint16, startTs uint32, startSeq uint16, tota nextTs := make([][]uint32, spatials) tsStep := make([][]uint32, spatials) - for i := 0; i < spatials; i++ { + for i := range spatials { nextTs[i] = make([]uint32, temporals) tsStep[i] = make([]uint32, temporals) for j := 0; j < temporals; j++ { @@ -85,11 +86,11 @@ func createFrames(startFrameNumber uint16, startTs uint32, startSeq uint16, tota } currentTs := make([]uint32, spatials) - for i := 0; i < spatials; i++ { + for i := range spatials { currentTs[i] = startTs } - for i := 0; i < totalFramesPerSpatial; i++ { - for s := 0; s < spatials; s++ { + for range totalFramesPerSpatial { + for s := range spatials { frame := &testFrameInfo{ header: rtp.Header{Timestamp: currentTs[s], SequenceNumber: startSeq}, framenumber: fn, @@ -134,7 +135,7 @@ func createFrames(startFrameNumber uint16, startTs uint32, startSeq uint16, tota func verifyFps(t *testing.T, expect, got []float32) { require.Equal(t, len(expect), len(got)) - for i := 0; i < len(expect); i++ { + for i := range expect { require.GreaterOrEqual(t, got[i], expect[i]*0.9, "expect %v, got %v", expect, got) require.LessOrEqual(t, got[i], expect[i]*1.1, "expect %v, got %v", expect, got) } diff --git a/pkg/sfu/buffer/helpers.go b/pkg/sfu/buffer/helpers.go deleted file mode 100644 index d8c862d..0000000 --- a/pkg/sfu/buffer/helpers.go +++ /dev/null @@ -1,455 +0,0 @@ -// Copyright 2023 LiveKit, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package buffer - -import ( - "encoding/binary" - "errors" - - "github.com/pion/rtp/codecs" - - "github.com/livekit/protocol/logger" -) - -var ( - errShortPacket = errors.New("packet is not large enough") - errNilPacket = errors.New("invalid nil packet") - errInvalidPacket = errors.New("invalid packet") -) - -// VP8 is a helper to get temporal data from VP8 packet header -/* - VP8 Payload Descriptor - 0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7 - +-+-+-+-+-+-+-+-+ +-+-+-+-+-+-+-+-+ - |X|R|N|S|R| PID | (REQUIRED) |X|R|N|S|R| PID | (REQUIRED) - +-+-+-+-+-+-+-+-+ +-+-+-+-+-+-+-+-+ - X: |I|L|T|K| RSV | (OPTIONAL) X: |I|L|T|K| RSV | (OPTIONAL) - +-+-+-+-+-+-+-+-+ +-+-+-+-+-+-+-+-+ - I: |M| PictureID | (OPTIONAL) I: |M| PictureID | (OPTIONAL) - +-+-+-+-+-+-+-+-+ +-+-+-+-+-+-+-+-+ - L: | TL0PICIDX | (OPTIONAL) | PictureID | - +-+-+-+-+-+-+-+-+ +-+-+-+-+-+-+-+-+ - T/K:|TID|Y| KEYIDX | (OPTIONAL) L: | TL0PICIDX | (OPTIONAL) - +-+-+-+-+-+-+-+-+ +-+-+-+-+-+-+-+-+ - T/K:|TID|Y| KEYIDX | (OPTIONAL) - +-+-+-+-+-+-+-+-+ -*/ -type VP8 struct { - FirstByte byte - S bool - - I bool - M bool - PictureID uint16 /* 7 or 15 bits, picture ID */ - - L bool - TL0PICIDX uint8 /* 8 bits temporal level zero index */ - - // Optional Header If either of the T or K bits are set to 1, - // the TID/Y/KEYIDX extension field MUST be present. - T bool - TID uint8 /* 2 bits temporal layer idx */ - Y bool - - K bool - KEYIDX uint8 /* 5 bits of key frame idx */ - - HeaderSize int - - // IsKeyFrame is a helper to detect if current packet is a keyframe - IsKeyFrame bool -} - -// Unmarshal parses the passed byte slice and stores the result in the VP8 this method is called upon -func (v *VP8) Unmarshal(payload []byte) error { - if payload == nil { - return errNilPacket - } - - payloadLen := len(payload) - if payloadLen < 1 { - return errShortPacket - } - - idx := 0 - v.FirstByte = payload[idx] - v.S = payload[idx]&0x10 > 0 - // Check for extended bit control - if payload[idx]&0x80 > 0 { - idx++ - if payloadLen < idx+1 { - return errShortPacket - } - v.I = payload[idx]&0x80 > 0 - v.L = payload[idx]&0x40 > 0 - v.T = payload[idx]&0x20 > 0 - v.K = payload[idx]&0x10 > 0 - if v.L && !v.T { - return errInvalidPacket - } - - if v.I { - idx++ - if payloadLen < idx+1 { - return errShortPacket - } - pid := payload[idx] & 0x7f - // if m is 1, then Picture ID is 15 bits - v.M = payload[idx]&0x80 > 0 - if v.M { - idx++ - if payloadLen < idx+1 { - return errShortPacket - } - v.PictureID = binary.BigEndian.Uint16([]byte{pid, payload[idx]}) - } else { - v.PictureID = uint16(pid) - } - } - - if v.L { - idx++ - if payloadLen < idx+1 { - return errShortPacket - } - v.TL0PICIDX = payload[idx] - } - - if v.T || v.K { - idx++ - if payloadLen < idx+1 { - return errShortPacket - } - - if v.T { - v.TID = (payload[idx] & 0xc0) >> 6 - v.Y = (payload[idx] & 0x20) > 0 - } - - if v.K { - v.KEYIDX = payload[idx] & 0x1f - } - } - idx++ - if payloadLen < idx+1 { - return errShortPacket - } - - // Check is packet is a keyframe by looking at P bit in vp8 payload - v.IsKeyFrame = payload[idx]&0x01 == 0 && v.S - } else { - idx++ - if payloadLen < idx+1 { - return errShortPacket - } - // Check is packet is a keyframe by looking at P bit in vp8 payload - v.IsKeyFrame = payload[idx]&0x01 == 0 && v.S - } - v.HeaderSize = idx - return nil -} - -func (v *VP8) Marshal() ([]byte, error) { - buf := make([]byte, v.HeaderSize) - n, err := v.MarshalTo(buf) - if err != nil { - return nil, err - } - return buf[:n], err -} - -func (v *VP8) MarshalTo(buf []byte) (int, error) { - if len(buf) < v.HeaderSize { - return 0, errShortPacket - } - - idx := 0 - buf[idx] = v.FirstByte - if v.I || v.L || v.T || v.K { - buf[idx] |= 0x80 // X bit - idx++ - - xpos := idx - xval := byte(0) - - idx++ - if v.I { - xval |= (1 << 7) - if v.M { - buf[idx] = 0x80 | byte((v.PictureID>>8)&0x7f) - buf[idx+1] = byte(v.PictureID & 0xff) - idx += 2 - } else { - buf[idx] = byte(v.PictureID) - idx++ - } - } - - if v.L { - xval |= (1 << 6) - buf[idx] = v.TL0PICIDX - idx++ - } - - if v.T || v.K { - buf[idx] = 0 - if v.T { - xval |= (1 << 5) - buf[idx] = v.TID << 6 - if v.Y { - buf[idx] |= (1 << 5) - } - } - - if v.K { - xval |= (1 << 4) - buf[idx] |= v.KEYIDX & 0x1f - } - idx++ - } - - buf[xpos] = xval - } else { - buf[idx] &^= 0x80 // X bit - idx++ - } - - return idx, nil -} - -// ------------------------------------- - -func VPxPictureIdSizeDiff(mBit1 bool, mBit2 bool) int { - if mBit1 == mBit2 { - return 0 - } - - if mBit1 { - return 1 - } - - return -1 -} - -// ------------------------------------- - -// IsH264KeyFrame detects if h264 payload is a keyframe -// this code was taken from https://github.com/jech/galene/blob/codecs/rtpconn/rtpreader.go#L45 -// all credits belongs to Juliusz Chroboczek @jech and the awesome Galene SFU -func IsH264KeyFrame(payload []byte) bool { - if len(payload) < 1 { - return false - } - nalu := payload[0] & 0x1F - if nalu == 0 { - // reserved - return false - } else if nalu <= 23 { - // simple NALU - return nalu == 7 - } else if nalu == 24 || nalu == 25 || nalu == 26 || nalu == 27 { - // STAP-A, STAP-B, MTAP16 or MTAP24 - i := 1 - if nalu == 25 || nalu == 26 || nalu == 27 { - // skip DON - i += 2 - } - for i < len(payload) { - if i+2 > len(payload) { - return false - } - length := uint16(payload[i])<<8 | - uint16(payload[i+1]) - i += 2 - if i+int(length) > len(payload) { - return false - } - offset := 0 - if nalu == 26 { - offset = 3 - } else if nalu == 27 { - offset = 4 - } - if offset >= int(length) { - return false - } - n := payload[i+offset] & 0x1F - if n == 7 { - return true - } else if n >= 24 { - // is this legal? - logger.Debugw("Non-simple NALU within a STAP") - } - i += int(length) - } - if i == len(payload) { - return false - } - return false - } else if nalu == 28 || nalu == 29 { - // FU-A or FU-B - if len(payload) < 2 { - return false - } - if (payload[1] & 0x80) == 0 { - // not a starting fragment - return false - } - return payload[1]&0x1F == 7 - } - return false -} - -// ------------------------------------- - -// IsVP9KeyFrame detects if vp9 payload is a keyframe -// taken from https://github.com/jech/galene/blob/master/codecs/codecs.go -// all credits belongs to Juliusz Chroboczek @jech and the awesome Galene SFU -func IsVP9KeyFrame(payload []byte) bool { - var vp9 codecs.VP9Packet - _, err := vp9.Unmarshal(payload) - if err != nil || len(vp9.Payload) < 1 { - return false - } - if !vp9.B { - return false - } - - if (vp9.Payload[0] & 0xc0) != 0x80 { - return false - } - - profile := (vp9.Payload[0] >> 4) & 0x3 - if profile != 3 { - return (vp9.Payload[0] & 0xC) == 0 - } - return (vp9.Payload[0] & 0x6) == 0 -} - -// ------------------------------------- - -// IsAV1KeyFrame detects if av1 payload is a keyframe -// taken from https://github.com/jech/galene/blob/master/codecs/codecs.go -// all credits belongs to Juliusz Chroboczek @jech and the awesome Galene SFU -func IsAV1KeyFrame(payload []byte) bool { - if len(payload) < 2 { - return false - } - // Z=0, N=1 - if (payload[0] & 0x88) != 0x08 { - return false - } - w := (payload[0] & 0x30) >> 4 - - getObu := func(data []byte, last bool) ([]byte, int, bool) { - if last { - return data, len(data), false - } - offset := 0 - length := 0 - for { - if len(data) <= offset { - return nil, offset, offset > 0 - } - l := data[offset] - length |= int(l&0x7f) << (offset * 7) - offset++ - if (l & 0x80) == 0 { - break - } - } - if len(data) < offset+length { - return data[offset:], len(data), true - } - return data[offset : offset+length], - offset + length, false - } - offset := 1 - i := 0 - for { - obu, length, truncated := - getObu(payload[offset:], int(w) == i+1) - if len(obu) < 1 { - return false - } - tpe := (obu[0] & 0x38) >> 3 - switch i { - case 0: - // OBU_SEQUENCE_HEADER - if tpe != 1 { - return false - } - default: - // OBU_FRAME_HEADER or OBU_FRAME - if tpe == 3 || tpe == 6 { - if len(obu) < 2 { - return false - } - // show_existing_frame == 0 - if (obu[1] & 0x80) != 0 { - return false - } - // frame_type == KEY_FRAME - return (obu[1] & 0x60) == 0 - } - } - if truncated || i >= int(w) { - // the first frame header is in a second - // packet, give up. - return false - } - offset += length - i++ - } -} - -func IsH265KeyFrame(payload []byte) (kf bool) { - if len(payload) < 2 { - return false - } - naluType := (payload[0] & 0x7E) >> 1 - switch { - case naluType == 33 || naluType == 34: - return true - case naluType == 48: // AP - idx := 2 - for idx < len(payload)-2 { - // TODO: check the DONL field (controled by sprop-max-don-diff) - size := binary.BigEndian.Uint16(payload[idx:]) - idx += 2 - if idx >= len(payload) { - return false - } - naluType = (payload[idx] & 0x7E) >> 1 - if naluType == 33 || naluType == 34 { - return true - } - idx += int(size) - } - return false - - case naluType == 49: // FU - if len(payload) < 3 { - return false - } - naluType = (payload[2] & 0x7E) >> 1 - return naluType == 33 || naluType == 34 - default: - return false - } -} - -// ------------------------------------- diff --git a/pkg/sfu/buffer/helpers_test.go b/pkg/sfu/buffer/helpers_test.go deleted file mode 100644 index 378bfbe..0000000 --- a/pkg/sfu/buffer/helpers_test.go +++ /dev/null @@ -1,110 +0,0 @@ -// Copyright 2023 LiveKit, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package buffer - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -func TestVP8Helper_Unmarshal(t *testing.T) { - type args struct { - payload []byte - } - tests := []struct { - name string - args args - wantErr bool - checkTemporal bool - temporalSupport bool - checkKeyFrame bool - keyFrame bool - checkPictureID bool - pictureID uint16 - checkTlzIdx bool - tlzIdx uint8 - checkTempID bool - temporalID uint8 - }{ - { - name: "Empty or nil payload must return error", - args: args{payload: []byte{}}, - wantErr: true, - }, - { - name: "Temporal must be supported by setting T bit to 1", - args: args{payload: []byte{0xff, 0x20, 0x1, 0x2, 0x3, 0x4}}, - checkTemporal: true, - temporalSupport: true, - }, - { - name: "Picture must be ID 7 bits by setting M bit to 0 and present by I bit set to 1", - args: args{payload: []byte{0xff, 0xff, 0x11, 0x2, 0x3, 0x4}}, - checkPictureID: true, - pictureID: 17, - }, - { - name: "Picture ID must be 15 bits by setting M bit to 1 and present by I bit set to 1", - args: args{payload: []byte{0xff, 0xff, 0x92, 0x67, 0x3, 0x4, 0x5}}, - checkPictureID: true, - pictureID: 4711, - }, - { - name: "Temporal level zero index must be present if L set to 1", - args: args{payload: []byte{0xff, 0xff, 0xff, 0xfd, 0xb4, 0x4, 0x5}}, - checkTlzIdx: true, - tlzIdx: 180, - }, - { - name: "Temporal index must be present and used if T bit set to 1", - args: args{payload: []byte{0xff, 0xff, 0xff, 0xfd, 0xb4, 0x9f, 0x5, 0x6}}, - checkTempID: true, - temporalID: 2, - }, - { - name: "Check if packet is a keyframe by looking at P bit set to 0", - args: args{payload: []byte{0xff, 0xff, 0xff, 0xfd, 0xb4, 0x9f, 0x94, 0x1}}, - checkKeyFrame: true, - keyFrame: true, - }, - } - for _, tt := range tests { - tt := tt - t.Run(tt.name, func(t *testing.T) { - p := &VP8{} - if err := p.Unmarshal(tt.args.payload); (err != nil) != tt.wantErr { - t.Errorf("Unmarshal() error = %v, wantErr %v", err, tt.wantErr) - } - if tt.checkTemporal { - require.Equal(t, tt.temporalSupport, p.T) - } - if tt.checkKeyFrame { - require.Equal(t, tt.keyFrame, p.IsKeyFrame) - } - if tt.checkPictureID { - require.Equal(t, tt.pictureID, p.PictureID) - } - if tt.checkTlzIdx { - require.Equal(t, tt.tlzIdx, p.TL0PICIDX) - } - if tt.checkTempID { - require.Equal(t, tt.temporalID, p.TID) - } - }) - } -} - -// ------------------------------------------ diff --git a/pkg/sfu/buffer/rtcpreader.go b/pkg/sfu/buffer/rtcpreader.go index 1f8f365..32bffc7 100644 --- a/pkg/sfu/buffer/rtcpreader.go +++ b/pkg/sfu/buffer/rtcpreader.go @@ -31,6 +31,7 @@ func NewRTCPReader(ssrc uint32) *RTCPReader { return &RTCPReader{ssrc: ssrc} } +//go:noinline func (r *RTCPReader) Write(p []byte) (n int, err error) { if r.closed.Load() { err = io.EOF diff --git a/pkg/sfu/buffer/videolayerutils.go b/pkg/sfu/buffer/videolayerutils.go index f099c2e..d80c0a7 100644 --- a/pkg/sfu/buffer/videolayerutils.go +++ b/pkg/sfu/buffer/videolayerutils.go @@ -15,24 +15,43 @@ package buffer import ( + "slices" + + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" ) const ( - QuarterResolution = "q" - HalfResolution = "h" - FullResolution = "f" + quarterResolutionQ = "q" + halfResolutionH = "h" + fullResolutionF = "f" + + quarterResolution2 = "2" + halfResolution1 = "1" + fullResolution0 = "0" ) -// SIMULCAST-CODEC-TODO: these need to be codec mime aware if and when each codec suppports different layers -func LayerPresenceFromTrackInfo(trackInfo *livekit.TrackInfo) *[livekit.VideoQuality_HIGH + 1]bool { - if trackInfo == nil || len(trackInfo.Layers) == 0 { +type VideoLayersRid [DefaultMaxLayerSpatial + 1]string + +var ( + videoLayersRidQHF = VideoLayersRid{quarterResolutionQ, halfResolutionH, fullResolutionF} + videoLayersRid210 = VideoLayersRid{quarterResolution2, halfResolution1, fullResolution0} + DefaultVideoLayersRid = videoLayersRidQHF +) + +func LayerPresenceFromTrackInfo(mimeType mime.MimeType, trackInfo *livekit.TrackInfo) *[livekit.VideoQuality_HIGH + 1]bool { + if trackInfo == nil { + return nil + } + + layers := GetVideoLayersForMimeType(mimeType, trackInfo) + if len(layers) == 0 { return nil } var layerPresence [livekit.VideoQuality_HIGH + 1]bool - for _, layer := range trackInfo.Layers { + for _, layer := range layers { // WARNING: comparing protobuf enum if layer.Quality <= livekit.VideoQuality_HIGH { layerPresence[layer.Quality] = true @@ -44,15 +63,15 @@ func LayerPresenceFromTrackInfo(trackInfo *livekit.TrackInfo) *[livekit.VideoQua return &layerPresence } -func RidToSpatialLayer(rid string, trackInfo *livekit.TrackInfo) int32 { - lp := LayerPresenceFromTrackInfo(trackInfo) +func RidToSpatialLayer(mimeType mime.MimeType, rid string, trackInfo *livekit.TrackInfo, ridSpace VideoLayersRid) int32 { + lp := LayerPresenceFromTrackInfo(mimeType, trackInfo) if lp == nil { switch rid { - case QuarterResolution: + case quarterResolutionQ: return 0 - case HalfResolution: + case halfResolutionH: return 1 - case FullResolution: + case fullResolutionF: return 2 default: return 0 @@ -60,7 +79,7 @@ func RidToSpatialLayer(rid string, trackInfo *livekit.TrackInfo) int32 { } switch rid { - case QuarterResolution: + case ridSpace[0]: switch { case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]: fallthrough @@ -76,7 +95,7 @@ func RidToSpatialLayer(rid string, trackInfo *livekit.TrackInfo) int32 { return 0 } - case HalfResolution: + case ridSpace[1]: switch { case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]: fallthrough @@ -92,19 +111,19 @@ func RidToSpatialLayer(rid string, trackInfo *livekit.TrackInfo) int32 { return 0 } - case FullResolution: + case ridSpace[2]: switch { case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]: return 2 case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM]: - logger.Warnw("unexpected rid f with only two qualities, low and medium", nil, "trackID", trackInfo.Sid, "trackInfo", logger.Proto(trackInfo)) + logger.Warnw("unexpected rid with only two qualities, low and medium", nil, "trackID", trackInfo.Sid, "trackInfo", logger.Proto(trackInfo), "rid", ridSpace[2]) return 1 case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_HIGH]: - logger.Warnw("unexpected rid f with only two qualities, low and high", nil, "trackID", trackInfo.Sid, "trackInfo", logger.Proto(trackInfo)) + logger.Warnw("unexpected rid with only two qualities, low and high", nil, "trackID", trackInfo.Sid, "trackInfo", logger.Proto(trackInfo), "rid", ridSpace[2]) return 1 case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]: - logger.Warnw("unexpected rid f with only two qualities, medium and high", nil, "trackID", trackInfo.Sid, "trackInfo", logger.Proto(trackInfo)) + logger.Warnw("unexpected rid with only two qualities, medium and high", nil, "trackID", trackInfo.Sid, "trackInfo", logger.Proto(trackInfo), "rid", ridSpace[2]) return 1 default: @@ -118,18 +137,18 @@ func RidToSpatialLayer(rid string, trackInfo *livekit.TrackInfo) int32 { } } -func SpatialLayerToRid(layer int32, trackInfo *livekit.TrackInfo) string { - lp := LayerPresenceFromTrackInfo(trackInfo) +func SpatialLayerToRid(mimeType mime.MimeType, layer int32, trackInfo *livekit.TrackInfo, ridSpace VideoLayersRid) string { + lp := LayerPresenceFromTrackInfo(mimeType, trackInfo) if lp == nil { switch layer { case 0: - return QuarterResolution + return quarterResolutionQ case 1: - return HalfResolution + return halfResolutionH case 2: - return FullResolution + return fullResolutionF default: - return QuarterResolution + return quarterResolutionQ } } @@ -143,10 +162,10 @@ func SpatialLayerToRid(layer int32, trackInfo *livekit.TrackInfo) string { case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_HIGH]: fallthrough case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]: - return QuarterResolution + return ridSpace[0] default: - return QuarterResolution + return ridSpace[0] } case 1: @@ -158,42 +177,42 @@ func SpatialLayerToRid(layer int32, trackInfo *livekit.TrackInfo) string { case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_HIGH]: fallthrough case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]: - return HalfResolution + return ridSpace[1] default: - return QuarterResolution + return ridSpace[0] } case 2: switch { case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]: - return FullResolution + return ridSpace[2] case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM]: logger.Warnw("unexpected layer 2 with only two qualities, low and medium", nil, "trackID", trackInfo.Sid, "trackInfo", logger.Proto(trackInfo)) - return HalfResolution + return ridSpace[1] case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_HIGH]: logger.Warnw("unexpected layer 2 with only two qualities, low and high", nil, "trackID", trackInfo.Sid, "trackInfo", logger.Proto(trackInfo)) - return HalfResolution + return ridSpace[1] case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]: logger.Warnw("unexpected layer 2 with only two qualities, medium and high", nil, "trackID", trackInfo.Sid, "trackInfo", logger.Proto(trackInfo)) - return HalfResolution + return ridSpace[1] default: - return QuarterResolution + return ridSpace[0] } default: - return QuarterResolution + return ridSpace[0] } } -func VideoQualityToRid(quality livekit.VideoQuality, trackInfo *livekit.TrackInfo) string { - return SpatialLayerToRid(VideoQualityToSpatialLayer(quality, trackInfo), trackInfo) +func VideoQualityToRid(mimeType mime.MimeType, quality livekit.VideoQuality, trackInfo *livekit.TrackInfo, ridSpace VideoLayersRid) string { + return SpatialLayerToRid(mimeType, VideoQualityToSpatialLayer(mimeType, quality, trackInfo), trackInfo, ridSpace) } -func SpatialLayerToVideoQuality(layer int32, trackInfo *livekit.TrackInfo) livekit.VideoQuality { - lp := LayerPresenceFromTrackInfo(trackInfo) +func SpatialLayerToVideoQuality(mimeType mime.MimeType, layer int32, trackInfo *livekit.TrackInfo) livekit.VideoQuality { + lp := LayerPresenceFromTrackInfo(mimeType, trackInfo) if lp == nil { switch layer { case 0: @@ -259,8 +278,8 @@ func SpatialLayerToVideoQuality(layer int32, trackInfo *livekit.TrackInfo) livek return livekit.VideoQuality_OFF } -func VideoQualityToSpatialLayer(quality livekit.VideoQuality, trackInfo *livekit.TrackInfo) int32 { - lp := LayerPresenceFromTrackInfo(trackInfo) +func VideoQualityToSpatialLayer(mimeType mime.MimeType, quality livekit.VideoQuality, trackInfo *livekit.TrackInfo) int32 { + lp := LayerPresenceFromTrackInfo(mimeType, trackInfo) if lp == nil { switch quality { case livekit.VideoQuality_LOW: @@ -324,3 +343,178 @@ func VideoQualityToSpatialLayer(quality livekit.VideoQuality, trackInfo *livekit return InvalidLayerSpatial } + +func GetVideoLayerModeForMimeType(mimeType mime.MimeType, ti *livekit.TrackInfo) livekit.VideoLayer_Mode { + if ti != nil { + for _, codec := range ti.Codecs { + if mime.NormalizeMimeType(codec.MimeType) == mimeType { + return codec.VideoLayerMode + } + } + } + + return livekit.VideoLayer_MODE_UNUSED +} + +func GetVideoLayersForMimeType(mimeType mime.MimeType, ti *livekit.TrackInfo) []*livekit.VideoLayer { + var layers []*livekit.VideoLayer + if ti != nil { + for _, codec := range ti.Codecs { + if mime.NormalizeMimeType(codec.MimeType) == mimeType { + layers = codec.Layers + break + } + } + if len(layers) == 0 { + layers = ti.Layers + } + } + return layers +} + +func GetSpatialLayerForRid(mimeType mime.MimeType, rid string, ti *livekit.TrackInfo) int32 { + if ti == nil { + return InvalidLayerSpatial + } + + if rid == "" { + // single layer without RID + return 0 + } + + layers := GetVideoLayersForMimeType(mimeType, ti) + for _, layer := range layers { + if layer.Rid == rid { + return layer.SpatialLayer + } + } + + if len(layers) != 0 { + // RID present in codec, but may not be specified via signalling + // (happens with older browsers setting a rid for SVC codecs) + hasRid := false + for _, layer := range layers { + if layer.Rid != "" { + hasRid = true + break + } + } + if !hasRid { + return 0 + } + } + + // SIMULCAST-CODEC-TODO - ideally should return invalid, but there are + // VP9 publishers using rid = f, if there are only two layers + // in TrackInfo, that will be q;h and f will become invalid. + // + // Actually, there should be no rids for VP9 in SDP and hence + // the above check should take effect. However, as simulcast + // codec/back up codec does not update rids from SDP, + // the default rids are used when vp9 (primary codec) + // is published. Due to that the above check gets bypassed. + // + // The full proper sequence would be + // 1. For primary codec using SVC, there will be no rids. + // The above check should take effect and it should + // return 0 even if some publisher uses a rid like `f`. + // 2. When secondary codec is published, rids for the codec + // corresponding to the back up codec mime type should + // be updated in `TrackInfo`. This is a bit tricky + // for a couple of cases + // a. Browsers like Firefox use a different CID everytime. + // So, it cannot be matched between `AddTrack` and SDP. + // One option is to look for a published track with + // back up codec and apply it there. But, that becomes + // a challenge if there are multiple published tracks + // with pending back up codec. + // b. The back up codec publish SDP will have the full + // codec list. It should be okay to assume that the + // codec that will be published is the back up codec, + // but just something to be aware of. + // 3. Use of this function with proper mime so that proper + // codec section can be looked up in `TrackInfo`. + // return InvalidLayerSpatial + logger.Infow( + "invalid layer for rid, returning default", + "trackID", ti.Sid, + "rid", rid, + "mimeType", mimeType, + "trackInfo", logger.Proto(ti), + ) + return 0 +} + +func GetSpatialLayerForVideoQuality(mimeType mime.MimeType, quality livekit.VideoQuality, ti *livekit.TrackInfo) int32 { + if ti == nil || quality == livekit.VideoQuality_OFF { + return InvalidLayerSpatial + } + + layers := GetVideoLayersForMimeType(mimeType, ti) + for _, layer := range layers { + if layer.Quality == quality { + return layer.SpatialLayer + } + } + + if len(layers) == 0 { + // single layer + return 0 + } + + // requested quality is higher than available layers, return the highest available layer + return VideoQualityToSpatialLayer(mimeType, quality, ti) +} + +func GetVideoQualityForSpatialLayer(mimeType mime.MimeType, spatialLayer int32, ti *livekit.TrackInfo) livekit.VideoQuality { + if spatialLayer == InvalidLayerSpatial || ti == nil { + return livekit.VideoQuality_OFF + } + + layers := GetVideoLayersForMimeType(mimeType, ti) + for _, layer := range layers { + if layer.SpatialLayer == spatialLayer { + return layer.Quality + } + } + + return livekit.VideoQuality_OFF +} + +func isVideoLayersRidKnown(rids VideoLayersRid, knownRids VideoLayersRid) bool { + for _, rid := range rids { + if rid == "" { + continue + } + + if !slices.Contains(knownRids[:], rid) { + return false + } + } + + return true +} + +func NormalizeVideoLayersRid(rids VideoLayersRid) VideoLayersRid { + out := rids + + normalize := func(knownRids VideoLayersRid) { + idx := 0 + for _, known := range knownRids { + if slices.Contains(rids[:], known) { + out[idx] = known + idx++ + } + } + } + + if isVideoLayersRidKnown(rids, videoLayersRidQHF) { + normalize(videoLayersRidQHF) + } + + if isVideoLayersRidKnown(rids, videoLayersRid210) { + normalize(videoLayersRid210) + } + + return out +} diff --git a/pkg/sfu/buffer/videolayerutils_test.go b/pkg/sfu/buffer/videolayerutils_test.go index bb103f7..9382ec4 100644 --- a/pkg/sfu/buffer/videolayerutils_test.go +++ b/pkg/sfu/buffer/videolayerutils_test.go @@ -19,6 +19,7 @@ import ( "github.com/stretchr/testify/require" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" ) @@ -30,129 +31,174 @@ func TestRidConversion(t *testing.T) { tests := []struct { name string trackInfo *livekit.TrackInfo + mimeType mime.MimeType ridToLayer map[string]RidAndLayer }{ { "no track info", nil, + mime.MimeTypeVP8, map[string]RidAndLayer{ - "": {rid: QuarterResolution, layer: 0}, - QuarterResolution: {rid: QuarterResolution, layer: 0}, - HalfResolution: {rid: HalfResolution, layer: 1}, - FullResolution: {rid: FullResolution, layer: 2}, + "": {rid: quarterResolutionQ, layer: 0}, + quarterResolutionQ: {rid: quarterResolutionQ, layer: 0}, + halfResolutionH: {rid: halfResolutionH, layer: 1}, + fullResolutionF: {rid: fullResolutionF, layer: 2}, }, }, { "no layers", &livekit.TrackInfo{}, + mime.MimeTypeVP8, map[string]RidAndLayer{ - "": {rid: QuarterResolution, layer: 0}, - QuarterResolution: {rid: QuarterResolution, layer: 0}, - HalfResolution: {rid: HalfResolution, layer: 1}, - FullResolution: {rid: FullResolution, layer: 2}, + "": {rid: quarterResolutionQ, layer: 0}, + quarterResolutionQ: {rid: quarterResolutionQ, layer: 0}, + halfResolutionH: {rid: halfResolutionH, layer: 1}, + fullResolutionF: {rid: fullResolutionF, layer: 2}, }, }, { "single layer, low", &livekit.TrackInfo{ - Layers: []*livekit.VideoLayer{ - {Quality: livekit.VideoQuality_LOW}, + Codecs: []*livekit.SimulcastCodecInfo{ + { + MimeType: mime.MimeTypeVP8.String(), + Layers: []*livekit.VideoLayer{ + {Quality: livekit.VideoQuality_LOW}, + }, + }, }, }, + mime.MimeTypeVP8, map[string]RidAndLayer{ - "": {rid: QuarterResolution, layer: 0}, - QuarterResolution: {rid: QuarterResolution, layer: 0}, - HalfResolution: {rid: QuarterResolution, layer: 0}, - FullResolution: {rid: QuarterResolution, layer: 0}, + "": {rid: quarterResolutionQ, layer: 0}, + quarterResolutionQ: {rid: quarterResolutionQ, layer: 0}, + halfResolutionH: {rid: quarterResolutionQ, layer: 0}, + fullResolutionF: {rid: quarterResolutionQ, layer: 0}, }, }, { "single layer, medium", &livekit.TrackInfo{ - Layers: []*livekit.VideoLayer{ - {Quality: livekit.VideoQuality_MEDIUM}, + Codecs: []*livekit.SimulcastCodecInfo{ + { + MimeType: mime.MimeTypeVP8.String(), + Layers: []*livekit.VideoLayer{ + {Quality: livekit.VideoQuality_MEDIUM}, + }, + }, }, }, + mime.MimeTypeVP8, map[string]RidAndLayer{ - "": {rid: QuarterResolution, layer: 0}, - QuarterResolution: {rid: QuarterResolution, layer: 0}, - HalfResolution: {rid: QuarterResolution, layer: 0}, - FullResolution: {rid: QuarterResolution, layer: 0}, + "": {rid: quarterResolutionQ, layer: 0}, + quarterResolutionQ: {rid: quarterResolutionQ, layer: 0}, + halfResolutionH: {rid: quarterResolutionQ, layer: 0}, + fullResolutionF: {rid: quarterResolutionQ, layer: 0}, }, }, { "single layer, high", &livekit.TrackInfo{ - Layers: []*livekit.VideoLayer{ - {Quality: livekit.VideoQuality_HIGH}, + Codecs: []*livekit.SimulcastCodecInfo{ + { + MimeType: mime.MimeTypeVP8.String(), + Layers: []*livekit.VideoLayer{ + {Quality: livekit.VideoQuality_MEDIUM}, + }, + }, }, }, + mime.MimeTypeVP8, map[string]RidAndLayer{ - "": {rid: QuarterResolution, layer: 0}, - QuarterResolution: {rid: QuarterResolution, layer: 0}, - HalfResolution: {rid: QuarterResolution, layer: 0}, - FullResolution: {rid: QuarterResolution, layer: 0}, + "": {rid: quarterResolutionQ, layer: 0}, + quarterResolutionQ: {rid: quarterResolutionQ, layer: 0}, + halfResolutionH: {rid: quarterResolutionQ, layer: 0}, + fullResolutionF: {rid: quarterResolutionQ, layer: 0}, }, }, { "two layers, low and medium", &livekit.TrackInfo{ - Layers: []*livekit.VideoLayer{ - {Quality: livekit.VideoQuality_LOW}, - {Quality: livekit.VideoQuality_MEDIUM}, + Codecs: []*livekit.SimulcastCodecInfo{ + { + MimeType: mime.MimeTypeVP8.String(), + Layers: []*livekit.VideoLayer{ + {Quality: livekit.VideoQuality_LOW}, + {Quality: livekit.VideoQuality_MEDIUM}, + }, + }, }, }, + mime.MimeTypeVP8, map[string]RidAndLayer{ - "": {rid: QuarterResolution, layer: 0}, - QuarterResolution: {rid: QuarterResolution, layer: 0}, - HalfResolution: {rid: HalfResolution, layer: 1}, - FullResolution: {rid: HalfResolution, layer: 1}, + "": {rid: quarterResolutionQ, layer: 0}, + quarterResolutionQ: {rid: quarterResolutionQ, layer: 0}, + halfResolutionH: {rid: halfResolutionH, layer: 1}, + fullResolutionF: {rid: halfResolutionH, layer: 1}, }, }, { "two layers, low and high", &livekit.TrackInfo{ - Layers: []*livekit.VideoLayer{ - {Quality: livekit.VideoQuality_LOW}, - {Quality: livekit.VideoQuality_HIGH}, + Codecs: []*livekit.SimulcastCodecInfo{ + { + MimeType: mime.MimeTypeVP8.String(), + Layers: []*livekit.VideoLayer{ + {Quality: livekit.VideoQuality_LOW}, + {Quality: livekit.VideoQuality_HIGH}, + }, + }, }, }, + mime.MimeTypeVP8, map[string]RidAndLayer{ - "": {rid: QuarterResolution, layer: 0}, - QuarterResolution: {rid: QuarterResolution, layer: 0}, - HalfResolution: {rid: HalfResolution, layer: 1}, - FullResolution: {rid: HalfResolution, layer: 1}, + "": {rid: quarterResolutionQ, layer: 0}, + quarterResolutionQ: {rid: quarterResolutionQ, layer: 0}, + halfResolutionH: {rid: halfResolutionH, layer: 1}, + fullResolutionF: {rid: halfResolutionH, layer: 1}, }, }, { "two layers, medium and high", &livekit.TrackInfo{ - Layers: []*livekit.VideoLayer{ - {Quality: livekit.VideoQuality_MEDIUM}, - {Quality: livekit.VideoQuality_HIGH}, + Codecs: []*livekit.SimulcastCodecInfo{ + { + MimeType: mime.MimeTypeVP8.String(), + Layers: []*livekit.VideoLayer{ + {Quality: livekit.VideoQuality_MEDIUM}, + {Quality: livekit.VideoQuality_HIGH}, + }, + }, }, }, + mime.MimeTypeVP8, map[string]RidAndLayer{ - "": {rid: QuarterResolution, layer: 0}, - QuarterResolution: {rid: QuarterResolution, layer: 0}, - HalfResolution: {rid: HalfResolution, layer: 1}, - FullResolution: {rid: HalfResolution, layer: 1}, + "": {rid: quarterResolutionQ, layer: 0}, + quarterResolutionQ: {rid: quarterResolutionQ, layer: 0}, + halfResolutionH: {rid: halfResolutionH, layer: 1}, + fullResolutionF: {rid: halfResolutionH, layer: 1}, }, }, { "three layers", &livekit.TrackInfo{ - Layers: []*livekit.VideoLayer{ - {Quality: livekit.VideoQuality_LOW}, - {Quality: livekit.VideoQuality_MEDIUM}, - {Quality: livekit.VideoQuality_HIGH}, + Codecs: []*livekit.SimulcastCodecInfo{ + { + MimeType: mime.MimeTypeVP8.String(), + Layers: []*livekit.VideoLayer{ + {Quality: livekit.VideoQuality_LOW}, + {Quality: livekit.VideoQuality_MEDIUM}, + {Quality: livekit.VideoQuality_HIGH}, + }, + }, }, }, + mime.MimeTypeVP8, map[string]RidAndLayer{ - "": {rid: QuarterResolution, layer: 0}, - QuarterResolution: {rid: QuarterResolution, layer: 0}, - HalfResolution: {rid: HalfResolution, layer: 1}, - FullResolution: {rid: FullResolution, layer: 2}, + "": {rid: quarterResolutionQ, layer: 0}, + quarterResolutionQ: {rid: quarterResolutionQ, layer: 0}, + halfResolutionH: {rid: halfResolutionH, layer: 1}, + fullResolutionF: {rid: fullResolutionF, layer: 2}, }, }, } @@ -160,10 +206,10 @@ func TestRidConversion(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { for testRid, expectedResult := range test.ridToLayer { - actualLayer := RidToSpatialLayer(testRid, test.trackInfo) + actualLayer := RidToSpatialLayer(test.mimeType, testRid, test.trackInfo, DefaultVideoLayersRid) require.Equal(t, expectedResult.layer, actualLayer) - actualRid := SpatialLayerToRid(actualLayer, test.trackInfo) + actualRid := SpatialLayerToRid(test.mimeType, actualLayer, test.trackInfo, DefaultVideoLayersRid) require.Equal(t, expectedResult.rid, actualRid) } }) @@ -178,11 +224,13 @@ func TestQualityConversion(t *testing.T) { tests := []struct { name string trackInfo *livekit.TrackInfo + mimeType mime.MimeType qualityToLayer map[livekit.VideoQuality]QualityAndLayer }{ { "no track info", nil, + mime.MimeTypeVP8, map[livekit.VideoQuality]QualityAndLayer{ livekit.VideoQuality_LOW: {quality: livekit.VideoQuality_LOW, layer: 0}, livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_MEDIUM, layer: 1}, @@ -192,6 +240,7 @@ func TestQualityConversion(t *testing.T) { { "no layers", &livekit.TrackInfo{}, + mime.MimeTypeVP8, map[livekit.VideoQuality]QualityAndLayer{ livekit.VideoQuality_LOW: {quality: livekit.VideoQuality_LOW, layer: 0}, livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_MEDIUM, layer: 1}, @@ -201,10 +250,16 @@ func TestQualityConversion(t *testing.T) { { "single layer, low", &livekit.TrackInfo{ - Layers: []*livekit.VideoLayer{ - {Quality: livekit.VideoQuality_LOW}, + Codecs: []*livekit.SimulcastCodecInfo{ + { + MimeType: mime.MimeTypeVP8.String(), + Layers: []*livekit.VideoLayer{ + {Quality: livekit.VideoQuality_LOW}, + }, + }, }, }, + mime.MimeTypeVP8, map[livekit.VideoQuality]QualityAndLayer{ livekit.VideoQuality_LOW: {quality: livekit.VideoQuality_LOW, layer: 0}, livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_LOW, layer: 0}, @@ -214,10 +269,16 @@ func TestQualityConversion(t *testing.T) { { "single layer, medium", &livekit.TrackInfo{ - Layers: []*livekit.VideoLayer{ - {Quality: livekit.VideoQuality_MEDIUM}, + Codecs: []*livekit.SimulcastCodecInfo{ + { + MimeType: mime.MimeTypeVP8.String(), + Layers: []*livekit.VideoLayer{ + {Quality: livekit.VideoQuality_MEDIUM}, + }, + }, }, }, + mime.MimeTypeVP8, map[livekit.VideoQuality]QualityAndLayer{ livekit.VideoQuality_LOW: {quality: livekit.VideoQuality_MEDIUM, layer: 0}, livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_MEDIUM, layer: 0}, @@ -227,10 +288,16 @@ func TestQualityConversion(t *testing.T) { { "single layer, high", &livekit.TrackInfo{ - Layers: []*livekit.VideoLayer{ - {Quality: livekit.VideoQuality_HIGH}, + Codecs: []*livekit.SimulcastCodecInfo{ + { + MimeType: mime.MimeTypeVP8.String(), + Layers: []*livekit.VideoLayer{ + {Quality: livekit.VideoQuality_HIGH}, + }, + }, }, }, + mime.MimeTypeVP8, map[livekit.VideoQuality]QualityAndLayer{ livekit.VideoQuality_LOW: {quality: livekit.VideoQuality_HIGH, layer: 0}, livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_HIGH, layer: 0}, @@ -240,11 +307,17 @@ func TestQualityConversion(t *testing.T) { { "two layers, low and medium", &livekit.TrackInfo{ - Layers: []*livekit.VideoLayer{ - {Quality: livekit.VideoQuality_LOW}, - {Quality: livekit.VideoQuality_MEDIUM}, + Codecs: []*livekit.SimulcastCodecInfo{ + { + MimeType: mime.MimeTypeVP8.String(), + Layers: []*livekit.VideoLayer{ + {Quality: livekit.VideoQuality_LOW}, + {Quality: livekit.VideoQuality_MEDIUM}, + }, + }, }, }, + mime.MimeTypeVP8, map[livekit.VideoQuality]QualityAndLayer{ livekit.VideoQuality_LOW: {quality: livekit.VideoQuality_LOW, layer: 0}, livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_MEDIUM, layer: 1}, @@ -254,11 +327,17 @@ func TestQualityConversion(t *testing.T) { { "two layers, low and high", &livekit.TrackInfo{ - Layers: []*livekit.VideoLayer{ - {Quality: livekit.VideoQuality_LOW}, - {Quality: livekit.VideoQuality_HIGH}, + Codecs: []*livekit.SimulcastCodecInfo{ + { + MimeType: mime.MimeTypeVP8.String(), + Layers: []*livekit.VideoLayer{ + {Quality: livekit.VideoQuality_LOW}, + {Quality: livekit.VideoQuality_HIGH}, + }, + }, }, }, + mime.MimeTypeVP8, map[livekit.VideoQuality]QualityAndLayer{ livekit.VideoQuality_LOW: {quality: livekit.VideoQuality_LOW, layer: 0}, livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_HIGH, layer: 1}, @@ -268,11 +347,17 @@ func TestQualityConversion(t *testing.T) { { "two layers, medium and high", &livekit.TrackInfo{ - Layers: []*livekit.VideoLayer{ - {Quality: livekit.VideoQuality_MEDIUM}, - {Quality: livekit.VideoQuality_HIGH}, + Codecs: []*livekit.SimulcastCodecInfo{ + { + MimeType: mime.MimeTypeVP8.String(), + Layers: []*livekit.VideoLayer{ + {Quality: livekit.VideoQuality_MEDIUM}, + {Quality: livekit.VideoQuality_HIGH}, + }, + }, }, }, + mime.MimeTypeVP8, map[livekit.VideoQuality]QualityAndLayer{ livekit.VideoQuality_LOW: {quality: livekit.VideoQuality_MEDIUM, layer: 0}, livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_MEDIUM, layer: 0}, @@ -282,12 +367,18 @@ func TestQualityConversion(t *testing.T) { { "three layers", &livekit.TrackInfo{ - Layers: []*livekit.VideoLayer{ - {Quality: livekit.VideoQuality_LOW}, - {Quality: livekit.VideoQuality_MEDIUM}, - {Quality: livekit.VideoQuality_HIGH}, + Codecs: []*livekit.SimulcastCodecInfo{ + { + MimeType: mime.MimeTypeVP8.String(), + Layers: []*livekit.VideoLayer{ + {Quality: livekit.VideoQuality_LOW}, + {Quality: livekit.VideoQuality_MEDIUM}, + {Quality: livekit.VideoQuality_HIGH}, + }, + }, }, }, + mime.MimeTypeVP8, map[livekit.VideoQuality]QualityAndLayer{ livekit.VideoQuality_LOW: {quality: livekit.VideoQuality_LOW, layer: 0}, livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_MEDIUM, layer: 1}, @@ -299,10 +390,10 @@ func TestQualityConversion(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { for testQuality, expectedResult := range test.qualityToLayer { - actualLayer := VideoQualityToSpatialLayer(testQuality, test.trackInfo) + actualLayer := VideoQualityToSpatialLayer(test.mimeType, testQuality, test.trackInfo) require.Equal(t, expectedResult.layer, actualLayer) - actualQuality := SpatialLayerToVideoQuality(actualLayer, test.trackInfo) + actualQuality := SpatialLayerToVideoQuality(test.mimeType, actualLayer, test.trackInfo) require.Equal(t, expectedResult.quality, actualQuality) } }) @@ -313,120 +404,165 @@ func TestVideoQualityToRidConversion(t *testing.T) { tests := []struct { name string trackInfo *livekit.TrackInfo + mimeTye mime.MimeType qualityToRid map[livekit.VideoQuality]string }{ { "no track info", nil, + mime.MimeTypeVP8, map[livekit.VideoQuality]string{ - livekit.VideoQuality_LOW: QuarterResolution, - livekit.VideoQuality_MEDIUM: HalfResolution, - livekit.VideoQuality_HIGH: FullResolution, + livekit.VideoQuality_LOW: quarterResolutionQ, + livekit.VideoQuality_MEDIUM: halfResolutionH, + livekit.VideoQuality_HIGH: fullResolutionF, }, }, { "no layers", &livekit.TrackInfo{}, + mime.MimeTypeVP8, map[livekit.VideoQuality]string{ - livekit.VideoQuality_LOW: QuarterResolution, - livekit.VideoQuality_MEDIUM: HalfResolution, - livekit.VideoQuality_HIGH: FullResolution, + livekit.VideoQuality_LOW: quarterResolutionQ, + livekit.VideoQuality_MEDIUM: halfResolutionH, + livekit.VideoQuality_HIGH: fullResolutionF, }, }, { "single layer, low", &livekit.TrackInfo{ - Layers: []*livekit.VideoLayer{ - {Quality: livekit.VideoQuality_LOW}, + Codecs: []*livekit.SimulcastCodecInfo{ + { + MimeType: mime.MimeTypeVP8.String(), + Layers: []*livekit.VideoLayer{ + {Quality: livekit.VideoQuality_LOW}, + }, + }, }, }, + mime.MimeTypeVP8, map[livekit.VideoQuality]string{ - livekit.VideoQuality_LOW: QuarterResolution, - livekit.VideoQuality_MEDIUM: QuarterResolution, - livekit.VideoQuality_HIGH: QuarterResolution, + livekit.VideoQuality_LOW: quarterResolutionQ, + livekit.VideoQuality_MEDIUM: quarterResolutionQ, + livekit.VideoQuality_HIGH: quarterResolutionQ, }, }, { "single layer, medium", &livekit.TrackInfo{ - Layers: []*livekit.VideoLayer{ - {Quality: livekit.VideoQuality_MEDIUM}, + Codecs: []*livekit.SimulcastCodecInfo{ + { + MimeType: mime.MimeTypeVP8.String(), + Layers: []*livekit.VideoLayer{ + {Quality: livekit.VideoQuality_MEDIUM}, + }, + }, }, }, + mime.MimeTypeVP8, map[livekit.VideoQuality]string{ - livekit.VideoQuality_LOW: QuarterResolution, - livekit.VideoQuality_MEDIUM: QuarterResolution, - livekit.VideoQuality_HIGH: QuarterResolution, + livekit.VideoQuality_LOW: quarterResolutionQ, + livekit.VideoQuality_MEDIUM: quarterResolutionQ, + livekit.VideoQuality_HIGH: quarterResolutionQ, }, }, { "single layer, high", &livekit.TrackInfo{ - Layers: []*livekit.VideoLayer{ - {Quality: livekit.VideoQuality_HIGH}, + Codecs: []*livekit.SimulcastCodecInfo{ + { + MimeType: mime.MimeTypeVP8.String(), + Layers: []*livekit.VideoLayer{ + {Quality: livekit.VideoQuality_HIGH}, + }, + }, }, }, + mime.MimeTypeVP8, map[livekit.VideoQuality]string{ - livekit.VideoQuality_LOW: QuarterResolution, - livekit.VideoQuality_MEDIUM: QuarterResolution, - livekit.VideoQuality_HIGH: QuarterResolution, + livekit.VideoQuality_LOW: quarterResolutionQ, + livekit.VideoQuality_MEDIUM: quarterResolutionQ, + livekit.VideoQuality_HIGH: quarterResolutionQ, }, }, { "two layers, low and medium", &livekit.TrackInfo{ - Layers: []*livekit.VideoLayer{ - {Quality: livekit.VideoQuality_LOW}, - {Quality: livekit.VideoQuality_MEDIUM}, + Codecs: []*livekit.SimulcastCodecInfo{ + { + MimeType: mime.MimeTypeVP8.String(), + Layers: []*livekit.VideoLayer{ + {Quality: livekit.VideoQuality_LOW}, + {Quality: livekit.VideoQuality_MEDIUM}, + }, + }, }, }, + mime.MimeTypeVP8, map[livekit.VideoQuality]string{ - livekit.VideoQuality_LOW: QuarterResolution, - livekit.VideoQuality_MEDIUM: HalfResolution, - livekit.VideoQuality_HIGH: HalfResolution, + livekit.VideoQuality_LOW: quarterResolutionQ, + livekit.VideoQuality_MEDIUM: halfResolutionH, + livekit.VideoQuality_HIGH: halfResolutionH, }, }, { "two layers, low and high", &livekit.TrackInfo{ - Layers: []*livekit.VideoLayer{ - {Quality: livekit.VideoQuality_LOW}, - {Quality: livekit.VideoQuality_HIGH}, + Codecs: []*livekit.SimulcastCodecInfo{ + { + MimeType: mime.MimeTypeVP8.String(), + Layers: []*livekit.VideoLayer{ + {Quality: livekit.VideoQuality_LOW}, + {Quality: livekit.VideoQuality_HIGH}, + }, + }, }, }, + mime.MimeTypeVP8, map[livekit.VideoQuality]string{ - livekit.VideoQuality_LOW: QuarterResolution, - livekit.VideoQuality_MEDIUM: HalfResolution, - livekit.VideoQuality_HIGH: HalfResolution, + livekit.VideoQuality_LOW: quarterResolutionQ, + livekit.VideoQuality_MEDIUM: halfResolutionH, + livekit.VideoQuality_HIGH: halfResolutionH, }, }, { "two layers, medium and high", &livekit.TrackInfo{ - Layers: []*livekit.VideoLayer{ - {Quality: livekit.VideoQuality_MEDIUM}, - {Quality: livekit.VideoQuality_HIGH}, + Codecs: []*livekit.SimulcastCodecInfo{ + { + MimeType: mime.MimeTypeVP8.String(), + Layers: []*livekit.VideoLayer{ + {Quality: livekit.VideoQuality_MEDIUM}, + {Quality: livekit.VideoQuality_HIGH}, + }, + }, }, }, + mime.MimeTypeVP8, map[livekit.VideoQuality]string{ - livekit.VideoQuality_LOW: QuarterResolution, - livekit.VideoQuality_MEDIUM: QuarterResolution, - livekit.VideoQuality_HIGH: HalfResolution, + livekit.VideoQuality_LOW: quarterResolutionQ, + livekit.VideoQuality_MEDIUM: quarterResolutionQ, + livekit.VideoQuality_HIGH: halfResolutionH, }, }, { "three layers", &livekit.TrackInfo{ - Layers: []*livekit.VideoLayer{ - {Quality: livekit.VideoQuality_LOW}, - {Quality: livekit.VideoQuality_MEDIUM}, - {Quality: livekit.VideoQuality_HIGH}, + Codecs: []*livekit.SimulcastCodecInfo{ + { + MimeType: mime.MimeTypeVP8.String(), + Layers: []*livekit.VideoLayer{ + {Quality: livekit.VideoQuality_LOW}, + {Quality: livekit.VideoQuality_MEDIUM}, + {Quality: livekit.VideoQuality_HIGH}, + }, + }, }, }, + mime.MimeTypeVP8, map[livekit.VideoQuality]string{ - livekit.VideoQuality_LOW: QuarterResolution, - livekit.VideoQuality_MEDIUM: HalfResolution, - livekit.VideoQuality_HIGH: FullResolution, + livekit.VideoQuality_LOW: quarterResolutionQ, + livekit.VideoQuality_MEDIUM: halfResolutionH, + livekit.VideoQuality_HIGH: fullResolutionF, }, }, } @@ -434,9 +570,323 @@ func TestVideoQualityToRidConversion(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { for testQuality, expectedRid := range test.qualityToRid { - actualRid := VideoQualityToRid(testQuality, test.trackInfo) + actualRid := VideoQualityToRid(test.mimeTye, testQuality, test.trackInfo, DefaultVideoLayersRid) require.Equal(t, expectedRid, actualRid) } }) } } + +func TestGetSpatialLayerForRid(t *testing.T) { + tests := []struct { + name string + trackInfo *livekit.TrackInfo + mimeType mime.MimeType + ridToSpatialLayer map[string]int32 + }{ + { + "no track info", + nil, + mime.MimeTypeVP8, + map[string]int32{ + quarterResolutionQ: InvalidLayerSpatial, + halfResolutionH: InvalidLayerSpatial, + fullResolutionF: InvalidLayerSpatial, + }, + }, + { + "no layers", + &livekit.TrackInfo{}, + mime.MimeTypeVP8, + map[string]int32{ + // SIMULCAST-CODEC-TODO + // quarterResolutionQ: InvalidLayerSpatial, + // halfResolutionH: InvalidLayerSpatial, + // fullResolutionF: InvalidLayerSpatial, + quarterResolutionQ: 0, + halfResolutionH: 0, + fullResolutionF: 0, + }, + }, + { + "no rid", + &livekit.TrackInfo{}, + mime.MimeTypeVP8, + map[string]int32{ + "": 0, + }, + }, + { + "single layer", + &livekit.TrackInfo{ + Codecs: []*livekit.SimulcastCodecInfo{ + { + MimeType: mime.MimeTypeVP8.String(), + Layers: []*livekit.VideoLayer{ + {Quality: livekit.VideoQuality_LOW, SpatialLayer: 0}, + }, + }, + }, + }, + mime.MimeTypeVP8, + map[string]int32{ + quarterResolutionQ: 0, + halfResolutionH: 0, + fullResolutionF: 0, + }, + }, + { + "layers", + &livekit.TrackInfo{ + Codecs: []*livekit.SimulcastCodecInfo{ + { + MimeType: mime.MimeTypeVP8.String(), + Layers: []*livekit.VideoLayer{ + {Quality: livekit.VideoQuality_LOW, SpatialLayer: 0, Rid: quarterResolutionQ}, + {Quality: livekit.VideoQuality_MEDIUM, SpatialLayer: 1, Rid: halfResolutionH}, + }, + }, + }, + }, + mime.MimeTypeVP8, + map[string]int32{ + quarterResolutionQ: 0, + halfResolutionH: 1, + // SIMULCAST-CODEC-TODO + // fullResolutionF: InvalidLayerSpatial, + fullResolutionF: 0, + }, + }, + { + "layers - no rid", + &livekit.TrackInfo{ + Codecs: []*livekit.SimulcastCodecInfo{ + { + MimeType: mime.MimeTypeVP8.String(), + Layers: []*livekit.VideoLayer{ + {Quality: livekit.VideoQuality_LOW, SpatialLayer: 0}, + {Quality: livekit.VideoQuality_MEDIUM, SpatialLayer: 1}, + }, + }, + }, + }, + mime.MimeTypeVP8, + map[string]int32{ + quarterResolutionQ: 0, + halfResolutionH: 0, + fullResolutionF: 0, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + for testRid, expectedSpatialLayer := range test.ridToSpatialLayer { + actualSpatialLayer := GetSpatialLayerForRid(test.mimeType, testRid, test.trackInfo) + require.Equal(t, expectedSpatialLayer, actualSpatialLayer) + } + }) + } +} + +func TestGetSpatialLayerForVideoQuality(t *testing.T) { + tests := []struct { + name string + trackInfo *livekit.TrackInfo + mimeType mime.MimeType + videoQualityToSpatialLayer map[livekit.VideoQuality]int32 + }{ + { + "no track info", + nil, + mime.MimeTypeVP8, + map[livekit.VideoQuality]int32{ + livekit.VideoQuality_LOW: InvalidLayerSpatial, + livekit.VideoQuality_MEDIUM: InvalidLayerSpatial, + livekit.VideoQuality_HIGH: InvalidLayerSpatial, + livekit.VideoQuality_OFF: InvalidLayerSpatial, + }, + }, + { + "no layers", + &livekit.TrackInfo{}, + mime.MimeTypeVP8, + map[livekit.VideoQuality]int32{ + livekit.VideoQuality_LOW: 0, + livekit.VideoQuality_MEDIUM: 0, + livekit.VideoQuality_HIGH: 0, + livekit.VideoQuality_OFF: InvalidLayerSpatial, + }, + }, + { + "not all layers", + &livekit.TrackInfo{ + Codecs: []*livekit.SimulcastCodecInfo{ + { + MimeType: mime.MimeTypeVP8.String(), + Layers: []*livekit.VideoLayer{ + {Quality: livekit.VideoQuality_LOW, SpatialLayer: 0, Rid: quarterResolutionQ}, + {Quality: livekit.VideoQuality_MEDIUM, SpatialLayer: 1, Rid: halfResolutionH}, + }, + }, + }, + }, + mime.MimeTypeVP8, + map[livekit.VideoQuality]int32{ + livekit.VideoQuality_LOW: 0, + livekit.VideoQuality_MEDIUM: 1, + livekit.VideoQuality_HIGH: 1, + livekit.VideoQuality_OFF: InvalidLayerSpatial, + }, + }, + { + "all layers", + &livekit.TrackInfo{ + Codecs: []*livekit.SimulcastCodecInfo{ + { + MimeType: mime.MimeTypeVP8.String(), + Layers: []*livekit.VideoLayer{ + {Quality: livekit.VideoQuality_LOW, SpatialLayer: 0, Rid: quarterResolutionQ}, + {Quality: livekit.VideoQuality_MEDIUM, SpatialLayer: 1, Rid: halfResolutionH}, + {Quality: livekit.VideoQuality_HIGH, SpatialLayer: 2, Rid: fullResolutionF}, + }, + }, + }, + }, + mime.MimeTypeVP8, + map[livekit.VideoQuality]int32{ + livekit.VideoQuality_LOW: 0, + livekit.VideoQuality_MEDIUM: 1, + livekit.VideoQuality_HIGH: 2, + livekit.VideoQuality_OFF: InvalidLayerSpatial, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + for testVideoQuality, expectedSpatialLayer := range test.videoQualityToSpatialLayer { + actualSpatialLayer := GetSpatialLayerForVideoQuality(test.mimeType, testVideoQuality, test.trackInfo) + require.Equal(t, expectedSpatialLayer, actualSpatialLayer) + } + }) + } +} + +func TestGetVideoQualityorSpatialLayer(t *testing.T) { + tests := []struct { + name string + trackInfo *livekit.TrackInfo + mimeType mime.MimeType + spatialLayerToVideoQuality map[int32]livekit.VideoQuality + }{ + { + "no track info", + nil, + mime.MimeTypeVP8, + map[int32]livekit.VideoQuality{ + InvalidLayerSpatial: livekit.VideoQuality_OFF, + 0: livekit.VideoQuality_OFF, + 1: livekit.VideoQuality_OFF, + 2: livekit.VideoQuality_OFF, + }, + }, + { + "no layers", + &livekit.TrackInfo{}, + mime.MimeTypeVP8, + map[int32]livekit.VideoQuality{ + InvalidLayerSpatial: livekit.VideoQuality_OFF, + 0: livekit.VideoQuality_OFF, + 1: livekit.VideoQuality_OFF, + 2: livekit.VideoQuality_OFF, + }, + }, + { + "layers", + &livekit.TrackInfo{ + Codecs: []*livekit.SimulcastCodecInfo{ + { + MimeType: mime.MimeTypeVP8.String(), + Layers: []*livekit.VideoLayer{ + {Quality: livekit.VideoQuality_LOW, SpatialLayer: 0, Rid: quarterResolutionQ}, + {Quality: livekit.VideoQuality_MEDIUM, SpatialLayer: 1, Rid: halfResolutionH}, + }, + }, + }, + }, + mime.MimeTypeVP8, + map[int32]livekit.VideoQuality{ + InvalidLayerSpatial: livekit.VideoQuality_OFF, + 0: livekit.VideoQuality_LOW, + 1: livekit.VideoQuality_MEDIUM, + 2: livekit.VideoQuality_OFF, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + for testSpatialLayer, expectedVideoQuality := range test.spatialLayerToVideoQuality { + actualVideoQuality := GetVideoQualityForSpatialLayer(test.mimeType, testSpatialLayer, test.trackInfo) + require.Equal(t, expectedVideoQuality, actualVideoQuality) + } + }) + } +} + +func TestNormalizeVideoLayersRid(t *testing.T) { + tests := []struct { + name string + rids VideoLayersRid + normalized VideoLayersRid + }{ + { + "empty", + VideoLayersRid{}, + VideoLayersRid{}, + }, + { + "unknown pattern", + VideoLayersRid{"3", "2", "1"}, + VideoLayersRid{"3", "2", "1"}, + }, + { + "qhf", + videoLayersRidQHF, + videoLayersRidQHF, + }, + { + "scrambled qhf", + VideoLayersRid{"f", "h", "q"}, + videoLayersRidQHF, + }, + { + "partial qhf", + VideoLayersRid{"h", "q"}, + VideoLayersRid{"q", "h", ""}, + }, + { + "210", + videoLayersRid210, + videoLayersRid210, + }, + { + "scrambled 210", + VideoLayersRid{"2", "0", "1"}, + videoLayersRid210, + }, + { + "partial 210", + VideoLayersRid{"1", "2"}, + VideoLayersRid{"2", "1", ""}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + normalizedRids := NormalizeVideoLayersRid(test.rids) + require.Equal(t, test.normalized, normalizedRids) + }) + } +} diff --git a/pkg/sfu/bwe/bwe.go b/pkg/sfu/bwe/bwe.go index 5c806df..ad61a70 100644 --- a/pkg/sfu/bwe/bwe.go +++ b/pkg/sfu/bwe/bwe.go @@ -31,6 +31,29 @@ const ( // ------------------------------------------------ +type BWEType int + +const ( + BWETypeNone BWEType = iota + BWETypeRemote + BWETypeSendSide +) + +func (b BWEType) String() string { + switch b { + case BWETypeNone: + return "NONE" + case BWETypeRemote: + return "REMOTE" + case BWETypeSendSide: + return "SEND_SIDE" + default: + return fmt.Sprintf("%d", int(b)) + } +} + +// ------------------------------------------------ + type CongestionState int const ( @@ -55,6 +78,8 @@ func (c CongestionState) String() string { // ------------------------------------------------ type BWE interface { + Type() BWEType + SetBWEListener(bweListner BWEListener) Reset() diff --git a/pkg/sfu/bwe/remotebwe/remote_bwe.go b/pkg/sfu/bwe/remotebwe/remote_bwe.go index 30a9f70..f6eb4ba 100644 --- a/pkg/sfu/bwe/remotebwe/remote_bwe.go +++ b/pkg/sfu/bwe/remotebwe/remote_bwe.go @@ -24,6 +24,8 @@ import ( "github.com/livekit/protocol/utils/mono" ) +var _ bwe.BWE = (*RemoteBWE)(nil) + // --------------------------------------------------------------------------- type RemoteBWEConfig struct { @@ -81,6 +83,10 @@ func NewRemoteBWE(params RemoteBWEParams) *RemoteBWE { return r } +func (r *RemoteBWE) Type() bwe.BWEType { + return bwe.BWETypeRemote +} + func (r *RemoteBWE) SetBWEListener(bweListener bwe.BWEListener) { r.lock.Lock() defer r.lock.Unlock() diff --git a/pkg/sfu/bwe/sendsidebwe/congestion_detector.go b/pkg/sfu/bwe/sendsidebwe/congestion_detector.go index 20869df..720d4ad 100644 --- a/pkg/sfu/bwe/sendsidebwe/congestion_detector.go +++ b/pkg/sfu/bwe/sendsidebwe/congestion_detector.go @@ -121,8 +121,8 @@ var ( MinBytesRatio: 0.5, MinDurationRatio: 0.5, - JQRMinDelay: 40 * time.Millisecond, - DQRMaxDelay: 15 * time.Millisecond, + JQRMinDelay: 50 * time.Millisecond, + DQRMaxDelay: 20 * time.Millisecond, WeightedLoss: defaultWeightedLossConfig, JQRMinWeightedLoss: 0.25, @@ -177,17 +177,20 @@ func (c congestionReason) String() string { } // ------------------------------------------------------------------------------- -type qdMeasurement struct { - jqrConfig CongestionSignalConfig - dqrConfig CongestionSignalConfig - jqrMin int64 - dqrMax int64 - numGroups int - numJQRGroups int - numDQRGroups int - minSendTime int64 - maxSendTime int64 +type qdMeasurement struct { + jqrConfig CongestionSignalConfig + dqrConfig CongestionSignalConfig + jqrMinDelay int64 + jqrMinTrendCoefficient float64 + dqrMaxDelay int64 + + numGroups int + numJQRGroups int + numDQRGroups int + minSendTime int64 + maxSendTime int64 + propagatedQueuingDelays []int64 isSealed bool minGroupIdx int @@ -196,13 +199,14 @@ type qdMeasurement struct { queuingRegion queuingRegion } -func newQDMeasurement(jqrConfig CongestionSignalConfig, dqrConfig CongestionSignalConfig, jqrMin int64, dqrMax int64) *qdMeasurement { +func newQDMeasurement(jqrConfig CongestionSignalConfig, dqrConfig CongestionSignalConfig, jqrMinDelay int64, jqrMinTrendCoefficient float64, dqrMaxDelay int64) *qdMeasurement { return &qdMeasurement{ - jqrConfig: jqrConfig, - dqrConfig: dqrConfig, - jqrMin: jqrMin, - dqrMax: dqrMax, - queuingRegion: queuingRegionIndeterminate, + jqrConfig: jqrConfig, + dqrConfig: dqrConfig, + jqrMinDelay: jqrMinDelay, + jqrMinTrendCoefficient: jqrMinTrendCoefficient, + dqrMaxDelay: dqrMaxDelay, + queuingRegion: queuingRegionIndeterminate, } } @@ -228,8 +232,10 @@ func (q *qdMeasurement) ProcessPacketGroup(pg *packetGroup, groupIdx int) { } q.maxSendTime = max(q.maxSendTime, maxSendTime) + q.propagatedQueuingDelays = append(q.propagatedQueuingDelays, pqd) + switch { - case pqd < q.dqrMax: + case pqd < q.dqrMaxDelay: q.numDQRGroups++ if q.numJQRGroups > 0 { // broken continuity, seal @@ -239,12 +245,12 @@ func (q *qdMeasurement) ProcessPacketGroup(pg *packetGroup, groupIdx int) { q.queuingRegion = queuingRegionDQR } - case pqd > q.jqrMin: + case pqd > q.jqrMinDelay: q.numJQRGroups++ if q.numDQRGroups > 0 { // broken continuity, seal q.isSealed = true - } else if q.jqrConfig.IsTriggered(q.numJQRGroups, q.maxSendTime-q.minSendTime) { + } else if q.jqrConfig.IsTriggered(q.numJQRGroups, q.maxSendTime-q.minSendTime) && q.trendCoefficient() > q.jqrMinTrendCoefficient { q.isSealed = true q.queuingRegion = queuingRegionJQR } @@ -279,6 +285,8 @@ func (q *qdMeasurement) MarshalLogObject(e zapcore.ObjectEncoder) error { e.AddInt("numDQRGroups", q.numDQRGroups) e.AddInt64("minSendTime", q.minSendTime) e.AddInt64("maxSendTime", q.maxSendTime) + e.AddArray("propagatedQueuingDelays", logger.Int64Slice(q.propagatedQueuingDelays)) + e.AddFloat64("trendCoefficient", q.trendCoefficient()) e.AddDuration("duration", time.Duration((q.maxSendTime-q.minSendTime)*1000)) e.AddBool("isSealed", q.isSealed) e.AddInt("minGroupIdx", q.minGroupIdx) @@ -287,6 +295,35 @@ func (q *qdMeasurement) MarshalLogObject(e zapcore.ObjectEncoder) error { return nil } +func (q *qdMeasurement) trendCoefficient() float64 { + concordantPairs := 0 + discordantPairs := 0 + + // the packet groups are processed from newest to oldest, + // so a concordant pair is when the value drops, + // i. e. the propagated queuing delay is increasing if newer (earlier entity in slice) is higher than older (later entity in slice) + for i := 0; i < len(q.propagatedQueuingDelays)-1; i++ { + for j := i + 1; j < len(q.propagatedQueuingDelays); j++ { + if q.propagatedQueuingDelays[i] > q.propagatedQueuingDelays[j] { + concordantPairs++ + } else if q.propagatedQueuingDelays[i] < q.propagatedQueuingDelays[j] { + discordantPairs++ + } + } + } + + if (concordantPairs + discordantPairs) == 0 { + // if the min requirements is only one sample, trend calculation is not possible, declare highest trend value + if len(q.propagatedQueuingDelays) == 1 { + return 1.0 + } + + return 0.0 + } + + return (float64(concordantPairs) - float64(discordantPairs)) / (float64(concordantPairs) + float64(discordantPairs)) +} + // ------------------------------------------------------------------------------- type lossMeasurement struct { @@ -405,8 +442,9 @@ type CongestionDetectorConfig struct { ProbeRegulator ccutils.ProbeRegulatorConfig `yaml:"probe_regulator,omitempty"` ProbeSignal ProbeSignalConfig `yaml:"probe_signal,omitempty"` - JQRMinDelay time.Duration `yaml:"jqr_min_delay,omitempty"` - DQRMaxDelay time.Duration `yaml:"dqr_max_delay,omitempty"` + JQRMinDelay time.Duration `yaml:"jqr_min_delay,omitempty"` + JQRMinTrendCoefficient float64 `yaml:"jqr_min_trend_coefficient,omitempty"` + DQRMaxDelay time.Duration `yaml:"dqr_max_delay,omitempty"` WeightedLoss WeightedLossConfig `yaml:"weighted_loss,omitempty"` JQRMinWeightedLoss float64 `yaml:"jqr_min_weighted_loss,omitempty"` @@ -452,8 +490,9 @@ var ( ProbeRegulator: ccutils.DefaultProbeRegulatorConfig, ProbeSignal: defaultProbeSignalConfig, - JQRMinDelay: 40 * time.Millisecond, - DQRMaxDelay: 15 * time.Millisecond, + JQRMinDelay: 50 * time.Millisecond, + JQRMinTrendCoefficient: 0.8, + DQRMaxDelay: 20 * time.Millisecond, WeightedLoss: defaultWeightedLossConfig, JQRMinWeightedLoss: 0.25, @@ -570,10 +609,7 @@ func (c *congestionDetector) getBWEListener() bwe.BWEListener { func (c *congestionDetector) HandleTWCCFeedback(report *rtcp.TransportLayerCC) { c.lock.Lock() - recvRefTime, isOutOfOrder := c.twccFeedback.ProcessReport(report, mono.Now()) - if isOutOfOrder { - c.params.Logger.Infow("send side bwe: received out-of-order feedback report") - } + recvRefTime, _ := c.twccFeedback.ProcessReport(report, mono.Now()) if len(c.packetGroups) == 0 { c.packetGroups = append( @@ -635,7 +671,7 @@ func (c *congestionDetector) HandleTWCCFeedback(report *rtcp.TransportLayerCC) { } } - // 1. go through the TWCC feedback report and record recive time as reported by remote + // 1. go through the TWCC feedback report and record receive time as reported by remote // 2. process acknowledged packet and group them // // losses are not recorded if a feedback report is completely lost. @@ -852,6 +888,7 @@ func (c *congestionDetector) updateCongestionSignal( qdJQRConfig, qdDQRConfig, c.params.Config.JQRMinDelay.Microseconds(), + c.params.Config.JQRMinTrendCoefficient, c.params.Config.DQRMaxDelay.Microseconds(), ) c.lossMeasurement = newLossMeasurement( @@ -1109,7 +1146,7 @@ func (c *congestionDetector) updateCongestionState(state bwe.CongestionState) (b // when in congested state, monitor changes in captured traffic ratio (CTR) // to ensure allocations are in line with latest estimates, it is possible that // the estimate is incorrect when congestion starts and the allocation may be - // sub-optimal and not enough to reduce/relieve congestion, by monitoing CTR + // sub-optimal and not enough to reduce/relieve congestion, by monitoring CTR // on a continuous basis allocations can be adjusted in the direction of // reducing/relieving congestion if state == bwe.CongestionStateCongested && fromState != bwe.CongestionStateCongested { diff --git a/pkg/sfu/bwe/sendsidebwe/send_side_bwe.go b/pkg/sfu/bwe/sendsidebwe/send_side_bwe.go index 1e3cc74..4279046 100644 --- a/pkg/sfu/bwe/sendsidebwe/send_side_bwe.go +++ b/pkg/sfu/bwe/sendsidebwe/send_side_bwe.go @@ -23,6 +23,8 @@ import ( "github.com/pion/rtcp" ) +var _ bwe.BWE = (*SendSideBWE)(nil) + // // Based on a simplified/modified version of JitterPath paper // (https://homepage.iis.sinica.edu.tw/papers/lcs/2114-F.pdf) @@ -88,6 +90,10 @@ func NewSendSideBWE(params SendSideBWEParams) *SendSideBWE { } } +func (s *SendSideBWE) Type() bwe.BWEType { + return bwe.BWETypeSendSide +} + func (s *SendSideBWE) SetBWEListener(bweListener bwe.BWEListener) { s.congestionDetector.SetBWEListener(bweListener) } diff --git a/pkg/sfu/bwe/sendsidebwe/twcc_feedback.go b/pkg/sfu/bwe/sendsidebwe/twcc_feedback.go index f6e10e1..2b2c843 100644 --- a/pkg/sfu/bwe/sendsidebwe/twcc_feedback.go +++ b/pkg/sfu/bwe/sendsidebwe/twcc_feedback.go @@ -15,7 +15,6 @@ package sendsidebwe import ( - "errors" "time" "github.com/livekit/protocol/logger" @@ -35,12 +34,6 @@ const ( // ------------------------------------------------------ -var ( - errFeedbackReportOutOfOrder = errors.New("feedback report out-of-order") -) - -// ------------------------------------------------------ - type twccFeedbackParams struct { Logger logger.Logger } @@ -78,6 +71,12 @@ func (t *twccFeedback) ProcessReport(report *rtcp.TransportLayerCC, at time.Time if (report.FbPktCount - t.highestFeedbackCount) > (1 << 7) { t.numReportsOutOfOrder++ isOutOfOrder = true + t.params.Logger.Infow( + "send side bwe: received out-of-order feedback report", + "highestFeedbackCount", t.highestFeedbackCount, + "fbPktCount", report.FbPktCount, + "numReportsOutOfOrder", t.numReportsOutOfOrder, + ) } // reference time wrap around handling diff --git a/pkg/sfu/codecmunger/codecmunger.go b/pkg/sfu/codecmunger/codecmunger.go index 8f22538..413af96 100644 --- a/pkg/sfu/codecmunger/codecmunger.go +++ b/pkg/sfu/codecmunger/codecmunger.go @@ -27,8 +27,8 @@ var ( ) type CodecMunger interface { - GetState() interface{} - SeedState(state interface{}) + GetState() any + SeedState(state any) SetLast(extPkt *buffer.ExtPacket) UpdateOffsets(extPkt *buffer.ExtPacket) diff --git a/pkg/sfu/codecmunger/null.go b/pkg/sfu/codecmunger/null.go index 9d57570..616786a 100644 --- a/pkg/sfu/codecmunger/null.go +++ b/pkg/sfu/codecmunger/null.go @@ -20,22 +20,22 @@ import ( ) type Null struct { - seededState interface{} + seededState any } func NewNull(_logger logger.Logger) *Null { return &Null{} } -func (n *Null) GetState() interface{} { +func (n *Null) GetState() any { return nil } -func (n *Null) SeedState(state interface{}) { +func (n *Null) SeedState(state any) { n.seededState = state } -func (n *Null) GetSeededState() interface{} { +func (n *Null) GetSeededState() any { return n.seededState } diff --git a/pkg/sfu/codecmunger/vp8.go b/pkg/sfu/codecmunger/vp8.go index 696c355..f6f45e1 100644 --- a/pkg/sfu/codecmunger/vp8.go +++ b/pkg/sfu/codecmunger/vp8.go @@ -15,8 +15,9 @@ package codecmunger import ( - "github.com/elliotchance/orderedmap/v2" + "github.com/elliotchance/orderedmap/v3" + "github.com/livekit/mediatransportutil/pkg/codec" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" @@ -60,13 +61,18 @@ func NewVP8(logger logger.Logger) *VP8 { } } -func NewVP8FromNull(cm CodecMunger, logger logger.Logger) *VP8 { +func NewVP8FromOther(cm CodecMunger, logger logger.Logger) *VP8 { v := NewVP8(logger) - v.SeedState(cm.(*Null).GetSeededState()) + switch cm := cm.(type) { + case *Null: + v.SeedState(cm.GetSeededState()) + case *VP8: + v.SeedState(cm.GetState()) + } return v } -func (v *VP8) GetState() interface{} { +func (v *VP8) GetState() any { return &livekit.VP8MungerState{ ExtLastPictureId: v.extLastPictureId, PictureIdUsed: v.pictureIdUsed, @@ -78,10 +84,15 @@ func (v *VP8) GetState() interface{} { } } -func (v *VP8) SeedState(seed interface{}) { +func (v *VP8) SeedState(seed any) { + var state *livekit.VP8MungerState switch cm := seed.(type) { case *livekit.RTPForwarderState_Vp8Munger: - state := cm.Vp8Munger + state = cm.Vp8Munger + case *livekit.VP8MungerState: + state = cm + } + if state != nil { v.extLastPictureId = state.ExtLastPictureId v.pictureIdUsed = state.PictureIdUsed v.lastTl0PicIdx = uint8(state.LastTl0PicIdx) @@ -93,7 +104,7 @@ func (v *VP8) SeedState(seed interface{}) { } func (v *VP8) SetLast(extPkt *buffer.ExtPacket) { - vp8, ok := extPkt.Payload.(buffer.VP8) + vp8, ok := extPkt.Payload.(codec.VP8) if !ok { return } @@ -118,7 +129,7 @@ func (v *VP8) SetLast(extPkt *buffer.ExtPacket) { } func (v *VP8) UpdateOffsets(extPkt *buffer.ExtPacket) { - vp8, ok := extPkt.Payload.(buffer.VP8) + vp8, ok := extPkt.Payload.(codec.VP8) if !ok { return } @@ -143,7 +154,7 @@ func (v *VP8) UpdateOffsets(extPkt *buffer.ExtPacket) { } func (v *VP8) UpdateAndGet(extPkt *buffer.ExtPacket, snOutOfOrder bool, snHasGap bool, maxTemporalLayer int32) (int, []byte, error) { - vp8, ok := extPkt.Payload.(buffer.VP8) + vp8, ok := extPkt.Payload.(codec.VP8) if !ok { return 0, nil, ErrNotVP8 } @@ -164,7 +175,7 @@ func (v *VP8) UpdateAndGet(extPkt *buffer.ExtPacket, snOutOfOrder bool, snHasGap // when it reaches a certain size. mungedPictureId := uint16((extPictureId - pictureIdOffset) & 0x7fff) - vp8Packet := &buffer.VP8{ + vp8Packet := codec.VP8{ FirstByte: vp8.FirstByte, I: vp8.I, M: mungedPictureId > 127, @@ -177,7 +188,7 @@ func (v *VP8) UpdateAndGet(extPkt *buffer.ExtPacket, snOutOfOrder bool, snHasGap K: vp8.K, KEYIDX: vp8.KEYIDX - v.keyIdxOffset, IsKeyFrame: vp8.IsKeyFrame, - HeaderSize: vp8.HeaderSize + buffer.VPxPictureIdSizeDiff(mungedPictureId > 127, vp8.M), + HeaderSize: vp8.HeaderSize + codec.VPxPictureIdSizeDiff(mungedPictureId > 127, vp8.M), } vp8HeaderBytes, err := vp8Packet.Marshal() if err != nil { @@ -271,7 +282,7 @@ func (v *VP8) UpdateAndGet(extPkt *buffer.ExtPacket, snOutOfOrder bool, snHasGap v.lastTl0PicIdx = mungedTl0PicIdx v.lastKeyIdx = mungedKeyIdx - vp8Packet := &buffer.VP8{ + vp8Packet := codec.VP8{ FirstByte: vp8.FirstByte, I: vp8.I, M: mungedPictureId > 127, @@ -284,7 +295,7 @@ func (v *VP8) UpdateAndGet(extPkt *buffer.ExtPacket, snOutOfOrder bool, snHasGap K: vp8.K, KEYIDX: mungedKeyIdx, IsKeyFrame: vp8.IsKeyFrame, - HeaderSize: vp8.HeaderSize + buffer.VPxPictureIdSizeDiff(mungedPictureId > 127, vp8.M), + HeaderSize: vp8.HeaderSize + codec.VPxPictureIdSizeDiff(mungedPictureId > 127, vp8.M), } vp8HeaderBytes, err := vp8Packet.Marshal() if err != nil { @@ -336,7 +347,7 @@ func (v *VP8) UpdateAndGetPadding(newPicture bool) ([]byte, error) { v.keyIdxOffset -= uint8(offset) } - vp8Packet := &buffer.VP8{ + vp8Packet := &codec.VP8{ FirstByte: 0x10, // partition 0, start of VP8 Partition, reference frame I: v.pictureIdUsed, M: pictureId > 127, diff --git a/pkg/sfu/codecmunger/vp8_test.go b/pkg/sfu/codecmunger/vp8_test.go index 31d33dd..2c5714b 100644 --- a/pkg/sfu/codecmunger/vp8_test.go +++ b/pkg/sfu/codecmunger/vp8_test.go @@ -20,9 +20,9 @@ import ( "github.com/stretchr/testify/require" + "github.com/livekit/mediatransportutil/pkg/codec" "github.com/livekit/protocol/logger" - "github.com/livekit/livekit-server/pkg/sfu/buffer" "github.com/livekit/livekit-server/pkg/sfu/testutils" ) @@ -52,7 +52,7 @@ func TestSetLast(t *testing.T) { Timestamp: 0xabcdef, SSRC: 0x12345678, } - vp8 := &buffer.VP8{ + vp8 := &codec.VP8{ FirstByte: 25, I: true, M: true, @@ -102,7 +102,7 @@ func TestUpdateOffsets(t *testing.T) { Timestamp: 0xabcdef, SSRC: 0x12345678, } - vp8 := &buffer.VP8{ + vp8 := &codec.VP8{ FirstByte: 25, I: true, M: true, @@ -125,7 +125,7 @@ func TestUpdateOffsets(t *testing.T) { Timestamp: 0xabcdef, SSRC: 0x87654321, } - vp8 = &buffer.VP8{ + vp8 = &codec.VP8{ FirstByte: 25, I: true, M: true, @@ -172,7 +172,7 @@ func TestOutOfOrderPictureId(t *testing.T) { Timestamp: 0xabcdef, SSRC: 0x12345678, } - vp8 := &buffer.VP8{ + vp8 := &codec.VP8{ FirstByte: 25, I: true, M: true, @@ -205,7 +205,7 @@ func TestOutOfOrderPictureId(t *testing.T) { vp8.PictureID = 13469 extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8) - expectedVP8 := &buffer.VP8{ + expectedVP8 := &codec.VP8{ FirstByte: 25, I: true, M: true, @@ -244,7 +244,7 @@ func TestOutOfOrderPictureId(t *testing.T) { vp8.PictureID = 13468 extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8) - expectedVP8 = &buffer.VP8{ + expectedVP8 = &codec.VP8{ FirstByte: 25, I: true, M: true, @@ -275,7 +275,7 @@ func TestTemporalLayerFiltering(t *testing.T) { Timestamp: 0xabcdef, SSRC: 0x12345678, } - vp8 := &buffer.VP8{ + vp8 := &codec.VP8{ FirstByte: 25, I: true, M: true, @@ -341,7 +341,7 @@ func TestGapInSequenceNumberSamePicture(t *testing.T) { SSRC: 0x12345678, PayloadSize: 33, } - vp8 := &buffer.VP8{ + vp8 := &codec.VP8{ FirstByte: 25, I: true, M: true, @@ -359,7 +359,7 @@ func TestGapInSequenceNumberSamePicture(t *testing.T) { extPkt, _ := testutils.GetTestExtPacketVP8(params, vp8) v.SetLast(extPkt) - expectedVP8 := &buffer.VP8{ + expectedVP8 := &codec.VP8{ FirstByte: 25, I: true, M: true, @@ -382,7 +382,7 @@ func TestGapInSequenceNumberSamePicture(t *testing.T) { require.Equal(t, marshalledVP8, buf) // telling there is a gap in sequence number will add pictures to missing picture cache - expectedVP8 = &buffer.VP8{ + expectedVP8 = &codec.VP8{ FirstByte: 25, I: true, M: true, @@ -418,7 +418,7 @@ func TestUpdateAndGetPadding(t *testing.T) { SSRC: 0x12345678, PayloadSize: 20, } - vp8 := &buffer.VP8{ + vp8 := &codec.VP8{ FirstByte: 25, I: true, M: true, @@ -440,7 +440,7 @@ func TestUpdateAndGetPadding(t *testing.T) { // getting padding with repeat of last picture buf, err := v.UpdateAndGetPadding(false) require.NoError(t, err) - expectedVP8 := buffer.VP8{ + expectedVP8 := codec.VP8{ FirstByte: 16, I: true, M: true, @@ -462,7 +462,7 @@ func TestUpdateAndGetPadding(t *testing.T) { // getting padding with new picture buf, err = v.UpdateAndGetPadding(true) require.NoError(t, err) - expectedVP8 = buffer.VP8{ + expectedVP8 = codec.VP8{ FirstByte: 16, I: true, M: true, diff --git a/pkg/sfu/connectionquality/connectionstats.go b/pkg/sfu/connectionquality/connectionstats.go index 8015abe..f0c71bd 100644 --- a/pkg/sfu/connectionquality/connectionstats.go +++ b/pkg/sfu/connectionquality/connectionstats.go @@ -22,11 +22,11 @@ import ( "go.uber.org/atomic" "google.golang.org/protobuf/types/known/timestamppb" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" "github.com/livekit/livekit-server/pkg/sfu/buffer" - "github.com/livekit/livekit-server/pkg/sfu/mime" "github.com/livekit/livekit-server/pkg/sfu/rtpstats" ) @@ -276,29 +276,30 @@ func (cs *ConnectionStats) updateScoreFromReceiverReport(at time.Time) (float32, return cs.updateScoreWithAggregate(agg, time.Time{}, at), streams } -func (cs *ConnectionStats) updateScoreAt(at time.Time) (float32, map[uint32]*buffer.StreamStatsWithLayers) { +func (cs *ConnectionStats) updateScoreAt(at time.Time) (float32, map[uint32]*buffer.StreamStatsWithLayers, bool) { if cs.params.SenderProvider != nil { // receiver report based quality scoring, use stats from receiver report for scoring - return cs.updateScoreFromReceiverReport(at) + score, streams := cs.updateScoreFromReceiverReport(at) + return score, streams, true } if cs.params.ReceiverProvider == nil { - return MinMOS, nil + return MinMOS, nil, false } streams := cs.params.ReceiverProvider.GetDeltaStats() if len(streams) == 0 { mos, _ := cs.scorer.GetMOSAndQuality() - return mos, nil + return mos, nil, false } agg := toAggregateDeltaInfo(streams, false) if agg == nil { // no receiver report in the window mos, _ := cs.scorer.GetMOSAndQuality() - return mos, streams + return mos, streams, false } - return cs.updateScoreWithAggregate(agg, cs.params.ReceiverProvider.GetLastSenderReportTime(), at), streams + return cs.updateScoreWithAggregate(agg, cs.params.ReceiverProvider.GetLastSenderReportTime(), at), streams, false } func (cs *ConnectionStats) updateStreamingStart(at time.Time) time.Time { @@ -324,12 +325,12 @@ func (cs *ConnectionStats) updateStreamingStart(at time.Time) time.Time { } func (cs *ConnectionStats) getStat() { - score, streams := cs.updateScoreAt(time.Time{}) + score, streams, isSender := cs.updateScoreAt(time.Time{}) if cs.onStatsUpdate != nil && len(streams) != 0 { analyticsStreams := make([]*livekit.AnalyticsStream, 0, len(streams)) for ssrc, stream := range streams { - as := toAnalyticsStream(ssrc, stream.RTPStats, stream.RTPStatsRemoteView) + as := toAnalyticsStream(ssrc, stream.RTPStats, stream.RTPStatsRemoteView, isSender) if as == nil { continue } @@ -373,6 +374,7 @@ func (cs *ConnectionStats) updateStatsWorker() { for { select { case <-cs.done.Watch(): + cs.getStat() return case <-tk.C: @@ -429,6 +431,13 @@ func toAggregateDeltaInfo(streams map[uint32]*buffer.StreamStatsWithLayers, useR for _, s := range streams { if useRemoteView { if s.RTPStatsRemoteView != nil { + // discount jitter from publisher side + internal processing while reporting downstream jitter + if s.RTPStats != nil { + s.RTPStatsRemoteView.JitterMax -= s.RTPStats.JitterMax + if s.RTPStatsRemoteView.JitterMax < 0.0 { + s.RTPStatsRemoteView.JitterMax = 0.0 + } + } deltaInfoList = append(deltaInfoList, s.RTPStatsRemoteView) } } else { @@ -444,13 +453,17 @@ func toAnalyticsStream( ssrc uint32, deltaStats *rtpstats.RTPDeltaInfo, deltaStatsRemoteView *rtpstats.RTPDeltaInfo, + isSender bool, ) *livekit.AnalyticsStream { if deltaStats == nil { return nil } // discount the feed side loss when reporting forwarded track stats, + // discount jitter from publisher side + internal processing while reporting downstream jitter packetsLost := deltaStats.PacketsLost + rtt := deltaStats.RttMax + maxJitter := deltaStats.JitterMax if deltaStatsRemoteView != nil { packetsLost = deltaStatsRemoteView.PacketsLost if deltaStatsRemoteView.PacketsMissing > packetsLost { @@ -458,7 +471,14 @@ func toAnalyticsStream( } else { packetsLost -= deltaStatsRemoteView.PacketsMissing } + + rtt = deltaStatsRemoteView.RttMax + maxJitter = deltaStatsRemoteView.JitterMax + } else if isSender { + rtt = 0 + maxJitter = 0 } + return &livekit.AnalyticsStream{ StartTime: timestamppb.New(deltaStats.StartTime), EndTime: timestamppb.New(deltaStats.EndTime), @@ -472,8 +492,8 @@ func toAnalyticsStream( PacketsLost: packetsLost, PacketsOutOfOrder: deltaStats.PacketsOutOfOrder, Frames: deltaStats.Frames, - Rtt: deltaStats.RttMax, - Jitter: uint32(deltaStats.JitterMax), + Rtt: rtt, + Jitter: uint32(maxJitter), Nacks: deltaStats.Nacks, Plis: deltaStats.Plis, Firs: deltaStats.Firs, diff --git a/pkg/sfu/connectionquality/connectionstats_test.go b/pkg/sfu/connectionquality/connectionstats_test.go index 1e74eb4..13b61a6 100644 --- a/pkg/sfu/connectionquality/connectionstats_test.go +++ b/pkg/sfu/connectionquality/connectionstats_test.go @@ -22,8 +22,8 @@ import ( "github.com/stretchr/testify/require" "github.com/livekit/livekit-server/pkg/sfu/buffer" - "github.com/livekit/livekit-server/pkg/sfu/mime" "github.com/livekit/livekit-server/pkg/sfu/rtpstats" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" ) @@ -319,7 +319,7 @@ func TestConnectionQuality(t *testing.T) { // unmute and send packets to bring quality back up now = now.Add(duration) cs.UpdateMuteAt(false, now.Add(2*time.Second)) - for i := 0; i < 3; i++ { + for range 3 { trp.setStreams(map[uint32]*buffer.StreamStatsWithLayers{ 1: { RTPStats: &rtpstats.RTPDeltaInfo{ diff --git a/pkg/sfu/connectionquality/scorer.go b/pkg/sfu/connectionquality/scorer.go index af92900..b415f75 100644 --- a/pkg/sfu/connectionquality/scorer.go +++ b/pkg/sfu/connectionquality/scorer.go @@ -482,7 +482,7 @@ func (q *qualityScorer) updateAtLocked(stat *windowStat, at time.Time) { ulgr.Debugw("quality rise") default: packets := stat.packets + stat.packetsPadding - if packets != 0 && (stat.packetsLost*100/packets) > 10 { + if packets != 0 && ((stat.packetsLost-stat.packetsMissing-stat.packetsOutOfOrder)*100/packets) > 10 { ulgr.Debugw("quality hold - high loss") } } @@ -525,10 +525,7 @@ func (q *qualityScorer) isUnmutedEnough(at time.Time) bool { sinceLayerUnmute = at.Sub(q.layerUnmutedAt) } - validDuration := sinceUnmute - if sinceLayerUnmute < validDuration { - validDuration = sinceLayerUnmute - } + validDuration := min(sinceLayerUnmute, sinceUnmute) sinceLastUpdate := at.Sub(q.lastUpdateAt) @@ -567,7 +564,7 @@ func (q *qualityScorer) getAdjustedPacketLossWeight(stat *windowStat) float64 { // 2. enough time has elapsed since last calculation if q.numPPSReadings > cPPSMinReadings && time.Since(q.modeCalculatedAt) > cModeCalculationInterval { q.ppsMode = 0 - for i := 0; i < len(q.ppsHistogram); i++ { + for i := range len(q.ppsHistogram) { if q.ppsHistogram[i] > q.ppsMode { q.ppsMode = i } diff --git a/pkg/sfu/datachannel/datachannel_writer.go b/pkg/sfu/datachannel/datachannel_writer.go index 707c920..8fbbb3c 100644 --- a/pkg/sfu/datachannel/datachannel_writer.go +++ b/pkg/sfu/datachannel/datachannel_writer.go @@ -15,24 +15,30 @@ const ( singleWriteTimeout = 50 * time.Millisecond ) -var ErrDataDroppedBySlowReader = errors.New("data dropped by slow reader") +var ( + ErrDataDroppedBySlowReader = errors.New("data dropped by slow reader") + ErrDataDroppedByHighBufferedAmount = errors.New("data dropped due to high buffered amount") +) type BufferedAmountGetter interface { BufferedAmount() uint64 } type DataChannelWriter[T BufferedAmountGetter] struct { - bufferGetter T - rawDC datachannel.ReadWriteCloserDeadliner - slowThreshold int - rate *BitrateCalculator + bufferGetter T + rawDC datachannel.ReadWriteCloserDeadliner + slowThreshold int + rate *BitrateCalculator + reliable bool + targetLatency time.Duration + minBufferedAmount uint64 } -// NewDataChannelWriter creates a new DataChannelWriter by detaching the data channel, when -// writing to the datachanel times out, it will block and retry if the receiver's bitrate is -// above the slowThreshold or drop the data if it's below the threshold. If the slowThreshold -// is 0, it will never retry on write timeout. -func NewDataChannelWriter[T BufferedAmountGetter](bufferGetter T, rawDC datachannel.ReadWriteCloserDeadliner, slowThreshold int) *DataChannelWriter[T] { +// NewDataChannelWriterReliable creates a new DataChannelWriter for reliable data channel by +// detaching it, when writing to the datachanel times out, it will block and retry if the +// receiver's bitrate is above the slowThreshold or drop the data if it's below the threshold. +// If the slowThreshold is 0, it will never retry on write timeout. +func NewDataChannelWriterReliable[T BufferedAmountGetter](bufferGetter T, rawDC datachannel.ReadWriteCloserDeadliner, slowThreshold int) *DataChannelWriter[T] { var rate *BitrateCalculator if slowThreshold > 0 { rate = NewBitrateCalculator(BitrateDuration, BitrateWindow) @@ -42,6 +48,26 @@ func NewDataChannelWriter[T BufferedAmountGetter](bufferGetter T, rawDC datachan rawDC: rawDC, slowThreshold: slowThreshold, rate: rate, + reliable: true, + } +} + +// NewDataChannelWriterUnreliable creates a new DataChannelWriter for unreliable data channel. +// It will drop data when the buffered amount is too high to maintain the target latency. +// The latency is estimated based on the bitrate in past 1 second. If targetLatency is 0, no +// buffering control is applied. +func NewDataChannelWriterUnreliable[T BufferedAmountGetter](bufferGetter T, rawDC datachannel.ReadWriteCloserDeadliner, targetLatency time.Duration, minBufferedAmount uint64) *DataChannelWriter[T] { + var rate *BitrateCalculator + if targetLatency > 0 { + rate = NewBitrateCalculator(BitrateDuration, BitrateWindow) + } + return &DataChannelWriter[T]{ + bufferGetter: bufferGetter, + rawDC: rawDC, + rate: rate, + targetLatency: targetLatency, + minBufferedAmount: minBufferedAmount, + reliable: false, } } @@ -50,6 +76,14 @@ func (w *DataChannelWriter[T]) BufferedAmountGetter() T { } func (w *DataChannelWriter[T]) Write(p []byte) (n int, err error) { + if w.reliable { + return w.writeReliable(p) + } else { + return w.writeUnreliable(p) + } +} + +func (w *DataChannelWriter[T]) writeReliable(p []byte) (n int, err error) { for { err = w.rawDC.SetWriteDeadline(time.Now().Add(singleWriteTimeout)) if err != nil { @@ -75,6 +109,34 @@ func (w *DataChannelWriter[T]) Write(p []byte) (n int, err error) { } } +func (w *DataChannelWriter[T]) writeUnreliable(p []byte) (n int, err error) { + if w.targetLatency == 0 { + err = w.rawDC.SetWriteDeadline(time.Now().Add(singleWriteTimeout)) + if err != nil { + return 0, err + } + return w.rawDC.Write(p) + } + + if bitrate, ok := w.rate.Bitrate(time.Now()); ok { + // control buffer latency to ~100ms + if w.bufferGetter.BufferedAmount() > uint64(time.Duration(bitrate)*w.targetLatency/8/time.Second) && w.bufferGetter.BufferedAmount() > w.minBufferedAmount { + return 0, ErrDataDroppedByHighBufferedAmount + } + } + + err = w.rawDC.SetWriteDeadline(time.Now().Add(singleWriteTimeout)) + if err != nil { + return 0, err + } + n, err = w.rawDC.Write(p) + if err != nil { + w.rate.AddBytes(n, int(w.bufferGetter.BufferedAmount()), mono.Now()) + } + + return +} + func (w *DataChannelWriter[T]) Close() error { return w.rawDC.Close() } diff --git a/pkg/sfu/datachannel/datachannel_writer_test.go b/pkg/sfu/datachannel/datachannel_writer_test.go index 45459ce..9b99b89 100644 --- a/pkg/sfu/datachannel/datachannel_writer_test.go +++ b/pkg/sfu/datachannel/datachannel_writer_test.go @@ -2,18 +2,19 @@ package datachannel import ( "context" + "sync/atomic" "testing" "time" "github.com/pion/datachannel" - "github.com/pion/transport/v3/deadline" + "github.com/pion/transport/v4/deadline" "github.com/stretchr/testify/require" ) func TestDataChannelWriter(t *testing.T) { mockDC := newMockDataChannelWriter() // slow threshold is 1000B/s - w := NewDataChannelWriter(mockDC, mockDC, 8000) + w := NewDataChannelWriterReliable(mockDC, mockDC, 8000) require.Equal(t, mockDC, w.BufferedAmountGetter()) buf := make([]byte, 2000) // write 2000 bytes so it should not drop in 2 seconds @@ -23,7 +24,7 @@ func TestDataChannelWriter(t *testing.T) { require.Equal(t, 2000, n) t1 := time.Now() - mockDC.SetNextWriteCompleteAt(t0.Add(time.Second)) + mockDC.SetNextWriteCompleteAt(t0.Add(time.Second * 3 / 2)) n, err = w.Write(buf[:10]) require.NoError(t, err) require.Equal(t, 10, n) @@ -38,7 +39,7 @@ func TestDataChannelWriter(t *testing.T) { func TestDataChannelWriter_NoSlowThreshold(t *testing.T) { mockDC := newMockDataChannelWriter() - w := NewDataChannelWriter(mockDC, mockDC, 0) + w := NewDataChannelWriterReliable(mockDC, mockDC, 0) buf := make([]byte, 2000) n, err := w.Write(buf) require.NoError(t, err) @@ -55,6 +56,24 @@ func TestDataChannelWriter_NoSlowThreshold(t *testing.T) { require.Equal(t, 0, n) } +func TestDataChannelWriter_Unreliable(t *testing.T) { + mockDC := newMockLossyDataChannelWriter(8192) + w := NewDataChannelWriterUnreliable(mockDC, mockDC, 100*time.Millisecond, 2000) + for range 10 { + buf := make([]byte, 128) + _, err := w.Write(buf) + require.NoError(t, err) + time.Sleep(100 * time.Millisecond) + } + buf := make([]byte, 4096) + _, err := w.Write(buf) + require.NoError(t, err) + // should drop due to high buffered amount + _, err = w.Write(buf) + require.ErrorIs(t, err, ErrDataDroppedByHighBufferedAmount) +} + +// mockDataChannelWriter type mockDataChannelWriter struct { datachannel.ReadWriteCloserDeadliner nextWriteCompleteAt time.Time @@ -92,3 +111,44 @@ func (m *mockDataChannelWriter) SetWriteDeadline(t time.Time) error { func (m *mockDataChannelWriter) SetNextWriteCompleteAt(t time.Time) { m.nextWriteCompleteAt = t } + +// mockLossyDataChannelWriter +type mockLossyDataChannelWriter struct { + datachannel.ReadWriteCloserDeadliner + bufferedAmount atomic.Int64 + targetBitrate int + lastWriteAt time.Time +} + +func newMockLossyDataChannelWriter(targetBitrate int) *mockLossyDataChannelWriter { + return &mockLossyDataChannelWriter{ + targetBitrate: targetBitrate, + lastWriteAt: time.Now(), + } +} + +func (m *mockLossyDataChannelWriter) BufferedAmount() uint64 { + return uint64(m.bufferedAmount.Load()) +} + +func (m *mockLossyDataChannelWriter) Write(b []byte) (int, error) { + m.bufferedAmount.Add(int64(len(b))) + if time.Now().Before(m.lastWriteAt) { + return len(b), nil + } + + // drain buffer based on target bitrate + canWriteBytes := time.Since(m.lastWriteAt) * time.Duration(m.targetBitrate) / time.Second / 8 + if m.bufferedAmount.Load() <= int64(canWriteBytes) { + m.lastWriteAt = m.lastWriteAt.Add(time.Duration(int64(time.Second) * int64(m.BufferedAmount()) / (int64(m.targetBitrate) / 8))) + m.bufferedAmount.Store(0) + } else { + m.lastWriteAt = time.Now() + m.bufferedAmount.Add(-int64(canWriteBytes)) + } + return len(b), nil +} + +func (m *mockLossyDataChannelWriter) SetWriteDeadline(t time.Time) error { + return nil +} diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go index ba59770..38cf315 100644 --- a/pkg/sfu/downtrack.go +++ b/pkg/sfu/downtrack.go @@ -28,20 +28,22 @@ import ( "github.com/pion/rtcp" "github.com/pion/rtp" "github.com/pion/sdp/v3" - "github.com/pion/transport/v3/packetio" + "github.com/pion/transport/v4/packetio" "github.com/pion/webrtc/v4" "go.uber.org/atomic" "go.uber.org/zap/zapcore" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" "github.com/livekit/protocol/utils/mono" "github.com/livekit/livekit-server/pkg/sfu/buffer" + "github.com/livekit/livekit-server/pkg/sfu/bwe" "github.com/livekit/livekit-server/pkg/sfu/ccutils" "github.com/livekit/livekit-server/pkg/sfu/connectionquality" - "github.com/livekit/livekit-server/pkg/sfu/mime" "github.com/livekit/livekit-server/pkg/sfu/pacer" + "github.com/livekit/livekit-server/pkg/sfu/packettrailer" act "github.com/livekit/livekit-server/pkg/sfu/rtpextension/abscapturetime" dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor" pd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/playoutdelay" @@ -56,7 +58,7 @@ type TrackSender interface { UpTrackMaxPublishedLayerChange(maxPublishedLayer int32) UpTrackMaxTemporalLayerSeenChange(maxTemporalLayerSeen int32) UpTrackBitrateReport(availableLayers []int32, bitrates Bitrates) - WriteRTP(p *buffer.ExtPacket, layer int32) error + WriteRTP(p *buffer.ExtPacket, layer int32) int32 Close() IsClosed() bool // ID is the globally unique identifier for this Track. @@ -64,12 +66,12 @@ type TrackSender interface { SubscriberID() livekit.ParticipantID HandleRTCPSenderReportData( payloadType webrtc.PayloadType, - isSVC bool, layer int32, publisherSRData *livekit.RTCPSenderReportState, ) error Resync() SetReceiver(TrackReceiver) + ReceiverRestart(TrackReceiver) } // ------------------------------------------------------------------- @@ -88,18 +90,19 @@ const ( waitBeforeSendPaddingOnMute = 100 * time.Millisecond maxPaddingOnMuteDuration = 5 * time.Second + paddingOnMuteInterval = 100 * time.Millisecond ) // ------------------------------------------------------------------- var ( - ErrUnknownKind = errors.New("unknown kind of codec") - ErrOutOfOrderSequenceNumberCacheMiss = errors.New("out-of-order sequence number not found in cache") - ErrPaddingOnlyPacket = errors.New("padding only packet that need not be forwarded") - ErrDuplicatePacket = errors.New("duplicate packet") - ErrPaddingNotOnFrameBoundary = errors.New("padding cannot send on non-frame boundary") - ErrDownTrackAlreadyBound = errors.New("already bound") - ErrPayloadOverflow = errors.New("payload overflow") + errUnknownKind = errors.New("unknown kind of codec") + errOutOfOrderSequenceNumberCacheMiss = errors.New("out-of-order sequence number not found in cache") + errPaddingOnlyPacket = errors.New("padding only packet that need not be forwarded") + errDuplicatePacket = errors.New("duplicate packet") + errPaddingNotOnFrameBoundary = errors.New("padding cannot send on non-frame boundary") + errDownTrackAlreadyBound = errors.New("already bound") + errPayloadOverflow = errors.New("payload overflow") ) var ( @@ -137,6 +140,56 @@ var ( 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, } + // PCMU (G.711 µ-law) silence frame - 0xff represents zero amplitude + // 160 samples = 20ms at 8kHz sample rate + PCMUSilenceFrame = []byte{ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + } + + // PCMA (G.711 A-law) silence frame - 0xd5 represents zero amplitude + // 160 samples = 20ms at 8kHz sample rate + PCMASilenceFrame = []byte{ + 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, + 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, + 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, + 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, + 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, + 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, + 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, + 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, + 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, + 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, + 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, + 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, + 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, + 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, + 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, + 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, + 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, + 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, + 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, + 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, 0xd5, + } + dummyAbsSendTimeExt, _ = rtp.NewAbsSendTimeExtension(mono.Now()).Marshal() dummyTransportCCExt, _ = rtp.TransportCCExtension{TransportSequence: 12345}.Marshal() ) @@ -193,12 +246,27 @@ type DownTrackStreamAllocatorListener interface { // check if track should participate in BWE IsBWEEnabled(dt *DownTrack) bool + // get the BWE type in use + BWEType() bwe.BWEType + // check if subscription mute can be applied IsSubscribeMutable(dt *DownTrack) bool } // ------------------------------------------------------------------- +type DownTrackListener interface { + OnBindAndConnected() + OnStatsUpdate(stat *livekit.AnalyticsStat) + OnMaxSubscribedLayerChanged(layer int32) + OnRttUpdate(rtt uint32) + OnCodecNegotiated(webrtc.RTPCodecCapability) + OnDownTrackClose(isExpectedToResume bool) + OnStreamStarted() +} + +// ------------------------------------------------------------------- + type bindState int const ( @@ -223,10 +291,13 @@ func (bs bindState) String() string { // ------------------------------------------------------------------- +var _ TrackSender = (*DownTrack)(nil) + type ReceiverReportListener func(dt *DownTrack, report *rtcp.ReceiverReport) -type DowntrackParams struct { +type DownTrackParams struct { Codecs []webrtc.RTPCodecParameters + IsEncrypted bool Source livekit.TrackSource Receiver TrackReceiver BufferFactory *buffer.Factory @@ -240,9 +311,11 @@ type DowntrackParams struct { RTCPWriter func([]rtcp.Packet) error DisableSenderReportPassThrough bool SupportsCodecChange bool + StripPacketTrailer bool + Listener DownTrackListener } -// DownTrack implements TrackLocal, is the track used to write packets +// DownTrack implements webrtc.TrackLocal, is the track used to write packets // to SFU Subscriber, the track handle the packets for simple, simulcast // and SVC Publisher. // A DownTrack has the following lifecycle @@ -251,7 +324,7 @@ type DowntrackParams struct { // - closed // once closed, a DownTrack cannot be re-used. type DownTrack struct { - params DowntrackParams + params DownTrackParams id livekit.TrackID kind webrtc.RTPCodecType ssrc uint32 @@ -267,7 +340,7 @@ type DownTrack struct { forwarder *Forwarder upstreamCodecs []webrtc.RTPCodecParameters - codec webrtc.RTPCodecCapability + codec atomic.Value // webrtc.RTPCodecCapability clockRate uint32 negotiatedCodecParameters []webrtc.RTPCodecParameters @@ -312,6 +385,7 @@ type DownTrack struct { blankFramesGeneration atomic.Uint32 connectionStats *connectionquality.ConnectionStats + onStatsUpdate atomic.Value // func(d *DownTrack, stat *livekit.AnalyticsStat) isNACKThrottled atomic.Bool @@ -333,20 +407,12 @@ type DownTrack struct { keyFrameRequesterCh chan struct{} keyFrameRequesterChClosed bool - cbMu sync.RWMutex - onStatsUpdate func(dt *DownTrack, stat *livekit.AnalyticsStat) - onMaxSubscribedLayerChanged func(dt *DownTrack, layer int32) - onRttUpdate func(dt *DownTrack, rtt uint32) - onCloseHandler func(isExpectedToResume bool) - onCodecNegotiated func(webrtc.RTPCodecCapability) - createdAt int64 } // NewDownTrack returns a DownTrack. -func NewDownTrack(params DowntrackParams) (*DownTrack, error) { - codecs := params.Codecs - mimeType := mime.NormalizeMimeType(codecs[0].MimeType) +func NewDownTrack(params DownTrackParams) (*DownTrack, error) { + mimeType := mime.NormalizeMimeType(params.Codecs[0].MimeType) var kind webrtc.RTPCodecType switch { case mime.IsMimeTypeAudio(mimeType): @@ -357,19 +423,20 @@ func NewDownTrack(params DowntrackParams) (*DownTrack, error) { kind = webrtc.RTPCodecType(0) } + codec := params.Codecs[0].RTPCodecCapability d := &DownTrack{ params: params, id: params.Receiver.TrackID(), - upstreamCodecs: codecs, + upstreamCodecs: params.Codecs, kind: kind, - codec: codecs[0].RTPCodecCapability, - clockRate: codecs[0].ClockRate, + clockRate: codec.ClockRate, pacer: params.Pacer, maxLayerNotifierCh: make(chan string, 1), keyFrameRequesterCh: make(chan struct{}, 1), createdAt: time.Now().UnixNano(), receiver: params.Receiver, } + d.codec.Store(codec) d.bindState.Store(bindStateUnbound) d.params.Logger = params.Logger.WithValues( "subscriberID", d.SubscriberID(), @@ -381,27 +448,21 @@ func NewDownTrack(params DowntrackParams) (*DownTrack, error) { } else { mdCacheSize, mdCacheSizeRTX = 8192, 1024 } - d.rtpStats = rtpstats.NewRTPStatsSender(rtpstats.RTPStatsParams{ - ClockRate: d.codec.ClockRate, - Logger: d.params.Logger.WithValues( - "stream", "primary", - ), - }, mdCacheSize) + d.rtpStats = rtpstats.NewRTPStatsSender(rtpstats.RTPStatsParams{}, mdCacheSize) + // clock rate will be set on bind or codec change with matching codec's clock rate + d.rtpStats.SetLogger(d.params.Logger.WithValues("stream", "primary")) d.deltaStatsSenderSnapshotId = d.rtpStats.NewSenderSnapshotId() - d.rtpStatsRTX = rtpstats.NewRTPStatsSender(rtpstats.RTPStatsParams{ - ClockRate: d.codec.ClockRate, - IsRTX: true, - Logger: d.params.Logger.WithValues( - "stream", "rtx", - ), - }, mdCacheSizeRTX) + d.rtpStatsRTX = rtpstats.NewRTPStatsSender(rtpstats.RTPStatsParams{IsRTX: true}, mdCacheSizeRTX) + // clock rate will be set on bind or codec change with matching codec's clock rate + d.rtpStatsRTX.SetLogger(d.params.Logger.WithValues("stream", "rtx")) d.deltaStatsRTXSenderSnapshotId = d.rtpStatsRTX.NewSenderSnapshotId() d.forwarder = NewForwarder( d.kind, d.params.Logger, - false, + false, // skipReferenceTS + false, // disableOpportunisticAllocation d.rtpStats, ) @@ -410,8 +471,9 @@ func NewDownTrack(params DowntrackParams) (*DownTrack, error) { Logger: d.params.Logger.WithValues("direction", "down"), }) d.connectionStats.OnStatsUpdate(func(_cs *connectionquality.ConnectionStats, stat *livekit.AnalyticsStat) { - if onStatsUpdate := d.getOnStatsUpdate(); onStatsUpdate != nil { - onStatsUpdate(d, stat) + d.params.Listener.OnStatsUpdate(stat) + if fn, ok := d.onStatsUpdate.Load().(func(*DownTrack, *livekit.AnalyticsStat)); ok && fn != nil { + fn(d, stat) } }) @@ -434,12 +496,6 @@ func NewDownTrack(params DowntrackParams) (*DownTrack, error) { return d, nil } -func (d *DownTrack) OnCodecNegotiated(f func(webrtc.RTPCodecCapability)) { - d.bindLock.Lock() - d.onCodecNegotiated = f - d.bindLock.Unlock() -} - // Bind is called by the PeerConnection after negotiation is complete // This asserts that the code requested is supported by the remote peer. // If so it sets up all the state (SSRC and PayloadType) to have a call @@ -447,10 +503,11 @@ func (d *DownTrack) Bind(t webrtc.TrackLocalContext) (webrtc.RTPCodecParameters, d.bindLock.Lock() if d.bindState.Load() != bindStateUnbound { d.bindLock.Unlock() - return webrtc.RTPCodecParameters{}, ErrDownTrackAlreadyBound + return webrtc.RTPCodecParameters{}, errDownTrackAlreadyBound } - // the context's codec parameters will be set to the binded codec after Bind return so we keep - // a copy of the codec parameters here to use it later + + // the TrackLocalContext's codec parameters will be set to the bound codec after Bind returns, + // so keep a copy of the codec parameters here to use it later d.negotiatedCodecParameters = append([]webrtc.RTPCodecParameters{}, t.CodecParameters()...) var codec, matchedUpstreamCodec webrtc.RTPCodecParameters for _, c := range d.upstreamCodecs { @@ -459,6 +516,21 @@ func (d *DownTrack) Bind(t webrtc.TrackLocalContext) (webrtc.RTPCodecParameters, codec = matchCodec matchedUpstreamCodec = c break + } else { + // for encrypyted tracks, should match on primary codec, + // i. e. codec at index 0 if the combination of upstream codecs is opus and RED + if d.params.IsEncrypted { + isRedAndOpus := true + for _, u := range d.upstreamCodecs { + if !mime.IsMimeTypeStringOpus(u.MimeType) || !mime.IsMimeTypeStringRED(u.MimeType) { + isRedAndOpus = false + break + } + } + if isRedAndOpus { + break + } + } } } @@ -466,7 +538,11 @@ func (d *DownTrack) Bind(t webrtc.TrackLocalContext) (webrtc.RTPCodecParameters, err := webrtc.ErrUnsupportedCodec onBinding := d.onBinding d.bindLock.Unlock() - d.params.Logger.Infow("bind error for unsupported codec", "codecs", d.upstreamCodecs, "remoteParameters", d.negotiatedCodecParameters) + d.params.Logger.Infow( + "bind error for unsupported codec", + "codecs", d.upstreamCodecs, + "remoteParameters", d.negotiatedCodecParameters, + ) if onBinding != nil { onBinding(err) } @@ -481,16 +557,10 @@ func (d *DownTrack) Bind(t webrtc.TrackLocalContext) (webrtc.RTPCodecParameters, return codec, nil } - // Bind is called under RTPSender.mu lock, call the RTPSender.GetParameters in goroutine to avoid deadlock - go func() { - if tr := d.transceiver.Load(); tr != nil { - if sender := tr.Sender(); sender != nil { - extensions := sender.GetParameters().HeaderExtensions - d.params.Logger.Debugw("negotiated downtrack extensions", "extensions", extensions) - d.SetRTPHeaderExtensions(extensions) - } - } - }() + // Bind is called under RTPSender.mu lock, + // call the RTPSender.GetParameters (which setRTPHeaderExtensions invokes) + // in goroutine to avoid deadlock + go d.setRTPHeaderExtensions() doBind := func() { d.bindLock.Lock() @@ -500,12 +570,6 @@ func (d *DownTrack) Bind(t webrtc.TrackLocalContext) (webrtc.RTPCodecParameters, return } - if bs := d.bindState.Load(); bs != bindStateWaitForReceiverReady { - d.bindLock.Unlock() - d.params.Logger.Debugw("DownTrack.Bind: not in wait for receiver state", "state", bs) - return - } - isFECEnabled := false if mime.IsMimeTypeStringRED(matchedUpstreamCodec.MimeType) { d.isRED = true @@ -519,19 +583,26 @@ func (d *DownTrack) Bind(t webrtc.TrackLocalContext) (webrtc.RTPCodecParameters, } } if d.upstreamPrimaryPT == 0 { - d.params.Logger.Errorw("failed to find upstream primary opus payload type for RED", nil, "matchedCodec", codec, "upstreamCodec", d.upstreamCodecs) + d.params.Logger.Errorw( + "failed to find upstream primary opus payload type for RED", nil, + "matchedCodec", codec, + "upstreamCodec", d.upstreamCodecs, + ) } var primaryPT, secondaryPT int if n, err := fmt.Sscanf(codec.SDPFmtpLine, "%d/%d", &primaryPT, &secondaryPT); err != nil || n != 2 { - d.params.Logger.Errorw("failed to parse primary and secondary payload type for RED", err, "matchedCodec", codec) + d.params.Logger.Errorw( + "failed to parse primary and secondary payload type for RED", err, + "matchedCodec", codec, + ) } d.primaryPT = uint8(primaryPT) } else if mime.IsMimeTypeStringAudio(matchedUpstreamCodec.MimeType) { isFECEnabled = strings.Contains(strings.ToLower(matchedUpstreamCodec.SDPFmtpLine), "fec") } - logFields := []interface{}{ + logFields := []any{ "codecs", d.upstreamCodecs, "matchCodec", codec, "ssrc", t.SSRC(), @@ -553,8 +624,8 @@ func (d *DownTrack) Bind(t webrtc.TrackLocalContext) (webrtc.RTPCodecParameters, d.payloadTypeRTX.Store(uint32(utils.FindRTXPayloadType(codec.PayloadType, d.negotiatedCodecParameters))) logFields = append( logFields, - "payloadType", d.payloadType, - "payloadTypeRTX", d.payloadTypeRTX, + "payloadType", d.payloadType.Load(), + "payloadTypeRTX", d.payloadTypeRTX.Load(), "codecParameters", d.negotiatedCodecParameters, ) d.params.Logger.Debugw("DownTrack.Bind", logFields...) @@ -577,16 +648,19 @@ func (d *DownTrack) Bind(t webrtc.TrackLocalContext) (webrtc.RTPCodecParameters, d.sequencer = newSequencer(d.params.MaxTrack, d.kind == webrtc.RTPCodecTypeVideo, d.params.Logger) - d.codec = codec.RTPCodecCapability + d.codec.Store(codec.RTPCodecCapability) + d.rtpStats.SetClockRate(codec.RTPCodecCapability.ClockRate) + d.rtpStatsRTX.SetClockRate(codec.RTPCodecCapability.ClockRate) + if d.onBinding != nil { d.onBinding(nil) } d.setBindStateLocked(bindStateBound) - mimeType := d.mimeTypeLocked() d.bindLock.Unlock() - d.forwarder.DetermineCodec(codec.RTPCodecCapability, d.Receiver().HeaderExtensions()) - d.connectionStats.Start(mimeType, isFECEnabled) + receiver := d.Receiver() + d.forwarder.DetermineCodec(codec.RTPCodecCapability, receiver.HeaderExtensions(), receiver.VideoLayerMode()) + d.connectionStats.Start(d.Mime(), isFECEnabled) d.params.Logger.Debugw("downtrack bound") } @@ -596,13 +670,9 @@ func (d *DownTrack) Bind(t webrtc.TrackLocalContext) (webrtc.RTPCodecParameters, d.bindOnReceiverReady = doBind d.setBindStateLocked(bindStateWaitForReceiverReady) } - - onCodecNegotiated := d.onCodecNegotiated d.bindLock.Unlock() - if onCodecNegotiated != nil { - onCodecNegotiated(codec.RTPCodecCapability) - } + d.params.Listener.OnCodecNegotiated(codec.RTPCodecCapability) if isReceiverReady { doBind() @@ -640,7 +710,8 @@ func (d *DownTrack) handleReceiverReady() { func (d *DownTrack) handleUpstreamCodecChange(mimeType string) { d.bindLock.Lock() - if mime.IsMimeTypeStringEqual(d.codec.MimeType, mimeType) { + existingMimeType := d.codec.Load().(webrtc.RTPCodecCapability).MimeType + if mime.IsMimeTypeStringEqual(existingMimeType, mimeType) { d.bindLock.Unlock() return } @@ -652,7 +723,7 @@ func (d *DownTrack) handleUpstreamCodecChange(mimeType string) { return } - oldPT, oldRtxPT, oldCodec := d.payloadType.Load(), d.payloadTypeRTX.Load(), d.codec + oldPT, oldRtxPT, oldCodec := d.payloadType.Load(), d.payloadTypeRTX.Load(), d.codec.Load().(webrtc.RTPCodecCapability) var codec webrtc.RTPCodecParameters for _, c := range d.upstreamCodecs { @@ -681,9 +752,12 @@ func (d *DownTrack) handleUpstreamCodecChange(mimeType string) { d.payloadType.Store(uint32(codec.PayloadType)) d.payloadTypeRTX.Store(uint32(utils.FindRTXPayloadType(codec.PayloadType, d.negotiatedCodecParameters))) - d.codec = codec.RTPCodecCapability - newMimeType := d.mimeTypeLocked() - isFECEnabled := strings.Contains(strings.ToLower(d.codec.SDPFmtpLine), "fec") + + d.codec.Store(codec.RTPCodecCapability) + d.rtpStats.SetClockRate(codec.RTPCodecCapability.ClockRate) + d.rtpStatsRTX.SetClockRate(codec.RTPCodecCapability.ClockRate) + + isFECEnabled := strings.Contains(strings.ToLower(codec.SDPFmtpLine), "fec") d.bindLock.Unlock() d.params.Logger.Infow( @@ -693,9 +767,11 @@ func (d *DownTrack) handleUpstreamCodecChange(mimeType string) { "oldCodec", oldCodec, "newCodec", codec.RTPCodecCapability, ) + receiver := d.Receiver() d.forwarder.Restart() - d.forwarder.DetermineCodec(codec.RTPCodecCapability, d.Receiver().HeaderExtensions()) - d.connectionStats.UpdateCodec(newMimeType, isFECEnabled) + d.forwarder.DetermineCodec(codec.RTPCodecCapability, receiver.HeaderExtensions(), receiver.VideoLayerMode()) + + d.connectionStats.UpdateCodec(d.Mime(), isFECEnabled) } // Unbind implements the teardown logic when the track is no longer needed. This happens @@ -712,12 +788,9 @@ func (d *DownTrack) SetStreamAllocatorListener(listener DownTrackStreamAllocator d.streamAllocatorListener = listener d.streamAllocatorLock.Unlock() - if listener != nil { - if !listener.IsBWEEnabled(d) { - d.absSendTimeExtID = 0 - d.transportWideExtID = 0 - } + d.setRTPHeaderExtensions() + if listener != nil { // kick off a gratuitous allocation listener.OnSubscriptionChanged(d) } @@ -745,19 +818,11 @@ func (d *DownTrack) ID() string { return string(d.id) } // Codec returns current track codec capability func (d *DownTrack) Codec() webrtc.RTPCodecCapability { - d.bindLock.Lock() - defer d.bindLock.Unlock() - return d.codec + return d.codec.Load().(webrtc.RTPCodecCapability) } func (d *DownTrack) Mime() mime.MimeType { - d.bindLock.Lock() - defer d.bindLock.Unlock() - return d.mimeTypeLocked() -} - -func (d *DownTrack) mimeTypeLocked() mime.MimeType { - return mime.NormalizeMimeType(d.codec.MimeType) + return mime.NormalizeMimeType(d.codec.Load().(webrtc.RTPCodecCapability).MimeType) } // StreamID is the group this track belongs too. This must be unique @@ -788,46 +853,104 @@ func (d *DownTrack) SetReceiver(r TrackReceiver) { d.receiverLock.Unlock() old.DeleteDownTrack(d.SubscriberID()) + d.bindLock.Unlock() + + r.AddOnReady(d.handleReceiverReady) + d.handleUpstreamCodecChange(r.Codec().MimeType) + + d.bindLock.Lock() if err := r.AddDownTrack(d); err != nil { d.params.Logger.Warnw("failed to add downtrack to receiver", err) } d.bindLock.Unlock() - r.AddOnReady(d.handleReceiverReady) - d.handleUpstreamCodecChange(r.Codec().MimeType) if sal := d.getStreamAllocatorListener(); sal != nil { sal.OnSubscribedLayerChanged(d, d.forwarder.MaxLayer()) } } // Sets RTP header extensions for this track -func (d *DownTrack) SetRTPHeaderExtensions(rtpHeaderExtensions []webrtc.RTPHeaderExtensionParameter) { - isBWEEnabled := true - if sal := d.getStreamAllocatorListener(); sal != nil { +func (d *DownTrack) setRTPHeaderExtensions() { + isBWEEnabled := false + bweType := bwe.BWETypeNone + sal := d.getStreamAllocatorListener() + if sal != nil { isBWEEnabled = sal.IsBWEEnabled(d) + bweType = sal.BWEType() } - for _, ext := range rtpHeaderExtensions { + + tr := d.transceiver.Load() + if tr == nil { + return + } + + var extensions []webrtc.RTPHeaderExtensionParameter + if sender := tr.Sender(); sender != nil { + extensions = sender.GetParameters().HeaderExtensions + d.params.Logger.Debugw("negotiated downtrack extensions", "extensions", extensions) + } + + d.bindLock.Lock() + for _, ext := range extensions { switch ext.URI { case sdp.ABSSendTimeURI: - if isBWEEnabled { - d.absSendTimeExtID = ext.ID - } else { - d.absSendTimeExtID = 0 + if sal != nil { + if isBWEEnabled && bweType == bwe.BWETypeRemote { + if d.absSendTimeExtID != 0 && d.absSendTimeExtID != ext.ID { + d.params.Logger.Infow("absSendTimeExtID mismatch", "current", d.absSendTimeExtID, "negotiated", ext.ID) + } + d.absSendTimeExtID = ext.ID + } else { + if d.absSendTimeExtID != 0 { + d.params.Logger.Infow("absSendTimeExtID disabled unexpectedly", "negotiated", ext.ID) + } + d.absSendTimeExtID = 0 + } } + case dd.ExtensionURI: - d.dependencyDescriptorExtID = ext.ID - case pd.PlayoutDelayURI: - d.playoutDelayExtID = ext.ID - case sdp.TransportCCURI: - if isBWEEnabled { - d.transportWideExtID = ext.ID - } else { - d.transportWideExtID = 0 + if d.dependencyDescriptorExtID != 0 && d.dependencyDescriptorExtID != ext.ID { + d.params.Logger.Infow("dependencyDescriptorExtID mismatch", "current", d.dependencyDescriptorExtID, "negotiated", ext.ID) } + d.dependencyDescriptorExtID = ext.ID + + case pd.PlayoutDelayURI: + if d.playoutDelayExtID != 0 && d.playoutDelayExtID != ext.ID { + d.params.Logger.Infow("playoutDelayExtID mismatch", "current", d.playoutDelayExtID, "negotiated", ext.ID) + } + d.playoutDelayExtID = ext.ID + + case sdp.TransportCCURI: + if sal != nil { + if isBWEEnabled && bweType == bwe.BWETypeSendSide { + if d.transportWideExtID != 0 && d.transportWideExtID != ext.ID { + d.params.Logger.Infow("transportWideExtID mismatch", "current", d.transportWideExtID, "negotiated", ext.ID) + } + d.transportWideExtID = ext.ID + } else { + if d.transportWideExtID != 0 { + d.params.Logger.Infow("transportWideExtID disabled unexpectedly", "negotiated", ext.ID) + } + d.transportWideExtID = 0 + } + } + case act.AbsCaptureTimeURI: + if d.absCaptureTimeExtID != 0 && d.absCaptureTimeExtID != ext.ID { + d.params.Logger.Infow("absCaptureTimeExtID mismatch", "current", d.absCaptureTimeExtID, "negotiated", ext.ID) + } d.absCaptureTimeExtID = ext.ID } } + d.params.Logger.Debugw( + "negotiated extension ids", + "absSendTimeExtID", d.absSendTimeExtID, + "dependencyDescriptorExtID", d.dependencyDescriptorExtID, + "playoutDelayExtID", d.playoutDelayExtID, + "transportWideExtID", d.transportWideExtID, + "absCaptureTimeExtID", d.absCaptureTimeExtID, + ) + d.bindLock.Unlock() } // Kind controls if this TrackLocal is audio or video @@ -848,15 +971,9 @@ func (d *DownTrack) SSRCRTX() uint32 { return d.ssrcRTX } -func (d *DownTrack) Stop() error { - if tr := d.transceiver.Load(); tr != nil { - return tr.Stop() - } - return errors.New("downtrack transceiver does not exist") -} - func (d *DownTrack) SetTransceiver(transceiver *webrtc.RTPTransceiver) { d.transceiver.Store(transceiver) + d.setRTPHeaderExtensions() } func (d *DownTrack) GetTransceiver() *webrtc.RTPTransceiver { @@ -880,13 +997,7 @@ func (d *DownTrack) postKeyFrameRequestEvent() { func (d *DownTrack) keyFrameRequester() { getInterval := func() time.Duration { - interval := 2 * d.rtpStats.GetRtt() - if interval < keyFrameIntervalMin { - interval = keyFrameIntervalMin - } - if interval > keyFrameIntervalMax { - interval = keyFrameIntervalMax - } + interval := min(max(2*d.rtpStats.GetRtt(), keyFrameIntervalMin), keyFrameIntervalMax) return time.Duration(interval) * time.Millisecond } @@ -939,30 +1050,26 @@ func (d *DownTrack) maxLayerNotifierWorker() { maxLayerSpatial := d.forwarder.GetMaxSubscribedSpatial() d.params.Logger.Debugw("max subscribed layer processed", "layer", maxLayerSpatial, "event", event) - if onMaxSubscribedLayerChanged := d.getOnMaxLayerChanged(); onMaxSubscribedLayerChanged != nil { - d.params.Logger.Debugw( - "notifying max subscribed layer", - "layer", maxLayerSpatial, - "event", event, - ) - onMaxSubscribedLayerChanged(d, maxLayerSpatial) - } - } - - if onMaxSubscribedLayerChanged := d.getOnMaxLayerChanged(); onMaxSubscribedLayerChanged != nil { d.params.Logger.Debugw( "notifying max subscribed layer", - "layer", buffer.InvalidLayerSpatial, - "event", "close", + "layer", maxLayerSpatial, + "event", event, ) - onMaxSubscribedLayerChanged(d, buffer.InvalidLayerSpatial) + d.params.Listener.OnMaxSubscribedLayerChanged(maxLayerSpatial) } + + d.params.Logger.Debugw( + "notifying max subscribed layer", + "layer", buffer.InvalidLayerSpatial, + "event", "close", + ) + d.params.Listener.OnMaxSubscribedLayerChanged(buffer.InvalidLayerSpatial) } // WriteRTP writes an RTP Packet to the DownTrack -func (d *DownTrack) WriteRTP(extPkt *buffer.ExtPacket, layer int32) error { +func (d *DownTrack) WriteRTP(extPkt *buffer.ExtPacket, layer int32) int32 { if !d.writable.Load() { - return nil + return 0 } tp, err := d.forwarder.GetTranslationParams(extPkt, layer) @@ -970,7 +1077,7 @@ func (d *DownTrack) WriteRTP(extPkt *buffer.ExtPacket, layer int32) error { if err != nil { d.params.Logger.Errorw("could not get translation params", err) } - return err + return 0 } poolEntity := PacketFactory.Get().(*[]byte) @@ -978,24 +1085,33 @@ func (d *DownTrack) WriteRTP(extPkt *buffer.ExtPacket, layer int32) error { copy(payload, tp.codecBytes) n := copy(payload[len(tp.codecBytes):], extPkt.Packet.Payload[tp.incomingHeaderSize:]) if n != len(extPkt.Packet.Payload[tp.incomingHeaderSize:]) { - d.params.Logger.Errorw("payload overflow", nil, "want", len(extPkt.Packet.Payload[tp.incomingHeaderSize:]), "have", n) + d.params.Logger.Errorw( + "payload overflow", errPayloadOverflow, + "want", len(extPkt.Packet.Payload[tp.incomingHeaderSize:]), + "have", n, + ) PacketFactory.Put(poolEntity) - return ErrPayloadOverflow + return 0 } payload = payload[:len(tp.codecBytes)+n] + if d.params.StripPacketTrailer { + if strip := packettrailer.StripTrailer(payload, tp.marker); strip > 0 { + payload = payload[:len(payload)-strip] + } + } + // translate RTP header - hdr := &rtp.Header{ + hdr := RTPHeaderFactory.Get().(*rtp.Header) + *hdr = rtp.Header{ Version: extPkt.Packet.Version, Padding: extPkt.Packet.Padding, + Marker: tp.marker, PayloadType: d.getTranslatedPayloadType(extPkt.Packet.PayloadType), SequenceNumber: uint16(tp.rtp.extSequenceNumber), Timestamp: uint32(tp.rtp.extTimestamp), SSRC: d.ssrc, } - if tp.marker { - hdr.Marker = tp.marker - } // add extensions if d.dependencyDescriptorExtID != 0 && tp.ddBytes != nil { @@ -1022,7 +1138,7 @@ func (d *DownTrack) WriteRTP(extPkt *buffer.ExtPacket, layer int32) error { // clock domain of publisher. SFU is normalising sender reports of publisher // to SFU clock before sending to subscribers. So, capture time should be // normalized to the same clock. Clear out any offset. - _, _, refSenderReport := d.forwarder.GetSenderReportParams() + _, _, _, refSenderReport := d.forwarder.GetSenderReportParams() if refSenderReport != nil { actExtCopy := *extPkt.AbsCaptureTimeExt if err = actExtCopy.Rewrite( @@ -1066,8 +1182,10 @@ func (d *DownTrack) WriteRTP(extPkt *buffer.ExtPacket, layer int32) error { 0, extPkt.IsOutOfOrder, ) - d.pacer.Enqueue(&pacer.Packet{ + pacerPacket := pacer.PacketFactory.Get().(*pacer.Packet) + *pacerPacket = pacer.Packet{ Header: hdr, + HeaderPool: RTPHeaderFactory, HeaderSize: headerSize, Payload: payload, ProbeClusterId: ccutils.ProbeClusterId(d.probeClusterId.Load()), @@ -1076,9 +1194,10 @@ func (d *DownTrack) WriteRTP(extPkt *buffer.ExtPacket, layer int32) error { WriteStream: d.writeStream, Pool: PacketFactory, PoolEntity: poolEntity, - }) + } + d.pacer.Enqueue(pacerPacket) - if extPkt.KeyFrame { + if extPkt.IsKeyFrame { d.isNACKThrottled.Store(false) d.rtpStats.UpdateKeyFrame(1) d.params.Logger.Debugw( @@ -1098,7 +1217,11 @@ func (d *DownTrack) WriteRTP(extPkt *buffer.ExtPacket, layer int32) error { sal.OnResume(d) } } - return nil + + if tp.isStarting { + d.params.Listener.OnStreamStarted() + } + return 1 } // WritePaddingRTP tries to write as many padding only RTP packets as necessary @@ -1108,32 +1231,34 @@ func (d *DownTrack) WritePaddingRTP(bytesToSend int, paddingOnMute bool, forceMa return 0 } - if !d.rtpStats.IsActive() && !paddingOnMute { - return 0 - } + if !paddingOnMute { + if !d.rtpStats.IsActive() { + return 0 + } - // Ideally should look at header extensions negotiated for - // track and decide if padding can be sent. But, browsers behave - // in unexpected ways when using audio for bandwidth estimation and - // padding is mainly used to probe for excess available bandwidth. - // So, to be safe, limit to video tracks - if d.kind == webrtc.RTPCodecTypeAudio { - return 0 - } + // Ideally should look at header extensions negotiated for + // track and decide if padding can be sent. But, browsers behave + // in unexpected ways when using audio for bandwidth estimation and + // padding is mainly used to probe for excess available bandwidth. + // So, to be safe, limit to video tracks + if d.kind == webrtc.RTPCodecTypeAudio { + return 0 + } - // LK-TODO-START - // Potentially write padding even if muted. Given that padding - // can be sent only on frame boundaries, writing on disabled tracks - // will give more options. - // LK-TODO-END - if d.forwarder.IsMuted() && !paddingOnMute { - return 0 - } + // LK-TODO-START + // Potentially write padding even if muted. Given that padding + // can be sent only on frame boundaries, writing on disabled tracks + // will give more options. + // LK-TODO-END + if d.forwarder.IsMuted() { + return 0 + } - // Hold sending padding packets till first RTCP-RR is received for this RTP stream. - // That is definitive proof that the remote side knows about this RTP stream. - if d.rtpStats.LastReceiverReportTime() == 0 && !paddingOnMute { - return 0 + // Hold sending padding packets till first RTCP-RR is received for this RTP stream. + // That is definitive proof that the remote side knows about this RTP stream. + if d.rtpStats.LastReceiverReportTime() == 0 { + return 0 + } } // RTP padding maximum is 255 bytes. Break it up. @@ -1143,7 +1268,14 @@ func (d *DownTrack) WritePaddingRTP(bytesToSend int, paddingOnMute bool, forceMa return 0 } - snts, err := d.forwarder.GetSnTsForPadding(num, forceMarker) + frameRate := uint32(0) + if paddingOnMute { + // advance timestamps when sending dummy padding packets to start a stream + // to ensure receiver sees proper timestamp and starts the stream + frameRate = uint32(time.Second / paddingOnMuteInterval) + } + + snts, err := d.forwarder.GetSnTsForPadding(num, frameRate, forceMarker) if err != nil { return 0 } @@ -1159,8 +1291,9 @@ func (d *DownTrack) WritePaddingRTP(bytesToSend int, paddingOnMute bool, forceMa bytesSent := 0 payloads := make([]byte, RTPPaddingMaxPayloadSize*len(snts)) - for i := 0; i < len(snts); i++ { - hdr := &rtp.Header{ + for i := range snts { + hdr := RTPHeaderFactory.Get().(*rtp.Header) + *hdr = rtp.Header{ Version: 2, Padding: true, Marker: false, @@ -1187,8 +1320,11 @@ func (d *DownTrack) WritePaddingRTP(bytesToSend int, paddingOnMute bool, forceMa payloadSize, false, ) - d.pacer.Enqueue(&pacer.Packet{ + + pacerPacket := pacer.PacketFactory.Get().(*pacer.Packet) + *pacerPacket = pacer.Packet{ Header: hdr, + HeaderPool: RTPHeaderFactory, HeaderSize: hdrSize, Payload: payload, ProbeClusterId: ccutils.ProbeClusterId(d.probeClusterId.Load()), @@ -1196,7 +1332,8 @@ func (d *DownTrack) WritePaddingRTP(bytesToSend int, paddingOnMute bool, forceMa AbsSendTimeExtID: uint8(d.absSendTimeExtID), TransportWideExtID: uint8(d.transportWideExtID), WriteStream: d.writeStream, - }) + } + d.pacer.Enqueue(pacerPacket) bytesSent += hdrSize + payloadSize } @@ -1237,8 +1374,8 @@ func (d *DownTrack) handleMute(muted bool, changed bool) { // no layers required due to publisher mute (bit of circular dependency), // there will be a delay in layers turning back on when unmute happens. // Unmute path will require - // 1. unmute signalling out-of-band from publisher received by down track(s) - // 2. down track(s) notifying max layer + // 1. unmute signalling out-of-band from publisher received by downtrack(s) + // 2. downtrack(s) notifying max layer // 3. out-of-band notification about max layer sent back to the publisher // 4. publisher starts layer(s) // Ideally, on publisher mute, whatever layers were active remain active and @@ -1275,23 +1412,24 @@ func (d *DownTrack) IsClosed() bool { } func (d *DownTrack) Close() { - d.CloseWithFlush(true) + d.CloseWithFlush(true, true) } -// CloseWithFlush - flush used to indicate whether send blank frame to flush +// CloseWithFlush - `flush` used to indicate whether send blank frame to flush // decoder of client. -// 1. When transceiver is reused by other participant's video track, +// 1. When transceiver of this track is reused by some other participant's video track, // set flush=true to avoid previous video shows before new stream is displayed. // 2. in case of session migration, participant migrate from other node, video track should // be resumed with same participant, set flush=false since we don't need to flush decoder. -func (d *DownTrack) CloseWithFlush(flush bool) { +func (d *DownTrack) CloseWithFlush(flush bool, isEnding bool) { + d.bindLock.Lock() if d.isClosed.Swap(true) { // already closed + d.bindLock.Unlock() return } - d.bindLock.Lock() - d.params.Logger.Debugw("close down track", "flushBlankFrame", flush) + d.params.Logger.Debugw("close downtrack", "flushBlankFrame", flush, "isEnding", isEnding) if d.bindState.Load() == bindStateBound { d.forwarder.Mute(true, true) @@ -1315,29 +1453,30 @@ func (d *DownTrack) CloseWithFlush(flush bool) { d.params.Logger.Debugw("closing sender", "kind", d.kind) } + d.setBindStateLocked(bindStateUnbound) d.Receiver().DeleteDownTrack(d.SubscriberID()) - if d.rtcpReader != nil && flush { + if d.rtcpReader != nil && isEnding { d.params.Logger.Debugw("downtrack close rtcp reader") d.rtcpReader.Close() d.rtcpReader.OnPacket(nil) } - if d.rtcpReaderRTX != nil && flush { + if d.rtcpReaderRTX != nil && isEnding { d.params.Logger.Debugw("downtrack close rtcp rtx reader") d.rtcpReaderRTX.Close() d.rtcpReaderRTX.OnPacket(nil) } - mime := d.codec.MimeType d.bindLock.Unlock() d.connectionStats.Close() d.rtpStats.Stop() d.rtpStatsRTX.Stop() - d.params.Logger.Debugw("rtp stats", + d.params.Logger.Debugw( + "rtp stats", "direction", "downstream", - "mime", mime, + "mime", d.Mime().String(), "ssrc", d.ssrc, "stats", d.rtpStats, "statsRTX", d.rtpStatsRTX, @@ -1353,9 +1492,7 @@ func (d *DownTrack) CloseWithFlush(flush bool) { close(d.keyFrameRequesterCh) d.keyFrameRequesterChMu.Unlock() - if onCloseHandler := d.getOnCloseHandler(); onCloseHandler != nil { - onCloseHandler(!flush) - } + d.params.Listener.OnDownTrackClose(!flush) } func (d *DownTrack) SetMaxSpatialLayer(spatialLayer int32) { @@ -1408,7 +1545,7 @@ func (d *DownTrack) SeedState(state DownTrackState) { } if state.RTPStats != nil || state.ForwarderState != nil { - d.params.Logger.Debugw("seeding down track state", "state", state) + d.params.Logger.Debugw("seeding downtrack state", "state", state) } if state.RTPStats != nil { d.rtpStats.Seed(state.RTPStats) @@ -1486,21 +1623,6 @@ func (d *DownTrack) UpTrackBitrateReport(availableLayers []int32, bitrates Bitra ) } -// OnCloseHandler method to be called on remote tracked removed -func (d *DownTrack) OnCloseHandler(fn func(isExpectedToResume bool)) { - d.cbMu.Lock() - defer d.cbMu.Unlock() - - d.onCloseHandler = fn -} - -func (d *DownTrack) getOnCloseHandler() func(isExpectedToResume bool) { - d.cbMu.RLock() - defer d.cbMu.RUnlock() - - return d.onCloseHandler -} - func (d *DownTrack) OnBinding(fn func(error)) { d.bindLock.Lock() defer d.bindLock.Unlock() @@ -1515,48 +1637,6 @@ func (d *DownTrack) AddReceiverReportListener(listener ReceiverReportListener) { d.receiverReportListeners = append(d.receiverReportListeners, listener) } -func (d *DownTrack) OnStatsUpdate(fn func(dt *DownTrack, stat *livekit.AnalyticsStat)) { - d.cbMu.Lock() - defer d.cbMu.Unlock() - - d.onStatsUpdate = fn -} - -func (d *DownTrack) getOnStatsUpdate() func(dt *DownTrack, stat *livekit.AnalyticsStat) { - d.cbMu.RLock() - defer d.cbMu.RUnlock() - - return d.onStatsUpdate -} - -func (d *DownTrack) OnRttUpdate(fn func(dt *DownTrack, rtt uint32)) { - d.cbMu.Lock() - defer d.cbMu.Unlock() - - d.onRttUpdate = fn -} - -func (d *DownTrack) getOnRttUpdate() func(dt *DownTrack, rtt uint32) { - d.cbMu.RLock() - defer d.cbMu.RUnlock() - - return d.onRttUpdate -} - -func (d *DownTrack) OnMaxLayerChanged(fn func(dt *DownTrack, layer int32)) { - d.cbMu.Lock() - defer d.cbMu.Unlock() - - d.onMaxSubscribedLayerChanged = fn -} - -func (d *DownTrack) getOnMaxLayerChanged() func(dt *DownTrack, layer int32) { - d.cbMu.RLock() - defer d.cbMu.RUnlock() - - return d.onMaxSubscribedLayerChanged -} - func (d *DownTrack) IsDeficient() bool { return d.forwarder.IsDeficient() } @@ -1653,6 +1733,22 @@ func (d *DownTrack) Resync() { d.forwarder.Resync() } +func (d *DownTrack) ReceiverRestart(rcvr TrackReceiver) { + if rcvr.Mime() != d.Receiver().Mime() { + d.params.Logger.Infow("upstream receiver restart - skipped", "mime", d.Receiver().Mime().String(), "newMime", rcvr.Mime().String()) + return + } + + d.bindLock.Lock() + codec := d.codec.Load().(webrtc.RTPCodecCapability) + d.bindLock.Unlock() + + receiver := d.Receiver() + d.params.Logger.Infow("upstream receiver restart", "mime", receiver.Mime().String()) + d.forwarder.Restart() + d.forwarder.DetermineCodec(codec, receiver.HeaderExtensions(), receiver.VideoLayerMode()) +} + func (d *DownTrack) CreateSourceDescriptionChunks() []rtcp.SourceDescriptionChunk { transceiver := d.transceiver.Load() if d.bindState.Load() != bindStateBound || transceiver == nil { @@ -1680,7 +1776,7 @@ func (d *DownTrack) CreateSenderReport() *rtcp.SenderReport { return nil } - _, tsOffset, refSenderReport := d.forwarder.GetSenderReportParams() + _, _, tsOffset, refSenderReport := d.forwarder.GetSenderReportParams() return d.rtpStats.GetRtcpSenderReport(d.ssrc, refSenderReport, tsOffset, !d.params.DisableSenderReportPassThrough) // not sending RTCP Sender Report for RTX @@ -1699,9 +1795,13 @@ func (d *DownTrack) writeBlankFrameRTP(duration float32, generation uint32) chan var getBlankFrame func(bool) ([]byte, error) switch mimeType { case mime.MimeTypeOpus: - getBlankFrame = d.getOpusBlankFrame + getBlankFrame = d.getAudioBlankFrameFunc(OpusSilenceFrame) case mime.MimeTypeRED: getBlankFrame = d.getOpusRedBlankFrame + case mime.MimeTypePCMU: + getBlankFrame = d.getAudioBlankFrameFunc(PCMUSilenceFrame) + case mime.MimeTypePCMA: + getBlankFrame = d.getAudioBlankFrameFunc(PCMASilenceFrame) case mime.MimeTypeVP8: getBlankFrame = d.getVP8BlankFrame case mime.MimeTypeH264: @@ -1712,7 +1812,8 @@ func (d *DownTrack) writeBlankFrameRTP(duration float32, generation uint32) chan } frameRate := uint32(30) - if mimeType == mime.MimeTypeOpus || mimeType == mime.MimeTypeRED { + if mimeType == mime.MimeTypeOpus || mimeType == mime.MimeTypeRED || + mimeType == mime.MimeTypePCMU || mimeType == mime.MimeTypePCMA { frameRate = 50 } @@ -1738,7 +1839,7 @@ func (d *DownTrack) writeBlankFrameRTP(duration float32, generation uint32) chan return } - for i := 0; i < len(snts); i++ { + for i := range snts { hdr := &rtp.Header{ Version: 2, Padding: false, @@ -1768,7 +1869,8 @@ func (d *DownTrack) writeBlankFrameRTP(duration float32, generation uint32) chan 0, false, ) - d.pacer.Enqueue(&pacer.Packet{ + pacerPacket := pacer.PacketFactory.Get().(*pacer.Packet) + *pacerPacket = pacer.Packet{ Header: hdr, HeaderSize: headerSize, Payload: payload, @@ -1776,7 +1878,8 @@ func (d *DownTrack) writeBlankFrameRTP(duration float32, generation uint32) chan AbsSendTimeExtID: uint8(d.absSendTimeExtID), TransportWideExtID: uint8(d.transportWideExtID), WriteStream: d.writeStream, - }) + } + d.pacer.Enqueue(pacerPacket) // only the first frame will need frameEndNeeded to close out the // previous picture, rest are small key frames (for the video case) @@ -1801,15 +1904,17 @@ func (d *DownTrack) maybeAddTrailer(buf []byte) int { return len(d.params.Trailer) } -func (d *DownTrack) getOpusBlankFrame(_frameEndNeeded bool) ([]byte, error) { +func (d *DownTrack) getAudioBlankFrameFunc(silentPayload []byte) func(_frameEndNeeded bool) ([]byte, error) { // silence frame // Used shortly after muting to ensure residual noise does not keep // generating noise at the decoder after the stream is stopped // i. e. comfort noise generation actually not producing something comfortable. - payload := make([]byte, 1000) - copy(payload[0:], OpusSilenceFrame) - trailerLen := d.maybeAddTrailer(payload[len(OpusSilenceFrame):]) - return payload[:len(OpusSilenceFrame)+trailerLen], nil + return func(_frameEndNeeded bool) ([]byte, error) { + payload := make([]byte, 1000) + copy(payload[0:], silentPayload) + trailerLen := d.maybeAddTrailer(payload[len(silentPayload):]) + return payload[:len(silentPayload)+trailerLen], nil + } } func (d *DownTrack) getOpusRedBlankFrame(_frameEndNeeded bool) ([]byte, error) { @@ -1917,6 +2022,7 @@ func (d *DownTrack) handleRTCP(bytes []byte) { if r.SSRC != d.ssrc { continue } + rr.Reports = append(rr.Reports, r) rtt, isRttChanged := d.rtpStats.UpdateFromReceiverReport(r) if isRttChanged { @@ -2006,9 +2112,7 @@ func (d *DownTrack) handleRTCP(bytes []byte) { d.sequencer.setRTT(rttToReport) } - if onRttUpdate := d.getOnRttUpdate(); onRttUpdate != nil { - onRttUpdate(d, rttToReport) - } + d.params.Listener.OnRttUpdate(rttToReport) } } @@ -2067,7 +2171,8 @@ func (d *DownTrack) retransmitPacket(epm *extPacketMeta, sourcePkt []byte, isPro d.params.Logger.Errorw("could not unmarshal rtp packet to send via RTX", err) return 0, err } - hdr := &rtp.Header{ + hdr := RTPHeaderFactory.Get().(*rtp.Header) + *hdr = rtp.Header{ Version: pkt.Header.Version, Padding: pkt.Header.Padding, Marker: epm.marker, @@ -2119,6 +2224,12 @@ func (d *DownTrack) retransmitPacket(epm *extPacketMeta, sourcePkt []byte, isPro payload = payload[:rtxOffset+int(epm.numCodecBytesOut)+len(pkt.Payload)-int(epm.numCodecBytesIn)] } + if d.params.StripPacketTrailer { + if strip := packettrailer.StripTrailer(payload[rtxOffset:], epm.marker); strip > 0 { + payload = payload[:len(payload)-strip] + } + } + headerSize := hdr.MarshalSize() var ( payloadSize, paddingSize int @@ -2154,8 +2265,10 @@ func (d *DownTrack) retransmitPacket(epm *extPacketMeta, sourcePkt []byte, isPro isOutOfOrder, ) } - d.pacer.Enqueue(&pacer.Packet{ + pacerPacket := pacer.PacketFactory.Get().(*pacer.Packet) + *pacerPacket = pacer.Packet{ Header: hdr, + HeaderPool: RTPHeaderFactory, HeaderSize: headerSize, Payload: payload, ProbeClusterId: ccutils.ProbeClusterId(d.probeClusterId.Load()), @@ -2166,7 +2279,8 @@ func (d *DownTrack) retransmitPacket(epm *extPacketMeta, sourcePkt []byte, isPro WriteStream: d.writeStream, Pool: PacketFactory, PoolEntity: poolEntity, - }) + } + d.pacer.Enqueue(pacerPacket) return headerSize + len(payload), nil } @@ -2187,6 +2301,8 @@ func (d *DownTrack) retransmitPackets(nacks []uint16) { src := PacketFactory.Get().(*[]byte) defer PacketFactory.Put(src) + receiver := d.Receiver() + nackAcks := uint32(0) nackMisses := uint32(0) numRepeatedNACKs := uint32(0) @@ -2198,7 +2314,7 @@ func (d *DownTrack) retransmitPackets(nacks []uint16) { nackAcks++ pktBuff := *src - n, err := d.Receiver().ReadRTP(pktBuff, uint8(epm.layer), epm.sourceSeqNo) + n, err := receiver.ReadRTP(pktBuff, uint8(epm.layer), epm.sourceSeqNo) if err != nil { if err == io.EOF { break @@ -2242,9 +2358,10 @@ func (d *DownTrack) WriteProbePackets(bytesToSend int, usePadding bool) int { } payloads := make([]byte, RTPPaddingMaxPayloadSize*num) - for i := 0; i < num; i++ { + for i := range num { rtxExtSequenceNumber := d.rtxSequenceNumber.Inc() - hdr := &rtp.Header{ + hdr := RTPHeaderFactory.Get().(*rtp.Header) + *hdr = rtp.Header{ Version: 2, Padding: true, Marker: false, @@ -2271,8 +2388,10 @@ func (d *DownTrack) WriteProbePackets(bytesToSend int, usePadding bool) int { payloadSize, false, ) - d.pacer.Enqueue(&pacer.Packet{ + pacerPacket := pacer.PacketFactory.Get().(*pacer.Packet) + *pacerPacket = pacer.Packet{ Header: hdr, + HeaderPool: RTPHeaderFactory, HeaderSize: hdrSize, Payload: payload, ProbeClusterId: ccutils.ProbeClusterId(d.probeClusterId.Load()), @@ -2280,7 +2399,8 @@ func (d *DownTrack) WriteProbePackets(bytesToSend int, usePadding bool) int { AbsSendTimeExtID: uint8(d.absSendTimeExtID), TransportWideExtID: uint8(d.transportWideExtID), WriteStream: d.writeStream, - }) + } + d.pacer.Enqueue(pacerPacket) bytesSent += hdrSize + payloadSize } @@ -2288,6 +2408,8 @@ func (d *DownTrack) WriteProbePackets(bytesToSend int, usePadding bool) int { src := PacketFactory.Get().(*[]byte) defer PacketFactory.Put(src) + receiver := d.Receiver() + endExtHighestSequenceNumber := d.rtpStats.ExtHighestSequenceNumber() startExtHighestSequenceNumber := endExtHighestSequenceNumber - 5 for esn := startExtHighestSequenceNumber; esn <= endExtHighestSequenceNumber; esn++ { @@ -2297,7 +2419,7 @@ func (d *DownTrack) WriteProbePackets(bytesToSend int, usePadding bool) int { } pktBuff := *src - n, err := d.Receiver().ReadRTP(pktBuff, uint8(epm.layer), epm.sourceSeqNo) + n, err := receiver.ReadRTP(pktBuff, uint8(epm.layer), epm.sourceSeqNo) if err != nil { if err == io.EOF { break @@ -2326,17 +2448,17 @@ func (d *DownTrack) addDummyExtensions(hdr *rtp.Header) { } } -func (d *DownTrack) getTranslatedPayloadType(src uint8) uint8 { - // send primary codec to subscriber if the publisher send primary codec to us when red is negotiated, +func (d *DownTrack) getTranslatedPayloadType(srcPT uint8) uint8 { + // send primary codec to subscriber if the publisher sent primary codec when red is negotiated, // this will happen when the payload is too large to encode into red payload (exceeds mtu). - if d.isRED && src == d.upstreamPrimaryPT && d.primaryPT != 0 { + if d.isRED && srcPT == d.upstreamPrimaryPT && d.primaryPT != 0 { return d.primaryPT } return uint8(d.payloadType.Load()) } -func (d *DownTrack) DebugInfo() map[string]interface{} { - stats := map[string]interface{}{ +func (d *DownTrack) DebugInfo() map[string]any { + stats := map[string]any{ "LastPli": d.rtpStats.LastPli(), } stats["RTPMunger"] = d.forwarder.RTPMungerDebugInfo() @@ -2348,12 +2470,12 @@ func (d *DownTrack) DebugInfo() map[string]interface{} { stats["PacketCount"] = senderReport.PacketCount } - return map[string]interface{}{ + return map[string]any{ "SubscriberID": d.params.SubID, "TrackID": d.id, "StreamID": d.params.StreamID, "SSRC": d.ssrc, - "MimeType": d.codec.MimeType, + "MimeType": d.Mime().String(), "BindState": d.bindState.Load().(bindState), "Muted": d.forwarder.IsMuted(), "PubMuted": d.forwarder.IsPubMuted(), @@ -2366,6 +2488,13 @@ func (d *DownTrack) GetConnectionScoreAndQuality() (float32, livekit.ConnectionQ return d.connectionStats.GetScoreAndQuality() } +// OnStatsUpdate registers an additional callback that fires alongside the +// configured DownTrackListener whenever connection-quality stats are produced. +// Intended for tests and observers; the production listener path is unaffected. +func (d *DownTrack) OnStatsUpdate(fn func(d *DownTrack, stat *livekit.AnalyticsStat)) { + d.onStatsUpdate.Store(fn) +} + func (d *DownTrack) GetTrackStats() *livekit.RTPStats { return rtpstats.ReconcileRTPStatsWithRTX(d.rtpStats.ToProto(), d.rtpStatsRTX.ToProto()) } @@ -2416,6 +2545,8 @@ func (d *DownTrack) onBindAndConnectedChange() { } d.writable.Store(d.connected.Load() && d.bindState.Load() == bindStateBound) if d.connected.Load() && d.bindState.Load() == bindStateBound && !d.bindAndConnectedOnce.Swap(true) { + go d.params.Listener.OnBindAndConnected() + if d.activePaddingOnMuteUpTrack.Load() { go d.sendPaddingOnMute() } @@ -2431,15 +2562,14 @@ func (d *DownTrack) sendPaddingOnMute() { if d.kind == webrtc.RTPCodecTypeVideo { d.sendPaddingOnMuteForVideo() - } else if d.Mime() == mime.MimeTypeOpus { - d.sendSilentFrameOnMuteForOpus() + } else { + d.sendSilentFrameOnMuteForAudio() } } func (d *DownTrack) sendPaddingOnMuteForVideo() { - paddingOnMuteInterval := 100 * time.Millisecond numPackets := maxPaddingOnMuteDuration / paddingOnMuteInterval - for i := 0; i < int(numPackets); i++ { + for i := range int(numPackets) { if d.rtpStats.IsActive() || d.IsClosed() { return } @@ -2451,7 +2581,29 @@ func (d *DownTrack) sendPaddingOnMuteForVideo() { } } -func (d *DownTrack) sendSilentFrameOnMuteForOpus() { +func (d *DownTrack) sendSilentFrameOnMuteForAudio() { + var ( + payload []byte + err error + ) + switch d.Mime() { + case mime.MimeTypeOpus: + payload, err = d.getAudioBlankFrameFunc(OpusSilenceFrame)(false) + case mime.MimeTypeRED: + payload, err = d.getOpusRedBlankFrame(false) + case mime.MimeTypePCMU: + payload, err = d.getAudioBlankFrameFunc(PCMUSilenceFrame)(false) + case mime.MimeTypePCMA: + payload, err = d.getAudioBlankFrameFunc(PCMASilenceFrame)(false) + default: + d.params.Logger.Infow("unsupported mime type for silent frame on mute", "mimeType", d.Mime()) + return + } + if err != nil { + d.params.Logger.Warnw("could not get blank frame", err) + return + } + frameRate := uint32(50) frameDuration := time.Duration(1000/frameRate) * time.Millisecond numFrames := frameRate * uint32(maxPaddingOnMuteDuration/time.Second) @@ -2469,7 +2621,7 @@ func (d *DownTrack) sendSilentFrameOnMuteForOpus() { d.params.Logger.Warnw("could not get SN/TS for blank frame", err) return } - for i := 0; i < len(snts); i++ { + for i := range len(snts) { hdr := &rtp.Header{ Version: 2, Padding: false, @@ -2480,13 +2632,6 @@ func (d *DownTrack) sendSilentFrameOnMuteForOpus() { SSRC: d.ssrc, } d.addDummyExtensions(hdr) - - payload, err := d.getOpusBlankFrame(false) - if err != nil { - d.params.Logger.Warnw("could not get blank frame", err) - return - } - headerSize := hdr.MarshalSize() d.rtpStats.Update( mono.UnixNano(), @@ -2498,7 +2643,8 @@ func (d *DownTrack) sendSilentFrameOnMuteForOpus() { len(payload), // although this is using empty frames, mark as padding as these are used to trigger Pion OnTrack only false, ) - d.pacer.Enqueue(&pacer.Packet{ + pacerPacket := pacer.PacketFactory.Get().(*pacer.Packet) + *pacerPacket = pacer.Packet{ Header: hdr, HeaderSize: headerSize, Payload: payload, @@ -2506,7 +2652,8 @@ func (d *DownTrack) sendSilentFrameOnMuteForOpus() { AbsSendTimeExtID: uint8(d.absSendTimeExtID), TransportWideExtID: uint8(d.transportWideExtID), WriteStream: d.writeStream, - }) + } + d.pacer.Enqueue(pacerPacket) } numFrames-- @@ -2516,14 +2663,13 @@ func (d *DownTrack) sendSilentFrameOnMuteForOpus() { func (d *DownTrack) HandleRTCPSenderReportData( _payloadType webrtc.PayloadType, - isSVC bool, layer int32, publisherSRData *livekit.RTCPSenderReportState, ) error { - d.forwarder.SetRefSenderReport(isSVC, layer, publisherSRData) + d.forwarder.SetRefSenderReport(layer, publisherSRData) - currentLayer, tsOffset, refSenderReport := d.forwarder.GetSenderReportParams() - if layer == currentLayer || (layer == 0 && isSVC) { + currentLayer, isSingleStream, tsOffset, refSenderReport := d.forwarder.GetSenderReportParams() + if layer == currentLayer || (layer == 0 && isSingleStream) { d.handleRTCPSenderReportData(refSenderReport, tsOffset) } return nil diff --git a/pkg/sfu/forwarder.go b/pkg/sfu/forwarder.go index 9aa39fe..4fac359 100644 --- a/pkg/sfu/forwarder.go +++ b/pkg/sfu/forwarder.go @@ -27,6 +27,7 @@ import ( "go.uber.org/zap/zapcore" "github.com/livekit/mediatransportutil" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" "github.com/livekit/protocol/utils" @@ -34,14 +35,13 @@ import ( "github.com/livekit/livekit-server/pkg/sfu/buffer" "github.com/livekit/livekit-server/pkg/sfu/codecmunger" - "github.com/livekit/livekit-server/pkg/sfu/mime" dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor" "github.com/livekit/livekit-server/pkg/sfu/rtpstats" + sfuutils "github.com/livekit/livekit-server/pkg/sfu/utils" "github.com/livekit/livekit-server/pkg/sfu/videolayerselector" "github.com/livekit/livekit-server/pkg/sfu/videolayerselector/temporallayerselector" ) -// Forwarder const ( FlagPauseOnDowngrade = true FlagFilterRTX = false @@ -54,6 +54,11 @@ const ( SwitchAheadThresholdSeconds = float64(0.025) // 25ms ) +var ( + errSkipStartOnOutOfOrderPacket = errors.New("skip start on out-of-order packet") + errSwitchPointTooFarBehind = errors.New("switch point too far behind") +) + // ------------------------------------------------------------------- type VideoPauseReason int @@ -180,6 +185,7 @@ func (v *VideoTransition) MarshalLogObject(e zapcore.ObjectEncoder) error { type TranslationParams struct { shouldDrop bool + isStarting bool isResuming bool isSwitching bool rtp TranslationParamsRTP @@ -209,13 +215,14 @@ func (r refInfo) MarshalLogObject(e zapcore.ObjectEncoder) error { // ------------------------------------------------------------------- type Forwarder struct { - lock sync.RWMutex - mime mime.MimeType - clockRate uint32 - kind webrtc.RTPCodecType - logger logger.Logger - skipReferenceTS bool - rtpStats *rtpstats.RTPStatsSender + lock sync.RWMutex + mime mime.MimeType + clockRate uint32 + kind webrtc.RTPCodecType + logger logger.Logger + skipReferenceTS bool + disableOpportunisticAllocation bool + rtpStats *rtpstats.RTPStatsSender muted bool pubMuted bool @@ -230,7 +237,8 @@ type Forwarder struct { referenceLayerSpatial int32 dummyStartTSOffset uint64 refInfos [buffer.DefaultMaxLayerSpatial + 1]refInfo - refIsSVC bool + refVideoLayerMode livekit.VideoLayer_Mode + isDDAvailable bool provisional *VideoAllocationProvisional @@ -247,20 +255,22 @@ func NewForwarder( kind webrtc.RTPCodecType, logger logger.Logger, skipReferenceTS bool, + disableOpportunisticAllocation bool, rtpStats *rtpstats.RTPStatsSender, ) *Forwarder { f := &Forwarder{ - mime: mime.MimeTypeUnknown, - kind: kind, - logger: logger, - skipReferenceTS: skipReferenceTS, - rtpStats: rtpStats, - referenceLayerSpatial: buffer.InvalidLayerSpatial, - lastAllocation: VideoAllocationDefault, - lastReferencePayloadType: -1, - rtpMunger: NewRTPMunger(logger), - vls: videolayerselector.NewNull(logger), - codecMunger: codecmunger.NewNull(logger), + mime: mime.MimeTypeUnknown, + kind: kind, + logger: logger, + skipReferenceTS: skipReferenceTS, + disableOpportunisticAllocation: disableOpportunisticAllocation, + rtpStats: rtpStats, + referenceLayerSpatial: buffer.InvalidLayerSpatial, + lastAllocation: VideoAllocationDefault, + lastReferencePayloadType: -1, + rtpMunger: NewRTPMunger(logger), + vls: videolayerselector.NewNull(logger), + codecMunger: codecmunger.NewNull(logger), } if f.kind == webrtc.RTPCodecTypeVideo { @@ -297,10 +307,14 @@ func (f *Forwarder) SetMaxTemporalLayerSeen(maxTemporalLayerSeen int32) bool { return true } -func (f *Forwarder) DetermineCodec(codec webrtc.RTPCodecCapability, extensions []webrtc.RTPHeaderExtensionParameter) { +func (f *Forwarder) DetermineCodec(codec webrtc.RTPCodecCapability, extensions []webrtc.RTPHeaderExtensionParameter, videoLayerMode livekit.VideoLayer_Mode) { f.lock.Lock() defer f.lock.Unlock() + if videoLayerMode == livekit.VideoLayer_ONE_SPATIAL_LAYER_PER_STREAM_INCOMPLETE_RTCP_SR { + f.skipReferenceTS = true + } + toMimeType := mime.NormalizeMimeType(codec.MimeType) codecChanged := f.mime != mime.MimeTypeUnknown && f.mime != toMimeType if codecChanged { @@ -308,6 +322,7 @@ func (f *Forwarder) DetermineCodec(codec webrtc.RTPCodecCapability, extensions [ } f.mime = toMimeType f.clockRate = codec.ClockRate + f.refVideoLayerMode = videoLayerMode ddAvailable := func(exts []webrtc.RTPHeaderExtensionParameter) bool { for _, ext := range exts { @@ -320,7 +335,7 @@ func (f *Forwarder) DetermineCodec(codec webrtc.RTPCodecCapability, extensions [ switch f.mime { case mime.MimeTypeVP8: - f.codecMunger = codecmunger.NewVP8FromNull(f.codecMunger, f.logger) + f.codecMunger = codecmunger.NewVP8FromOther(f.codecMunger, f.logger) if f.vls != nil { if vls := videolayerselector.NewSimulcastFromOther(f.vls); vls != nil { f.vls = vls @@ -333,47 +348,66 @@ func (f *Forwarder) DetermineCodec(codec webrtc.RTPCodecCapability, extensions [ f.vls.SetTemporalLayerSelector(temporallayerselector.NewVP8(f.logger)) case mime.MimeTypeH264, mime.MimeTypeH265: + f.codecMunger = codecmunger.NewNull(f.logger) if f.vls != nil { - f.vls = videolayerselector.NewSimulcastFromOther(f.vls) + if vls := videolayerselector.NewSimulcastFromOther(f.vls); vls != nil { + f.vls = vls + } else { + f.logger.Errorw("failed to create simulcast on codec change", nil) + } } else { f.vls = videolayerselector.NewSimulcast(f.logger) } case mime.MimeTypeVP9: - // DD-TODO : we only enable dd layer selector for av1/vp9 now, in the future we can enable it for vp8 too - isDDAvailable := ddAvailable(extensions) - if isDDAvailable { + f.codecMunger = codecmunger.NewNull(f.logger) + if sfuutils.IsSimulcastMode(videoLayerMode) { if f.vls != nil { - f.vls = videolayerselector.NewDependencyDescriptorFromOther(f.vls) + f.vls = videolayerselector.NewSimulcastFromOther(f.vls) } else { f.vls = videolayerselector.NewDependencyDescriptor(f.logger) } } else { - if f.vls != nil { - f.vls = videolayerselector.NewVP9FromOther(f.vls) + f.isDDAvailable = ddAvailable(extensions) + if f.isDDAvailable { + if f.vls != nil { + f.vls = videolayerselector.NewDependencyDescriptorFromOther(f.vls) + } else { + f.vls = videolayerselector.NewDependencyDescriptor(f.logger) + } } else { - f.vls = videolayerselector.NewVP9(f.logger) + if f.vls != nil { + f.vls = videolayerselector.NewVP9FromOther(f.vls) + } else { + f.vls = videolayerselector.NewVP9(f.logger) + } } } - // SVC-TODO: Support for VP9 simulcast. When DD is not available, have to pick selector based on VP9 SVC or Simulcast case mime.MimeTypeAV1: - // DD-TODO : we only enable dd layer selector for av1/vp9 now, in the future we can enable it for vp8 too - isDDAvailable := ddAvailable(extensions) - if isDDAvailable { - if f.vls != nil { - f.vls = videolayerselector.NewDependencyDescriptorFromOther(f.vls) - } else { - f.vls = videolayerselector.NewDependencyDescriptor(f.logger) - } - } else { + f.codecMunger = codecmunger.NewNull(f.logger) + if sfuutils.IsSimulcastMode(videoLayerMode) { if f.vls != nil { f.vls = videolayerselector.NewSimulcastFromOther(f.vls) } else { f.vls = videolayerselector.NewSimulcast(f.logger) } + } else { + f.isDDAvailable = ddAvailable(extensions) + if f.isDDAvailable { + if f.vls != nil { + f.vls = videolayerselector.NewDependencyDescriptorFromOther(f.vls) + } else { + f.vls = videolayerselector.NewDependencyDescriptor(f.logger) + } + } else { + if f.vls != nil { + f.vls = videolayerselector.NewSimulcastFromOther(f.vls) + } else { + f.vls = videolayerselector.NewSimulcast(f.logger) + } + } } - // SVC-TODO: Support for AV1 Simulcast } } @@ -460,7 +494,12 @@ func (f *Forwarder) Mute(muted bool, isSubscribeMutable bool) bool { // It could result in some bandwidth consumed for stream without visibility in // the case of intentional mute. if muted && !isSubscribeMutable { - f.logger.Debugw("ignoring forwarder mute, paused due to congestion") + f.logger.Infow( + "ignoring forwarder mute, paused due to congestion", + "targetLayers", f.vls.GetTarget(), + "currentLayers", f.vls.GetCurrent(), + "lastAllocation", f.lastAllocation, + ) return false } @@ -529,6 +568,41 @@ func (f *Forwarder) SetMaxSpatialLayer(spatialLayer int32) (bool, buffer.VideoLa f.logger.Debugw("setting max spatial layer", "layer", spatialLayer) f.vls.SetMaxSpatial(spatialLayer) + if f.disableOpportunisticAllocation { + return true, f.vls.GetMax() + } + + if f.vls.GetTarget().Spatial != buffer.InvalidLayerSpatial || + f.isDeficientLocked() || + f.lastAllocation.PauseReason == VideoPauseReasonMuted || + f.lastAllocation.PauseReason == VideoPauseReasonPubMuted { + return true, f.vls.GetMax() + } + + f.logger.Debugw("opportunistically setting target spatial layer", "layer", spatialLayer) + + alloc := f.lastAllocation + + // bitrates are not known + alloc.BandwidthRequested = 0 + alloc.BandwidthDelta = 0 + alloc.Bitrates = Bitrates{} + + alloc.TargetLayer = f.vls.GetMax() + alloc.RequestLayerSpatial = f.vls.GetMax().Spatial + alloc.MaxLayer = f.vls.GetMax() + + alloc.DistanceToDesired = getDistanceToDesired( + f.muted, + f.pubMuted, + f.vls.GetMaxSeen(), + nil, + alloc.Bitrates, + alloc.TargetLayer, + f.vls.GetMax(), + ) + + f.updateAllocation(alloc, "opportunistic") return true, f.vls.GetMax() } @@ -577,14 +651,10 @@ func (f *Forwarder) GetMaxSubscribedSpatial() int32 { layer := buffer.InvalidLayerSpatial // covers muted case if !f.muted { - layer = f.vls.GetMax().Spatial - // If current is higher, mark the current layer as max subscribed layer // to prevent the current layer from stopping before forwarder switches // to the new and lower max layer, - if layer < f.vls.GetCurrent().Spatial { - layer = f.vls.GetCurrent().Spatial - } + layer = max(f.vls.GetMax().Spatial, f.vls.GetCurrent().Spatial) // if reference layer is higher, hold there until an RTCP Sender Report from // publisher is available as that is used for reference time stamp between layers. @@ -612,21 +682,24 @@ func (f *Forwarder) getRefLayer() (int32, int32) { return buffer.InvalidLayerSpatial, buffer.InvalidLayerSpatial } - if f.refIsSVC { + if f.refVideoLayerMode == livekit.VideoLayer_MULTIPLE_SPATIAL_LAYERS_PER_STREAM { return 0, currentLayerSpatial } return currentLayerSpatial, currentLayerSpatial } -func (f *Forwarder) SetRefSenderReport(isSVC bool, layer int32, srData *livekit.RTCPSenderReportState) { +func (f *Forwarder) SetRefSenderReport(layer int32, srData *livekit.RTCPSenderReportState) { f.lock.Lock() defer f.lock.Unlock() - f.refIsSVC = isSVC if layer >= 0 && int(layer) < len(f.refInfos) { if layer == f.referenceLayerSpatial && f.refInfos[layer].senderReport == nil { - f.logger.Debugw("received RTCP sender report for reference layer spatial", "layer", layer) + f.logger.Debugw( + "received RTCP sender report for reference layer spatial", + "layer", layer, + "srData", rtpstats.WrappedRTCPSenderReportStateLogger{RTCPSenderReportState: srData}, + ) } f.refInfos[layer] = refInfo{srData, 0, false} @@ -641,7 +714,7 @@ func (f *Forwarder) SetRefSenderReport(isSVC bool, layer int32, srData *livekit. // // It could happen like this // 1. Normal operation: publisher sending sender reports and - // suscribers use reports from publisher to calculate and send + // subscribers use reports from publisher to calculate and send // RTCP sender report. // 2. Publisher pauses: there are no more reports. // 3. When paused, subscriber can still use the publisher side sender @@ -665,7 +738,7 @@ func (f *Forwarder) SetRefSenderReport(isSVC bool, layer int32, srData *livekit. } } -func (f *Forwarder) GetSenderReportParams() (int32, uint64, *livekit.RTCPSenderReportState) { +func (f *Forwarder) GetSenderReportParams() (int32, bool, uint64, *livekit.RTCPSenderReportState) { f.lock.RLock() defer f.lock.RUnlock() @@ -673,10 +746,10 @@ func (f *Forwarder) GetSenderReportParams() (int32, uint64, *livekit.RTCPSenderR if refLayer == buffer.InvalidLayerSpatial || f.refInfos[refLayer].senderReport == nil || !f.refInfos[refLayer].isTSOffsetValid { - return buffer.InvalidLayerSpatial, 0, nil + return buffer.InvalidLayerSpatial, false, 0, nil } - return currentLayerSpatial, f.refInfos[refLayer].tsOffset, f.refInfos[refLayer].senderReport + return currentLayerSpatial, f.refVideoLayerMode == livekit.VideoLayer_MULTIPLE_SPATIAL_LAYERS_PER_STREAM, f.refInfos[refLayer].tsOffset, f.refInfos[refLayer].senderReport } func (f *Forwarder) isDeficientLocked() bool { @@ -896,18 +969,24 @@ func (f *Forwarder) ProvisionalAllocatePrepare(availableLayers []int32, bitrates f.lock.Lock() defer f.lock.Unlock() - f.provisional = &VideoAllocationProvisional{ - allocatedLayer: buffer.InvalidLayer, - muted: f.muted, - pubMuted: f.pubMuted, - maxSeenLayer: f.vls.GetMaxSeen(), - bitrates: bitrates, - maxLayer: f.vls.GetMax(), - currentLayer: f.vls.GetCurrent(), + if f.provisional == nil { + f.provisional = &VideoAllocationProvisional{} } - f.provisional.availableLayers = make([]int32, len(availableLayers)) - copy(f.provisional.availableLayers, availableLayers) + p := f.provisional + p.allocatedLayer = buffer.InvalidLayer + p.muted = f.muted + p.pubMuted = f.pubMuted + p.maxSeenLayer = f.vls.GetMaxSeen() + p.bitrates = bitrates + p.maxLayer = f.vls.GetMax() + p.currentLayer = f.vls.GetCurrent() + if cap(p.availableLayers) >= len(availableLayers) { + p.availableLayers = p.availableLayers[:len(availableLayers)] + } else { + p.availableLayers = make([]int32, len(availableLayers)) + } + copy(p.availableLayers, availableLayers) } func (f *Forwarder) ProvisionalAllocateReset() { @@ -1523,7 +1602,11 @@ func (f *Forwarder) updateAllocation(alloc VideoAllocation, reason string) Video alloc.PauseReason != f.lastAllocation.PauseReason || alloc.TargetLayer != f.lastAllocation.TargetLayer || alloc.RequestLayerSpatial != f.lastAllocation.RequestLayerSpatial { - f.logger.Debugw(fmt.Sprintf("stream allocation: %s", reason), "allocation", &alloc) + f.logger.Debugw( + fmt.Sprintf("stream allocation: %s", reason), + "allocation", &alloc, + "lastAllocation", &f.lastAllocation, + ) } f.lastAllocation = alloc @@ -1563,18 +1646,7 @@ func (f *Forwarder) CheckSync() (bool, int32) { f.lock.RLock() defer f.lock.RUnlock() - locked, layer := f.vls.CheckSync() - if !locked { - return locked, layer - } - - // max published layer (as seen by this forwarder) could be - // lower than max subscribed, mark de-synced if not deficient - if !f.isDeficientLocked() && f.vls.GetMax().Spatial > f.vls.GetTarget().Spatial { - return false, layer - } - - return true, layer + return f.vls.CheckSync() } func (f *Forwarder) Restart() { @@ -1586,11 +1658,11 @@ func (f *Forwarder) Restart() { f.referenceLayerSpatial = buffer.InvalidLayerSpatial f.lastReferencePayloadType = -1 - for layer := 0; layer < len(f.refInfos); layer++ { + for layer := range len(f.refInfos) { f.refInfos[layer] = refInfo{} } f.lastSwitchExtIncomingTS = 0 - f.refIsSVC = false + f.refVideoLayerMode = livekit.VideoLayer_MODE_UNUSED } func (f *Forwarder) FilterRTX(nacks []uint16) (filtered []uint16, disallowedLayers [buffer.DefaultMaxLayerSpatial + 1]bool) { @@ -1614,7 +1686,7 @@ func (f *Forwarder) FilterRTX(nacks []uint16) (filtered []uint16, disallowedLaye if FlagFilterRTXLayers { currentLayer := f.vls.GetCurrent() targetLayer := f.vls.GetTarget() - for layer := int32(0); layer < buffer.DefaultMaxLayerSpatial+1; layer++ { + for layer := range buffer.DefaultMaxLayerSpatial + 1 { if f.isDeficientLocked() && (targetLayer.Spatial < currentLayer.Spatial || layer > currentLayer.Spatial) { disallowedLayers[layer] = true } @@ -1643,7 +1715,7 @@ func (f *Forwarder) GetTranslationParams(extPkt *buffer.ExtPacket, layer int32) return TranslationParams{ shouldDrop: true, - }, ErrUnknownKind + }, errUnknownKind } func (f *Forwarder) getRefLayerRTPTimestamp(ts uint32, refLayer, targetLayer int32) (uint32, error) { @@ -1651,7 +1723,7 @@ func (f *Forwarder) getRefLayerRTPTimestamp(ts uint32, refLayer, targetLayer int return 0, fmt.Errorf("invalid layer(s), refLayer: %d, targetLayer: %d", refLayer, targetLayer) } - if refLayer == targetLayer || f.refIsSVC { + if refLayer == targetLayer || f.refVideoLayerMode == livekit.VideoLayer_MULTIPLE_SPATIAL_LAYERS_PER_STREAM { return ts, nil } @@ -1676,8 +1748,13 @@ func (f *Forwarder) getRefLayerRTPTimestamp(ts uint32, refLayer, targetLayer int return ts + offset, nil } -func (f *Forwarder) processSourceSwitch(extPkt *buffer.ExtPacket, layer int32) error { +func (f *Forwarder) processSourceSwitch(extPkt *buffer.ExtPacket, layer int32) (bool, error) { + starting := false if !f.started { + if extPkt.IsOutOfOrder { + return starting, errSkipStartOnOutOfOrderPacket + } + f.started = true f.referenceLayerSpatial = layer f.rtpMunger.SetLastSnTs(extPkt) @@ -1691,8 +1768,13 @@ func (f *Forwarder) processSourceSwitch(extPkt *buffer.ExtPacket, layer int32) e "layer", layer, "referenceLayerSpatial", f.referenceLayerSpatial, ) - return nil + starting = true + return starting, nil } else if f.referenceLayerSpatial == buffer.InvalidLayerSpatial { + if extPkt.IsOutOfOrder { + return starting, errSkipStartOnOutOfOrderPacket + } + f.referenceLayerSpatial = layer f.codecMunger.SetLast(extPkt) f.logger.Debugw( @@ -1704,6 +1786,8 @@ func (f *Forwarder) processSourceSwitch(extPkt *buffer.ExtPacket, layer int32) e "layer", layer, "referenceLayerSpatial", f.referenceLayerSpatial, ) + + starting = true } logTransition := func(message string, extExpectedTS, extRefTS, extLastTS uint64, diffSeconds float64) { @@ -1722,23 +1806,6 @@ func (f *Forwarder) processSourceSwitch(extPkt *buffer.ExtPacket, layer int32) e "rtpStats", f.rtpStats, ) } - // TODO-REMOVE-AFTER-DATA-COLLECTION - logTransitionInfo := func(message string, extExpectedTS, extRefTS, extLastTS uint64, diffSeconds float64) { - f.logger.Infow( - message, - "layer", layer, - "referenceLayerSpatial", f.referenceLayerSpatial, - "extExpectedTS", extExpectedTS, - "incomingTS", extPkt.Packet.Timestamp, - "extIncomingTS", extPkt.ExtTimestamp, - "extRefTS", extRefTS, - "extLastTS", extLastTS, - "diffSeconds", math.Abs(diffSeconds), - "refInfos", logger.ObjectSlice(f.refInfos[:]), - "lastSwitchExtIncomingTS", f.lastSwitchExtIncomingTS, - "rtpStats", f.rtpStats, - ) - } // Compute how much time passed between the previous forwarded packet // and the current incoming (to be forwarded) packet and calculate @@ -1750,8 +1817,7 @@ func (f *Forwarder) processSourceSwitch(extPkt *buffer.ExtPacket, layer int32) e // 3. extExpectedTS -> expected timestamp of this packet calculated based on elapsed time since first packet // Ideally, extRefTS and extExpectedTS should be very close and extLastTS should be before both of those. // But, cases like muting/unmuting, clock vagaries, pacing, etc. make them not satisfy those conditions always. - rtpMungerState := f.rtpMunger.GetState() - extLastTS := rtpMungerState.ExtLastTimestamp + extLastTS := f.rtpMunger.GetExtLastTimestamp() extExpectedTS := extLastTS extRefTS := extLastTS refTS := uint32(extRefTS) @@ -1771,7 +1837,7 @@ func (f *Forwarder) processSourceSwitch(extPkt *buffer.ExtPacket, layer int32) e "layer", layer, "error", err, ) - return err + return starting, err } } @@ -1818,7 +1884,6 @@ func (f *Forwarder) processSourceSwitch(extPkt *buffer.ExtPacket, layer int32) e } } - bigJump := false var extNextTS uint64 if f.lastSSRC == 0 { // If resuming (e. g. on unmute), keep next timestamp close to expected timestamp. @@ -1844,14 +1909,12 @@ func (f *Forwarder) processSourceSwitch(extPkt *buffer.ExtPacket, layer int32) e diffSeconds := float64(int64(extExpectedTS-extRefTS)) / float64(f.clockRate) if diffSeconds >= 0.0 { if f.resumeBehindThreshold > 0 && diffSeconds > f.resumeBehindThreshold { - logTransitionInfo("resume, reference too far behind", extExpectedTS, extRefTS, extLastTS, diffSeconds) + logTransition("resume, reference too far behind", extExpectedTS, extRefTS, extLastTS, diffSeconds) extNextTS = extExpectedTS - bigJump = true } else if diffSeconds > ResumeBehindHighThresholdSeconds { // could be due to incoming time stamp lagging a lot, like an unpause of the track - logTransitionInfo("resume, reference very far behind", extExpectedTS, extRefTS, extLastTS, diffSeconds) + logTransition("resume, reference very far behind", extExpectedTS, extRefTS, extLastTS, diffSeconds) extNextTS = extExpectedTS - bigJump = true } else { extNextTS = extRefTS } @@ -1872,7 +1935,7 @@ func (f *Forwarder) processSourceSwitch(extPkt *buffer.ExtPacket, layer int32) e // (like "have waited for too long for layer switch, nothing available, switch to whatever is available" kind of condition). logTransition("layer switch, reference too far behind", extExpectedTS, extRefTS, extLastTS, diffSeconds) - return errors.New("switch point too far behind") + return starting, errSwitchPointTooFarBehind } // use a nominal increase to ensure that timestamp is always moving forward @@ -1893,54 +1956,37 @@ func (f *Forwarder) processSourceSwitch(extPkt *buffer.ExtPacket, layer int32) e // nominal increase extNextTS = extLastTS + 1 } - if bigJump { // TODO-REMOVE-AFTER-DATA-COLLECTION - f.logger.Infow( - "next timestamp on switch", - "switchingAt", switchingAt, - "layer", layer, - "extLastTS", extLastTS, - "lastMarker", rtpMungerState.LastMarker, - "extRefTS", extRefTS, - "dummyStartTSOffset", f.dummyStartTSOffset, - "referenceLayerSpatial", f.referenceLayerSpatial, - "extExpectedTS", extExpectedTS, - "extNextTS", extNextTS, - "tsJump", extNextTS-extLastTS, - "nextSN", rtpMungerState.ExtLastSequenceNumber+1, - "extIncomingSN", extPkt.ExtSequenceNumber, - "incomingTS", extPkt.Packet.Timestamp, - "extIncomingTS", extPkt.ExtTimestamp, - "rtpStats", f.rtpStats, - ) - } else { - f.logger.Debugw( - "next timestamp on switch", - "switchingAt", switchingAt, - "layer", layer, - "extLastTS", extLastTS, - "lastMarker", rtpMungerState.LastMarker, - "extRefTS", extRefTS, - "dummyStartTSOffset", f.dummyStartTSOffset, - "referenceLayerSpatial", f.referenceLayerSpatial, - "extExpectedTS", extExpectedTS, - "extNextTS", extNextTS, - "tsJump", extNextTS-extLastTS, - "nextSN", rtpMungerState.ExtLastSequenceNumber+1, - "extIncomingSN", extPkt.ExtSequenceNumber, - "extIncomingTS", extPkt.ExtTimestamp, - "rtpStats", f.rtpStats, - ) - } + f.logger.Debugw( + "next timestamp on switch", + "switchingAt", switchingAt, + "layer", layer, + "extLastTS", extLastTS, + "lastMarker", f.rtpMunger.GetState().LastMarker, + "extRefTS", extRefTS, + "dummyStartTSOffset", f.dummyStartTSOffset, + "referenceLayerSpatial", f.referenceLayerSpatial, + "extExpectedTS", extExpectedTS, + "extNextTS", extNextTS, + "tsJump", extNextTS-extLastTS, + "nextSN", f.rtpMunger.GetState().ExtLastSequenceNumber+1, + "extIncomingSN", extPkt.ExtSequenceNumber, + "extIncomingTS", extPkt.ExtTimestamp, + "rtpStats", f.rtpStats, + ) f.rtpMunger.UpdateSnTsOffsets(extPkt, 1, extNextTS-extLastTS) f.codecMunger.UpdateOffsets(extPkt) - return nil + return starting, nil } // should be called with lock held -func (f *Forwarder) getTranslationParamsCommon(extPkt *buffer.ExtPacket, layer int32, tp *TranslationParams) error { +func (f *Forwarder) getTranslationParamsCommon(extPkt *buffer.ExtPacket, layer int32, tp *TranslationParams) (bool, error) { + var ( + starting bool + err error + ) if f.lastSSRC != extPkt.Packet.SSRC { - if err := f.processSourceSwitch(extPkt, layer); err != nil { + if starting, err = f.processSourceSwitch(extPkt, layer); err != nil { f.logger.Debugw( "could not switch feed", "error", err, @@ -1954,7 +2000,7 @@ func (f *Forwarder) getTranslationParamsCommon(extPkt *buffer.ExtPacket, layer i ) tp.shouldDrop = true f.vls.Rollback() - return nil + return starting, nil } f.logger.Debugw( "switching feed", @@ -1977,29 +2023,30 @@ func (f *Forwarder) getTranslationParamsCommon(extPkt *buffer.ExtPacket, layer i tpRTP, err := f.rtpMunger.UpdateAndGetSnTs(extPkt, tp.marker) if err != nil { tp.shouldDrop = true - if err == ErrPaddingOnlyPacket || err == ErrDuplicatePacket || err == ErrOutOfOrderSequenceNumberCacheMiss { - return nil + if err == errPaddingOnlyPacket || err == errDuplicatePacket || err == errOutOfOrderSequenceNumberCacheMiss { + return starting, nil } - return err + return starting, err } tp.rtp = tpRTP if len(extPkt.Packet.Payload) > 0 { - return f.translateCodecHeader(extPkt, tp) + return starting, f.translateCodecHeader(extPkt, tp) } - return nil + return starting, nil } // should be called with lock held func (f *Forwarder) getTranslationParamsAudio(extPkt *buffer.ExtPacket, layer int32) (TranslationParams, error) { tp := TranslationParams{} - if err := f.getTranslationParamsCommon(extPkt, layer, &tp); err != nil { + starting, err := f.getTranslationParamsCommon(extPkt, layer, &tp) + tp.isStarting = starting + if err != nil { tp.shouldDrop = true - return tp, err } - return tp, nil + return tp, err } // should be called with lock held @@ -2013,6 +2060,24 @@ func (f *Forwarder) getTranslationParamsVideo(extPkt *buffer.ExtPacket, layer in result := f.vls.Select(extPkt, layer) if !result.IsSelected { + if f.isDDAvailable && extPkt.DependencyDescriptor == nil { + f.logger.Infow( + "turning off dependency descriptor", + "layer", layer, + "refInfos", logger.ObjectSlice(f.refInfos[:]), + "lastSwitchExtIncomingTS", f.lastSwitchExtIncomingTS, + "currentLayer", f.vls.GetCurrent(), + "targetLayer", f.vls.GetCurrent(), + "maxLayer", f.vls.GetMax(), + ) + f.isDDAvailable = false + switch f.mime { + case mime.MimeTypeVP9: + f.vls = videolayerselector.NewVP9FromOther(f.vls) + case mime.MimeTypeAV1: + f.vls = videolayerselector.NewSimulcastFromOther(f.vls) + } + } tp.shouldDrop = true if f.started && result.IsRelevant { // call to update highest incoming sequence number and other internal structures @@ -2029,7 +2094,8 @@ func (f *Forwarder) getTranslationParamsVideo(extPkt *buffer.ExtPacket, layer in tp.ddBytes = result.DependencyDescriptorExtension tp.marker = result.RTPMarker - err := f.getTranslationParamsCommon(extPkt, layer, &tp) + starting, err := f.getTranslationParamsCommon(extPkt, layer, &tp) + tp.isStarting = starting if tp.shouldDrop { return tp, err } @@ -2117,7 +2183,7 @@ func (f *Forwarder) maybeStart() { ) } -func (f *Forwarder) GetSnTsForPadding(num int, forceMarker bool) ([]SnTs, error) { +func (f *Forwarder) GetSnTsForPadding(num int, frameRate uint32, forceMarker bool) ([]SnTs, error) { f.lock.Lock() defer f.lock.Unlock() @@ -2131,7 +2197,13 @@ func (f *Forwarder) GetSnTsForPadding(num int, forceMarker bool) ([]SnTs, error) if !f.vls.GetTarget().IsValid() { forceMarker = true } - return f.rtpMunger.UpdateAndGetPaddingSnTs(num, 0, 0, forceMarker, 0) + return f.rtpMunger.UpdateAndGetPaddingSnTs( + num, + f.clockRate, + frameRate, + forceMarker, + f.rtpMunger.GetExtLastTimestamp(), + ) } func (f *Forwarder) GetSnTsForBlankFrames(frameRate uint32, numPackets int) ([]SnTs, bool, error) { @@ -2145,7 +2217,7 @@ func (f *Forwarder) GetSnTsForBlankFrames(frameRate uint32, numPackets int) ([]S numPackets++ } - extLastTS := f.rtpMunger.GetState().ExtLastTimestamp + extLastTS := f.rtpMunger.GetExtLastTimestamp() extExpectedTS := extLastTS if f.rtpStats != nil { tsExt, err := f.rtpStats.GetExpectedRTPTimestamp(mono.Now()) @@ -2156,7 +2228,13 @@ func (f *Forwarder) GetSnTsForBlankFrames(frameRate uint32, numPackets int) ([]S if int64(extExpectedTS-extLastTS) <= 0 { extExpectedTS = extLastTS + 1 } - snts, err := f.rtpMunger.UpdateAndGetPaddingSnTs(numPackets, f.clockRate, frameRate, frameEndNeeded, extExpectedTS) + snts, err := f.rtpMunger.UpdateAndGetPaddingSnTs( + numPackets, + f.clockRate, + frameRate, + frameEndNeeded, + extExpectedTS, + ) return snts, frameEndNeeded, err } @@ -2167,7 +2245,7 @@ func (f *Forwarder) GetPadding(frameEndNeeded bool) ([]byte, error) { return f.codecMunger.UpdateAndGetPadding(!frameEndNeeded) } -func (f *Forwarder) RTPMungerDebugInfo() map[string]interface{} { +func (f *Forwarder) RTPMungerDebugInfo() map[string]any { f.lock.RLock() defer f.lock.RUnlock() diff --git a/pkg/sfu/forwarder_test.go b/pkg/sfu/forwarder_test.go index 722c767..cca1e69 100644 --- a/pkg/sfu/forwarder_test.go +++ b/pkg/sfu/forwarder_test.go @@ -20,6 +20,8 @@ import ( "github.com/pion/webrtc/v4" "github.com/stretchr/testify/require" + "github.com/livekit/mediatransportutil/pkg/codec" + "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" "github.com/livekit/livekit-server/pkg/sfu/buffer" @@ -32,8 +34,14 @@ func disable(f *Forwarder) { } func newForwarder(codec webrtc.RTPCodecCapability, kind webrtc.RTPCodecType) *Forwarder { - f := NewForwarder(kind, logger.GetLogger(), true, nil) - f.DetermineCodec(codec, nil) + f := NewForwarder( + kind, + logger.GetLogger(), + true, // skipReferenceTS + true, // disableOpportunisticAllocation + nil, + ) + f.DetermineCodec(codec, nil, livekit.VideoLayer_MODE_UNUSED) return f } @@ -463,6 +471,9 @@ func TestForwarderProvisionalAllocate(t *testing.T) { f.SetMaxPublishedLayer(buffer.DefaultMaxLayerSpatial) f.SetMaxTemporalLayerSeen(buffer.DefaultMaxLayerTemporal) + // Reset to invalid layers for testing allocation from scratch + disable(f) + bitrates := Bitrates{ {1, 2, 3, 4}, {5, 6, 7, 8}, @@ -681,6 +692,9 @@ func TestForwarderProvisionalAllocateMute(t *testing.T) { f.SetMaxSpatialLayer(buffer.DefaultMaxLayerSpatial) f.SetMaxTemporalLayer(buffer.DefaultMaxLayerTemporal) + // Reset to invalid layers for testing muted state + disable(f) + bitrates := Bitrates{ {1, 2, 3, 4}, {5, 6, 7, 8}, @@ -722,6 +736,9 @@ func TestForwarderProvisionalAllocateGetCooperativeTransition(t *testing.T) { f.SetMaxPublishedLayer(buffer.DefaultMaxLayerSpatial) f.SetMaxTemporalLayerSeen(buffer.DefaultMaxLayerTemporal) + // Reset to invalid layers for testing cooperative transition from scratch + disable(f) + availableLayers := []int32{0, 1, 2} bitrates := Bitrates{ {1, 2, 3, 4}, @@ -1272,22 +1289,42 @@ func TestForwarderGetTranslationParamsAudio(t *testing.T) { f := newForwarder(testutils.TestOpusCodec, webrtc.RTPCodecTypeAudio) params := &testutils.TestExtPacketParams{ + SequenceNumber: 23332, + Timestamp: 0xabcdef, + SSRC: 0x12345678, + PayloadSize: 20, + IsOutOfOrder: true, + } + extPkt, _ := testutils.GetTestExtPacket(params) + + // should not start on an out-of-order packet + expectedTP := TranslationParams{ + shouldDrop: true, + } + actualTP, err := f.GetTranslationParams(extPkt, 0) + require.NoError(t, err) + require.Equal(t, expectedTP, actualTP) + require.False(t, f.started) + require.Zero(t, f.lastSSRC) + + params = &testutils.TestExtPacketParams{ SequenceNumber: 23333, Timestamp: 0xabcdef, SSRC: 0x12345678, PayloadSize: 20, } - extPkt, _ := testutils.GetTestExtPacket(params) + extPkt, _ = testutils.GetTestExtPacket(params) - // should lock onto the first packet - expectedTP := TranslationParams{ + // should lock onto the first in-order packet + expectedTP = TranslationParams{ + isStarting: true, rtp: TranslationParamsRTP{ snOrdering: SequenceNumberOrderingContiguous, extSequenceNumber: 23333, extTimestamp: 0xabcdef, }, } - actualTP, err := f.GetTranslationParams(extPkt, 0) + actualTP, err = f.GetTranslationParams(extPkt, 0) require.NoError(t, err) require.Equal(t, expectedTP, actualTP) require.True(t, f.started) @@ -1436,13 +1473,14 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { f := newForwarder(testutils.TestVP8Codec, webrtc.RTPCodecTypeVideo) params := &testutils.TestExtPacketParams{ - SequenceNumber: 23333, + SequenceNumber: 23332, Timestamp: 0xabcdef, SSRC: 0x12345678, PayloadSize: 20, - SetMarker: true, + Marker: true, + IsOutOfOrder: true, } - vp8 := &buffer.VP8{ + vp8 := &codec.VP8{ FirstByte: 25, I: true, M: true, @@ -1459,13 +1497,47 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { } extPkt, _ := testutils.GetTestExtPacketVP8(params, vp8) - // no target layers, should drop + // should not start on an out-of-order packet expectedTP := TranslationParams{ shouldDrop: true, } actualTP, err := f.GetTranslationParams(extPkt, 0) require.NoError(t, err) require.Equal(t, expectedTP, actualTP) + require.False(t, f.started) + require.Zero(t, f.lastSSRC) + + params = &testutils.TestExtPacketParams{ + SequenceNumber: 23333, + Timestamp: 0xabcdef, + SSRC: 0x12345678, + PayloadSize: 20, + Marker: true, + } + vp8 = &codec.VP8{ + FirstByte: 25, + I: true, + M: true, + PictureID: 13467, + L: true, + TL0PICIDX: 233, + T: true, + TID: 0, + Y: true, + K: true, + KEYIDX: 23, + HeaderSize: 6, + IsKeyFrame: false, + } + extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8) + + // no target layers, should drop + expectedTP = TranslationParams{ + shouldDrop: true, + } + actualTP, err = f.GetTranslationParams(extPkt, 0) + require.NoError(t, err) + require.Equal(t, expectedTP, actualTP) // although target layer matches, not a key frame, so should drop f.vls.SetTarget(buffer.VideoLayer{ @@ -1480,7 +1552,7 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { require.Equal(t, expectedTP, actualTP) // should lock onto packet (key frame) - vp8 = &buffer.VP8{ + vp8 = &codec.VP8{ FirstByte: 25, I: true, M: true, @@ -1496,7 +1568,7 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { IsKeyFrame: true, } extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8) - expectedVP8 := &buffer.VP8{ + expectedVP8 := &codec.VP8{ FirstByte: 25, I: true, M: true, @@ -1511,8 +1583,9 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { HeaderSize: 6, IsKeyFrame: true, } - marshalledVP8, err := expectedVP8.Marshal() + marshalledVP8, _ := expectedVP8.Marshal() expectedTP = TranslationParams{ + isStarting: true, isSwitching: true, isResuming: true, rtp: TranslationParamsRTP{ @@ -1576,7 +1649,7 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { PayloadSize: 20, } extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8) - expectedVP8 = &buffer.VP8{ + expectedVP8 = &codec.VP8{ FirstByte: 25, I: true, M: true, @@ -1613,7 +1686,7 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { SSRC: 0x12345678, PayloadSize: 20, } - vp8 = &buffer.VP8{ + vp8 = &codec.VP8{ FirstByte: 25, S: true, I: true, @@ -1630,7 +1703,7 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { IsKeyFrame: true, } extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8) - expectedVP8 = &buffer.VP8{ + expectedVP8 = &codec.VP8{ FirstByte: 25, I: true, M: true, @@ -1667,7 +1740,7 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { SSRC: 0x12345678, PayloadSize: 20, } - vp8 = &buffer.VP8{ + vp8 = &codec.VP8{ FirstByte: 25, I: true, M: true, @@ -1702,7 +1775,7 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { SSRC: 0x12345678, PayloadSize: 20, } - vp8 = &buffer.VP8{ + vp8 = &codec.VP8{ FirstByte: 25, I: true, M: true, @@ -1718,7 +1791,7 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { IsKeyFrame: false, } extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8) - expectedVP8 = &buffer.VP8{ + expectedVP8 = &codec.VP8{ FirstByte: 25, I: true, M: true, @@ -1799,7 +1872,7 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { SSRC: 0x87654321, PayloadSize: 20, } - vp8 = &buffer.VP8{ + vp8 = &codec.VP8{ FirstByte: 25, I: true, M: false, @@ -1816,7 +1889,7 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) { } extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8) - expectedVP8 = &buffer.VP8{ + expectedVP8 = &codec.VP8{ FirstByte: 25, I: true, M: true, @@ -1858,7 +1931,7 @@ func TestForwarderGetSnTsForPadding(t *testing.T) { SSRC: 0x12345678, PayloadSize: 20, } - vp8 := &buffer.VP8{ + vp8 := &codec.VP8{ FirstByte: 25, I: true, M: true, @@ -1888,14 +1961,14 @@ func TestForwarderGetSnTsForPadding(t *testing.T) { disable(f) // should get back frame end needed as the last packet did not have RTP marker set - snts, err := f.GetSnTsForPadding(5, false) + snts, err := f.GetSnTsForPadding(5, 0, false) require.NoError(t, err) numPadding := 5 clockRate := uint32(0) frameRate := uint32(5) var sntsExpected = make([]SnTs, numPadding) - for i := 0; i < numPadding; i++ { + for i := range numPadding { sntsExpected[i] = SnTs{ extSequenceNumber: 23333 + uint64(i) + 1, extTimestamp: 0xabcdef + (uint64(i)*uint64(clockRate))/uint64(frameRate), @@ -1904,10 +1977,10 @@ func TestForwarderGetSnTsForPadding(t *testing.T) { require.Equal(t, sntsExpected, snts) // now that there is a marker, timestamp should jump on first padding when asked again - snts, err = f.GetSnTsForPadding(numPadding, false) + snts, err = f.GetSnTsForPadding(numPadding, 0, false) require.NoError(t, err) - for i := 0; i < numPadding; i++ { + for i := range numPadding { sntsExpected[i] = SnTs{ extSequenceNumber: 23338 + uint64(i) + 1, extTimestamp: 0xabcdef + (uint64(i+1)*uint64(clockRate))/uint64(frameRate), @@ -1925,7 +1998,7 @@ func TestForwarderGetSnTsForBlankFrames(t *testing.T) { SSRC: 0x12345678, PayloadSize: 20, } - vp8 := &buffer.VP8{ + vp8 := &codec.VP8{ FirstByte: 25, I: true, M: true, @@ -2002,7 +2075,7 @@ func TestForwarderGetPaddingVP8(t *testing.T) { SSRC: 0x12345678, PayloadSize: 20, } - vp8 := &buffer.VP8{ + vp8 := &codec.VP8{ FirstByte: 25, I: true, M: true, @@ -2029,7 +2102,7 @@ func TestForwarderGetPaddingVP8(t *testing.T) { _, _ = f.GetTranslationParams(extPkt, 0) // getting padding with frame end needed, should repeat the last picture id - expectedVP8 := buffer.VP8{ + expectedVP8 := codec.VP8{ FirstByte: 16, I: true, M: true, @@ -2051,7 +2124,7 @@ func TestForwarderGetPaddingVP8(t *testing.T) { require.Equal(t, marshalledVP8, buf) // getting padding with no frame end needed, should get next picture id - expectedVP8 = buffer.VP8{ + expectedVP8 = codec.VP8{ FirstByte: 16, I: true, M: true, diff --git a/pkg/sfu/forwardstats.go b/pkg/sfu/forwardstats.go index f594b41..dd41d0b 100644 --- a/pkg/sfu/forwardstats.go +++ b/pkg/sfu/forwardstats.go @@ -1,27 +1,33 @@ package sfu import ( - "fmt" "sync" "time" - "go.uber.org/atomic" - "github.com/livekit/livekit-server/pkg/telemetry/prometheus" "github.com/livekit/protocol/logger" "github.com/livekit/protocol/utils" + "github.com/livekit/protocol/utils/mono" +) + +const ( + cHighForwardingLatency = 20 * time.Millisecond + cSkewFactor = 10 ) type ForwardStats struct { lock sync.Mutex - lastLeftNano atomic.Int64 latency *utils.LatencyAggregate + lowest int64 + highest int64 + lastUpdateAt int64 closeCh chan struct{} } func NewForwardStats(latencyUpdateInterval, reportInterval, latencyWindowLength time.Duration) *ForwardStats { s := &ForwardStats{ latency: utils.NewLatencyAggregate(latencyUpdateInterval, latencyWindowLength), + lowest: time.Second.Nanoseconds(), closeCh: make(chan struct{}), } @@ -29,43 +35,58 @@ func NewForwardStats(latencyUpdateInterval, reportInterval, latencyWindowLength return s } -func (s *ForwardStats) Update(arrival, left int64) { +func (s *ForwardStats) Update(arrival, left int64) (int64, bool) { transit := left - arrival - - // ignore if transit is too large or negative, this could happen if system time is adjusted - if transit < 0 || time.Duration(transit) > 5*time.Second { - return - } - lastLeftNano := s.lastLeftNano.Load() - if left < lastLeftNano || !s.lastLeftNano.CompareAndSwap(lastLeftNano, left) { - return - } + isHighForwardingLatency := time.Duration(transit) > cHighForwardingLatency s.lock.Lock() - defer s.lock.Unlock() s.latency.Update(time.Duration(arrival), float64(transit)) + s.lowest = min(transit, s.lowest) + s.highest = max(transit, s.highest) + s.lastUpdateAt = arrival + s.lock.Unlock() + + prometheus.RecordForwardLatencySample(transit) + return transit, isHighForwardingLatency } -func (s *ForwardStats) GetStats() (latency, jitter time.Duration) { +func (s *ForwardStats) GetStats(shortDuration time.Duration) (time.Duration, time.Duration) { s.lock.Lock() - w := s.latency.Summarize() + // a dummy sample to flush the pipe to current time + now := mono.UnixNano() + if (now - s.lastUpdateAt) > shortDuration.Nanoseconds() { + s.latency.Update(time.Duration(now), 0) + } + + wLong := s.latency.Summarize() + + lowest := s.lowest + s.lowest = time.Second.Nanoseconds() + + highest := s.highest + s.highest = 0 s.lock.Unlock() - latency, jitter = time.Duration(w.Mean()), time.Duration(w.StdDev()) - // TODO: remove this check after debugging unexpected jitter issue - if jitter > 10*time.Second { - logger.Infow("unexpected forward jitter", - "jitter", jitter, - "stats", fmt.Sprintf("count %.2f, mean %.2f, stdDev %.2f", w.Count(), w.Mean(), w.StdDev()), + + latencyLong, jitterLong := time.Duration(wLong.Mean()), time.Duration(wLong.StdDev()) + if jitterLong > latencyLong*cSkewFactor { + logger.Infow( + "high jitter in forwarding path", + "lowest", time.Duration(lowest), + "highest", time.Duration(highest), + "countLong", wLong.Count(), + "latencyLong", latencyLong, + "jitterLong", jitterLong, ) } - return + return latencyLong, jitterLong } -func (s *ForwardStats) GetLastStats(duration time.Duration) (latency, jitter time.Duration) { +func (s *ForwardStats) GetShortStats(shortDuration time.Duration) (time.Duration, time.Duration) { s.lock.Lock() - defer s.lock.Unlock() - w := s.latency.SummarizeLast(duration) - return time.Duration(w.Mean()), time.Duration(w.StdDev()) + wShort := s.latency.SummarizeLast(shortDuration) + s.lock.Unlock() + + return time.Duration(wShort.Mean()), time.Duration(wShort.StdDev()) } func (s *ForwardStats) Stop() { @@ -75,15 +96,16 @@ func (s *ForwardStats) Stop() { func (s *ForwardStats) report(reportInterval time.Duration) { ticker := time.NewTicker(reportInterval) defer ticker.Stop() + for { select { case <-s.closeCh: return + case <-ticker.C: - latency, jitter := s.GetLastStats(reportInterval) - latencySlow, jitterSlow := s.GetStats() - prometheus.RecordForwardJitter(uint32(jitter/time.Millisecond), uint32(jitterSlow/time.Millisecond)) - prometheus.RecordForwardLatency(uint32(latency/time.Millisecond), uint32(latencySlow/time.Millisecond)) + latencyLong, jitterLong := s.GetStats(reportInterval) + prometheus.RecordForwardJitter(uint32(jitterLong.Nanoseconds())) + prometheus.RecordForwardLatency(uint32(latencyLong.Nanoseconds())) } } } diff --git a/pkg/sfu/interceptor/rtx.go b/pkg/sfu/interceptor/rtx.go index 69e00df..1451ffa 100644 --- a/pkg/sfu/interceptor/rtx.go +++ b/pkg/sfu/interceptor/rtx.go @@ -39,13 +39,17 @@ type streamInfo struct { type RTXInfoExtractorFactory struct { onStreamFound func(*interceptor.StreamInfo) - onRTXPairFound func(repair, base uint32) + onRTXPairFound func(repair, base uint32, rsid string) lock sync.Mutex streams map[uint32]streamInfo logger logger.Logger } -func NewRTXInfoExtractorFactory(onStreamFound func(*interceptor.StreamInfo), onRTXPairFound func(repair, base uint32), logger logger.Logger) *RTXInfoExtractorFactory { +func NewRTXInfoExtractorFactory( + onStreamFound func(*interceptor.StreamInfo), + onRTXPairFound func(repair, base uint32, rsid string), + logger logger.Logger, +) *RTXInfoExtractorFactory { return &RTXInfoExtractorFactory{ onStreamFound: onStreamFound, onRTXPairFound: onRTXPairFound, @@ -61,16 +65,23 @@ func (f *RTXInfoExtractorFactory) NewInterceptor(id string) (interceptor.Interce }, nil } -func (f *RTXInfoExtractorFactory) setStreamInfo(ssrc uint32, mid, rid, rsid string) { +func (f *RTXInfoExtractorFactory) SetStreamInfo(ssrc uint32, mid, rid, rsid string) { var repairSsrc, baseSsrc uint32 + var repairSid string f.lock.Lock() + if mid == "" || (rid == "" && rsid == "") { + f.lock.Unlock() + return + } + if rsid != "" { // repair stream found, find base stream for base, info := range f.streams { if info.mid == mid && info.rid == rsid { repairSsrc = ssrc baseSsrc = base + repairSid = rsid delete(f.streams, base) break } @@ -81,6 +92,7 @@ func (f *RTXInfoExtractorFactory) setStreamInfo(ssrc uint32, mid, rid, rsid stri if info.mid == mid && info.rsid == rid { repairSsrc = repair baseSsrc = ssrc + repairSid = rid delete(f.streams, repair) break } @@ -99,10 +111,12 @@ func (f *RTXInfoExtractorFactory) setStreamInfo(ssrc uint32, mid, rid, rsid stri f.lock.Unlock() if repairSsrc != 0 && baseSsrc != 0 { - f.onRTXPairFound(repairSsrc, baseSsrc) + f.onRTXPairFound(repairSsrc, baseSsrc, repairSid) } } +// ------------------------------------------ + type RTXInfoExtractor struct { interceptor.NoOp @@ -131,6 +145,8 @@ func (u *RTXInfoExtractor) BindRemoteStream(info *interceptor.StreamInfo, reader } } +// ------------------------------------------ + type rtxInfoReader struct { tryTimes int reader interceptor.RTPReader @@ -171,10 +187,10 @@ func (r *rtxInfoReader) Read(b []byte, a interceptor.Attributes) (int, intercept if mid != "" && (rid != "" || rsid != "") { r.logger.Debugw("stream found", "mid", mid, "rid", rid, "rsid", rsid, "ssrc", header.SSRC) r.tryTimes = -1 - go r.factory.setStreamInfo(header.SSRC, mid, rid, rsid) + go r.factory.SetStreamInfo(header.SSRC, mid, rid, rsid) } else { // ignore padding only packet for probe count - if !(header.Padding && n-header.MarshalSize()-int(b[n-1]) == 0) { + if !header.Padding || n-header.MarshalSize()-int(b[n-1]) != 0 { r.tryTimes-- } } diff --git a/pkg/sfu/interceptor/unhandlesimulcast.go b/pkg/sfu/interceptor/unhandlesimulcast.go new file mode 100644 index 0000000..27a5333 --- /dev/null +++ b/pkg/sfu/interceptor/unhandlesimulcast.go @@ -0,0 +1,183 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package interceptor + +import ( + "github.com/pion/interceptor" + "github.com/pion/rtp" + "github.com/pion/sdp/v3" + "github.com/pion/webrtc/v4" + "go.uber.org/zap/zapcore" + + "github.com/livekit/livekit-server/pkg/sfu/utils" + "github.com/livekit/protocol/logger" +) + +const ( + simulcastProbeCount = 10 +) + +type SimulcastTrackInfo struct { + Mid string + StreamID string + RepairSSRC uint32 // set only when `IsRepairStream: false`, i. e. RTX SSRC for the primary stream + IsRepairStream bool +} + +func (s *SimulcastTrackInfo) MarshalLogObject(e zapcore.ObjectEncoder) error { + e.AddString("Mid", s.Mid) + e.AddString("StreamID", s.StreamID) + e.AddUint32("RepairSSRC", s.RepairSSRC) + e.AddBool("IsRepairStream", s.IsRepairStream) + return nil +} + +// ------------------------------------------------------------------- + +type UnhandleSimulcastOption func(u *UnhandleSimulcastInterceptor) error + +func UnhandleSimulcastTracks(logger logger.Logger, tracks map[uint32]SimulcastTrackInfo) UnhandleSimulcastOption { + return func(u *UnhandleSimulcastInterceptor) error { + u.logger = logger + u.simTracks = tracks + return nil + } +} + +type UnhandleSimulcastInterceptorFactory struct { + opts []UnhandleSimulcastOption +} + +func (f *UnhandleSimulcastInterceptorFactory) NewInterceptor(id string) (interceptor.Interceptor, error) { + i := &UnhandleSimulcastInterceptor{simTracks: map[uint32]SimulcastTrackInfo{}} + for _, o := range f.opts { + if err := o(i); err != nil { + return nil, err + } + } + return i, nil +} + +func NewUnhandleSimulcastInterceptorFactory(opts ...UnhandleSimulcastOption) (*UnhandleSimulcastInterceptorFactory, error) { + return &UnhandleSimulcastInterceptorFactory{opts: opts}, nil +} + +type unhandleSimulcastRTPReader struct { + SimulcastTrackInfo + logger logger.Logger + tryTimes int + reader interceptor.RTPReader + midExtID uint8 + ridExtID uint8 + rsidExtID uint8 +} + +func (u *unhandleSimulcastRTPReader) Read(b []byte, a interceptor.Attributes) (int, interceptor.Attributes, error) { + n, a, err := u.reader.Read(b, a) + if u.tryTimes < 0 || err != nil { + return n, a, err + } + + header := rtp.Header{} + hsize, err := header.Unmarshal(b[:n]) + if err != nil { + return n, a, nil + } + var mid, rid, rsid string + if payload := header.GetExtension(u.midExtID); payload != nil { + mid = string(payload) + } + + if payload := header.GetExtension(u.ridExtID); payload != nil { + rid = string(payload) + } + + if payload := header.GetExtension(u.rsidExtID); payload != nil { + rid = string(payload) + } + + if mid != "" && (rid != "" || rsid != "") { + u.logger.Debugw( + "unhandle stream found", + "mid", mid, + "rid", rid, + "rsid", rsid, + "ssrc", header.SSRC, + "simulcastTrackInfo", u.SimulcastTrackInfo, + ) + u.tryTimes = -1 + return n, a, nil + } else { + // ignore padding only packet for probe count + if !header.Padding || n-header.MarshalSize()-int(b[n-1]) != 0 { + u.tryTimes-- + } + } + + if mid == "" { + header.SetExtension(u.midExtID, []byte(u.Mid)) + } + if rid == "" && !u.IsRepairStream { + header.SetExtension(u.ridExtID, []byte(u.StreamID)) + } + if rsid == "" && u.IsRepairStream { + header.SetExtension(u.rsidExtID, []byte(u.StreamID)) + } + + hsize2 := header.MarshalSize() + + if hsize2-hsize+n > len(b) { // no enough buf to set extension + return n, a, nil + } + copy(b[hsize2:], b[hsize:n]) + header.MarshalTo(b) + u.logger.Debugw( + "unhandle stream injecting", + "mid", mid, + "rid", rid, + "rsid", rsid, + "ssrc", header.SSRC, + "simulcastTrackInfo", u.SimulcastTrackInfo, + ) + return hsize2 - hsize + n, a, nil +} + +type UnhandleSimulcastInterceptor struct { + interceptor.NoOp + logger logger.Logger + simTracks map[uint32]SimulcastTrackInfo +} + +func (u *UnhandleSimulcastInterceptor) BindRemoteStream(info *interceptor.StreamInfo, reader interceptor.RTPReader) interceptor.RTPReader { + if t, ok := u.simTracks[info.SSRC]; ok { + midExtensionID := utils.GetHeaderExtensionID(info.RTPHeaderExtensions, webrtc.RTPHeaderExtensionCapability{URI: sdp.SDESMidURI}) + streamIDExtensionID := utils.GetHeaderExtensionID(info.RTPHeaderExtensions, webrtc.RTPHeaderExtensionCapability{URI: sdp.SDESRTPStreamIDURI}) + repairStreamIDExtensionID := utils.GetHeaderExtensionID(info.RTPHeaderExtensions, webrtc.RTPHeaderExtensionCapability{URI: sdp.SDESRepairRTPStreamIDURI}) + if midExtensionID == 0 || streamIDExtensionID == 0 || repairStreamIDExtensionID == 0 { + return reader + } + + return &unhandleSimulcastRTPReader{ + SimulcastTrackInfo: t, + logger: u.logger, + reader: reader, + tryTimes: simulcastProbeCount, + midExtID: uint8(midExtensionID), + ridExtID: uint8(streamIDExtensionID), + rsidExtID: uint8(repairStreamIDExtensionID), + } + } + return reader +} diff --git a/pkg/sfu/mime/mimetype.go b/pkg/sfu/mime/mimetype.go deleted file mode 100644 index 118a7ba..0000000 --- a/pkg/sfu/mime/mimetype.go +++ /dev/null @@ -1,281 +0,0 @@ -// Copyright 2023 LiveKit, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package mime - -import ( - "strings" - - "github.com/pion/webrtc/v4" -) - -const ( - MimeTypePrefixAudio = "audio/" - MimeTypePrefixVideo = "video/" -) - -type MimeTypeCodec int - -const ( - MimeTypeCodecUnknown MimeTypeCodec = iota - MimeTypeCodecH264 - MimeTypeCodecH265 - MimeTypeCodecOpus - MimeTypeCodecRED - MimeTypeCodecVP8 - MimeTypeCodecVP9 - MimeTypeCodecAV1 - MimeTypeCodecG722 - MimeTypeCodecPCMU - MimeTypeCodecPCMA - MimeTypeCodecRTX - MimeTypeCodecFlexFEC - MimeTypeCodecULPFEC -) - -func (m MimeTypeCodec) String() string { - switch m { - case MimeTypeCodecUnknown: - return "MimeTypeCodecUnknown" - case MimeTypeCodecH264: - return "H264" - case MimeTypeCodecH265: - return "H265" - case MimeTypeCodecOpus: - return "opus" - case MimeTypeCodecRED: - return "red" - case MimeTypeCodecVP8: - return "VP8" - case MimeTypeCodecVP9: - return "VP9" - case MimeTypeCodecAV1: - return "AV1" - case MimeTypeCodecG722: - return "G722" - case MimeTypeCodecPCMU: - return "PCMU" - case MimeTypeCodecPCMA: - return "PCMA" - case MimeTypeCodecRTX: - return "rtx" - case MimeTypeCodecFlexFEC: - return "flexfec" - case MimeTypeCodecULPFEC: - return "ulpfec" - } - - return "MimeTypeCodecUnknown" -} - -func NormalizeMimeTypeCodec(codec string) MimeTypeCodec { - switch { - case strings.EqualFold(codec, "h264"): - return MimeTypeCodecH264 - case strings.EqualFold(codec, "h265"): - return MimeTypeCodecH265 - case strings.EqualFold(codec, "opus"): - return MimeTypeCodecOpus - case strings.EqualFold(codec, "red"): - return MimeTypeCodecRED - case strings.EqualFold(codec, "vp8"): - return MimeTypeCodecVP8 - case strings.EqualFold(codec, "vp9"): - return MimeTypeCodecVP9 - case strings.EqualFold(codec, "av1"): - return MimeTypeCodecAV1 - case strings.EqualFold(codec, "g722"): - return MimeTypeCodecG722 - case strings.EqualFold(codec, "pcmu"): - return MimeTypeCodecPCMU - case strings.EqualFold(codec, "pcma"): - return MimeTypeCodecPCMA - case strings.EqualFold(codec, "rtx"): - return MimeTypeCodecRTX - case strings.EqualFold(codec, "flexfec"): - return MimeTypeCodecFlexFEC - case strings.EqualFold(codec, "ulpfec"): - return MimeTypeCodecULPFEC - } - - return MimeTypeCodecUnknown -} - -func GetMimeTypeCodec(mime string) MimeTypeCodec { - i := strings.IndexByte(mime, '/') - if i == -1 { - return MimeTypeCodecUnknown - } - - return NormalizeMimeTypeCodec(mime[i+1:]) -} - -func IsMimeTypeCodecStringOpus(codec string) bool { - return NormalizeMimeTypeCodec(codec) == MimeTypeCodecOpus -} - -func IsMimeTypeCodecStringRED(codec string) bool { - return NormalizeMimeTypeCodec(codec) == MimeTypeCodecRED -} - -func IsMimeTypeCodecStringH264(codec string) bool { - return NormalizeMimeTypeCodec(codec) == MimeTypeCodecH264 -} - -type MimeType int - -const ( - MimeTypeUnknown MimeType = iota - MimeTypeH264 - MimeTypeH265 - MimeTypeOpus - MimeTypeRED - MimeTypeVP8 - MimeTypeVP9 - MimeTypeAV1 - MimeTypeG722 - MimeTypePCMU - MimeTypePCMA - MimeTypeRTX - MimeTypeFlexFEC - MimeTypeULPFEC -) - -func (m MimeType) String() string { - switch m { - case MimeTypeUnknown: - return "MimeTypeUnknown" - case MimeTypeH264: - return webrtc.MimeTypeH264 - case MimeTypeH265: - return webrtc.MimeTypeH265 - case MimeTypeOpus: - return webrtc.MimeTypeOpus - case MimeTypeRED: - return "audio/red" - case MimeTypeVP8: - return webrtc.MimeTypeVP8 - case MimeTypeVP9: - return webrtc.MimeTypeVP9 - case MimeTypeAV1: - return webrtc.MimeTypeAV1 - case MimeTypeG722: - return webrtc.MimeTypeG722 - case MimeTypePCMU: - return webrtc.MimeTypePCMU - case MimeTypePCMA: - return webrtc.MimeTypePCMA - case MimeTypeRTX: - return webrtc.MimeTypeRTX - case MimeTypeFlexFEC: - return webrtc.MimeTypeFlexFEC - case MimeTypeULPFEC: - return "video/ulpfec" - } - - return "MimeTypeUnknown" -} - -func NormalizeMimeType(mime string) MimeType { - switch { - case strings.EqualFold(mime, webrtc.MimeTypeH264): - return MimeTypeH264 - case strings.EqualFold(mime, webrtc.MimeTypeH265): - return MimeTypeH265 - case strings.EqualFold(mime, webrtc.MimeTypeOpus): - return MimeTypeOpus - case strings.EqualFold(mime, "audio/red"): - return MimeTypeRED - case strings.EqualFold(mime, webrtc.MimeTypeVP8): - return MimeTypeVP8 - case strings.EqualFold(mime, webrtc.MimeTypeVP9): - return MimeTypeVP9 - case strings.EqualFold(mime, webrtc.MimeTypeAV1): - return MimeTypeAV1 - case strings.EqualFold(mime, webrtc.MimeTypeG722): - return MimeTypeG722 - case strings.EqualFold(mime, webrtc.MimeTypePCMU): - return MimeTypePCMU - case strings.EqualFold(mime, webrtc.MimeTypePCMA): - return MimeTypePCMA - case strings.EqualFold(mime, webrtc.MimeTypeRTX): - return MimeTypeRTX - case strings.EqualFold(mime, webrtc.MimeTypeFlexFEC): - return MimeTypeFlexFEC - case strings.EqualFold(mime, "video/ulpfec"): - return MimeTypeULPFEC - } - - return MimeTypeUnknown -} - -func IsMimeTypeStringEqual(mime1 string, mime2 string) bool { - return NormalizeMimeType(mime1) == NormalizeMimeType(mime2) -} - -func IsMimeTypeStringAudio(mime string) bool { - return strings.HasPrefix(mime, MimeTypePrefixAudio) -} - -func IsMimeTypeAudio(mimeType MimeType) bool { - return strings.HasPrefix(mimeType.String(), MimeTypePrefixAudio) -} - -func IsMimeTypeStringVideo(mime string) bool { - return strings.HasPrefix(mime, MimeTypePrefixVideo) -} - -func IsMimeTypeVideo(mimeType MimeType) bool { - return strings.HasPrefix(mimeType.String(), MimeTypePrefixVideo) -} - -// SVC-TODO: Have to use more conditions to differentiate between -// SVC-TODO: SVC and non-SVC (could be single layer or simulcast). -// SVC-TODO: May only need to differentiate between simulcast and non-simulcast -// SVC-TODO: i. e. may be possible to treat single layer as SVC to get proper/intended functionality. -func IsMimeTypeSVC(mimeType MimeType) bool { - switch mimeType { - case MimeTypeAV1, MimeTypeVP9: - return true - } - return false -} - -func IsMimeTypeStringSVC(mime string) bool { - return IsMimeTypeSVC(NormalizeMimeType(mime)) -} - -func IsMimeTypeStringRED(mime string) bool { - return NormalizeMimeType(mime) == MimeTypeRED -} - -func IsMimeTypeStringOpus(mime string) bool { - return NormalizeMimeType(mime) == MimeTypeOpus -} - -func IsMimeTypeStringRTX(mime string) bool { - return NormalizeMimeType(mime) == MimeTypeRTX -} - -func IsMimeTypeStringVP8(mime string) bool { - return NormalizeMimeType(mime) == MimeTypeVP8 -} - -func IsMimeTypeStringVP9(mime string) bool { - return NormalizeMimeType(mime) == MimeTypeVP9 -} - -func IsMimeTypeStringH264(mime string) bool { - return NormalizeMimeType(mime) == MimeTypeH264 -} diff --git a/pkg/sfu/pacer/base.go b/pkg/sfu/pacer/base.go index b709778..96e3be2 100644 --- a/pkg/sfu/pacer/base.go +++ b/pkg/sfu/pacer/base.go @@ -20,6 +20,7 @@ import ( "time" "github.com/livekit/livekit-server/pkg/sfu/bwe" + "github.com/livekit/mediatransportutil" "github.com/livekit/protocol/logger" "github.com/livekit/protocol/utils/mono" "github.com/pion/rtp" @@ -56,9 +57,17 @@ func (b *Base) TimeSinceLastSentPacket() time.Duration { func (b *Base) SendPacket(p *Packet) (int, error) { defer func() { + if p.HeaderPool != nil && p.Header != nil { + *p.Header = rtp.Header{} + p.HeaderPool.Put(p.Header) + } + if p.Pool != nil && p.PoolEntity != nil { p.Pool.Put(p.PoolEntity) } + + *p = Packet{} + PacketFactory.Put(p) }() err := b.patchRTPHeaderExtensions(p) @@ -83,8 +92,10 @@ func (b *Base) SendPacket(p *Packet) (int, error) { func (b *Base) patchRTPHeaderExtensions(p *Packet) error { sendingAt := mono.Now() if p.AbsSendTimeExtID != 0 { - absSendTime := rtp.NewAbsSendTimeExtension(sendingAt) - absSendTimeBytes, err := absSendTime.Marshal() + absSendTimeExt := rtp.AbsSendTimeExtension{ + Timestamp: uint64(mediatransportutil.ToNtpTime(sendingAt) >> 14), + } + absSendTimeBytes, err := absSendTimeExt.Marshal() if err != nil { return err } diff --git a/pkg/sfu/pacer/leaky_bucket.go b/pkg/sfu/pacer/leaky_bucket.go index db5c893..6866d44 100644 --- a/pkg/sfu/pacer/leaky_bucket.go +++ b/pkg/sfu/pacer/leaky_bucket.go @@ -80,8 +80,8 @@ func (l *LeakyBucket) Enqueue(p *Packet) { func (l *LeakyBucket) sendWorker() { l.lock.RLock() interval := l.interval - bitrate := l.bitrate l.lock.RUnlock() + var bitrate int timer := time.NewTimer(interval) overage := 0 diff --git a/pkg/sfu/pacer/pacer.go b/pkg/sfu/pacer/pacer.go index 8919dd6..f232125 100644 --- a/pkg/sfu/pacer/pacer.go +++ b/pkg/sfu/pacer/pacer.go @@ -23,8 +23,27 @@ import ( "github.com/pion/webrtc/v4" ) +var ( + PacketFactory = &sync.Pool{ + New: func() any { + return &Packet{} + }, + } +) + +// -------------------------------------- + +type PacerBehavior string + +const ( + PacerBehaviorPassThrough PacerBehavior = "pass-through" + PacerBehaviorNoQueue PacerBehavior = "no-queue" + PacerBehaviorLeakybucket PacerBehavior = "leaky-bucket" +) + type Packet struct { Header *rtp.Header + HeaderPool *sync.Pool HeaderSize int Payload []byte IsRTX bool diff --git a/pkg/sfu/packettrailer/packet_trailer.go b/pkg/sfu/packettrailer/packet_trailer.go new file mode 100644 index 0000000..3b58f66 --- /dev/null +++ b/pkg/sfu/packettrailer/packet_trailer.go @@ -0,0 +1,46 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package packettrailer + +var Magic = [4]byte{'L', 'K', 'T', 'S'} + +const ( + xorByte = 0xFF + + envelopeSize = 5 // 1B trailer_len + 4B magic +) + +// StripTrailer returns the number of bytes to strip from the end of an RTP +// payload if it contains an LKTS trailer. The trailer is located by checking +// for the "LKTS" magic suffix and then reading the XORed trailer_len byte +// immediately before it. Returns 0 if absent or ineligible. +func StripTrailer(payload []byte, marker bool) int { + if !marker || len(payload) < envelopeSize { + return 0 + } + + tail := payload[len(payload)-4:] + if tail[0] != Magic[0] || tail[1] != Magic[1] || + tail[2] != Magic[2] || tail[3] != Magic[3] { + return 0 + } + + trailerLen := int(payload[len(payload)-5] ^ xorByte) + if trailerLen < envelopeSize || trailerLen > len(payload) { + return 0 + } + + return trailerLen +} diff --git a/pkg/sfu/packettrailer/packet_trailer_test.go b/pkg/sfu/packettrailer/packet_trailer_test.go new file mode 100644 index 0000000..653b71f --- /dev/null +++ b/pkg/sfu/packettrailer/packet_trailer_test.go @@ -0,0 +1,182 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package packettrailer + +import ( + "encoding/binary" + "testing" +) + +const ( + tagTimestampUs = 0x01 + tagFrameID = 0x02 +) + +// appendTLV appends a single XORed TLV element to dst. +func appendTLV(dst []byte, tag byte, value []byte) []byte { + dst = append(dst, tag^xorByte, byte(len(value))^xorByte) + for _, b := range value { + dst = append(dst, b^xorByte) + } + return dst +} + +// appendEnvelope appends the 5-byte envelope (XORed trailer_len + magic). +func appendEnvelope(dst []byte, trailerLen byte) []byte { + dst = append(dst, trailerLen^xorByte) + dst = append(dst, Magic[:]...) + return dst +} + +// makeTrailer builds a complete LKTS trailer with both timestamp and frame_id TLVs. +func makeTrailer(timestampUs int64, frameID uint32) []byte { + var trailer []byte + + var tsBuf [8]byte + binary.BigEndian.PutUint64(tsBuf[:], uint64(timestampUs)) + trailer = appendTLV(trailer, tagTimestampUs, tsBuf[:]) + + var fidBuf [4]byte + binary.BigEndian.PutUint32(fidBuf[:], frameID) + trailer = appendTLV(trailer, tagFrameID, fidBuf[:]) + + trailerLen := byte(len(trailer) + envelopeSize) + trailer = appendEnvelope(trailer, trailerLen) + return trailer +} + +// makePayloadWithTrailer builds a fake video payload followed by a full LKTS trailer. +func makePayloadWithTrailer(videoLen int, timestampUs int64, frameID uint32) []byte { + video := make([]byte, videoLen) + for i := range video { + video[i] = byte(i) + } + return append(video, makeTrailer(timestampUs, frameID)...) +} + +// makeTimestampOnlyTrailer builds a trailer with only the timestamp TLV. +func makeTimestampOnlyTrailer(timestampUs int64) []byte { + var trailer []byte + var tsBuf [8]byte + binary.BigEndian.PutUint64(tsBuf[:], uint64(timestampUs)) + trailer = appendTLV(trailer, tagTimestampUs, tsBuf[:]) + trailerLen := byte(len(trailer) + envelopeSize) + trailer = appendEnvelope(trailer, trailerLen) + return trailer +} + +func TestStripTrailer(t *testing.T) { + fullTrailerSize := 21 // (1+1+8) + (1+1+4) + 5 + tsOnlyTrailerSize := 15 // (1+1+8) + 5 + + tests := []struct { + name string + payload []byte + marker bool + wantStrip int + }{ + { + name: "marker set with full trailer (timestamp + frame_id)", + payload: makePayloadWithTrailer(20, 1700000000000000, 42), + marker: true, + wantStrip: fullTrailerSize, + }, + { + name: "marker set with timestamp-only trailer", + payload: func() []byte { + video := make([]byte, 20) + return append(video, makeTimestampOnlyTrailer(1700000000000000)...) + }(), + marker: true, + wantStrip: tsOnlyTrailerSize, + }, + { + name: "marker not set with valid trailer", + payload: makePayloadWithTrailer(20, 1700000000000000, 42), + marker: false, + wantStrip: 0, + }, + { + name: "marker set without magic", + payload: make([]byte, 32), + marker: true, + wantStrip: 0, + }, + { + name: "marker set but payload too short for envelope", + payload: []byte{0x4C, 0x4B, 0x54, 0x53}, + marker: true, + wantStrip: 0, + }, + { + name: "marker set with partial magic mismatch", + payload: func() []byte { + p := makePayloadWithTrailer(20, 1700000000000000, 42) + p[len(p)-1] = 'x' + return p + }(), + marker: true, + wantStrip: 0, + }, + { + name: "trailer_len exceeds payload length", + payload: func() []byte { + var trailer []byte + var tsBuf [8]byte + binary.BigEndian.PutUint64(tsBuf[:], 42) + trailer = appendTLV(trailer, tagTimestampUs, tsBuf[:]) + trailer = appendEnvelope(trailer, 200) + return trailer + }(), + marker: true, + wantStrip: 0, + }, + { + name: "trailer_len smaller than envelope (invalid)", + payload: func() []byte { + video := make([]byte, 20) + var trailer []byte + var tsBuf [8]byte + binary.BigEndian.PutUint64(tsBuf[:], 42) + trailer = appendTLV(trailer, tagTimestampUs, tsBuf[:]) + trailer = appendEnvelope(trailer, 3) + return append(video, trailer...) + }(), + marker: true, + wantStrip: 0, + }, + { + name: "exactly envelope-only trailer", + payload: appendEnvelope(nil, byte(envelopeSize)), + marker: true, + wantStrip: envelopeSize, + }, + { + name: "empty payload", + payload: []byte{}, + marker: true, + wantStrip: 0, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := StripTrailer(tt.payload, tt.marker) + if got != tt.wantStrip { + t.Errorf("StripTrailer() = %d, want %d", got, tt.wantStrip) + } + }) + } +} diff --git a/pkg/sfu/playoutdelay_test.go b/pkg/sfu/playoutdelay_test.go index 419f473..94babe8 100644 --- a/pkg/sfu/playoutdelay_test.go +++ b/pkg/sfu/playoutdelay_test.go @@ -26,7 +26,9 @@ import ( ) func TestPlayoutDelay(t *testing.T) { - stats := rtpstats.NewRTPStatsSender(rtpstats.RTPStatsParams{ClockRate: 900000, Logger: logger.GetLogger()}, 128) + stats := rtpstats.NewRTPStatsSender(rtpstats.RTPStatsParams{}, 128) + stats.SetClockRate(90000) + c, err := NewPlayoutDelayController(100, 120, logger.GetLogger(), stats) require.NoError(t, err) diff --git a/pkg/sfu/receiver.go b/pkg/sfu/receiver.go index ee307a8..0bc5663 100644 --- a/pkg/sfu/receiver.go +++ b/pkg/sfu/receiver.go @@ -15,181 +15,38 @@ package sfu import ( - "errors" - "io" "strings" "sync" "time" "github.com/pion/rtcp" "github.com/pion/webrtc/v4" - "go.uber.org/atomic" - "github.com/livekit/mediatransportutil/pkg/bucket" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" - "github.com/livekit/protocol/utils" - "github.com/livekit/livekit-server/pkg/sfu/audio" "github.com/livekit/livekit-server/pkg/sfu/buffer" "github.com/livekit/livekit-server/pkg/sfu/connectionquality" - "github.com/livekit/livekit-server/pkg/sfu/mime" - dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor" "github.com/livekit/livekit-server/pkg/sfu/rtpstats" - "github.com/livekit/livekit-server/pkg/sfu/streamtracker" ) -var ( - ErrReceiverClosed = errors.New("receiver closed") - ErrDownTrackAlreadyExist = errors.New("DownTrack already exist") - ErrBufferNotFound = errors.New("buffer not found") - ErrDuplicateLayer = errors.New("duplicate layer") -) - -// -------------------------------------- - -type PLIThrottleConfig struct { - LowQuality time.Duration `yaml:"low_quality,omitempty"` - MidQuality time.Duration `yaml:"mid_quality,omitempty"` - HighQuality time.Duration `yaml:"high_quality,omitempty"` -} - -var ( - DefaultPLIThrottleConfig = PLIThrottleConfig{ - LowQuality: 500 * time.Millisecond, - MidQuality: time.Second, - HighQuality: time.Second, - } -) - -// -------------------------------------- - -type AudioConfig struct { - audio.AudioLevelConfig `yaml:",inline"` - - // enable red encoding downtrack for opus only audio up track - ActiveREDEncoding bool `yaml:"active_red_encoding,omitempty"` - // enable proxying weakest subscriber loss to publisher in RTCP Receiver Report - EnableLossProxying bool `yaml:"enable_loss_proxying,omitempty"` -} - -var ( - DefaultAudioConfig = AudioConfig{ - AudioLevelConfig: audio.DefaultAudioLevelConfig, - } -) - -// -------------------------------------- - -type AudioLevelHandle func(level uint8, duration uint32) - -type Bitrates [buffer.DefaultMaxLayerSpatial + 1][buffer.DefaultMaxLayerTemporal + 1]int64 - -type ReceiverCodecState int - -const ( - ReceiverCodecStateNormal ReceiverCodecState = iota - ReceiverCodecStateSuspended - ReceiverCodecStateInvalid -) - -// TrackReceiver defines an interface receive media from remote peer -type TrackReceiver interface { - TrackID() livekit.TrackID - StreamID() string - - // returns the initial codec of the receiver, it is determined by the track's codec - // and will not change if the codec changes during the session (publisher changes codec) - Codec() webrtc.RTPCodecParameters - Mime() mime.MimeType - HeaderExtensions() []webrtc.RTPHeaderExtensionParameter - IsClosed() bool - - ReadRTP(buf []byte, layer uint8, esn uint64) (int, error) - GetLayeredBitrate() ([]int32, Bitrates) - - GetAudioLevel() (float64, bool) - - SendPLI(layer int32, force bool) - - SetUpTrackPaused(paused bool) - SetMaxExpectedSpatialLayer(layer int32) - - AddDownTrack(track TrackSender) error - DeleteDownTrack(participantID livekit.ParticipantID) - GetDownTracks() []TrackSender - - DebugInfo() map[string]interface{} - - TrackInfo() *livekit.TrackInfo - UpdateTrackInfo(ti *livekit.TrackInfo) - - // Get primary receiver if this receiver represents a RED codec; otherwise it will return itself - GetPrimaryReceiverForRed() TrackReceiver - - // Get red receiver for primary codec, used by forward red encodings for opus only codec - GetRedReceiver() TrackReceiver - - GetTemporalLayerFpsForSpatial(layer int32) []float32 - - GetTrackStats() *livekit.RTPStats - - // AddOnReady adds a function to be called when the receiver is ready, the callback - // could be called immediately if the receiver is ready when the callback is added - AddOnReady(func()) - - AddOnCodecStateChange(func(webrtc.RTPCodecParameters, ReceiverCodecState)) - CodecState() ReceiverCodecState -} - -type redPktWriteFunc func(pkt *buffer.ExtPacket, spatialLayer int32) int +var _ TrackReceiver = (*WebRTCReceiver)(nil) // WebRTCReceiver receives a media track type WebRTCReceiver struct { - logger logger.Logger + *ReceiverBase - pliThrottleConfig PLIThrottleConfig - audioConfig AudioConfig - - trackID livekit.TrackID - streamID string - kind webrtc.RTPCodecType - receiver *webrtc.RTPReceiver - codec webrtc.RTPCodecParameters - codecState ReceiverCodecState - codecStateLock sync.Mutex - onCodecStateChange []func(webrtc.RTPCodecParameters, ReceiverCodecState) - isSVC bool - isRED bool - onCloseHandler func() - closeOnce sync.Once - closed atomic.Bool - useTrackers bool - trackInfo atomic.Pointer[livekit.TrackInfo] + receiver *webrtc.RTPReceiver + onCloseHandler func() onRTCP func([]rtcp.Packet) - bufferMu sync.RWMutex - buffers [buffer.DefaultMaxLayerSpatial + 1]*buffer.Buffer - upTracks [buffer.DefaultMaxLayerSpatial + 1]TrackRemote - rtt uint32 - - lbThreshold int - - streamTrackerManager *StreamTrackerManager - - downTrackSpreader *DownTrackSpreader + upTracksMu sync.Mutex + upTracks [buffer.DefaultMaxLayerSpatial + 1]TrackRemote connectionStats *connectionquality.ConnectionStats - - onStatsUpdate func(w *WebRTCReceiver, stat *livekit.AnalyticsStat) - onMaxLayerChange func(maxLayer int32) - - primaryReceiver atomic.Pointer[RedPrimaryReceiver] - redReceiver atomic.Pointer[RedReceiver] - redPktWriter atomic.Value // redPktWriteFunc - - forwardStats *ForwardStats + onStatsUpdate func(w *WebRTCReceiver, stat *livekit.AnalyticsStat) } type ReceiverOpts func(w *WebRTCReceiver) *WebRTCReceiver @@ -197,7 +54,7 @@ type ReceiverOpts func(w *WebRTCReceiver) *WebRTCReceiver // WithPliThrottleConfig indicates minimum time(ms) between sending PLIs func WithPliThrottleConfig(pliThrottleConfig PLIThrottleConfig) ReceiverOpts { return func(w *WebRTCReceiver) *WebRTCReceiver { - w.pliThrottleConfig = pliThrottleConfig + w.ReceiverBase.SetPLIThrottleConfig(pliThrottleConfig) return w } } @@ -205,15 +62,14 @@ func WithPliThrottleConfig(pliThrottleConfig PLIThrottleConfig) ReceiverOpts { // WithAudioConfig sets up parameters for active speaker detection func WithAudioConfig(audioConfig AudioConfig) ReceiverOpts { return func(w *WebRTCReceiver) *WebRTCReceiver { - w.audioConfig = audioConfig + w.ReceiverBase.SetAudioConfig(audioConfig) return w } } -// WithStreamTrackers enables StreamTracker use for simulcast -func WithStreamTrackers() ReceiverOpts { +func WithEnableRTPStreamRestartDetection(enable bool) ReceiverOpts { return func(w *WebRTCReceiver) *WebRTCReceiver { - w.useTrackers = true + w.ReceiverBase.SetEnableRTPStreamRestartDetection(enable) return w } } @@ -225,14 +81,14 @@ func WithStreamTrackers() ReceiverOpts { // Set to 0 (disabled) by default. func WithLoadBalanceThreshold(downTracks int) ReceiverOpts { return func(w *WebRTCReceiver) *WebRTCReceiver { - w.lbThreshold = downTracks + w.ReceiverBase.SetLBThreshold(downTracks) return w } } func WithForwardStats(forwardStats *ForwardStats) ReceiverOpts { return func(w *WebRTCReceiver) *WebRTCReceiver { - w.forwardStats = forwardStats + w.ReceiverBase.SetForwardStats(forwardStats) return w } } @@ -248,243 +104,119 @@ func NewWebRTCReceiver( opts ...ReceiverOpts, ) *WebRTCReceiver { w := &WebRTCReceiver{ - logger: logger, - receiver: receiver, - trackID: livekit.TrackID(track.ID()), - streamID: track.StreamID(), - codec: track.Codec(), - codecState: ReceiverCodecStateNormal, - kind: track.Kind(), - onRTCP: onRTCP, - isSVC: mime.IsMimeTypeStringSVC(track.Codec().MimeType), - isRED: mime.IsMimeTypeStringRED(track.Codec().MimeType), + receiver: receiver, + onRTCP: onRTCP, } + w.ReceiverBase = NewReceiverBase( + ReceiverBaseParams{ + TrackID: livekit.TrackID(track.ID()), + StreamID: track.StreamID(), + Kind: track.Kind(), + Codec: track.Codec(), + HeaderExtensions: receiver.GetParameters().HeaderExtensions, + Logger: logger, + StreamTrackerManagerConfig: streamTrackerManagerConfig, + StreamTrackerManagerListener: w, + IsSelfClosing: true, + OnClosed: w.onClosed, + }, + trackInfo, + ReceiverCodecStateNormal, + ) + for _, opt := range opts { w = opt(w) } - w.trackInfo.Store(utils.CloneProto(trackInfo)) - - w.downTrackSpreader = NewDownTrackSpreader(DownTrackSpreaderParams{ - Threshold: w.lbThreshold, - Logger: logger, - }) w.connectionStats = connectionquality.NewConnectionStats(connectionquality.ConnectionStatsParams{ ReceiverProvider: w, - Logger: w.logger.WithValues("direction", "up"), + Logger: logger.WithValues("direction", "up"), }) w.connectionStats.OnStatsUpdate(func(_cs *connectionquality.ConnectionStats, stat *livekit.AnalyticsStat) { if w.onStatsUpdate != nil { w.onStatsUpdate(w, stat) } }) + codec := track.Codec() w.connectionStats.Start( - mime.NormalizeMimeType(w.codec.MimeType), + mime.NormalizeMimeType(codec.MimeType), // TODO: technically not correct to declare FEC on when RED. Need the primary codec's fmtp line to check. - mime.IsMimeTypeStringRED(w.codec.MimeType) || strings.Contains(strings.ToLower(w.codec.SDPFmtpLine), "useinbandfec=1"), + mime.IsMimeTypeStringRED(codec.MimeType) || strings.Contains(strings.ToLower(codec.SDPFmtpLine), "useinbandfec=1"), ) - w.streamTrackerManager = NewStreamTrackerManager(logger, trackInfo, w.isSVC, w.codec.ClockRate, streamTrackerManagerConfig) - w.streamTrackerManager.SetListener(w) - // SVC-TODO: Handle DD for non-SVC cases??? - if w.isSVC { - for _, ext := range receiver.GetParameters().HeaderExtensions { - if ext.URI == dd.ExtensionURI { - w.streamTrackerManager.AddDependencyDescriptorTrackers() - break - } - } - } + w.UpdateTrackInfo(trackInfo) return w } -func (w *WebRTCReceiver) TrackInfo() *livekit.TrackInfo { - return w.trackInfo.Load() -} - -func (w *WebRTCReceiver) UpdateTrackInfo(ti *livekit.TrackInfo) { - w.trackInfo.Store(utils.CloneProto(ti)) - w.streamTrackerManager.UpdateTrackInfo(ti) -} - func (w *WebRTCReceiver) OnStatsUpdate(fn func(w *WebRTCReceiver, stat *livekit.AnalyticsStat)) { w.onStatsUpdate = fn } -func (w *WebRTCReceiver) OnMaxLayerChange(fn func(maxLayer int32)) { - w.bufferMu.Lock() - w.onMaxLayerChange = fn - w.bufferMu.Unlock() -} - -func (w *WebRTCReceiver) getOnMaxLayerChange() func(maxLayer int32) { - w.bufferMu.RLock() - defer w.bufferMu.RUnlock() - - return w.onMaxLayerChange -} - func (w *WebRTCReceiver) GetConnectionScoreAndQuality() (float32, livekit.ConnectionQuality) { return w.connectionStats.GetScoreAndQuality() } -func (w *WebRTCReceiver) IsClosed() bool { - return w.closed.Load() -} - -func (w *WebRTCReceiver) SetRTT(rtt uint32) { - w.bufferMu.Lock() - if w.rtt == rtt { - w.bufferMu.Unlock() - return - } - - w.rtt = rtt - buffers := w.buffers - w.bufferMu.Unlock() - - for _, buff := range buffers { - if buff == nil { - continue - } - - buff.SetRTT(rtt) - } -} - -func (w *WebRTCReceiver) StreamID() string { - return w.streamID -} - -func (w *WebRTCReceiver) TrackID() livekit.TrackID { - return w.trackID -} - func (w *WebRTCReceiver) ssrc(layer int) uint32 { + w.upTracksMu.Lock() + defer w.upTracksMu.Unlock() + if track := w.upTracks[layer]; track != nil { return uint32(track.SSRC()) } return 0 } -func (w *WebRTCReceiver) Codec() webrtc.RTPCodecParameters { - return w.codec -} - -func (w *WebRTCReceiver) Mime() mime.MimeType { - return mime.NormalizeMimeType(w.codec.MimeType) -} - -func (w *WebRTCReceiver) HeaderExtensions() []webrtc.RTPHeaderExtensionParameter { - return w.receiver.GetParameters().HeaderExtensions -} - -func (w *WebRTCReceiver) Kind() webrtc.RTPCodecType { - return w.kind -} - func (w *WebRTCReceiver) AddUpTrack(track TrackRemote, buff *buffer.Buffer) error { - if w.closed.Load() { + if w.isClosed.Load() { return ErrReceiverClosed } layer := int32(0) - if w.Kind() == webrtc.RTPCodecTypeVideo && !w.isSVC { - layer = buffer.RidToSpatialLayer(track.RID(), w.trackInfo.Load()) + if w.Kind() == webrtc.RTPCodecTypeVideo && w.videoLayerMode != livekit.VideoLayer_MULTIPLE_SPATIAL_LAYERS_PER_STREAM { + layer = buffer.GetSpatialLayerForRid(w.Mime(), track.RID(), w.ReceiverBase.TrackInfo()) } - buff.SetLogger(w.logger.WithValues("layer", layer)) - buff.SetAudioLevelParams(audio.AudioLevelParams{ - Config: w.audioConfig.AudioLevelConfig, - }) - buff.SetAudioLossProxying(w.audioConfig.EnableLossProxying) - buff.OnRtcpFeedback(w.sendRTCP) - buff.OnRtcpSenderReport(func() { - srData := buff.GetSenderReportData() - w.downTrackSpreader.Broadcast(func(dt TrackSender) { - _ = dt.HandleRTCPSenderReportData(w.codec.PayloadType, w.isSVC, layer, srData) - }) - }) - - if w.Kind() == webrtc.RTPCodecTypeVideo && layer == 0 { - buff.OnCodecChange(w.handleCodecChange) + if layer < 0 { + w.ReceiverBase.Logger().Warnw( + "invalid layer", nil, + "rid", track.RID(), + "trackInfo", logger.Proto(w.ReceiverBase.TrackInfo()), + ) + return ErrInvalidLayer } - var duration time.Duration - switch layer { - case 2: - duration = w.pliThrottleConfig.HighQuality - case 1: - duration = w.pliThrottleConfig.MidQuality - case 0: - duration = w.pliThrottleConfig.LowQuality - default: - duration = w.pliThrottleConfig.MidQuality - } - if duration != 0 { - buff.SetPLIThrottle(duration.Nanoseconds()) - } - - w.bufferMu.Lock() + w.upTracksMu.Lock() if w.upTracks[layer] != nil { - w.bufferMu.Unlock() + w.upTracksMu.Unlock() return ErrDuplicateLayer } w.upTracks[layer] = track - w.buffers[layer] = buff - rtt := w.rtt - w.bufferMu.Unlock() + w.upTracksMu.Unlock() - buff.SetRTT(rtt) - buff.SetPaused(w.streamTrackerManager.IsPaused()) - - if w.Kind() == webrtc.RTPCodecTypeVideo && w.useTrackers { - w.streamTrackerManager.AddTracker(layer) - } - - go w.forwardRTP(layer, buff) + w.ReceiverBase.AddBuffer(buff, layer) + buff.OnRtcpFeedback(w.sendRTCP) + w.ReceiverBase.StartBuffer(buff, layer) return nil } -// SetUpTrackPaused indicates upstream will not be sending any data. -// this will reflect the "muted" status and will pause streamtracker to ensure we don't turn off -// the layer -func (w *WebRTCReceiver) SetUpTrackPaused(paused bool) { - w.streamTrackerManager.SetPaused(paused) +func (w *WebRTCReceiver) NumUpTracks() int { + numUpTracks := 0 - w.bufferMu.RLock() - for _, buff := range w.buffers { - if buff == nil { - continue + w.upTracksMu.Lock() + for _, track := range w.upTracks { + if track != nil { + numUpTracks++ } - - buff.SetPaused(paused) } - w.bufferMu.RUnlock() + w.upTracksMu.Unlock() - w.connectionStats.UpdateMute(paused) + return numUpTracks } -func (w *WebRTCReceiver) AddDownTrack(track TrackSender) error { - if w.closed.Load() { - return ErrReceiverClosed - } - - if w.downTrackSpreader.HasDownTrack(track.SubscriberID()) { - w.logger.Infow("subscriberID already exists, replacing downtrack", "subscriberID", track.SubscriberID()) - } - - track.UpTrackMaxPublishedLayerChange(w.streamTrackerManager.GetMaxPublishedLayer()) - track.UpTrackMaxTemporalLayerSeenChange(w.streamTrackerManager.GetMaxTemporalLayerSeen()) - - w.downTrackSpreader.Store(track) - w.logger.Debugw("downtrack added", "subscriberID", track.SubscriberID()) - return nil -} - -func (w *WebRTCReceiver) GetDownTracks() []TrackSender { - return w.downTrackSpreader.GetDownTracks() +func (w *WebRTCReceiver) UpdateTrackInfo(ti *livekit.TrackInfo) { + w.ReceiverBase.UpdateTrackInfo(ti) + w.connectionStats.UpdateMute(ti.GetMuted()) } func (w *WebRTCReceiver) notifyMaxExpectedLayer(layer int32) { @@ -499,9 +231,8 @@ func (w *WebRTCReceiver) notifyMaxExpectedLayer(layer int32) { } expectedBitrate := int64(0) - for _, vl := range ti.Layers { - l := buffer.VideoQualityToSpatialLayer(vl.Quality, ti) - if l <= layer { + for _, vl := range buffer.GetVideoLayersForMimeType(w.Mime(), ti) { + if vl.SpatialLayer <= layer { expectedBitrate += int64(vl.Bitrate) } } @@ -510,89 +241,54 @@ func (w *WebRTCReceiver) notifyMaxExpectedLayer(layer int32) { } func (w *WebRTCReceiver) SetMaxExpectedSpatialLayer(layer int32) { - w.streamTrackerManager.SetMaxExpectedSpatialLayer(layer) + w.ReceiverBase.SetMaxExpectedSpatialLayer(layer) + w.notifyMaxExpectedLayer(layer) if layer == buffer.InvalidLayerSpatial { w.connectionStats.UpdateLayerMute(true) } else { w.connectionStats.UpdateLayerMute(false) - w.connectionStats.AddLayerTransition(w.streamTrackerManager.DistanceToDesired()) + w.connectionStats.AddLayerTransition(w.ReceiverBase.StreamTrackerManager().DistanceToDesired()) } } // StreamTrackerManagerListener.OnAvailableLayersChanged func (w *WebRTCReceiver) OnAvailableLayersChanged() { - w.downTrackSpreader.Broadcast(func(dt TrackSender) { - dt.UpTrackLayersChange() - }) - - w.connectionStats.AddLayerTransition(w.streamTrackerManager.DistanceToDesired()) + w.connectionStats.AddLayerTransition(w.ReceiverBase.StreamTrackerManager().DistanceToDesired()) } // StreamTrackerManagerListener.OnBitrateAvailabilityChanged func (w *WebRTCReceiver) OnBitrateAvailabilityChanged() { - w.downTrackSpreader.Broadcast(func(dt TrackSender) { - dt.UpTrackBitrateAvailabilityChange() - }) } // StreamTrackerManagerListener.OnMaxPublishedLayerChanged func (w *WebRTCReceiver) OnMaxPublishedLayerChanged(maxPublishedLayer int32) { - w.downTrackSpreader.Broadcast(func(dt TrackSender) { - dt.UpTrackMaxPublishedLayerChange(maxPublishedLayer) - }) - w.notifyMaxExpectedLayer(maxPublishedLayer) - w.connectionStats.AddLayerTransition(w.streamTrackerManager.DistanceToDesired()) + w.connectionStats.AddLayerTransition(w.ReceiverBase.StreamTrackerManager().DistanceToDesired()) } // StreamTrackerManagerListener.OnMaxTemporalLayerSeenChanged func (w *WebRTCReceiver) OnMaxTemporalLayerSeenChanged(maxTemporalLayerSeen int32) { - w.downTrackSpreader.Broadcast(func(dt TrackSender) { - dt.UpTrackMaxTemporalLayerSeenChange(maxTemporalLayerSeen) - }) - - w.connectionStats.AddLayerTransition(w.streamTrackerManager.DistanceToDesired()) + w.connectionStats.AddLayerTransition(w.ReceiverBase.StreamTrackerManager().DistanceToDesired()) } // StreamTrackerManagerListener.OnMaxAvailableLayerChanged func (w *WebRTCReceiver) OnMaxAvailableLayerChanged(maxAvailableLayer int32) { - if onMaxLayerChange := w.getOnMaxLayerChange(); onMaxLayerChange != nil { - onMaxLayerChange(maxAvailableLayer) - } } // StreamTrackerManagerListener.OnBitrateReport func (w *WebRTCReceiver) OnBitrateReport(availableLayers []int32, bitrates Bitrates) { - w.downTrackSpreader.Broadcast(func(dt TrackSender) { - dt.UpTrackBitrateReport(availableLayers, bitrates) - }) - - w.connectionStats.AddLayerTransition(w.streamTrackerManager.DistanceToDesired()) + w.connectionStats.AddLayerTransition(w.ReceiverBase.StreamTrackerManager().DistanceToDesired()) } -func (w *WebRTCReceiver) GetLayeredBitrate() ([]int32, Bitrates) { - return w.streamTrackerManager.GetLayeredBitrate() -} - -// OnCloseHandler method to be called on remote tracked removed +// OnCloseHandler method to be called on remote track removed func (w *WebRTCReceiver) OnCloseHandler(fn func()) { w.onCloseHandler = fn } -// DeleteDownTrack removes a DownTrack from a Receiver -func (w *WebRTCReceiver) DeleteDownTrack(subscriberID livekit.ParticipantID) { - if w.closed.Load() { - return - } - - w.downTrackSpreader.Free(subscriberID) - w.logger.Debugw("downtrack deleted", "subscriberID", subscriberID) -} - func (w *WebRTCReceiver) sendRTCP(packets []rtcp.Packet) { - if packets == nil || w.closed.Load() { + if packets == nil || w.isClosed.Load() { return } @@ -601,93 +297,10 @@ func (w *WebRTCReceiver) sendRTCP(packets []rtcp.Packet) { } } -func (w *WebRTCReceiver) SendPLI(layer int32, force bool) { - // SVC-TODO : should send LRR (Layer Refresh Request) instead of PLI - buff := w.getBuffer(layer) - if buff == nil { - return - } - - buff.SendPLI(force) -} - -func (w *WebRTCReceiver) getBuffer(layer int32) *buffer.Buffer { - w.bufferMu.RLock() - defer w.bufferMu.RUnlock() - - return w.getBufferLocked(layer) -} - -func (w *WebRTCReceiver) getBufferLocked(layer int32) *buffer.Buffer { - // for svc codecs, use layer = 0 always. - // spatial layers are in-built and handled by single buffer - if w.isSVC { - layer = 0 - } - - if layer < 0 || int(layer) >= len(w.buffers) { - return nil - } - - return w.buffers[layer] -} - -func (w *WebRTCReceiver) ReadRTP(buf []byte, layer uint8, esn uint64) (int, error) { - b := w.getBuffer(int32(layer)) - if b == nil { - return 0, ErrBufferNotFound - } - - return b.GetPacket(buf, esn) -} - -func (w *WebRTCReceiver) GetTrackStats() *livekit.RTPStats { - w.bufferMu.RLock() - defer w.bufferMu.RUnlock() - - stats := make([]*livekit.RTPStats, 0, len(w.buffers)) - for _, buff := range w.buffers { - if buff == nil { - continue - } - - sswl := buff.GetStats() - if sswl == nil { - continue - } - - stats = append(stats, sswl) - } - - return rtpstats.AggregateRTPStats(stats) -} - -func (w *WebRTCReceiver) GetAudioLevel() (float64, bool) { - if w.Kind() == webrtc.RTPCodecTypeVideo { - return 0, false - } - - w.bufferMu.RLock() - defer w.bufferMu.RUnlock() - - for _, buff := range w.buffers { - if buff == nil { - continue - } - - return buff.GetAudioLevel() - } - - return 0, false -} - func (w *WebRTCReceiver) GetDeltaStats() map[uint32]*buffer.StreamStatsWithLayers { - w.bufferMu.RLock() - defer w.bufferMu.RUnlock() - - deltaStats := make(map[uint32]*buffer.StreamStatsWithLayers, len(w.buffers)) - - for layer, buff := range w.buffers { + buffers := w.ReceiverBase.GetAllBuffers() + deltaStats := make(map[uint32]*buffer.StreamStatsWithLayers, len(buffers)) + for layer, buff := range buffers { if buff == nil { continue } @@ -709,11 +322,9 @@ func (w *WebRTCReceiver) GetDeltaStats() map[uint32]*buffer.StreamStatsWithLayer } func (w *WebRTCReceiver) GetLastSenderReportTime() time.Time { - w.bufferMu.RLock() - defer w.bufferMu.RUnlock() - + buffers := w.ReceiverBase.GetAllBuffers() latestSRTime := time.Time{} - for _, buff := range w.buffers { + for _, buff := range buffers { if buff == nil { continue } @@ -727,120 +338,22 @@ func (w *WebRTCReceiver) GetLastSenderReportTime() time.Time { return latestSRTime } -func (w *WebRTCReceiver) forwardRTP(layer int32, buff *buffer.Buffer) { - defer func() { - w.closeOnce.Do(func() { - w.closed.Store(true) - w.closeTracks() - if pr := w.primaryReceiver.Load(); pr != nil { - pr.Close() - } - if pr := w.redReceiver.Load(); pr != nil { - pr.Close() - } - }) - - w.streamTrackerManager.RemoveTracker(layer) - if w.isSVC { - w.streamTrackerManager.RemoveAllTrackers() - } - }() - - var spatialTrackers [buffer.DefaultMaxLayerSpatial + 1]streamtracker.StreamTrackerWorker - if layer < 0 || int(layer) >= len(spatialTrackers) { - w.logger.Errorw("invalid layer", nil, "layer", layer) - return - } - spatialTrackers[layer] = w.streamTrackerManager.GetTracker(layer) - - pktBuf := make([]byte, bucket.MaxPktSize) - for { - pkt, err := buff.ReadExtended(pktBuf) - if err == io.EOF { - return - } - - if pkt.Packet.PayloadType != uint8(w.codec.PayloadType) { - // drop packets as we don't support codec fallback directly - continue - } - - spatialLayer := layer - if pkt.Spatial >= 0 { - // svc packet, take spatial layer info from packet - spatialLayer = pkt.Spatial - } - if int(spatialLayer) >= len(spatialTrackers) { - w.logger.Errorw( - "unexpected spatial layer", nil, - "spatialLayer", spatialLayer, - "pktSpatialLayer", pkt.Spatial, - ) - continue - } - - writeCount := w.downTrackSpreader.Broadcast(func(dt TrackSender) { - _ = dt.WriteRTP(pkt, spatialLayer) - }) - - if f := w.redPktWriter.Load(); f != nil { - writeCount += f.(redPktWriteFunc)(pkt, spatialLayer) - } - - // track delay/jitter - if writeCount > 0 && w.forwardStats != nil { - w.forwardStats.Update(pkt.Arrival, time.Now().UnixNano()) - } - - // track video layers - if w.Kind() == webrtc.RTPCodecTypeVideo { - if spatialTrackers[spatialLayer] == nil { - spatialTrackers[spatialLayer] = w.streamTrackerManager.GetTracker(spatialLayer) - if spatialTrackers[spatialLayer] == nil { - spatialTrackers[spatialLayer] = w.streamTrackerManager.AddTracker(spatialLayer) - } - } - if spatialTrackers[spatialLayer] != nil { - spatialTrackers[spatialLayer].Observe( - pkt.Temporal, - len(pkt.RawPacket), - len(pkt.Packet.Payload), - pkt.Packet.Marker, - pkt.Packet.Timestamp, - pkt.DependencyDescriptor, - ) - } - } - } -} - -// closeTracks close all tracks from Receiver -func (w *WebRTCReceiver) closeTracks() { +func (w *WebRTCReceiver) onClosed() { w.connectionStats.Close() - w.streamTrackerManager.Close() - - closeTrackSenders(w.downTrackSpreader.ResetAndGetDownTracks()) if w.onCloseHandler != nil { w.onCloseHandler() } } -func (w *WebRTCReceiver) DebugInfo() map[string]interface{} { - isSimulcast := !w.isSVC - if ti := w.trackInfo.Load(); ti != nil { - isSimulcast = isSimulcast && len(ti.Layers) > 1 - } - info := map[string]interface{}{ - "SVC": w.isSVC, - "Simulcast": isSimulcast, - } +func (w *WebRTCReceiver) DebugInfo() map[string]any { + info := w.ReceiverBase.DebugInfo() - w.bufferMu.RLock() - upTrackInfo := make([]map[string]interface{}, 0, len(w.upTracks)) + w.upTracksMu.Lock() + upTrackInfo := make([]map[string]any, 0, len(w.upTracks)) for layer, ut := range w.upTracks { if ut != nil { - upTrackInfo = append(upTrackInfo, map[string]interface{}{ + upTrackInfo = append(upTrackInfo, map[string]any{ "Layer": layer, "SSRC": ut.SSRC(), "Msid": ut.Msid(), @@ -848,110 +361,10 @@ func (w *WebRTCReceiver) DebugInfo() map[string]interface{} { }) } } - w.bufferMu.RUnlock() + w.upTracksMu.Unlock() info["UpTracks"] = upTrackInfo return info } -func (w *WebRTCReceiver) GetPrimaryReceiverForRed() TrackReceiver { - if !w.isRED || w.closed.Load() { - return w - } - - if w.primaryReceiver.Load() == nil { - pr := NewRedPrimaryReceiver(w, DownTrackSpreaderParams{ - Threshold: w.lbThreshold, - Logger: w.logger, - }) - if w.primaryReceiver.CompareAndSwap(nil, pr) { - w.redPktWriter.Store(redPktWriteFunc(pr.ForwardRTP)) - } - } - return w.primaryReceiver.Load() -} - -func (w *WebRTCReceiver) GetRedReceiver() TrackReceiver { - if w.isRED || w.closed.Load() { - return w - } - - if w.redReceiver.Load() == nil { - pr := NewRedReceiver(w, DownTrackSpreaderParams{ - Threshold: w.lbThreshold, - Logger: w.logger, - }) - if w.redReceiver.CompareAndSwap(nil, pr) { - w.redPktWriter.Store(redPktWriteFunc(pr.ForwardRTP)) - } - } - return w.redReceiver.Load() -} - -func (w *WebRTCReceiver) GetTemporalLayerFpsForSpatial(layer int32) []float32 { - b := w.getBuffer(layer) - if b == nil { - return nil - } - - if !w.isSVC { - return b.GetTemporalLayerFpsForSpatial(0) - } - - return b.GetTemporalLayerFpsForSpatial(layer) -} - -func (w *WebRTCReceiver) AddOnReady(fn func()) { - // webRTCReceiver is always ready after created - fn() -} - -func (w *WebRTCReceiver) handleCodecChange(newCodec webrtc.RTPCodecParameters) { - // we don't support the codec fallback directly, set the codec state to invalid once it happens - w.SetCodecState(ReceiverCodecStateInvalid) -} - -func (w *WebRTCReceiver) AddOnCodecStateChange(f func(webrtc.RTPCodecParameters, ReceiverCodecState)) { - w.codecStateLock.Lock() - w.onCodecStateChange = append(w.onCodecStateChange, f) - w.codecStateLock.Unlock() -} - -func (w *WebRTCReceiver) CodecState() ReceiverCodecState { - w.codecStateLock.Lock() - defer w.codecStateLock.Unlock() - - return w.codecState -} - -func (w *WebRTCReceiver) SetCodecState(state ReceiverCodecState) { - w.codecStateLock.Lock() - if w.codecState == state || w.codecState == ReceiverCodecStateInvalid { - w.codecStateLock.Unlock() - return - } - - w.codecState = state - fns := w.onCodecStateChange - w.codecStateLock.Unlock() - - for _, f := range fns { - f(w.codec, state) - } -} - // ----------------------------------------------------------- - -// closes all track senders in parallel, returns when all are closed -func closeTrackSenders(senders []TrackSender) { - wg := sync.WaitGroup{} - for _, dt := range senders { - dt := dt - wg.Add(1) - go func() { - defer wg.Done() - dt.Close() - }() - } - wg.Wait() -} diff --git a/pkg/sfu/receiver_base.go b/pkg/sfu/receiver_base.go new file mode 100644 index 0000000..d056d25 --- /dev/null +++ b/pkg/sfu/receiver_base.go @@ -0,0 +1,1294 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sfu + +import ( + "errors" + "fmt" + "io" + "slices" + "strings" + "sync" + "time" + + "github.com/pion/webrtc/v4" + "go.uber.org/atomic" + + "github.com/livekit/mediatransportutil/pkg/bucket" + "github.com/livekit/mediatransportutil/pkg/codec" + "github.com/livekit/protocol/codecs/mime" + "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/logger" + "github.com/livekit/protocol/utils" + "github.com/livekit/protocol/utils/mono" + + "github.com/livekit/livekit-server/pkg/sfu/audio" + "github.com/livekit/livekit-server/pkg/sfu/buffer" + "github.com/livekit/livekit-server/pkg/sfu/rtpstats" + "github.com/livekit/livekit-server/pkg/sfu/streamtracker" + sfuutils "github.com/livekit/livekit-server/pkg/sfu/utils" +) + +var ( + ErrReceiverClosed = errors.New("receiver closed") + ErrDownTrackAlreadyExist = errors.New("DownTrack already exist") + ErrDuplicateLayer = errors.New("duplicate layer") + ErrInvalidLayer = errors.New("invalid layer") +) + +// -------------------------------------- + +type PLIThrottleConfig struct { + LowQuality time.Duration `yaml:"low_quality,omitempty"` + MidQuality time.Duration `yaml:"mid_quality,omitempty"` + HighQuality time.Duration `yaml:"high_quality,omitempty"` +} + +var ( + DefaultPLIThrottleConfig = PLIThrottleConfig{ + LowQuality: 500 * time.Millisecond, + MidQuality: time.Second, + HighQuality: time.Second, + } +) + +// -------------------------------------- + +type AudioConfig struct { + audio.AudioLevelConfig `yaml:",inline"` + + // enable red encoding downtrack for opus only audio up track + ActiveREDEncoding bool `yaml:"active_red_encoding,omitempty"` + // enable proxying weakest subscriber loss to publisher in RTCP Receiver Report + EnableLossProxying bool `yaml:"enable_loss_proxying,omitempty"` +} + +var ( + DefaultAudioConfig = AudioConfig{ + AudioLevelConfig: audio.DefaultAudioLevelConfig, + } +) + +// -------------------------------------- + +type Bitrates [buffer.DefaultMaxLayerSpatial + 1][buffer.DefaultMaxLayerTemporal + 1]int64 + +// -------------------------------------- + +type ReceiverCodecState int + +const ( + ReceiverCodecStateNormal ReceiverCodecState = iota + ReceiverCodecStateSuspended + ReceiverCodecStateInvalid +) + +// -------------------------------------- + +// TrackReceiver defines an interface receive media from remote peer +type TrackReceiver interface { + TrackID() livekit.TrackID + StreamID() string + + // returns the initial codec of the receiver, it is determined by the track's codec + // and will not change if the codec changes during the session (publisher changes codec) + Codec() webrtc.RTPCodecParameters + Mime() mime.MimeType + VideoLayerMode() livekit.VideoLayer_Mode + HeaderExtensions() []webrtc.RTPHeaderExtensionParameter + IsClosed() bool + + ReadRTP(buf []byte, layer uint8, esn uint64) (int, error) + GetLayeredBitrate() ([]int32, Bitrates) + + GetAudioLevel() (float64, bool) + + SendPLI(layer int32, force bool) + + SetMaxExpectedSpatialLayer(layer int32) + + AddDownTrack(track TrackSender) error + DeleteDownTrack(participantID livekit.ParticipantID) + GetDownTracks() []TrackSender + + DebugInfo() map[string]any + + TrackInfo() *livekit.TrackInfo + UpdateTrackInfo(ti *livekit.TrackInfo) + + // Get primary receiver if this receiver represents a RED codec; otherwise it will return itself + GetPrimaryReceiverForRed() TrackReceiver + + // Get red receiver for primary codec, used by forward red encodings for opus only codec + GetRedReceiver() TrackReceiver + + GetTemporalLayerFpsForSpatial(layer int32) []float32 + + GetTrackStats() *livekit.RTPStats + + // AddOnReady adds a function to be called when the receiver is ready, the callback + // could be called immediately if the receiver is ready when the callback is added + AddOnReady(func()) + + AddOnCodecStateChange(func(webrtc.RTPCodecParameters, ReceiverCodecState)) + CodecState() ReceiverCodecState + + // VideoSizes returns the video size parsed from rtp packet for each spatial layer. + VideoSizes() []codec.VideoSize + + // closes all associated buffers and issues a resync to all attached downtracks so that + // they can resync and have proper sequncing without gaps in sequence numbers / timestamps + Restart(reason string) +} + +// -------------------------------------- + +type REDTransformer interface { + TrackReceiver + ForwardRTP(pkt *buffer.ExtPacket, spatialLayer int32) int32 + ForwardRTCPSenderReport( + payloadType webrtc.PayloadType, + layer int32, + publisherSRData *livekit.RTCPSenderReportState, + ) + GetDownTracks() []TrackSender + ResyncDownTracks() + OnStreamRestart() + CanClose() bool + Close() +} + +// -------------------------------------- + +type bufferPromise struct { + ready chan struct{} +} + +type ReceiverBaseParams struct { + TrackID livekit.TrackID + StreamID string + Kind webrtc.RTPCodecType + Codec webrtc.RTPCodecParameters + HeaderExtensions []webrtc.RTPHeaderExtensionParameter + Logger logger.Logger + StreamTrackerManagerConfig StreamTrackerManagerConfig + StreamTrackerManagerListener StreamTrackerManagerListener + IsSelfClosing bool + OnNewBufferNeeded func(int32, *livekit.TrackInfo) (buffer.BufferProvider, error) + OnClosed func() +} + +type ReceiverBase struct { + params ReceiverBaseParams + + pliThrottleConfig PLIThrottleConfig + audioConfig AudioConfig + enableRTPStreamRestartDetection bool + lbThreshold int + forwardStats *ForwardStats + + codecStateLock sync.Mutex + codecState ReceiverCodecState + onCodecStateChange []func(webrtc.RTPCodecParameters, ReceiverCodecState) + + isRED bool + videoLayerMode livekit.VideoLayer_Mode + + bufferMu sync.RWMutex + buffers [buffer.DefaultMaxLayerSpatial + 1]buffer.BufferProvider + bufferPromises [buffer.DefaultMaxLayerSpatial + 1]*bufferPromise + trackInfo *livekit.TrackInfo + + videoSizeMu sync.RWMutex + videoSizes [buffer.DefaultMaxLayerSpatial + 1]codec.VideoSize + onVideoSizeChanged func() + + rtt uint32 + + streamTrackerManager *StreamTrackerManager + + downTrackSpreader *sfuutils.DownTrackSpreader[TrackSender] + + onMaxLayerChange func(mimeType mime.MimeType, maxLayer int32) + + redTransformer atomic.Pointer[REDTransformer] + + forwardersGeneration atomic.Uint32 + forwardersWaitGroup *sync.WaitGroup + restartInProgress bool + + isClosed atomic.Bool +} + +func NewReceiverBase(params ReceiverBaseParams, trackInfo *livekit.TrackInfo, codecState ReceiverCodecState) *ReceiverBase { + r := &ReceiverBase{ + params: params, + codecState: codecState, + isRED: mime.IsMimeTypeStringRED(params.Codec.MimeType), + trackInfo: utils.CloneProto(trackInfo), + videoLayerMode: buffer.GetVideoLayerModeForMimeType(mime.NormalizeMimeType(params.Codec.MimeType), trackInfo), + } + + r.downTrackSpreader = sfuutils.NewDownTrackSpreader[TrackSender](sfuutils.DownTrackSpreaderParams{ + Threshold: r.lbThreshold, + Logger: params.Logger, + }) + + r.streamTrackerManager = NewStreamTrackerManager( + params.Logger, + trackInfo, + r.Mime(), + r.params.Codec.ClockRate, + params.StreamTrackerManagerConfig, + ) + r.streamTrackerManager.SetListener(r) + + r.startForwardersGeneration() + + return r +} + +func (r *ReceiverBase) Close(reason string, clearBuffers bool) { + if r.isClosed.Swap(true) { + return + } + + if clearBuffers { + r.ClearAllBuffers(reason) + } + r.streamTrackerManager.Close() + + closeTrackSenders(r.downTrackSpreader.ResetAndGetDownTracks()) + + if rt := r.loadREDTransformer(); rt != nil { + rt.Close() + } + + if r.params.OnClosed != nil { + r.params.OnClosed() + } +} + +func (r *ReceiverBase) CanClose() bool { + if r.IsClosed() { + return true + } + + if r.downTrackSpreader.DownTrackCount() != 0 { + return false + } + + if rt := r.loadREDTransformer(); rt != nil { + return rt.CanClose() + } + + return true +} + +func (r *ReceiverBase) SetPLIThrottleConfig(pliThrottleConfig PLIThrottleConfig) { + r.pliThrottleConfig = pliThrottleConfig +} + +func (r *ReceiverBase) SetAudioConfig(audioConfig AudioConfig) { + r.audioConfig = audioConfig +} + +func (r *ReceiverBase) SetEnableRTPStreamRestartDetection(enableRTPStremRestartDetection bool) { + r.enableRTPStreamRestartDetection = enableRTPStremRestartDetection +} + +func (r *ReceiverBase) SetLBThreshold(lbThreshold int) { + r.lbThreshold = lbThreshold +} + +func (r *ReceiverBase) SetForwardStats(forwardStats *ForwardStats) { + r.forwardStats = forwardStats +} + +func (r *ReceiverBase) Logger() logger.Logger { + return r.params.Logger +} + +func (r *ReceiverBase) TrackInfo() *livekit.TrackInfo { + r.bufferMu.RLock() + defer r.bufferMu.RUnlock() + + return utils.CloneProto(r.trackInfo) +} + +func (r *ReceiverBase) UpdateTrackInfo(ti *livekit.TrackInfo) { + r.bufferMu.Lock() + existingVersion := utils.TimedVersionFromProto(r.trackInfo.Version) + updateVersion := utils.TimedVersionFromProto(ti.Version) + if updateVersion.Compare(existingVersion) < 0 { + r.bufferMu.Unlock() + r.params.Logger.Debugw( + "not updating to older version", + "existing", logger.Proto(r.trackInfo), + "updated", logger.Proto(ti), + ) + return + } + + shouldResync := utils.TimedVersionFromProto(r.trackInfo.Version) != utils.TimedVersionFromProto(ti.Version) + if shouldResync { + r.params.Logger.Debugw( + "updating track info", + "existing", logger.Proto(r.trackInfo), + "updated", logger.Proto(ti), + "shouldResync", shouldResync, + ) + } + r.trackInfo = utils.CloneProto(ti) + + paused := r.trackInfo.GetMuted() + for _, buff := range r.buffers { + if buff == nil { + continue + } + + buff.SetPaused(paused) + } + r.bufferMu.Unlock() + + r.streamTrackerManager.UpdateTrackInfo(ti) + + if shouldResync { + r.Restart("update-track-info") + } +} + +func (r *ReceiverBase) Restart(reason string) { + r.params.Logger.Infow("restarting receiver", "reason", reason) + r.restartInternal(reason, false) +} + +func (r *ReceiverBase) restartInternal(reason string, isDetected bool) { + r.params.Logger.Debugw( + "restart receiver", + "reason", reason, + "isDetected", isDetected, + "isClosed", r.IsClosed(), + ) + + if r.IsClosed() { + return + } + + // 1. guard against concurrent restarts + r.bufferMu.Lock() + if r.restartInProgress { + r.params.Logger.Debugw("restart receiver, skipping duplicate") + r.bufferMu.Unlock() + return + } + r.restartInProgress = true + + // 2. advance forwarder generation + r.forwardersGeneration.Inc() + r.params.Logger.Debugw( + "restart receiver, advanced forwarder generation", + "forwardersGeneration", r.forwardersGeneration.Load(), + ) + r.bufferMu.Unlock() + + // 3. mark for restart all the buffers + // if a stream restart was detected, skip external restart + // + // NOTE: The case of external restart and detected restart (which usually comes from one buffer) + // racing will miss restart on all buffers if detected restart from one buffer adds the guard + // against concurrent restart. But, that condition should be very rare if at all. + // External restart happens when the underlying track changes or when seeking + if !isDetected { + for layer, buff := range r.GetAllBuffers() { + if buff == nil { + continue + } + + r.params.Logger.Debugw("restart receiver, marking buffer for restart", "layer", layer) + buff.MarkForRestartStream(reason) + } + r.params.Logger.Debugw("restart receiver, marked buffers for restart") + } + + // 4. wait for the forwarders to finish + r.waitForForwardersStop() + r.params.Logger.Debugw("restart receiver, forwarders stopped") + + // 5. restart all the buffers + // Two phase restart - mark, followed by restart to ensure + // a fresh start after existing forwarder is stopped + if !isDetected { + for layer, buff := range r.GetAllBuffers() { + if buff == nil { + continue + } + + r.params.Logger.Debugw("restart receiver, restarting buffer", "layer", layer) + buff.RestartStream(reason) + } + r.params.Logger.Debugw("restart receiver, restarted buffers") + } + + // 6. reset stream tracker + r.streamTrackerManager.RemoveAllTrackers() + r.params.Logger.Debugw("restart receiver, stream trackers removed") + + // 7. signal attached downtracks to resync so that they can have proper sequencing on a receiver restart + r.downTrackSpreader.Broadcast(func(dt TrackSender) { + dt.ReceiverRestart(r) + }) + if rt := r.loadREDTransformer(); rt != nil { + rt.OnStreamRestart() + } + r.params.Logger.Debugw("restart receiver, down tracks signalled") + + // 8. move forwarder generation ahead + r.startForwardersGeneration() + r.params.Logger.Debugw( + "restart receiver, restarted forwarder generation", + "forwardersGeneration", r.forwardersGeneration.Load(), + ) + + r.bufferMu.Lock() + // 9. release restart hold + r.restartInProgress = false + + // 10. restart forwarders + for layer, buff := range r.buffers { + if buff == nil { + continue + } + + r.params.Logger.Debugw("restart receiver, restarting forwarder", "layer", layer) + r.startForwarderForBufferLocked(int32(layer), buff) + } + r.params.Logger.Debugw("restart receiver, restarted forwarders") + r.bufferMu.Unlock() +} + +func (r *ReceiverBase) OnMaxLayerChange(fn func(mimeType mime.MimeType, maxLayer int32)) { + r.bufferMu.Lock() + r.onMaxLayerChange = fn + r.bufferMu.Unlock() +} + +func (r *ReceiverBase) getOnMaxLayerChange() func(mimeType mime.MimeType, maxLayer int32) { + r.bufferMu.RLock() + defer r.bufferMu.RUnlock() + + return r.onMaxLayerChange +} + +func (r *ReceiverBase) IsClosed() bool { + return r.isClosed.Load() +} + +func (r *ReceiverBase) SetRTT(rtt uint32) { + r.bufferMu.Lock() + if r.rtt == rtt || rtt == 0 { + r.bufferMu.Unlock() + return + } + + r.rtt = rtt + buffers := r.buffers + r.bufferMu.Unlock() + + for _, buff := range buffers { + if buff == nil { + continue + } + + buff.SetRTT(rtt) + } +} + +func (r *ReceiverBase) TrackID() livekit.TrackID { + return r.params.TrackID +} + +func (r *ReceiverBase) StreamID() string { + return r.params.StreamID +} + +func (r *ReceiverBase) Codec() webrtc.RTPCodecParameters { + return r.params.Codec +} + +func (r *ReceiverBase) Mime() mime.MimeType { + return mime.NormalizeMimeType(r.params.Codec.MimeType) +} + +func (r *ReceiverBase) VideoLayerMode() livekit.VideoLayer_Mode { + return r.videoLayerMode +} + +func (r *ReceiverBase) HeaderExtensions() []webrtc.RTPHeaderExtensionParameter { + return r.params.HeaderExtensions +} + +func (r *ReceiverBase) Kind() webrtc.RTPCodecType { + return r.params.Kind +} + +func (r *ReceiverBase) StreamTrackerManager() *StreamTrackerManager { + return r.streamTrackerManager +} + +func (r *ReceiverBase) AddDownTrack(track TrackSender) error { + if r.IsClosed() { + return ErrReceiverClosed + } + + if r.downTrackSpreader.HasDownTrack(track.SubscriberID()) { + r.params.Logger.Infow("subscriberID already exists, replacing downtrack", "subscriberID", track.SubscriberID()) + } + + track.UpTrackMaxPublishedLayerChange(r.streamTrackerManager.GetMaxPublishedLayer()) + track.UpTrackMaxTemporalLayerSeenChange(r.streamTrackerManager.GetMaxTemporalLayerSeen()) + + r.downTrackSpreader.Store(track) + r.params.Logger.Debugw("downtrack added", "subscriberID", track.SubscriberID()) + return nil +} + +func (r *ReceiverBase) DeleteDownTrack(subscriberID livekit.ParticipantID) { + r.downTrackSpreader.Free(subscriberID) + r.params.Logger.Debugw("downtrack deleted", "subscriberID", subscriberID) +} + +func (r *ReceiverBase) GetDownTracks() []TrackSender { + downTracks := r.downTrackSpreader.GetDownTracks() + if rt := r.loadREDTransformer(); rt != nil { + downTracks = append(downTracks, rt.GetDownTracks()...) + } + return downTracks +} + +func (r *ReceiverBase) SetMaxExpectedSpatialLayer(layer int32) { + prevMax := r.streamTrackerManager.SetMaxExpectedSpatialLayer(layer) + r.params.Logger.Debugw("max expected layer change", "layer", layer, "prevMax", prevMax) + + r.bufferMu.RLock() + // stop key frame seeders of stopped layers + for idx := layer + 1; idx <= prevMax; idx++ { + if r.buffers[idx] != nil { + r.buffers[idx].StopKeyFrameSeeder() + } + } + + // start key frame seeders of newly expected layers + for idx := prevMax + 1; idx <= layer; idx++ { + if r.buffers[idx] != nil { + r.buffers[idx].StartKeyFrameSeeder() + } + } + r.bufferMu.RUnlock() +} + +// StreamTrackerManagerListener.OnAvailableLayersChanged +func (r *ReceiverBase) OnAvailableLayersChanged() { + r.downTrackSpreader.Broadcast(func(dt TrackSender) { + dt.UpTrackLayersChange() + }) + + if r.params.StreamTrackerManagerListener != nil { + r.params.StreamTrackerManagerListener.OnAvailableLayersChanged() + } +} + +// StreamTrackerManagerListener.OnBitrateAvailabilityChanged +func (r *ReceiverBase) OnBitrateAvailabilityChanged() { + r.downTrackSpreader.Broadcast(func(dt TrackSender) { + dt.UpTrackBitrateAvailabilityChange() + }) + + if r.params.StreamTrackerManagerListener != nil { + r.params.StreamTrackerManagerListener.OnBitrateAvailabilityChanged() + } +} + +// StreamTrackerManagerListener.OnMaxPublishedLayerChanged +func (r *ReceiverBase) OnMaxPublishedLayerChanged(maxPublishedLayer int32) { + r.downTrackSpreader.Broadcast(func(dt TrackSender) { + dt.UpTrackMaxPublishedLayerChange(maxPublishedLayer) + }) + + if r.params.StreamTrackerManagerListener != nil { + r.params.StreamTrackerManagerListener.OnMaxPublishedLayerChanged(maxPublishedLayer) + } +} + +// StreamTrackerManagerListener.OnMaxTemporalLayerSeenChanged +func (r *ReceiverBase) OnMaxTemporalLayerSeenChanged(maxTemporalLayerSeen int32) { + r.downTrackSpreader.Broadcast(func(dt TrackSender) { + dt.UpTrackMaxTemporalLayerSeenChange(maxTemporalLayerSeen) + }) + + if r.params.StreamTrackerManagerListener != nil { + r.params.StreamTrackerManagerListener.OnMaxTemporalLayerSeenChanged(maxTemporalLayerSeen) + } +} + +// StreamTrackerManagerListener.OnMaxAvailableLayerChanged +func (r *ReceiverBase) OnMaxAvailableLayerChanged(maxAvailableLayer int32) { + if onMaxLayerChange := r.getOnMaxLayerChange(); onMaxLayerChange != nil { + onMaxLayerChange(r.Mime(), maxAvailableLayer) + } + + if r.params.StreamTrackerManagerListener != nil { + r.params.StreamTrackerManagerListener.OnMaxAvailableLayerChanged(maxAvailableLayer) + } +} + +// StreamTrackerManagerListener.OnBitrateReport +func (r *ReceiverBase) OnBitrateReport(availableLayers []int32, bitrates Bitrates) { + r.downTrackSpreader.Broadcast(func(dt TrackSender) { + dt.UpTrackBitrateReport(availableLayers, bitrates) + }) + + if r.params.StreamTrackerManagerListener != nil { + r.params.StreamTrackerManagerListener.OnBitrateReport(availableLayers, bitrates) + } +} + +func (r *ReceiverBase) GetLayeredBitrate() ([]int32, Bitrates) { + return r.streamTrackerManager.GetLayeredBitrate() +} + +func (r *ReceiverBase) SendPLI(layer int32, force bool) { + // SVC-TODO : should send LRR (Layer Refresh Request) instead of PLI + buff := r.GetOrCreateBuffer(layer) + if buff == nil { + return + } + + buff.SendPLI(force) +} + +func (r *ReceiverBase) getBuffer(layer int32) (buffer.BufferProvider, int32) { + r.bufferMu.RLock() + defer r.bufferMu.RUnlock() + + return r.getBufferLocked(layer) +} + +func (r *ReceiverBase) getBufferLocked(layer int32) (buffer.BufferProvider, int32) { + // for svc codecs, use layer = 0 always. + // spatial layers are in-built and handled by single buffer + if r.videoLayerMode == livekit.VideoLayer_MULTIPLE_SPATIAL_LAYERS_PER_STREAM { + layer = 0 + } + + if layer < 0 || int(layer) >= len(r.buffers) { + return nil, layer + } + + return r.buffers[layer], layer +} + +func (r *ReceiverBase) GetOrCreateBuffer(layer int32) buffer.BufferProvider { + r.bufferMu.Lock() + + if r.IsClosed() { + r.bufferMu.Unlock() + return nil + } + + var buff buffer.BufferProvider + if buff, layer = r.getBufferLocked(layer); buff != nil { + r.bufferMu.Unlock() + return buff + } + + if r.params.OnNewBufferNeeded == nil { + r.bufferMu.Unlock() + return nil + } + + if bp := r.bufferPromises[layer]; bp != nil { + r.bufferMu.Unlock() + <-bp.ready + + buff, _ := r.getBuffer(layer) + return buff + } + + bp := &bufferPromise{ + ready: make(chan struct{}), + } + r.bufferPromises[layer] = bp + + ti := utils.CloneProto(r.trackInfo) + r.bufferMu.Unlock() + + defer close(bp.ready) + + buff, err := r.params.OnNewBufferNeeded(layer, ti) + if err != nil { + r.params.Logger.Errorw("could not create buffer", err) + + r.bufferMu.Lock() + r.bufferPromises[layer] = nil + r.bufferMu.Unlock() + + return nil + } + + r.bufferMu.Lock() + r.buffers[layer] = buff + rtt := r.rtt + paused := r.trackInfo.GetMuted() + r.bufferMu.Unlock() + + r.setupBuffer(buff, layer, rtt, paused) + return buff +} + +func (r *ReceiverBase) setupBuffer(buff buffer.BufferProvider, layer int32, rtt uint32, paused bool) { + buff.SetLogger(r.params.Logger.WithValues("layer", layer)) + buff.SetAudioLevelConfig(r.audioConfig.AudioLevelConfig) + buff.SetStreamRestartDetection(r.enableRTPStreamRestartDetection) + buff.OnRtcpSenderReport(func() { + srData := buff.GetSenderReportData() + r.downTrackSpreader.Broadcast(func(dt TrackSender) { + _ = dt.HandleRTCPSenderReportData(r.params.Codec.PayloadType, layer, srData) + }) + + if rt := r.loadREDTransformer(); rt != nil { + rt.ForwardRTCPSenderReport(r.params.Codec.PayloadType, layer, srData) + } + }) + buff.OnVideoSizeChanged(func(videoSize []codec.VideoSize) { + r.videoSizeMu.Lock() + if r.videoLayerMode == livekit.VideoLayer_MULTIPLE_SPATIAL_LAYERS_PER_STREAM { + copy(r.videoSizes[:], videoSize) + } else { + r.videoSizes[layer] = videoSize[0] + } + r.params.Logger.Debugw("video size changed", "size", r.videoSizes) + cb := r.onVideoSizeChanged + r.videoSizeMu.Unlock() + + if cb != nil { + cb() + } + }) + if r.Kind() == webrtc.RTPCodecTypeVideo && layer == 0 { + buff.OnCodecChange(r.handleCodecChange) + } + buff.OnStreamRestart(func(reason string) { + r.restartInternal(reason, true) + }) + + var duration time.Duration + switch layer { + case 2: + duration = r.pliThrottleConfig.HighQuality + case 1: + duration = r.pliThrottleConfig.MidQuality + case 0: + duration = r.pliThrottleConfig.LowQuality + default: + duration = r.pliThrottleConfig.MidQuality + } + if duration != 0 { + buff.SetPLIThrottle(duration.Nanoseconds()) + } + + buff.SetRTT(rtt) + buff.SetPaused(paused) +} + +func (r *ReceiverBase) AddBuffer(buff buffer.BufferProvider, layer int32) { + r.bufferMu.Lock() + r.buffers[layer] = buff + rtt := r.rtt + paused := r.trackInfo.GetMuted() + r.bufferMu.Unlock() + + r.setupBuffer(buff, layer, rtt, paused) +} + +func (r *ReceiverBase) StartBuffer(buff buffer.BufferProvider, layer int32) { + r.bufferMu.Lock() + r.startForwarderForBufferLocked(layer, buff) + r.bufferMu.Unlock() +} + +func (r *ReceiverBase) GetAllBuffers() [buffer.DefaultMaxLayerSpatial + 1]buffer.BufferProvider { + buffers := [buffer.DefaultMaxLayerSpatial + 1]buffer.BufferProvider{} + + r.bufferMu.RLock() + defer r.bufferMu.RUnlock() + + for i := range buffers { + buffers[i] = r.buffers[i] + } + return buffers +} + +func (r *ReceiverBase) ClearAllBuffers(reason string) { + r.bufferMu.Lock() + buffers := r.buffers + for idx := range r.buffers { + r.buffers[idx] = nil + r.bufferPromises[idx] = nil + } + r.bufferMu.Unlock() + + for _, buff := range buffers { + if buff == nil { + continue + } + buff.CloseWithReason(reason) + } + + r.streamTrackerManager.RemoveAllTrackers() +} + +func (r *ReceiverBase) ReadRTP(buf []byte, layer uint8, esn uint64) (int, error) { + b, _ := r.getBuffer(int32(layer)) + if b == nil { + return 0, bucket.ErrPacketMismatch + } + + return b.GetPacket(buf, esn) +} + +func (r *ReceiverBase) GetTrackStats() *livekit.RTPStats { + r.bufferMu.RLock() + defer r.bufferMu.RUnlock() + + allStats := make([]*livekit.RTPStats, 0, len(r.buffers)) + for _, buff := range r.buffers { + if buff == nil { + continue + } + + stats := buff.GetStats() + if stats == nil { + continue + } + + allStats = append(allStats, stats) + } + + return rtpstats.AggregateRTPStats(allStats) +} + +func (r *ReceiverBase) GetAudioLevel() (float64, bool) { + if r.Kind() == webrtc.RTPCodecTypeVideo { + return 0, false + } + + r.bufferMu.RLock() + defer r.bufferMu.RUnlock() + + for _, buff := range r.buffers { + if buff == nil { + continue + } + + return buff.GetAudioLevel() + } + + return 0, false +} + +func (r *ReceiverBase) startForwardersGeneration() { + r.bufferMu.Lock() + defer r.bufferMu.Unlock() + + r.forwardersGeneration.Inc() + r.forwardersWaitGroup = &sync.WaitGroup{} +} + +func (r *ReceiverBase) waitForForwardersStop() { + r.bufferMu.Lock() + forwardersWaitGroup := r.forwardersWaitGroup + r.bufferMu.Unlock() + + if forwardersWaitGroup != nil { + forwardersWaitGroup.Wait() + } +} + +func (r *ReceiverBase) startForwarderForBufferLocked(layer int32, buff buffer.BufferProvider) { + if r.restartInProgress { + r.params.Logger.Debugw("restart in progress, deferring starting forwarder", "layer", layer) + return + } + + r.forwardersWaitGroup.Add(1) + + forwarderGeneration := r.forwardersGeneration.Load() + r.params.Logger.Debugw("starting forwarder", "layer", layer, "forwarderGeneration", forwarderGeneration) + go r.forwardRTP(layer, buff, forwarderGeneration, r.forwardersWaitGroup) +} + +func (r *ReceiverBase) forwardRTP( + layer int32, + buff buffer.BufferProvider, + forwarderGeneration uint32, + wg *sync.WaitGroup, +) { + var ( + extPkt *buffer.ExtPacket + err error + ) + + numPacketsForwarded := 0 + numPacketsDropped := 0 + defer func() { + if err == io.EOF { + if r.params.IsSelfClosing { + r.Close("forwarder-done", false) + + r.streamTrackerManager.RemoveTracker(layer) + if r.videoLayerMode == livekit.VideoLayer_MULTIPLE_SPATIAL_LAYERS_PER_STREAM { + r.streamTrackerManager.RemoveAllTrackers() + } + } + } + + r.params.Logger.Debugw( + "closing forwarder", + "layer", layer, + "numPacketsForwarded", numPacketsForwarded, + "numPacketsDropped", numPacketsDropped, + "forwarderGeneration", forwarderGeneration, + "forwardersGeneration", r.forwardersGeneration.Load(), + "error", err, + ) + wg.Done() + }() + + var spatialTrackers [buffer.DefaultMaxLayerSpatial + 1]streamtracker.StreamTrackerWorker + if layer < 0 || int(layer) >= len(spatialTrackers) { + r.params.Logger.Errorw("invalid layer", nil, "layer", layer) + return + } + + pktBuf := make([]byte, bucket.RTPMaxPktSize) + r.params.Logger.Debugw( + "starting forwarding", + "layer", layer, + "forwarderGeneration", forwarderGeneration, + "forwardersGeneration", r.forwardersGeneration.Load(), + ) + + for r.forwardersGeneration.Load() == forwarderGeneration { + extPkt, err = buff.ReadExtended(pktBuf) + if err == io.EOF { + return + } + if extPkt == nil { + continue + } + dequeuedAt := mono.UnixNano() + + if extPkt.Packet.PayloadType != uint8(r.params.Codec.PayloadType) { + // drop packets as we don't support codec fallback directly + r.params.Logger.Debugw( + "dropping packet - payload mismatch", + "packetPayloadType", extPkt.Packet.PayloadType, + "payloadType", r.params.Codec.PayloadType, + ) + numPacketsDropped++ + continue + } + + spatialLayer := layer + if extPkt.Spatial >= 0 { + // svc packet, take spatial layer info from packet + spatialLayer = extPkt.Spatial + } + if int(spatialLayer) >= len(spatialTrackers) { + r.params.Logger.Errorw( + "unexpected spatial layer", nil, + "spatialLayer", spatialLayer, + "pktSpatialLayer", extPkt.Spatial, + ) + numPacketsDropped++ + continue + } + + var writeCount atomic.Int32 + r.downTrackSpreader.Broadcast(func(dt TrackSender) { + writeCount.Add(dt.WriteRTP(extPkt, spatialLayer)) + }) + if rt := r.loadREDTransformer(); rt != nil { + writeCount.Add(rt.ForwardRTP(extPkt, spatialLayer)) + } + + // track delay/jitter + if writeCount.Load() > 0 && r.forwardStats != nil && !extPkt.IsBuffered { + if latency, isHigh := r.forwardStats.Update(extPkt.Arrival, mono.UnixNano()); isHigh { + r.params.Logger.Debugw( + "high forwarding latency", + "latency", time.Duration(latency), + "queuingLatency", time.Duration(dequeuedAt-extPkt.Arrival), + "writeCount", writeCount.Load(), + "isOutOfOrder", extPkt.IsOutOfOrder, + "layer", layer, + ) + } + } + + // track video layers + if r.Kind() == webrtc.RTPCodecTypeVideo { + if spatialTrackers[spatialLayer] == nil { + spatialTrackers[spatialLayer] = r.streamTrackerManager.GetTracker(spatialLayer) + if spatialTrackers[spatialLayer] == nil { + if r.videoLayerMode == livekit.VideoLayer_MULTIPLE_SPATIAL_LAYERS_PER_STREAM && extPkt.DependencyDescriptor != nil { + r.streamTrackerManager.AddDependencyDescriptorTrackers() + } + spatialTrackers[spatialLayer] = r.streamTrackerManager.AddTracker(spatialLayer) + } + } + if spatialTrackers[spatialLayer] != nil { + spatialTrackers[spatialLayer].Observe( + extPkt.Temporal, + len(extPkt.RawPacket), + len(extPkt.Packet.Payload), + extPkt.Packet.Marker, + extPkt.Packet.Timestamp, + extPkt.DependencyDescriptor, + ) + } + } + + numPacketsForwarded++ + + buffer.ReleaseExtPacket(extPkt) + } +} + +func (r *ReceiverBase) DebugInfo() map[string]any { + videoLayerMode := buffer.GetVideoLayerModeForMimeType(r.Mime(), r.TrackInfo()) + info := map[string]any{ + "Mime": r.Mime().String(), + "VideoLayerMode": videoLayerMode.String(), + } + + return info +} + +func (r *ReceiverBase) GetPrimaryReceiverForRed() TrackReceiver { + r.bufferMu.Lock() + defer r.bufferMu.Unlock() + + if !r.isRED || r.IsClosed() { + return r + } + + rt := r.loadREDTransformer() + if rt == nil { + pr := NewRedPrimaryReceiver(r, sfuutils.DownTrackSpreaderParams{ + Threshold: r.lbThreshold, + Logger: r.params.Logger, + }) + r.redTransformer.Store(&pr) + return pr + } else { + if pr, ok := rt.(*RedPrimaryReceiver); ok { + return pr + } + } + return nil +} + +func (r *ReceiverBase) GetRedReceiver() TrackReceiver { + r.bufferMu.Lock() + defer r.bufferMu.Unlock() + + if r.isRED || r.IsClosed() { + return r + } + + rt := r.loadREDTransformer() + if rt == nil { + pr := NewRedReceiver(r, sfuutils.DownTrackSpreaderParams{ + Threshold: r.lbThreshold, + Logger: r.params.Logger, + }) + r.redTransformer.Store(&pr) + return pr + } else { + if pr, ok := rt.(*RedReceiver); ok { + return pr + } + } + return nil +} + +func (r *ReceiverBase) GetTemporalLayerFpsForSpatial(layer int32) []float32 { + b, _ := r.getBuffer(layer) + if b == nil { + return nil + } + + if r.videoLayerMode != livekit.VideoLayer_MULTIPLE_SPATIAL_LAYERS_PER_STREAM { + return b.GetTemporalLayerFpsForSpatial(0) + } + + return b.GetTemporalLayerFpsForSpatial(layer) +} + +func (r *ReceiverBase) AddOnReady(fn func()) { + // receiver is always ready after created + fn() +} + +func (r *ReceiverBase) handleCodecChange(newCodec webrtc.RTPCodecParameters) { + // codec fallback is not supported mid-session, i.e. change of codec via payload type change, + // set the codec state to invalid once it happens + r.SetCodecState(ReceiverCodecStateInvalid) +} + +func (r *ReceiverBase) AddOnCodecStateChange(f func(webrtc.RTPCodecParameters, ReceiverCodecState)) { + r.codecStateLock.Lock() + r.onCodecStateChange = append(r.onCodecStateChange, f) + r.codecStateLock.Unlock() +} + +func (r *ReceiverBase) CodecState() ReceiverCodecState { + r.codecStateLock.Lock() + defer r.codecStateLock.Unlock() + + return r.codecState +} + +func (r *ReceiverBase) SetCodecState(state ReceiverCodecState) { + r.codecStateLock.Lock() + if r.codecState == state || r.codecState == ReceiverCodecStateInvalid { + r.codecStateLock.Unlock() + return + } + + r.codecState = state + fns := r.onCodecStateChange + r.codecStateLock.Unlock() + + for _, f := range fns { + f(r.params.Codec, state) + } +} + +func (r *ReceiverBase) SetCodecWithState(codec webrtc.RTPCodecParameters, headerExtensions []webrtc.RTPHeaderExtensionParameter, codecState ReceiverCodecState) { + r.checkCodecChanged(codec, headerExtensions) + + r.codecStateLock.Lock() + if codecState == r.codecState { + r.codecStateLock.Unlock() + return + } + + var fireChange bool + var reason string + onCodecStateChange := r.onCodecStateChange + r.params.Logger.Infow("codec state changed", "from", r.codecState, "to", codecState) + switch codecState { + case ReceiverCodecStateNormal: + // TODO: support codec recovery + r.codecStateLock.Unlock() + return + + case ReceiverCodecStateSuspended: + reason = "codec suspended" + fallthrough + + case ReceiverCodecStateInvalid: + r.codecState = codecState + fireChange = true + reason = "codec invalid" + } + r.codecStateLock.Unlock() + + if fireChange { + r.ClearAllBuffers(reason) + + for _, fn := range onCodecStateChange { + fn(r.params.Codec, codecState) + } + } +} + +func (r *ReceiverBase) checkCodecChanged(codec webrtc.RTPCodecParameters, headerExtensions []webrtc.RTPHeaderExtensionParameter) { + existingFmtp := strings.Split(r.params.Codec.SDPFmtpLine, ";") + slices.Sort(existingFmtp) + checkFmtp := strings.Split(codec.SDPFmtpLine, ";") + slices.Sort(checkFmtp) + if !mime.IsMimeTypeStringEqual(r.params.Codec.MimeType, codec.MimeType) || !slices.Equal(existingFmtp, checkFmtp) || + r.params.Codec.ClockRate != codec.ClockRate { + err := fmt.Errorf("mime: %s -> %s, fmtp: %s -> %s, clockRate: %d -> %d", + r.params.Codec.MimeType, codec.MimeType, + r.params.Codec.SDPFmtpLine, codec.SDPFmtpLine, + r.params.Codec.ClockRate, codec.ClockRate, + ) + r.params.Logger.Errorw("unexpected change in codec", err) + } + + if len(r.params.HeaderExtensions) != len(headerExtensions) { + err := fmt.Errorf("extensions: %d -> %d", len(r.params.HeaderExtensions), len(headerExtensions)) + r.params.Logger.Errorw("unexpected change in extensions length", err) + } +} + +func (r *ReceiverBase) VideoSizes() []codec.VideoSize { + var sizes []codec.VideoSize + r.videoSizeMu.RLock() + defer r.videoSizeMu.RUnlock() + for _, v := range r.videoSizes { + if v.Width == 0 || v.Height == 0 { + break + } + sizes = append(sizes, v) + } + + return sizes +} + +func (r *ReceiverBase) OnVideoSizeChanged(f func()) { + r.videoSizeMu.Lock() + r.onVideoSizeChanged = f + r.videoSizeMu.Unlock() +} + +func (r *ReceiverBase) loadREDTransformer() REDTransformer { + if rt := r.redTransformer.Load(); rt != nil { + return *rt + } + + return nil +} + +// ----------------------------------------------------------- + +// closes all track senders in parallel, returns when all are closed +func closeTrackSenders(senders []TrackSender) { + wg := sync.WaitGroup{} + for _, dt := range senders { + dt := dt + wg.Add(1) + go func() { + defer wg.Done() + dt.Close() + }() + } + wg.Wait() +} diff --git a/pkg/sfu/receiver_test.go b/pkg/sfu/receiver_test.go index a1f475e..3cfde07 100644 --- a/pkg/sfu/receiver_test.go +++ b/pkg/sfu/receiver_test.go @@ -102,15 +102,15 @@ func BenchmarkWriteRTP(b *testing.B) { } func benchmarkNoPool(b *testing.B, downTracks int) { - for i := 0; i < b.N; i++ { - for dt := 0; dt < downTracks; dt++ { + for b.Loop() { + for range downTracks { writeRTP() } } } func benchmarkPool(b *testing.B, wp *workerpool.WorkerPool, buckets []int) { - for i := 0; i < b.N; i++ { + for b.Loop() { var wg sync.WaitGroup for j := range buckets { downTracks := buckets[j] @@ -130,7 +130,7 @@ func benchmarkPool(b *testing.B, wp *workerpool.WorkerPool, buckets []int) { } func benchmarkGoroutine(b *testing.B, buckets []int) { - for i := 0; i < b.N; i++ { + for b.Loop() { var wg sync.WaitGroup for j := range buckets { downTracks := buckets[j] @@ -150,7 +150,7 @@ func benchmarkGoroutine(b *testing.B, buckets []int) { } func benchmarkLoadBalanced(b *testing.B, numProcs, step, downTracks int) { - for i := 0; i < b.N; i++ { + for b.Loop() { start := atomic.NewUint64(0) step := uint64(step) end := uint64(downTracks) @@ -177,7 +177,7 @@ func benchmarkLoadBalanced(b *testing.B, numProcs, step, downTracks int) { } func benchmarkLoadBalancedPool(b *testing.B, wp *workerpool.WorkerPool, numProcs, step, downTracks int) { - for i := 0; i < b.N; i++ { + for b.Loop() { start := atomic.NewUint64(0) step := uint64(step) end := uint64(downTracks) diff --git a/pkg/sfu/redprimaryreceiver.go b/pkg/sfu/redprimaryreceiver.go index f066205..4963a14 100644 --- a/pkg/sfu/redprimaryreceiver.go +++ b/pkg/sfu/redprimaryreceiver.go @@ -21,12 +21,16 @@ import ( "go.uber.org/atomic" "github.com/pion/rtp" + "github.com/pion/webrtc/v4" "github.com/livekit/livekit-server/pkg/sfu/buffer" + "github.com/livekit/livekit-server/pkg/sfu/utils" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" ) +var _ REDTransformer = (*RedPrimaryReceiver)(nil) + var ( ErrIncompleteRedHeader = errors.New("incomplete red block header") ErrIncompleteRedBlock = errors.New("incomplete red block payload") @@ -34,28 +38,30 @@ var ( type RedPrimaryReceiver struct { TrackReceiver - downTrackSpreader *DownTrackSpreader + downTrackSpreader *utils.DownTrackSpreader[TrackSender] logger logger.Logger closed atomic.Bool redPT uint8 - firstPktReceived bool - lastSeq uint16 + firstPktReceived bool + lastSeq uint16 + highestForwardedExtSequenceNumber uint64 + highestForwardedExtTimestamp uint64 // bitset for upstream packet receive history [lastSeq-8, lastSeq-1], bit 1 represents packet received pktHistory byte } -func NewRedPrimaryReceiver(receiver TrackReceiver, dsp DownTrackSpreaderParams) *RedPrimaryReceiver { +func NewRedPrimaryReceiver(receiver TrackReceiver, dsp utils.DownTrackSpreaderParams) REDTransformer { return &RedPrimaryReceiver{ TrackReceiver: receiver, - downTrackSpreader: NewDownTrackSpreader(dsp), + downTrackSpreader: utils.NewDownTrackSpreader[TrackSender](dsp), logger: dsp.Logger, redPT: uint8(receiver.Codec().PayloadType), } } -func (r *RedPrimaryReceiver) ForwardRTP(pkt *buffer.ExtPacket, spatialLayer int32) int { +func (r *RedPrimaryReceiver) ForwardRTP(pkt *buffer.ExtPacket, spatialLayer int32) int32 { // extract primary payload from RED and forward to downtracks if r.downTrackSpreader.DownTrackCount() == 0 { return 0 @@ -63,9 +69,11 @@ func (r *RedPrimaryReceiver) ForwardRTP(pkt *buffer.ExtPacket, spatialLayer int3 if pkt.Packet.PayloadType != r.redPT { // forward non-red packet directly - return r.downTrackSpreader.Broadcast(func(dt TrackSender) { - _ = dt.WriteRTP(pkt, spatialLayer) + var writeCount atomic.Int32 + r.downTrackSpreader.Broadcast(func(dt TrackSender) { + writeCount.Add(dt.WriteRTP(pkt, spatialLayer)) }) + return writeCount.Load() } pkts, err := r.getSendPktsFromRed(pkt.Packet) @@ -74,7 +82,7 @@ func (r *RedPrimaryReceiver) ForwardRTP(pkt *buffer.ExtPacket, spatialLayer int3 return 0 } - var count int + var writeCount atomic.Int32 for i, sendPkt := range pkts { pPkt := *pkt if i != len(pkts)-1 { @@ -85,13 +93,53 @@ func (r *RedPrimaryReceiver) ForwardRTP(pkt *buffer.ExtPacket, spatialLayer int3 } pPkt.Packet = sendPkt + if r.highestForwardedExtTimestamp != 0 { + if (pPkt.ExtSequenceNumber > r.highestForwardedExtSequenceNumber && pPkt.ExtTimestamp < r.highestForwardedExtTimestamp) || + (pPkt.ExtSequenceNumber < r.highestForwardedExtSequenceNumber && pPkt.ExtTimestamp > r.highestForwardedExtTimestamp) { + r.logger.Warnw( + "sequence number OR timestamp inversion, dropping", nil, + "numPackets", len(pkts), + "primaryIncomingSN", pkt.Packet.Header.SequenceNumber, + "primaryIncomingTS", pkt.Packet.Header.Timestamp, + "primaryExtractedSN", pkts[len(pkts)-1].SequenceNumber, + "primaryExtractedTS", pkts[len(pkts)-1].Timestamp, + "primaryESN", pkt.ExtSequenceNumber, + "primaryETS", pkt.ExtTimestamp, + "packetIndex", i, + "packetExtractedSN", pkts[i].SequenceNumber, + "packetESN", pPkt.ExtSequenceNumber, + "packetExtractedTS", pkts[i].Timestamp, + "packetETS", pPkt.ExtTimestamp, + "pktHistory", r.pktHistory, + "redHeader", pkt.Packet.Payload[:10], + "payloadSize", len(pkt.Packet.Payload), + "lastSeq", r.lastSeq, + "highestFowardedExtSequenceNumber", r.highestForwardedExtSequenceNumber, + "highestFowardedExtTimestamp", r.highestForwardedExtTimestamp, + ) + continue // drop the packet which causes the inversion + } + } + r.highestForwardedExtSequenceNumber = max(r.highestForwardedExtSequenceNumber, pPkt.ExtSequenceNumber) + r.highestForwardedExtTimestamp = max(r.highestForwardedExtTimestamp, pPkt.ExtTimestamp) + // not modify the ExtPacket.RawPacket here for performance since it is not used by the DownTrack, // otherwise it should be set to the correct value (marshal the primary rtp packet) - count += r.downTrackSpreader.Broadcast(func(dt TrackSender) { - _ = dt.WriteRTP(&pPkt, spatialLayer) + r.downTrackSpreader.Broadcast(func(dt TrackSender) { + writeCount.Add(dt.WriteRTP(&pPkt, spatialLayer)) }) } - return count + return writeCount.Load() +} + +func (r *RedPrimaryReceiver) ForwardRTCPSenderReport( + payloadType webrtc.PayloadType, + layer int32, + publisherSRData *livekit.RTCPSenderReportState, +) { + r.downTrackSpreader.Broadcast(func(dt TrackSender) { + _ = dt.HandleRTCPSenderReportData(payloadType, layer, publisherSRData) + }) } func (r *RedPrimaryReceiver) AddDownTrack(track TrackSender) error { @@ -127,6 +175,12 @@ func (r *RedPrimaryReceiver) ResyncDownTracks() { }) } +func (r *RedPrimaryReceiver) OnStreamRestart() { + r.downTrackSpreader.Broadcast(func(dt TrackSender) { + dt.ReceiverRestart(r) + }) +} + func (r *RedPrimaryReceiver) IsClosed() bool { return r.closed.Load() } @@ -191,7 +245,7 @@ func (r *RedPrimaryReceiver) getSendPktsFromRed(rtp *rtp.Packet) ([]*rtp.Packet, var recoverBits byte if needRecover { bitIndex := r.lastSeq - rtp.SequenceNumber - for i := 0; i < maxRedCount; i++ { + for i := range maxRedCount { if bitIndex > 7 { break } @@ -205,6 +259,8 @@ func (r *RedPrimaryReceiver) getSendPktsFromRed(rtp *rtp.Packet) ([]*rtp.Packet, return extractPktsFromRed(rtp, recoverBits) } +// --------------------------------------- + type block struct { tsOffset uint32 length int diff --git a/pkg/sfu/redreceiver.go b/pkg/sfu/redreceiver.go index 7fbd9a6..b93f343 100644 --- a/pkg/sfu/redreceiver.go +++ b/pkg/sfu/redreceiver.go @@ -21,13 +21,17 @@ import ( "go.uber.org/atomic" "github.com/pion/rtp" + "github.com/pion/webrtc/v4" "github.com/livekit/livekit-server/pkg/sfu/buffer" + "github.com/livekit/livekit-server/pkg/sfu/utils" "github.com/livekit/mediatransportutil/pkg/bucket" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" ) +var _ REDTransformer = (*RedReceiver)(nil) + const ( maxRedCount = 2 mtuSize = 1500 @@ -42,22 +46,22 @@ const ( type RedReceiver struct { TrackReceiver - downTrackSpreader *DownTrackSpreader + downTrackSpreader *utils.DownTrackSpreader[TrackSender] logger logger.Logger closed atomic.Bool pktBuff [maxRedCount]*rtp.Packet redPayloadBuf [mtuSize]byte } -func NewRedReceiver(receiver TrackReceiver, dsp DownTrackSpreaderParams) *RedReceiver { +func NewRedReceiver(receiver TrackReceiver, dsp utils.DownTrackSpreaderParams) REDTransformer { return &RedReceiver{ TrackReceiver: receiver, - downTrackSpreader: NewDownTrackSpreader(dsp), + downTrackSpreader: utils.NewDownTrackSpreader[TrackSender](dsp), logger: dsp.Logger, } } -func (r *RedReceiver) ForwardRTP(pkt *buffer.ExtPacket, spatialLayer int32) int { +func (r *RedReceiver) ForwardRTP(pkt *buffer.ExtPacket, spatialLayer int32) int32 { // encode RED payload from primary payload and forward to downtracks if r.downTrackSpreader.DownTrackCount() == 0 { return 0 @@ -65,9 +69,11 @@ func (r *RedReceiver) ForwardRTP(pkt *buffer.ExtPacket, spatialLayer int32) int // fallback to primary codec if payload size exceeds redundant block length if len(pkt.Packet.Payload) >= maxRedPayload { - return r.downTrackSpreader.Broadcast(func(dt TrackSender) { - _ = dt.WriteRTP(pkt, spatialLayer) + var writeCount atomic.Int32 + r.downTrackSpreader.Broadcast(func(dt TrackSender) { + writeCount.Add(dt.WriteRTP(pkt, spatialLayer)) }) + return writeCount.Load() } redLen, err := r.encodeRedForPrimary(pkt.Packet, r.redPayloadBuf[:]) @@ -78,14 +84,26 @@ func (r *RedReceiver) ForwardRTP(pkt *buffer.ExtPacket, spatialLayer int32) int pPkt := *pkt redRtpPacket := *pkt.Packet - redRtpPacket.PayloadType = 63 + redRtpPacket.PayloadType = opusRedPT redRtpPacket.Payload = r.redPayloadBuf[:redLen] pPkt.Packet = &redRtpPacket // not modify the ExtPacket.RawPacket here for performance since it is not used by the DownTrack, // otherwise it should be set to the correct value (marshal the primary rtp packet) - return r.downTrackSpreader.Broadcast(func(dt TrackSender) { - _ = dt.WriteRTP(&pPkt, spatialLayer) + var writeCount atomic.Int32 + r.downTrackSpreader.Broadcast(func(dt TrackSender) { + writeCount.Add(dt.WriteRTP(&pPkt, spatialLayer)) + }) + return writeCount.Load() +} + +func (r *RedReceiver) ForwardRTCPSenderReport( + payloadType webrtc.PayloadType, + layer int32, + publisherSRData *livekit.RTCPSenderReportState, +) { + r.downTrackSpreader.Broadcast(func(dt TrackSender) { + _ = dt.HandleRTCPSenderReportData(payloadType, layer, publisherSRData) }) } @@ -122,6 +140,12 @@ func (r *RedReceiver) ResyncDownTracks() { }) } +func (r *RedReceiver) OnStreamRestart() { + r.downTrackSpreader.Broadcast(func(dt TrackSender) { + dt.ReceiverRestart(r) + }) +} + func (r *RedReceiver) CanClose() bool { return r.closed.Load() || r.downTrackSpreader.DownTrackCount() == 0 } diff --git a/pkg/sfu/redreceiver_test.go b/pkg/sfu/redreceiver_test.go index 16a1340..06df497 100644 --- a/pkg/sfu/redreceiver_test.go +++ b/pkg/sfu/redreceiver_test.go @@ -26,8 +26,7 @@ import ( ) const ( - tsStep = uint32(48000 / 1000 * 10) - opusREDPT = 63 + tsStep = uint32(48000 / 1000 * 10) ) type dummyDowntrack struct { @@ -36,24 +35,26 @@ type dummyDowntrack struct { receivedPkts []*rtp.Packet } -func (dt *dummyDowntrack) WriteRTP(p *buffer.ExtPacket, _ int32) error { +func (dt *dummyDowntrack) WriteRTP(p *buffer.ExtPacket, _ int32) int32 { dt.lastReceivedPkt = p.Packet dt.receivedPkts = append(dt.receivedPkts, p.Packet) - return nil + return 1 } -func (dt *dummyDowntrack) TrackInfoAvailable() {} - func TestRedReceiver(t *testing.T) { dt := &dummyDowntrack{TrackSender: &DownTrack{}} t.Run("normal", func(t *testing.T) { w := &WebRTCReceiver{ - isRED: true, - kind: webrtc.RTPCodecTypeAudio, - logger: logger.GetLogger(), + ReceiverBase: &ReceiverBase{ + params: ReceiverBaseParams{ + Kind: webrtc.RTPCodecTypeAudio, + Logger: logger.GetLogger(), + }, + isRED: true, + }, } - require.Equal(t, w.GetRedReceiver(), w) + require.Equal(t, w.GetRedReceiver(), w.ReceiverBase) w.isRED = false red := w.GetRedReceiver().(*RedReceiver) require.NotNil(t, red) @@ -75,15 +76,19 @@ func TestRedReceiver(t *testing.T) { t.Run("packet lost and jump", func(t *testing.T) { w := &WebRTCReceiver{ - kind: webrtc.RTPCodecTypeAudio, - logger: logger.GetLogger(), + ReceiverBase: &ReceiverBase{ + params: ReceiverBaseParams{ + Kind: webrtc.RTPCodecTypeAudio, + Logger: logger.GetLogger(), + }, + }, } red := w.GetRedReceiver().(*RedReceiver) require.NoError(t, red.AddDownTrack(dt)) header := rtp.Header{SequenceNumber: 65534, Timestamp: (uint32(1) << 31) - 2*tsStep, PayloadType: 111} expectPkt := make([]*rtp.Packet, 0, maxRedCount+1) - for i := 0; i < 10; i++ { + for i := range 10 { if i%2 == 0 { header.SequenceNumber++ header.Timestamp += tsStep @@ -126,8 +131,12 @@ func TestRedReceiver(t *testing.T) { t.Run("unorder and repeat", func(t *testing.T) { w := &WebRTCReceiver{ - kind: webrtc.RTPCodecTypeAudio, - logger: logger.GetLogger(), + ReceiverBase: &ReceiverBase{ + params: ReceiverBaseParams{ + Kind: webrtc.RTPCodecTypeAudio, + Logger: logger.GetLogger(), + }, + }, } red := w.GetRedReceiver().(*RedReceiver) require.NoError(t, red.AddDownTrack(dt)) @@ -158,11 +167,15 @@ func TestRedReceiver(t *testing.T) { t.Run("encoding exceed space", func(t *testing.T) { w := &WebRTCReceiver{ - isRED: true, - kind: webrtc.RTPCodecTypeAudio, - logger: logger.GetLogger(), + ReceiverBase: &ReceiverBase{ + params: ReceiverBaseParams{ + Kind: webrtc.RTPCodecTypeAudio, + Logger: logger.GetLogger(), + }, + isRED: true, + }, } - require.Equal(t, w.GetRedReceiver(), w) + require.Equal(t, w.GetRedReceiver(), w.ReceiverBase) w.isRED = false red := w.GetRedReceiver().(*RedReceiver) require.NotNil(t, red) @@ -183,11 +196,15 @@ func TestRedReceiver(t *testing.T) { t.Run("large timestamp gap", func(t *testing.T) { w := &WebRTCReceiver{ - isRED: true, - kind: webrtc.RTPCodecTypeAudio, - logger: logger.GetLogger(), + ReceiverBase: &ReceiverBase{ + params: ReceiverBaseParams{ + Kind: webrtc.RTPCodecTypeAudio, + Logger: logger.GetLogger(), + }, + isRED: true, + }, } - require.Equal(t, w.GetRedReceiver(), w) + require.Equal(t, w.GetRedReceiver(), w.ReceiverBase) w.isRED = false red := w.GetRedReceiver().(*RedReceiver) require.NotNil(t, red) @@ -247,7 +264,7 @@ func verifyEncodingEqual(t *testing.T, p1, p2 *rtp.Packet) { func generatePkts(header rtp.Header, count int, tsStep uint32) []*rtp.Packet { pkts := make([]*rtp.Packet, 0, count) - for i := 0; i < count; i++ { + for range count { hbuf, _ := header.Marshal() pkts = append(pkts, &rtp.Packet{ Header: header, @@ -271,7 +288,7 @@ func generateRedPkts(t *testing.T, pkts []*rtp.Packet, redCount int) []*rtp.Pack } buf := make([]byte, mtuSize) redPkt := *pkt - redPkt.PayloadType = opusREDPT + redPkt.PayloadType = opusRedPT encoded, err := encodeRedForPrimary(encodingPkts, pkt, buf) require.NoError(t, err) redPkt.Payload = buf[:encoded] @@ -283,11 +300,15 @@ func generateRedPkts(t *testing.T, pkts []*rtp.Packet, redCount int) []*rtp.Pack func testRedRedPrimaryReceiver(t *testing.T, maxPktCount, redCount int, sendPktIdx, expectPktIdx []int) { dt := &dummyDowntrack{TrackSender: &DownTrack{}} w := &WebRTCReceiver{ - kind: webrtc.RTPCodecTypeAudio, - logger: logger.GetLogger(), - codec: webrtc.RTPCodecParameters{PayloadType: opusREDPT, RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: "audio/red"}}, + ReceiverBase: &ReceiverBase{ + params: ReceiverBaseParams{ + Kind: webrtc.RTPCodecTypeAudio, + Logger: logger.GetLogger(), + Codec: webrtc.RTPCodecParameters{PayloadType: opusRedPT, RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: "audio/red"}}, + }, + }, } - require.Equal(t, w.GetPrimaryReceiverForRed(), w) + require.Equal(t, w.GetPrimaryReceiverForRed(), w.ReceiverBase) w.isRED = true red := w.GetPrimaryReceiverForRed().(*RedPrimaryReceiver) require.NotNil(t, red) @@ -313,10 +334,14 @@ func testRedRedPrimaryReceiver(t *testing.T, maxPktCount, redCount int, sendPktI func TestRedPrimaryReceiver(t *testing.T) { w := &WebRTCReceiver{ - kind: webrtc.RTPCodecTypeAudio, - logger: logger.GetLogger(), + ReceiverBase: &ReceiverBase{ + params: ReceiverBaseParams{ + Kind: webrtc.RTPCodecTypeAudio, + Logger: logger.GetLogger(), + }, + }, } - require.Equal(t, w.GetPrimaryReceiverForRed(), w) + require.Equal(t, w.GetPrimaryReceiverForRed(), w.ReceiverBase) w.isRED = true red := w.GetPrimaryReceiverForRed().(*RedPrimaryReceiver) require.NotNil(t, red) @@ -324,7 +349,7 @@ func TestRedPrimaryReceiver(t *testing.T) { t.Run("packet should send only once", func(t *testing.T) { maxPktCount := 19 var sendPktIndex []int - for i := 0; i < maxPktCount; i++ { + for i := range maxPktCount { sendPktIndex = append(sendPktIndex, i) } testRedRedPrimaryReceiver(t, maxPktCount, maxRedCount, sendPktIndex, sendPktIndex) @@ -333,7 +358,7 @@ func TestRedPrimaryReceiver(t *testing.T) { t.Run("packet duplicate and unorder", func(t *testing.T) { maxPktCount := 19 var sendPktIndex []int - for i := 0; i < maxPktCount; i++ { + for i := range maxPktCount { sendPktIndex = append(sendPktIndex, i) if i > 0 { sendPktIndex = append(sendPktIndex, i-1) @@ -346,7 +371,7 @@ func TestRedPrimaryReceiver(t *testing.T) { t.Run("full recover", func(t *testing.T) { maxPktCount := 19 var sendPktIndex, recvPktIndex []int - for i := 0; i < maxPktCount; i++ { + for i := range maxPktCount { recvPktIndex = append(recvPktIndex, i) // drop packets covered by red encoding @@ -383,11 +408,15 @@ func TestRedPrimaryReceiver(t *testing.T) { t.Run("mixed primary codec", func(t *testing.T) { dt := &dummyDowntrack{TrackSender: &DownTrack{}} w := &WebRTCReceiver{ - kind: webrtc.RTPCodecTypeAudio, - logger: logger.GetLogger(), - codec: webrtc.RTPCodecParameters{PayloadType: opusREDPT, RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: "audio/red"}}, + ReceiverBase: &ReceiverBase{ + params: ReceiverBaseParams{ + Kind: webrtc.RTPCodecTypeAudio, + Logger: logger.GetLogger(), + Codec: webrtc.RTPCodecParameters{PayloadType: opusRedPT, RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: "audio/red"}}, + }, + }, } - require.Equal(t, w.GetPrimaryReceiverForRed(), w) + require.Equal(t, w.GetPrimaryReceiverForRed(), w.ReceiverBase) w.isRED = true red := w.GetPrimaryReceiverForRed().(*RedPrimaryReceiver) require.NotNil(t, red) diff --git a/pkg/sfu/rtpextension/dependencydescriptor/bitstreamwriter.go b/pkg/sfu/rtpextension/dependencydescriptor/bitstreamwriter.go index 1e5ffac..b7d1c5d 100644 --- a/pkg/sfu/rtpextension/dependencydescriptor/bitstreamwriter.go +++ b/pkg/sfu/rtpextension/dependencydescriptor/bitstreamwriter.go @@ -103,7 +103,7 @@ func (w *BitStreamWriter) writePartialByte(source uint8, sourceBitCount int, tar } func (w *BitStreamWriter) WriteNonSymmetric(val, numValues uint32) error { - if !(val < numValues && numValues <= 1<<31) { + if val >= numValues || numValues > 1<<31 { return fmt.Errorf("invalid argument, val %d, numValues %d", val, numValues) } if numValues == 1 { diff --git a/pkg/sfu/rtpextension/dependencydescriptor/dependencydescriptorreader.go b/pkg/sfu/rtpextension/dependencydescriptor/dependencydescriptorreader.go index 2ca21ff..b3baf6f 100644 --- a/pkg/sfu/rtpextension/dependencydescriptor/dependencydescriptorreader.go +++ b/pkg/sfu/rtpextension/dependencydescriptor/dependencydescriptorreader.go @@ -229,12 +229,13 @@ func (r *DependencyDescriptorReader) readTemplateLayers() error { } nextLayerIdc = nextLayerIdcType(idc) - if nextLayerIdc == nextTemporalLayer { + switch nextLayerIdc { + case nextTemporalLayer: temporalId++ if temporalId >= MaxTemporalIds { return ErrDDReaderTooManyTemporalLayers } - } else if nextLayerIdc == nextSpatialLayer { + case nextSpatialLayer: spatialId++ temporalId = 0 if spatialId >= MaxSpatialIds { @@ -242,7 +243,7 @@ func (r *DependencyDescriptorReader) readTemplateLayers() error { } } - if !(nextLayerIdc != noMoreLayer && r.buffer.Ok()) { + if nextLayerIdc == noMoreLayer || !r.buffer.Ok() { break } } diff --git a/pkg/sfu/rtpextension/dependencydescriptor/dependencydescriptorwriter.go b/pkg/sfu/rtpextension/dependencydescriptor/dependencydescriptorwriter.go index d392c90..3c6ba72 100644 --- a/pkg/sfu/rtpextension/dependencydescriptor/dependencydescriptorwriter.go +++ b/pkg/sfu/rtpextension/dependencydescriptor/dependencydescriptorwriter.go @@ -16,8 +16,7 @@ package dependencydescriptor import ( "fmt" - - "golang.org/x/exp/slices" + "slices" ) type TemplateMatch struct { @@ -246,8 +245,8 @@ func (w *DependencyDescriptorWriter) writeExtendedFields() error { } func (w *DependencyDescriptorWriter) writeTemplateDependencyStructure() error { - if !(w.structure.StructureId >= 0 && w.structure.StructureId < MaxTemplates && - w.structure.NumDecodeTargets > 0 && w.structure.NumDecodeTargets <= MaxDecodeTargets) { + if w.structure.StructureId < 0 || w.structure.StructureId >= MaxTemplates || + w.structure.NumDecodeTargets <= 0 || w.structure.NumDecodeTargets > MaxDecodeTargets { return fmt.Errorf("invalid arguments, structureId: %d, numDecodeTargets: %d", w.structure.StructureId, w.structure.NumDecodeTargets) } @@ -283,8 +282,8 @@ func (w *DependencyDescriptorWriter) writeTemplateDependencyStructure() error { } func (w *DependencyDescriptorWriter) writeTemplateLayers() error { - if !(len(w.structure.Templates) > 0 && len(w.structure.Templates) <= MaxTemplates && - w.structure.Templates[0].SpatialId == 0 && w.structure.Templates[0].TemporalId == 0) { + if len(w.structure.Templates) <= 0 || len(w.structure.Templates) > MaxTemplates || + w.structure.Templates[0].SpatialId != 0 || w.structure.Templates[0].TemporalId != 0 { return fmt.Errorf("invalid templates, len %d, templates[0]: spatialId %d, temporalId %d", len(w.structure.Templates), w.structure.Templates[0].SpatialId, w.structure.Templates[0].TemporalId) } for i := 1; i < len(w.structure.Templates); i++ { diff --git a/pkg/sfu/rtpmunger.go b/pkg/sfu/rtpmunger.go index 7eb41d7..f897a70 100644 --- a/pkg/sfu/rtpmunger.go +++ b/pkg/sfu/rtpmunger.go @@ -77,8 +77,8 @@ func NewRTPMunger(logger logger.Logger) *RTPMunger { } } -func (r *RTPMunger) DebugInfo() map[string]interface{} { - return map[string]interface{}{ +func (r *RTPMunger) DebugInfo() map[string]any { + return map[string]any{ "ExtHighestIncomingSN": r.extHighestIncomingSN, "ExtLastSN": r.extLastSN, "ExtSecondLastSN": r.extSecondLastSN, @@ -102,6 +102,10 @@ func (r *RTPMunger) GetState() *livekit.RTPMungerState { } } +func (r *RTPMunger) GetExtLastTimestamp() uint64 { + return r.extLastTS +} + func (r *RTPMunger) GetTSOffset() uint64 { return r.tsOffset } @@ -185,7 +189,7 @@ func (r *RTPMunger) UpdateAndGetSnTs(extPkt *buffer.ExtPacket, marker bool) (Tra r.secondLastMarker = r.lastMarker r.lastMarker = marker - if extPkt.KeyFrame { + if extPkt.IsKeyFrame { r.extRtxGateSn = extMungedSN r.isInRtxGateRegion = true } @@ -207,7 +211,7 @@ func (r *RTPMunger) UpdateAndGetSnTs(extPkt *buffer.ExtPacket, marker bool) (Tra if err != nil { return TranslationParamsRTP{ snOrdering: SequenceNumberOrderingOutOfOrder, - }, ErrOutOfOrderSequenceNumberCacheMiss + }, errOutOfOrderSequenceNumberCacheMiss } extSequenceNumber := extPkt.ExtSequenceNumber - snOffset @@ -223,7 +227,7 @@ func (r *RTPMunger) UpdateAndGetSnTs(extPkt *buffer.ExtPacket, marker bool) (Tra ) return TranslationParamsRTP{ snOrdering: SequenceNumberOrderingOutOfOrder, - }, ErrOutOfOrderSequenceNumberCacheMiss + }, errOutOfOrderSequenceNumberCacheMiss } return TranslationParamsRTP{ @@ -245,13 +249,13 @@ func (r *RTPMunger) UpdateAndGetSnTs(extPkt *buffer.ExtPacket, marker bool) (Tra return TranslationParamsRTP{ snOrdering: SequenceNumberOrderingContiguous, - }, ErrPaddingOnlyPacket + }, errPaddingOnlyPacket } // can get duplicate packet due to FEC return TranslationParamsRTP{ snOrdering: SequenceNumberOrderingDuplicate, - }, ErrDuplicatePacket + }, errDuplicatePacket } func (r *RTPMunger) FilterRTX(nacks []uint16) []uint16 { @@ -269,7 +273,13 @@ func (r *RTPMunger) FilterRTX(nacks []uint16) []uint16 { return filtered } -func (r *RTPMunger) UpdateAndGetPaddingSnTs(num int, clockRate uint32, frameRate uint32, forceMarker bool, extRtpTimestamp uint64) ([]SnTs, error) { +func (r *RTPMunger) UpdateAndGetPaddingSnTs( + num int, + clockRate uint32, + frameRate uint32, + forceMarker bool, + extRtpTimestamp uint64, +) ([]SnTs, error) { if num == 0 { return nil, nil } @@ -278,7 +288,7 @@ func (r *RTPMunger) UpdateAndGetPaddingSnTs(num int, clockRate uint32, frameRate tsOffset := 0 if !r.lastMarker { if !forceMarker { - return nil, ErrPaddingNotOnFrameBoundary + return nil, errPaddingNotOnFrameBoundary } // if forcing frame end, use timestamp of latest received frame for the first one @@ -289,7 +299,7 @@ func (r *RTPMunger) UpdateAndGetPaddingSnTs(num int, clockRate uint32, frameRate extLastSN := r.extLastSN extLastTS := r.extLastTS vals := make([]SnTs, num) - for i := 0; i < num; i++ { + for i := range num { extLastSN++ vals[i].extSequenceNumber = extLastSN @@ -322,6 +332,7 @@ func (r *RTPMunger) UpdateAndGetPaddingSnTs(num int, clockRate uint32, frameRate r.tsOffset -= extLastTS - r.extLastTS r.extLastTS = extLastTS + r.secondLastMarker = r.lastMarker if forceMarker { r.lastMarker = true } diff --git a/pkg/sfu/rtpmunger_test.go b/pkg/sfu/rtpmunger_test.go index 08e51a1..bf0c27e 100644 --- a/pkg/sfu/rtpmunger_test.go +++ b/pkg/sfu/rtpmunger_test.go @@ -44,9 +44,9 @@ func TestSetLastSnTs(t *testing.T) { require.Equal(t, uint64(23332), r.extHighestIncomingSN) require.Equal(t, uint64(23333), r.extLastSN) require.Equal(t, uint64(0xabcdef), r.extLastTS) - snOffset, err := r.snRangeMap.GetValue(r.extHighestIncomingSN) + _, err = r.snRangeMap.GetValue(r.extHighestIncomingSN) require.Error(t, err) - snOffset, err = r.snRangeMap.GetValue(r.extLastSN) + snOffset, err := r.snRangeMap.GetValue(r.extLastSN) require.NoError(t, err) require.Equal(t, uint64(0), snOffset) require.Equal(t, uint64(0), r.snOffset) @@ -96,9 +96,9 @@ func TestPacketDropped(t *testing.T) { require.Equal(t, uint64(23332), r.extHighestIncomingSN) require.Equal(t, uint64(23333), r.extLastSN) require.Equal(t, uint64(0xabcdef), r.extLastTS) - snOffset, err := r.snRangeMap.GetValue(r.extHighestIncomingSN) + _, err := r.snRangeMap.GetValue(r.extHighestIncomingSN) require.Error(t, err) - snOffset, err = r.snRangeMap.GetValue(r.extLastSN) + snOffset, err := r.snRangeMap.GetValue(r.extLastSN) require.NoError(t, err) require.Equal(t, uint64(0), snOffset) require.Equal(t, uint64(0), r.tsOffset) @@ -133,7 +133,7 @@ func TestPacketDropped(t *testing.T) { r.PacketDropped(extPkt) require.Equal(t, uint64(23333), r.extLastSN) - snOffset, err = r.snRangeMap.GetValue(r.extHighestIncomingSN) + _, err = r.snRangeMap.GetValue(r.extHighestIncomingSN) require.Error(t, err) snOffset, err = r.snRangeMap.GetValue(r.extHighestIncomingSN + 1) require.NoError(t, err) @@ -180,9 +180,10 @@ func TestOutOfOrderSequenceNumber(t *testing.T) { } extPkt, _ = testutils.GetTestExtPacket(params) - tp, err := r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker) + _, err = r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker) require.Error(t, err) + var tp TranslationParamsRTP // add a missing sequence number to the cache err = r.snRangeMap.ExcludeRange(23334, 23335) require.NoError(t, err) @@ -228,7 +229,7 @@ func TestOutOfOrderSequenceNumber(t *testing.T) { } tp, err = r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker) - require.Error(t, err, ErrOutOfOrderSequenceNumberCacheMiss) + require.Error(t, err, errOutOfOrderSequenceNumberCacheMiss) require.Equal(t, tpExpected, tp) } @@ -252,7 +253,7 @@ func TestDuplicateSequenceNumber(t *testing.T) { } tp, err := r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker) - require.ErrorIs(t, err, ErrDuplicatePacket) + require.ErrorIs(t, err, errDuplicatePacket) require.Equal(t, tpExpected, tp) } @@ -274,13 +275,14 @@ func TestPaddingOnlyPacket(t *testing.T) { tp, err := r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker) require.Error(t, err) - require.ErrorIs(t, err, ErrPaddingOnlyPacket) + require.ErrorIs(t, err, errPaddingOnlyPacket) require.Equal(t, tpExpected, tp) require.Equal(t, uint64(23333), r.extHighestIncomingSN) require.Equal(t, uint64(23333), r.extLastSN) - snOffset, err := r.snRangeMap.GetValue(r.extHighestIncomingSN) + _, err = r.snRangeMap.GetValue(r.extHighestIncomingSN) require.Error(t, err) + var snOffset uint64 // padding only packet with a gap should not report an error params = &testutils.TestExtPacketParams{ SequenceNumber: 23335, @@ -366,11 +368,11 @@ func TestGapInSequenceNumber(t *testing.T) { } tp, err = r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker) - require.ErrorIs(t, err, ErrPaddingOnlyPacket) + require.ErrorIs(t, err, errPaddingOnlyPacket) require.Equal(t, tpExpected, tp) require.Equal(t, uint64(65536+2), r.extHighestIncomingSN) require.Equal(t, uint64(65536+1), r.extLastSN) - snOffset, err = r.snRangeMap.GetValue(r.extHighestIncomingSN) + _, err = r.snRangeMap.GetValue(r.extHighestIncomingSN) require.Error(t, err) // a packet with a gap should be adjusting for dropped padding packet @@ -417,11 +419,11 @@ func TestGapInSequenceNumber(t *testing.T) { } tp, err = r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker) - require.ErrorIs(t, err, ErrPaddingOnlyPacket) + require.ErrorIs(t, err, errPaddingOnlyPacket) require.Equal(t, tpExpected, tp) require.Equal(t, uint64(65536+5), r.extHighestIncomingSN) require.Equal(t, uint64(65536+3), r.extLastSN) - snOffset, err = r.snRangeMap.GetValue(r.extHighestIncomingSN) + _, err = r.snRangeMap.GetValue(r.extHighestIncomingSN) require.Error(t, err) // a packet with a gap should be adjusting for dropped packets @@ -521,7 +523,7 @@ func TestUpdateAndGetPaddingSnTs(t *testing.T) { // getting padding without forcing marker should fail _, err := r.UpdateAndGetPaddingSnTs(10, 10, 5, false, 0) require.Error(t, err) - require.ErrorIs(t, err, ErrPaddingNotOnFrameBoundary) + require.ErrorIs(t, err, errPaddingNotOnFrameBoundary) // forcing a marker should not error out. // And timestamp on first padding should be the same as the last one. @@ -529,7 +531,7 @@ func TestUpdateAndGetPaddingSnTs(t *testing.T) { clockRate := uint64(10) frameRate := uint64(5) var sntsExpected = make([]SnTs, numPadding) - for i := 0; i < numPadding; i++ { + for i := range numPadding { sntsExpected[i] = SnTs{ extSequenceNumber: uint64(params.SequenceNumber) + uint64(i) + 1, extTimestamp: uint64(params.Timestamp) + ((uint64(i)*clockRate)+frameRate-1)/frameRate, @@ -540,7 +542,7 @@ func TestUpdateAndGetPaddingSnTs(t *testing.T) { require.Equal(t, sntsExpected, snts) // now that there is a marker, timestamp should jump on first padding when asked again - for i := 0; i < numPadding; i++ { + for i := range numPadding { sntsExpected[i] = SnTs{ extSequenceNumber: uint64(params.SequenceNumber) + uint64(len(snts)) + uint64(i) + 1, extTimestamp: snts[len(snts)-1].extTimestamp + ((uint64(i+1)*clockRate)+frameRate-1)/frameRate, @@ -570,7 +572,7 @@ func TestIsOnFrameBoundary(t *testing.T) { // packet with RTP marker params = &testutils.TestExtPacketParams{ - SetMarker: true, + Marker: true, SequenceNumber: 23334, Timestamp: 0xabcdef, SSRC: 0x12345678, diff --git a/pkg/sfu/rtpstats/rtpstats_base.go b/pkg/sfu/rtpstats/rtpstats_base.go index d8f5c8e..b14a92b 100644 --- a/pkg/sfu/rtpstats/rtpstats_base.go +++ b/pkg/sfu/rtpstats/rtpstats_base.go @@ -25,13 +25,12 @@ import ( "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/utils" "github.com/livekit/protocol/utils/mono" + "github.com/livekit/protocol/utils/rtputil" ) const ( cFirstPacketTimeAdjustWindow = 2 * time.Minute cFirstPacketTimeAdjustThreshold = 15 * 1e9 - - cSequenceNumberLargeJumpThreshold = 100 ) // ------------------------------------------------------- @@ -192,6 +191,8 @@ func (w WrappedRTCPSenderReportStateLogger) MarshalLogObject(e zapcore.ObjectEnc return nil } +// ------------------------------------------------------------------ + func RTCPSenderReportPropagationDelay(rsrs *livekit.RTCPSenderReportState, passThrough bool) time.Duration { if passThrough { return 0 @@ -205,6 +206,8 @@ func RTCPSenderReportPropagationDelay(rsrs *livekit.RTCPSenderReportState, passT type rtpStatsBase struct { *rtpStatsBaseLite + rtpConverter *rtputil.RTPConverter + firstTime int64 firstTimeAdjustment time.Duration highestTime int64 @@ -251,6 +254,18 @@ func newRTPStatsBase(params RTPStatsParams) *rtpStatsBase { } } +func (r *rtpStatsBase) SetClockRate(clockRate uint32) { + r.lock.Lock() + defer r.lock.Unlock() + + r.setClockRateLocked(clockRate) +} + +func (r *rtpStatsBase) setClockRateLocked(clockRate uint32) { + r.rtpConverter = rtputil.NewRTPConverter(int64(clockRate)) + r.rtpStatsBaseLite.setClockRateLocked(clockRate) +} + func (r *rtpStatsBase) seed(from *rtpStatsBase) bool { if !r.rtpStatsBaseLite.seed(from.rtpStatsBaseLite) { return false @@ -346,6 +361,13 @@ func (r *rtpStatsBase) UpdateKeyFrame(kfCount uint32) { r.lastKeyFrame = time.Now() } +func (r *rtpStatsBase) KeyFrame() (uint32, time.Time) { + r.lock.RLock() + defer r.lock.RUnlock() + + return r.keyFrames, r.lastKeyFrame +} + func (r *rtpStatsBase) UpdateRtt(rtt uint32) { r.lock.Lock() defer r.lock.Unlock() @@ -374,7 +396,11 @@ func (r *rtpStatsBase) GetRtt() uint32 { return r.rtt } -func (r *rtpStatsBase) maybeAdjustFirstPacketTime(srData *livekit.RTCPSenderReportState, tsOffset uint64, extStartTS uint64) (err error, loggingFields []interface{}) { +func (r *rtpStatsBase) maybeAdjustFirstPacketTime( + srData *livekit.RTCPSenderReportState, + tsOffset uint64, + extStartTS uint64, +) (adjustment int64, loggingFields []any, err error) { nowNano := mono.UnixNano() if time.Duration(nowNano-r.startTime) > cFirstPacketTimeAdjustWindow { return @@ -387,49 +413,52 @@ func (r *rtpStatsBase) maybeAdjustFirstPacketTime(srData *livekit.RTCPSenderRepo // in some network element along the way), push back first time // to an earlier instance. timeSinceReceive := time.Duration(nowNano - srData.AtAdjusted) - extNowTS := srData.RtpTimestampExt - tsOffset + uint64(timeSinceReceive.Nanoseconds()*int64(r.params.ClockRate)/1e9) + extNowTS := srData.RtpTimestampExt - tsOffset + r.rtpConverter.ToRTPExt(timeSinceReceive) samplesDiff := int64(extNowTS - extStartTS) if samplesDiff < 0 { // out-of-order, skip return } - samplesDuration := time.Duration(float64(samplesDiff) / float64(r.params.ClockRate) * float64(time.Second)) + samplesDuration := r.rtpConverter.ToDurationExt(uint64(samplesDiff)) timeSinceFirst := time.Duration(nowNano - r.firstTime) now := r.firstTime + timeSinceFirst.Nanoseconds() firstTime := now - samplesDuration.Nanoseconds() - - getFields := func() []interface{} { - return []interface{}{ - "startTime", time.Unix(0, r.startTime), - "nowTime", time.Unix(0, now), - "before", time.Unix(0, r.firstTime), - "after", time.Unix(0, firstTime), - "adjustment", time.Duration(r.firstTime - firstTime), - "extNowTS", extNowTS, - "extStartTS", extStartTS, - "srData", WrappedRTCPSenderReportStateLogger{srData}, - "tsOffset", tsOffset, - "timeSinceReceive", timeSinceReceive, - "timeSinceFirst", timeSinceFirst, - "samplesDiff", samplesDiff, - "samplesDuration", samplesDuration, - } - } + adjustment = r.firstTime - firstTime if firstTime < r.firstTime { - if r.firstTime-firstTime > cFirstPacketTimeAdjustThreshold { + if adjustment > cFirstPacketTimeAdjustThreshold { err = errors.New("adjusting first packet time, too big, ignoring") - loggingFields = getFields() + loggingFields = r.firstPacketTimeAdjustLogFields(now, firstTime, adjustment, extNowTS, extStartTS, srData, tsOffset, timeSinceReceive, timeSinceFirst, samplesDuration, samplesDiff) } else { - r.logger.Debugw("adjusting first packet time", getFields()...) - r.firstTimeAdjustment += time.Duration(r.firstTime - firstTime) + r.firstTimeAdjustment += time.Duration(adjustment) + r.logger.Debugw("adjusting first packet time", r.firstPacketTimeAdjustLogFields(now, firstTime, adjustment, extNowTS, extStartTS, srData, tsOffset, timeSinceReceive, timeSinceFirst, samplesDuration, samplesDiff)...) r.firstTime = firstTime } } + return } +func (r *rtpStatsBase) firstPacketTimeAdjustLogFields(now, firstTime int64, adjustment int64, extNowTS, extStartTS uint64, srData *livekit.RTCPSenderReportState, tsOffset uint64, timeSinceReceive, timeSinceFirst, samplesDuration time.Duration, samplesDiff int64) []any { + return []any{ + "startTime", time.Unix(0, r.startTime), + "nowTime", time.Unix(0, now), + "before", time.Unix(0, r.firstTime), + "after", time.Unix(0, firstTime), + "adjustment", time.Duration(adjustment), + "firstTimeAdjustment", r.firstTimeAdjustment, + "extNowTS", extNowTS, + "extStartTS", extStartTS, + "srData", WrappedRTCPSenderReportStateLogger{srData}, + "tsOffset", tsOffset, + "timeSinceReceive", timeSinceReceive, + "timeSinceFirst", timeSinceFirst, + "samplesDiff", samplesDiff, + "samplesDuration", samplesDuration, + } +} + func (r *rtpStatsBase) getPacketsSeenMinusPadding(extStartSN, extHighestSN uint64) uint64 { packetsSeen := r.getPacketsSeen(extStartSN, extHighestSN) if r.packetsPadding > packetsSeen { @@ -447,9 +476,13 @@ func (r *rtpStatsBase) deltaInfo( snapshotID uint32, extStartSN uint64, extHighestSN uint64, -) (deltaInfo *RTPDeltaInfo, err error, loggingFields []interface{}) { - then, now := r.getAndResetSnapshot(snapshotID, extStartSN, extHighestSN) - if now == nil || then == nil { +) (deltaInfo *RTPDeltaInfo, loggingFields []any, err error) { + if r.clockRate == 0 { + return + } + + then, now, ok := r.getAndResetSnapshot(snapshotID, extStartSN, extHighestSN) + if !ok { return } @@ -461,7 +494,7 @@ func (r *rtpStatsBase) deltaInfo( packetsExpected = 0 } if packetsExpected > cNumSequenceNumbers { - loggingFields = []interface{}{ + loggingFields = []any{ "snapshotID", snapshotID, "snapshotNow", now, "snapshotThen", then, @@ -487,7 +520,7 @@ func (r *rtpStatsBase) deltaInfo( // padding packets delta could be higher than expected due to out-of-order padding packets packetsPadding := now.packetsPadding - then.packetsPadding if packetsExpected < packetsPadding { - loggingFields = []interface{}{ + loggingFields = []any{ "snapshotID", snapshotID, "snapshotNow", now, "snapshotThen", then, @@ -518,7 +551,7 @@ func (r *rtpStatsBase) deltaInfo( PacketsOutOfOrder: uint32(now.packetsOutOfOrder - then.packetsOutOfOrder), Frames: now.frames - then.frames, RttMax: then.maxRtt, - JitterMax: then.maxJitter / float64(r.params.ClockRate) * 1e6, + JitterMax: then.maxJitter / float64(r.clockRate) * 1e6, Nacks: now.nacks - then.nacks, Plis: now.plis - then.plis, Firs: now.firs - then.firs, @@ -614,8 +647,8 @@ func (r *rtpStatsBase) toProto( p.KeyFrames = r.keyFrames p.LastKeyFrame = timestamppb.New(r.lastKeyFrame) - p.JitterCurrent = jitter / float64(r.params.ClockRate) * 1e6 - p.JitterMax = maxJitter / float64(r.params.ClockRate) * 1e6 + p.JitterCurrent = jitter / float64(r.clockRate) * 1e6 + p.JitterMax = maxJitter / float64(r.clockRate) * 1e6 p.Firs = r.firs p.LastFir = timestamppb.New(r.lastFir) @@ -635,9 +668,9 @@ func (r *rtpStatsBase) updateJitter(ets uint64, packetTime int64) float64 { // p1f1 -> p1f2 -> p2f1 // In this case, p2f1 (packet 2, frame 1) will still be used in jitter calculation // although it is the second packet of a frame because of out-of-order receival. - if r.lastJitterExtTimestamp != ets { + if r.lastJitterExtTimestamp != ets && r.rtpConverter != nil { timeSinceFirst := packetTime - r.firstTime - packetTimeRTP := uint64(timeSinceFirst * int64(r.params.ClockRate) / 1e9) + packetTimeRTP := r.rtpConverter.ToRTPExt(time.Duration(timeSinceFirst)) transit := packetTimeRTP - ets if r.lastTransit != 0 { @@ -661,9 +694,9 @@ func (r *rtpStatsBase) updateJitter(ets uint64, packetTime int64) float64 { return r.jitter } -func (r *rtpStatsBase) getAndResetSnapshot(snapshotID uint32, extStartSN uint64, extHighestSN uint64) (*snapshot, *snapshot) { - if !r.initialized { - return nil, nil +func (r *rtpStatsBase) getAndResetSnapshot(snapshotID uint32, extStartSN uint64, extHighestSN uint64) (snapshot, snapshot, bool) { + if !r.initialized || snapshotID < cFirstSnapshotID { + return snapshot{}, snapshot{}, false } idx := snapshotID - cFirstSnapshotID @@ -676,7 +709,7 @@ func (r *rtpStatsBase) getAndResetSnapshot(snapshotID uint32, extStartSN uint64, // snapshot now now := r.getSnapshot(mono.UnixNano(), extHighestSN+1) r.snapshots[idx] = now - return &then, &now + return then, now, true } func (r *rtpStatsBase) getDrift(extStartTS, extHighestTS uint64) ( @@ -686,21 +719,20 @@ func (r *rtpStatsBase) getDrift(extStartTS, extHighestTS uint64) ( rebasedReportDrift *livekit.RTPDrift, ) { if r.firstTime != 0 { - elapsed := r.highestTime - r.firstTime + elapsed := time.Duration(r.highestTime - r.firstTime) rtpClockTicks := extHighestTS - extStartTS - driftSamples := int64(rtpClockTicks - uint64(elapsed*int64(r.params.ClockRate)/1e9)) + driftSamples := int64(rtpClockTicks - r.rtpConverter.ToRTPExt(elapsed)) if elapsed > 0 { - elapsedSeconds := time.Duration(elapsed).Seconds() packetDrift = &livekit.RTPDrift{ StartTime: timestamppb.New(time.Unix(0, r.firstTime)), EndTime: timestamppb.New(time.Unix(0, r.highestTime)), - Duration: elapsedSeconds, + Duration: elapsed.Seconds(), StartTimestamp: extStartTS, EndTimestamp: extHighestTS, RtpClockTicks: rtpClockTicks, DriftSamples: driftSamples, - DriftMs: (float64(driftSamples) * 1000) / float64(r.params.ClockRate), - ClockRate: float64(rtpClockTicks) / elapsedSeconds, + DriftMs: (float64(driftSamples) * 1000) / float64(r.clockRate), + ClockRate: float64(rtpClockTicks) / elapsed.Seconds(), } } } @@ -708,25 +740,27 @@ func (r *rtpStatsBase) getDrift(extStartTS, extHighestTS uint64) ( if r.srFirst != nil && r.srNewest != nil && r.srFirst.RtpTimestamp != r.srNewest.RtpTimestamp { rtpClockTicks := r.srNewest.RtpTimestampExt - r.srFirst.RtpTimestampExt - elapsed := mediatransportutil.NtpTime(r.srNewest.NtpTimestamp).Time().Sub(mediatransportutil.NtpTime(r.srFirst.NtpTimestamp).Time()) + srFirstTime := mediatransportutil.NtpTime(r.srFirst.NtpTimestamp).Time() + srNewestTime := mediatransportutil.NtpTime(r.srNewest.NtpTimestamp).Time() + elapsed := srNewestTime.Sub(srFirstTime) if elapsed.Seconds() > 0.0 { - driftSamples := int64(rtpClockTicks - uint64(elapsed.Nanoseconds()*int64(r.params.ClockRate)/1e9)) + driftSamples := int64(rtpClockTicks - r.rtpConverter.ToRTPExt(elapsed)) ntpReportDrift = &livekit.RTPDrift{ - StartTime: timestamppb.New(mediatransportutil.NtpTime(r.srFirst.NtpTimestamp).Time()), - EndTime: timestamppb.New(mediatransportutil.NtpTime(r.srNewest.NtpTimestamp).Time()), + StartTime: timestamppb.New(srFirstTime), + EndTime: timestamppb.New(srNewestTime), Duration: elapsed.Seconds(), StartTimestamp: r.srFirst.RtpTimestampExt, EndTimestamp: r.srNewest.RtpTimestampExt, RtpClockTicks: rtpClockTicks, DriftSamples: driftSamples, - DriftMs: (float64(driftSamples) * 1000) / float64(r.params.ClockRate), + DriftMs: (float64(driftSamples) * 1000) / float64(r.clockRate), ClockRate: float64(rtpClockTicks) / elapsed.Seconds(), } } elapsed = time.Duration(r.srNewest.At - r.srFirst.At) if elapsed.Seconds() > 0.0 { - driftSamples := int64(rtpClockTicks - uint64(elapsed.Nanoseconds()*int64(r.params.ClockRate)/1e9)) + driftSamples := int64(rtpClockTicks - r.rtpConverter.ToRTPExt(elapsed)) receivedReportDrift = &livekit.RTPDrift{ StartTime: timestamppb.New(time.Unix(0, r.srFirst.At)), EndTime: timestamppb.New(time.Unix(0, r.srNewest.At)), @@ -735,14 +769,14 @@ func (r *rtpStatsBase) getDrift(extStartTS, extHighestTS uint64) ( EndTimestamp: r.srNewest.RtpTimestampExt, RtpClockTicks: rtpClockTicks, DriftSamples: driftSamples, - DriftMs: (float64(driftSamples) * 1000) / float64(r.params.ClockRate), + DriftMs: (float64(driftSamples) * 1000) / float64(r.clockRate), ClockRate: float64(rtpClockTicks) / elapsed.Seconds(), } } elapsed = time.Duration(r.srNewest.AtAdjusted - r.srFirst.AtAdjusted) if elapsed.Seconds() > 0.0 { - driftSamples := int64(rtpClockTicks - uint64(elapsed.Nanoseconds()*int64(r.params.ClockRate)/1e9)) + driftSamples := int64(rtpClockTicks - r.rtpConverter.ToRTPExt(elapsed)) rebasedReportDrift = &livekit.RTPDrift{ StartTime: timestamppb.New(time.Unix(0, r.srFirst.AtAdjusted)), EndTime: timestamppb.New(time.Unix(0, r.srNewest.AtAdjusted)), @@ -751,7 +785,7 @@ func (r *rtpStatsBase) getDrift(extStartTS, extHighestTS uint64) ( EndTimestamp: r.srNewest.RtpTimestampExt, RtpClockTicks: rtpClockTicks, DriftSamples: driftSamples, - DriftMs: (float64(driftSamples) * 1000) / float64(r.params.ClockRate), + DriftMs: (float64(driftSamples) * 1000) / float64(r.clockRate), ClockRate: float64(rtpClockTicks) / elapsed.Seconds(), } } diff --git a/pkg/sfu/rtpstats/rtpstats_base_lite.go b/pkg/sfu/rtpstats/rtpstats_base_lite.go index c54b366..9180dfd 100644 --- a/pkg/sfu/rtpstats/rtpstats_base_lite.go +++ b/pkg/sfu/rtpstats/rtpstats_base_lite.go @@ -17,6 +17,7 @@ package rtpstats import ( "errors" "fmt" + "strings" "sync" "time" @@ -95,14 +96,13 @@ func (s *snapshotLite) MarshalLogObject(e zapcore.ObjectEncoder) error { // ------------------------------------------------------------------ type RTPStatsParams struct { - ClockRate uint32 - IsRTX bool - Logger logger.Logger + IsRTX bool } type rtpStatsBaseLite struct { - params RTPStatsParams - logger logger.Logger + params RTPStatsParams + clockRate uint32 + logger logger.Logger lock sync.RWMutex @@ -134,7 +134,7 @@ type rtpStatsBaseLite struct { func newRTPStatsBaseLite(params RTPStatsParams) *rtpStatsBaseLite { return &rtpStatsBaseLite{ params: params, - logger: params.Logger, + logger: logger.GetLogger(), nextSnapshotLiteID: cFirstSnapshotID, snapshotLites: make([]snapshotLite, 2), } @@ -172,7 +172,21 @@ func (r *rtpStatsBaseLite) seed(from *rtpStatsBaseLite) bool { return true } +func (r *rtpStatsBaseLite) SetClockRate(clockRate uint32) { + r.lock.Lock() + defer r.lock.Unlock() + + r.setClockRateLocked(clockRate) +} + +func (r *rtpStatsBaseLite) setClockRateLocked(clockRate uint32) { + r.clockRate = clockRate +} + func (r *rtpStatsBaseLite) SetLogger(logger logger.Logger) { + r.lock.Lock() + defer r.lock.Unlock() + r.logger = logger } @@ -305,9 +319,9 @@ func (r *rtpStatsBaseLite) deltaInfoLite( snapshotLiteID uint32, extStartSN uint64, extHighestSN uint64, -) (deltaInfoLite *RTPDeltaInfoLite, err error, loggingFields []interface{}) { - then, now := r.getAndResetSnapshotLite(snapshotLiteID, extStartSN, extHighestSN) - if now == nil || then == nil { +) (deltaInfoLite *RTPDeltaInfoLite, loggingFields []any, err error) { + then, now, ok := r.getAndResetSnapshotLite(snapshotLiteID, extStartSN, extHighestSN) + if !ok { return } @@ -319,7 +333,7 @@ func (r *rtpStatsBaseLite) deltaInfoLite( packetsExpected = 0 } if packetsExpected > cNumSequenceNumbers { - loggingFields = []interface{}{ + loggingFields = []any{ "snapshotLiteID", snapshotLiteID, "snapshotLiteNow", now, "snapshotLiteThen", then, @@ -342,7 +356,7 @@ func (r *rtpStatsBaseLite) deltaInfoLite( packetsLost = 0 } if packetsLost > packetsExpected { - loggingFields = []interface{}{ + loggingFields = []any{ "snapshotLiteID", snapshotLiteID, "snapshotLiteNow", now, "snapshotLiteThen", then, @@ -366,7 +380,7 @@ func (r *rtpStatsBaseLite) deltaInfoLite( } func (r *rtpStatsBaseLite) marshalLogObject(e zapcore.ObjectEncoder, packetsExpected, packetsSeenMinusPadding uint64) (float64, error) { - if r == nil || !r.initialized { + if r == nil || !r.initialized || r.clockRate == 0 { return 0, errors.New("not initialized") } @@ -399,25 +413,18 @@ func (r *rtpStatsBaseLite) marshalLogObject(e zapcore.ObjectEncoder, packetsExpe e.AddFloat32("packetLostPercentage", float32(r.packetsLost)/float32(packetsExpected)*100.0) } - hasLoss := false - first := true - str := "[" + var sb strings.Builder for burst, count := range r.gapHistogram { if count == 0 { continue } - - hasLoss = true - - if !first { - str += ", " + if sb.Len() > 0 { + sb.WriteString(", ") } - first = false - str += fmt.Sprintf("%d:%d", burst+1, count) + fmt.Fprintf(&sb, "%d:%d", burst+1, count) } - str += "]" - if hasLoss { - e.AddString("gapHistogram", str) + if sb.Len() > 0 { + e.AddString("gapHistogram", "["+sb.String()+"]") } e.AddUint32("nacks", r.nacks) @@ -431,7 +438,7 @@ func (r *rtpStatsBaseLite) marshalLogObject(e zapcore.ObjectEncoder, packetsExpe } func (r *rtpStatsBaseLite) toProto(packetsExpected, packetsSeenMinusPadding, packetsLost uint64) *livekit.RTPStats { - if r.startTime == 0 { + if r.startTime == 0 || r.clockRate == 0 { return nil } @@ -473,33 +480,22 @@ func (r *rtpStatsBaseLite) toProto(packetsExpected, packetsSeenMinusPadding, pac LastPli: timestamppb.New(time.Unix(0, r.lastPli)), } - gapsPresent := false - for i := 0; i < len(r.gapHistogram); i++ { + for i := range len(r.gapHistogram) { if r.gapHistogram[i] == 0 { continue } - - gapsPresent = true - break - } - - if gapsPresent { - p.GapHistogram = make(map[int32]uint32, len(r.gapHistogram)) - for i := 0; i < len(r.gapHistogram); i++ { - if r.gapHistogram[i] == 0 { - continue - } - - p.GapHistogram[int32(i+1)] = r.gapHistogram[i] + if p.GapHistogram == nil { + p.GapHistogram = make(map[int32]uint32, len(r.gapHistogram)) } + p.GapHistogram[int32(i+1)] = r.gapHistogram[i] } return p } -func (r *rtpStatsBaseLite) getAndResetSnapshotLite(snapshotLiteID uint32, extStartSN uint64, extHighestSN uint64) (*snapshotLite, *snapshotLite) { +func (r *rtpStatsBaseLite) getAndResetSnapshotLite(snapshotLiteID uint32, extStartSN uint64, extHighestSN uint64) (snapshotLite, snapshotLite, bool) { if !r.initialized { - return nil, nil + return snapshotLite{}, snapshotLite{}, false } idx := snapshotLiteID - cFirstSnapshotID @@ -512,7 +508,7 @@ func (r *rtpStatsBaseLite) getAndResetSnapshotLite(snapshotLiteID uint32, extSta // snapshot now now := r.getSnapshotLite(mono.UnixNano(), extHighestSN+1) r.snapshotLites[idx] = now - return &then, &now + return then, now, true } func (r *rtpStatsBaseLite) updateGapHistogram(gap int) { @@ -554,4 +550,8 @@ func getPacketsExpected(extStartSN, extHighestSN uint64) uint64 { return extHighestSN - extStartSN + 1 } +func shouldLog(count int) bool { + return count < 20 || count%200 == 0 +} + // ---------------------------------- diff --git a/pkg/sfu/rtpstats/rtpstats_receiver.go b/pkg/sfu/rtpstats/rtpstats_receiver.go index 35531e6..1fc1fc8 100644 --- a/pkg/sfu/rtpstats/rtpstats_receiver.go +++ b/pkg/sfu/rtpstats/rtpstats_receiver.go @@ -20,10 +20,12 @@ import ( "time" "github.com/pion/rtcp" + "go.uber.org/zap" "go.uber.org/zap/zapcore" - "github.com/livekit/livekit-server/pkg/sfu/utils" "github.com/livekit/mediatransportutil" + "github.com/livekit/mediatransportutil/pkg/latency" + "github.com/livekit/mediatransportutil/pkg/utils" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" protoutils "github.com/livekit/protocol/utils" @@ -37,12 +39,53 @@ const ( cReportSlack = float64(60.0) cTSJumpTooHighFactor = float64(1.5) + + restartThreshold = 5 ) // --------------------------------------------------------------------- +type RTPFlowUnhandledReason int + +const ( + RTPFlowUnhandledReasonNone RTPFlowUnhandledReason = iota + RTPFlowUnhandledReasonEnded + RTPFlowUnhandledReasonUnconfigured + RTPFlowUnhandledReasonPaddingOnly + RTPFlowUnhandledReasonPreStartTimestamp + RTPFlowUnhandledReasonOldTimestamp + RTPFlowUnhandledReasonPreStartSequenceNumber + RTPFlowUnhandledReasonOldSequenceNumber + RTPFlowUnhandledReasonRestart +) + +func (r RTPFlowUnhandledReason) String() string { + switch r { + case RTPFlowUnhandledReasonNone: + return "NONE" + case RTPFlowUnhandledReasonEnded: + return "ENDED" + case RTPFlowUnhandledReasonUnconfigured: + return "UNCONFIGURED" + case RTPFlowUnhandledReasonPaddingOnly: + return "PADDING_ONLY" + case RTPFlowUnhandledReasonPreStartTimestamp: + return "PRE_START_TIMESTAMP" + case RTPFlowUnhandledReasonOldTimestamp: + return "OLD_TIMESTAMP" + case RTPFlowUnhandledReasonPreStartSequenceNumber: + return "PRE_START_SEQUENCE_NUMBER" + case RTPFlowUnhandledReasonOldSequenceNumber: + return "OLD_SEQUENCE_NUMBER" + case RTPFlowUnhandledReasonRestart: + return "RESTART" + default: + return fmt.Sprintf("UNKNOWN: %d", int(r)) + } +} + type RTPFlowState struct { - IsNotHandled bool + UnhandledReason RTPFlowUnhandledReason LossStartInclusive uint64 LossEndExclusive uint64 @@ -59,7 +102,7 @@ func (r *RTPFlowState) MarshalLogObject(e zapcore.ObjectEncoder) error { return nil } - e.AddBool("IsNotHandled", r.IsNotHandled) + e.AddString("UnhandledReason", r.UnhandledReason.String()) e.AddUint64("LossStartInclusive", r.LossStartInclusive) e.AddUint64("LossEndExclusive", r.LossEndExclusive) e.AddBool("IsDuplicate", r.IsDuplicate) @@ -71,6 +114,62 @@ func (r *RTPFlowState) MarshalLogObject(e zapcore.ObjectEncoder) error { // --------------------------------------------------------------------- +type packet struct { + sequenceNumber uint16 + timestamp uint32 +} + +func (p packet) MarshalLogObject(e zapcore.ObjectEncoder) error { + e.AddUint16("sequenceNumber", p.sequenceNumber) + e.AddUint32("timestamp", p.timestamp) + return nil +} + +// --------------------------------------------------------------------- + +type receiverUpdateLoggingFields struct { + packetTime int64 + sequenceNumber uint16 + timestamp uint32 + marker bool + hdrSize int + payloadSize int + paddingSize int + resSN utils.WrapAroundUpdateResult[uint64] + gapSN int64 + resTS utils.WrapAroundUpdateResult[uint64] + gapTS int64 + snRolloverCount int + expectedTSJump int64 + tsRolloverCount int + timeSinceHighest int64 + rtpStats *RTPStatsReceiver +} + +func (rulf *receiverUpdateLoggingFields) MarshalLogObject(e zapcore.ObjectEncoder) error { + if rulf != nil { + e.AddObject("resSN", &rulf.resSN) + e.AddInt64("gapSN", rulf.gapSN) + e.AddObject("resTS", &rulf.resTS) + e.AddInt64("gapTS", rulf.gapTS) + e.AddInt("snRolloverCount", rulf.snRolloverCount) + e.AddInt64("expectedTSJump", rulf.expectedTSJump) + e.AddInt("tsRolloverCount", rulf.tsRolloverCount) + e.AddTime("packetTime", time.Unix(0, rulf.packetTime)) + e.AddDuration("timeSinceHighest", time.Duration(rulf.timeSinceHighest)) + e.AddUint16("sequenceNumber", rulf.sequenceNumber) + e.AddUint32("timestamp", rulf.timestamp) + e.AddBool("marker", rulf.marker) + e.AddInt("hdrSize", rulf.hdrSize) + e.AddInt("payloadSize", rulf.payloadSize) + e.AddInt("paddingSize", rulf.paddingSize) + e.AddObject("rtpStats", lockedRTPStatsReceiverLogEncoder{rulf.rtpStats}) + } + return nil +} + +// --------------------------------------------------------------------- + type RTPStatsReceiver struct { *rtpStatsBase @@ -81,27 +180,40 @@ type RTPStatsReceiver struct { history *protoutils.Bitmap[uint64] - propagationDelayEstimator *utils.OWDEstimator + propagationDelayEstimator *latency.OWDEstimator clockSkewCount int clockSkewMediaPathCount int outOfOrderSenderReportCount int - largeJumpCount int - largeJumpNegativeCount int timeReversedCount int + + packetsDroppedPreStartTimestamp int + packetsDroppedOldTimestamp int + packetsDroppedPreStartSequenceNumber int + packetsDroppedOldSequenceNumber int + + restartPacketsBuf [restartThreshold]packet + restartPacketsN int } func NewRTPStatsReceiver(params RTPStatsParams) *RTPStatsReceiver { return &RTPStatsReceiver{ rtpStatsBase: newRTPStatsBase(params), sequenceNumber: utils.NewWrapAround[uint16, uint64](utils.WrapAroundParams{IsRestartAllowed: false}), - tsRolloverThreshold: (1 << 31) * 1e9 / int64(params.ClockRate), timestamp: utils.NewWrapAround[uint32, uint64](utils.WrapAroundParams{IsRestartAllowed: false}), history: protoutils.NewBitmap[uint64](cHistorySize), - propagationDelayEstimator: utils.NewOWDEstimator(utils.OWDEstimatorParamsDefault), + propagationDelayEstimator: latency.NewOWDEstimator(latency.OWDEstimatorParamsDefault), } } +func (r *RTPStatsReceiver) SetClockRate(clockRate uint32) { + r.lock.Lock() + defer r.lock.Unlock() + + r.tsRolloverThreshold = (1 << 31) * 1e9 / int64(clockRate) + r.rtpStatsBase.setClockRateLocked(clockRate) +} + func (r *RTPStatsReceiver) NewSnapshotId() uint32 { r.lock.Lock() defer r.lock.Unlock() @@ -115,17 +227,19 @@ func (r *RTPStatsReceiver) getTSRolloverCount(diffNano int64, ts uint32) int { return -1 } - excess := (diffNano - r.tsRolloverThreshold*2) * int64(r.params.ClockRate) / 1e9 - roc := excess / (1 << 32) - if roc < 0 { - roc = 0 - } + excess := (diffNano - r.tsRolloverThreshold*2) * int64(r.clockRate) / 1e9 + roc := max(excess/(1<<32), 0) if r.timestamp.GetHighest() > ts { roc++ } return int(roc) } +func (r *RTPStatsReceiver) undoUpdatesLocked(resSN utils.WrapAroundUpdateResult[uint64], resTS utils.WrapAroundUpdateResult[uint64]) { + r.sequenceNumber.UndoUpdate(resSN) + r.timestamp.UndoUpdate(resTS) +} + func (r *RTPStatsReceiver) Update( packetTime int64, sequenceNumber uint16, @@ -139,7 +253,11 @@ func (r *RTPStatsReceiver) Update( defer r.lock.Unlock() if r.endTime != 0 { - flowState.IsNotHandled = true + flowState.UnhandledReason = RTPFlowUnhandledReasonEnded + return + } + if r.clockRate == 0 { + flowState.UnhandledReason = RTPFlowUnhandledReasonUnconfigured return } @@ -152,31 +270,10 @@ func (r *RTPStatsReceiver) Update( var tsRolloverCount int var snRolloverCount int - logger := func() logger.UnlikelyLogger { - return r.logger.WithUnlikelyValues( - "resSN", resSN, - "gapSN", gapSN, - "resTS", resTS, - "gapTS", gapTS, - "snRolloverCount", snRolloverCount, - "expectedTSJump", expectedTSJump, - "tsRolloverCount", tsRolloverCount, - "packetTime", time.Unix(0, packetTime), - "timeSinceHighest", time.Duration(timeSinceHighest), - "sequenceNumber", sequenceNumber, - "timestamp", timestamp, - "marker", marker, - "hdrSize", hdrSize, - "payloadSize", payloadSize, - "paddingSize", paddingSize, - "rtpStats", lockedRTPStatsReceiverLogEncoder{r}, - ) - } - if !r.initialized { if payloadSize == 0 { // do not start on a padding only packet - flowState.IsNotHandled = true + flowState.UnhandledReason = RTPFlowUnhandledReasonPaddingOnly return } @@ -201,57 +298,164 @@ func (r *RTPStatsReceiver) Update( ) } else { resSN = r.sequenceNumber.Update(sequenceNumber) - if resSN.IsUnhandled { - flowState.IsNotHandled = true - return - } gapSN = int64(resSN.ExtendedVal - resSN.PreExtendedHighest) timeSinceHighest = packetTime - r.highestTime + expectedTSJump = int64(r.rtpConverter.ToRTPExt(time.Duration(timeSinceHighest))) tsRolloverCount = r.getTSRolloverCount(timeSinceHighest, timestamp) - if tsRolloverCount >= 0 { - logger().Warnw("potential time stamp roll over", nil) - } resTS = r.timestamp.Rollover(timestamp, tsRolloverCount) + gapTS = int64(resTS.ExtendedVal - resTS.PreExtendedHighest) if resTS.IsUnhandled { - flowState.IsNotHandled = true + r.undoUpdatesLocked(resSN, resTS) + + r.packetsDroppedPreStartTimestamp++ + rulf := &receiverUpdateLoggingFields{ + packetTime: packetTime, + sequenceNumber: sequenceNumber, + timestamp: timestamp, + marker: marker, + hdrSize: hdrSize, + payloadSize: payloadSize, + paddingSize: paddingSize, + resSN: resSN, + gapSN: gapSN, + resTS: resTS, + gapTS: gapTS, + snRolloverCount: snRolloverCount, + expectedTSJump: expectedTSJump, + tsRolloverCount: tsRolloverCount, + timeSinceHighest: timeSinceHighest, + rtpStats: r, + } + if shouldLog(r.packetsDroppedPreStartTimestamp) { + r.logger.Warnw("dropping packet, pre-start timestamp", nil, zap.Inline(rulf)) + } + + if r.maybeRestart(sequenceNumber, timestamp, payloadSize) { + r.logger.Infow("potential restart", zap.Inline(rulf)) + r.resetRestart() + flowState.UnhandledReason = RTPFlowUnhandledReasonRestart + } else { + flowState.UnhandledReason = RTPFlowUnhandledReasonPreStartTimestamp + } return } - gapTS = int64(resTS.ExtendedVal - resTS.PreExtendedHighest) - // it is possible to receive old packets in two different scenarios - // as it is not possible to detect how far to roll back, ignore old packets - // - // Case 1: - // Very old time stamp, happens under the following conditions - // - resume after long mute, big time stamp jump - // - an out of order packet from before the mute arrives (unsure what causes this - // very old packet to be trasmitted from remote), causing time stamp to jump back - // to before mute, but it appears like it has rolled over. - // Use a threshold against expected to ignore these. - if gapSN < 0 && gapTS > 0 { - expectedTSJump = timeSinceHighest * int64(r.params.ClockRate) / 1e9 - if gapTS > int64(float64(expectedTSJump)*cTSJumpTooHighFactor) { - r.sequenceNumber.UndoUpdate(resSN) - r.timestamp.UndoUpdate(resTS) - logger().Warnw("dropping old packet, timestamp", nil) - flowState.IsNotHandled = true + if tsRolloverCount >= 0 && payloadSize > 0 { + rulf := &receiverUpdateLoggingFields{ + packetTime: packetTime, + sequenceNumber: sequenceNumber, + timestamp: timestamp, + marker: marker, + hdrSize: hdrSize, + payloadSize: payloadSize, + paddingSize: paddingSize, + resSN: resSN, + gapSN: gapSN, + resTS: resTS, + gapTS: gapTS, + snRolloverCount: snRolloverCount, + expectedTSJump: expectedTSJump, + tsRolloverCount: tsRolloverCount, + timeSinceHighest: timeSinceHighest, + rtpStats: r, + } + r.logger.Warnw("potential time stamp roll over", nil, zap.Inline(rulf)) + } + + if !resSN.IsUnhandled { + // it is possible to receive old packets in two different scenarios + // as it is not possible to detect how far to roll back, ignore old packets + // + // Case 1: + // Very old time stamp, happens under the following conditions + // - resume after long mute, this casues big time stamp jump ahead for the packets + // after unmute + // - an out of order packet from before the mute arrives (unsure what causes this + // very old packet to be transmitted from remote), causing time stamp to jump back + // to before mute, but it appears like it has rolled over. + // Use a threshold against expected to ignore these. + if gapSN < 0 && gapTS > 0 { + if gapTS > int64(float64(expectedTSJump)*cTSJumpTooHighFactor) { + r.undoUpdatesLocked(resSN, resTS) + + r.packetsDroppedOldTimestamp++ + rulf := &receiverUpdateLoggingFields{ + packetTime: packetTime, + sequenceNumber: sequenceNumber, + timestamp: timestamp, + marker: marker, + hdrSize: hdrSize, + payloadSize: payloadSize, + paddingSize: paddingSize, + resSN: resSN, + gapSN: gapSN, + resTS: resTS, + gapTS: gapTS, + snRolloverCount: snRolloverCount, + expectedTSJump: expectedTSJump, + tsRolloverCount: tsRolloverCount, + timeSinceHighest: timeSinceHighest, + rtpStats: r, + } + if shouldLog(r.packetsDroppedOldTimestamp) { + r.logger.Warnw("dropping packet, old timestamp", nil, zap.Inline(rulf)) + } + + if r.maybeRestart(sequenceNumber, timestamp, payloadSize) { + r.logger.Infow("potential restart", zap.Inline(rulf)) + r.resetRestart() + flowState.UnhandledReason = RTPFlowUnhandledReasonRestart + } else { + flowState.UnhandledReason = RTPFlowUnhandledReasonOldTimestamp + } + return + } + } + + // Case 2: + // Sequence number looks like it is moving forward, but it is actually a very old packet. + if gapTS < 0 && gapSN > 0 { + r.undoUpdatesLocked(resSN, resTS) + + expectedTSJump = int64(r.rtpConverter.ToRTPExt(time.Duration(timeSinceHighest))) + + r.packetsDroppedOldSequenceNumber++ + rulf := &receiverUpdateLoggingFields{ + packetTime: packetTime, + sequenceNumber: sequenceNumber, + timestamp: timestamp, + marker: marker, + hdrSize: hdrSize, + payloadSize: payloadSize, + paddingSize: paddingSize, + resSN: resSN, + gapSN: gapSN, + resTS: resTS, + gapTS: gapTS, + snRolloverCount: snRolloverCount, + expectedTSJump: expectedTSJump, + tsRolloverCount: tsRolloverCount, + timeSinceHighest: timeSinceHighest, + rtpStats: r, + } + if shouldLog(r.packetsDroppedOldSequenceNumber) { + r.logger.Warnw("dropping packet, old sequence number", nil, zap.Inline(rulf)) + } + + if r.maybeRestart(sequenceNumber, timestamp, payloadSize) { + r.logger.Infow("potential restart", zap.Inline(rulf)) + r.resetRestart() + flowState.UnhandledReason = RTPFlowUnhandledReasonRestart + } else { + flowState.UnhandledReason = RTPFlowUnhandledReasonOldSequenceNumber + } return } } - // Case 2: - // Sequence number looks like it is moving forward, but it is actually a very old packet. - if gapTS < 0 && gapSN > 0 { - r.sequenceNumber.UndoUpdate(resSN) - r.timestamp.UndoUpdate(resTS) - logger().Warnw("dropping old packet, sequence number", nil) - flowState.IsNotHandled = true - return - } - // it is possible that sequence number has rolled over too - if gapSN < 0 && gapTS > 0 && payloadSize > 0 { + if (gapSN < 0 || gapSN > (1<<15)) && gapTS > 0 && payloadSize > 0 { // not possible to know how many cycles of sequence number roll over could have happened, // ensure that it at least does not go backwards snRolloverCount = 0 @@ -259,12 +463,63 @@ func (r *RTPStatsReceiver) Update( snRolloverCount = 1 } resSN = r.sequenceNumber.Rollover(sequenceNumber, snRolloverCount) - if resSN.IsUnhandled { - flowState.IsNotHandled = true - return + if !resSN.IsUnhandled { + rulf := &receiverUpdateLoggingFields{ + packetTime: packetTime, + sequenceNumber: sequenceNumber, + timestamp: timestamp, + marker: marker, + hdrSize: hdrSize, + payloadSize: payloadSize, + paddingSize: paddingSize, + resSN: resSN, + gapSN: gapSN, + resTS: resTS, + gapTS: gapTS, + snRolloverCount: snRolloverCount, + expectedTSJump: expectedTSJump, + tsRolloverCount: tsRolloverCount, + timeSinceHighest: timeSinceHighest, + rtpStats: r, + } + r.logger.Warnw("forcing sequence number rollover", nil, zap.Inline(rulf)) + } + } + + if resSN.IsUnhandled { + r.undoUpdatesLocked(resSN, resTS) + + r.packetsDroppedPreStartSequenceNumber++ + rulf := &receiverUpdateLoggingFields{ + packetTime: packetTime, + sequenceNumber: sequenceNumber, + timestamp: timestamp, + marker: marker, + hdrSize: hdrSize, + payloadSize: payloadSize, + paddingSize: paddingSize, + resSN: resSN, + gapSN: gapSN, + resTS: resTS, + gapTS: gapTS, + snRolloverCount: snRolloverCount, + expectedTSJump: expectedTSJump, + tsRolloverCount: tsRolloverCount, + timeSinceHighest: timeSinceHighest, + rtpStats: r, + } + if shouldLog(r.packetsDroppedPreStartSequenceNumber) { + r.logger.Warnw("dropping packet, pre-start sequence number", nil, zap.Inline(rulf)) } - logger().Warnw("forcing sequence number rollover", nil) + if r.maybeRestart(sequenceNumber, timestamp, payloadSize) { + r.logger.Infow("potential restart", zap.Inline(rulf)) + r.resetRestart() + flowState.UnhandledReason = RTPFlowUnhandledReasonRestart + } else { + flowState.UnhandledReason = RTPFlowUnhandledReasonPreStartSequenceNumber + } + return } } gapSN = int64(resSN.ExtendedVal - resSN.PreExtendedHighest) @@ -287,32 +542,31 @@ func (r *RTPStatsReceiver) Update( } flowState.IsOutOfOrder = true - - if !flowState.IsDuplicate && -gapSN >= cSequenceNumberLargeJumpThreshold { - r.largeJumpNegativeCount++ - if (r.largeJumpNegativeCount-1)%100 == 0 { - logger().Warnw( - "large sequence number gap negative", nil, - "count", r.largeJumpNegativeCount, - ) - } - } } else { // in-order - if gapSN >= cSequenceNumberLargeJumpThreshold { - r.largeJumpCount++ - if (r.largeJumpCount-1)%100 == 0 { - logger().Warnw( - "large sequence number gap", nil, - "count", r.largeJumpCount, - ) - } - } - - if resTS.ExtendedVal < resTS.PreExtendedHighest { + if resTS.ExtendedVal < resTS.PreExtendedHighest && r.bytes > 0 { r.timeReversedCount++ - if (r.timeReversedCount-1)%100 == 0 { - logger().Warnw( + if shouldLog(r.timeReversedCount) { + rulf := &receiverUpdateLoggingFields{ + packetTime: packetTime, + sequenceNumber: sequenceNumber, + timestamp: timestamp, + marker: marker, + hdrSize: hdrSize, + payloadSize: payloadSize, + paddingSize: paddingSize, + resSN: resSN, + gapSN: gapSN, + resTS: resTS, + gapTS: gapTS, + snRolloverCount: snRolloverCount, + expectedTSJump: expectedTSJump, + tsRolloverCount: tsRolloverCount, + timeSinceHighest: timeSinceHighest, + rtpStats: r, + } + r.logger.Warnw( "time reversed", nil, + zap.Inline(rulf), "count", r.timeReversedCount, ) } @@ -355,6 +609,7 @@ func (r *RTPStatsReceiver) Update( r.updateJitter(resTS.ExtendedVal, packetTime) } } + r.resetRestart() return } @@ -363,10 +618,12 @@ func (r *RTPStatsReceiver) getExtendedSenderReport(srData *livekit.RTCPSenderRep if r.srNewest != nil { // use time since last sender report to ensure long gaps where the time stamp might // jump more than half the range - timeSinceLastReport := mediatransportutil.NtpTime(srData.NtpTimestamp).Time().Sub(mediatransportutil.NtpTime(r.srNewest.NtpTimestamp).Time()) - expectedRTPTimestampExt := r.srNewest.RtpTimestampExt + uint64(timeSinceLastReport.Nanoseconds()*int64(r.params.ClockRate)/1e9) - lbound := expectedRTPTimestampExt - uint64(cReportSlack*float64(r.params.ClockRate)) - ubound := expectedRTPTimestampExt + uint64(cReportSlack*float64(r.params.ClockRate)) + srTime := mediatransportutil.NtpTime(srData.NtpTimestamp).Time() + srNewestTime := mediatransportutil.NtpTime(r.srNewest.NtpTimestamp).Time() + timeSinceLastReport := srTime.Sub(srNewestTime) + expectedRTPTimestampExt := r.srNewest.RtpTimestampExt + r.rtpConverter.ToRTPExt(timeSinceLastReport) + lbound := expectedRTPTimestampExt - uint64(cReportSlack*float64(r.clockRate)) + ubound := expectedRTPTimestampExt + uint64(cReportSlack*float64(r.clockRate)) isInRange := (srData.RtpTimestamp-uint32(lbound) < (1 << 31)) && (uint32(ubound)-srData.RtpTimestamp < (1 << 31)) if isInRange { lbTSCycles := lbound & 0xFFFF_FFFF_0000_0000 @@ -409,7 +666,7 @@ func (r *RTPStatsReceiver) checkOutOfOrderSenderReport(srData *livekit.RTCPSende // Or it could be due bad report generation. // In any case, ignore out-of-order reports. r.outOfOrderSenderReportCount++ - if (r.outOfOrderSenderReportCount-1)%10 == 0 { + if shouldLog(r.outOfOrderSenderReportCount) { r.logger.Infow( "received sender report, out-of-order, skipping", "current", WrappedRTCPSenderReportStateLogger{srData}, @@ -428,18 +685,22 @@ func (r *RTPStatsReceiver) checkRTPClockSkewForSenderReport(srData *livekit.RTCP return } - timeSinceLast := mediatransportutil.NtpTime(srData.NtpTimestamp).Time().Sub(mediatransportutil.NtpTime(r.srNewest.NtpTimestamp).Time()).Seconds() + srTime := mediatransportutil.NtpTime(srData.NtpTimestamp).Time() + srNewestTime := mediatransportutil.NtpTime(r.srNewest.NtpTimestamp).Time() + srFirstTime := mediatransportutil.NtpTime(r.srFirst.NtpTimestamp).Time() + + timeSinceLast := srTime.Sub(srNewestTime).Seconds() rtpDiffSinceLast := srData.RtpTimestampExt - r.srNewest.RtpTimestampExt calculatedClockRateFromLast := float64(rtpDiffSinceLast) / timeSinceLast - timeSinceFirst := mediatransportutil.NtpTime(srData.NtpTimestamp).Time().Sub(mediatransportutil.NtpTime(r.srFirst.NtpTimestamp).Time()).Seconds() + timeSinceFirst := srTime.Sub(srFirstTime).Seconds() rtpDiffSinceFirst := srData.RtpTimestampExt - r.srFirst.RtpTimestampExt calculatedClockRateFromFirst := float64(rtpDiffSinceFirst) / timeSinceFirst - if (timeSinceLast > 0.2 && math.Abs(float64(r.params.ClockRate)-calculatedClockRateFromLast) > 0.2*float64(r.params.ClockRate)) || - (timeSinceFirst > 0.2 && math.Abs(float64(r.params.ClockRate)-calculatedClockRateFromFirst) > 0.2*float64(r.params.ClockRate)) { + if (timeSinceLast > 0.2 && math.Abs(float64(r.clockRate)-calculatedClockRateFromLast) > 0.2*float64(r.clockRate)) || + (timeSinceFirst > 0.2 && math.Abs(float64(r.clockRate)-calculatedClockRateFromFirst) > 0.2*float64(r.clockRate)) { r.clockSkewCount++ - if (r.clockSkewCount-1)%100 == 0 { + if shouldLog(r.clockSkewCount) { r.logger.Infow( "received sender report, clock skew", "current", WrappedRTCPSenderReportStateLogger{srData}, @@ -463,20 +724,20 @@ func (r *RTPStatsReceiver) checkRTPClockSkewAgainstMediaPathForSenderReport(srDa nowNano := mono.UnixNano() timeSinceSR := time.Duration(nowNano - srData.AtAdjusted) - extNowTSSR := srData.RtpTimestampExt + uint64(timeSinceSR.Nanoseconds()*int64(r.params.ClockRate)/1e9) + extNowTSSR := srData.RtpTimestampExt + r.rtpConverter.ToRTPExt(timeSinceSR) timeSinceHighest := time.Duration(nowNano - r.highestTime) - extNowTSHighest := r.timestamp.GetExtendedHighest() + uint64(timeSinceHighest.Nanoseconds()*int64(r.params.ClockRate)/1e9) + extNowTSHighest := r.timestamp.GetExtendedHighest() + r.rtpConverter.ToRTPExt(timeSinceHighest) diffHighest := extNowTSSR - extNowTSHighest timeSinceFirst := time.Duration(nowNano - r.firstTime) - extNowTSFirst := r.timestamp.GetExtendedStart() + uint64(timeSinceFirst.Nanoseconds()*int64(r.params.ClockRate)/1e9) + extNowTSFirst := r.timestamp.GetExtendedStart() + r.rtpConverter.ToRTPExt(timeSinceFirst) diffFirst := extNowTSSR - extNowTSFirst // is it more than 5 seconds off? - if uint32(math.Abs(float64(int64(diffHighest)))) > 5*r.params.ClockRate || uint32(math.Abs(float64(int64(diffFirst)))) > 5*r.params.ClockRate { + if uint32(math.Abs(float64(int64(diffHighest)))) > 5*r.clockRate || uint32(math.Abs(float64(int64(diffFirst)))) > 5*r.clockRate { r.clockSkewMediaPathCount++ - if (r.clockSkewMediaPathCount-1)%100 == 0 { + if shouldLog(r.clockSkewMediaPathCount) { r.logger.Infow( "received sender report, clock skew against media path", "current", WrappedRTCPSenderReportStateLogger{srData}, @@ -496,7 +757,8 @@ func (r *RTPStatsReceiver) checkRTPClockSkewAgainstMediaPathForSenderReport(srDa } func (r *RTPStatsReceiver) updatePropagationDelayAndRecordSenderReport(srData *livekit.RTCPSenderReportState) { - senderClockTime := mediatransportutil.NtpTime(srData.NtpTimestamp).Time().UnixNano() + srTime := mediatransportutil.NtpTime(srData.NtpTimestamp).Time() + senderClockTime := srTime.UnixNano() estimatedPropagationDelay, stepChange := r.propagationDelayEstimator.Update(senderClockTime, srData.At) if stepChange { r.logger.Debugw( @@ -518,7 +780,7 @@ func (r *RTPStatsReceiver) SetRtcpSenderReportData(srData *livekit.RTCPSenderRep r.lock.Lock() defer r.lock.Unlock() - if srData == nil || !r.initialized { + if srData == nil || !r.initialized || r.clockRate == 0 { return false } @@ -542,9 +804,11 @@ func (r *RTPStatsReceiver) SetRtcpSenderReportData(srData *livekit.RTCPSenderRep r.updatePropagationDelayAndRecordSenderReport(srDataExt) r.checkRTPClockSkewAgainstMediaPathForSenderReport(srDataExt) - if err, loggingFields := r.maybeAdjustFirstPacketTime(r.srNewest, 0, r.timestamp.GetExtendedStart()); err != nil { + adjustment, loggingFields, err := r.maybeAdjustFirstPacketTime(r.srNewest, 0, r.timestamp.GetExtendedStart()) + if err != nil { r.logger.Infow(err.Error(), append(loggingFields, "rtpStats", lockedRTPStatsReceiverLogEncoder{r})...) } + r.propagationDelayEstimator.InitialAdjustment(adjustment) return true } @@ -571,8 +835,8 @@ func (r *RTPStatsReceiver) GetRtcpReceptionReport(ssrc uint32, proxyFracLost uin defer r.lock.Unlock() extHighestSN := r.sequenceNumber.GetExtendedHighest() - then, now := r.getAndResetSnapshot(snapshotID, r.sequenceNumber.GetExtendedStart(), extHighestSN) - if now == nil || then == nil { + then, now, ok := r.getAndResetSnapshot(snapshotID, r.sequenceNumber.GetExtendedStart(), extHighestSN) + if !ok { return nil } @@ -594,15 +858,9 @@ func (r *RTPStatsReceiver) GetRtcpReceptionReport(ssrc uint32, proxyFracLost uin packetsLost = 0 } lossRate := float32(packetsLost) / float32(packetsExpected) - fracLost := uint8(lossRate * 256.0) - if proxyFracLost > fracLost { - fracLost = proxyFracLost - } + fracLost := max(proxyFracLost, uint8(lossRate*256.0)) - totalLost := r.packetsLost - if totalLost > 0xffffff { // 24-bits max - totalLost = 0xffffff - } + totalLost := min(r.packetsLost, 0xffffff) // 24-bits max lastSR := uint32(0) dlsr := uint32(0) @@ -629,7 +887,7 @@ func (r *RTPStatsReceiver) DeltaInfo(snapshotID uint32) *RTPDeltaInfo { r.lock.Lock() defer r.lock.Unlock() - deltaInfo, err, loggingFields := r.deltaInfo( + deltaInfo, loggingFields, err := r.deltaInfo( snapshotID, r.sequenceNumber.GetExtendedStart(), r.sequenceNumber.GetExtendedHighest(), @@ -653,6 +911,10 @@ func (r *RTPStatsReceiver) MarshalLogObject(e zapcore.ObjectEncoder) error { } func (r *RTPStatsReceiver) ToProto() *livekit.RTPStats { + if r == nil { + return nil + } + r.lock.RLock() defer r.lock.RUnlock() @@ -696,6 +958,37 @@ func (r *RTPStatsReceiver) ExtendedHighestSequenceNumber() uint64 { return r.sequenceNumber.GetExtendedHighest() } +func (r *RTPStatsReceiver) maybeRestart(sn uint16, ts uint32, payloadSize int) bool { + if payloadSize > 0 { + if r.restartPacketsN < restartThreshold { + r.restartPacketsBuf[r.restartPacketsN] = packet{sn, ts} + r.restartPacketsN++ + } else { + // keep last restartThreshold entries: shift left and append + copy(r.restartPacketsBuf[:], r.restartPacketsBuf[1:]) + r.restartPacketsBuf[restartThreshold-1] = packet{sn, ts} + } + } + if r.restartPacketsN < restartThreshold { + return false + } + + // check for contiguous sequence numbers and equal or increasing timestamps + for i := 1; i < r.restartPacketsN; i++ { + p := &r.restartPacketsBuf[i] + prev := &r.restartPacketsBuf[i-1] + if p.sequenceNumber != prev.sequenceNumber+1 || (p.timestamp-prev.timestamp) > (1<<31) { + return false + } + } + + return true +} + +func (r *RTPStatsReceiver) resetRestart() { + r.restartPacketsN = 0 +} + // ---------------------------------- type lockedRTPStatsReceiverLogEncoder struct { @@ -725,6 +1018,18 @@ func (r lockedRTPStatsReceiverLogEncoder) MarshalLogObject(e zapcore.ObjectEncod e.AddUint64("extHighestTS", extHighestTS) e.AddObject("propagationDelayEstimator", r.propagationDelayEstimator) + + e.AddInt("clockSkewCount", r.clockSkewCount) + e.AddInt("clockSkewMediaPathCount", r.clockSkewMediaPathCount) + e.AddInt("outOfOrderSenderReportCount", r.outOfOrderSenderReportCount) + e.AddInt("timeReversedCount", r.timeReversedCount) + + e.AddInt("packetsDroppedPreStartTimestamp", r.packetsDroppedPreStartTimestamp) + e.AddInt("packetsDroppedOldTimestamp", r.packetsDroppedOldTimestamp) + e.AddInt("packetsDroppedPreStartSequenceNumber", r.packetsDroppedPreStartSequenceNumber) + e.AddInt("packetsDroppedOldSequenceNumber", r.packetsDroppedOldSequenceNumber) + + e.AddArray("restartPackets", logger.ObjectSlice(r.restartPacketsBuf[:r.restartPacketsN])) return nil } diff --git a/pkg/sfu/rtpstats/rtpstats_receiver_lite.go b/pkg/sfu/rtpstats/rtpstats_receiver_lite.go index 84812a3..82e669b 100644 --- a/pkg/sfu/rtpstats/rtpstats_receiver_lite.go +++ b/pkg/sfu/rtpstats/rtpstats_receiver_lite.go @@ -17,7 +17,7 @@ package rtpstats import ( "go.uber.org/zap/zapcore" - "github.com/livekit/livekit-server/pkg/sfu/utils" + "github.com/livekit/mediatransportutil/pkg/utils" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/utils/mono" ) @@ -102,7 +102,9 @@ func (r *RTPStatsReceiverLite) Update(packetTime int64, packetSize int, sequence gapSN := int64(resSN.ExtendedVal - resSN.PreExtendedHighest) if gapSN <= 0 { // duplicate OR out-of-order r.packetsOutOfOrder++ // counting duplicate as out-of-order - r.packetsLost-- + if gapSN != 0 && r.packetsLost > 0 { + r.packetsLost-- + } } else { // in-order r.updateGapHistogram(int(gapSN)) r.packetsLost += uint64(gapSN - 1) @@ -119,7 +121,7 @@ func (r *RTPStatsReceiverLite) DeltaInfoLite(snapshotLiteID uint32) *RTPDeltaInf r.lock.Lock() defer r.lock.Unlock() - deltaInfoLite, err, loggingFields := r.deltaInfoLite( + deltaInfoLite, loggingFields, err := r.deltaInfoLite( snapshotLiteID, r.sequenceNumber.GetExtendedStart(), r.sequenceNumber.GetExtendedHighest(), diff --git a/pkg/sfu/rtpstats/rtpstats_sender.go b/pkg/sfu/rtpstats/rtpstats_sender.go index cb5d8b7..eafa7bc 100644 --- a/pkg/sfu/rtpstats/rtpstats_sender.go +++ b/pkg/sfu/rtpstats/rtpstats_sender.go @@ -21,12 +21,12 @@ import ( "time" "github.com/pion/rtcp" + "go.uber.org/zap" "go.uber.org/zap/zapcore" "google.golang.org/protobuf/types/known/timestamppb" "github.com/livekit/mediatransportutil" "github.com/livekit/protocol/livekit" - "github.com/livekit/protocol/logger" "github.com/livekit/protocol/utils/mono" ) @@ -40,6 +40,9 @@ const ( snInfoFlagOutOfOrder ) +// max number of receiver reports kept per sender snapshot for logging +const cMaxProcessedReceptionReports = 64 + type snInfo struct { pktSize uint16 hdrSize uint8 @@ -103,7 +106,8 @@ type wrappedReceptionReportsLogger struct { } func (w wrappedReceptionReportsLogger) MarshalLogObject(e zapcore.ObjectEncoder) error { - for i, rr := range w.senderSnapshotReceiverView.processedReceptionReports { + for i := 0; i < w.senderSnapshotReceiverView.processedReceptionReportsSize; i++ { + rr := w.senderSnapshotReceiverView.processedReceptionReports[(w.senderSnapshotReceiverView.processedReceptionReportsHead+i)%cMaxProcessedReceptionReports] e.AddReflected(fmt.Sprintf("%d", i), rr) } @@ -192,9 +196,13 @@ type senderSnapshotReceiverView struct { maxRtt uint32 maxJitter float64 - extLastRRSN uint64 - intervalStats intervalStats - processedReceptionReports []rtcp.ReceptionReport + extLastRRSN uint64 + intervalStats intervalStats + + processedReceptionReports [cMaxProcessedReceptionReports]rtcp.ReceptionReport + processedReceptionReportsHead int + processedReceptionReportsSize int + metadataCacheOverflowCount int } @@ -272,6 +280,70 @@ func (s *senderSnapshot) maybeUpdateMaxJitter(jitter float64) { // ------------------------------------------------------------------- +type senderUpdateLoggingFields struct { + packetTime int64 + extSequenceNumber uint64 + extTimestamp uint64 + marker bool + hdrSize int + payloadSize int + paddingSize int + gapSN int64 + gapTS int64 + timeSinceHighest int64 + rtpStats *RTPStatsSender +} + +func (sulf *senderUpdateLoggingFields) MarshalLogObject(e zapcore.ObjectEncoder) error { + if sulf != nil { + e.AddUint64("currSN", sulf.extSequenceNumber) + e.AddInt64("gapSN", sulf.gapSN) + e.AddUint64("currTS", sulf.extTimestamp) + e.AddInt64("gapTS", sulf.gapTS) + e.AddTime("packetTime", time.Unix(0, sulf.packetTime)) + e.AddDuration("timeSinceHighest", time.Duration(sulf.timeSinceHighest)) + e.AddBool("marker", sulf.marker) + e.AddInt("hdrSize", sulf.hdrSize) + e.AddInt("payloadSize", sulf.payloadSize) + e.AddInt("paddingSize", sulf.paddingSize) + e.AddObject("rtpStats", lockedRTPStatsSenderLogEncoder{sulf.rtpStats}) + } + return nil +} + +// ------------------------------------------------------------------- + +type senderReportLoggingFields struct { + srData *livekit.RTCPSenderReportState + publisherSRData *livekit.RTCPSenderReportState + tsOffset uint64 + reportTime int64 + reportTimeAdjusted int64 + nowNano int64 + nowRTPExt uint64 + rtpStats *RTPStatsSender +} + +func (srlf *senderReportLoggingFields) MarshalLogObject(e zapcore.ObjectEncoder) error { + if srlf != nil { + e.AddObject("curr", WrappedRTCPSenderReportStateLogger{srlf.srData}) + e.AddObject("feed", WrappedRTCPSenderReportStateLogger{srlf.publisherSRData}) + e.AddUint64("tsOffset", srlf.tsOffset) + e.AddTime("timeNow", time.Now()) + e.AddInt64("reportTime", srlf.reportTime) + e.AddInt64("reportTimeAdjusted", srlf.reportTimeAdjusted) + e.AddDuration("timeSinceHighest", time.Duration(srlf.nowNano-srlf.rtpStats.highestTime)) + e.AddDuration("timeSinceFirst", time.Duration(srlf.nowNano-srlf.rtpStats.firstTime)) + e.AddDuration("timeSincePublisherSRAdjusted", time.Duration(srlf.nowNano-srlf.publisherSRData.AtAdjusted)) + e.AddDuration("timeSincePublisherSR", time.Duration(srlf.nowNano-srlf.publisherSRData.At)) + e.AddUint64("nowRTPExt", srlf.nowRTPExt) + e.AddObject("rtpStats", lockedRTPStatsSenderLogEncoder{srlf.rtpStats}) + } + return nil +} + +// ------------------------------------------------------------------- + type rttMarker struct { ntpTime mediatransportutil.NtpTime sentAt time.Time @@ -308,10 +380,8 @@ type RTPStatsSender struct { nextSenderSnapshotID uint32 senderSnapshots []senderSnapshot - clockSkewCount int - largeJumpNegativeCount int - largeJumpCount int - timeReversedCount int + clockSkewCount int + timeReversedCount int } func NewRTPStatsSender(params RTPStatsParams, cacheSize int) *RTPStatsSender { @@ -399,7 +469,7 @@ func (r *RTPStatsSender) Update( r.lock.Lock() defer r.lock.Unlock() - if r.endTime != 0 { + if r.endTime != 0 || r.clockRate == 0 { return } @@ -444,21 +514,6 @@ func (r *RTPStatsSender) Update( pktSize := uint64(hdrSize + payloadSize + paddingSize) isDuplicate := false gapSN := int64(extSequenceNumber - r.extHighestSN) - ulgr := func() logger.UnlikelyLogger { - return r.logger.WithUnlikelyValues( - "currSN", extSequenceNumber, - "gapSN", gapSN, - "currTS", extTimestamp, - "gapTS", int64(extTimestamp-r.extHighestTS), - "packetTime", time.Unix(0, packetTime), - "timeSinceHighest", time.Duration(packetTime-r.highestTime), - "marker", marker, - "hdrSize", hdrSize, - "payloadSize", payloadSize, - "paddingSize", paddingSize, - "rtpStats", lockedRTPStatsSenderLogEncoder{r}, - ) - } if gapSN <= 0 { // duplicate OR out-of-order if payloadSize == 0 && extSequenceNumber < r.extStartSN && !r.params.IsRTX { // do not start on a padding only packet @@ -479,8 +534,22 @@ func (r *RTPStatsSender) Update( r.senderSnapshots[i].maybeReinit(r.extStartSN, extSequenceNumber) } - ulgr().Infow( + sulf := &senderUpdateLoggingFields{ + packetTime: packetTime, + extSequenceNumber: extSequenceNumber, + extTimestamp: extTimestamp, + marker: marker, + hdrSize: hdrSize, + payloadSize: payloadSize, + paddingSize: paddingSize, + gapSN: gapSN, + gapTS: int64(extTimestamp - r.extHighestTS), + timeSinceHighest: packetTime - r.highestTime, + rtpStats: r, + } + r.logger.Infow( "adjusting start sequence number", + zap.Inline(sulf), "snAfter", extSequenceNumber, "tsAfter", extTimestamp, ) @@ -500,32 +569,26 @@ func (r *RTPStatsSender) Update( r.packetsLost-- r.setSnInfo(extSequenceNumber, r.extHighestSN, uint16(pktSize), uint8(hdrSize), uint16(payloadSize), marker, true) } - - if !isDuplicate && -gapSN >= cSequenceNumberLargeJumpThreshold { - r.largeJumpNegativeCount++ - if (r.largeJumpNegativeCount-1)%100 == 0 { - ulgr().Warnw( - "large sequence number gap negative", nil, - "count", r.largeJumpNegativeCount, - ) - } - } } else { // in-order - if gapSN >= cSequenceNumberLargeJumpThreshold { - r.largeJumpCount++ - if (r.largeJumpCount-1)%100 == 0 { - ulgr().Warnw( - "large sequence number gap", nil, - "count", r.largeJumpCount, - ) - } - } - if extTimestamp < r.extHighestTS { r.timeReversedCount++ - if (r.timeReversedCount-1)%100 == 0 { - ulgr().Warnw( + if shouldLog(r.timeReversedCount) { + sulf := &senderUpdateLoggingFields{ + packetTime: packetTime, + extSequenceNumber: extSequenceNumber, + extTimestamp: extTimestamp, + marker: marker, + hdrSize: hdrSize, + payloadSize: payloadSize, + paddingSize: paddingSize, + gapSN: gapSN, + gapTS: int64(extTimestamp - r.extHighestTS), + timeSinceHighest: packetTime - r.highestTime, + rtpStats: r, + } + r.logger.Warnw( "time reversed", nil, + zap.Inline(sulf), "count", r.timeReversedCount, ) } @@ -544,8 +607,22 @@ func (r *RTPStatsSender) Update( } if extTimestamp < r.extStartTS { - ulgr().Infow( + sulf := &senderUpdateLoggingFields{ + packetTime: packetTime, + extSequenceNumber: extSequenceNumber, + extTimestamp: extTimestamp, + marker: marker, + hdrSize: hdrSize, + payloadSize: payloadSize, + paddingSize: paddingSize, + gapSN: gapSN, + gapTS: int64(extTimestamp - r.extHighestTS), + timeSinceHighest: packetTime - r.highestTime, + rtpStats: r, + } + r.logger.Infow( "adjusting start timestamp", + zap.Inline(sulf), "snAfter", extSequenceNumber, "tsAfter", extTimestamp, ) @@ -628,14 +705,24 @@ func (r *RTPStatsSender) UpdateFromReceiverReport(rr rtcp.ReceptionReport) (rtt r.lastRR = rr }() - timeSinceLastRR := func() time.Duration { - if r.lastRRTime != 0 { - return time.Duration(nowNano - r.lastRRTime) - } - return time.Duration(nowNano - r.startTime) + timeSinceLastRR := time.Duration(nowNano - r.startTime) + if r.lastRRTime != 0 { + timeSinceLastRR = time.Duration(nowNano - r.lastRRTime) } extReceivedRRSN := extHighestSNFromRR + (r.extStartSN & 0xFFFF_FFFF_FFFF_0000) + if int64(r.extHighestSN-extReceivedRRSN) < 0 || int64(r.extHighestSN-extReceivedRRSN) > (1<<32) { + r.logger.Infow( + "receiver report runaway, dropping", + "timeSinceLastRR", timeSinceLastRR, + "receivedRR", rr, + "extHighestSNFromRR", extHighestSNFromRR, + "extReceivedRRSN", extReceivedRRSN, + "rtpStats", lockedRTPStatsSenderLogEncoder{r}, + ) + return + } + if r.extHighestSNFromRR != extHighestSNFromRR && int64(r.extHighestSN-extReceivedRRSN) >= (1<<16) { // there are cases where remote does not send RTCP Receiver Report for extended periods of time, // some times several minutes, in that interval the sequence number rolls over, @@ -645,7 +732,7 @@ func (r *RTPStatsSender) UpdateFromReceiverReport(rr rtcp.ReceptionReport) (rtt // should not roll over for long time, for e. g. it will approximately take 100 days at 500 pps). // So, there seems to be a remote reporter issue where the sequence number rollover is missed. // - // catch up till diffrence between highest sent and highest received via receiver report is + // catch up till difference between highest sent and highest received via receiver report is // less than full 16-bit range. // // in a different flavor, there are clients that do not report properly, @@ -654,7 +741,7 @@ func (r *RTPStatsSender) UpdateFromReceiverReport(rr rtcp.ReceptionReport) (rtt // RTCP RR does not change. r.logger.Infow( "receiver report missed rollover, adjusting", - "timeSinceLastRR", timeSinceLastRR(), + "timeSinceLastRR", timeSinceLastRR, "receivedRR", rr, "extHighestSNFromRR", extHighestSNFromRR, "extReceivedRRSN", extReceivedRRSN, @@ -667,7 +754,7 @@ func (r *RTPStatsSender) UpdateFromReceiverReport(rr rtcp.ReceptionReport) (rtt } r.logger.Infow( "receiver report missed rollover, adjusted", - "timeSinceLastRR", timeSinceLastRR(), + "timeSinceLastRR", timeSinceLastRR, "receivedRR", rr, "extHighestSNFromRR", extHighestSNFromRR, "extReceivedRRSN", extReceivedRRSN, @@ -679,7 +766,7 @@ func (r *RTPStatsSender) UpdateFromReceiverReport(rr rtcp.ReceptionReport) (rtt // if remote adjusts somehow, roll back alignment r.logger.Infow( "receiver report caught up rollover, adjusting", - "timeSinceLastRR", timeSinceLastRR(), + "timeSinceLastRR", timeSinceLastRR, "receivedRR", rr, "extHighestSNFromRR", extHighestSNFromRR, "extReceivedRRSN", extReceivedRRSN, @@ -692,7 +779,7 @@ func (r *RTPStatsSender) UpdateFromReceiverReport(rr rtcp.ReceptionReport) (rtt } r.logger.Infow( "receiver report caught up rollover, adjusted", - "timeSinceLastRR", timeSinceLastRR(), + "timeSinceLastRR", timeSinceLastRR, "receivedRR", rr, "extHighestSNFromRR", extHighestSNFromRR, "extReceivedRRSN", extReceivedRRSN, @@ -701,9 +788,9 @@ func (r *RTPStatsSender) UpdateFromReceiverReport(rr rtcp.ReceptionReport) (rtt } if r.extHighestSNFromRR > extHighestSNFromRR { - r.logger.Infow( + r.logger.Debugw( "receiver report out-of-order, dropping", - "timeSinceLastRR", timeSinceLastRR(), + "timeSinceLastRR", timeSinceLastRR, "receivedRR", rr, "extHighestSNFromRR", extHighestSNFromRR, "extReceivedRRSN", extReceivedRRSN, @@ -768,7 +855,7 @@ func (r *RTPStatsSender) UpdateFromReceiverReport(rr rtcp.ReceptionReport) (rtt "metadata cache overflow", "senderSnapshotID", i+cFirstSnapshotID, "senderSnapshot", s, - "timeSinceLastRR", timeSinceLastRR(), + "timeSinceLastRR", timeSinceLastRR, "receivedRR", rr, "extReceivedRRSN", extReceivedRRSN, "packetsInInterval", extReceivedRRSN-s.receiverView.extLastRRSN, @@ -779,7 +866,16 @@ func (r *RTPStatsSender) UpdateFromReceiverReport(rr rtcp.ReceptionReport) (rtt } } s.receiverView.extLastRRSN = extReceivedRRSN - s.receiverView.processedReceptionReports = append(s.receiverView.processedReceptionReports, rr) + + s.receiverView.processedReceptionReports[s.receiverView.processedReceptionReportsHead] = rr + s.receiverView.processedReceptionReportsHead++ + if s.receiverView.processedReceptionReportsHead >= cMaxProcessedReceptionReports { + s.receiverView.processedReceptionReportsHead = 0 + } + s.receiverView.processedReceptionReportsSize++ + if s.receiverView.processedReceptionReportsSize > cMaxProcessedReceptionReports { + s.receiverView.processedReceptionReportsSize = cMaxProcessedReceptionReports + } } return @@ -796,11 +892,11 @@ func (r *RTPStatsSender) MaybeAdjustFirstPacketTime(publisherSRData *livekit.RTC r.lock.Lock() defer r.lock.Unlock() - if !r.initialized || publisherSRData == nil { + if !r.initialized || publisherSRData == nil || r.clockRate == 0 { return } - if err, loggingFields := r.maybeAdjustFirstPacketTime(publisherSRData, tsOffset, r.extStartTS); err != nil { + if _, loggingFields, err := r.maybeAdjustFirstPacketTime(publisherSRData, tsOffset, r.extStartTS); err != nil { r.logger.Infow(err.Error(), append(loggingFields, "rtpStats", lockedRTPStatsSenderLogEncoder{r})...) } } @@ -809,14 +905,13 @@ func (r *RTPStatsSender) GetExpectedRTPTimestamp(at time.Time) (expectedTSExt ui r.lock.RLock() defer r.lock.RUnlock() - if r.firstTime == 0 { + if r.firstTime == 0 || r.clockRate == 0 { err = errors.New("uninitialized") return } timeDiff := at.Sub(time.Unix(0, r.firstTime)) - expectedRTPDiff := timeDiff.Nanoseconds() * int64(r.params.ClockRate) / 1e9 - expectedTSExt = r.extStartTS + uint64(expectedRTPDiff) + expectedTSExt = r.extStartTS + r.rtpConverter.ToRTPExt(timeDiff) return } @@ -824,7 +919,7 @@ func (r *RTPStatsSender) GetRtcpSenderReport(ssrc uint32, publisherSRData *livek r.lock.Lock() defer r.lock.Unlock() - if !r.initialized || publisherSRData == nil { + if !r.initialized || publisherSRData == nil || r.clockRate == 0 { return nil } @@ -834,19 +929,21 @@ func (r *RTPStatsSender) GetRtcpSenderReport(ssrc uint32, publisherSRData *livek nowNTP mediatransportutil.NtpTime nowRTPExt uint64 ) + nowNano := mono.UnixNano() if passThrough { - reportTime = publisherSRData.At - reportTimeAdjusted = publisherSRData.AtAdjusted + timeSincePublisherSR := time.Duration(nowNano - publisherSRData.At) + reportTime = publisherSRData.At + timeSincePublisherSR.Nanoseconds() + reportTimeAdjusted = publisherSRData.AtAdjusted + timeSincePublisherSR.Nanoseconds() - nowNTP = mediatransportutil.NtpTime(publisherSRData.NtpTimestamp) - nowRTPExt = publisherSRData.RtpTimestampExt - tsOffset + nowNTP = mediatransportutil.ToNtpTime(mediatransportutil.NtpTime(publisherSRData.NtpTimestamp).Time().Add(timeSincePublisherSR)) + nowRTPExt = publisherSRData.RtpTimestampExt - tsOffset + r.rtpConverter.ToRTPExt(timeSincePublisherSR) } else { - timeSincePublisherSRAdjusted := time.Duration(mono.UnixNano() - publisherSRData.AtAdjusted) + timeSincePublisherSRAdjusted := time.Duration(nowNano - publisherSRData.AtAdjusted) reportTimeAdjusted = publisherSRData.AtAdjusted + timeSincePublisherSRAdjusted.Nanoseconds() reportTime = reportTimeAdjusted nowNTP = mediatransportutil.ToNtpTime(time.Unix(0, reportTime)) - nowRTPExt = publisherSRData.RtpTimestampExt - tsOffset + uint64(timeSincePublisherSRAdjusted.Nanoseconds()*int64(r.params.ClockRate)/1e9) + nowRTPExt = publisherSRData.RtpTimestampExt - tsOffset + r.rtpConverter.ToRTPExt(timeSincePublisherSRAdjusted) } packetCount := uint32(r.getPacketsSeenPlusDuplicates(r.extStartSN, r.extHighestSN)) @@ -861,33 +958,28 @@ func (r *RTPStatsSender) GetRtcpSenderReport(ssrc uint32, publisherSRData *livek Octets: octetCount, } - ulgr := func() logger.UnlikelyLogger { - nowNano := mono.UnixNano() - return r.logger.WithUnlikelyValues( - "curr", WrappedRTCPSenderReportStateLogger{srData}, - "feed", WrappedRTCPSenderReportStateLogger{publisherSRData}, - "tsOffset", tsOffset, - "timeNow", mono.Now(), - "reportTime", time.Unix(0, reportTime), - "reportTimeAdjusted", time.Unix(0, reportTimeAdjusted), - "timeSinceHighest", time.Duration(nowNano-r.highestTime), - "timeSinceFirst", time.Duration(nowNano-r.firstTime), - "timeSincePublisherSRAdjusted", time.Duration(nowNano-publisherSRData.AtAdjusted), - "timeSincePublisherSR", time.Duration(nowNano-publisherSRData.At), - "nowRTPExt", nowRTPExt, - "rtpStats", lockedRTPStatsSenderLogEncoder{r}, - ) - } - - if r.srNewest != nil && nowRTPExt >= r.srNewest.RtpTimestampExt { - timeSinceLastReport := nowNTP.Time().Sub(mediatransportutil.NtpTime(r.srNewest.NtpTimestamp).Time()) + if r.srNewest != nil && nowRTPExt >= r.srNewest.RtpTimestampExt && r.clockRate != 0 { + nowTime := nowNTP.Time() + srNewestTime := mediatransportutil.NtpTime(r.srNewest.NtpTimestamp).Time() + timeSinceLastReport := nowTime.Sub(srNewestTime) rtpDiffSinceLastReport := nowRTPExt - r.srNewest.RtpTimestampExt windowClockRate := float64(rtpDiffSinceLastReport) / timeSinceLastReport.Seconds() - if timeSinceLastReport.Seconds() > 0.2 && math.Abs(float64(r.params.ClockRate)-windowClockRate) > 0.2*float64(r.params.ClockRate) { + if timeSinceLastReport.Seconds() > 0.2 && math.Abs(float64(r.clockRate)-windowClockRate) > 0.2*float64(r.clockRate) { r.clockSkewCount++ - if (r.clockSkewCount-1)%100 == 0 { - ulgr().Infow( + if shouldLog(r.clockSkewCount) { + srlf := &senderReportLoggingFields{ + srData: srData, + publisherSRData: publisherSRData, + tsOffset: tsOffset, + reportTime: reportTime, + reportTimeAdjusted: reportTimeAdjusted, + nowNano: nowNano, + nowRTPExt: nowRTPExt, + rtpStats: r, + } + r.logger.Infow( "sending sender report, clock skew", + zap.Inline(srlf), "timeSinceLastReport", timeSinceLastReport, "rtpDiffSinceLastReport", rtpDiffSinceLastReport, "windowClockRate", windowClockRate, @@ -900,7 +992,20 @@ func (r *RTPStatsSender) GetRtcpSenderReport(ssrc uint32, publisherSRData *livek if r.srNewest != nil && nowRTPExt < r.srNewest.RtpTimestampExt { // If report being generated is behind the last report, skip it. // Should not happen. - ulgr().Infow("sending sender report, out-of-order, skipping") + srlf := &senderReportLoggingFields{ + srData: srData, + publisherSRData: publisherSRData, + tsOffset: tsOffset, + reportTime: reportTime, + reportTimeAdjusted: reportTimeAdjusted, + nowNano: nowNano, + nowRTPExt: nowRTPExt, + rtpStats: r, + } + r.logger.Infow( + "sending sender report, out-of-order, skipping", + zap.Inline(srlf), + ) return nil } @@ -911,7 +1016,7 @@ func (r *RTPStatsSender) GetRtcpSenderReport(ssrc uint32, publisherSRData *livek r.rttMarker = rttMarker{ ntpTime: nowNTP, - sentAt: mono.Now(), + sentAt: time.Now(), } return &rtcp.SenderReport{ @@ -927,7 +1032,7 @@ func (r *RTPStatsSender) DeltaInfo(snapshotID uint32) *RTPDeltaInfo { r.lock.Lock() defer r.lock.Unlock() - deltaInfo, err, loggingFields := r.deltaInfo( + deltaInfo, loggingFields, err := r.deltaInfo( snapshotID, r.extStartSN, r.extHighestSN, @@ -943,9 +1048,13 @@ func (r *RTPStatsSender) DeltaInfoSender(senderSnapshotID uint32) (*RTPDeltaInfo r.lock.Lock() defer r.lock.Unlock() + if r.clockRate == 0 { + return nil, nil + } + var deltaStatsSenderView *RTPDeltaInfo - thenSenderView, nowSenderView := r.getAndResetSenderSnapshotWindow(senderSnapshotID) - if thenSenderView != nil && nowSenderView != nil { + thenSenderView, nowSenderView, ok := r.getAndResetSenderSnapshotWindow(senderSnapshotID) + if ok { startTime := thenSenderView.startTime endTime := nowSenderView.startTime @@ -979,7 +1088,7 @@ func (r *RTPStatsSender) DeltaInfoSender(senderSnapshotID uint32) (*RTPDeltaInfo packetsLostFeed = packetsExpected } - maxJitterTime := thenSenderView.maxJitterFeed / float64(r.params.ClockRate) * 1e6 + maxJitterTime := thenSenderView.maxJitterFeed / float64(r.clockRate) * 1e6 deltaStatsSenderView = &RTPDeltaInfo{ StartTime: time.Unix(0, startTime), @@ -1007,8 +1116,8 @@ func (r *RTPStatsSender) DeltaInfoSender(senderSnapshotID uint32) (*RTPDeltaInfo var deltaStatsReceiverView *RTPDeltaInfo if r.lastRRTime != 0 { - thenReceiverView, nowReceiverView := r.getAndResetSenderSnapshotReceiverView(senderSnapshotID) - if thenReceiverView != nil && nowReceiverView != nil { + thenReceiverView, nowReceiverView, ok := r.getAndResetSenderSnapshotReceiverView(senderSnapshotID) + if ok { startTime := thenReceiverView.startTime endTime := nowReceiverView.startTime @@ -1048,12 +1157,7 @@ func (r *RTPStatsSender) DeltaInfoSender(senderSnapshotID uint32) (*RTPDeltaInfo packetsLost = packetsExpected } - // discount jitter from publisher side + internal processing - maxJitter := thenReceiverView.maxJitter - thenReceiverView.maxJitterFeed - if maxJitter < 0.0 { - maxJitter = 0.0 - } - maxJitterTime := maxJitter / float64(r.params.ClockRate) * 1e6 + maxJitterTime := thenReceiverView.maxJitter / float64(r.clockRate) * 1e6 deltaStatsReceiverView = &RTPDeltaInfo{ StartTime: time.Unix(0, startTime), @@ -1117,9 +1221,9 @@ func (r *RTPStatsSender) ToProto() *livekit.RTPStats { return p } -func (r *RTPStatsSender) getAndResetSenderSnapshotWindow(senderSnapshotID uint32) (*senderSnapshotWindow, *senderSnapshotWindow) { - if !r.initialized { - return nil, nil +func (r *RTPStatsSender) getAndResetSenderSnapshotWindow(senderSnapshotID uint32) (senderSnapshotWindow, senderSnapshotWindow, bool) { + if !r.initialized || senderSnapshotID < cFirstSnapshotID { + return senderSnapshotWindow{}, senderSnapshotWindow{}, false } idx := senderSnapshotID - cFirstSnapshotID @@ -1131,7 +1235,7 @@ func (r *RTPStatsSender) getAndResetSenderSnapshotWindow(senderSnapshotID uint32 // snapshot now r.senderSnapshots[idx].senderView = r.getSenderSnapshotWindow(mono.UnixNano()) - return &then.senderView, &r.senderSnapshots[idx].senderView + return then.senderView, r.senderSnapshots[idx].senderView, true } func (r *RTPStatsSender) getSenderSnapshotWindow(startTime int64) senderSnapshotWindow { @@ -1158,9 +1262,9 @@ func (r *RTPStatsSender) getSenderSnapshotWindow(startTime int64) senderSnapshot } } -func (r *RTPStatsSender) getAndResetSenderSnapshotReceiverView(senderSnapshotID uint32) (*senderSnapshotReceiverView, *senderSnapshotReceiverView) { - if !r.initialized || r.lastRRTime == 0 { - return nil, nil +func (r *RTPStatsSender) getAndResetSenderSnapshotReceiverView(senderSnapshotID uint32) (senderSnapshotReceiverView, senderSnapshotReceiverView, bool) { + if !r.initialized || r.lastRRTime == 0 || senderSnapshotID < cFirstSnapshotID { + return senderSnapshotReceiverView{}, senderSnapshotReceiverView{}, false } idx := senderSnapshotID - cFirstSnapshotID @@ -1172,7 +1276,7 @@ func (r *RTPStatsSender) getAndResetSenderSnapshotReceiverView(senderSnapshotID // snapshot now r.senderSnapshots[idx].receiverView = r.getSenderSnapshotReceiverView(r.lastRRTime, &then.receiverView) - return &then.receiverView, &r.senderSnapshots[idx].receiverView + return then.receiverView, r.senderSnapshots[idx].receiverView, true } func (r *RTPStatsSender) getSenderSnapshotReceiverView(startTime int64, s *senderSnapshotReceiverView) senderSnapshotReceiverView { @@ -1251,11 +1355,13 @@ func (r *RTPStatsSender) clearSnInfos(extStartInclusive uint64, extEndExclusive return } + if extEndExclusive-extStartInclusive > uint64(len(r.snInfos)) { + clear(r.snInfos) + return + } + for esn := extStartInclusive; esn != extEndExclusive; esn++ { - snInfo := &r.snInfos[int(esn)%len(r.snInfos)] - snInfo.pktSize = 0 - snInfo.hdrSize = 0 - snInfo.flags = 0 + r.snInfos[int(esn)%len(r.snInfos)] = snInfo{} } } @@ -1273,14 +1379,18 @@ func (r *RTPStatsSender) getIntervalStats( extEndExclusive uint64, ehsn uint64, ) (intervalStats intervalStats) { - processESN := func(esn uint64, ehsn uint64) { - slot := r.getSnInfoOutOfOrderSlot(esn, ehsn) - if slot < 0 { - intervalStats.packetsNotFoundMetadata++ - return - } + upperBound := ehsn + 1 + lowerBound := uint64(0) + if n := uint64(len(r.snInfos)); n != 0 && ehsn >= n-1 { + lowerBound = ehsn - n + 1 + } + extStartInclusiveClamped := max(min(extStartInclusive, upperBound), lowerBound) + extEndExclusiveClamped := max(min(extEndExclusive, upperBound), extStartInclusiveClamped) - snInfo := &r.snInfos[slot] + intervalStats.packetsNotFoundMetadata = (extEndExclusive - extStartInclusive) - (extEndExclusiveClamped - extStartInclusiveClamped) + + for esn := extStartInclusiveClamped; esn != extEndExclusiveClamped; esn++ { + snInfo := &r.snInfos[int(esn)%len(r.snInfos)] switch { case snInfo.pktSize == 0: intervalStats.packetsLostFeed++ @@ -1303,10 +1413,6 @@ func (r *RTPStatsSender) getIntervalStats( intervalStats.frames++ } } - - for esn := extStartInclusive; esn != extEndExclusive; esn++ { - processESN(esn, ehsn) - } return } diff --git a/pkg/sfu/rtpstats/rtpstats_receiver_test.go b/pkg/sfu/rtpstats/rtpstats_test.go similarity index 74% rename from pkg/sfu/rtpstats/rtpstats_receiver_test.go rename to pkg/sfu/rtpstats/rtpstats_test.go index d895b57..9039ad7 100644 --- a/pkg/sfu/rtpstats/rtpstats_receiver_test.go +++ b/pkg/sfu/rtpstats/rtpstats_test.go @@ -21,8 +21,6 @@ import ( "github.com/pion/rtp" "github.com/stretchr/testify/require" - - "github.com/livekit/protocol/logger" ) func getPacket(sn uint16, ts uint32, payloadSize int) *rtp.Packet { @@ -37,10 +35,8 @@ func getPacket(sn uint16, ts uint32, payloadSize int) *rtp.Packet { func Test_RTPStatsReceiver_Update(t *testing.T) { clockRate := uint32(90000) - r := NewRTPStatsReceiver(RTPStatsParams{ - ClockRate: clockRate, - Logger: logger.GetLogger(), - }) + r := NewRTPStatsReceiver(RTPStatsParams{}) + r.SetClockRate(clockRate) sequenceNumber := uint16(rand.Float64() * float64(1<<16)) timestamp := uint32(rand.Float64() * float64(1<<32)) @@ -89,7 +85,7 @@ func Test_RTPStatsReceiver_Update(t *testing.T) { len(packet.Payload), 0, ) - require.True(t, flowState.IsNotHandled) + require.Equal(t, RTPFlowUnhandledReasonPreStartTimestamp, flowState.UnhandledReason) require.Equal(t, sequenceNumber, r.sequenceNumber.GetHighest()) require.Equal(t, sequenceNumber, uint16(r.sequenceNumber.GetExtendedHighest())) require.Equal(t, timestamp, r.timestamp.GetHighest()) @@ -108,7 +104,7 @@ func Test_RTPStatsReceiver_Update(t *testing.T) { len(packet.Payload), 0, ) - require.True(t, flowState.IsNotHandled) + require.Equal(t, RTPFlowUnhandledReasonPreStartTimestamp, flowState.UnhandledReason) require.Equal(t, sequenceNumber, r.sequenceNumber.GetHighest()) require.Equal(t, sequenceNumber, uint16(r.sequenceNumber.GetExtendedHighest())) require.Equal(t, timestamp, r.timestamp.GetHighest()) @@ -218,7 +214,7 @@ func Test_RTPStatsReceiver_Update(t *testing.T) { len(packet.Payload), 0, ) - require.True(t, flowState.IsNotHandled) + require.Equal(t, RTPFlowUnhandledReasonOldSequenceNumber, flowState.UnhandledReason) require.Equal(t, sequenceNumber, r.sequenceNumber.GetHighest()) require.Equal(t, sequenceNumber, uint16(r.sequenceNumber.GetExtendedHighest())) require.Equal(t, timestamp, r.timestamp.GetHighest()) @@ -226,3 +222,65 @@ func Test_RTPStatsReceiver_Update(t *testing.T) { r.Stop() } + +func Test_RTPStatsReceiver_Restart(t *testing.T) { + clockRate := uint32(90000) + r := NewRTPStatsReceiver(RTPStatsParams{}) + r.SetClockRate(clockRate) + + // should not restart till there are at least threshold packets + require.False(t, r.maybeRestart(10, 20, 1000)) + require.False(t, r.maybeRestart(11, 20, 1000)) + require.False(t, r.maybeRestart(13, 20, 1000)) + require.False(t, r.maybeRestart(14, 20, 1000)) + // although adding 5th packet should have enough packets for a check, + // still should not restart as there is a sequence number gap between 11 and 13 + require.False(t, r.maybeRestart(15, 20, 1000)) + require.False(t, r.maybeRestart(16, 19, 1000)) + // has enough packets, but still cannot restart because timestamps are not increasing + require.False(t, r.maybeRestart(17, 21, 1000)) + require.False(t, r.maybeRestart(18, 21, 1000)) + require.False(t, r.maybeRestart(19, 21, 1000)) + // can restart as there are enough packets with proper sequencing + require.True(t, r.maybeRestart(20, 21, 1000)) + require.Equal(t, restartThreshold, r.restartPacketsN) + + r.resetRestart() + require.Zero(t, r.restartPacketsN) + + r.Stop() +} + +func Test_RTPStatsSender_getIntervalStats(t *testing.T) { + t.Run("packetsNotFoundMetadata should match lost packets", func(t *testing.T) { + r := NewRTPStatsSender(RTPStatsParams{}, 1024) + stats := r.getIntervalStats(0, 10000, 10000) + require.EqualValues(t, 8977, stats.packetsNotFoundMetadata) + }) +} + +func BenchmarkRTPStatsReceiver_Update(b *testing.B) { + const clockRate = 90000 + const hdrSize = 12 + const payloadSize = 1200 + const paddingSize = 0 + const tsIncrement = 3000 // ~33ms at 90kHz + + r := NewRTPStatsReceiver(RTPStatsParams{}) + r.SetClockRate(clockRate) + + // initialize before running benchmark loop + packetTime := time.Now().UnixNano() + sn := uint16(0) + ts := uint32(0) + r.Update(packetTime, sn, ts, false, hdrSize, payloadSize, paddingSize) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + sn++ + ts += tsIncrement + packetTime += 33 * int64(time.Millisecond) // ~30fps + marker := (i+1)%100 == 0 // marker every 100 packets + r.Update(packetTime, sn, ts, marker, hdrSize, payloadSize, paddingSize) + } +} diff --git a/pkg/sfu/sequencer.go b/pkg/sfu/sequencer.go index 5ba788b..1e08ef9 100644 --- a/pkg/sfu/sequencer.go +++ b/pkg/sfu/sequencer.go @@ -119,6 +119,10 @@ type sequencer struct { } func newSequencer(size int, maybeSparse bool, logger logger.Logger) *sequencer { + if size == 0 { + return nil + } + s := &sequencer{ size: size, startTime: time.Now().UnixNano(), diff --git a/pkg/sfu/sfu.go b/pkg/sfu/sfu.go index 648c1e5..c8760d0 100644 --- a/pkg/sfu/sfu.go +++ b/pkg/sfu/sfu.go @@ -16,18 +16,21 @@ package sfu import ( "sync" + + "github.com/pion/rtp" ) var ( - PacketFactory *sync.Pool -) - -func init() { - // Init packet factory PacketFactory = &sync.Pool{ - New: func() interface{} { + New: func() any { b := make([]byte, 1460) return &b }, } -} + + RTPHeaderFactory = &sync.Pool{ + New: func() any { + return &rtp.Header{} + }, + } +) diff --git a/pkg/sfu/streamallocator/streamallocator.go b/pkg/sfu/streamallocator/streamallocator.go index 0f1bd65..31d58f1 100644 --- a/pkg/sfu/streamallocator/streamallocator.go +++ b/pkg/sfu/streamallocator/streamallocator.go @@ -16,6 +16,7 @@ package streamallocator import ( "fmt" + "slices" "sort" "sync" "time" @@ -37,21 +38,24 @@ import ( ) const ( - ChannelCapacityInfinity = 100 * 1000 * 1000 // 100 Mbps + cChannelCapacityInfinity = 100 * 1000 * 1000 // 100 Mbps - PriorityMin = uint8(1) - PriorityMax = uint8(255) - PriorityDefaultScreenshare = PriorityMax - PriorityDefaultVideo = PriorityMin + cPriorityMin = uint8(1) + cPriorityMax = uint8(255) + cPriorityDefaultScreenshare = cPriorityMax + cPriorityDefaultVideo = cPriorityMin - FlagAllowOvershootWhileOptimal = true - FlagAllowOvershootWhileDeficient = false - FlagAllowOvershootExemptTrackWhileDeficient = true - FlagAllowOvershootInProbe = true - FlagAllowOvershootInCatchup = false - FlagAllowOvershootInBoost = true + cFlagAllowOvershootWhileOptimal = true + cFlagAllowOvershootWhileDeficient = false + cFlagAllowOvershootExemptTrackWhileDeficient = true + cFlagAllowOvershootInProbe = true + cFlagAllowOvershootInCatchup = false + cFlagAllowOvershootInBoost = true cRTTPullInterval = 30 * time.Second + + cPingLong = cRTTPullInterval / 2 + cPingShort = 100 * time.Millisecond ) // --------------------------------------------------------------------------- @@ -131,13 +135,45 @@ func (s streamAllocatorSignal) String() string { type Event struct { *StreamAllocator - Signal streamAllocatorSignal - TrackID livekit.TrackID - Data interface{} + signal streamAllocatorSignal + + // only one of these fields will be set based on event type + trackID livekit.TrackID + allowPause bool + channelCapacity int64 + twccFeedback *rtcp.TransportLayerCC + congestionStateChangeData congestionStateChangeData + probeBytesToSend int + probeClusterId ccutils.ProbeClusterId + probeClusterInfo ccutils.ProbeClusterInfo } func (e Event) String() string { - return fmt.Sprintf("StreamAllocator:Event{signal: %s, trackID: %s, data: %+v}", e.Signal, e.TrackID, e.Data) + value := "" + switch e.signal { + case streamAllocatorSignalAllocateTrack, streamAllocatorSignalResume: + value = fmt.Sprintf("trackID: %s", e.trackID) + case streamAllocatorSignalEstimate, streamAllocatorSignalSetChannelCapacity: + value = fmt.Sprintf("channelCapacity: %d", e.channelCapacity) + case streamAllocatorSignalFeedback: + value = fmt.Sprintf("twccFeedback: %+v", e.twccFeedback) + case streamAllocatorSignalProbeClusterSwitch: + value = fmt.Sprintf("probeClusterInfo: %+v", e.probeClusterInfo) + case streamAllocatorSignalSendProbe: + value = fmt.Sprintf("probeBytesToSend: %d", e.probeBytesToSend) + case streamAllocatorSignalPacerProbeObserverClusterComplete: + value = fmt.Sprintf("probeClusterId: %v", e.probeClusterId) + case streamAllocatorSignalSetAllowPause: + value = fmt.Sprintf("allowPause: %v", e.allowPause) + case streamAllocatorSignalCongestionStateChange: + value = fmt.Sprintf("congestStatecChangeData: %+v", e.congestionStateChangeData) + } + + if value == "" { + return fmt.Sprintf("StreamAllocator:Event{signal: %s}", e.signal) + } + + return fmt.Sprintf("StreamAllocator:Event{signal: %s, %s}", e.signal, value) } // --------------------------------------------------------------------------- @@ -202,6 +238,7 @@ type StreamAllocator struct { videoTracksMu sync.RWMutex videoTracks map[livekit.TrackID]*Track + videoTracksShadow []*Track isAllocateAllPending bool rembTrackingSSRC uint32 @@ -215,6 +252,8 @@ type StreamAllocator struct { lastRTTTime time.Time + pingGeneration atomic.Uint32 + isStopped atomic.Bool } @@ -247,7 +286,7 @@ func NewStreamAllocator(params StreamAllocatorParams, enabled bool, allowPause b func (s *StreamAllocator) Start() { s.eventsQueue.Start() - go s.ping() + go s.ping(s.pingGeneration.Inc(), cPingLong) } func (s *StreamAllocator) Stop() { @@ -273,10 +312,10 @@ func (s *StreamAllocator) SetSendSideBWEInterceptor(sendSideBWEInterceptor cc.Ba } type AddTrackParams struct { - Source livekit.TrackSource - Priority uint8 - IsSimulcast bool - PublisherID livekit.ParticipantID + Source livekit.TrackSource + Priority uint8 + IsMultiLayered bool + PublisherID livekit.ParticipantID } func (s *StreamAllocator) AddTrack(downTrack *sfu.DownTrack, params AddTrackParams) { @@ -284,13 +323,14 @@ func (s *StreamAllocator) AddTrack(downTrack *sfu.DownTrack, params AddTrackPara return } - track := NewTrack(downTrack, params.Source, params.IsSimulcast, params.PublisherID, s.params.Logger) + track := NewTrack(downTrack, params.Source, params.IsMultiLayered, params.PublisherID, s.params.Logger) track.SetPriority(params.Priority) trackID := livekit.TrackID(downTrack.ID()) s.videoTracksMu.Lock() oldTrack := s.videoTracks[trackID] s.videoTracks[trackID] = track + s.shadowVideoTracksLocked() s.videoTracksMu.Unlock() if oldTrack != nil { @@ -307,12 +347,13 @@ func (s *StreamAllocator) RemoveTrack(downTrack *sfu.DownTrack) { s.videoTracksMu.Lock() if existing := s.videoTracks[livekit.TrackID(downTrack.ID())]; existing != nil && existing.DownTrack() == downTrack { delete(s.videoTracks, livekit.TrackID(downTrack.ID())) + s.shadowVideoTracksLocked() } s.videoTracksMu.Unlock() // STREAM-ALLOCATOR-TODO: use any saved bandwidth to re-distribute s.postEvent(Event{ - Signal: streamAllocatorSignalAdjustState, + signal: streamAllocatorSignalAdjustState, }) } @@ -324,7 +365,7 @@ func (s *StreamAllocator) SetTrackPriority(downTrack *sfu.DownTrack, priority ui // do a full allocation on a track priority change to keep it simple s.isAllocateAllPending = true s.postEvent(Event{ - Signal: streamAllocatorSignalAllocateAllTracks, + signal: streamAllocatorSignalAllocateAllTracks, }) } } @@ -333,15 +374,15 @@ func (s *StreamAllocator) SetTrackPriority(downTrack *sfu.DownTrack, priority ui func (s *StreamAllocator) SetAllowPause(allowPause bool) { s.postEvent(Event{ - Signal: streamAllocatorSignalSetAllowPause, - Data: allowPause, + signal: streamAllocatorSignalSetAllowPause, + allowPause: allowPause, }) } func (s *StreamAllocator) SetChannelCapacity(channelCapacity int64) { s.postEvent(Event{ - Signal: streamAllocatorSignalSetChannelCapacity, - Data: channelCapacity, + signal: streamAllocatorSignalSetChannelCapacity, + channelCapacity: channelCapacity, }) } @@ -385,13 +426,7 @@ func (s *StreamAllocator) OnREMB(downTrack *sfu.DownTrack, remb *rtcp.ReceiverEs downTrackSSRCRTX = track.DownTrack().SSRCRTX() } - found := false - for _, ssrc := range remb.SSRCs { - if ssrc == s.rembTrackingSSRC { - found = true - break - } - } + found := slices.Contains(remb.SSRCs, s.rembTrackingSSRC) if !found { if len(remb.SSRCs) == 0 { s.params.Logger.Warnw("stream allocator: no SSRC to track REMB", nil) @@ -429,24 +464,24 @@ func (s *StreamAllocator) OnREMB(downTrack *sfu.DownTrack, remb *rtcp.ReceiverEs s.videoTracksMu.Unlock() s.postEvent(Event{ - Signal: streamAllocatorSignalEstimate, - Data: int64(remb.Bitrate), + signal: streamAllocatorSignalEstimate, + channelCapacity: int64(remb.Bitrate), }) } // called when a new transport-cc feedback is received func (s *StreamAllocator) OnTransportCCFeedback(downTrack *sfu.DownTrack, fb *rtcp.TransportLayerCC) { s.postEvent(Event{ - Signal: streamAllocatorSignalFeedback, - Data: fb, + signal: streamAllocatorSignalFeedback, + twccFeedback: fb, }) } // called when target bitrate changes (send side bandwidth estimation) func (s *StreamAllocator) onTargetBitrateChange(bitrate int) { s.postEvent(Event{ - Signal: streamAllocatorSignalEstimate, - Data: int64(bitrate), + signal: streamAllocatorSignalEstimate, + channelCapacity: int64(bitrate), }) } @@ -460,8 +495,8 @@ type congestionStateChangeData struct { // BWEListener implementation func (s *StreamAllocator) OnCongestionStateChange(fromState bwe.CongestionState, toState bwe.CongestionState, estimatedAvailableChannelCapacity int64) { s.postEvent(Event{ - Signal: streamAllocatorSignalCongestionStateChange, - Data: congestionStateChangeData{fromState, toState, estimatedAvailableChannelCapacity}, + signal: streamAllocatorSignalCongestionStateChange, + congestionStateChangeData: congestionStateChangeData{fromState, toState, estimatedAvailableChannelCapacity}, }) } @@ -503,8 +538,8 @@ func (s *StreamAllocator) OnSubscribedLayerChanged(downTrack *sfu.DownTrack, lay if shouldPost { s.postEvent(Event{ - Signal: streamAllocatorSignalAllocateTrack, - TrackID: livekit.TrackID(downTrack.ID()), + signal: streamAllocatorSignalAllocateTrack, + trackID: livekit.TrackID(downTrack.ID()), }) } } @@ -512,32 +547,32 @@ func (s *StreamAllocator) OnSubscribedLayerChanged(downTrack *sfu.DownTrack, lay // called when forwarder resumes a track func (s *StreamAllocator) OnResume(downTrack *sfu.DownTrack) { s.postEvent(Event{ - Signal: streamAllocatorSignalResume, - TrackID: livekit.TrackID(downTrack.ID()), + signal: streamAllocatorSignalResume, + trackID: livekit.TrackID(downTrack.ID()), }) } // called when probe cluster changes func (s *StreamAllocator) OnProbeClusterSwitch(pci ccutils.ProbeClusterInfo) { s.postEvent(Event{ - Signal: streamAllocatorSignalProbeClusterSwitch, - Data: pci, + signal: streamAllocatorSignalProbeClusterSwitch, + probeClusterInfo: pci, }) } // called when prober wants to send packet(s) func (s *StreamAllocator) OnSendProbe(bytesToSend int) { s.postEvent(Event{ - Signal: streamAllocatorSignalSendProbe, - Data: bytesToSend, + signal: streamAllocatorSignalSendProbe, + probeBytesToSend: bytesToSend, }) } // called when pacer probe observer observes a cluster completion func (s *StreamAllocator) OnPacerProbeObserverClusterComplete(probeClusterId ccutils.ProbeClusterId) { s.postEvent(Event{ - Signal: streamAllocatorSignalPacerProbeObserverClusterComplete, - Data: probeClusterId, + signal: streamAllocatorSignalPacerProbeObserverClusterComplete, + probeClusterId: probeClusterId, }) } @@ -557,6 +592,10 @@ func (s *StreamAllocator) IsBWEEnabled(downTrack *sfu.DownTrack) bool { return true } +func (s *StreamAllocator) BWEType() bwe.BWEType { + return s.params.BWE.Type() +} + // called to check if track subscription mute can be applied func (s *StreamAllocator) IsSubscribeMutable(downTrack *sfu.DownTrack) bool { s.videoTracksMu.Lock() @@ -579,24 +618,24 @@ func (s *StreamAllocator) maybePostEventAllocateTrack(downTrack *sfu.DownTrack) if shouldPost { s.postEvent(Event{ - Signal: streamAllocatorSignalAllocateTrack, - TrackID: livekit.TrackID(downTrack.ID()), + signal: streamAllocatorSignalAllocateTrack, + trackID: livekit.TrackID(downTrack.ID()), }) } } -func (s *StreamAllocator) ping() { - ticker := time.NewTicker(100 * time.Millisecond) +func (s *StreamAllocator) ping(pingGeneration uint32, interval time.Duration) { + ticker := time.NewTicker(interval) defer ticker.Stop() for { <-ticker.C - if s.isStopped.Load() { + if s.isStopped.Load() || (pingGeneration != s.pingGeneration.Load()) { return } s.postEvent(Event{ - Signal: streamAllocatorSignalPeriodicPing, + signal: streamAllocatorSignalPeriodicPing, }) } } @@ -604,7 +643,7 @@ func (s *StreamAllocator) ping() { func (s *StreamAllocator) postEvent(event Event) { event.StreamAllocator = s s.eventsQueue.Enqueue(func(event Event) { - switch event.Signal { + switch event.signal { case streamAllocatorSignalAllocateTrack: event.handleSignalAllocateTrack(event) case streamAllocatorSignalAllocateAllTracks: @@ -637,7 +676,7 @@ func (s *StreamAllocator) postEvent(event Event) { func (s *StreamAllocator) handleSignalAllocateTrack(event Event) { s.videoTracksMu.Lock() - track := s.videoTracks[event.TrackID] + track := s.videoTracks[event.trackID] if track != nil { track.SetDirty(false) } @@ -663,13 +702,11 @@ func (s *StreamAllocator) handleSignalAdjustState(Event) { } func (s *StreamAllocator) handleSignalEstimate(event Event) { - receivedEstimate := event.Data.(int64) - // always update NACKs packetDelta, repeatedNackDelta := s.getNackDelta() s.params.BWE.HandleREMB( - receivedEstimate, + event.channelCapacity, s.getExpectedBandwidthUsage(), packetDelta, repeatedNackDelta, @@ -677,7 +714,7 @@ func (s *StreamAllocator) handleSignalEstimate(event Event) { } func (s *StreamAllocator) handleSignalFeedback(event Event) { - fb := event.Data.(*rtcp.TransportLayerCC) + fb := event.twccFeedback if s.sendSideBWEInterceptor != nil { s.sendSideBWEInterceptor.WriteRTCP([]rtcp.Packet{fb}, nil) } @@ -745,7 +782,7 @@ func (s *StreamAllocator) handleSignalPeriodicPing(Event) { } func (s *StreamAllocator) handleSignalProbeClusterSwitch(event Event) { - pci := event.Data.(ccutils.ProbeClusterInfo) + pci := event.probeClusterInfo s.activeProbeClusterId = pci.Id s.activeProbeGoalReached = false s.activeProbeCongesting = false @@ -754,19 +791,19 @@ func (s *StreamAllocator) handleSignalProbeClusterSwitch(event Event) { s.params.Pacer.StartProbeCluster(pci) - for _, t := range s.getTracks() { + for _, t := range s.getVideoTracks() { t.DownTrack().SetProbeClusterId(pci.Id) } } func (s *StreamAllocator) handleSignalSendProbe(event Event) { - bytesToSend := event.Data.(int) + bytesToSend := event.probeBytesToSend if bytesToSend <= 0 { return } bytesSent := 0 - for _, track := range s.getTracks() { + for _, track := range s.getVideoTracks() { sent := track.WriteProbePackets(bytesToSend) bytesSent += sent bytesToSend -= sent @@ -779,10 +816,9 @@ func (s *StreamAllocator) handleSignalSendProbe(event Event) { } func (s *StreamAllocator) handleSignalPacerProbeObserverClusterComplete(event Event) { - probeClusterId, _ := event.Data.(ccutils.ProbeClusterId) - pci := s.params.Pacer.EndProbeCluster(probeClusterId) + pci := s.params.Pacer.EndProbeCluster(event.probeClusterId) - for _, t := range s.getTracks() { + for _, t := range s.getVideoTracks() { t.DownTrack().SwapProbeClusterId(pci.Id, ccutils.ProbeClusterIdInvalid) } @@ -792,7 +828,7 @@ func (s *StreamAllocator) handleSignalPacerProbeObserverClusterComplete(event Ev func (s *StreamAllocator) handleSignalResume(event Event) { s.videoTracksMu.Lock() - track := s.videoTracks[event.TrackID] + track := s.videoTracks[event.trackID] updated := track != nil && track.SetStreamState(StreamStateActive) s.videoTracksMu.Unlock() @@ -804,11 +840,11 @@ func (s *StreamAllocator) handleSignalResume(event Event) { } func (s *StreamAllocator) handleSignalSetAllowPause(event Event) { - s.allowPause = event.Data.(bool) + s.allowPause = event.allowPause } func (s *StreamAllocator) handleSignalSetChannelCapacity(event Event) { - s.overriddenChannelCapacity = event.Data.(int64) + s.overriddenChannelCapacity = event.channelCapacity if s.overriddenChannelCapacity > 0 { s.params.Logger.Infow("allocating on override channel capacity", "override", s.overriddenChannelCapacity) s.allocateAllTracks() @@ -818,7 +854,7 @@ func (s *StreamAllocator) handleSignalSetChannelCapacity(event Event) { } func (s *StreamAllocator) handleSignalCongestionStateChange(event Event) { - cscd := event.Data.(congestionStateChangeData) + cscd := event.congestionStateChangeData if cscd.toState != bwe.CongestionStateNone { // end/abort any running probe if channel is not clear s.maybeStopProbe() @@ -828,8 +864,8 @@ func (s *StreamAllocator) handleSignalCongestionStateChange(event Event) { // during early warning hold (if there was one) if isHoldableCongestionState(cscd.fromState) && cscd.toState == bwe.CongestionStateNone && s.state == streamAllocatorStateStable { update := NewStreamStateUpdate() - for _, track := range s.getTracks() { - allocation := track.AllocateOptimal(FlagAllowOvershootWhileOptimal, false) + for _, track := range s.getVideoTracks() { + allocation := track.AllocateOptimal(cFlagAllowOvershootWhileOptimal, false) updateStreamStateChange(track, allocation, update) } s.maybeSendUpdate(update) @@ -875,11 +911,14 @@ func (s *StreamAllocator) setState(state streamAllocatorState) { s.params.BWE.Reset() s.activeProbeClusterId = ccutils.ProbeClusterIdInvalid + go s.ping(s.pingGeneration.Inc(), cPingLong) + } else { + go s.ping(s.pingGeneration.Inc(), cPingShort) } } func (s *StreamAllocator) adjustState() { - for _, track := range s.getTracks() { + for _, track := range s.getVideoTracks() { if track.IsDeficient() { s.setState(streamAllocatorStateDeficient) return @@ -897,7 +936,7 @@ func (s *StreamAllocator) allocateTrack(track *Track) { bweCongestionState := s.params.BWE.CongestionState() if !s.enabled || (s.state == streamAllocatorStateStable && !isDeficientCongestionState(bweCongestionState)) || !track.IsManaged() { update := NewStreamStateUpdate() - allocation := track.AllocateOptimal(FlagAllowOvershootWhileOptimal, isHoldableCongestionState(bweCongestionState)) + allocation := track.AllocateOptimal(cFlagAllowOvershootWhileOptimal, isHoldableCongestionState(bweCongestionState)) updateStreamStateChange(track, allocation, update) s.maybeSendUpdate(update) return @@ -923,7 +962,7 @@ func (s *StreamAllocator) allocateTrack(track *Track) { // track needing change is not currently streaming, i. e. it has to be resumed. // track.ProvisionalAllocatePrepare() - transition := track.ProvisionalAllocateGetCooperativeTransition(FlagAllowOvershootWhileDeficient) + transition := track.ProvisionalAllocateGetCooperativeTransition(cFlagAllowOvershootWhileDeficient) // downgrade, giving back bits if transition.From.GreaterThan(transition.To) { @@ -969,7 +1008,7 @@ func (s *StreamAllocator) allocateTrack(track *Track) { availableChannelCapacity, layer, s.allowPause, - FlagAllowOvershootWhileDeficient, + cFlagAllowOvershootWhileDeficient, ) if availableChannelCapacity < usedChannelCapacity { break alloc_loop @@ -995,14 +1034,14 @@ func (s *StreamAllocator) allocateTrack(track *Track) { } track.ProvisionalAllocateReset() - transition = track.ProvisionalAllocateGetCooperativeTransition(FlagAllowOvershootWhileDeficient) // get transition again to reset above allocation attempt using available headroom + transition = track.ProvisionalAllocateGetCooperativeTransition(cFlagAllowOvershootWhileDeficient) // get transition again to reset above allocation attempt using available headroom } // if there is not enough headroom, try to redistribute starting with tracks that are closest to their desired. bandwidthAcquired := int64(0) var contributingTracks []*Track - minDistanceSorted := s.getMinDistanceSorted(track) + minDistanceSorted := s.getMinDistanceSortedVideoTracks(track) for _, t := range minDistanceSorted { t.ProvisionalAllocatePrepare() } @@ -1052,7 +1091,7 @@ func (s *StreamAllocator) maybeStopProbe() { pci := s.params.Pacer.EndProbeCluster(s.activeProbeClusterId) - for _, t := range s.getTracks() { + for _, t := range s.getVideoTracks() { t.DownTrack().SwapProbeClusterId(pci.Id, ccutils.ProbeClusterIdInvalid) } @@ -1074,11 +1113,11 @@ func (s *StreamAllocator) maybeBoostDeficientTracks() { update := NewStreamStateUpdate() - sortedTracks := s.getMaxDistanceSortedDeficient() + sortedTracks := s.getMaxDistanceSortedDeficientVideoTracks() boost_loop: for { for idx, track := range sortedTracks { - allocation, boosted := track.AllocateNextHigher(availableChannelCapacity, FlagAllowOvershootInCatchup) + allocation, boosted := track.AllocateNextHigher(availableChannelCapacity, cFlagAllowOvershootInCatchup) if !boosted { if idx == len(sortedTracks)-1 { // all tracks tried @@ -1096,7 +1135,7 @@ boost_loop: break // sort again below as the track that was just boosted could still be farthest from its desired } - sortedTracks = s.getMaxDistanceSortedDeficient() + sortedTracks = s.getMaxDistanceSortedDeficientVideoTracks() if len(sortedTracks) == 0 { break // nothing available to boost } @@ -1133,13 +1172,13 @@ func (s *StreamAllocator) allocateAllTracks() { // This pass is to find out if there is any leftover channel capacity after allocating exempt tracks. // Exempt tracks are given optimal allocation (i. e. no bandwidth constraint) so that they do not fail allocation. // - videoTracks := s.getTracks() + videoTracks := s.getVideoTracks() for _, track := range videoTracks { if track.IsManaged() { continue } - allocation := track.AllocateOptimal(FlagAllowOvershootExemptTrackWhileDeficient, false) + allocation := track.AllocateOptimal(cFlagAllowOvershootExemptTrackWhileDeficient, false) updateStreamStateChange(track, allocation, update) // STREAM-ALLOCATOR-TODO: optimistic allocation before bitrate is available will return 0. How to account for that? @@ -1162,7 +1201,7 @@ func (s *StreamAllocator) allocateAllTracks() { updateStreamStateChange(track, allocation, update) } } else { - sorted := s.getSorted() + sorted := s.getSortedVideoTracks() for _, track := range sorted { track.ProvisionalAllocatePrepare() } @@ -1175,7 +1214,7 @@ func (s *StreamAllocator) allocateAllTracks() { } for _, track := range sorted { - _, usedChannelCapacity := track.ProvisionalAllocate(availableChannelCapacity, layer, s.allowPause, FlagAllowOvershootWhileDeficient) + _, usedChannelCapacity := track.ProvisionalAllocate(availableChannelCapacity, layer, s.allowPause, cFlagAllowOvershootWhileDeficient) availableChannelCapacity -= usedChannelCapacity if availableChannelCapacity < 0 { availableChannelCapacity = 0 @@ -1239,7 +1278,7 @@ func (s *StreamAllocator) getAvailableChannelCapacity(allowOverride bool) int64 func (s *StreamAllocator) getExpectedBandwidthUsage() int64 { expected := int64(0) - for _, track := range s.getTracks() { + for _, track := range s.getVideoTracks() { expected += track.BandwidthRequested() } @@ -1248,14 +1287,8 @@ func (s *StreamAllocator) getExpectedBandwidthUsage() int64 { func (s *StreamAllocator) getExpectedBandwidthUsageWithoutTracks(filteredTracks []*Track) int64 { expected := int64(0) - for _, track := range s.getTracks() { - filtered := false - for _, ft := range filteredTracks { - if ft == track { - filtered = true - break - } - } + for _, track := range s.getVideoTracks() { + filtered := slices.Contains(filteredTracks, track) if !filtered { expected += track.BandwidthRequested() } @@ -1275,7 +1308,7 @@ func (s *StreamAllocator) getAvailableHeadroomWithoutTracks(allowOverride bool, func (s *StreamAllocator) getNackDelta() (uint32, uint32) { aggPacketDelta := uint32(0) aggRepeatedNackDelta := uint32(0) - for _, track := range s.getTracks() { + for _, track := range s.getVideoTracks() { packetDelta, nackDelta := track.GetNackDelta() aggPacketDelta += packetDelta aggRepeatedNackDelta += nackDelta @@ -1305,8 +1338,8 @@ func (s *StreamAllocator) maybeProbe() { func (s *StreamAllocator) maybeProbeWithMedia() { // boost deficient track farthest from desired layer - for _, track := range s.getMaxDistanceSortedDeficient() { - allocation, boosted := track.AllocateNextHigher(ChannelCapacityInfinity, FlagAllowOvershootInBoost) + for _, track := range s.getMaxDistanceSortedDeficientVideoTracks() { + allocation, boosted := track.AllocateNextHigher(cChannelCapacityInfinity, cFlagAllowOvershootInBoost) if !boosted { continue } @@ -1322,17 +1355,14 @@ func (s *StreamAllocator) maybeProbeWithMedia() { func (s *StreamAllocator) maybeProbeWithPadding() { // use deficient track farthest from desired layer to find how much to probe - for _, track := range s.getMaxDistanceSortedDeficient() { - transition, available := track.GetNextHigherTransition(FlagAllowOvershootInProbe) + for _, track := range s.getMaxDistanceSortedDeficientVideoTracks() { + transition, available := track.GetNextHigherTransition(cFlagAllowOvershootInProbe) if !available || transition.BandwidthDelta < 0 { continue } // overshoot a bit to account for noise (in measurement/estimate etc) - desiredIncreaseBps := (transition.BandwidthDelta * s.params.Config.ProbeOveragePct) / 100 - if desiredIncreaseBps < s.params.Config.ProbeMinBps { - desiredIncreaseBps = s.params.Config.ProbeMinBps - } + desiredIncreaseBps := max((transition.BandwidthDelta*s.params.Config.ProbeOveragePct)/100, s.params.Config.ProbeMinBps) expectedBandwidthUsage := s.getExpectedBandwidthUsage() pci := s.prober.AddCluster( ccutils.ProbeClusterModeUniform, @@ -1351,68 +1381,65 @@ func (s *StreamAllocator) maybeProbeWithPadding() { } } -func (s *StreamAllocator) getTracks() []*Track { +func (s *StreamAllocator) getVideoTracks() []*Track { s.videoTracksMu.RLock() - tracks := make([]*Track, 0, len(s.videoTracks)) - for _, track := range s.videoTracks { - tracks = append(tracks, track) - } - s.videoTracksMu.RUnlock() + defer s.videoTracksMu.RUnlock() - return tracks + return s.videoTracksShadow } -func (s *StreamAllocator) getSorted() TrackSorter { - s.videoTracksMu.RLock() +func (s *StreamAllocator) getSortedVideoTracks() TrackSorter { var trackSorter TrackSorter - for _, track := range s.videoTracks { + for _, track := range s.getVideoTracks() { if !track.IsManaged() { continue } trackSorter = append(trackSorter, track) } - s.videoTracksMu.RUnlock() sort.Sort(trackSorter) return trackSorter } -func (s *StreamAllocator) getMinDistanceSorted(exclude *Track) MinDistanceSorter { - s.videoTracksMu.RLock() +func (s *StreamAllocator) getMinDistanceSortedVideoTracks(exclude *Track) MinDistanceSorter { var minDistanceSorter MinDistanceSorter - for _, track := range s.videoTracks { + for _, track := range s.getVideoTracks() { if !track.IsManaged() || track == exclude { continue } minDistanceSorter = append(minDistanceSorter, track) } - s.videoTracksMu.RUnlock() sort.Sort(minDistanceSorter) return minDistanceSorter } -func (s *StreamAllocator) getMaxDistanceSortedDeficient() MaxDistanceSorter { - s.videoTracksMu.RLock() +func (s *StreamAllocator) getMaxDistanceSortedDeficientVideoTracks() MaxDistanceSorter { var maxDistanceSorter MaxDistanceSorter - for _, track := range s.videoTracks { + for _, track := range s.getVideoTracks() { if !track.IsManaged() || !track.IsDeficient() { continue } maxDistanceSorter = append(maxDistanceSorter, track) } - s.videoTracksMu.RUnlock() sort.Sort(maxDistanceSorter) return maxDistanceSorter } +func (s *StreamAllocator) shadowVideoTracksLocked() { + s.videoTracksShadow = make([]*Track, 0, len(s.videoTracks)) + for _, t := range s.videoTracks { + s.videoTracksShadow = append(s.videoTracksShadow, t) + } +} + // ------------------------------------------------ func updateStreamStateChange(track *Track, allocation sfu.VideoAllocation, update *StreamStateUpdate) { diff --git a/pkg/sfu/streamallocator/track.go b/pkg/sfu/streamallocator/track.go index 237faf5..a094aa5 100644 --- a/pkg/sfu/streamallocator/track.go +++ b/pkg/sfu/streamallocator/track.go @@ -23,12 +23,12 @@ import ( ) type Track struct { - downTrack *sfu.DownTrack - source livekit.TrackSource - isSimulcast bool - priority uint8 - publisherID livekit.ParticipantID - logger logger.Logger + downTrack *sfu.DownTrack + source livekit.TrackSource + isMultiLayered bool + priority uint8 + publisherID livekit.ParticipantID + logger logger.Logger maxLayer buffer.VideoLayer @@ -43,17 +43,17 @@ type Track struct { func NewTrack( downTrack *sfu.DownTrack, source livekit.TrackSource, - isSimulcast bool, + isMultiLayered bool, publisherID livekit.ParticipantID, logger logger.Logger, ) *Track { t := &Track{ - downTrack: downTrack, - source: source, - isSimulcast: isSimulcast, - publisherID: publisherID, - logger: logger, - streamState: StreamStateInactive, + downTrack: downTrack, + source: source, + isMultiLayered: isMultiLayered, + publisherID: publisherID, + logger: logger, + streamState: StreamStateInactive, } t.SetPriority(0) t.SetMaxLayer(downTrack.MaxLayer()) @@ -87,9 +87,9 @@ func (t *Track) SetPriority(priority uint8) bool { if priority == 0 { switch t.source { case livekit.TrackSource_SCREEN_SHARE: - priority = PriorityDefaultScreenshare + priority = cPriorityDefaultScreenshare default: - priority = PriorityDefaultVideo + priority = cPriorityDefaultVideo } } @@ -110,7 +110,7 @@ func (t *Track) DownTrack() *sfu.DownTrack { } func (t *Track) IsManaged() bool { - return t.source != livekit.TrackSource_SCREEN_SHARE || t.isSimulcast + return t.source != livekit.TrackSource_SCREEN_SHARE || t.isMultiLayered } func (t *Track) ID() livekit.TrackID { diff --git a/pkg/sfu/streamtracker/streamtracker.go b/pkg/sfu/streamtracker/streamtracker.go index b45723a..0a2229e 100644 --- a/pkg/sfu/streamtracker/streamtracker.go +++ b/pkg/sfu/streamtracker/streamtracker.go @@ -69,8 +69,8 @@ type StreamTracker struct { lastNotifiedStatus StreamStatus lastBitrateReport time.Time - bytesForBitrate [4]int64 - bitrate [4]int64 + bytesForBitrate [buffer.DefaultMaxLayerTemporal + 1]int64 + bitrate [buffer.DefaultMaxLayerTemporal + 1]int64 isStopped bool } @@ -158,10 +158,10 @@ func (s *StreamTracker) resetLocked() { s.setStatusLocked(StreamStatusStopped) - for i := 0; i < len(s.bytesForBitrate); i++ { + for i := range len(s.bytesForBitrate) { s.bytesForBitrate[i] = 0 } - for i := 0; i < len(s.bitrate); i++ { + for i := range len(s.bitrate) { s.bitrate[i] = 0 } @@ -207,8 +207,17 @@ func (s *StreamTracker) Observe( go s.worker(s.generation.Load()) } - if temporalLayer >= 0 { + if temporalLayer >= 0 && int(temporalLayer) < len(s.bytesForBitrate) { s.bytesForBitrate[temporalLayer] += int64(pktSize) + } else if int(temporalLayer) >= len(s.bytesForBitrate) { + s.params.Logger.Warnw( + "invalid temporal layer", nil, + "temporalLayer", temporalLayer, + "pktSize", pktSize, + "payloadSize", payloadSize, + "hasMarker", hasMarker, + "ts", ts, + ) } s.lock.Unlock() @@ -235,7 +244,7 @@ func (s *StreamTracker) BitrateTemporalCumulative() []int64 { } // clear higher layers - for i := 0; i < len(brs); i++ { + for i := range brs { if brs[i] == 0 { for j := i + 1; j < len(brs); j++ { brs[j] = 0 @@ -291,7 +300,7 @@ func (s *StreamTracker) bitrateReport() { s.lastBitrateReport = now bitrateAvailabilityChanged := false - for i := 0; i < len(s.bytesForBitrate); i++ { + for i := range len(s.bytesForBitrate) { bitrate := int64(float64(s.bytesForBitrate[i]*8) / diff.Seconds()) if (s.bitrate[i] == 0 && bitrate > 0) || (s.bitrate[i] > 0 && bitrate == 0) { bitrateAvailabilityChanged = true diff --git a/pkg/sfu/streamtracker/streamtracker_dd.go b/pkg/sfu/streamtracker/streamtracker_dd.go index 5feb33d..a87f2e7 100644 --- a/pkg/sfu/streamtracker/streamtracker_dd.go +++ b/pkg/sfu/streamtracker/streamtracker_dd.go @@ -107,13 +107,13 @@ func (s *StreamTrackerDependencyDescriptor) resetLocked() { // bump generation to trigger exit of current worker s.generation.Inc() - for i := 0; i < len(s.bytesForBitrate); i++ { - for j := 0; j < len(s.bytesForBitrate[i]); j++ { + for i := range len(s.bytesForBitrate) { + for j := range len(s.bytesForBitrate[i]) { s.bytesForBitrate[i][j] = 0 } } - for i := 0; i < len(s.bitrate); i++ { - for j := 0; j < len(s.bitrate[i]); j++ { + for i := range len(s.bitrate) { + for j := range len(s.bitrate[i]) { s.bitrate[i][j] = 0 } } @@ -248,11 +248,11 @@ func (s *StreamTrackerDependencyDescriptor) bitrateReport() { s.lastBitrateReport = now var availableChangedFns []func() - for spatial := 0; spatial < len(s.bytesForBitrate); spatial++ { + for spatial := range len(s.bytesForBitrate) { bytesForBitrate := s.bytesForBitrate[spatial][:] bitrateAvailabilityChanged := false bitrates := s.bitrate[spatial][:] - for i := 0; i < len(bytesForBitrate); i++ { + for i := range bytesForBitrate { bitrate := int64(float64(bytesForBitrate[i]*8) / diff.Seconds()) if (bitrates[i] == 0 && bitrate > 0) || (bitrates[i] > 0 && bitrate == 0) { bitrateAvailabilityChanged = true diff --git a/pkg/sfu/streamtrackermanager.go b/pkg/sfu/streamtrackermanager.go index a0e35d6..6eb28e5 100644 --- a/pkg/sfu/streamtrackermanager.go +++ b/pkg/sfu/streamtrackermanager.go @@ -15,13 +15,13 @@ package sfu import ( - "sort" + "slices" "sync" "time" "github.com/frostbyte73/core" - "go.uber.org/atomic" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" "github.com/livekit/protocol/utils" @@ -110,14 +110,15 @@ var ( // --------------------------------------------------- type StreamTrackerManager struct { - logger logger.Logger - trackInfo atomic.Pointer[livekit.TrackInfo] - isSVC bool - clockRate uint32 + logger logger.Logger + mimeType mime.MimeType + videoLayerMode livekit.VideoLayer_Mode + clockRate uint32 trackerConfig StreamTrackerConfig lock sync.RWMutex + trackInfo *livekit.TrackInfo maxPublishedLayer int32 maxTemporalLayerSeen int32 @@ -126,7 +127,6 @@ type StreamTrackerManager struct { availableLayers []int32 maxExpectedLayer int32 - paused bool closed core.Fuse @@ -136,18 +136,19 @@ type StreamTrackerManager struct { func NewStreamTrackerManager( logger logger.Logger, trackInfo *livekit.TrackInfo, - isSVC bool, + mimeType mime.MimeType, clockRate uint32, config StreamTrackerManagerConfig, ) *StreamTrackerManager { s := &StreamTrackerManager{ logger: logger, - isSVC: isSVC, + mimeType: mimeType, + videoLayerMode: buffer.GetVideoLayerModeForMimeType(mimeType, trackInfo), + trackInfo: utils.CloneProto(trackInfo), maxPublishedLayer: buffer.InvalidLayerSpatial, maxTemporalLayerSeen: buffer.InvalidLayerTemporal, clockRate: clockRate, } - s.trackInfo.Store(utils.CloneProto(trackInfo)) switch trackInfo.Source { case livekit.TrackSource_SCREEN_SHARE: @@ -158,7 +159,8 @@ func NewStreamTrackerManager( s.trackerConfig = config.Video } - s.maxExpectedLayerFromTrackInfo() + s.maxExpectedLayer = buffer.InvalidLayerSpatial + s.maxExpectedLayerFromTrackInfo(true) if trackInfo.Type == livekit.TrackType_VIDEO { go s.bitrateReporter() @@ -290,7 +292,7 @@ func (s *StreamTrackerManager) AddTracker(layer int32) streamtracker.StreamTrack }) s.lock.Lock() - paused := s.paused + paused := s.trackInfo.GetMuted() s.trackers[layer] = tracker notify := false @@ -329,8 +331,10 @@ func (s *StreamTrackerManager) RemoveAllTrackers() { s.trackers[layer] = nil } s.availableLayers = make([]int32, 0) - s.maxExpectedLayerFromTrackInfoLocked() - s.paused = false + + s.maxExpectedLayer = buffer.InvalidLayerSpatial + s.maxExpectedLayerFromTrackInfoLocked(true) + ddTracker := s.ddTracker s.ddTracker = nil s.lock.Unlock() @@ -356,9 +360,8 @@ func (s *StreamTrackerManager) GetTracker(layer int32) streamtracker.StreamTrack return s.trackers[layer] } -func (s *StreamTrackerManager) SetPaused(paused bool) { +func (s *StreamTrackerManager) setPaused(paused bool) { s.lock.Lock() - s.paused = paused trackers := s.trackers s.lock.Unlock() @@ -369,16 +372,15 @@ func (s *StreamTrackerManager) SetPaused(paused bool) { } } -func (s *StreamTrackerManager) IsPaused() bool { - s.lock.RLock() - defer s.lock.RUnlock() - - return s.paused -} - func (s *StreamTrackerManager) UpdateTrackInfo(ti *livekit.TrackInfo) { - s.trackInfo.Store(utils.CloneProto(ti)) - s.maxExpectedLayerFromTrackInfo() + s.lock.Lock() + s.trackInfo = utils.CloneProto(ti) + s.maxExpectedLayerFromTrackInfoLocked(false) + + paused := s.trackInfo.GetMuted() + s.lock.Unlock() + + s.setPaused(paused) } func (s *StreamTrackerManager) SetMaxExpectedSpatialLayer(layer int32) int32 { @@ -417,7 +419,7 @@ func (s *StreamTrackerManager) DistanceToDesired() float64 { s.lock.RLock() defer s.lock.RUnlock() - if s.paused || s.maxExpectedLayer < 0 || s.maxTemporalLayerSeen < 0 { + if s.trackInfo.GetMuted() || s.maxExpectedLayer < 0 || s.maxTemporalLayerSeen < 0 { return 0 } @@ -480,18 +482,18 @@ func (s *StreamTrackerManager) getLayeredBitrateLocked() ([]int32, Bitrates) { for i, tracker := range s.trackers { if tracker != nil { tls := make([]int64, buffer.DefaultMaxLayerTemporal+1) - if s.hasSpatialLayerLocked(int32(i)) { + if slices.Contains(s.availableLayers, int32(i)) { tls = tracker.BitrateTemporalCumulative() } - for j := 0; j < len(br[i]); j++ { + for j := range len(br[i]) { br[i][j] = tls[j] } } } // accumulate bitrates for SVC streams without dependency descriptor - if s.isSVC && s.ddTracker == nil { + if s.videoLayerMode == livekit.VideoLayer_MULTIPLE_SPATIAL_LAYERS_PER_STREAM && s.ddTracker == nil { for i := len(br) - 1; i >= 1; i-- { for j := len(br[i]) - 1; j >= 0; j-- { if br[i][j] != 0 { @@ -509,32 +511,15 @@ func (s *StreamTrackerManager) getLayeredBitrateLocked() ([]int32, Bitrates) { return availableLayers, br } -func (s *StreamTrackerManager) hasSpatialLayerLocked(layer int32) bool { - for _, l := range s.availableLayers { - if l == layer { - return true - } - } - - return false -} - func (s *StreamTrackerManager) addAvailableLayer(layer int32) { s.lock.Lock() - hasLayer := false - for _, l := range s.availableLayers { - if l == layer { - hasLayer = true - break - } - } - if hasLayer { + if slices.Contains(s.availableLayers, layer) { s.lock.Unlock() return } s.availableLayers = append(s.availableLayers, layer) - sort.Slice(s.availableLayers, func(i, j int) bool { return s.availableLayers[i] < s.availableLayers[j] }) + slices.Sort(s.availableLayers) // check if new layer is the max layer isMaxLayerChange := s.availableLayers[len(s.availableLayers)-1] == layer @@ -568,7 +553,7 @@ func (s *StreamTrackerManager) removeAvailableLayer(layer int32) { newLayers = append(newLayers, l) } } - sort.Slice(newLayers, func(i, j int) bool { return newLayers[i] < newLayers[j] }) + slices.Sort(newLayers) s.availableLayers = newLayers s.logger.Debugw( @@ -594,24 +579,29 @@ func (s *StreamTrackerManager) removeAvailableLayer(layer int32) { } } -func (s *StreamTrackerManager) maxExpectedLayerFromTrackInfo() { +func (s *StreamTrackerManager) maxExpectedLayerFromTrackInfo(force bool) { s.lock.Lock() defer s.lock.Unlock() - s.maxExpectedLayerFromTrackInfoLocked() + s.maxExpectedLayerFromTrackInfoLocked(force) } -func (s *StreamTrackerManager) maxExpectedLayerFromTrackInfoLocked() { - s.maxExpectedLayer = buffer.InvalidLayerSpatial - ti := s.trackInfo.Load() - if ti != nil { - for _, layer := range ti.Layers { - spatialLayer := buffer.VideoQualityToSpatialLayer(layer.Quality, ti) - if spatialLayer > s.maxExpectedLayer { - s.maxExpectedLayer = spatialLayer +func (s *StreamTrackerManager) maxExpectedLayerFromTrackInfoLocked(force bool) { + maxExpectedLayer := buffer.InvalidLayerSpatial + if s.trackInfo != nil { + for _, layer := range buffer.GetVideoLayersForMimeType(s.mimeType, s.trackInfo) { + if layer.SpatialLayer > maxExpectedLayer { + maxExpectedLayer = layer.SpatialLayer } } } + + // when max expected is higher than current max, trackers are reset + // which allows a layer start to be detected on initial packets from that higher layer, + // so update max only on track info max being lower than current max + if force || maxExpectedLayer < s.maxExpectedLayer { + s.maxExpectedLayer = maxExpectedLayer + } } func (s *StreamTrackerManager) GetMaxTemporalLayerSeen() int32 { diff --git a/pkg/sfu/testutils/data.go b/pkg/sfu/testutils/data.go index 7f7c968..34b1fca 100644 --- a/pkg/sfu/testutils/data.go +++ b/pkg/sfu/testutils/data.go @@ -21,12 +21,13 @@ import ( "github.com/pion/webrtc/v4" "github.com/livekit/livekit-server/pkg/sfu/buffer" + "github.com/livekit/mediatransportutil/pkg/codec" ) // ----------------------------------------------------------- type TestExtPacketParams struct { - SetMarker bool + Marker bool IsKeyFrame bool PayloadType uint8 SequenceNumber uint16 @@ -38,6 +39,7 @@ type TestExtPacketParams struct { PaddingSize byte ArrivalTime time.Time VideoLayer buffer.VideoLayer + IsOutOfOrder bool } // ----------------------------------------------------------- @@ -47,7 +49,7 @@ func GetTestExtPacket(params *TestExtPacketParams) (*buffer.ExtPacket, error) { Header: rtp.Header{ Version: 2, Padding: params.PaddingSize != 0, - Marker: params.SetMarker, + Marker: params.Marker, PayloadType: params.PayloadType, SequenceNumber: params.SequenceNumber, Timestamp: params.Timestamp, @@ -68,8 +70,9 @@ func GetTestExtPacket(params *TestExtPacketParams) (*buffer.ExtPacket, error) { ExtTimestamp: uint64(params.TSCycles<<32) + uint64(params.Timestamp), Arrival: params.ArrivalTime.UnixNano(), Packet: &packet, - KeyFrame: params.IsKeyFrame, + IsKeyFrame: params.IsKeyFrame, RawPacket: raw, + IsOutOfOrder: params.IsOutOfOrder, } return ep, nil @@ -77,13 +80,13 @@ func GetTestExtPacket(params *TestExtPacketParams) (*buffer.ExtPacket, error) { // -------------------------------------- -func GetTestExtPacketVP8(params *TestExtPacketParams, vp8 *buffer.VP8) (*buffer.ExtPacket, error) { +func GetTestExtPacketVP8(params *TestExtPacketParams, vp8 *codec.VP8) (*buffer.ExtPacket, error) { ep, err := GetTestExtPacket(params) if err != nil { return nil, err } - ep.KeyFrame = vp8.IsKeyFrame + ep.IsKeyFrame = vp8.IsKeyFrame ep.Payload = *vp8 if ep.DependencyDescriptor == nil { ep.Temporal = int32(vp8.TID) diff --git a/pkg/sfu/track_remote.go b/pkg/sfu/track_remote.go index d868d0f..70b08a9 100644 --- a/pkg/sfu/track_remote.go +++ b/pkg/sfu/track_remote.go @@ -1,3 +1,17 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package sfu import "github.com/pion/webrtc/v4" @@ -7,6 +21,7 @@ type TrackRemote interface { RID() string Msid() string SSRC() webrtc.SSRC + RtxSSRC() webrtc.SSRC StreamID() string Kind() webrtc.RTPCodecType Codec() webrtc.RTPCodecParameters diff --git a/pkg/sfu/downtrackspreader.go b/pkg/sfu/utils/downtrackspreader.go similarity index 66% rename from pkg/sfu/downtrackspreader.go rename to pkg/sfu/utils/downtrackspreader.go index 98057a0..996889c 100644 --- a/pkg/sfu/downtrackspreader.go +++ b/pkg/sfu/utils/downtrackspreader.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package sfu +package utils import ( "sync" @@ -22,55 +22,59 @@ import ( "github.com/livekit/protocol/utils" ) +type sender interface { + SubscriberID() livekit.ParticipantID +} + type DownTrackSpreaderParams struct { Threshold int Logger logger.Logger } -type DownTrackSpreader struct { +type DownTrackSpreader[T sender] struct { params DownTrackSpreaderParams downTrackMu sync.RWMutex - downTracks map[livekit.ParticipantID]TrackSender - downTracksShadow []TrackSender + downTracks map[livekit.ParticipantID]T + downTracksShadow []T } -func NewDownTrackSpreader(params DownTrackSpreaderParams) *DownTrackSpreader { - d := &DownTrackSpreader{ +func NewDownTrackSpreader[T sender](params DownTrackSpreaderParams) *DownTrackSpreader[T] { + d := &DownTrackSpreader[T]{ params: params, - downTracks: make(map[livekit.ParticipantID]TrackSender), + downTracks: make(map[livekit.ParticipantID]T), } return d } -func (d *DownTrackSpreader) GetDownTracks() []TrackSender { +func (d *DownTrackSpreader[T]) GetDownTracks() []T { d.downTrackMu.RLock() defer d.downTrackMu.RUnlock() return d.downTracksShadow } -func (d *DownTrackSpreader) ResetAndGetDownTracks() []TrackSender { +func (d *DownTrackSpreader[T]) ResetAndGetDownTracks() []T { d.downTrackMu.Lock() defer d.downTrackMu.Unlock() downTracks := d.downTracksShadow - d.downTracks = make(map[livekit.ParticipantID]TrackSender) + d.downTracks = make(map[livekit.ParticipantID]T) d.downTracksShadow = nil return downTracks } -func (d *DownTrackSpreader) Store(ts TrackSender) { +func (d *DownTrackSpreader[T]) Store(sender T) { d.downTrackMu.Lock() defer d.downTrackMu.Unlock() - d.downTracks[ts.SubscriberID()] = ts + d.downTracks[sender.SubscriberID()] = sender d.shadowDownTracks() } -func (d *DownTrackSpreader) Free(subscriberID livekit.ParticipantID) { +func (d *DownTrackSpreader[T]) Free(subscriberID livekit.ParticipantID) { d.downTrackMu.Lock() defer d.downTrackMu.Unlock() @@ -78,7 +82,7 @@ func (d *DownTrackSpreader) Free(subscriberID livekit.ParticipantID) { d.shadowDownTracks() } -func (d *DownTrackSpreader) HasDownTrack(subscriberID livekit.ParticipantID) bool { +func (d *DownTrackSpreader[T]) HasDownTrack(subscriberID livekit.ParticipantID) bool { d.downTrackMu.RLock() defer d.downTrackMu.RUnlock() @@ -86,10 +90,10 @@ func (d *DownTrackSpreader) HasDownTrack(subscriberID livekit.ParticipantID) boo return ok } -func (d *DownTrackSpreader) Broadcast(writer func(TrackSender)) int { +func (d *DownTrackSpreader[T]) Broadcast(writer func(T)) { downTracks := d.GetDownTracks() if len(downTracks) == 0 { - return 0 + return } threshold := uint64(d.params.Threshold) @@ -101,17 +105,16 @@ func (d *DownTrackSpreader) Broadcast(writer func(TrackSender)) int { // WriteRTP takes about 50µs on average, so we write to 2 down tracks per loop. step := uint64(2) utils.ParallelExec(downTracks, threshold, step, writer) - return len(downTracks) } -func (d *DownTrackSpreader) DownTrackCount() int { +func (d *DownTrackSpreader[T]) DownTrackCount() int { d.downTrackMu.RLock() defer d.downTrackMu.RUnlock() return len(d.downTracksShadow) } -func (d *DownTrackSpreader) shadowDownTracks() { - d.downTracksShadow = make([]TrackSender, 0, len(d.downTracks)) +func (d *DownTrackSpreader[T]) shadowDownTracks() { + d.downTracksShadow = make([]T, 0, len(d.downTracks)) for _, dt := range d.downTracks { d.downTracksShadow = append(d.downTracksShadow, dt) } diff --git a/pkg/sfu/utils/helpers.go b/pkg/sfu/utils/helpers.go index d237d51..7ce7eaa 100644 --- a/pkg/sfu/utils/helpers.go +++ b/pkg/sfu/utils/helpers.go @@ -18,10 +18,12 @@ import ( "errors" "fmt" - "github.com/livekit/livekit-server/pkg/sfu/mime" "github.com/pion/interceptor" "github.com/pion/rtp" "github.com/pion/webrtc/v4" + + "github.com/livekit/protocol/codecs/mime" + "github.com/livekit/protocol/livekit" ) // Do a fuzzy find for a codec in the list of codecs @@ -89,3 +91,7 @@ func ValidateRTPPacket(pkt *rtp.Packet, expectedPayloadType uint8, expectedSSRC return nil } + +func IsSimulcastMode(m livekit.VideoLayer_Mode) bool { + return m == livekit.VideoLayer_ONE_SPATIAL_LAYER_PER_STREAM || m == livekit.VideoLayer_ONE_SPATIAL_LAYER_PER_STREAM_INCOMPLETE_RTCP_SR +} diff --git a/pkg/sfu/utils/owd_estimator.go b/pkg/sfu/utils/owd_estimator.go deleted file mode 100644 index b25e260..0000000 --- a/pkg/sfu/utils/owd_estimator.go +++ /dev/null @@ -1,179 +0,0 @@ -// Copyright 2023 LiveKit, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package utils - -import ( - "time" - - "go.uber.org/zap/zapcore" -) - -type OWDEstimatorParams struct { - PropagationDelayFallFactor float64 - PropagationDelayRiseFactor float64 - - PropagationDelaySpikeAdaptationFactor float64 - - PropagationDelayDeltaThresholdMin time.Duration - PropagationDelayDeltaThresholdMaxFactor int64 - PropagationDelayDeltaHighResetNumReports int - PropagationDelayDeltaHighResetWait time.Duration - PropagationDelayDeltaLongTermAdaptationThreshold time.Duration -} - -var OWDEstimatorParamsDefault = OWDEstimatorParams{ - // OWD (One-Way-Delay) Estimator is used to estimate propagation delay between sender and receicer. - // As they operate on different clock domains, it is not possible to get exact propagation delay easily. - // So, this module is an estimator using a simple approach explained below. It should not be used for - // things that require high accuracy. - // - // One example is RTCP Sender Reports getting re-based to SFU time base so that all subscriber side - // can have the same time base (i. e. SFU time base). To convert publisher side - // RTCP Sender Reports to SFU timebase, a propagation delay is maintained. - // propagation_delay = time_of_report_reception - ntp_timestamp_in_report - // - // Propagation delay is adapted continuously. If it falls, adapt quickly to the - // lower value as that could be the real propagation delay. If it rises, adapt slowly - // as it might be a temporary change or slow drift. See below for handling of high deltas - // which could be a result of a path change. - PropagationDelayFallFactor: 0.9, - PropagationDelayRiseFactor: 0.1, - - PropagationDelaySpikeAdaptationFactor: 0.5, - - // To account for path changes mid-stream, if the delta of the propagation delay is consistently higher, reset. - // Reset at whichever of the below happens later. - // 1. 10 seconds of persistent high delta. - // 2. at least 2 consecutive reports with high delta. - // - // A long term estimate of delta of propagation delay is maintained and delta propagation delay exceeding - // a factor of the long term estimate is considered a sharp increase. That will trigger the start of the - // path change condition and if it persists, propagation delay will be reset. - PropagationDelayDeltaThresholdMin: 10 * time.Millisecond, - PropagationDelayDeltaThresholdMaxFactor: 2, - PropagationDelayDeltaHighResetNumReports: 2, - PropagationDelayDeltaHighResetWait: 10 * time.Second, - PropagationDelayDeltaLongTermAdaptationThreshold: 50 * time.Millisecond, -} - -type OWDEstimator struct { - params OWDEstimatorParams - - initialized bool - lastSenderClockTimeNs int64 - lastPropagationDelayNs int64 - lastDeltaPropagationDelayNs int64 - estimatedPropagationDelayNs int64 - longTermDeltaPropagationDelayNs int64 - propagationDelayDeltaHighCount int - propagationDelayDeltaHighStartTime time.Time - propagationDelaySpikeNs int64 -} - -func NewOWDEstimator(params OWDEstimatorParams) *OWDEstimator { - return &OWDEstimator{ - params: params, - } -} - -func (o *OWDEstimator) MarshalLogObject(e zapcore.ObjectEncoder) error { - if o != nil { - e.AddTime("lastSenderClockTimeNs", time.Unix(0, o.lastSenderClockTimeNs)) - e.AddDuration("lastPropagationDelayNs", time.Duration(o.lastPropagationDelayNs)) - e.AddDuration("lastDeltaPropagationDelayNs", time.Duration(o.lastDeltaPropagationDelayNs)) - e.AddDuration("estimatedPropagationDelayNs", time.Duration(o.estimatedPropagationDelayNs)) - e.AddDuration("longTermDeltaPropagationDelayNs", time.Duration(o.longTermDeltaPropagationDelayNs)) - e.AddInt("propagationDelayDeltaHighCount", o.propagationDelayDeltaHighCount) - e.AddTime("propagationDelayDeltaHighStartTime", o.propagationDelayDeltaHighStartTime) - e.AddDuration("propagationDelaySpikeNs", time.Duration(o.propagationDelaySpikeNs)) - } - return nil -} - -func (o *OWDEstimator) Update(senderClockTimeNs int64, receiverClockTimeNs int64) (int64, bool) { - resetDelta := func() { - o.propagationDelayDeltaHighCount = 0 - o.propagationDelayDeltaHighStartTime = time.Time{} - o.propagationDelaySpikeNs = 0 - } - - initPropagationDelay := func(pd int64) { - o.estimatedPropagationDelayNs = pd - o.longTermDeltaPropagationDelayNs = 0 - resetDelta() - } - - o.lastPropagationDelayNs = receiverClockTimeNs - senderClockTimeNs - if !o.initialized { - o.initialized = true - o.lastSenderClockTimeNs = senderClockTimeNs - initPropagationDelay(o.lastPropagationDelayNs) - return o.estimatedPropagationDelayNs, true - } - - stepChange := false - o.lastDeltaPropagationDelayNs = o.lastPropagationDelayNs - o.estimatedPropagationDelayNs - // check for path changes, i. e. a step jump increase in propagation delay observed over time - if o.lastDeltaPropagationDelayNs > o.params.PropagationDelayDeltaThresholdMin.Nanoseconds() { // ignore small changes for path change consideration - if o.longTermDeltaPropagationDelayNs != 0 && - o.lastDeltaPropagationDelayNs > o.longTermDeltaPropagationDelayNs*o.params.PropagationDelayDeltaThresholdMaxFactor { - o.propagationDelayDeltaHighCount++ - if o.propagationDelayDeltaHighStartTime.IsZero() { - o.propagationDelayDeltaHighStartTime = time.Now() - } - if o.propagationDelaySpikeNs == 0 { - o.propagationDelaySpikeNs = o.lastPropagationDelayNs - } else { - o.propagationDelaySpikeNs += int64(o.params.PropagationDelaySpikeAdaptationFactor * float64(o.lastPropagationDelayNs-o.propagationDelaySpikeNs)) - } - - if o.propagationDelayDeltaHighCount >= o.params.PropagationDelayDeltaHighResetNumReports && time.Since(o.propagationDelayDeltaHighStartTime) >= o.params.PropagationDelayDeltaHighResetWait { - stepChange = true - initPropagationDelay(o.propagationDelaySpikeNs) - } - } else { - resetDelta() - } - } else { - resetDelta() - - factor := o.params.PropagationDelayFallFactor - if o.lastPropagationDelayNs > o.estimatedPropagationDelayNs { - factor = o.params.PropagationDelayRiseFactor - } - o.estimatedPropagationDelayNs += int64(factor * float64(o.lastPropagationDelayNs-o.estimatedPropagationDelayNs)) - } - - if o.lastDeltaPropagationDelayNs < o.params.PropagationDelayDeltaLongTermAdaptationThreshold.Nanoseconds() { - if o.longTermDeltaPropagationDelayNs == 0 { - o.longTermDeltaPropagationDelayNs = o.lastDeltaPropagationDelayNs - } else { - // do not adapt to large +ve spikes, can happen when channel is congested and reports are delivered very late - // if the spike is in fact a path change, it will persist and handled by path change detection above - sinceLast := senderClockTimeNs - o.lastSenderClockTimeNs - adaptationFactor := min(1.0, float64(sinceLast)/float64(o.params.PropagationDelayDeltaHighResetWait)) - o.longTermDeltaPropagationDelayNs += int64(adaptationFactor * float64(o.lastDeltaPropagationDelayNs-o.longTermDeltaPropagationDelayNs)) - } - } - if o.longTermDeltaPropagationDelayNs < 0 { - o.longTermDeltaPropagationDelayNs = 0 - } - o.lastSenderClockTimeNs = senderClockTimeNs - return o.estimatedPropagationDelayNs, stepChange -} - -func (o *OWDEstimator) EstimatedPropagationDelay() int64 { - return o.estimatedPropagationDelayNs -} diff --git a/pkg/sfu/utils/rangemap.go b/pkg/sfu/utils/rangemap.go index 6ca2332..8fb36d5 100644 --- a/pkg/sfu/utils/rangemap.go +++ b/pkg/sfu/utils/rangemap.go @@ -80,10 +80,7 @@ func (r *RangeMap[RT, VT]) MarshalLogObject(e zapcore.ObjectEncoder) error { e.AddInt("numRanges", len(r.ranges)) // just the last 10 ranges max - startIdx := len(r.ranges) - 10 - if startIdx < 0 { - startIdx = 0 - } + startIdx := max(len(r.ranges)-10, 0) for i := startIdx; i < len(r.ranges); i++ { e.AddObject(fmt.Sprintf("range[%d]", i), r.ranges[i]) } diff --git a/pkg/sfu/utils/rangemap_test.go b/pkg/sfu/utils/rangemap_test.go index 657481f..767c05c 100644 --- a/pkg/sfu/utils/rangemap_test.go +++ b/pkg/sfu/utils/rangemap_test.go @@ -66,7 +66,7 @@ func TestRangeMapUint32(t *testing.T) { require.Equal(t, uint32(1), value) // excluded range should return error - value, err = r.GetValue(10) + _, err = r.GetValue(10) require.ErrorIs(t, err, errKeyExcluded) // out-of-order exclusion should return error @@ -98,11 +98,11 @@ func TestRangeMapUint32(t *testing.T) { require.Equal(t, expectedRanges, r.ranges) // excluded range should return error, now is excluded because exclusion range could be extended - value, err = r.GetValue(11) + _, err = r.GetValue(11) require.ErrorIs(t, err, errKeyExcluded) // getting value in old range should return 0 - value, err = r.GetValue(6) + _, err = r.GetValue(6) require.NoError(t, err) // newer should return 2 @@ -129,7 +129,7 @@ func TestRangeMapUint32(t *testing.T) { require.Equal(t, expectedRanges, r.ranges) // excluded range should return error, now is excluded because exclusion range could be extended - value, err = r.GetValue(15) + _, err = r.GetValue(15) require.ErrorIs(t, err, errKeyExcluded) // newer should return 12 @@ -190,15 +190,15 @@ func TestRangeMapUint32(t *testing.T) { require.Equal(t, expectedRanges, r.ranges) // excluded range should return error - value, err = r.GetValue(50) + _, err = r.GetValue(50) require.ErrorIs(t, err, errKeyExcluded) - value, err = r.GetValue(28) + _, err = r.GetValue(28) require.ErrorIs(t, err, errKeyExcluded) - value, err = r.GetValue(17) + _, err = r.GetValue(17) require.ErrorIs(t, err, errKeyTooOld) // previously valid, but aged out key should return error - value, err = r.GetValue(5) + _, err = r.GetValue(5) require.ErrorIs(t, err, errKeyTooOld) // valid range access should return values @@ -279,7 +279,7 @@ func TestRangeMapUint32(t *testing.T) { require.Equal(t, expectedRanges, r.ranges) // before first range access - value, err = r.GetValue(5) + _, err = r.GetValue(5) require.ErrorIs(t, err, errKeyTooOld) // first range access @@ -319,7 +319,7 @@ func TestRangeMapUint32(t *testing.T) { require.Equal(t, expectedRanges, r.ranges) // aged out range access - value, err = r.GetValue(25) + _, err = r.GetValue(25) require.ErrorIs(t, err, errKeyTooOld) // access closed range before decrementing value diff --git a/pkg/sfu/utils/wraparound.go b/pkg/sfu/utils/wraparound.go deleted file mode 100644 index c75db34..0000000 --- a/pkg/sfu/utils/wraparound.go +++ /dev/null @@ -1,224 +0,0 @@ -// Copyright 2023 LiveKit, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package utils - -import ( - "unsafe" - - "go.uber.org/zap/zapcore" -) - -type number interface { - uint16 | uint32 -} - -type extendedNumber interface { - uint32 | uint64 -} - -type WrapAroundParams struct { - IsRestartAllowed bool -} - -type WrapAround[T number, ET extendedNumber] struct { - params WrapAroundParams - fullRange ET - - initialized bool - start T - highest T - cycles ET - extendedHighest ET -} - -func NewWrapAround[T number, ET extendedNumber](params WrapAroundParams) *WrapAround[T, ET] { - var t T - return &WrapAround[T, ET]{ - params: params, - fullRange: 1 << (unsafe.Sizeof(t) * 8), - } -} - -func (w *WrapAround[T, ET]) Seed(from *WrapAround[T, ET]) { - w.initialized = from.initialized - w.start = from.start - w.highest = from.highest - w.cycles = from.cycles - w.updateExtendedHighest() -} - -type WrapAroundUpdateResult[ET extendedNumber] struct { - IsUnhandled bool // when set, other fields are invalid - IsRestart bool - PreExtendedStart ET // valid only if IsRestart = true - PreExtendedHighest ET - ExtendedVal ET -} - -func (w *WrapAroundUpdateResult[ET]) MarshalLogObject(e zapcore.ObjectEncoder) error { - if w == nil { - return nil - } - - e.AddBool("IsUnhandled", w.IsUnhandled) - e.AddBool("IsRestart", w.IsRestart) - e.AddUint64("PreExtendedStart", uint64(w.PreExtendedStart)) - e.AddUint64("PreExtendedHighest", uint64(w.PreExtendedHighest)) - e.AddUint64("ExtendedVal", uint64(w.ExtendedVal)) - return nil -} - -func (w *WrapAround[T, ET]) Update(val T) (result WrapAroundUpdateResult[ET]) { - if !w.initialized { - result.PreExtendedHighest = ET(val) - 1 - result.ExtendedVal = ET(val) - - w.start = val - w.highest = val - w.updateExtendedHighest() - w.initialized = true - return - } - - gap := val - w.highest - if gap > T(w.fullRange>>1) { - // out-of-order - return w.maybeAdjustStart(val) - } - - // in-order - result.PreExtendedHighest = w.extendedHighest - - if val < w.highest { - w.cycles += w.fullRange - } - w.highest = val - - w.updateExtendedHighest() - result.ExtendedVal = w.extendedHighest - return -} - -func (w *WrapAround[T, ET]) UndoUpdate(result WrapAroundUpdateResult[ET]) { - if !w.initialized || result.PreExtendedHighest >= result.ExtendedVal { - return - } - - w.ResetHighest(result.PreExtendedHighest) -} - -func (w *WrapAround[T, ET]) Rollover(val T, numCycles int) (result WrapAroundUpdateResult[ET]) { - if numCycles < 0 || !w.initialized { - return w.Update(val) - } - - result.PreExtendedHighest = w.extendedHighest - - w.cycles += ET(numCycles) * w.fullRange - w.highest = val - - w.updateExtendedHighest() - result.ExtendedVal = w.extendedHighest - return -} - -func (w *WrapAround[T, ET]) RollbackRestart(ev ET) { - if w.isWrapBack(w.start, T(ev)) { - w.cycles -= w.fullRange - w.updateExtendedHighest() - } - w.start = T(ev) -} - -func (w *WrapAround[T, ET]) ResetHighest(ev ET) { - w.highest = T(ev) - w.cycles = ev & ^(w.fullRange - 1) - w.updateExtendedHighest() -} - -func (w *WrapAround[T, ET]) GetStart() T { - return w.start -} - -func (w *WrapAround[T, ET]) GetExtendedStart() ET { - return ET(w.start) -} - -func (w *WrapAround[T, ET]) GetHighest() T { - return w.highest -} - -func (w *WrapAround[T, ET]) GetExtendedHighest() ET { - return w.extendedHighest -} - -func (w *WrapAround[T, ET]) updateExtendedHighest() { - w.extendedHighest = getExtended(w.cycles, w.highest) -} - -func (w *WrapAround[T, ET]) maybeAdjustStart(val T) (result WrapAroundUpdateResult[ET]) { - // re-adjust start if necessary. The conditions are - // 1. Not seen more than half the range yet - // 1. wrap back compared to start and not completed a half cycle, sequences like (10, 65530) in uint16 space - // 2. no wrap around, but out-of-order compared to start and not completed a half cycle , sequences like (10, 9), (65530, 65528) in uint16 space - cycles := w.cycles - totalNum := w.GetExtendedHighest() - w.GetExtendedStart() + 1 - if totalNum > (w.fullRange >> 1) { - if w.isWrapBack(val, w.highest) { - cycles -= w.fullRange - } - result.PreExtendedHighest = w.extendedHighest - result.ExtendedVal = getExtended(cycles, val) - return - } - - if val-w.start > T(w.fullRange>>1) { - if w.params.IsRestartAllowed { - // out-of-order with existing start => a new start - result.IsRestart = true - if val > w.start { - result.PreExtendedStart = w.fullRange + ET(w.start) - } else { - result.PreExtendedStart = ET(w.start) - } - - if w.isWrapBack(val, w.highest) { - w.cycles = w.fullRange - w.updateExtendedHighest() - cycles = 0 - } - w.start = val - } else { - result.IsUnhandled = true - } - } else { - if w.isWrapBack(val, w.highest) { - cycles -= w.fullRange - } - } - result.PreExtendedHighest = w.extendedHighest - result.ExtendedVal = getExtended(cycles, val) - return -} - -func (w *WrapAround[T, ET]) isWrapBack(earlier T, later T) bool { - return ET(later) < (w.fullRange>>1) && ET(earlier) >= (w.fullRange>>1) -} - -// ------------------------------------ - -func getExtended[T number, ET extendedNumber](cycles ET, val T) ET { - return cycles + ET(val) -} diff --git a/pkg/sfu/utils/wraparound_test.go b/pkg/sfu/utils/wraparound_test.go deleted file mode 100644 index 4f2e505..0000000 --- a/pkg/sfu/utils/wraparound_test.go +++ /dev/null @@ -1,679 +0,0 @@ -// Copyright 2023 LiveKit, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package utils - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -func TestWrapAroundUint16(t *testing.T) { - w := NewWrapAround[uint16, uint32](WrapAroundParams{IsRestartAllowed: true}) - testCases := []struct { - name string - input uint16 - updated WrapAroundUpdateResult[uint32] - start uint16 - extendedStart uint32 - highest uint16 - extendedHighest uint32 - }{ - // initialize - { - name: "initialize", - input: 10, - updated: WrapAroundUpdateResult[uint32]{ - IsRestart: false, - PreExtendedStart: 0, - PreExtendedHighest: 9, - ExtendedVal: 10, - }, - start: 10, - extendedStart: 10, - highest: 10, - extendedHighest: 10, - }, - // an older number without wrap around should reset start point - { - name: "reset start no wrap around", - input: 8, - updated: WrapAroundUpdateResult[uint32]{ - IsRestart: true, - PreExtendedStart: 10, - PreExtendedHighest: 10, - ExtendedVal: 8, - }, - start: 8, - extendedStart: 8, - highest: 10, - extendedHighest: 10, - }, - // an older number with wrap around should reset start point - { - name: "reset start wrap around", - input: (1 << 16) - 6, - updated: WrapAroundUpdateResult[uint32]{ - IsRestart: true, - PreExtendedStart: (1 << 16) + 8, - PreExtendedHighest: (1 << 16) + 10, - ExtendedVal: (1 << 16) - 6, - }, - start: (1 << 16) - 6, - extendedStart: (1 << 16) - 6, - highest: 10, - extendedHighest: (1 << 16) + 10, - }, - // an older number with wrap around should reset start point again - { - name: "reset start again", - input: (1 << 16) - 12, - updated: WrapAroundUpdateResult[uint32]{ - IsRestart: true, - PreExtendedStart: (1 << 16) - 6, - PreExtendedHighest: (1 << 16) + 10, - ExtendedVal: (1 << 16) - 12, - }, - start: (1 << 16) - 12, - extendedStart: (1 << 16) - 12, - highest: 10, - extendedHighest: (1 << 16) + 10, - }, - // out of order with highest, wrap back, but no restart - { - name: "out of order - no restart", - input: (1 << 16) - 3, - updated: WrapAroundUpdateResult[uint32]{ - IsRestart: false, - PreExtendedStart: 0, - PreExtendedHighest: (1 << 16) + 10, - ExtendedVal: (1 << 16) - 3, - }, - start: (1 << 16) - 12, - extendedStart: (1 << 16) - 12, - highest: 10, - extendedHighest: (1 << 16) + 10, - }, - // duplicate should return same as highest - { - name: "duplicate", - input: 10, - updated: WrapAroundUpdateResult[uint32]{ - IsRestart: false, - PreExtendedStart: 0, - PreExtendedHighest: (1 << 16) + 10, - ExtendedVal: (1 << 16) + 10, - }, - start: (1 << 16) - 12, - extendedStart: (1 << 16) - 12, - highest: 10, - extendedHighest: (1 << 16) + 10, - }, - // a significant jump in order should not reset start - { - name: "big in-order jump", - input: (1 << 15) - 10, - updated: WrapAroundUpdateResult[uint32]{ - IsRestart: false, - PreExtendedStart: 0, - PreExtendedHighest: (1 << 16) + 10, - ExtendedVal: (1 << 16) + (1 << 15) - 10, - }, - start: (1 << 16) - 12, - extendedStart: (1 << 16) - 12, - highest: (1 << 15) - 10, - extendedHighest: (1 << 16) + (1 << 15) - 10, - }, - // now out-of-order should not reset start as half the range has been seen - { - name: "out-of-order after half range", - input: (1 << 15) - 11, - updated: WrapAroundUpdateResult[uint32]{ - IsRestart: false, - PreExtendedStart: 0, - PreExtendedHighest: (1 << 16) + (1 << 15) - 10, - ExtendedVal: (1 << 16) + (1 << 15) - 11, - }, - start: (1 << 16) - 12, - extendedStart: (1 << 16) - 12, - highest: (1 << 15) - 10, - extendedHighest: (1 << 16) + (1 << 15) - 10, - }, - // wrap back out-of-order - { - name: "wrap back out-of-order after half range", - input: (1 << 16) - 1, - updated: WrapAroundUpdateResult[uint32]{ - IsRestart: false, - PreExtendedStart: 0, - PreExtendedHighest: (1 << 16) + (1 << 15) - 10, - ExtendedVal: (1 << 16) - 1, - }, - start: (1 << 16) - 12, - extendedStart: (1 << 16) - 12, - highest: (1 << 15) - 10, - extendedHighest: (1 << 16) + (1 << 15) - 10, - }, - // in-order, should update highest - { - name: "in-order", - input: (1 << 15) + 3, - updated: WrapAroundUpdateResult[uint32]{ - IsRestart: false, - PreExtendedStart: 0, - PreExtendedHighest: (1 << 16) + (1 << 15) - 10, - ExtendedVal: (1 << 16) + (1 << 15) + 3, - }, - start: (1 << 16) - 12, - extendedStart: (1 << 16) - 12, - highest: (1 << 15) + 3, - extendedHighest: (1 << 16) + (1 << 15) + 3, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - require.Equal(t, tc.updated, w.Update(tc.input)) - require.Equal(t, tc.start, w.GetStart()) - require.Equal(t, tc.extendedStart, w.GetExtendedStart()) - require.Equal(t, tc.highest, w.GetHighest()) - require.Equal(t, tc.extendedHighest, w.GetExtendedHighest()) - }) - } -} - -func TestWrapAroundUint16NoRestart(t *testing.T) { - w := NewWrapAround[uint16, uint32](WrapAroundParams{IsRestartAllowed: false}) - testCases := []struct { - name string - input uint16 - updated WrapAroundUpdateResult[uint32] - start uint16 - extendedStart uint32 - highest uint16 - extendedHighest uint32 - }{ - // initialize - { - name: "initialize", - input: 10, - updated: WrapAroundUpdateResult[uint32]{ - IsRestart: false, - PreExtendedStart: 0, - PreExtendedHighest: 9, - ExtendedVal: 10, - }, - start: 10, - extendedStart: 10, - highest: 10, - extendedHighest: 10, - }, - // an older number without wrap around should not reset start point - { - name: "no reset start no wrap around", - input: 8, - updated: WrapAroundUpdateResult[uint32]{ - IsUnhandled: true, - // the following fields are not valid when `IsUnhandled = true`, but code fills it in - // and they are filled in here for testing purposes - PreExtendedHighest: 10, - ExtendedVal: 8, - }, - start: 10, - extendedStart: 10, - highest: 10, - extendedHighest: 10, - }, - // an older number with wrap around should not reset start point - { - name: "no reset start wrap around", - input: (1 << 16) - 6, - updated: WrapAroundUpdateResult[uint32]{ - IsUnhandled: true, - PreExtendedHighest: 10, - ExtendedVal: (1 << 16) - 6, - }, - start: 10, - extendedStart: 10, - highest: 10, - extendedHighest: 10, - }, - // yet another older number with wrap around should not reset start point - { - name: "no reset start again", - input: (1 << 16) - 12, - updated: WrapAroundUpdateResult[uint32]{ - IsUnhandled: true, - PreExtendedHighest: 10, - ExtendedVal: (1 << 16) - 12, - }, - start: 10, - extendedStart: 10, - highest: 10, - extendedHighest: 10, - }, - // duplicate should return same as highest - { - name: "duplicate", - input: 10, - updated: WrapAroundUpdateResult[uint32]{ - PreExtendedHighest: 10, - ExtendedVal: 10, - }, - start: 10, - extendedStart: 10, - highest: 10, - extendedHighest: 10, - }, - // a significant jump in order should move highest to that - { - name: "big in-order jump", - input: (1 << 15) - 10, - updated: WrapAroundUpdateResult[uint32]{ - PreExtendedHighest: 10, - ExtendedVal: (1 << 15) - 10, - }, - start: 10, - extendedStart: 10, - highest: (1 << 15) - 10, - extendedHighest: (1 << 15) - 10, - }, - // in-order, should update highest - { - name: "in-order", - input: (1 << 15) + 13, - updated: WrapAroundUpdateResult[uint32]{ - PreExtendedHighest: (1 << 15) - 10, - ExtendedVal: (1 << 15) + 13, - }, - start: 10, - extendedStart: 10, - highest: (1 << 15) + 13, - extendedHighest: (1 << 15) + 13, - }, - // now out-of-order should not reset start as half the range has been seen - { - name: "out-of-order after half range", - input: (1 << 15) - 11, - updated: WrapAroundUpdateResult[uint32]{ - PreExtendedHighest: (1 << 15) + 13, - ExtendedVal: (1 << 15) - 11, - }, - start: 10, - extendedStart: 10, - highest: (1 << 15) + 13, - extendedHighest: (1 << 15) + 13, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - require.Equal(t, tc.updated, w.Update(tc.input)) - require.Equal(t, tc.start, w.GetStart()) - require.Equal(t, tc.extendedStart, w.GetExtendedStart()) - require.Equal(t, tc.highest, w.GetHighest()) - require.Equal(t, tc.extendedHighest, w.GetExtendedHighest()) - }) - } -} - -func TestWrapAroundUint16RollbackRestartAndResetHighest(t *testing.T) { - w := NewWrapAround[uint16, uint64](WrapAroundParams{IsRestartAllowed: true}) - - // initialize - w.Update(23) - require.Equal(t, uint16(23), w.GetStart()) - require.Equal(t, uint64(23), w.GetExtendedStart()) - require.Equal(t, uint16(23), w.GetHighest()) - require.Equal(t, uint64(23), w.GetExtendedHighest()) - - // an in-order update - w.Update(25) - require.Equal(t, uint16(23), w.GetStart()) - require.Equal(t, uint64(23), w.GetExtendedStart()) - require.Equal(t, uint16(25), w.GetHighest()) - require.Equal(t, uint64(25), w.GetExtendedHighest()) - - // force restart without wrap - res := w.Update(12) - expectedResult := WrapAroundUpdateResult[uint64]{ - IsRestart: true, - PreExtendedStart: 23, - PreExtendedHighest: 25, - ExtendedVal: 12, - } - require.Equal(t, expectedResult, res) - require.Equal(t, uint16(12), w.GetStart()) - require.Equal(t, uint64(12), w.GetExtendedStart()) - require.Equal(t, uint16(25), w.GetHighest()) - require.Equal(t, uint64(25), w.GetExtendedHighest()) - - // roll back restart - w.RollbackRestart(res.PreExtendedStart) - require.Equal(t, uint16(23), w.GetStart()) - require.Equal(t, uint64(23), w.GetExtendedStart()) - require.Equal(t, uint16(25), w.GetHighest()) - require.Equal(t, uint64(25), w.GetExtendedHighest()) - - // force restart with wrap - res = w.Update(65533) - expectedResult = WrapAroundUpdateResult[uint64]{ - IsRestart: true, - PreExtendedStart: (1 << 16) + 23, - PreExtendedHighest: (1 << 16) + 25, - ExtendedVal: 65533, - } - require.Equal(t, expectedResult, res) - require.Equal(t, uint16(65533), w.GetStart()) - require.Equal(t, uint64(65533), w.GetExtendedStart()) - require.Equal(t, uint16(25), w.GetHighest()) - require.Equal(t, uint64(65536+25), w.GetExtendedHighest()) - - // roll back restart - w.RollbackRestart(res.PreExtendedStart) - require.Equal(t, uint16(23), w.GetStart()) - require.Equal(t, uint64(23), w.GetExtendedStart()) - require.Equal(t, uint16(25), w.GetHighest()) - require.Equal(t, uint64(25), w.GetExtendedHighest()) - - // reset highest - w.ResetHighest(0x1234) - require.Equal(t, uint16(23), w.GetStart()) - require.Equal(t, uint64(23), w.GetExtendedStart()) - require.Equal(t, uint16(0x1234), w.GetHighest()) - require.Equal(t, uint64(0x1234), w.GetExtendedHighest()) - - w.ResetHighest(0x7f1234) - require.Equal(t, uint16(23), w.GetStart()) - require.Equal(t, uint64(23), w.GetExtendedStart()) - require.Equal(t, uint16(0x1234), w.GetHighest()) - require.Equal(t, uint64(0x7f1234), w.GetExtendedHighest()) -} - -func TestWrapAroundUint16WrapAroundRestartDuplicate(t *testing.T) { - w := NewWrapAround[uint16, uint64](WrapAroundParams{IsRestartAllowed: true}) - - // initialize - w.Update(65534) - require.Equal(t, uint16(65534), w.GetStart()) - require.Equal(t, uint64(65534), w.GetExtendedStart()) - require.Equal(t, uint16(65534), w.GetHighest()) - require.Equal(t, uint64(65534), w.GetExtendedHighest()) - - // an in-order update with a roll over - w.Update(32) - require.Equal(t, uint16(65534), w.GetStart()) - require.Equal(t, uint64(65534), w.GetExtendedStart()) - require.Equal(t, uint16(32), w.GetHighest()) - require.Equal(t, uint64(65568), w.GetExtendedHighest()) - - // duplicate of start - res := w.Update(65534) - expectedResult := WrapAroundUpdateResult[uint64]{ - IsRestart: false, - PreExtendedStart: 0, - PreExtendedHighest: 65568, - ExtendedVal: 65534, - } - require.Equal(t, expectedResult, res) - require.Equal(t, uint16(65534), w.GetStart()) - require.Equal(t, uint64(65534), w.GetExtendedStart()) - require.Equal(t, uint16(32), w.GetHighest()) - require.Equal(t, uint64(65568), w.GetExtendedHighest()) - - // duplicate of start - again - res = w.Update(65534) - expectedResult = WrapAroundUpdateResult[uint64]{ - IsRestart: false, - PreExtendedStart: 0, - PreExtendedHighest: 65568, - ExtendedVal: 65534, - } - require.Equal(t, expectedResult, res) - require.Equal(t, uint16(65534), w.GetStart()) - require.Equal(t, uint64(65534), w.GetExtendedStart()) - require.Equal(t, uint16(32), w.GetHighest()) - require.Equal(t, uint64(65568), w.GetExtendedHighest()) -} - -func TestWrapAroundUint16Rollover(t *testing.T) { - w := NewWrapAround[uint16, uint32](WrapAroundParams{IsRestartAllowed: false}) - testCases := []struct { - name string - input uint16 - numCycles int - updated WrapAroundUpdateResult[uint32] - start uint16 - extendedStart uint32 - highest uint16 - extendedHighest uint32 - }{ - // initialize - should initialize irrespective of numCycles - { - name: "initialize", - input: 10, - numCycles: 10, - updated: WrapAroundUpdateResult[uint32]{ - IsRestart: false, - PreExtendedStart: 0, - PreExtendedHighest: 9, - ExtendedVal: 10, - }, - start: 10, - extendedStart: 10, - highest: 10, - extendedHighest: 10, - }, - // negative cycles - should just do an update - { - name: "zero", - input: 8, - numCycles: -1, - updated: WrapAroundUpdateResult[uint32]{ - IsUnhandled: true, - // the following fields are not valid when `IsUnhandled = true`, but code fills it in - // and they are filled in here for testing purposes - PreExtendedHighest: 10, - ExtendedVal: 8, - }, - start: 10, - extendedStart: 10, - highest: 10, - extendedHighest: 10, - }, - // one cycle - { - name: "one cycle", - input: (1 << 16) - 6, - numCycles: 1, - updated: WrapAroundUpdateResult[uint32]{ - PreExtendedHighest: 10, - ExtendedVal: (1 << 16) - 6 + (1 << 16), - }, - start: 10, - extendedStart: 10, - highest: (1 << 16) - 6, - extendedHighest: (1 << 16) - 6 + (1 << 16), - }, - // two cycles - { - name: "two cycles", - input: (1 << 16) - 7, - numCycles: 2, - updated: WrapAroundUpdateResult[uint32]{ - PreExtendedHighest: (1 << 16) - 6 + (1 << 16), - ExtendedVal: (1 << 16) - 7 + 3*(1<<16), - }, - start: 10, - extendedStart: 10, - highest: (1 << 16) - 7, - extendedHighest: (1 << 16) - 7 + 3*(1<<16), - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - require.Equal(t, tc.updated, w.Rollover(tc.input, tc.numCycles)) - require.Equal(t, tc.start, w.GetStart()) - require.Equal(t, tc.extendedStart, w.GetExtendedStart()) - require.Equal(t, tc.highest, w.GetHighest()) - require.Equal(t, tc.extendedHighest, w.GetExtendedHighest()) - }) - } -} - -func TestWrapAroundUint32(t *testing.T) { - w := NewWrapAround[uint32, uint64](WrapAroundParams{IsRestartAllowed: true}) - testCases := []struct { - name string - input uint32 - updated WrapAroundUpdateResult[uint64] - start uint32 - extendedStart uint64 - highest uint32 - extendedHighest uint64 - }{ - // initialize - { - name: "initialize", - input: 10, - updated: WrapAroundUpdateResult[uint64]{ - IsRestart: false, - PreExtendedStart: 0, - PreExtendedHighest: 9, - ExtendedVal: 10, - }, - start: 10, - extendedStart: 10, - highest: 10, - extendedHighest: 10, - }, - // an older number without wrap around should reset start point - { - name: "reset start no wrap around", - input: 8, - updated: WrapAroundUpdateResult[uint64]{ - IsRestart: true, - PreExtendedStart: 10, - PreExtendedHighest: 10, - ExtendedVal: 8, - }, - start: 8, - extendedStart: 8, - highest: 10, - extendedHighest: 10, - }, - // an older number with wrap around should reset start point - { - name: "reset start wrap around", - input: (1 << 32) - 6, - updated: WrapAroundUpdateResult[uint64]{ - IsRestart: true, - PreExtendedStart: (1 << 32) + 8, - PreExtendedHighest: (1 << 32) + 10, - ExtendedVal: (1 << 32) - 6, - }, - start: (1 << 32) - 6, - extendedStart: (1 << 32) - 6, - highest: 10, - extendedHighest: (1 << 32) + 10, - }, - // an older number with wrap around should reset start point again - { - name: "reset start again", - input: (1 << 32) - 12, - updated: WrapAroundUpdateResult[uint64]{ - IsRestart: true, - PreExtendedStart: (1 << 32) - 6, - PreExtendedHighest: (1 << 32) + 10, - ExtendedVal: (1 << 32) - 12, - }, - start: (1 << 32) - 12, - extendedStart: (1 << 32) - 12, - highest: 10, - extendedHighest: (1 << 32) + 10, - }, - // duplicate should return same as highest - { - name: "duplicate", - input: 10, - updated: WrapAroundUpdateResult[uint64]{ - IsRestart: false, - PreExtendedStart: 0, - PreExtendedHighest: (1 << 32) + 10, - ExtendedVal: (1 << 32) + 10, - }, - start: (1 << 32) - 12, - extendedStart: (1 << 32) - 12, - highest: 10, - extendedHighest: (1 << 32) + 10, - }, - // a significant jump in order should not reset start - { - name: "big in-order jump", - input: 1 << 31, - updated: WrapAroundUpdateResult[uint64]{ - IsRestart: false, - PreExtendedStart: 0, - PreExtendedHighest: (1 << 32) + 10, - ExtendedVal: (1 << 32) + (1 << 31), - }, - start: (1 << 32) - 12, - extendedStart: (1 << 32) - 12, - highest: 1 << 31, - extendedHighest: (1 << 32) + (1 << 31), - }, - // now out-of-order should not reset start as half the range has been seen - { - name: "out-of-order after half range", - input: (1 << 31) - 1, - updated: WrapAroundUpdateResult[uint64]{ - IsRestart: false, - PreExtendedStart: 0, - PreExtendedHighest: (1 << 32) + (1 << 31), - ExtendedVal: (1 << 32) + (1 << 31) - 1, - }, - start: (1 << 32) - 12, - extendedStart: (1 << 32) - 12, - highest: 1 << 31, - extendedHighest: (1 << 32) + (1 << 31), - }, - // in-order, should update highest - { - name: "in-order", - input: (1 << 31) + 3, - updated: WrapAroundUpdateResult[uint64]{ - IsRestart: false, - PreExtendedStart: 0, - PreExtendedHighest: (1 << 32) + (1 << 31), - ExtendedVal: (1 << 32) + (1 << 31) + 3, - }, - start: (1 << 32) - 12, - extendedStart: (1 << 32) - 12, - highest: (1 << 31) + 3, - extendedHighest: (1 << 32) + (1 << 31) + 3, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - require.Equal(t, tc.updated, w.Update(tc.input)) - require.Equal(t, tc.start, w.GetStart()) - require.Equal(t, tc.extendedStart, w.GetExtendedStart()) - require.Equal(t, tc.highest, w.GetHighest()) - require.Equal(t, tc.extendedHighest, w.GetExtendedHighest()) - }) - } -} diff --git a/pkg/sfu/videolayerselector/dependencydescriptor.go b/pkg/sfu/videolayerselector/dependencydescriptor.go index 7d93417..3e2e090 100644 --- a/pkg/sfu/videolayerselector/dependencydescriptor.go +++ b/pkg/sfu/videolayerselector/dependencydescriptor.go @@ -24,6 +24,11 @@ import ( "github.com/livekit/protocol/logger" ) +const ( + decisionCacheMaxElements = 256 + decisionCacheNackEntries = 80 +) + type DependencyDescriptor struct { *Base @@ -40,12 +45,14 @@ type DependencyDescriptor struct { decodeTargetsLock sync.RWMutex decodeTargets []*DecodeTarget fnWrapper FrameNumberWrapper + + restartGeneration int } func NewDependencyDescriptor(logger logger.Logger) *DependencyDescriptor { return &DependencyDescriptor{ Base: NewBase(logger), - decisions: NewSelectorDecisionCache(256, 80), + decisions: NewSelectorDecisionCache(decisionCacheMaxElements, decisionCacheNackEntries), fnWrapper: FrameNumberWrapper{logger: logger}, } } @@ -75,6 +82,21 @@ func (d *DependencyDescriptor) Select(extPkt *buffer.ExtPacket, _layer int32) (r return } + if ddwdt.RestartGeneration > d.restartGeneration { + d.logger.Debugw( + "stream restarted", + "packet", ddwdt.RestartGeneration, + "current", d.restartGeneration, + "structureKeyFrame", d.extKeyFrameNum, + "efn", ddwdt.ExtFrameNum, + "lastEfn", d.fnWrapper.LastOrigin(), + ) + d.restart(ddwdt.RestartGeneration) + } else if ddwdt.RestartGeneration < d.restartGeneration { + // must not happen + d.logger.Warnw("packet from old generation", nil, "packet", ddwdt.RestartGeneration, "current", d.restartGeneration) + } + dd := ddwdt.Descriptor extFrameNum := ddwdt.ExtFrameNum @@ -269,7 +291,7 @@ func (d *DependencyDescriptor) Select(extPkt *buffer.ExtPacket, _layer int32) (r "fn", dd.FrameNumber, "efn", extFrameNum, "sn", extPkt.Packet.SequenceNumber, - "isKeyFrame", extPkt.KeyFrame, + "isKeyFrame", extPkt.IsKeyFrame, ) } @@ -286,7 +308,7 @@ func (d *DependencyDescriptor) Select(extPkt *buffer.ExtPacket, _layer int32) (r "fn", dd.FrameNumber, "efn", extFrameNum, "sn", extPkt.Packet.SequenceNumber, - "isKeyFrame", extPkt.KeyFrame, + "isKeyFrame", extPkt.IsKeyFrame, ) result.IsRelevant = true @@ -434,3 +456,9 @@ func (d *DependencyDescriptor) CheckSync() (locked bool, layer int32) { return false, layer } + +func (d *DependencyDescriptor) restart(generation int) { + d.restartGeneration = generation + d.invalidateKeyFrame() + d.decisions = NewSelectorDecisionCache(decisionCacheMaxElements, decisionCacheNackEntries) +} diff --git a/pkg/sfu/videolayerselector/dependencydescriptor_test.go b/pkg/sfu/videolayerselector/dependencydescriptor_test.go index a562988..925c017 100644 --- a/pkg/sfu/videolayerselector/dependencydescriptor_test.go +++ b/pkg/sfu/videolayerselector/dependencydescriptor_test.go @@ -15,7 +15,7 @@ package videolayerselector import ( - "sort" + "slices" "testing" "github.com/pion/rtp" @@ -143,7 +143,7 @@ func TestDependencyDescriptor(t *testing.T) { // non key frame, dropped ret = ddSelector.Select(&buffer.ExtPacket{ - KeyFrame: false, + IsKeyFrame: false, DependencyDescriptor: &buffer.ExtDependencyDescriptor{ Descriptor: &dd.DependencyDescriptor{ FrameNumber: 1, @@ -287,8 +287,14 @@ func createDDFrames(maxLayer buffer.VideoLayer, startFrameNumber uint16) []*buff activeBitMask |= 1 << uint(i*int(maxLayer.Temporal+1)+j) } } - sort.Slice(decodeTargets, func(i, j int) bool { - return decodeTargets[i].Layer.GreaterThan(decodeTargets[j].Layer) + slices.SortFunc(decodeTargets, func(a, b buffer.DependencyDescriptorDecodeTarget) int { + if a.Layer.GreaterThan(b.Layer) { + return -1 + } + if b.Layer.GreaterThan(a.Layer) { + return 1 + } + return 0 }) chainDiffs := make([]int, int(maxLayer.Spatial)+1) @@ -308,7 +314,7 @@ func createDDFrames(maxLayer buffer.VideoLayer, startFrameNumber uint16) []*buff } } keyFrame := &buffer.ExtPacket{ - KeyFrame: true, + IsKeyFrame: true, DependencyDescriptor: &buffer.ExtDependencyDescriptor{ Descriptor: &dd.DependencyDescriptor{ FrameNumber: startFrameNumber, diff --git a/pkg/sfu/videolayerselector/framechain.go b/pkg/sfu/videolayerselector/framechain.go index 19ddf7e..b54dd00 100644 --- a/pkg/sfu/videolayerselector/framechain.go +++ b/pkg/sfu/videolayerselector/framechain.go @@ -71,11 +71,11 @@ func (fc *FrameChain) OnFrame(extFrameNum uint64, fd *dd.FrameDependencyTemplate } var intact bool - switch { - case sd == selectorDecisionForwarded: + switch sd { + case selectorDecisionForwarded: intact = true - case sd == selectorDecisionUnknown: + case selectorDecisionUnknown: // If the previous frame is unknown, means it has not arrived but could be recovered by NACK / out-of-order arrival, // set up a expected callback here to determine if the chain is broken or intact if fc.decisions.ExpectDecision(prevFrameInChain, fc.OnExpectFrameChanged) { diff --git a/pkg/sfu/videolayerselector/framenumberwrapper.go b/pkg/sfu/videolayerselector/framenumberwrapper.go index f0997fc..43c9c02 100644 --- a/pkg/sfu/videolayerselector/framenumberwrapper.go +++ b/pkg/sfu/videolayerselector/framenumberwrapper.go @@ -52,3 +52,7 @@ func (f *FrameNumberWrapper) UpdateAndGet(new uint64, updateOffset bool) uint64 f.last = new return new + f.offset } + +func (f *FrameNumberWrapper) LastOrigin() uint64 { + return f.last +} diff --git a/pkg/sfu/videolayerselector/framenumberwrapper_test.go b/pkg/sfu/videolayerselector/framenumberwrapper_test.go index b18f509..4d0f6f2 100644 --- a/pkg/sfu/videolayerselector/framenumberwrapper_test.go +++ b/pkg/sfu/videolayerselector/framenumberwrapper_test.go @@ -21,7 +21,7 @@ import ( "github.com/stretchr/testify/require" - "github.com/livekit/livekit-server/pkg/sfu/utils" + "github.com/livekit/mediatransportutil/pkg/utils" "github.com/livekit/protocol/logger" ) @@ -65,7 +65,7 @@ func TestFrameNumberWrapper(t *testing.T) { // frame out of order firstF = secondF secondF = getFrame(firstF, false) - // it is possile that an out of order non-keyframe has been converted to in order frame number if the diff is 32768 + // it is possible that an out of order non-keyframe has been converted to in order frame number if the diff is 32768 // that is ok because the client can't decode in such case and always need to wait for the key frame. // so it is just a failure of test case and increase the frame number here. if secondF-firstF == 0x8000 { diff --git a/pkg/sfu/videolayerselector/simulcast.go b/pkg/sfu/videolayerselector/simulcast.go index 9720bbe..664b2a7 100644 --- a/pkg/sfu/videolayerselector/simulcast.go +++ b/pkg/sfu/videolayerselector/simulcast.go @@ -93,7 +93,7 @@ func (s *Simulcast) Select(extPkt *buffer.ExtPacket, layer int32) (result VideoL isActive := s.currentLayer.IsValid() found := false reason := "" - if extPkt.KeyFrame { + if extPkt.IsKeyFrame { if layer > s.currentLayer.Spatial && layer <= s.targetLayer.Spatial { reason = "upgrading layer" found = true @@ -124,7 +124,7 @@ func (s *Simulcast) Select(extPkt *buffer.ExtPacket, layer int32) (result VideoL } // if locked to higher than max layer due to overshoot, check if it can be dialed back - if s.currentLayer.Spatial > s.maxLayer.Spatial && layer <= s.maxLayer.Spatial && extPkt.KeyFrame { + if s.currentLayer.Spatial > s.maxLayer.Spatial && layer <= s.maxLayer.Spatial && extPkt.IsKeyFrame { s.previousLayer = s.currentLayer s.currentLayer.Spatial = layer diff --git a/pkg/sfu/videolayerselector/temporallayerselector/vp8.go b/pkg/sfu/videolayerselector/temporallayerselector/vp8.go index c0a86c4..39480c4 100644 --- a/pkg/sfu/videolayerselector/temporallayerselector/vp8.go +++ b/pkg/sfu/videolayerselector/temporallayerselector/vp8.go @@ -16,6 +16,7 @@ package temporallayerselector import ( "github.com/livekit/livekit-server/pkg/sfu/buffer" + "github.com/livekit/mediatransportutil/pkg/codec" "github.com/livekit/protocol/logger" ) @@ -36,7 +37,7 @@ func (v *VP8) Select(extPkt *buffer.ExtPacket, current int32, target int32) (thi return } - vp8, ok := extPkt.Payload.(buffer.VP8) + vp8, ok := extPkt.Payload.(codec.VP8) if !ok { return } diff --git a/pkg/sfu/videolayerselector/vp9.go b/pkg/sfu/videolayerselector/vp9.go index dc5999c..4b11d37 100644 --- a/pkg/sfu/videolayerselector/vp9.go +++ b/pkg/sfu/videolayerselector/vp9.go @@ -50,7 +50,7 @@ func (v *VP9) Select(extPkt *buffer.ExtPacket, _layer int32) (result VideoLayerS updatedLayer := v.currentLayer if !v.currentLayer.IsValid() { - if !extPkt.KeyFrame { + if !extPkt.IsKeyFrame { return } diff --git a/pkg/telemetry/analyticsservice.go b/pkg/telemetry/analyticsservice.go index 57873ac..5868ca8 100644 --- a/pkg/telemetry/analyticsservice.go +++ b/pkg/telemetry/analyticsservice.go @@ -22,6 +22,7 @@ import ( "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" + "github.com/livekit/protocol/observability/roomobs" "github.com/livekit/protocol/rpc" "github.com/livekit/protocol/utils/guid" @@ -34,8 +35,24 @@ type AnalyticsService interface { SendStats(ctx context.Context, stats []*livekit.AnalyticsStat) SendEvent(ctx context.Context, events *livekit.AnalyticsEvent) SendNodeRoomStates(ctx context.Context, nodeRooms *livekit.AnalyticsNodeRooms) + RoomProjectReporter(ctx context.Context) roomobs.ProjectReporter } +// ---------------------------- + +var _ AnalyticsService = &NullAnalyticService{} + +type NullAnalyticService struct{} + +func (n NullAnalyticService) SendStats(_ context.Context, _ []*livekit.AnalyticsStat) {} +func (n NullAnalyticService) SendEvent(_ context.Context, _ *livekit.AnalyticsEvent) {} +func (n NullAnalyticService) SendNodeRoomStates(_ context.Context, _ *livekit.AnalyticsNodeRooms) {} +func (n NullAnalyticService) RoomProjectReporter(_ctx context.Context) roomobs.ProjectReporter { + return nil +} + +// ---------------------------- + type analyticsService struct { analyticsKey string nodeID string @@ -95,3 +112,7 @@ func (a *analyticsService) SendNodeRoomStates(_ context.Context, nodeRooms *live logger.Errorw("failed to send node room states", err) } } + +func (a *analyticsService) RoomProjectReporter(_ context.Context) roomobs.ProjectReporter { + return roomobs.NewNoopProjectReporter() +} diff --git a/pkg/telemetry/events.go b/pkg/telemetry/events.go index d0ba938..55a21f6 100644 --- a/pkg/telemetry/events.go +++ b/pkg/telemetry/events.go @@ -20,15 +20,16 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" - "github.com/livekit/livekit-server/pkg/sfu/mime" "github.com/livekit/livekit-server/pkg/telemetry/prometheus" + "github.com/livekit/protocol/codecs/mime" + "github.com/livekit/protocol/egress" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" "github.com/livekit/protocol/utils/guid" "github.com/livekit/protocol/webhook" ) -func (t *telemetryService) NotifyEvent(ctx context.Context, event *livekit.WebhookEvent) { +func (t *telemetryService) NotifyEvent(ctx context.Context, event *livekit.WebhookEvent, opts ...webhook.NotifyOption) { if t.notifier == nil { return } @@ -36,7 +37,7 @@ func (t *telemetryService) NotifyEvent(ctx context.Context, event *livekit.Webho event.CreatedAt = time.Now().Unix() event.Id = guid.New("EV_") - if err := t.notifier.QueueNotify(ctx, event); err != nil { + if err := t.notifier.QueueNotify(ctx, event, opts...); err != nil { logger.Warnw("failed to notify webhook", err, "event", event.Event) } } @@ -79,6 +80,7 @@ func (t *telemetryService) ParticipantJoined( clientInfo *livekit.ClientInfo, clientMeta *livekit.AnalyticsClientMeta, shouldSendEvent bool, + guard *ReferenceGuard, ) { t.enqueue(func() { _, found := t.getOrCreateWorker( @@ -87,6 +89,7 @@ func (t *telemetryService) ParticipantJoined( livekit.RoomName(room.Name), livekit.ParticipantID(participant.Sid), livekit.ParticipantIdentity(participant.Identity), + guard, ) if !found { prometheus.IncrementParticipantRtcConnected(1) @@ -108,10 +111,11 @@ func (t *telemetryService) ParticipantActive( participant *livekit.ParticipantInfo, clientMeta *livekit.AnalyticsClientMeta, isMigration bool, + guard *ReferenceGuard, ) { t.enqueue(func() { if !isMigration { - // consider participant joined only when they became active + // a participant is considered "joined" only when they become "active" t.NotifyEvent(ctx, &livekit.WebhookEvent{ Event: webhook.EventParticipantJoined, Room: room, @@ -125,12 +129,13 @@ func (t *telemetryService) ParticipantActive( livekit.RoomName(room.Name), livekit.ParticipantID(participant.Sid), livekit.ParticipantIdentity(participant.Identity), + guard, ) if !found { - // need to also account for participant count prometheus.AddParticipant() } worker.SetConnected() + prometheus.IncrementParticipantRtcActive(1) ev := newParticipantEvent(livekit.AnalyticsEventType_PARTICIPANT_ACTIVE, room, participant) ev.ClientMeta = clientMeta @@ -161,6 +166,7 @@ func (t *telemetryService) ParticipantResumed( livekit.RoomName(room.Name), livekit.ParticipantID(participant.Sid), livekit.ParticipantIdentity(participant.Identity), + nil, ) if !found { prometheus.AddParticipant() @@ -179,37 +185,55 @@ func (t *telemetryService) ParticipantLeft(ctx context.Context, room *livekit.Room, participant *livekit.ParticipantInfo, shouldSendEvent bool, + guard *ReferenceGuard, ) { t.enqueue(func() { isConnected := false - if worker, ok := t.getWorker(livekit.ParticipantID(participant.Sid)); ok { + if worker, ok := t.getWorker(livekit.RoomID(room.Sid), livekit.ParticipantID(participant.Sid)); ok { isConnected = worker.IsConnected() - if worker.Close() { + if worker.Close(guard) { prometheus.SubParticipant() + } else { + logger.Infow( + "stats worker active", + "room", room.Name, + "roomID", room.Sid, + "participant", participant.Identity, + "participantID", participant.Sid, + "worker", worker, + ) } } - if isConnected && shouldSendEvent { + if shouldSendEvent { + webhookEvent := webhook.EventParticipantLeft + analyticsEvent := livekit.AnalyticsEventType_PARTICIPANT_LEFT + if !isConnected { + webhookEvent = webhook.EventParticipantConnectionAborted + analyticsEvent = livekit.AnalyticsEventType_PARTICIPANT_CONNECTION_ABORTED + } t.NotifyEvent(ctx, &livekit.WebhookEvent{ - Event: webhook.EventParticipantLeft, + Event: webhookEvent, Room: room, Participant: participant, }) - t.SendEvent(ctx, newParticipantEvent(livekit.AnalyticsEventType_PARTICIPANT_LEFT, room, participant)) + t.SendEvent(ctx, newParticipantEvent(analyticsEvent, room, participant)) } }) } func (t *telemetryService) TrackPublishRequested( ctx context.Context, + roomID livekit.RoomID, + roomName livekit.RoomName, participantID livekit.ParticipantID, identity livekit.ParticipantIdentity, track *livekit.TrackInfo, ) { t.enqueue(func() { - prometheus.AddPublishAttempt(track.Type.String()) - room := t.getRoomDetails(participantID) + prometheus.RecordTrackPublishAttempt(track.Type.String()) + room := toMinimalRoomProto(roomID, roomName) ev := newTrackEvent(livekit.AnalyticsEventType_TRACK_PUBLISH_REQUESTED, room, participantID, track) if ev.Participant != nil { ev.Participant.Identity = string(identity) @@ -220,15 +244,21 @@ func (t *telemetryService) TrackPublishRequested( func (t *telemetryService) TrackPublished( ctx context.Context, + roomID livekit.RoomID, + roomName livekit.RoomName, participantID livekit.ParticipantID, identity livekit.ParticipantIdentity, track *livekit.TrackInfo, + shouldSendEvent bool, ) { t.enqueue(func() { prometheus.AddPublishedTrack(track.Type.String()) - prometheus.AddPublishSuccess(track.Type.String()) + prometheus.RecordTrackPublishSuccess(track.Type.String()) + if !shouldSendEvent { + return + } - room := t.getRoomDetails(participantID) + room := toMinimalRoomProto(roomID, roomName) participant := &livekit.ParticipantInfo{ Sid: string(participantID), Identity: string(identity), @@ -246,22 +276,30 @@ func (t *telemetryService) TrackPublished( }) } -func (t *telemetryService) TrackPublishedUpdate(ctx context.Context, participantID livekit.ParticipantID, track *livekit.TrackInfo) { +func (t *telemetryService) TrackPublishedUpdate( + ctx context.Context, + roomID livekit.RoomID, + roomName livekit.RoomName, + participantID livekit.ParticipantID, + track *livekit.TrackInfo, +) { t.enqueue(func() { - room := t.getRoomDetails(participantID) + room := toMinimalRoomProto(roomID, roomName) t.SendEvent(ctx, newTrackEvent(livekit.AnalyticsEventType_TRACK_PUBLISHED_UPDATE, room, participantID, track)) }) } func (t *telemetryService) TrackMaxSubscribedVideoQuality( ctx context.Context, + roomID livekit.RoomID, + roomName livekit.RoomName, participantID livekit.ParticipantID, track *livekit.TrackInfo, mime mime.MimeType, maxQuality livekit.VideoQuality, ) { t.enqueue(func() { - room := t.getRoomDetails(participantID) + room := toMinimalRoomProto(roomID, roomName) ev := newTrackEvent(livekit.AnalyticsEventType_TRACK_MAX_SUBSCRIBED_VIDEO_QUALITY, room, participantID, track) ev.MaxSubscribedVideoQuality = maxQuality ev.Mime = mime.String() @@ -271,13 +309,15 @@ func (t *telemetryService) TrackMaxSubscribedVideoQuality( func (t *telemetryService) TrackSubscribeRequested( ctx context.Context, + roomID livekit.RoomID, + roomName livekit.RoomName, participantID livekit.ParticipantID, track *livekit.TrackInfo, ) { t.enqueue(func() { prometheus.RecordTrackSubscribeAttempt() - room := t.getRoomDetails(participantID) + room := toMinimalRoomProto(roomID, roomName) ev := newTrackEvent(livekit.AnalyticsEventType_TRACK_SUBSCRIBE_REQUESTED, room, participantID, track) t.SendEvent(ctx, ev) }) @@ -285,6 +325,8 @@ func (t *telemetryService) TrackSubscribeRequested( func (t *telemetryService) TrackSubscribed( ctx context.Context, + roomID livekit.RoomID, + roomName livekit.RoomName, participantID livekit.ParticipantID, track *livekit.TrackInfo, publisher *livekit.ParticipantInfo, @@ -297,7 +339,7 @@ func (t *telemetryService) TrackSubscribed( return } - room := t.getRoomDetails(participantID) + room := toMinimalRoomProto(roomID, roomName) ev := newTrackEvent(livekit.AnalyticsEventType_TRACK_SUBSCRIBED, room, participantID, track) ev.Publisher = publisher t.SendEvent(ctx, ev) @@ -306,6 +348,8 @@ func (t *telemetryService) TrackSubscribed( func (t *telemetryService) TrackSubscribeFailed( ctx context.Context, + roomID livekit.RoomID, + roomName livekit.RoomName, participantID livekit.ParticipantID, trackID livekit.TrackID, err error, @@ -314,7 +358,7 @@ func (t *telemetryService) TrackSubscribeFailed( t.enqueue(func() { prometheus.RecordTrackSubscribeFailure(err, isUserError) - room := t.getRoomDetails(participantID) + room := toMinimalRoomProto(roomID, roomName) ev := newTrackEvent(livekit.AnalyticsEventType_TRACK_SUBSCRIBE_FAILED, room, participantID, &livekit.TrackInfo{ Sid: string(trackID), }) @@ -325,6 +369,8 @@ func (t *telemetryService) TrackSubscribeFailed( func (t *telemetryService) TrackUnsubscribed( ctx context.Context, + roomID livekit.RoomID, + roomName livekit.RoomName, participantID livekit.ParticipantID, track *livekit.TrackInfo, shouldSendEvent bool, @@ -333,7 +379,7 @@ func (t *telemetryService) TrackUnsubscribed( prometheus.RecordTrackUnsubscribed(track.Type.String()) if shouldSendEvent { - room := t.getRoomDetails(participantID) + room := toMinimalRoomProto(roomID, roomName) t.SendEvent(ctx, newTrackEvent(livekit.AnalyticsEventType_TRACK_UNSUBSCRIBED, room, participantID, track)) } }) @@ -341,6 +387,8 @@ func (t *telemetryService) TrackUnsubscribed( func (t *telemetryService) TrackUnpublished( ctx context.Context, + roomID livekit.RoomID, + roomName livekit.RoomName, participantID livekit.ParticipantID, identity livekit.ParticipantIdentity, track *livekit.TrackInfo, @@ -352,7 +400,7 @@ func (t *telemetryService) TrackUnpublished( return } - room := t.getRoomDetails(participantID) + room := toMinimalRoomProto(roomID, roomName) participant := &livekit.ParticipantInfo{ Sid: string(participantID), Identity: string(identity), @@ -370,28 +418,34 @@ func (t *telemetryService) TrackUnpublished( func (t *telemetryService) TrackMuted( ctx context.Context, + roomID livekit.RoomID, + roomName livekit.RoomName, participantID livekit.ParticipantID, track *livekit.TrackInfo, ) { t.enqueue(func() { - room := t.getRoomDetails(participantID) + room := toMinimalRoomProto(roomID, roomName) t.SendEvent(ctx, newTrackEvent(livekit.AnalyticsEventType_TRACK_MUTED, room, participantID, track)) }) } func (t *telemetryService) TrackUnmuted( ctx context.Context, + roomID livekit.RoomID, + roomName livekit.RoomName, participantID livekit.ParticipantID, track *livekit.TrackInfo, ) { t.enqueue(func() { - room := t.getRoomDetails(participantID) + room := toMinimalRoomProto(roomID, roomName) t.SendEvent(ctx, newTrackEvent(livekit.AnalyticsEventType_TRACK_UNMUTED, room, participantID, track)) }) } func (t *telemetryService) TrackPublishRTPStats( ctx context.Context, + roomID livekit.RoomID, + roomName livekit.RoomName, participantID livekit.ParticipantID, trackID livekit.TrackID, mimeType mime.MimeType, @@ -399,7 +453,7 @@ func (t *telemetryService) TrackPublishRTPStats( stats *livekit.RTPStats, ) { t.enqueue(func() { - room := t.getRoomDetails(participantID) + room := toMinimalRoomProto(roomID, roomName) ev := newRoomEvent(livekit.AnalyticsEventType_TRACK_PUBLISH_STATS, room) ev.ParticipantId = string(participantID) ev.TrackId = string(trackID) @@ -412,13 +466,15 @@ func (t *telemetryService) TrackPublishRTPStats( func (t *telemetryService) TrackSubscribeRTPStats( ctx context.Context, + roomID livekit.RoomID, + roomName livekit.RoomName, participantID livekit.ParticipantID, trackID livekit.TrackID, mimeType mime.MimeType, stats *livekit.RTPStats, ) { t.enqueue(func() { - room := t.getRoomDetails(participantID) + room := toMinimalRoomProto(roomID, roomName) ev := newRoomEvent(livekit.AnalyticsEventType_TRACK_SUBSCRIBE_STATS, room) ev.ParticipantId = string(participantID) ev.TrackId = string(trackID) @@ -428,12 +484,19 @@ func (t *telemetryService) TrackSubscribeRTPStats( }) } +func (t *telemetryService) NotifyEgressEvent(ctx context.Context, event string, info *livekit.EgressInfo) { + opts := egress.GetEgressNotifyOptions(info) + + t.NotifyEvent(ctx, &livekit.WebhookEvent{ + Event: event, + EgressInfo: info, + }, opts...) +} + func (t *telemetryService) EgressStarted(ctx context.Context, info *livekit.EgressInfo) { + t.enqueue(func() { - t.NotifyEvent(ctx, &livekit.WebhookEvent{ - Event: webhook.EventEgressStarted, - EgressInfo: info, - }) + t.NotifyEgressEvent(ctx, webhook.EventEgressStarted, info) t.SendEvent(ctx, newEgressEvent(livekit.AnalyticsEventType_EGRESS_STARTED, info)) }) @@ -441,20 +504,15 @@ func (t *telemetryService) EgressStarted(ctx context.Context, info *livekit.Egre func (t *telemetryService) EgressUpdated(ctx context.Context, info *livekit.EgressInfo) { t.enqueue(func() { - t.NotifyEvent(ctx, &livekit.WebhookEvent{ - Event: webhook.EventEgressUpdated, - EgressInfo: info, - }) + t.NotifyEgressEvent(ctx, webhook.EventEgressUpdated, info) + t.SendEvent(ctx, newEgressEvent(livekit.AnalyticsEventType_EGRESS_UPDATED, info)) }) } func (t *telemetryService) EgressEnded(ctx context.Context, info *livekit.EgressInfo) { t.enqueue(func() { - t.NotifyEvent(ctx, &livekit.WebhookEvent{ - Event: webhook.EventEgressEnded, - EgressInfo: info, - }) + t.NotifyEgressEvent(ctx, webhook.EventEgressEnded, info) t.SendEvent(ctx, newEgressEvent(livekit.AnalyticsEventType_EGRESS_ENDED, info)) }) @@ -533,19 +591,6 @@ func (t *telemetryService) Webhook(ctx context.Context, webhookInfo *livekit.Web }) } -// returns a livekit.Room with only name and sid filled out -// returns nil if room is not found -func (t *telemetryService) getRoomDetails(participantID livekit.ParticipantID) *livekit.Room { - if worker, ok := t.getWorker(participantID); ok { - return &livekit.Room{ - Sid: string(worker.roomID), - Name: string(worker.roomName), - } - } - - return nil -} - func newRoomEvent(event livekit.AnalyticsEventType, room *livekit.Room) *livekit.AnalyticsEvent { ev := &livekit.AnalyticsEvent{ Type: event, @@ -596,3 +641,10 @@ func newIngressEvent(event livekit.AnalyticsEventType, ingress *livekit.IngressI Ingress: ingress, } } + +func toMinimalRoomProto(roomID livekit.RoomID, roomName livekit.RoomName) *livekit.Room { + return &livekit.Room{ + Sid: string(roomID), + Name: string(roomName), + } +} diff --git a/pkg/telemetry/events_test.go b/pkg/telemetry/events_test.go index e83b61b..c5bb062 100644 --- a/pkg/telemetry/events_test.go +++ b/pkg/telemetry/events_test.go @@ -21,6 +21,7 @@ import ( "github.com/stretchr/testify/require" + "github.com/livekit/livekit-server/pkg/telemetry" "github.com/livekit/protocol/livekit" ) @@ -46,9 +47,10 @@ func Test_OnParticipantJoin_EventIsSent(t *testing.T) { ClientConnectTime: 420, } participantInfo := &livekit.ParticipantInfo{Sid: partSID} + guard := &telemetry.ReferenceGuard{} // do - fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, clientInfo, clientMeta, true) + fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, clientInfo, clientMeta, true, guard) time.Sleep(time.Millisecond * 500) // test @@ -81,10 +83,11 @@ func Test_OnParticipantLeft_EventIsSent(t *testing.T) { room := &livekit.Room{Sid: "RoomSid", Name: "RoomName"} partSID := "part1" participantInfo := &livekit.ParticipantInfo{Sid: partSID} + guard := &telemetry.ReferenceGuard{} // do - fixture.sut.ParticipantActive(context.Background(), room, participantInfo, &livekit.AnalyticsClientMeta{}, false) - fixture.sut.ParticipantLeft(context.Background(), room, participantInfo, true) + fixture.sut.ParticipantActive(context.Background(), room, participantInfo, &livekit.AnalyticsClientMeta{}, false, guard) + fixture.sut.ParticipantLeft(context.Background(), room, participantInfo, true, guard) time.Sleep(time.Millisecond * 500) // test @@ -100,6 +103,8 @@ func Test_OnTrackUpdate_EventIsSent(t *testing.T) { fixture := createFixture() // prepare + roomID := "room1" + roomName := "RoomName" partID := "part1" trackID := "track1" layer := &livekit.VideoLayer{ @@ -119,7 +124,7 @@ func Test_OnTrackUpdate_EventIsSent(t *testing.T) { } // do - fixture.sut.TrackPublishedUpdate(context.Background(), livekit.ParticipantID(partID), trackInfo) + fixture.sut.TrackPublishedUpdate(context.Background(), livekit.RoomID(roomID), livekit.RoomName(roomName), livekit.ParticipantID(partID), trackInfo) time.Sleep(time.Millisecond * 500) // test @@ -158,9 +163,10 @@ func Test_OnParticipantActive_EventIsSent(t *testing.T) { ClientAddr: "127.0.0.1", } participantInfo := &livekit.ParticipantInfo{Sid: partSID} + guard := &telemetry.ReferenceGuard{} // do - fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, clientInfo, clientMeta, true) + fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, clientInfo, clientMeta, true, guard) time.Sleep(time.Millisecond * 500) // test @@ -173,7 +179,7 @@ func Test_OnParticipantActive_EventIsSent(t *testing.T) { ClientConnectTime: 420, } - fixture.sut.ParticipantActive(context.Background(), room, participantInfo, clientMetaConnect, false) + fixture.sut.ParticipantActive(context.Background(), room, participantInfo, clientMetaConnect, false, guard) time.Sleep(time.Millisecond * 500) require.Equal(t, 2, fixture.analytics.SendEventCallCount()) @@ -210,9 +216,10 @@ func Test_OnTrackSubscribed_EventIsSent(t *testing.T) { ClientAddr: "127.0.0.1", } participantInfo := &livekit.ParticipantInfo{Sid: partSID} + guard := &telemetry.ReferenceGuard{} // do - fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, clientInfo, clientMeta, true) + fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, clientInfo, clientMeta, true, guard) time.Sleep(time.Millisecond * 500) // test @@ -221,7 +228,7 @@ func Test_OnTrackSubscribed_EventIsSent(t *testing.T) { require.Equal(t, room, event.Room) // do - fixture.sut.TrackSubscribed(context.Background(), livekit.ParticipantID(partSID), trackInfo, publisherInfo, true) + fixture.sut.TrackSubscribed(context.Background(), livekit.RoomID(room.Sid), livekit.RoomName(room.Name), livekit.ParticipantID(partSID), trackInfo, publisherInfo, true) time.Sleep(time.Millisecond * 500) require.Eventually(t, func() bool { diff --git a/pkg/telemetry/prometheus/debug.go b/pkg/telemetry/prometheus/debug.go new file mode 100644 index 0000000..1f74598 --- /dev/null +++ b/pkg/telemetry/prometheus/debug.go @@ -0,0 +1,40 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package prometheus + +import ( + "github.com/prometheus/client_golang/prometheus" + + "github.com/livekit/protocol/livekit" +) + +var ( + refCounts *prometheus.GaugeVec +) + +func initDebugStats(nodeID string, nodeType livekit.NodeType) { + refCounts = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: livekitNamespace, + Subsystem: "debug", + Name: "ref_count", + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, + }, []string{"referrer"}) + + prometheus.MustRegister(refCounts) +} + +func AddRef(referrer string, n int) { + refCounts.WithLabelValues(referrer).Add(float64(n)) +} diff --git a/pkg/telemetry/prometheus/node.go b/pkg/telemetry/prometheus/node.go index e68dc6e..c3a3fc3 100644 --- a/pkg/telemetry/prometheus/node.go +++ b/pkg/telemetry/prometheus/node.go @@ -17,35 +17,34 @@ package prometheus import ( "time" - "github.com/mackerelio/go-osstat/memory" "github.com/prometheus/client_golang/prometheus" + "github.com/twitchtv/twirp" "go.uber.org/atomic" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/rpc" "github.com/livekit/protocol/utils/hwstats" + "github.com/livekit/protocol/webhook" ) const ( livekitNamespace string = "livekit" - - statsUpdateInterval = time.Second * 10 ) var ( initialized atomic.Bool - MessageCounter *prometheus.CounterVec - MessageBytes *prometheus.CounterVec - ServiceOperationCounter *prometheus.CounterVec - TwirpRequestStatusCounter *prometheus.CounterVec + promMessageCounter *prometheus.CounterVec + promServiceOperationCounter *prometheus.CounterVec + promTwirpRequestStatusCounter *prometheus.CounterVec + promTwirpRequestLatency *prometheus.HistogramVec - sysPacketsStart uint32 - sysDroppedPacketsStart uint32 - promSysPacketGauge *prometheus.GaugeVec - promSysDroppedPacketPctGauge prometheus.Gauge + sysPacketsStart uint32 + sysDroppedPacketsStart uint32 + promSysPacketGauge *prometheus.GaugeVec - cpuStats *hwstats.CPUStats + cpuStats *hwstats.CPUStats + memoryStats *hwstats.MemoryStats ) func Init(nodeID string, nodeType livekit.NodeType) error { @@ -53,27 +52,17 @@ func Init(nodeID string, nodeType livekit.NodeType) error { return nil } - MessageCounter = prometheus.NewCounterVec( + promMessageCounter = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: livekitNamespace, Subsystem: "node", Name: "messages", ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, }, - []string{"type", "status"}, + []string{"type", "status", "direction"}, ) - MessageBytes = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Namespace: livekitNamespace, - Subsystem: "node", - Name: "message_bytes", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, - }, - []string{"type", "message_type"}, - ) - - ServiceOperationCounter = prometheus.NewCounterVec( + promServiceOperationCounter = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: livekitNamespace, Subsystem: "node", @@ -83,7 +72,7 @@ func Init(nodeID string, nodeType livekit.NodeType) error { []string{"type", "status", "error_type"}, ) - TwirpRequestStatusCounter = prometheus.NewCounterVec( + promTwirpRequestStatusCounter = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: livekitNamespace, Subsystem: "node", @@ -93,6 +82,17 @@ func Init(nodeID string, nodeType livekit.NodeType) error { []string{"service", "method", "status", "code"}, ) + promTwirpRequestLatency = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: livekitNamespace, + Subsystem: "node", + Name: "twirp_request_latency_ms", + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, + Buckets: []float64{5, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10000, 30000, 60000}, + }, + []string{"service", "method"}, + ) + promSysPacketGauge = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Namespace: livekitNamespace, @@ -104,30 +104,21 @@ func Init(nodeID string, nodeType livekit.NodeType) error { []string{"type"}, ) - promSysDroppedPacketPctGauge = prometheus.NewGauge( - prometheus.GaugeOpts{ - Namespace: livekitNamespace, - Subsystem: "node", - Name: "dropped_packets", - ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, - Help: "System level dropped outgoing packet percentage.", - }, - ) - - prometheus.MustRegister(MessageCounter) - prometheus.MustRegister(MessageBytes) - prometheus.MustRegister(ServiceOperationCounter) - prometheus.MustRegister(TwirpRequestStatusCounter) + prometheus.MustRegister(promMessageCounter) + prometheus.MustRegister(promServiceOperationCounter) + prometheus.MustRegister(promTwirpRequestStatusCounter) + prometheus.MustRegister(promTwirpRequestLatency) prometheus.MustRegister(promSysPacketGauge) - prometheus.MustRegister(promSysDroppedPacketPctGauge) sysPacketsStart, sysDroppedPacketsStart, _ = getTCStats() initPacketStats(nodeID, nodeType) initRoomStats(nodeID, nodeType) rpc.InitPSRPCStats(prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}) + webhook.InitWebhookStats(prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}) initQualityStats(nodeID, nodeType) initDataPacketStats(nodeID, nodeType) + initDebugStats(nodeID, nodeType) var err error cpuStats, err = hwstats.NewCPUStats(nil) @@ -135,145 +126,187 @@ func Init(nodeID string, nodeType livekit.NodeType) error { return err } + memoryStats, err = hwstats.NewMemoryStats() + if err != nil { + return err + } + return nil } -func GetUpdatedNodeStats(prev *livekit.NodeStats, prevAverage *livekit.NodeStats) (*livekit.NodeStats, bool, error) { +func GetNodeStats(nodeStartedAt int64, prevStats []*livekit.NodeStats, rateIntervals []time.Duration) (*livekit.NodeStats, error) { loadAvg, err := getLoadAvg() if err != nil { - return nil, false, err - } - - var cpuLoad float64 - cpuIdle := cpuStats.GetCPUIdle() - if cpuIdle > 0 { - cpuLoad = 1 - (cpuIdle / cpuStats.NumCPU()) + return nil, err } // On MacOS, get "\"vm_stat\": executable file not found in $PATH" although it is in /usr/bin // So, do not error out. Use the information if it is available. - memTotal := uint64(0) - memUsed := uint64(0) - memInfo, _ := memory.Get() - if memInfo != nil { - memTotal = memInfo.Total - memUsed = memInfo.Used - } + memUsed, memTotal, _ := memoryStats.GetMemory() - // do not error out, and use the information if it is available sysPackets, sysDroppedPackets, _ := getTCStats() promSysPacketGauge.WithLabelValues("out").Set(float64(sysPackets - sysPacketsStart)) promSysPacketGauge.WithLabelValues("dropped").Set(float64(sysDroppedPackets - sysDroppedPacketsStart)) - bytesInNow := bytesIn.Load() - bytesOutNow := bytesOut.Load() - packetsInNow := packetsIn.Load() - packetsOutNow := packetsOut.Load() - nackTotalNow := nackTotal.Load() - retransmitBytesNow := retransmitBytes.Load() - retransmitPacketsNow := retransmitPackets.Load() - participantSignalConnectedNow := participantSignalConnected.Load() - participantRTCInitNow := participantRTCInit.Load() - participantRTConnectedCNow := participantRTCConnected.Load() - trackPublishAttemptsNow := trackPublishAttempts.Load() - trackPublishSuccessNow := trackPublishSuccess.Load() - trackSubscribeAttemptsNow := trackSubscribeAttempts.Load() - trackSubscribeSuccessNow := trackSubscribeSuccess.Load() - forwardLatencyNow := forwardLatency.Load() - forwardJitterNow := forwardJitter.Load() - - updatedAt := time.Now().Unix() - elapsed := updatedAt - prevAverage.UpdatedAt - // include sufficient buffer to be sure a stats update had taken place - computeAverage := elapsed > int64(statsUpdateInterval.Seconds()+2) - if bytesInNow != prevAverage.BytesIn || - bytesOutNow != prevAverage.BytesOut || - packetsInNow != prevAverage.PacketsIn || - packetsOutNow != prevAverage.PacketsOut || - retransmitBytesNow != prevAverage.RetransmitBytesOut || - retransmitPacketsNow != prevAverage.RetransmitPacketsOut { - computeAverage = true - } - stats := &livekit.NodeStats{ - StartedAt: prev.StartedAt, - UpdatedAt: updatedAt, - NumRooms: roomCurrent.Load(), - NumClients: participantCurrent.Load(), - NumTracksIn: trackPublishedCurrent.Load(), - NumTracksOut: trackSubscribedCurrent.Load(), - NumTrackPublishAttempts: trackPublishAttemptsNow, - NumTrackPublishSuccess: trackPublishSuccessNow, - NumTrackSubscribeAttempts: trackSubscribeAttemptsNow, - NumTrackSubscribeSuccess: trackSubscribeSuccessNow, - BytesIn: bytesInNow, - BytesOut: bytesOutNow, - PacketsIn: packetsInNow, - PacketsOut: packetsOutNow, - RetransmitBytesOut: retransmitBytesNow, - RetransmitPacketsOut: retransmitPacketsNow, - NackTotal: nackTotalNow, - ParticipantSignalConnected: participantSignalConnectedNow, - ParticipantRtcInit: participantRTCInitNow, - ParticipantRtcConnected: participantRTConnectedCNow, - BytesInPerSec: prevAverage.BytesInPerSec, - BytesOutPerSec: prevAverage.BytesOutPerSec, - PacketsInPerSec: prevAverage.PacketsInPerSec, - PacketsOutPerSec: prevAverage.PacketsOutPerSec, - RetransmitBytesOutPerSec: prevAverage.RetransmitBytesOutPerSec, - RetransmitPacketsOutPerSec: prevAverage.RetransmitPacketsOutPerSec, - NackPerSec: prevAverage.NackPerSec, - ForwardLatency: forwardLatencyNow, - ForwardJitter: forwardJitterNow, - ParticipantSignalConnectedPerSec: prevAverage.ParticipantSignalConnectedPerSec, - ParticipantRtcInitPerSec: prevAverage.ParticipantRtcInitPerSec, - ParticipantRtcConnectedPerSec: prevAverage.ParticipantRtcConnectedPerSec, - NumCpus: uint32(cpuStats.NumCPU()), // this will round down to the nearest integer - CpuLoad: float32(cpuLoad), - MemoryTotal: memTotal, - MemoryUsed: memUsed, - LoadAvgLast1Min: float32(loadAvg.Loadavg1), - LoadAvgLast5Min: float32(loadAvg.Loadavg5), - LoadAvgLast15Min: float32(loadAvg.Loadavg15), - SysPacketsOut: sysPackets, - SysPacketsDropped: sysDroppedPackets, - TrackPublishAttemptsPerSec: prevAverage.TrackPublishAttemptsPerSec, - TrackPublishSuccessPerSec: prevAverage.TrackPublishSuccessPerSec, - TrackSubscribeAttemptsPerSec: prevAverage.TrackSubscribeAttemptsPerSec, - TrackSubscribeSuccessPerSec: prevAverage.TrackSubscribeSuccessPerSec, + StartedAt: nodeStartedAt, + UpdatedAt: time.Now().Unix(), + NumRooms: roomCurrent.Load(), + NumClients: participantCurrent.Load(), + NumTracksIn: trackPublishedCurrent.Load(), + NumTracksOut: trackSubscribedCurrent.Load(), + NumTrackPublishAttempts: trackPublishAttempts.Load(), + NumTrackPublishSuccess: trackPublishSuccess.Load(), + NumTrackPublishCancels: trackPublishCancels.Load(), + NumTrackSubscribeAttempts: trackSubscribeAttempts.Load(), + NumTrackSubscribeSuccess: trackSubscribeSuccess.Load(), + NumTrackSubscribeCancels: trackSubscribeCancels.Load(), + BytesIn: bytesIn.Load(), + BytesOut: bytesOut.Load(), + PacketsIn: packetsIn.Load(), + PacketsOut: packetsOut.Load(), + RetransmitBytesOut: retransmitBytes.Load(), + RetransmitPacketsOut: retransmitPackets.Load(), + NackTotal: nackTotal.Load(), + ParticipantSignalConnected: participantSignalConnected.Load(), + ParticipantSignalFailed: participantSignalFailed.Load(), + ParticipantSignalValidationFailed: participantSignalValidationFailed.Load(), + ParticipantRtcInit: participantRTCInit.Load(), + ParticipantRtcConnected: participantRTCConnected.Load(), + ParticipantRtcCanceled: participantRTCCanceled.Load(), + ParticipantRtcActive: participantRTCActive.Load(), + ForwardLatency: forwardLatency.Load(), + ForwardJitter: forwardJitter.Load(), + NumCpus: uint32(cpuStats.NumCPU()), // this will round down to the nearest integer + CpuLoad: float32(cpuStats.GetCPULoad()), + MemoryTotal: memTotal, + MemoryUsed: memUsed, + LoadAvgLast1Min: float32(loadAvg.Loadavg1), + LoadAvgLast5Min: float32(loadAvg.Loadavg5), + LoadAvgLast15Min: float32(loadAvg.Loadavg15), + SysPacketsOut: sysPackets, + SysPacketsDropped: sysDroppedPackets, } - // update stats - if computeAverage { - stats.BytesInPerSec = perSec(prevAverage.BytesIn, bytesInNow, elapsed) - stats.BytesOutPerSec = perSec(prevAverage.BytesOut, bytesOutNow, elapsed) - stats.PacketsInPerSec = perSec(prevAverage.PacketsIn, packetsInNow, elapsed) - stats.PacketsOutPerSec = perSec(prevAverage.PacketsOut, packetsOutNow, elapsed) - stats.RetransmitBytesOutPerSec = perSec(prevAverage.RetransmitBytesOut, retransmitBytesNow, elapsed) - stats.RetransmitPacketsOutPerSec = perSec(prevAverage.RetransmitPacketsOut, retransmitPacketsNow, elapsed) - stats.NackPerSec = perSec(prevAverage.NackTotal, nackTotalNow, elapsed) - stats.ParticipantSignalConnectedPerSec = perSec(prevAverage.ParticipantSignalConnected, participantSignalConnectedNow, elapsed) - stats.ParticipantRtcInitPerSec = perSec(prevAverage.ParticipantRtcInit, participantRTCInitNow, elapsed) - stats.ParticipantRtcConnectedPerSec = perSec(prevAverage.ParticipantRtcConnected, participantRTConnectedCNow, elapsed) - stats.SysPacketsOutPerSec = perSec(uint64(prevAverage.SysPacketsOut), uint64(sysPackets), elapsed) - stats.SysPacketsDroppedPerSec = perSec(uint64(prevAverage.SysPacketsDropped), uint64(sysDroppedPackets), elapsed) - stats.TrackPublishAttemptsPerSec = perSec(uint64(prevAverage.NumTrackPublishAttempts), uint64(trackPublishAttemptsNow), elapsed) - stats.TrackPublishSuccessPerSec = perSec(uint64(prevAverage.NumTrackPublishSuccess), uint64(trackPublishSuccessNow), elapsed) - stats.TrackSubscribeAttemptsPerSec = perSec(uint64(prevAverage.NumTrackSubscribeAttempts), uint64(trackSubscribeAttemptsNow), elapsed) - stats.TrackSubscribeSuccessPerSec = perSec(uint64(prevAverage.NumTrackSubscribeSuccess), uint64(trackSubscribeSuccessNow), elapsed) + for _, rateInterval := range rateIntervals { + for idx := len(prevStats) - 1; idx >= 0; idx-- { + prev := prevStats[idx] + if prev == nil { + continue + } - packetTotal := stats.SysPacketsOutPerSec + stats.SysPacketsDroppedPerSec - if packetTotal == 0 { - stats.SysPacketsDroppedPctPerSec = 0 - } else { - stats.SysPacketsDroppedPctPerSec = stats.SysPacketsDroppedPerSec / packetTotal + if stats.UpdatedAt-prev.UpdatedAt >= int64(rateInterval.Seconds()) { + if rate := getNodeStatsRate(append(prevStats[idx:], stats)); rate != nil { + stats.Rates = append(stats.Rates, rate) + } + break + } } - promSysDroppedPacketPctGauge.Set(float64(stats.SysPacketsDroppedPctPerSec)) } - return stats, computeAverage, nil + return stats, nil +} + +func getNodeStatsRate(statsHistory []*livekit.NodeStats) *livekit.NodeStatsRate { + if len(statsHistory) == 0 { + return nil + } + + elapsed := statsHistory[len(statsHistory)-1].UpdatedAt - statsHistory[0].UpdatedAt + if elapsed <= 0 { + return nil + } + + // time weighted averages + var cpuLoad, memoryUsed, memoryTotal, memoryLoad float32 + for idx := len(statsHistory) - 1; idx > 0; idx-- { + stats := statsHistory[idx] + prevStats := statsHistory[idx-1] + if stats == nil || prevStats == nil { + continue + } + + spanElapsed := stats.UpdatedAt - prevStats.UpdatedAt + if spanElapsed <= 0 { + continue + } + + cpuLoad += stats.CpuLoad * float32(spanElapsed) + memoryUsed += float32(stats.MemoryUsed) * float32(spanElapsed) + memoryTotal += float32(stats.MemoryTotal) * float32(spanElapsed) + if stats.MemoryTotal > 0 { + memoryLoad += float32(stats.MemoryUsed) / float32(stats.MemoryTotal) * float32(spanElapsed) + } + } + + earlier := statsHistory[0] + later := statsHistory[len(statsHistory)-1] + rate := &livekit.NodeStatsRate{ + StartedAt: earlier.UpdatedAt, + EndedAt: later.UpdatedAt, + Duration: elapsed, + BytesIn: perSec(earlier.BytesIn, later.BytesIn, elapsed), + BytesOut: perSec(earlier.BytesOut, later.BytesOut, elapsed), + PacketsIn: perSec(earlier.PacketsIn, later.PacketsIn, elapsed), + PacketsOut: perSec(earlier.PacketsOut, later.PacketsOut, elapsed), + RetransmitBytesOut: perSec(earlier.RetransmitBytesOut, later.RetransmitBytesOut, elapsed), + RetransmitPacketsOut: perSec(earlier.RetransmitPacketsOut, later.RetransmitPacketsOut, elapsed), + NackTotal: perSec(earlier.NackTotal, later.NackTotal, elapsed), + ParticipantSignalConnected: perSec(earlier.ParticipantSignalConnected, later.ParticipantSignalConnected, elapsed), + ParticipantSignalFailed: perSec(earlier.ParticipantSignalFailed, later.ParticipantSignalFailed, elapsed), + ParticipantSignalValidationFailed: perSec(earlier.ParticipantSignalValidationFailed, later.ParticipantSignalValidationFailed, elapsed), + ParticipantRtcInit: perSec(earlier.ParticipantRtcInit, later.ParticipantRtcInit, elapsed), + ParticipantRtcConnected: perSec(earlier.ParticipantRtcConnected, later.ParticipantRtcConnected, elapsed), + ParticipantRtcCanceled: perSec(earlier.ParticipantRtcCanceled, later.ParticipantRtcCanceled, elapsed), + ParticipantRtcActive: perSec(earlier.ParticipantRtcActive, later.ParticipantRtcActive, elapsed), + SysPacketsOut: perSec(uint64(earlier.SysPacketsOut), uint64(later.SysPacketsOut), elapsed), + SysPacketsDropped: perSec(uint64(earlier.SysPacketsDropped), uint64(later.SysPacketsDropped), elapsed), + TrackPublishAttempts: perSec(uint64(earlier.NumTrackPublishAttempts), uint64(later.NumTrackPublishAttempts), elapsed), + TrackPublishSuccess: perSec(uint64(earlier.NumTrackPublishSuccess), uint64(later.NumTrackPublishSuccess), elapsed), + TrackPublishCancels: perSec(uint64(earlier.NumTrackPublishCancels), uint64(later.NumTrackPublishCancels), elapsed), + TrackSubscribeAttempts: perSec(uint64(earlier.NumTrackSubscribeAttempts), uint64(later.NumTrackSubscribeAttempts), elapsed), + TrackSubscribeSuccess: perSec(uint64(earlier.NumTrackSubscribeSuccess), uint64(later.NumTrackSubscribeSuccess), elapsed), + TrackSubscribeCancels: perSec(uint64(earlier.NumTrackSubscribeCancels), uint64(later.NumTrackSubscribeCancels), elapsed), + CpuLoad: cpuLoad / float32(elapsed), + MemoryLoad: memoryLoad / float32(elapsed), + MemoryUsed: memoryUsed / float32(elapsed), + MemoryTotal: memoryTotal / float32(elapsed), + } + return rate } func perSec(prev, curr uint64, secs int64) float32 { return float32(curr-prev) / float32(secs) } + +func RecordSignalRequestSuccess() { + promMessageCounter.WithLabelValues("signal", "success", "request").Add(1) +} + +func RecordSignalRequestFailure() { + promMessageCounter.WithLabelValues("signal", "failure", "request").Add(1) +} + +func RecordSignalResponseSuccess() { + promMessageCounter.WithLabelValues("signal", "success", "response").Add(1) +} + +func RecordSignalResponseFailure() { + promMessageCounter.WithLabelValues("signal", "failure", "response").Add(1) +} + +func RecordServiceOperationSuccess(op string) { + promServiceOperationCounter.WithLabelValues(op, "success", "").Add(1) +} + +func RecordServiceOperationError(op string, error string) { + promServiceOperationCounter.WithLabelValues(op, "error", error).Add(1) +} + +func RecordTwirpRequestStatus(service string, method string, statusFamily string, code twirp.ErrorCode) { + promTwirpRequestStatusCounter.WithLabelValues(service, method, statusFamily, string(code)).Add(1) +} + +func RecordTwirpRequestLatency(service, method string, duration time.Duration) { + promTwirpRequestLatency.WithLabelValues(service, method).Observe(float64(duration.Milliseconds())) +} diff --git a/pkg/telemetry/prometheus/packets.go b/pkg/telemetry/prometheus/packets.go index ce3f07c..a9f6684 100644 --- a/pkg/telemetry/prometheus/packets.go +++ b/pkg/telemetry/prometheus/packets.go @@ -24,31 +24,40 @@ import ( type Direction string const ( - Incoming Direction = "incoming" - Outgoing Direction = "outgoing" - transmissionInitial = "initial" - transmissionRetransmit = "retransmit" + Incoming Direction = "incoming" + Outgoing Direction = "outgoing" +) + +type TransmissionType string + +const ( + TransmissionInitial TransmissionType = "initial" + TransmissionRetransmit TransmissionType = "retransmit" ) var ( - bytesIn atomic.Uint64 - bytesOut atomic.Uint64 - packetsIn atomic.Uint64 - packetsOut atomic.Uint64 - nackTotal atomic.Uint64 - retransmitBytes atomic.Uint64 - retransmitPackets atomic.Uint64 - participantSignalConnected atomic.Uint64 - participantRTCConnected atomic.Uint64 - participantRTCInit atomic.Uint64 - forwardLatency atomic.Uint32 - forwardJitter atomic.Uint32 + bytesIn atomic.Uint64 + bytesOut atomic.Uint64 + packetsIn atomic.Uint64 + packetsOut atomic.Uint64 + nackTotal atomic.Uint64 + retransmitBytes atomic.Uint64 + retransmitPackets atomic.Uint64 + participantSignalConnected atomic.Uint64 + participantSignalFailed atomic.Uint64 + participantSignalValidationFailed atomic.Uint64 + participantRTCConnected atomic.Uint64 + participantRTCInit atomic.Uint64 + participantRTCCanceled atomic.Uint64 + participantRTCActive atomic.Uint64 + forwardLatency atomic.Uint32 + forwardJitter atomic.Uint32 - promPacketLabels = []string{"direction", "transmission"} + promPacketLabels = []string{"direction", "transmission", "country"} promPacketTotal *prometheus.CounterVec promPacketBytes *prometheus.CounterVec - promRTCPLabels = []string{"direction"} - promStreamLabels = []string{"direction", "source", "type"} + promRTCPLabels = []string{"direction", "country"} + promStreamLabels = []string{"direction", "source", "type", "country"} promNackTotal *prometheus.CounterVec promPliTotal *prometheus.CounterVec promFirTotal *prometheus.CounterVec @@ -62,15 +71,7 @@ var ( promConnections *prometheus.GaugeVec promForwardLatency prometheus.Gauge promForwardJitter prometheus.Gauge - - promPacketTotalIncomingInitial prometheus.Counter - promPacketTotalIncomingRetransmit prometheus.Counter - promPacketTotalOutgoingInitial prometheus.Counter - promPacketTotalOutgoingRetransmit prometheus.Counter - promPacketBytesIncomingInitial prometheus.Counter - promPacketBytesIncomingRetransmit prometheus.Counter - promPacketBytesOutgoingInitial prometheus.Counter - promPacketBytesOutgoingRetransmit prometheus.Counter + promForwardLatencyHist prometheus.Histogram ) func initPacketStats(nodeID string, nodeType livekit.NodeType) { @@ -135,7 +136,6 @@ func initPacketStats(nodeID string, nodeType livekit.NodeType) { Subsystem: "jitter", Name: "us", ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, - // 1ms, 10ms, 30ms, 50ms, 70ms, 100ms, 300ms, 600ms, 1s Buckets: []float64{1000, 10000, 30000, 50000, 70000, 100000, 300000, 600000, 1000000}, }, promStreamLabels) @@ -170,6 +170,25 @@ func initPacketStats(nodeID string, nodeType livekit.NodeType) { Name: "jitter", ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, }) + promForwardLatencyHist = prometheus.NewHistogram(prometheus.HistogramOpts{ + Namespace: livekitNamespace, + Subsystem: "forward_latency", + Name: "ns", + ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()}, + // 50us, 100us, 250us, 500us, 1ms, 2ms, 3ms, 5ms, 10ms, 20ms + Buckets: []float64{ + 50 * 1000, + 100 * 1000, + 250 * 1000, + 500 * 1000, + 1 * 1000 * 1000, + 2 * 1000 * 1000, + 3 * 1000 * 1000, + 5 * 1000 * 1000, + 10 * 1000 * 1000, + 20 * 1000 * 1000, + }, + }) prometheus.MustRegister(promPacketTotal) prometheus.MustRegister(promPacketBytes) @@ -186,31 +205,17 @@ func initPacketStats(nodeID string, nodeType livekit.NodeType) { prometheus.MustRegister(promConnections) prometheus.MustRegister(promForwardLatency) prometheus.MustRegister(promForwardJitter) - - promPacketTotalIncomingInitial = promPacketTotal.WithLabelValues(string(Incoming), transmissionInitial) - promPacketTotalIncomingRetransmit = promPacketTotal.WithLabelValues(string(Incoming), transmissionRetransmit) - promPacketTotalOutgoingInitial = promPacketTotal.WithLabelValues(string(Outgoing), transmissionInitial) - promPacketTotalOutgoingRetransmit = promPacketTotal.WithLabelValues(string(Outgoing), transmissionRetransmit) - promPacketBytesIncomingInitial = promPacketBytes.WithLabelValues(string(Incoming), transmissionInitial) - promPacketBytesIncomingRetransmit = promPacketBytes.WithLabelValues(string(Incoming), transmissionRetransmit) - promPacketBytesOutgoingInitial = promPacketBytes.WithLabelValues(string(Outgoing), transmissionInitial) - promPacketBytesOutgoingRetransmit = promPacketBytes.WithLabelValues(string(Outgoing), transmissionRetransmit) + prometheus.MustRegister(promForwardLatencyHist) } -func IncrementPackets(direction Direction, count uint64, retransmit bool) { - if direction == Incoming { - if retransmit { - promPacketTotalIncomingRetransmit.Add(float64(count)) - } else { - promPacketTotalIncomingInitial.Add(float64(count)) - } +func IncrementPackets(country string, direction Direction, count uint64, retransmit bool) { + var transmission TransmissionType + if retransmit { + transmission = TransmissionRetransmit } else { - if retransmit { - promPacketTotalOutgoingRetransmit.Add(float64(count)) - } else { - promPacketTotalOutgoingInitial.Add(float64(count)) - } + transmission = TransmissionInitial } + promPacketTotal.WithLabelValues(string(direction), string(transmission), country).Add(float64(count)) if direction == Incoming { packetsIn.Add(count) @@ -222,20 +227,14 @@ func IncrementPackets(direction Direction, count uint64, retransmit bool) { } } -func IncrementBytes(direction Direction, count uint64, retransmit bool) { - if direction == Incoming { - if retransmit { - promPacketBytesIncomingRetransmit.Add(float64(count)) - } else { - promPacketBytesIncomingInitial.Add(float64(count)) - } +func IncrementBytes(country string, direction Direction, count uint64, retransmit bool) { + var transmission TransmissionType + if retransmit { + transmission = TransmissionRetransmit } else { - if retransmit { - promPacketBytesOutgoingRetransmit.Add(float64(count)) - } else { - promPacketBytesOutgoingInitial.Add(float64(count)) - } + transmission = TransmissionInitial } + promPacketBytes.WithLabelValues(string(direction), string(transmission), country).Add(float64(count)) if direction == Incoming { bytesIn.Add(count) @@ -247,46 +246,53 @@ func IncrementBytes(direction Direction, count uint64, retransmit bool) { } } -func IncrementRTCP(direction Direction, nack, pli, fir uint32) { +func IncrementRTCP(country string, direction Direction, nack, pli, fir uint32) { if nack > 0 { - promNackTotal.WithLabelValues(string(direction)).Add(float64(nack)) + promNackTotal.WithLabelValues(string(direction), country).Add(float64(nack)) nackTotal.Add(uint64(nack)) } if pli > 0 { - promPliTotal.WithLabelValues(string(direction)).Add(float64(pli)) + promPliTotal.WithLabelValues(string(direction), country).Add(float64(pli)) } if fir > 0 { - promFirTotal.WithLabelValues(string(direction)).Add(float64(fir)) + promFirTotal.WithLabelValues(string(direction), country).Add(float64(fir)) } } -func RecordPacketLoss(direction Direction, trackSource livekit.TrackSource, trackType livekit.TrackType, lost, total uint32) { +func RecordPacketLoss( + country string, + direction Direction, + trackSource livekit.TrackSource, + trackType livekit.TrackType, + lost uint32, + total uint32, +) { if total > 0 { - promPacketLoss.WithLabelValues(string(direction), trackSource.String(), trackType.String()).Observe(float64(lost) / float64(total) * 100) + promPacketLoss.WithLabelValues(string(direction), trackSource.String(), trackType.String(), country).Observe(float64(lost) / float64(total) * 100) } if lost > 0 { - promPacketLossTotal.WithLabelValues(string(direction), trackSource.String(), trackType.String()).Add(float64(lost)) + promPacketLossTotal.WithLabelValues(string(direction), trackSource.String(), trackType.String(), country).Add(float64(lost)) } } -func RecordPacketOutOfOrder(direction Direction, trackSource livekit.TrackSource, trackType livekit.TrackType, ooo, total uint32) { +func RecordPacketOutOfOrder(country string, direction Direction, trackSource livekit.TrackSource, trackType livekit.TrackType, ooo, total uint32) { if total > 0 { - promPacketOutOfOrder.WithLabelValues(string(direction), trackSource.String(), trackType.String()).Observe(float64(ooo) / float64(total) * 100) + promPacketOutOfOrder.WithLabelValues(string(direction), trackSource.String(), trackType.String(), country).Observe(float64(ooo) / float64(total) * 100) } if ooo > 0 { - promPacketOutOfOrderTotal.WithLabelValues(string(direction), trackSource.String(), trackType.String()).Add(float64(ooo)) + promPacketOutOfOrderTotal.WithLabelValues(string(direction), trackSource.String(), trackType.String(), country).Add(float64(ooo)) } } -func RecordJitter(direction Direction, trackSource livekit.TrackSource, trackType livekit.TrackType, jitter uint32) { +func RecordJitter(country string, direction Direction, trackSource livekit.TrackSource, trackType livekit.TrackType, jitter uint32) { if jitter > 0 { - promJitter.WithLabelValues(string(direction), trackSource.String(), trackType.String()).Observe(float64(jitter)) + promJitter.WithLabelValues(string(direction), trackSource.String(), trackType.String(), country).Observe(float64(jitter)) } } -func RecordRTT(direction Direction, trackSource livekit.TrackSource, trackType livekit.TrackType, rtt uint32) { +func RecordRTT(country string, direction Direction, trackSource livekit.TrackSource, trackType livekit.TrackType, rtt uint32) { if rtt > 0 { - promRTT.WithLabelValues(string(direction), trackSource.String(), trackType.String()).Observe(float64(rtt)) + promRTT.WithLabelValues(string(direction), trackSource.String(), trackType.String(), country).Observe(float64(rtt)) } } @@ -297,9 +303,17 @@ func IncrementParticipantJoin(join uint32) { } } -func IncrementParticipantJoinFail(join uint32) { - if join > 0 { - promParticipantJoin.WithLabelValues("signal_failed").Add(float64(join)) +func IncrementParticipantJoinFail(fail uint32) { + if fail > 0 { + participantSignalFailed.Add(uint64(fail)) + promParticipantJoin.WithLabelValues("signal_failed").Add(float64(fail)) + } +} + +func IncrementParticipantJoinValidationFail(validationFail uint32) { + if validationFail > 0 { + participantSignalValidationFailed.Add(uint64(validationFail)) + promParticipantJoin.WithLabelValues("signal_validation_failed").Add(float64(validationFail)) } } @@ -317,6 +331,20 @@ func IncrementParticipantRtcConnected(join uint32) { } } +func IncrementParticipantRtcActive(active uint32) { + if active > 0 { + participantRTCActive.Add(uint64(active)) + promParticipantJoin.WithLabelValues("rtc_active").Add(float64(active)) + } +} + +func IncrementParticipantRtcCanceled(numCancels uint64) { + if numCancels > 0 { + participantRTCCanceled.Add(numCancels) + promParticipantJoin.WithLabelValues("rtc_canceled").Add(float64(numCancels)) + } +} + func AddConnection(direction Direction) { promConnections.WithLabelValues(string(direction)).Add(1) } @@ -325,12 +353,16 @@ func SubConnection(direction Direction) { promConnections.WithLabelValues(string(direction)).Sub(1) } -func RecordForwardLatency(_, latencyAvg uint32) { - forwardLatency.Store(latencyAvg) - promForwardLatency.Set(float64(latencyAvg)) +func RecordForwardLatencySample(forwardLatency int64) { + promForwardLatencyHist.Observe(float64(forwardLatency)) } -func RecordForwardJitter(_, jitterAvg uint32) { - forwardJitter.Store(jitterAvg) - promForwardJitter.Set(float64(jitterAvg)) +func RecordForwardLatency(longTermLatencyAvg uint32) { + forwardLatency.Store(longTermLatencyAvg) + promForwardLatency.Set(float64(longTermLatencyAvg)) +} + +func RecordForwardJitter(longTermJitterAvg uint32) { + forwardJitter.Store(longTermJitterAvg) + promForwardJitter.Set(float64(longTermJitterAvg)) } diff --git a/pkg/telemetry/prometheus/rooms.go b/pkg/telemetry/prometheus/rooms.go index e0d4847..00ec4b1 100644 --- a/pkg/telemetry/prometheus/rooms.go +++ b/pkg/telemetry/prometheus/rooms.go @@ -31,8 +31,10 @@ var ( trackSubscribedCurrent atomic.Int32 trackPublishAttempts atomic.Int32 trackPublishSuccess atomic.Int32 + trackPublishCancels atomic.Int32 trackSubscribeAttempts atomic.Int32 trackSubscribeSuccess atomic.Int32 + trackSubscribeCancels atomic.Int32 // count the number of failures that are due to user error (permissions, track doesn't exist), so we could compute // success rate by subtracting this from total attempts trackSubscribeUserError atomic.Int32 @@ -162,30 +164,67 @@ func SubPublishedTrack(kind string) { trackPublishedCurrent.Dec() } -func AddPublishAttempt(kind string) { +func RecordTrackPublishAttempt(kind string) { trackPublishAttempts.Inc() promTrackPublishCounter.WithLabelValues(kind, "attempt").Inc() } -func AddPublishSuccess(kind string) { +func RecordTrackPublishSuccess(kind string) { trackPublishSuccess.Inc() promTrackPublishCounter.WithLabelValues(kind, "success").Inc() } -func RecordPublishTime(source livekit.TrackSource, trackType livekit.TrackType, d time.Duration, sdk livekit.ClientInfo_SDK, kind livekit.ParticipantInfo_Kind) { - recordPubSubTime(true, source, trackType, d, sdk, kind, 1) +func RecordTrackPublishCancels(kind string, numCancels int32) { + trackPublishCancels.Add(numCancels) + promTrackPublishCounter.WithLabelValues(kind, "cancel").Add(float64(numCancels)) } -func RecordSubscribeTime(source livekit.TrackSource, trackType livekit.TrackType, d time.Duration, sdk livekit.ClientInfo_SDK, kind livekit.ParticipantInfo_Kind, count int) { - recordPubSubTime(false, source, trackType, d, sdk, kind, count) +func RecordPublishTime( + country string, + source livekit.TrackSource, + trackType livekit.TrackType, + d time.Duration, + sdk livekit.ClientInfo_SDK, + kind livekit.ParticipantInfo_Kind, +) { + recordPubSubTime(true, country, source, trackType, d, sdk, kind, 1) } -func recordPubSubTime(isPublish bool, source livekit.TrackSource, trackType livekit.TrackType, d time.Duration, sdk livekit.ClientInfo_SDK, kind livekit.ParticipantInfo_Kind, count int) { +func RecordSubscribeTime( + country string, + source livekit.TrackSource, + trackType livekit.TrackType, + d time.Duration, + sdk livekit.ClientInfo_SDK, + kind livekit.ParticipantInfo_Kind, + count int, +) { + recordPubSubTime(false, country, source, trackType, d, sdk, kind, count) +} + +func recordPubSubTime( + isPublish bool, + country string, + source livekit.TrackSource, + trackType livekit.TrackType, + d time.Duration, + sdk livekit.ClientInfo_SDK, + kind livekit.ParticipantInfo_Kind, + count int, +) { direction := "subscribe" if isPublish { direction = "publish" } - promPubSubTime.WithLabelValues(direction, source.String(), trackType.String(), sdk.String(), kind.String(), strconv.Itoa(count)).Observe(float64(d.Milliseconds())) + promPubSubTime.WithLabelValues( + direction, + source.String(), + trackType.String(), + country, + sdk.String(), + kind.String(), + strconv.Itoa(count), + ).Observe(float64(d.Milliseconds())) } func RecordTrackSubscribeSuccess(kind string) { @@ -214,9 +253,15 @@ func RecordTrackSubscribeFailure(err error, isUserError bool) { if isUserError { trackSubscribeUserError.Inc() + trackSubscribeCancels.Inc() } } +func RecordTrackSubscribeCancels(numCancels int32) { + trackSubscribeCancels.Add(numCancels) + promTrackSubscribeCounter.WithLabelValues("cancel", "").Add(float64(numCancels)) +} + func RecordSessionStartTime(protocolVersion int, d time.Duration) { promSessionStartTime.WithLabelValues(strconv.Itoa(protocolVersion)).Observe(float64(d.Milliseconds())) } diff --git a/pkg/telemetry/signalanddatastats.go b/pkg/telemetry/signalanddatastats.go deleted file mode 100644 index 2c728f0..0000000 --- a/pkg/telemetry/signalanddatastats.go +++ /dev/null @@ -1,205 +0,0 @@ -// Copyright 2023 LiveKit, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package telemetry - -import ( - "context" - "fmt" - "sync" - "time" - - "github.com/frostbyte73/core" - "go.uber.org/atomic" - - "github.com/livekit/protocol/livekit" - "github.com/livekit/protocol/utils" -) - -type BytesTrackType string - -const ( - BytesTrackTypeData BytesTrackType = "DT" - BytesTrackTypeSignal BytesTrackType = "SG" -) - -// ------------------------------- - -type TrafficTotals struct { - At time.Time - SendBytes uint64 - SendMessages uint32 - RecvBytes uint64 - RecvMessages uint32 -} - -// -------------------------------- - -// stats for signal and data channel -type BytesTrackStats struct { - trackID livekit.TrackID - pID livekit.ParticipantID - send, recv atomic.Uint64 - sendMessages, recvMessages atomic.Uint32 - totalSendBytes, totalRecvBytes atomic.Uint64 - totalSendMessages, totalRecvMessages atomic.Uint32 - telemetry TelemetryService - done core.Fuse -} - -func NewBytesTrackStats(trackID livekit.TrackID, pID livekit.ParticipantID, telemetry TelemetryService) *BytesTrackStats { - s := &BytesTrackStats{ - trackID: trackID, - pID: pID, - telemetry: telemetry, - } - go s.reporter() - return s -} - -func (s *BytesTrackStats) AddBytes(bytes uint64, isSend bool) { - if isSend { - s.send.Add(bytes) - s.sendMessages.Inc() - s.totalSendBytes.Add(bytes) - s.totalSendMessages.Inc() - } else { - s.recv.Add(bytes) - s.recvMessages.Inc() - s.totalRecvBytes.Add(bytes) - s.totalRecvMessages.Inc() - } -} - -func (s *BytesTrackStats) GetTrafficTotals() *TrafficTotals { - return &TrafficTotals{ - At: time.Now(), - SendBytes: s.totalSendBytes.Load(), - SendMessages: s.totalSendMessages.Load(), - RecvBytes: s.totalRecvBytes.Load(), - RecvMessages: s.totalRecvMessages.Load(), - } -} - -func (s *BytesTrackStats) Stop() { - s.done.Break() -} - -func (s *BytesTrackStats) report() { - if recv := s.recv.Swap(0); recv > 0 { - s.telemetry.TrackStats(StatsKeyForData(livekit.StreamType_UPSTREAM, s.pID, s.trackID), &livekit.AnalyticsStat{ - Streams: []*livekit.AnalyticsStream{ - { - PrimaryBytes: recv, - PrimaryPackets: s.recvMessages.Swap(0), - }, - }, - }) - } - - if send := s.send.Swap(0); send > 0 { - s.telemetry.TrackStats(StatsKeyForData(livekit.StreamType_DOWNSTREAM, s.pID, s.trackID), &livekit.AnalyticsStat{ - Streams: []*livekit.AnalyticsStream{ - { - PrimaryBytes: send, - PrimaryPackets: s.sendMessages.Swap(0), - }, - }, - }) - } -} - -func (s *BytesTrackStats) reporter() { - ticker := time.NewTicker(telemetryNonMediaStatsUpdateInterval) - defer func() { - ticker.Stop() - s.report() - }() - - for { - select { - case <-s.done.Watch(): - return - case <-ticker.C: - s.report() - } - } -} - -// ----------------------------------------------------------------------- - -type BytesSignalStats struct { - BytesTrackStats - ctx context.Context - - mu sync.Mutex - ri *livekit.Room - pi *livekit.ParticipantInfo -} - -func NewBytesSignalStats(ctx context.Context, telemetry TelemetryService) *BytesSignalStats { - return &BytesSignalStats{ - BytesTrackStats: BytesTrackStats{ - telemetry: telemetry, - }, - ctx: ctx, - } -} - -func (s *BytesSignalStats) ResolveRoom(ri *livekit.Room) { - s.mu.Lock() - defer s.mu.Unlock() - if s.ri == nil && ri.GetSid() != "" { - s.ri = &livekit.Room{ - Sid: ri.Sid, - Name: ri.Name, - } - s.maybeStart() - } -} - -func (s *BytesSignalStats) ResolveParticipant(pi *livekit.ParticipantInfo) { - s.mu.Lock() - defer s.mu.Unlock() - if s.pi == nil && pi != nil { - s.pi = &livekit.ParticipantInfo{ - Sid: pi.Sid, - Identity: pi.Identity, - } - s.maybeStart() - } -} - -func (s *BytesSignalStats) maybeStart() { - if s.ri == nil || s.pi == nil { - return - } - - s.pID = livekit.ParticipantID(s.pi.Sid) - s.trackID = BytesTrackIDForParticipantID(BytesTrackTypeSignal, s.pID) - - s.telemetry.ParticipantJoined(s.ctx, s.ri, s.pi, nil, nil, false) - go s.reporter() -} - -func (s *BytesSignalStats) reporter() { - s.BytesTrackStats.reporter() - s.telemetry.ParticipantLeft(s.ctx, s.ri, s.pi, false) -} - -// ----------------------------------------------------------------------- - -func BytesTrackIDForParticipantID(typ BytesTrackType, participantID livekit.ParticipantID) livekit.TrackID { - return livekit.TrackID(fmt.Sprintf("%s_%s%s", utils.TrackPrefix, string(typ), participantID)) -} diff --git a/pkg/telemetry/stats.go b/pkg/telemetry/stats.go index b24b044..56dc530 100644 --- a/pkg/telemetry/stats.go +++ b/pkg/telemetry/stats.go @@ -20,6 +20,7 @@ import ( ) type StatsKey struct { + country string streamType livekit.StreamType participantID livekit.ParticipantID trackID livekit.TrackID @@ -28,8 +29,16 @@ type StatsKey struct { track bool } -func StatsKeyForTrack(streamType livekit.StreamType, participantID livekit.ParticipantID, trackID livekit.TrackID, trackSource livekit.TrackSource, trackType livekit.TrackType) StatsKey { +func StatsKeyForTrack( + country string, + streamType livekit.StreamType, + participantID livekit.ParticipantID, + trackID livekit.TrackID, + trackSource livekit.TrackSource, + trackType livekit.TrackType, +) StatsKey { return StatsKey{ + country: country, streamType: streamType, participantID: participantID, trackID: trackID, @@ -39,15 +48,21 @@ func StatsKeyForTrack(streamType livekit.StreamType, participantID livekit.Parti } } -func StatsKeyForData(streamType livekit.StreamType, participantID livekit.ParticipantID, trackID livekit.TrackID) StatsKey { +func StatsKeyForData( + country string, + streamType livekit.StreamType, + participantID livekit.ParticipantID, + trackID livekit.TrackID, +) StatsKey { return StatsKey{ + country: country, streamType: streamType, participantID: participantID, trackID: trackID, } } -func (t *telemetryService) TrackStats(key StatsKey, stat *livekit.AnalyticsStat) { +func (t *telemetryService) TrackStats(roomID livekit.RoomID, _roomName livekit.RoomName, key StatsKey, stat *livekit.AnalyticsStat) { t.enqueue(func() { direction := prometheus.Incoming if key.streamType == livekit.StreamType_DOWNSTREAM { @@ -76,23 +91,23 @@ func (t *telemetryService) TrackStats(key StatsKey, stat *livekit.AnalyticsStat) bytes += stream.RetransmitBytes } if key.track { - prometheus.RecordPacketLoss(direction, key.trackSource, key.trackType, stream.PacketsLost, stream.PrimaryPackets+stream.PaddingPackets) - prometheus.RecordPacketOutOfOrder(direction, key.trackSource, key.trackType, stream.PacketsOutOfOrder, stream.PrimaryPackets+stream.PaddingPackets) - prometheus.RecordRTT(direction, key.trackSource, key.trackType, stream.Rtt) - prometheus.RecordJitter(direction, key.trackSource, key.trackType, stream.Jitter) + prometheus.RecordPacketLoss(key.country, direction, key.trackSource, key.trackType, stream.PacketsLost, stream.PrimaryPackets+stream.PaddingPackets) + prometheus.RecordPacketOutOfOrder(key.country, direction, key.trackSource, key.trackType, stream.PacketsOutOfOrder, stream.PrimaryPackets+stream.PaddingPackets) + prometheus.RecordRTT(key.country, direction, key.trackSource, key.trackType, stream.Rtt) + prometheus.RecordJitter(key.country, direction, key.trackSource, key.trackType, stream.Jitter) } } - prometheus.IncrementRTCP(direction, nacks, plis, firs) - prometheus.IncrementPackets(direction, uint64(packets), false) - prometheus.IncrementBytes(direction, bytes, false) + prometheus.IncrementRTCP(key.country, direction, nacks, plis, firs) + prometheus.IncrementPackets(key.country, direction, uint64(packets), false) + prometheus.IncrementBytes(key.country, direction, bytes, false) if retransmitPackets != 0 { - prometheus.IncrementPackets(direction, uint64(retransmitPackets), true) + prometheus.IncrementPackets(key.country, direction, uint64(retransmitPackets), true) } if retransmitBytes != 0 { - prometheus.IncrementBytes(direction, retransmitBytes, true) + prometheus.IncrementBytes(key.country, direction, retransmitBytes, true) } - if worker, ok := t.getWorker(key.participantID); ok { + if worker, ok := t.getWorker(roomID, key.participantID); ok { worker.OnTrackStat(key.trackID, key.streamType, stat) } }) diff --git a/pkg/telemetry/stats_test.go b/pkg/telemetry/stats_test.go index 89051ee..eb70d04 100644 --- a/pkg/telemetry/stats_test.go +++ b/pkg/telemetry/stats_test.go @@ -52,12 +52,13 @@ func Test_ParticipantAndRoomDataAreSentWithAnalytics(t *testing.T) { partSID := livekit.ParticipantID("part1") clientInfo := &livekit.ClientInfo{Sdk: 2} participantInfo := &livekit.ParticipantInfo{Sid: string(partSID)} - fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, clientInfo, nil, true) + guard := &telemetry.ReferenceGuard{} + fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, clientInfo, nil, true, guard) // do packet := 33 stat := &livekit.AnalyticsStat{Streams: []*livekit.AnalyticsStream{{PrimaryBytes: uint64(packet)}}} - fixture.sut.TrackStats(telemetry.StatsKeyForData(livekit.StreamType_DOWNSTREAM, partSID, ""), stat) + fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_DOWNSTREAM, partSID, ""), stat) // flush fixture.flush() @@ -76,11 +77,12 @@ func Test_OnDownstreamPackets(t *testing.T) { fixture := createFixture() // prepare - room := &livekit.Room{} + room := &livekit.Room{Sid: "RoomSid", Name: "RoomName"} partSID := livekit.ParticipantID("part1") clientInfo := &livekit.ClientInfo{Sdk: 2} participantInfo := &livekit.ParticipantInfo{Sid: string(partSID)} - fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, clientInfo, nil, true) + guard := &telemetry.ReferenceGuard{} + fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, clientInfo, nil, true, guard) // do packets := []int{33, 23} @@ -89,7 +91,7 @@ func Test_OnDownstreamPackets(t *testing.T) { trackID := livekit.TrackID("trackID") for i := range packets { stat := &livekit.AnalyticsStat{Streams: []*livekit.AnalyticsStream{{PrimaryBytes: uint64(packets[i]), PrimaryPackets: uint32(1)}}} - fixture.sut.TrackStats(telemetry.StatsKeyForData(livekit.StreamType_DOWNSTREAM, partSID, trackID), stat) + fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_DOWNSTREAM, partSID, trackID), stat) } // flush @@ -109,22 +111,23 @@ func Test_OnDownstreamPackets_SeveralTracks(t *testing.T) { fixture := createFixture() // prepare - room := &livekit.Room{} + room := &livekit.Room{Sid: "RoomSid", Name: "RoomName"} partSID := livekit.ParticipantID("part1") clientInfo := &livekit.ClientInfo{Sdk: 2} participantInfo := &livekit.ParticipantInfo{Sid: string(partSID)} - fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, clientInfo, nil, true) + guard := &telemetry.ReferenceGuard{} + fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, clientInfo, nil, true, guard) // do packet1 := 33 trackID1 := livekit.TrackID("trackID1") stat1 := &livekit.AnalyticsStat{Streams: []*livekit.AnalyticsStream{{PrimaryBytes: uint64(packet1), PrimaryPackets: 1}}} - fixture.sut.TrackStats(telemetry.StatsKeyForData(livekit.StreamType_DOWNSTREAM, partSID, trackID1), stat1) + fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_DOWNSTREAM, partSID, trackID1), stat1) packet2 := 23 trackID2 := livekit.TrackID("trackID2") stat2 := &livekit.AnalyticsStat{Streams: []*livekit.AnalyticsStream{{PrimaryBytes: uint64(packet2), PrimaryPackets: 1}}} - fixture.sut.TrackStats(telemetry.StatsKeyForData(livekit.StreamType_DOWNSTREAM, partSID, trackID2), stat2) + fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_DOWNSTREAM, partSID, trackID2), stat2) // flush fixture.flush() @@ -155,10 +158,11 @@ func Test_OnDownStreamStat(t *testing.T) { fixture := createFixture() // prepare - room := &livekit.Room{} + room := &livekit.Room{Sid: "RoomSid", Name: "RoomName"} partSID := livekit.ParticipantID("part1") participantInfo := &livekit.ParticipantInfo{Sid: string(partSID)} - fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, nil, nil, true) + guard := &telemetry.ReferenceGuard{} + fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, nil, nil, true, guard) // do stat1 := &livekit.AnalyticsStat{ @@ -175,7 +179,7 @@ func Test_OnDownStreamStat(t *testing.T) { }, } trackID := livekit.TrackID("trackID1") - fixture.sut.TrackStats(telemetry.StatsKeyForData(livekit.StreamType_DOWNSTREAM, partSID, trackID), stat1) + fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_DOWNSTREAM, partSID, trackID), stat1) stat2 := &livekit.AnalyticsStat{ Streams: []*livekit.AnalyticsStream{ @@ -191,7 +195,7 @@ func Test_OnDownStreamStat(t *testing.T) { }, }, } - fixture.sut.TrackStats(telemetry.StatsKeyForData(livekit.StreamType_DOWNSTREAM, partSID, trackID), stat2) + fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_DOWNSTREAM, partSID, trackID), stat2) // flush fixture.flush() @@ -214,10 +218,11 @@ func Test_PacketLostDiffShouldBeSentToTelemetry(t *testing.T) { fixture := createFixture() // prepare - room := &livekit.Room{} + room := &livekit.Room{Sid: "RoomSid", Name: "RoomName"} partSID := livekit.ParticipantID("part1") participantInfo := &livekit.ParticipantInfo{Sid: string(partSID)} - fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, nil, nil, true) + guard := &telemetry.ReferenceGuard{} + fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, nil, nil, true, guard) // do trackID := livekit.TrackID("trackID1") @@ -230,7 +235,7 @@ func Test_PacketLostDiffShouldBeSentToTelemetry(t *testing.T) { }, }, } - fixture.sut.TrackStats(telemetry.StatsKeyForData(livekit.StreamType_DOWNSTREAM, partSID, trackID), stat1) // there should be bytes reported so that stats are sent + fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_DOWNSTREAM, partSID, trackID), stat1) // there should be bytes reported so that stats are sent // flush fixture.flush() @@ -244,7 +249,7 @@ func Test_PacketLostDiffShouldBeSentToTelemetry(t *testing.T) { }, }, } - fixture.sut.TrackStats(telemetry.StatsKeyForData(livekit.StreamType_DOWNSTREAM, partSID, trackID), stat2) + fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_DOWNSTREAM, partSID, trackID), stat2) // flush fixture.flush() @@ -266,10 +271,11 @@ func Test_OnDownStreamRTCP_SeveralTracks(t *testing.T) { fixture := createFixture() // prepare - room := &livekit.Room{} + room := &livekit.Room{Sid: "RoomSid", Name: "RoomName"} partSID := livekit.ParticipantID("part1") participantInfo := &livekit.ParticipantInfo{Sid: string(partSID)} - fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, nil, nil, true) + guard := &telemetry.ReferenceGuard{} + fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, nil, nil, true, guard) // do trackID1 := livekit.TrackID("trackID1") @@ -282,7 +288,7 @@ func Test_OnDownStreamRTCP_SeveralTracks(t *testing.T) { }, }, } - fixture.sut.TrackStats(telemetry.StatsKeyForData(livekit.StreamType_DOWNSTREAM, partSID, trackID1), stat1) // there should be bytes reported so that stats are sent + fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_DOWNSTREAM, partSID, trackID1), stat1) // there should be bytes reported so that stats are sent stat2 := &livekit.AnalyticsStat{ Streams: []*livekit.AnalyticsStream{ @@ -293,7 +299,7 @@ func Test_OnDownStreamRTCP_SeveralTracks(t *testing.T) { }, }, } - fixture.sut.TrackStats(telemetry.StatsKeyForData(livekit.StreamType_DOWNSTREAM, partSID, trackID1), stat2) + fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_DOWNSTREAM, partSID, trackID1), stat2) stat3 := &livekit.AnalyticsStat{ Streams: []*livekit.AnalyticsStream{ @@ -304,7 +310,7 @@ func Test_OnDownStreamRTCP_SeveralTracks(t *testing.T) { }, }, } - fixture.sut.TrackStats(telemetry.StatsKeyForData(livekit.StreamType_DOWNSTREAM, partSID, trackID2), stat3) + fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_DOWNSTREAM, partSID, trackID2), stat3) // flush fixture.flush() @@ -335,10 +341,11 @@ func Test_OnUpstreamStat(t *testing.T) { fixture := createFixture() // prepare - room := &livekit.Room{} + room := &livekit.Room{Sid: "RoomSid", Name: "RoomName"} partSID := livekit.ParticipantID("part1") participantInfo := &livekit.ParticipantInfo{Sid: string(partSID)} - fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, nil, nil, true) + guard := &telemetry.ReferenceGuard{} + fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, nil, nil, true, guard) // do stat1 := &livekit.AnalyticsStat{ @@ -357,7 +364,7 @@ func Test_OnUpstreamStat(t *testing.T) { } trackID := livekit.TrackID("trackID") - fixture.sut.TrackStats(telemetry.StatsKeyForData(livekit.StreamType_UPSTREAM, partSID, trackID), stat1) + fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_UPSTREAM, partSID, trackID), stat1) stat2 := &livekit.AnalyticsStat{ Streams: []*livekit.AnalyticsStream{ @@ -373,7 +380,7 @@ func Test_OnUpstreamStat(t *testing.T) { }, }, } - fixture.sut.TrackStats(telemetry.StatsKeyForData(livekit.StreamType_UPSTREAM, partSID, trackID), stat2) + fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_UPSTREAM, partSID, trackID), stat2) // flush fixture.flush() @@ -396,11 +403,12 @@ func Test_OnUpstreamRTCP_SeveralTracks(t *testing.T) { fixture := createFixture() // prepare - room := &livekit.Room{} + room := &livekit.Room{Sid: "RoomSid", Name: "RoomName"} partSID := livekit.ParticipantID("part1") identity := livekit.ParticipantIdentity("part1Identity") participantInfo := &livekit.ParticipantInfo{Sid: string(partSID), Identity: string(identity)} - fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, nil, nil, true) + guard := &telemetry.ReferenceGuard{} + fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, nil, nil, true, guard) // there should be bytes reported so that stats are sent totalBytes := 1 @@ -416,8 +424,8 @@ func Test_OnUpstreamRTCP_SeveralTracks(t *testing.T) { }, }, } - fixture.sut.TrackStats(telemetry.StatsKeyForData(livekit.StreamType_UPSTREAM, partSID, trackID1), stat1) - fixture.sut.TrackStats(telemetry.StatsKeyForData(livekit.StreamType_UPSTREAM, partSID, trackID2), stat1) // using same buffer is not correct but for test it is fine + fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_UPSTREAM, partSID, trackID1), stat1) + fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_UPSTREAM, partSID, trackID2), stat1) // using same buffer is not correct but for test it is fine // do totalBytes++ @@ -431,7 +439,7 @@ func Test_OnUpstreamRTCP_SeveralTracks(t *testing.T) { }, }, } - fixture.sut.TrackStats(telemetry.StatsKeyForData(livekit.StreamType_UPSTREAM, partSID, trackID1), stat2) + fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_UPSTREAM, partSID, trackID1), stat2) stat3 := &livekit.AnalyticsStat{ Streams: []*livekit.AnalyticsStream{ @@ -442,7 +450,7 @@ func Test_OnUpstreamRTCP_SeveralTracks(t *testing.T) { }, }, } - fixture.sut.TrackStats(telemetry.StatsKeyForData(livekit.StreamType_UPSTREAM, partSID, trackID2), stat3) + fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_UPSTREAM, partSID, trackID2), stat3) // flush fixture.flush() @@ -471,7 +479,7 @@ func Test_OnUpstreamRTCP_SeveralTracks(t *testing.T) { require.True(t, found2) // remove 1 track - track stats were flushed above, so no more calls to SendStats - fixture.sut.TrackUnpublished(context.Background(), partSID, identity, &livekit.TrackInfo{Sid: string(trackID2)}, true) + fixture.sut.TrackUnpublished(context.Background(), livekit.RoomID(room.Sid), livekit.RoomName(room.Name), partSID, identity, &livekit.TrackInfo{Sid: string(trackID2)}, true) // flush fixture.flush() @@ -486,10 +494,11 @@ func Test_AnalyticsSentWhenParticipantLeaves(t *testing.T) { room := &livekit.Room{} partSID := "part1" participantInfo := &livekit.ParticipantInfo{Sid: partSID} - fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, nil, nil, true) + guard := &telemetry.ReferenceGuard{} + fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, nil, nil, true, guard) // do - fixture.sut.ParticipantLeft(context.Background(), room, participantInfo, true) + fixture.sut.ParticipantLeft(context.Background(), room, participantInfo, true, guard) // should not be called if there are no track stats time.Sleep(time.Millisecond * 500) @@ -500,10 +509,11 @@ func Test_AddUpTrack(t *testing.T) { fixture := createFixture() // prepare - room := &livekit.Room{} + room := &livekit.Room{Sid: "RoomSid", Name: "RoomName"} partSID := livekit.ParticipantID("part1") participantInfo := &livekit.ParticipantInfo{Sid: string(partSID)} - fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, nil, nil, true) + guard := &telemetry.ReferenceGuard{} + fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, nil, nil, true, guard) // do var totalBytes uint64 = 3 @@ -518,7 +528,7 @@ func Test_AddUpTrack(t *testing.T) { }, } trackID := livekit.TrackID("trackID") - fixture.sut.TrackStats(telemetry.StatsKeyForData(livekit.StreamType_UPSTREAM, partSID, trackID), stat) + fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_UPSTREAM, partSID, trackID), stat) // flush fixture.flush() @@ -537,10 +547,11 @@ func Test_AddUpTrack_SeveralBuffers_Simulcast(t *testing.T) { fixture := createFixture() // prepare - room := &livekit.Room{} + room := &livekit.Room{Sid: "RoomSid", Name: "RoomName"} partSID := livekit.ParticipantID("part1") participantInfo := &livekit.ParticipantInfo{Sid: string(partSID)} - fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, nil, nil, true) + guard := &telemetry.ReferenceGuard{} + fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, nil, nil, true, guard) // do trackID := livekit.TrackID("trackID") @@ -556,7 +567,7 @@ func Test_AddUpTrack_SeveralBuffers_Simulcast(t *testing.T) { }, }, } - fixture.sut.TrackStats(telemetry.StatsKeyForData(livekit.StreamType_UPSTREAM, partSID, trackID), stat1) + fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_UPSTREAM, partSID, trackID), stat1) // flush fixture.flush() @@ -576,10 +587,11 @@ func Test_BothDownstreamAndUpstreamStatsAreSentTogether(t *testing.T) { fixture := createFixture() // prepare - room := &livekit.Room{} + room := &livekit.Room{Sid: "RoomSid", Name: "RoomName"} partSID := livekit.ParticipantID("part1") participantInfo := &livekit.ParticipantInfo{Sid: string(partSID)} - fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, nil, nil, true) + guard := &telemetry.ReferenceGuard{} + fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, nil, nil, true, guard) // do // upstream bytes @@ -591,7 +603,7 @@ func Test_BothDownstreamAndUpstreamStatsAreSentTogether(t *testing.T) { }, }, } - fixture.sut.TrackStats(telemetry.StatsKeyForData(livekit.StreamType_UPSTREAM, partSID, "trackID"), stat1) + fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_UPSTREAM, partSID, "trackID"), stat1) // downstream bytes stat2 := &livekit.AnalyticsStat{ Streams: []*livekit.AnalyticsStream{ @@ -601,7 +613,7 @@ func Test_BothDownstreamAndUpstreamStatsAreSentTogether(t *testing.T) { }, }, } - fixture.sut.TrackStats(telemetry.StatsKeyForData(livekit.StreamType_DOWNSTREAM, partSID, "trackID1"), stat2) + fixture.sut.TrackStats(livekit.RoomID(room.Sid), livekit.RoomName(room.Name), telemetry.StatsKeyForData("test", livekit.StreamType_DOWNSTREAM, partSID, "trackID1"), stat2) // flush fixture.flush() diff --git a/pkg/telemetry/statsconn.go b/pkg/telemetry/statsconn.go index b03d0bc..5219861 100644 --- a/pkg/telemetry/statsconn.go +++ b/pkg/telemetry/statsconn.go @@ -17,7 +17,7 @@ package telemetry import ( "net" - "github.com/pion/turn/v4" + "github.com/pion/turn/v5" "github.com/livekit/livekit-server/pkg/telemetry/prometheus" ) @@ -52,7 +52,8 @@ func NewConn(c net.Conn, direction prometheus.Direction) *Conn { func (c *Conn) Read(b []byte) (n int, err error) { n, err = c.Conn.Read(b) if n > 0 { - prometheus.IncrementBytes(prometheus.Incoming, uint64(n), false) + prometheus.IncrementBytes("", prometheus.Incoming, uint64(n), false) + prometheus.IncrementPackets("", prometheus.Incoming, 1, false) } return } @@ -60,7 +61,8 @@ func (c *Conn) Read(b []byte) (n int, err error) { func (c *Conn) Write(b []byte) (n int, err error) { n, err = c.Conn.Write(b) if n > 0 { - prometheus.IncrementBytes(prometheus.Outgoing, uint64(n), false) + prometheus.IncrementBytes("", prometheus.Outgoing, uint64(n), false) + prometheus.IncrementPackets("", prometheus.Outgoing, 1, false) } return } @@ -83,8 +85,8 @@ func NewPacketConn(c net.PacketConn, direction prometheus.Direction) *PacketConn func (c *PacketConn) ReadFrom(p []byte) (n int, addr net.Addr, err error) { n, addr, err = c.PacketConn.ReadFrom(p) if n > 0 { - prometheus.IncrementBytes(prometheus.Incoming, uint64(n), false) - prometheus.IncrementPackets(prometheus.Incoming, 1, false) + prometheus.IncrementBytes("", prometheus.Incoming, uint64(n), false) + prometheus.IncrementPackets("", prometheus.Incoming, 1, false) } return } @@ -92,8 +94,8 @@ func (c *PacketConn) ReadFrom(p []byte) (n int, addr net.Addr, err error) { func (c *PacketConn) WriteTo(p []byte, addr net.Addr) (n int, err error) { n, err = c.PacketConn.WriteTo(p, addr) if n > 0 { - prometheus.IncrementBytes(prometheus.Outgoing, uint64(n), false) - prometheus.IncrementPackets(prometheus.Outgoing, 1, false) + prometheus.IncrementBytes("", prometheus.Outgoing, uint64(n), false) + prometheus.IncrementPackets("", prometheus.Outgoing, 1, false) } return } @@ -111,8 +113,8 @@ func NewRelayAddressGenerator(g turn.RelayAddressGenerator) *RelayAddressGenerat return &RelayAddressGenerator{RelayAddressGenerator: g} } -func (g *RelayAddressGenerator) AllocatePacketConn(network string, requestedPort int) (net.PacketConn, net.Addr, error) { - conn, addr, err := g.RelayAddressGenerator.AllocatePacketConn(network, requestedPort) +func (g *RelayAddressGenerator) AllocatePacketConn(c turn.AllocateListenerConfig) (net.PacketConn, net.Addr, error) { + conn, addr, err := g.RelayAddressGenerator.AllocatePacketConn(c) if err != nil { return nil, addr, err } @@ -120,11 +122,20 @@ func (g *RelayAddressGenerator) AllocatePacketConn(network string, requestedPort return NewPacketConn(conn, prometheus.Outgoing), addr, err } -func (g *RelayAddressGenerator) AllocateConn(network string, requestedPort int) (net.Conn, net.Addr, error) { - conn, addr, err := g.RelayAddressGenerator.AllocateConn(network, requestedPort) +func (g *RelayAddressGenerator) AllocateConn(c turn.AllocateConnConfig) (net.Conn, error) { + conn, err := g.RelayAddressGenerator.AllocateConn(c) + if err != nil { + return nil, err + } + + return NewConn(conn, prometheus.Outgoing), err +} + +func (g *RelayAddressGenerator) AllocateListener(c turn.AllocateListenerConfig) (net.Listener, net.Addr, error) { + l, addr, err := g.RelayAddressGenerator.AllocateListener(c) if err != nil { return nil, addr, err } - return NewConn(conn, prometheus.Outgoing), addr, err + return NewListener(l), addr, err } diff --git a/pkg/telemetry/statsworker.go b/pkg/telemetry/statsworker.go index a72ceaf..59429a0 100644 --- a/pkg/telemetry/statsworker.go +++ b/pkg/telemetry/statsworker.go @@ -19,6 +19,7 @@ import ( "sync" "time" + "go.uber.org/zap/zapcore" "google.golang.org/protobuf/types/known/timestamppb" "github.com/livekit/livekit-server/pkg/utils" @@ -27,6 +28,37 @@ import ( protoutils "github.com/livekit/protocol/utils" ) +type ReferenceGuard struct { + activated, released bool +} + +type ReferenceCount struct { + count int +} + +func (s *ReferenceCount) Activate(guard *ReferenceGuard) { + if guard != nil && !guard.activated { + guard.activated = true + s.count++ + } +} + +func (s *ReferenceCount) Release(guard *ReferenceGuard) bool { + if guard == nil || !guard.activated || guard.released { + return false + } + guard.released = true + s.count-- + return s.count == 0 +} + +func (s ReferenceCount) MarshalLogObject(e zapcore.ObjectEncoder) error { + e.AddInt("count", s.count) + return nil +} + +// ---------------------------------------- + // StatsWorker handles participant stats type StatsWorker struct { next *StatsWorker @@ -42,6 +74,7 @@ type StatsWorker struct { lock sync.RWMutex outgoingPerTrack map[livekit.TrackID][]*livekit.AnalyticsStat incomingPerTrack map[livekit.TrackID][]*livekit.AnalyticsStat + refCount ReferenceCount closedAt time.Time } @@ -52,6 +85,7 @@ func newStatsWorker( roomName livekit.RoomName, participantID livekit.ParticipantID, identity livekit.ParticipantIdentity, + guard *ReferenceGuard, ) *StatsWorker { s := &StatsWorker{ ctx: ctx, @@ -63,6 +97,7 @@ func newStatsWorker( outgoingPerTrack: make(map[livekit.TrackID][]*livekit.AnalyticsStat), incomingPerTrack: make(map[livekit.TrackID][]*livekit.AnalyticsStat), } + s.refCount.Activate(guard) return s } @@ -93,7 +128,7 @@ func (s *StatsWorker) IsConnected() bool { return s.isConnected } -func (s *StatsWorker) Flush(now time.Time) bool { +func (s *StatsWorker) Flush(now time.Time, closeWait time.Duration) bool { ts := timestamppb.New(now) s.lock.Lock() @@ -105,7 +140,7 @@ func (s *StatsWorker) Flush(now time.Time) bool { outgoingPerTrack := s.outgoingPerTrack s.outgoingPerTrack = make(map[livekit.TrackID][]*livekit.AnalyticsStat) - closed := !s.closedAt.IsZero() && now.Sub(s.closedAt) > workerCleanupWait + closed := !s.closedAt.IsZero() && now.Sub(s.closedAt) > closeWait s.lock.Unlock() stats = s.collectStats(ts, livekit.StreamType_UPSTREAM, incomingPerTrack, stats) @@ -117,10 +152,14 @@ func (s *StatsWorker) Flush(now time.Time) bool { return closed } -func (s *StatsWorker) Close() bool { +func (s *StatsWorker) Close(guard *ReferenceGuard) bool { s.lock.Lock() defer s.lock.Unlock() + if !s.refCount.Release(guard) { + return false + } + ok := s.closedAt.IsZero() if ok { s.closedAt = time.Now() @@ -128,10 +167,14 @@ func (s *StatsWorker) Close() bool { return ok } -func (s *StatsWorker) Closed() bool { +func (s *StatsWorker) Closed(guard *ReferenceGuard) bool { s.lock.Lock() defer s.lock.Unlock() - return !s.closedAt.IsZero() + if s.closedAt.IsZero() { + s.refCount.Activate(guard) + return false + } + return true } func (s *StatsWorker) collectStats( @@ -157,6 +200,20 @@ func (s *StatsWorker) collectStats( return stats } +func (s *StatsWorker) MarshalLogObject(e zapcore.ObjectEncoder) error { + s.lock.RLock() + defer s.lock.RUnlock() + + e.AddString("room", string(s.roomName)) + e.AddString("roomID", string(s.roomID)) + e.AddString("participant", string(s.participantIdentity)) + e.AddString("participantID", string(s.participantID)) + e.AddBool("isConnected", s.isConnected) + e.AddTime("closedAt", s.closedAt) + e.AddObject("refCount", s.refCount) + return nil +} + // ------------------------------------------------------------------------- // create a single stream and single video layer post aggregation @@ -253,8 +310,9 @@ func coalesce(stats []*livekit.AnalyticsStat) *livekit.AnalyticsStat { stat := &livekit.AnalyticsStat{ MinScore: minScore, - MedianScore: utils.MedianFloat32(scores), + MedianScore: utils.Median(scores), Streams: []*livekit.AnalyticsStream{coalescedStream}, + Mime: stats[len(stats)-1].Mime, // use the latest Mime } numScores := len(scores) if numScores > 0 { @@ -263,6 +321,41 @@ func coalesce(stats []*livekit.AnalyticsStat) *livekit.AnalyticsStat { return stat } +type CondensedStat struct { + StartTime time.Time + EndTime time.Time + Bytes uint64 + Packets uint32 + PacketsLost uint32 + Frames uint32 +} + +func CondenseStat(stat *livekit.AnalyticsStat) (ps CondensedStat, ok bool) { + if ok = isValid(stat); !ok { + return + } + + for _, stream := range stat.Streams { + startTime := stream.StartTime.AsTime() + endTime := stream.EndTime.AsTime() + if ps.StartTime.IsZero() || startTime.Before(ps.StartTime) { + ps.StartTime = startTime + } + if endTime.After(ps.EndTime) { + ps.EndTime = endTime + } + + ps.Bytes += stream.PrimaryBytes + ps.Packets += stream.PrimaryPackets + ps.PacketsLost += stream.PacketsLost + if stream.Frames > ps.Frames { + ps.Frames = stream.Frames + } + } + + return +} + func isValid(stat *livekit.AnalyticsStat) bool { for _, analyticsStream := range stat.Streams { if int32(analyticsStream.PrimaryPackets) < 0 || diff --git a/pkg/telemetry/statsworker_test.go b/pkg/telemetry/statsworker_test.go new file mode 100644 index 0000000..f7ae488 --- /dev/null +++ b/pkg/telemetry/statsworker_test.go @@ -0,0 +1,19 @@ +package telemetry + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestStatsWorker(t *testing.T) { + t.Run("reference counted close works", func(t *testing.T) { + var g0, g1 ReferenceGuard + w := newStatsWorker(t.Context(), nil, "", "", "", "", &g0) + require.False(t, w.Closed(&g1)) + require.False(t, w.Close(&g0)) + require.False(t, w.Closed(&g1)) + require.True(t, w.Close(&g1)) + require.True(t, w.Closed(&g1)) + }) +} diff --git a/pkg/telemetry/telemetryfakes/fake_analytics_service.go b/pkg/telemetry/telemetryfakes/fake_analytics_service.go index 21b2bb6..14c4dea 100644 --- a/pkg/telemetry/telemetryfakes/fake_analytics_service.go +++ b/pkg/telemetry/telemetryfakes/fake_analytics_service.go @@ -7,9 +7,21 @@ import ( "github.com/livekit/livekit-server/pkg/telemetry" "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/observability/roomobs" ) type FakeAnalyticsService struct { + RoomProjectReporterStub func(context.Context) roomobs.ProjectReporter + roomProjectReporterMutex sync.RWMutex + roomProjectReporterArgsForCall []struct { + arg1 context.Context + } + roomProjectReporterReturns struct { + result1 roomobs.ProjectReporter + } + roomProjectReporterReturnsOnCall map[int]struct { + result1 roomobs.ProjectReporter + } SendEventStub func(context.Context, *livekit.AnalyticsEvent) sendEventMutex sync.RWMutex sendEventArgsForCall []struct { @@ -32,6 +44,67 @@ type FakeAnalyticsService struct { invocationsMutex sync.RWMutex } +func (fake *FakeAnalyticsService) RoomProjectReporter(arg1 context.Context) roomobs.ProjectReporter { + fake.roomProjectReporterMutex.Lock() + ret, specificReturn := fake.roomProjectReporterReturnsOnCall[len(fake.roomProjectReporterArgsForCall)] + fake.roomProjectReporterArgsForCall = append(fake.roomProjectReporterArgsForCall, struct { + arg1 context.Context + }{arg1}) + stub := fake.RoomProjectReporterStub + fakeReturns := fake.roomProjectReporterReturns + fake.recordInvocation("RoomProjectReporter", []interface{}{arg1}) + fake.roomProjectReporterMutex.Unlock() + if stub != nil { + return stub(arg1) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeAnalyticsService) RoomProjectReporterCallCount() int { + fake.roomProjectReporterMutex.RLock() + defer fake.roomProjectReporterMutex.RUnlock() + return len(fake.roomProjectReporterArgsForCall) +} + +func (fake *FakeAnalyticsService) RoomProjectReporterCalls(stub func(context.Context) roomobs.ProjectReporter) { + fake.roomProjectReporterMutex.Lock() + defer fake.roomProjectReporterMutex.Unlock() + fake.RoomProjectReporterStub = stub +} + +func (fake *FakeAnalyticsService) RoomProjectReporterArgsForCall(i int) context.Context { + fake.roomProjectReporterMutex.RLock() + defer fake.roomProjectReporterMutex.RUnlock() + argsForCall := fake.roomProjectReporterArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeAnalyticsService) RoomProjectReporterReturns(result1 roomobs.ProjectReporter) { + fake.roomProjectReporterMutex.Lock() + defer fake.roomProjectReporterMutex.Unlock() + fake.RoomProjectReporterStub = nil + fake.roomProjectReporterReturns = struct { + result1 roomobs.ProjectReporter + }{result1} +} + +func (fake *FakeAnalyticsService) RoomProjectReporterReturnsOnCall(i int, result1 roomobs.ProjectReporter) { + fake.roomProjectReporterMutex.Lock() + defer fake.roomProjectReporterMutex.Unlock() + fake.RoomProjectReporterStub = nil + if fake.roomProjectReporterReturnsOnCall == nil { + fake.roomProjectReporterReturnsOnCall = make(map[int]struct { + result1 roomobs.ProjectReporter + }) + } + fake.roomProjectReporterReturnsOnCall[i] = struct { + result1 roomobs.ProjectReporter + }{result1} +} + func (fake *FakeAnalyticsService) SendEvent(arg1 context.Context, arg2 *livekit.AnalyticsEvent) { fake.sendEventMutex.Lock() fake.sendEventArgsForCall = append(fake.sendEventArgsForCall, struct { @@ -139,12 +212,6 @@ func (fake *FakeAnalyticsService) SendStatsArgsForCall(i int) (context.Context, func (fake *FakeAnalyticsService) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() - fake.sendEventMutex.RLock() - defer fake.sendEventMutex.RUnlock() - fake.sendNodeRoomStatesMutex.RLock() - defer fake.sendNodeRoomStatesMutex.RUnlock() - fake.sendStatsMutex.RLock() - defer fake.sendStatsMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} for key, value := range fake.invocations { copiedInvocations[key] = value diff --git a/pkg/telemetry/telemetryfakes/fake_telemetry_service.go b/pkg/telemetry/telemetryfakes/fake_telemetry_service.go index 52ab445..f8eecd7 100644 --- a/pkg/telemetry/telemetryfakes/fake_telemetry_service.go +++ b/pkg/telemetry/telemetryfakes/fake_telemetry_service.go @@ -5,9 +5,10 @@ import ( "context" "sync" - "github.com/livekit/livekit-server/pkg/sfu/mime" "github.com/livekit/livekit-server/pkg/telemetry" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/observability/roomobs" ) type FakeTelemetryService struct { @@ -75,13 +76,14 @@ type FakeTelemetryService struct { arg1 context.Context arg2 *livekit.AnalyticsNodeRooms } - NotifyEventStub func(context.Context, *livekit.WebhookEvent) - notifyEventMutex sync.RWMutex - notifyEventArgsForCall []struct { + NotifyEgressEventStub func(context.Context, string, *livekit.EgressInfo) + notifyEgressEventMutex sync.RWMutex + notifyEgressEventArgsForCall []struct { arg1 context.Context - arg2 *livekit.WebhookEvent + arg2 string + arg3 *livekit.EgressInfo } - ParticipantActiveStub func(context.Context, *livekit.Room, *livekit.ParticipantInfo, *livekit.AnalyticsClientMeta, bool) + ParticipantActiveStub func(context.Context, *livekit.Room, *livekit.ParticipantInfo, *livekit.AnalyticsClientMeta, bool, *telemetry.ReferenceGuard) participantActiveMutex sync.RWMutex participantActiveArgsForCall []struct { arg1 context.Context @@ -89,8 +91,9 @@ type FakeTelemetryService struct { arg3 *livekit.ParticipantInfo arg4 *livekit.AnalyticsClientMeta arg5 bool + arg6 *telemetry.ReferenceGuard } - ParticipantJoinedStub func(context.Context, *livekit.Room, *livekit.ParticipantInfo, *livekit.ClientInfo, *livekit.AnalyticsClientMeta, bool) + ParticipantJoinedStub func(context.Context, *livekit.Room, *livekit.ParticipantInfo, *livekit.ClientInfo, *livekit.AnalyticsClientMeta, bool, *telemetry.ReferenceGuard) participantJoinedMutex sync.RWMutex participantJoinedArgsForCall []struct { arg1 context.Context @@ -99,14 +102,16 @@ type FakeTelemetryService struct { arg4 *livekit.ClientInfo arg5 *livekit.AnalyticsClientMeta arg6 bool + arg7 *telemetry.ReferenceGuard } - ParticipantLeftStub func(context.Context, *livekit.Room, *livekit.ParticipantInfo, bool) + ParticipantLeftStub func(context.Context, *livekit.Room, *livekit.ParticipantInfo, bool, *telemetry.ReferenceGuard) participantLeftMutex sync.RWMutex participantLeftArgsForCall []struct { arg1 context.Context arg2 *livekit.Room arg3 *livekit.ParticipantInfo arg4 bool + arg5 *telemetry.ReferenceGuard } ParticipantResumedStub func(context.Context, *livekit.Room, *livekit.ParticipantInfo, livekit.NodeID, livekit.ReconnectReason) participantResumedMutex sync.RWMutex @@ -129,6 +134,17 @@ type FakeTelemetryService struct { arg1 context.Context arg2 *livekit.Room } + RoomProjectReporterStub func(context.Context) roomobs.ProjectReporter + roomProjectReporterMutex sync.RWMutex + roomProjectReporterArgsForCall []struct { + arg1 context.Context + } + roomProjectReporterReturns struct { + result1 roomobs.ProjectReporter + } + roomProjectReporterReturnsOnCall map[int]struct { + result1 roomobs.ProjectReporter + } RoomStartedStub func(context.Context, *livekit.Room) roomStartedMutex sync.RWMutex roomStartedArgsForCall []struct { @@ -153,118 +169,147 @@ type FakeTelemetryService struct { arg1 context.Context arg2 []*livekit.AnalyticsStat } - TrackMaxSubscribedVideoQualityStub func(context.Context, livekit.ParticipantID, *livekit.TrackInfo, mime.MimeType, livekit.VideoQuality) + TrackMaxSubscribedVideoQualityStub func(context.Context, livekit.RoomID, livekit.RoomName, livekit.ParticipantID, *livekit.TrackInfo, mime.MimeType, livekit.VideoQuality) trackMaxSubscribedVideoQualityMutex sync.RWMutex trackMaxSubscribedVideoQualityArgsForCall []struct { arg1 context.Context - arg2 livekit.ParticipantID - arg3 *livekit.TrackInfo - arg4 mime.MimeType - arg5 livekit.VideoQuality + arg2 livekit.RoomID + arg3 livekit.RoomName + arg4 livekit.ParticipantID + arg5 *livekit.TrackInfo + arg6 mime.MimeType + arg7 livekit.VideoQuality } - TrackMutedStub func(context.Context, livekit.ParticipantID, *livekit.TrackInfo) + TrackMutedStub func(context.Context, livekit.RoomID, livekit.RoomName, livekit.ParticipantID, *livekit.TrackInfo) trackMutedMutex sync.RWMutex trackMutedArgsForCall []struct { arg1 context.Context - arg2 livekit.ParticipantID - arg3 *livekit.TrackInfo + arg2 livekit.RoomID + arg3 livekit.RoomName + arg4 livekit.ParticipantID + arg5 *livekit.TrackInfo } - TrackPublishRTPStatsStub func(context.Context, livekit.ParticipantID, livekit.TrackID, mime.MimeType, int, *livekit.RTPStats) + TrackPublishRTPStatsStub func(context.Context, livekit.RoomID, livekit.RoomName, livekit.ParticipantID, livekit.TrackID, mime.MimeType, int, *livekit.RTPStats) trackPublishRTPStatsMutex sync.RWMutex trackPublishRTPStatsArgsForCall []struct { arg1 context.Context - arg2 livekit.ParticipantID - arg3 livekit.TrackID - arg4 mime.MimeType - arg5 int - arg6 *livekit.RTPStats + arg2 livekit.RoomID + arg3 livekit.RoomName + arg4 livekit.ParticipantID + arg5 livekit.TrackID + arg6 mime.MimeType + arg7 int + arg8 *livekit.RTPStats } - TrackPublishRequestedStub func(context.Context, livekit.ParticipantID, livekit.ParticipantIdentity, *livekit.TrackInfo) + TrackPublishRequestedStub func(context.Context, livekit.RoomID, livekit.RoomName, livekit.ParticipantID, livekit.ParticipantIdentity, *livekit.TrackInfo) trackPublishRequestedMutex sync.RWMutex trackPublishRequestedArgsForCall []struct { arg1 context.Context - arg2 livekit.ParticipantID - arg3 livekit.ParticipantIdentity - arg4 *livekit.TrackInfo + arg2 livekit.RoomID + arg3 livekit.RoomName + arg4 livekit.ParticipantID + arg5 livekit.ParticipantIdentity + arg6 *livekit.TrackInfo } - TrackPublishedStub func(context.Context, livekit.ParticipantID, livekit.ParticipantIdentity, *livekit.TrackInfo) + TrackPublishedStub func(context.Context, livekit.RoomID, livekit.RoomName, livekit.ParticipantID, livekit.ParticipantIdentity, *livekit.TrackInfo, bool) trackPublishedMutex sync.RWMutex trackPublishedArgsForCall []struct { arg1 context.Context - arg2 livekit.ParticipantID - arg3 livekit.ParticipantIdentity - arg4 *livekit.TrackInfo + arg2 livekit.RoomID + arg3 livekit.RoomName + arg4 livekit.ParticipantID + arg5 livekit.ParticipantIdentity + arg6 *livekit.TrackInfo + arg7 bool } - TrackPublishedUpdateStub func(context.Context, livekit.ParticipantID, *livekit.TrackInfo) + TrackPublishedUpdateStub func(context.Context, livekit.RoomID, livekit.RoomName, livekit.ParticipantID, *livekit.TrackInfo) trackPublishedUpdateMutex sync.RWMutex trackPublishedUpdateArgsForCall []struct { arg1 context.Context - arg2 livekit.ParticipantID - arg3 *livekit.TrackInfo + arg2 livekit.RoomID + arg3 livekit.RoomName + arg4 livekit.ParticipantID + arg5 *livekit.TrackInfo } - TrackStatsStub func(telemetry.StatsKey, *livekit.AnalyticsStat) + TrackStatsStub func(livekit.RoomID, livekit.RoomName, telemetry.StatsKey, *livekit.AnalyticsStat) trackStatsMutex sync.RWMutex trackStatsArgsForCall []struct { - arg1 telemetry.StatsKey - arg2 *livekit.AnalyticsStat + arg1 livekit.RoomID + arg2 livekit.RoomName + arg3 telemetry.StatsKey + arg4 *livekit.AnalyticsStat } - TrackSubscribeFailedStub func(context.Context, livekit.ParticipantID, livekit.TrackID, error, bool) + TrackSubscribeFailedStub func(context.Context, livekit.RoomID, livekit.RoomName, livekit.ParticipantID, livekit.TrackID, error, bool) trackSubscribeFailedMutex sync.RWMutex trackSubscribeFailedArgsForCall []struct { arg1 context.Context - arg2 livekit.ParticipantID - arg3 livekit.TrackID - arg4 error - arg5 bool + arg2 livekit.RoomID + arg3 livekit.RoomName + arg4 livekit.ParticipantID + arg5 livekit.TrackID + arg6 error + arg7 bool } - TrackSubscribeRTPStatsStub func(context.Context, livekit.ParticipantID, livekit.TrackID, mime.MimeType, *livekit.RTPStats) + TrackSubscribeRTPStatsStub func(context.Context, livekit.RoomID, livekit.RoomName, livekit.ParticipantID, livekit.TrackID, mime.MimeType, *livekit.RTPStats) trackSubscribeRTPStatsMutex sync.RWMutex trackSubscribeRTPStatsArgsForCall []struct { arg1 context.Context - arg2 livekit.ParticipantID - arg3 livekit.TrackID - arg4 mime.MimeType - arg5 *livekit.RTPStats + arg2 livekit.RoomID + arg3 livekit.RoomName + arg4 livekit.ParticipantID + arg5 livekit.TrackID + arg6 mime.MimeType + arg7 *livekit.RTPStats } - TrackSubscribeRequestedStub func(context.Context, livekit.ParticipantID, *livekit.TrackInfo) + TrackSubscribeRequestedStub func(context.Context, livekit.RoomID, livekit.RoomName, livekit.ParticipantID, *livekit.TrackInfo) trackSubscribeRequestedMutex sync.RWMutex trackSubscribeRequestedArgsForCall []struct { arg1 context.Context - arg2 livekit.ParticipantID - arg3 *livekit.TrackInfo + arg2 livekit.RoomID + arg3 livekit.RoomName + arg4 livekit.ParticipantID + arg5 *livekit.TrackInfo } - TrackSubscribedStub func(context.Context, livekit.ParticipantID, *livekit.TrackInfo, *livekit.ParticipantInfo, bool) + TrackSubscribedStub func(context.Context, livekit.RoomID, livekit.RoomName, livekit.ParticipantID, *livekit.TrackInfo, *livekit.ParticipantInfo, bool) trackSubscribedMutex sync.RWMutex trackSubscribedArgsForCall []struct { arg1 context.Context - arg2 livekit.ParticipantID - arg3 *livekit.TrackInfo - arg4 *livekit.ParticipantInfo - arg5 bool + arg2 livekit.RoomID + arg3 livekit.RoomName + arg4 livekit.ParticipantID + arg5 *livekit.TrackInfo + arg6 *livekit.ParticipantInfo + arg7 bool } - TrackUnmutedStub func(context.Context, livekit.ParticipantID, *livekit.TrackInfo) + TrackUnmutedStub func(context.Context, livekit.RoomID, livekit.RoomName, livekit.ParticipantID, *livekit.TrackInfo) trackUnmutedMutex sync.RWMutex trackUnmutedArgsForCall []struct { arg1 context.Context - arg2 livekit.ParticipantID - arg3 *livekit.TrackInfo + arg2 livekit.RoomID + arg3 livekit.RoomName + arg4 livekit.ParticipantID + arg5 *livekit.TrackInfo } - TrackUnpublishedStub func(context.Context, livekit.ParticipantID, livekit.ParticipantIdentity, *livekit.TrackInfo, bool) + TrackUnpublishedStub func(context.Context, livekit.RoomID, livekit.RoomName, livekit.ParticipantID, livekit.ParticipantIdentity, *livekit.TrackInfo, bool) trackUnpublishedMutex sync.RWMutex trackUnpublishedArgsForCall []struct { arg1 context.Context - arg2 livekit.ParticipantID - arg3 livekit.ParticipantIdentity - arg4 *livekit.TrackInfo - arg5 bool + arg2 livekit.RoomID + arg3 livekit.RoomName + arg4 livekit.ParticipantID + arg5 livekit.ParticipantIdentity + arg6 *livekit.TrackInfo + arg7 bool } - TrackUnsubscribedStub func(context.Context, livekit.ParticipantID, *livekit.TrackInfo, bool) + TrackUnsubscribedStub func(context.Context, livekit.RoomID, livekit.RoomName, livekit.ParticipantID, *livekit.TrackInfo, bool) trackUnsubscribedMutex sync.RWMutex trackUnsubscribedArgsForCall []struct { arg1 context.Context - arg2 livekit.ParticipantID - arg3 *livekit.TrackInfo - arg4 bool + arg2 livekit.RoomID + arg3 livekit.RoomName + arg4 livekit.ParticipantID + arg5 *livekit.TrackInfo + arg6 bool } WebhookStub func(context.Context, *livekit.WebhookInfo) webhookMutex sync.RWMutex @@ -630,40 +675,41 @@ func (fake *FakeTelemetryService) LocalRoomStateArgsForCall(i int) (context.Cont return argsForCall.arg1, argsForCall.arg2 } -func (fake *FakeTelemetryService) NotifyEvent(arg1 context.Context, arg2 *livekit.WebhookEvent) { - fake.notifyEventMutex.Lock() - fake.notifyEventArgsForCall = append(fake.notifyEventArgsForCall, struct { +func (fake *FakeTelemetryService) NotifyEgressEvent(arg1 context.Context, arg2 string, arg3 *livekit.EgressInfo) { + fake.notifyEgressEventMutex.Lock() + fake.notifyEgressEventArgsForCall = append(fake.notifyEgressEventArgsForCall, struct { arg1 context.Context - arg2 *livekit.WebhookEvent - }{arg1, arg2}) - stub := fake.NotifyEventStub - fake.recordInvocation("NotifyEvent", []interface{}{arg1, arg2}) - fake.notifyEventMutex.Unlock() + arg2 string + arg3 *livekit.EgressInfo + }{arg1, arg2, arg3}) + stub := fake.NotifyEgressEventStub + fake.recordInvocation("NotifyEgressEvent", []interface{}{arg1, arg2, arg3}) + fake.notifyEgressEventMutex.Unlock() if stub != nil { - fake.NotifyEventStub(arg1, arg2) + fake.NotifyEgressEventStub(arg1, arg2, arg3) } } -func (fake *FakeTelemetryService) NotifyEventCallCount() int { - fake.notifyEventMutex.RLock() - defer fake.notifyEventMutex.RUnlock() - return len(fake.notifyEventArgsForCall) +func (fake *FakeTelemetryService) NotifyEgressEventCallCount() int { + fake.notifyEgressEventMutex.RLock() + defer fake.notifyEgressEventMutex.RUnlock() + return len(fake.notifyEgressEventArgsForCall) } -func (fake *FakeTelemetryService) NotifyEventCalls(stub func(context.Context, *livekit.WebhookEvent)) { - fake.notifyEventMutex.Lock() - defer fake.notifyEventMutex.Unlock() - fake.NotifyEventStub = stub +func (fake *FakeTelemetryService) NotifyEgressEventCalls(stub func(context.Context, string, *livekit.EgressInfo)) { + fake.notifyEgressEventMutex.Lock() + defer fake.notifyEgressEventMutex.Unlock() + fake.NotifyEgressEventStub = stub } -func (fake *FakeTelemetryService) NotifyEventArgsForCall(i int) (context.Context, *livekit.WebhookEvent) { - fake.notifyEventMutex.RLock() - defer fake.notifyEventMutex.RUnlock() - argsForCall := fake.notifyEventArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2 +func (fake *FakeTelemetryService) NotifyEgressEventArgsForCall(i int) (context.Context, string, *livekit.EgressInfo) { + fake.notifyEgressEventMutex.RLock() + defer fake.notifyEgressEventMutex.RUnlock() + argsForCall := fake.notifyEgressEventArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 } -func (fake *FakeTelemetryService) ParticipantActive(arg1 context.Context, arg2 *livekit.Room, arg3 *livekit.ParticipantInfo, arg4 *livekit.AnalyticsClientMeta, arg5 bool) { +func (fake *FakeTelemetryService) ParticipantActive(arg1 context.Context, arg2 *livekit.Room, arg3 *livekit.ParticipantInfo, arg4 *livekit.AnalyticsClientMeta, arg5 bool, arg6 *telemetry.ReferenceGuard) { fake.participantActiveMutex.Lock() fake.participantActiveArgsForCall = append(fake.participantActiveArgsForCall, struct { arg1 context.Context @@ -671,12 +717,13 @@ func (fake *FakeTelemetryService) ParticipantActive(arg1 context.Context, arg2 * arg3 *livekit.ParticipantInfo arg4 *livekit.AnalyticsClientMeta arg5 bool - }{arg1, arg2, arg3, arg4, arg5}) + arg6 *telemetry.ReferenceGuard + }{arg1, arg2, arg3, arg4, arg5, arg6}) stub := fake.ParticipantActiveStub - fake.recordInvocation("ParticipantActive", []interface{}{arg1, arg2, arg3, arg4, arg5}) + fake.recordInvocation("ParticipantActive", []interface{}{arg1, arg2, arg3, arg4, arg5, arg6}) fake.participantActiveMutex.Unlock() if stub != nil { - fake.ParticipantActiveStub(arg1, arg2, arg3, arg4, arg5) + fake.ParticipantActiveStub(arg1, arg2, arg3, arg4, arg5, arg6) } } @@ -686,20 +733,20 @@ func (fake *FakeTelemetryService) ParticipantActiveCallCount() int { return len(fake.participantActiveArgsForCall) } -func (fake *FakeTelemetryService) ParticipantActiveCalls(stub func(context.Context, *livekit.Room, *livekit.ParticipantInfo, *livekit.AnalyticsClientMeta, bool)) { +func (fake *FakeTelemetryService) ParticipantActiveCalls(stub func(context.Context, *livekit.Room, *livekit.ParticipantInfo, *livekit.AnalyticsClientMeta, bool, *telemetry.ReferenceGuard)) { fake.participantActiveMutex.Lock() defer fake.participantActiveMutex.Unlock() fake.ParticipantActiveStub = stub } -func (fake *FakeTelemetryService) ParticipantActiveArgsForCall(i int) (context.Context, *livekit.Room, *livekit.ParticipantInfo, *livekit.AnalyticsClientMeta, bool) { +func (fake *FakeTelemetryService) ParticipantActiveArgsForCall(i int) (context.Context, *livekit.Room, *livekit.ParticipantInfo, *livekit.AnalyticsClientMeta, bool, *telemetry.ReferenceGuard) { fake.participantActiveMutex.RLock() defer fake.participantActiveMutex.RUnlock() argsForCall := fake.participantActiveArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4, argsForCall.arg5 + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4, argsForCall.arg5, argsForCall.arg6 } -func (fake *FakeTelemetryService) ParticipantJoined(arg1 context.Context, arg2 *livekit.Room, arg3 *livekit.ParticipantInfo, arg4 *livekit.ClientInfo, arg5 *livekit.AnalyticsClientMeta, arg6 bool) { +func (fake *FakeTelemetryService) ParticipantJoined(arg1 context.Context, arg2 *livekit.Room, arg3 *livekit.ParticipantInfo, arg4 *livekit.ClientInfo, arg5 *livekit.AnalyticsClientMeta, arg6 bool, arg7 *telemetry.ReferenceGuard) { fake.participantJoinedMutex.Lock() fake.participantJoinedArgsForCall = append(fake.participantJoinedArgsForCall, struct { arg1 context.Context @@ -708,12 +755,13 @@ func (fake *FakeTelemetryService) ParticipantJoined(arg1 context.Context, arg2 * arg4 *livekit.ClientInfo arg5 *livekit.AnalyticsClientMeta arg6 bool - }{arg1, arg2, arg3, arg4, arg5, arg6}) + arg7 *telemetry.ReferenceGuard + }{arg1, arg2, arg3, arg4, arg5, arg6, arg7}) stub := fake.ParticipantJoinedStub - fake.recordInvocation("ParticipantJoined", []interface{}{arg1, arg2, arg3, arg4, arg5, arg6}) + fake.recordInvocation("ParticipantJoined", []interface{}{arg1, arg2, arg3, arg4, arg5, arg6, arg7}) fake.participantJoinedMutex.Unlock() if stub != nil { - fake.ParticipantJoinedStub(arg1, arg2, arg3, arg4, arg5, arg6) + fake.ParticipantJoinedStub(arg1, arg2, arg3, arg4, arg5, arg6, arg7) } } @@ -723,32 +771,33 @@ func (fake *FakeTelemetryService) ParticipantJoinedCallCount() int { return len(fake.participantJoinedArgsForCall) } -func (fake *FakeTelemetryService) ParticipantJoinedCalls(stub func(context.Context, *livekit.Room, *livekit.ParticipantInfo, *livekit.ClientInfo, *livekit.AnalyticsClientMeta, bool)) { +func (fake *FakeTelemetryService) ParticipantJoinedCalls(stub func(context.Context, *livekit.Room, *livekit.ParticipantInfo, *livekit.ClientInfo, *livekit.AnalyticsClientMeta, bool, *telemetry.ReferenceGuard)) { fake.participantJoinedMutex.Lock() defer fake.participantJoinedMutex.Unlock() fake.ParticipantJoinedStub = stub } -func (fake *FakeTelemetryService) ParticipantJoinedArgsForCall(i int) (context.Context, *livekit.Room, *livekit.ParticipantInfo, *livekit.ClientInfo, *livekit.AnalyticsClientMeta, bool) { +func (fake *FakeTelemetryService) ParticipantJoinedArgsForCall(i int) (context.Context, *livekit.Room, *livekit.ParticipantInfo, *livekit.ClientInfo, *livekit.AnalyticsClientMeta, bool, *telemetry.ReferenceGuard) { fake.participantJoinedMutex.RLock() defer fake.participantJoinedMutex.RUnlock() argsForCall := fake.participantJoinedArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4, argsForCall.arg5, argsForCall.arg6 + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4, argsForCall.arg5, argsForCall.arg6, argsForCall.arg7 } -func (fake *FakeTelemetryService) ParticipantLeft(arg1 context.Context, arg2 *livekit.Room, arg3 *livekit.ParticipantInfo, arg4 bool) { +func (fake *FakeTelemetryService) ParticipantLeft(arg1 context.Context, arg2 *livekit.Room, arg3 *livekit.ParticipantInfo, arg4 bool, arg5 *telemetry.ReferenceGuard) { fake.participantLeftMutex.Lock() fake.participantLeftArgsForCall = append(fake.participantLeftArgsForCall, struct { arg1 context.Context arg2 *livekit.Room arg3 *livekit.ParticipantInfo arg4 bool - }{arg1, arg2, arg3, arg4}) + arg5 *telemetry.ReferenceGuard + }{arg1, arg2, arg3, arg4, arg5}) stub := fake.ParticipantLeftStub - fake.recordInvocation("ParticipantLeft", []interface{}{arg1, arg2, arg3, arg4}) + fake.recordInvocation("ParticipantLeft", []interface{}{arg1, arg2, arg3, arg4, arg5}) fake.participantLeftMutex.Unlock() if stub != nil { - fake.ParticipantLeftStub(arg1, arg2, arg3, arg4) + fake.ParticipantLeftStub(arg1, arg2, arg3, arg4, arg5) } } @@ -758,17 +807,17 @@ func (fake *FakeTelemetryService) ParticipantLeftCallCount() int { return len(fake.participantLeftArgsForCall) } -func (fake *FakeTelemetryService) ParticipantLeftCalls(stub func(context.Context, *livekit.Room, *livekit.ParticipantInfo, bool)) { +func (fake *FakeTelemetryService) ParticipantLeftCalls(stub func(context.Context, *livekit.Room, *livekit.ParticipantInfo, bool, *telemetry.ReferenceGuard)) { fake.participantLeftMutex.Lock() defer fake.participantLeftMutex.Unlock() fake.ParticipantLeftStub = stub } -func (fake *FakeTelemetryService) ParticipantLeftArgsForCall(i int) (context.Context, *livekit.Room, *livekit.ParticipantInfo, bool) { +func (fake *FakeTelemetryService) ParticipantLeftArgsForCall(i int) (context.Context, *livekit.Room, *livekit.ParticipantInfo, bool, *telemetry.ReferenceGuard) { fake.participantLeftMutex.RLock() defer fake.participantLeftMutex.RUnlock() argsForCall := fake.participantLeftArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4 + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4, argsForCall.arg5 } func (fake *FakeTelemetryService) ParticipantResumed(arg1 context.Context, arg2 *livekit.Room, arg3 *livekit.ParticipantInfo, arg4 livekit.NodeID, arg5 livekit.ReconnectReason) { @@ -873,6 +922,67 @@ func (fake *FakeTelemetryService) RoomEndedArgsForCall(i int) (context.Context, return argsForCall.arg1, argsForCall.arg2 } +func (fake *FakeTelemetryService) RoomProjectReporter(arg1 context.Context) roomobs.ProjectReporter { + fake.roomProjectReporterMutex.Lock() + ret, specificReturn := fake.roomProjectReporterReturnsOnCall[len(fake.roomProjectReporterArgsForCall)] + fake.roomProjectReporterArgsForCall = append(fake.roomProjectReporterArgsForCall, struct { + arg1 context.Context + }{arg1}) + stub := fake.RoomProjectReporterStub + fakeReturns := fake.roomProjectReporterReturns + fake.recordInvocation("RoomProjectReporter", []interface{}{arg1}) + fake.roomProjectReporterMutex.Unlock() + if stub != nil { + return stub(arg1) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeTelemetryService) RoomProjectReporterCallCount() int { + fake.roomProjectReporterMutex.RLock() + defer fake.roomProjectReporterMutex.RUnlock() + return len(fake.roomProjectReporterArgsForCall) +} + +func (fake *FakeTelemetryService) RoomProjectReporterCalls(stub func(context.Context) roomobs.ProjectReporter) { + fake.roomProjectReporterMutex.Lock() + defer fake.roomProjectReporterMutex.Unlock() + fake.RoomProjectReporterStub = stub +} + +func (fake *FakeTelemetryService) RoomProjectReporterArgsForCall(i int) context.Context { + fake.roomProjectReporterMutex.RLock() + defer fake.roomProjectReporterMutex.RUnlock() + argsForCall := fake.roomProjectReporterArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeTelemetryService) RoomProjectReporterReturns(result1 roomobs.ProjectReporter) { + fake.roomProjectReporterMutex.Lock() + defer fake.roomProjectReporterMutex.Unlock() + fake.RoomProjectReporterStub = nil + fake.roomProjectReporterReturns = struct { + result1 roomobs.ProjectReporter + }{result1} +} + +func (fake *FakeTelemetryService) RoomProjectReporterReturnsOnCall(i int, result1 roomobs.ProjectReporter) { + fake.roomProjectReporterMutex.Lock() + defer fake.roomProjectReporterMutex.Unlock() + fake.RoomProjectReporterStub = nil + if fake.roomProjectReporterReturnsOnCall == nil { + fake.roomProjectReporterReturnsOnCall = make(map[int]struct { + result1 roomobs.ProjectReporter + }) + } + fake.roomProjectReporterReturnsOnCall[i] = struct { + result1 roomobs.ProjectReporter + }{result1} +} + func (fake *FakeTelemetryService) RoomStarted(arg1 context.Context, arg2 *livekit.Room) { fake.roomStartedMutex.Lock() fake.roomStartedArgsForCall = append(fake.roomStartedArgsForCall, struct { @@ -1010,20 +1120,22 @@ func (fake *FakeTelemetryService) SendStatsArgsForCall(i int) (context.Context, return argsForCall.arg1, argsForCall.arg2 } -func (fake *FakeTelemetryService) TrackMaxSubscribedVideoQuality(arg1 context.Context, arg2 livekit.ParticipantID, arg3 *livekit.TrackInfo, arg4 mime.MimeType, arg5 livekit.VideoQuality) { +func (fake *FakeTelemetryService) TrackMaxSubscribedVideoQuality(arg1 context.Context, arg2 livekit.RoomID, arg3 livekit.RoomName, arg4 livekit.ParticipantID, arg5 *livekit.TrackInfo, arg6 mime.MimeType, arg7 livekit.VideoQuality) { fake.trackMaxSubscribedVideoQualityMutex.Lock() fake.trackMaxSubscribedVideoQualityArgsForCall = append(fake.trackMaxSubscribedVideoQualityArgsForCall, struct { arg1 context.Context - arg2 livekit.ParticipantID - arg3 *livekit.TrackInfo - arg4 mime.MimeType - arg5 livekit.VideoQuality - }{arg1, arg2, arg3, arg4, arg5}) + arg2 livekit.RoomID + arg3 livekit.RoomName + arg4 livekit.ParticipantID + arg5 *livekit.TrackInfo + arg6 mime.MimeType + arg7 livekit.VideoQuality + }{arg1, arg2, arg3, arg4, arg5, arg6, arg7}) stub := fake.TrackMaxSubscribedVideoQualityStub - fake.recordInvocation("TrackMaxSubscribedVideoQuality", []interface{}{arg1, arg2, arg3, arg4, arg5}) + fake.recordInvocation("TrackMaxSubscribedVideoQuality", []interface{}{arg1, arg2, arg3, arg4, arg5, arg6, arg7}) fake.trackMaxSubscribedVideoQualityMutex.Unlock() if stub != nil { - fake.TrackMaxSubscribedVideoQualityStub(arg1, arg2, arg3, arg4, arg5) + fake.TrackMaxSubscribedVideoQualityStub(arg1, arg2, arg3, arg4, arg5, arg6, arg7) } } @@ -1033,31 +1145,33 @@ func (fake *FakeTelemetryService) TrackMaxSubscribedVideoQualityCallCount() int return len(fake.trackMaxSubscribedVideoQualityArgsForCall) } -func (fake *FakeTelemetryService) TrackMaxSubscribedVideoQualityCalls(stub func(context.Context, livekit.ParticipantID, *livekit.TrackInfo, mime.MimeType, livekit.VideoQuality)) { +func (fake *FakeTelemetryService) TrackMaxSubscribedVideoQualityCalls(stub func(context.Context, livekit.RoomID, livekit.RoomName, livekit.ParticipantID, *livekit.TrackInfo, mime.MimeType, livekit.VideoQuality)) { fake.trackMaxSubscribedVideoQualityMutex.Lock() defer fake.trackMaxSubscribedVideoQualityMutex.Unlock() fake.TrackMaxSubscribedVideoQualityStub = stub } -func (fake *FakeTelemetryService) TrackMaxSubscribedVideoQualityArgsForCall(i int) (context.Context, livekit.ParticipantID, *livekit.TrackInfo, mime.MimeType, livekit.VideoQuality) { +func (fake *FakeTelemetryService) TrackMaxSubscribedVideoQualityArgsForCall(i int) (context.Context, livekit.RoomID, livekit.RoomName, livekit.ParticipantID, *livekit.TrackInfo, mime.MimeType, livekit.VideoQuality) { fake.trackMaxSubscribedVideoQualityMutex.RLock() defer fake.trackMaxSubscribedVideoQualityMutex.RUnlock() argsForCall := fake.trackMaxSubscribedVideoQualityArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4, argsForCall.arg5 + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4, argsForCall.arg5, argsForCall.arg6, argsForCall.arg7 } -func (fake *FakeTelemetryService) TrackMuted(arg1 context.Context, arg2 livekit.ParticipantID, arg3 *livekit.TrackInfo) { +func (fake *FakeTelemetryService) TrackMuted(arg1 context.Context, arg2 livekit.RoomID, arg3 livekit.RoomName, arg4 livekit.ParticipantID, arg5 *livekit.TrackInfo) { fake.trackMutedMutex.Lock() fake.trackMutedArgsForCall = append(fake.trackMutedArgsForCall, struct { arg1 context.Context - arg2 livekit.ParticipantID - arg3 *livekit.TrackInfo - }{arg1, arg2, arg3}) + arg2 livekit.RoomID + arg3 livekit.RoomName + arg4 livekit.ParticipantID + arg5 *livekit.TrackInfo + }{arg1, arg2, arg3, arg4, arg5}) stub := fake.TrackMutedStub - fake.recordInvocation("TrackMuted", []interface{}{arg1, arg2, arg3}) + fake.recordInvocation("TrackMuted", []interface{}{arg1, arg2, arg3, arg4, arg5}) fake.trackMutedMutex.Unlock() if stub != nil { - fake.TrackMutedStub(arg1, arg2, arg3) + fake.TrackMutedStub(arg1, arg2, arg3, arg4, arg5) } } @@ -1067,34 +1181,36 @@ func (fake *FakeTelemetryService) TrackMutedCallCount() int { return len(fake.trackMutedArgsForCall) } -func (fake *FakeTelemetryService) TrackMutedCalls(stub func(context.Context, livekit.ParticipantID, *livekit.TrackInfo)) { +func (fake *FakeTelemetryService) TrackMutedCalls(stub func(context.Context, livekit.RoomID, livekit.RoomName, livekit.ParticipantID, *livekit.TrackInfo)) { fake.trackMutedMutex.Lock() defer fake.trackMutedMutex.Unlock() fake.TrackMutedStub = stub } -func (fake *FakeTelemetryService) TrackMutedArgsForCall(i int) (context.Context, livekit.ParticipantID, *livekit.TrackInfo) { +func (fake *FakeTelemetryService) TrackMutedArgsForCall(i int) (context.Context, livekit.RoomID, livekit.RoomName, livekit.ParticipantID, *livekit.TrackInfo) { fake.trackMutedMutex.RLock() defer fake.trackMutedMutex.RUnlock() argsForCall := fake.trackMutedArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4, argsForCall.arg5 } -func (fake *FakeTelemetryService) TrackPublishRTPStats(arg1 context.Context, arg2 livekit.ParticipantID, arg3 livekit.TrackID, arg4 mime.MimeType, arg5 int, arg6 *livekit.RTPStats) { +func (fake *FakeTelemetryService) TrackPublishRTPStats(arg1 context.Context, arg2 livekit.RoomID, arg3 livekit.RoomName, arg4 livekit.ParticipantID, arg5 livekit.TrackID, arg6 mime.MimeType, arg7 int, arg8 *livekit.RTPStats) { fake.trackPublishRTPStatsMutex.Lock() fake.trackPublishRTPStatsArgsForCall = append(fake.trackPublishRTPStatsArgsForCall, struct { arg1 context.Context - arg2 livekit.ParticipantID - arg3 livekit.TrackID - arg4 mime.MimeType - arg5 int - arg6 *livekit.RTPStats - }{arg1, arg2, arg3, arg4, arg5, arg6}) + arg2 livekit.RoomID + arg3 livekit.RoomName + arg4 livekit.ParticipantID + arg5 livekit.TrackID + arg6 mime.MimeType + arg7 int + arg8 *livekit.RTPStats + }{arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8}) stub := fake.TrackPublishRTPStatsStub - fake.recordInvocation("TrackPublishRTPStats", []interface{}{arg1, arg2, arg3, arg4, arg5, arg6}) + fake.recordInvocation("TrackPublishRTPStats", []interface{}{arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8}) fake.trackPublishRTPStatsMutex.Unlock() if stub != nil { - fake.TrackPublishRTPStatsStub(arg1, arg2, arg3, arg4, arg5, arg6) + fake.TrackPublishRTPStatsStub(arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8) } } @@ -1104,32 +1220,34 @@ func (fake *FakeTelemetryService) TrackPublishRTPStatsCallCount() int { return len(fake.trackPublishRTPStatsArgsForCall) } -func (fake *FakeTelemetryService) TrackPublishRTPStatsCalls(stub func(context.Context, livekit.ParticipantID, livekit.TrackID, mime.MimeType, int, *livekit.RTPStats)) { +func (fake *FakeTelemetryService) TrackPublishRTPStatsCalls(stub func(context.Context, livekit.RoomID, livekit.RoomName, livekit.ParticipantID, livekit.TrackID, mime.MimeType, int, *livekit.RTPStats)) { fake.trackPublishRTPStatsMutex.Lock() defer fake.trackPublishRTPStatsMutex.Unlock() fake.TrackPublishRTPStatsStub = stub } -func (fake *FakeTelemetryService) TrackPublishRTPStatsArgsForCall(i int) (context.Context, livekit.ParticipantID, livekit.TrackID, mime.MimeType, int, *livekit.RTPStats) { +func (fake *FakeTelemetryService) TrackPublishRTPStatsArgsForCall(i int) (context.Context, livekit.RoomID, livekit.RoomName, livekit.ParticipantID, livekit.TrackID, mime.MimeType, int, *livekit.RTPStats) { fake.trackPublishRTPStatsMutex.RLock() defer fake.trackPublishRTPStatsMutex.RUnlock() argsForCall := fake.trackPublishRTPStatsArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4, argsForCall.arg5, argsForCall.arg6 + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4, argsForCall.arg5, argsForCall.arg6, argsForCall.arg7, argsForCall.arg8 } -func (fake *FakeTelemetryService) TrackPublishRequested(arg1 context.Context, arg2 livekit.ParticipantID, arg3 livekit.ParticipantIdentity, arg4 *livekit.TrackInfo) { +func (fake *FakeTelemetryService) TrackPublishRequested(arg1 context.Context, arg2 livekit.RoomID, arg3 livekit.RoomName, arg4 livekit.ParticipantID, arg5 livekit.ParticipantIdentity, arg6 *livekit.TrackInfo) { fake.trackPublishRequestedMutex.Lock() fake.trackPublishRequestedArgsForCall = append(fake.trackPublishRequestedArgsForCall, struct { arg1 context.Context - arg2 livekit.ParticipantID - arg3 livekit.ParticipantIdentity - arg4 *livekit.TrackInfo - }{arg1, arg2, arg3, arg4}) + arg2 livekit.RoomID + arg3 livekit.RoomName + arg4 livekit.ParticipantID + arg5 livekit.ParticipantIdentity + arg6 *livekit.TrackInfo + }{arg1, arg2, arg3, arg4, arg5, arg6}) stub := fake.TrackPublishRequestedStub - fake.recordInvocation("TrackPublishRequested", []interface{}{arg1, arg2, arg3, arg4}) + fake.recordInvocation("TrackPublishRequested", []interface{}{arg1, arg2, arg3, arg4, arg5, arg6}) fake.trackPublishRequestedMutex.Unlock() if stub != nil { - fake.TrackPublishRequestedStub(arg1, arg2, arg3, arg4) + fake.TrackPublishRequestedStub(arg1, arg2, arg3, arg4, arg5, arg6) } } @@ -1139,32 +1257,35 @@ func (fake *FakeTelemetryService) TrackPublishRequestedCallCount() int { return len(fake.trackPublishRequestedArgsForCall) } -func (fake *FakeTelemetryService) TrackPublishRequestedCalls(stub func(context.Context, livekit.ParticipantID, livekit.ParticipantIdentity, *livekit.TrackInfo)) { +func (fake *FakeTelemetryService) TrackPublishRequestedCalls(stub func(context.Context, livekit.RoomID, livekit.RoomName, livekit.ParticipantID, livekit.ParticipantIdentity, *livekit.TrackInfo)) { fake.trackPublishRequestedMutex.Lock() defer fake.trackPublishRequestedMutex.Unlock() fake.TrackPublishRequestedStub = stub } -func (fake *FakeTelemetryService) TrackPublishRequestedArgsForCall(i int) (context.Context, livekit.ParticipantID, livekit.ParticipantIdentity, *livekit.TrackInfo) { +func (fake *FakeTelemetryService) TrackPublishRequestedArgsForCall(i int) (context.Context, livekit.RoomID, livekit.RoomName, livekit.ParticipantID, livekit.ParticipantIdentity, *livekit.TrackInfo) { fake.trackPublishRequestedMutex.RLock() defer fake.trackPublishRequestedMutex.RUnlock() argsForCall := fake.trackPublishRequestedArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4 + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4, argsForCall.arg5, argsForCall.arg6 } -func (fake *FakeTelemetryService) TrackPublished(arg1 context.Context, arg2 livekit.ParticipantID, arg3 livekit.ParticipantIdentity, arg4 *livekit.TrackInfo) { +func (fake *FakeTelemetryService) TrackPublished(arg1 context.Context, arg2 livekit.RoomID, arg3 livekit.RoomName, arg4 livekit.ParticipantID, arg5 livekit.ParticipantIdentity, arg6 *livekit.TrackInfo, arg7 bool) { fake.trackPublishedMutex.Lock() fake.trackPublishedArgsForCall = append(fake.trackPublishedArgsForCall, struct { arg1 context.Context - arg2 livekit.ParticipantID - arg3 livekit.ParticipantIdentity - arg4 *livekit.TrackInfo - }{arg1, arg2, arg3, arg4}) + arg2 livekit.RoomID + arg3 livekit.RoomName + arg4 livekit.ParticipantID + arg5 livekit.ParticipantIdentity + arg6 *livekit.TrackInfo + arg7 bool + }{arg1, arg2, arg3, arg4, arg5, arg6, arg7}) stub := fake.TrackPublishedStub - fake.recordInvocation("TrackPublished", []interface{}{arg1, arg2, arg3, arg4}) + fake.recordInvocation("TrackPublished", []interface{}{arg1, arg2, arg3, arg4, arg5, arg6, arg7}) fake.trackPublishedMutex.Unlock() if stub != nil { - fake.TrackPublishedStub(arg1, arg2, arg3, arg4) + fake.TrackPublishedStub(arg1, arg2, arg3, arg4, arg5, arg6, arg7) } } @@ -1174,31 +1295,33 @@ func (fake *FakeTelemetryService) TrackPublishedCallCount() int { return len(fake.trackPublishedArgsForCall) } -func (fake *FakeTelemetryService) TrackPublishedCalls(stub func(context.Context, livekit.ParticipantID, livekit.ParticipantIdentity, *livekit.TrackInfo)) { +func (fake *FakeTelemetryService) TrackPublishedCalls(stub func(context.Context, livekit.RoomID, livekit.RoomName, livekit.ParticipantID, livekit.ParticipantIdentity, *livekit.TrackInfo, bool)) { fake.trackPublishedMutex.Lock() defer fake.trackPublishedMutex.Unlock() fake.TrackPublishedStub = stub } -func (fake *FakeTelemetryService) TrackPublishedArgsForCall(i int) (context.Context, livekit.ParticipantID, livekit.ParticipantIdentity, *livekit.TrackInfo) { +func (fake *FakeTelemetryService) TrackPublishedArgsForCall(i int) (context.Context, livekit.RoomID, livekit.RoomName, livekit.ParticipantID, livekit.ParticipantIdentity, *livekit.TrackInfo, bool) { fake.trackPublishedMutex.RLock() defer fake.trackPublishedMutex.RUnlock() argsForCall := fake.trackPublishedArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4 + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4, argsForCall.arg5, argsForCall.arg6, argsForCall.arg7 } -func (fake *FakeTelemetryService) TrackPublishedUpdate(arg1 context.Context, arg2 livekit.ParticipantID, arg3 *livekit.TrackInfo) { +func (fake *FakeTelemetryService) TrackPublishedUpdate(arg1 context.Context, arg2 livekit.RoomID, arg3 livekit.RoomName, arg4 livekit.ParticipantID, arg5 *livekit.TrackInfo) { fake.trackPublishedUpdateMutex.Lock() fake.trackPublishedUpdateArgsForCall = append(fake.trackPublishedUpdateArgsForCall, struct { arg1 context.Context - arg2 livekit.ParticipantID - arg3 *livekit.TrackInfo - }{arg1, arg2, arg3}) + arg2 livekit.RoomID + arg3 livekit.RoomName + arg4 livekit.ParticipantID + arg5 *livekit.TrackInfo + }{arg1, arg2, arg3, arg4, arg5}) stub := fake.TrackPublishedUpdateStub - fake.recordInvocation("TrackPublishedUpdate", []interface{}{arg1, arg2, arg3}) + fake.recordInvocation("TrackPublishedUpdate", []interface{}{arg1, arg2, arg3, arg4, arg5}) fake.trackPublishedUpdateMutex.Unlock() if stub != nil { - fake.TrackPublishedUpdateStub(arg1, arg2, arg3) + fake.TrackPublishedUpdateStub(arg1, arg2, arg3, arg4, arg5) } } @@ -1208,30 +1331,32 @@ func (fake *FakeTelemetryService) TrackPublishedUpdateCallCount() int { return len(fake.trackPublishedUpdateArgsForCall) } -func (fake *FakeTelemetryService) TrackPublishedUpdateCalls(stub func(context.Context, livekit.ParticipantID, *livekit.TrackInfo)) { +func (fake *FakeTelemetryService) TrackPublishedUpdateCalls(stub func(context.Context, livekit.RoomID, livekit.RoomName, livekit.ParticipantID, *livekit.TrackInfo)) { fake.trackPublishedUpdateMutex.Lock() defer fake.trackPublishedUpdateMutex.Unlock() fake.TrackPublishedUpdateStub = stub } -func (fake *FakeTelemetryService) TrackPublishedUpdateArgsForCall(i int) (context.Context, livekit.ParticipantID, *livekit.TrackInfo) { +func (fake *FakeTelemetryService) TrackPublishedUpdateArgsForCall(i int) (context.Context, livekit.RoomID, livekit.RoomName, livekit.ParticipantID, *livekit.TrackInfo) { fake.trackPublishedUpdateMutex.RLock() defer fake.trackPublishedUpdateMutex.RUnlock() argsForCall := fake.trackPublishedUpdateArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4, argsForCall.arg5 } -func (fake *FakeTelemetryService) TrackStats(arg1 telemetry.StatsKey, arg2 *livekit.AnalyticsStat) { +func (fake *FakeTelemetryService) TrackStats(arg1 livekit.RoomID, arg2 livekit.RoomName, arg3 telemetry.StatsKey, arg4 *livekit.AnalyticsStat) { fake.trackStatsMutex.Lock() fake.trackStatsArgsForCall = append(fake.trackStatsArgsForCall, struct { - arg1 telemetry.StatsKey - arg2 *livekit.AnalyticsStat - }{arg1, arg2}) + arg1 livekit.RoomID + arg2 livekit.RoomName + arg3 telemetry.StatsKey + arg4 *livekit.AnalyticsStat + }{arg1, arg2, arg3, arg4}) stub := fake.TrackStatsStub - fake.recordInvocation("TrackStats", []interface{}{arg1, arg2}) + fake.recordInvocation("TrackStats", []interface{}{arg1, arg2, arg3, arg4}) fake.trackStatsMutex.Unlock() if stub != nil { - fake.TrackStatsStub(arg1, arg2) + fake.TrackStatsStub(arg1, arg2, arg3, arg4) } } @@ -1241,33 +1366,35 @@ func (fake *FakeTelemetryService) TrackStatsCallCount() int { return len(fake.trackStatsArgsForCall) } -func (fake *FakeTelemetryService) TrackStatsCalls(stub func(telemetry.StatsKey, *livekit.AnalyticsStat)) { +func (fake *FakeTelemetryService) TrackStatsCalls(stub func(livekit.RoomID, livekit.RoomName, telemetry.StatsKey, *livekit.AnalyticsStat)) { fake.trackStatsMutex.Lock() defer fake.trackStatsMutex.Unlock() fake.TrackStatsStub = stub } -func (fake *FakeTelemetryService) TrackStatsArgsForCall(i int) (telemetry.StatsKey, *livekit.AnalyticsStat) { +func (fake *FakeTelemetryService) TrackStatsArgsForCall(i int) (livekit.RoomID, livekit.RoomName, telemetry.StatsKey, *livekit.AnalyticsStat) { fake.trackStatsMutex.RLock() defer fake.trackStatsMutex.RUnlock() argsForCall := fake.trackStatsArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2 + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4 } -func (fake *FakeTelemetryService) TrackSubscribeFailed(arg1 context.Context, arg2 livekit.ParticipantID, arg3 livekit.TrackID, arg4 error, arg5 bool) { +func (fake *FakeTelemetryService) TrackSubscribeFailed(arg1 context.Context, arg2 livekit.RoomID, arg3 livekit.RoomName, arg4 livekit.ParticipantID, arg5 livekit.TrackID, arg6 error, arg7 bool) { fake.trackSubscribeFailedMutex.Lock() fake.trackSubscribeFailedArgsForCall = append(fake.trackSubscribeFailedArgsForCall, struct { arg1 context.Context - arg2 livekit.ParticipantID - arg3 livekit.TrackID - arg4 error - arg5 bool - }{arg1, arg2, arg3, arg4, arg5}) + arg2 livekit.RoomID + arg3 livekit.RoomName + arg4 livekit.ParticipantID + arg5 livekit.TrackID + arg6 error + arg7 bool + }{arg1, arg2, arg3, arg4, arg5, arg6, arg7}) stub := fake.TrackSubscribeFailedStub - fake.recordInvocation("TrackSubscribeFailed", []interface{}{arg1, arg2, arg3, arg4, arg5}) + fake.recordInvocation("TrackSubscribeFailed", []interface{}{arg1, arg2, arg3, arg4, arg5, arg6, arg7}) fake.trackSubscribeFailedMutex.Unlock() if stub != nil { - fake.TrackSubscribeFailedStub(arg1, arg2, arg3, arg4, arg5) + fake.TrackSubscribeFailedStub(arg1, arg2, arg3, arg4, arg5, arg6, arg7) } } @@ -1277,33 +1404,35 @@ func (fake *FakeTelemetryService) TrackSubscribeFailedCallCount() int { return len(fake.trackSubscribeFailedArgsForCall) } -func (fake *FakeTelemetryService) TrackSubscribeFailedCalls(stub func(context.Context, livekit.ParticipantID, livekit.TrackID, error, bool)) { +func (fake *FakeTelemetryService) TrackSubscribeFailedCalls(stub func(context.Context, livekit.RoomID, livekit.RoomName, livekit.ParticipantID, livekit.TrackID, error, bool)) { fake.trackSubscribeFailedMutex.Lock() defer fake.trackSubscribeFailedMutex.Unlock() fake.TrackSubscribeFailedStub = stub } -func (fake *FakeTelemetryService) TrackSubscribeFailedArgsForCall(i int) (context.Context, livekit.ParticipantID, livekit.TrackID, error, bool) { +func (fake *FakeTelemetryService) TrackSubscribeFailedArgsForCall(i int) (context.Context, livekit.RoomID, livekit.RoomName, livekit.ParticipantID, livekit.TrackID, error, bool) { fake.trackSubscribeFailedMutex.RLock() defer fake.trackSubscribeFailedMutex.RUnlock() argsForCall := fake.trackSubscribeFailedArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4, argsForCall.arg5 + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4, argsForCall.arg5, argsForCall.arg6, argsForCall.arg7 } -func (fake *FakeTelemetryService) TrackSubscribeRTPStats(arg1 context.Context, arg2 livekit.ParticipantID, arg3 livekit.TrackID, arg4 mime.MimeType, arg5 *livekit.RTPStats) { +func (fake *FakeTelemetryService) TrackSubscribeRTPStats(arg1 context.Context, arg2 livekit.RoomID, arg3 livekit.RoomName, arg4 livekit.ParticipantID, arg5 livekit.TrackID, arg6 mime.MimeType, arg7 *livekit.RTPStats) { fake.trackSubscribeRTPStatsMutex.Lock() fake.trackSubscribeRTPStatsArgsForCall = append(fake.trackSubscribeRTPStatsArgsForCall, struct { arg1 context.Context - arg2 livekit.ParticipantID - arg3 livekit.TrackID - arg4 mime.MimeType - arg5 *livekit.RTPStats - }{arg1, arg2, arg3, arg4, arg5}) + arg2 livekit.RoomID + arg3 livekit.RoomName + arg4 livekit.ParticipantID + arg5 livekit.TrackID + arg6 mime.MimeType + arg7 *livekit.RTPStats + }{arg1, arg2, arg3, arg4, arg5, arg6, arg7}) stub := fake.TrackSubscribeRTPStatsStub - fake.recordInvocation("TrackSubscribeRTPStats", []interface{}{arg1, arg2, arg3, arg4, arg5}) + fake.recordInvocation("TrackSubscribeRTPStats", []interface{}{arg1, arg2, arg3, arg4, arg5, arg6, arg7}) fake.trackSubscribeRTPStatsMutex.Unlock() if stub != nil { - fake.TrackSubscribeRTPStatsStub(arg1, arg2, arg3, arg4, arg5) + fake.TrackSubscribeRTPStatsStub(arg1, arg2, arg3, arg4, arg5, arg6, arg7) } } @@ -1313,31 +1442,33 @@ func (fake *FakeTelemetryService) TrackSubscribeRTPStatsCallCount() int { return len(fake.trackSubscribeRTPStatsArgsForCall) } -func (fake *FakeTelemetryService) TrackSubscribeRTPStatsCalls(stub func(context.Context, livekit.ParticipantID, livekit.TrackID, mime.MimeType, *livekit.RTPStats)) { +func (fake *FakeTelemetryService) TrackSubscribeRTPStatsCalls(stub func(context.Context, livekit.RoomID, livekit.RoomName, livekit.ParticipantID, livekit.TrackID, mime.MimeType, *livekit.RTPStats)) { fake.trackSubscribeRTPStatsMutex.Lock() defer fake.trackSubscribeRTPStatsMutex.Unlock() fake.TrackSubscribeRTPStatsStub = stub } -func (fake *FakeTelemetryService) TrackSubscribeRTPStatsArgsForCall(i int) (context.Context, livekit.ParticipantID, livekit.TrackID, mime.MimeType, *livekit.RTPStats) { +func (fake *FakeTelemetryService) TrackSubscribeRTPStatsArgsForCall(i int) (context.Context, livekit.RoomID, livekit.RoomName, livekit.ParticipantID, livekit.TrackID, mime.MimeType, *livekit.RTPStats) { fake.trackSubscribeRTPStatsMutex.RLock() defer fake.trackSubscribeRTPStatsMutex.RUnlock() argsForCall := fake.trackSubscribeRTPStatsArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4, argsForCall.arg5 + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4, argsForCall.arg5, argsForCall.arg6, argsForCall.arg7 } -func (fake *FakeTelemetryService) TrackSubscribeRequested(arg1 context.Context, arg2 livekit.ParticipantID, arg3 *livekit.TrackInfo) { +func (fake *FakeTelemetryService) TrackSubscribeRequested(arg1 context.Context, arg2 livekit.RoomID, arg3 livekit.RoomName, arg4 livekit.ParticipantID, arg5 *livekit.TrackInfo) { fake.trackSubscribeRequestedMutex.Lock() fake.trackSubscribeRequestedArgsForCall = append(fake.trackSubscribeRequestedArgsForCall, struct { arg1 context.Context - arg2 livekit.ParticipantID - arg3 *livekit.TrackInfo - }{arg1, arg2, arg3}) + arg2 livekit.RoomID + arg3 livekit.RoomName + arg4 livekit.ParticipantID + arg5 *livekit.TrackInfo + }{arg1, arg2, arg3, arg4, arg5}) stub := fake.TrackSubscribeRequestedStub - fake.recordInvocation("TrackSubscribeRequested", []interface{}{arg1, arg2, arg3}) + fake.recordInvocation("TrackSubscribeRequested", []interface{}{arg1, arg2, arg3, arg4, arg5}) fake.trackSubscribeRequestedMutex.Unlock() if stub != nil { - fake.TrackSubscribeRequestedStub(arg1, arg2, arg3) + fake.TrackSubscribeRequestedStub(arg1, arg2, arg3, arg4, arg5) } } @@ -1347,33 +1478,35 @@ func (fake *FakeTelemetryService) TrackSubscribeRequestedCallCount() int { return len(fake.trackSubscribeRequestedArgsForCall) } -func (fake *FakeTelemetryService) TrackSubscribeRequestedCalls(stub func(context.Context, livekit.ParticipantID, *livekit.TrackInfo)) { +func (fake *FakeTelemetryService) TrackSubscribeRequestedCalls(stub func(context.Context, livekit.RoomID, livekit.RoomName, livekit.ParticipantID, *livekit.TrackInfo)) { fake.trackSubscribeRequestedMutex.Lock() defer fake.trackSubscribeRequestedMutex.Unlock() fake.TrackSubscribeRequestedStub = stub } -func (fake *FakeTelemetryService) TrackSubscribeRequestedArgsForCall(i int) (context.Context, livekit.ParticipantID, *livekit.TrackInfo) { +func (fake *FakeTelemetryService) TrackSubscribeRequestedArgsForCall(i int) (context.Context, livekit.RoomID, livekit.RoomName, livekit.ParticipantID, *livekit.TrackInfo) { fake.trackSubscribeRequestedMutex.RLock() defer fake.trackSubscribeRequestedMutex.RUnlock() argsForCall := fake.trackSubscribeRequestedArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4, argsForCall.arg5 } -func (fake *FakeTelemetryService) TrackSubscribed(arg1 context.Context, arg2 livekit.ParticipantID, arg3 *livekit.TrackInfo, arg4 *livekit.ParticipantInfo, arg5 bool) { +func (fake *FakeTelemetryService) TrackSubscribed(arg1 context.Context, arg2 livekit.RoomID, arg3 livekit.RoomName, arg4 livekit.ParticipantID, arg5 *livekit.TrackInfo, arg6 *livekit.ParticipantInfo, arg7 bool) { fake.trackSubscribedMutex.Lock() fake.trackSubscribedArgsForCall = append(fake.trackSubscribedArgsForCall, struct { arg1 context.Context - arg2 livekit.ParticipantID - arg3 *livekit.TrackInfo - arg4 *livekit.ParticipantInfo - arg5 bool - }{arg1, arg2, arg3, arg4, arg5}) + arg2 livekit.RoomID + arg3 livekit.RoomName + arg4 livekit.ParticipantID + arg5 *livekit.TrackInfo + arg6 *livekit.ParticipantInfo + arg7 bool + }{arg1, arg2, arg3, arg4, arg5, arg6, arg7}) stub := fake.TrackSubscribedStub - fake.recordInvocation("TrackSubscribed", []interface{}{arg1, arg2, arg3, arg4, arg5}) + fake.recordInvocation("TrackSubscribed", []interface{}{arg1, arg2, arg3, arg4, arg5, arg6, arg7}) fake.trackSubscribedMutex.Unlock() if stub != nil { - fake.TrackSubscribedStub(arg1, arg2, arg3, arg4, arg5) + fake.TrackSubscribedStub(arg1, arg2, arg3, arg4, arg5, arg6, arg7) } } @@ -1383,31 +1516,33 @@ func (fake *FakeTelemetryService) TrackSubscribedCallCount() int { return len(fake.trackSubscribedArgsForCall) } -func (fake *FakeTelemetryService) TrackSubscribedCalls(stub func(context.Context, livekit.ParticipantID, *livekit.TrackInfo, *livekit.ParticipantInfo, bool)) { +func (fake *FakeTelemetryService) TrackSubscribedCalls(stub func(context.Context, livekit.RoomID, livekit.RoomName, livekit.ParticipantID, *livekit.TrackInfo, *livekit.ParticipantInfo, bool)) { fake.trackSubscribedMutex.Lock() defer fake.trackSubscribedMutex.Unlock() fake.TrackSubscribedStub = stub } -func (fake *FakeTelemetryService) TrackSubscribedArgsForCall(i int) (context.Context, livekit.ParticipantID, *livekit.TrackInfo, *livekit.ParticipantInfo, bool) { +func (fake *FakeTelemetryService) TrackSubscribedArgsForCall(i int) (context.Context, livekit.RoomID, livekit.RoomName, livekit.ParticipantID, *livekit.TrackInfo, *livekit.ParticipantInfo, bool) { fake.trackSubscribedMutex.RLock() defer fake.trackSubscribedMutex.RUnlock() argsForCall := fake.trackSubscribedArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4, argsForCall.arg5 + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4, argsForCall.arg5, argsForCall.arg6, argsForCall.arg7 } -func (fake *FakeTelemetryService) TrackUnmuted(arg1 context.Context, arg2 livekit.ParticipantID, arg3 *livekit.TrackInfo) { +func (fake *FakeTelemetryService) TrackUnmuted(arg1 context.Context, arg2 livekit.RoomID, arg3 livekit.RoomName, arg4 livekit.ParticipantID, arg5 *livekit.TrackInfo) { fake.trackUnmutedMutex.Lock() fake.trackUnmutedArgsForCall = append(fake.trackUnmutedArgsForCall, struct { arg1 context.Context - arg2 livekit.ParticipantID - arg3 *livekit.TrackInfo - }{arg1, arg2, arg3}) + arg2 livekit.RoomID + arg3 livekit.RoomName + arg4 livekit.ParticipantID + arg5 *livekit.TrackInfo + }{arg1, arg2, arg3, arg4, arg5}) stub := fake.TrackUnmutedStub - fake.recordInvocation("TrackUnmuted", []interface{}{arg1, arg2, arg3}) + fake.recordInvocation("TrackUnmuted", []interface{}{arg1, arg2, arg3, arg4, arg5}) fake.trackUnmutedMutex.Unlock() if stub != nil { - fake.TrackUnmutedStub(arg1, arg2, arg3) + fake.TrackUnmutedStub(arg1, arg2, arg3, arg4, arg5) } } @@ -1417,33 +1552,35 @@ func (fake *FakeTelemetryService) TrackUnmutedCallCount() int { return len(fake.trackUnmutedArgsForCall) } -func (fake *FakeTelemetryService) TrackUnmutedCalls(stub func(context.Context, livekit.ParticipantID, *livekit.TrackInfo)) { +func (fake *FakeTelemetryService) TrackUnmutedCalls(stub func(context.Context, livekit.RoomID, livekit.RoomName, livekit.ParticipantID, *livekit.TrackInfo)) { fake.trackUnmutedMutex.Lock() defer fake.trackUnmutedMutex.Unlock() fake.TrackUnmutedStub = stub } -func (fake *FakeTelemetryService) TrackUnmutedArgsForCall(i int) (context.Context, livekit.ParticipantID, *livekit.TrackInfo) { +func (fake *FakeTelemetryService) TrackUnmutedArgsForCall(i int) (context.Context, livekit.RoomID, livekit.RoomName, livekit.ParticipantID, *livekit.TrackInfo) { fake.trackUnmutedMutex.RLock() defer fake.trackUnmutedMutex.RUnlock() argsForCall := fake.trackUnmutedArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4, argsForCall.arg5 } -func (fake *FakeTelemetryService) TrackUnpublished(arg1 context.Context, arg2 livekit.ParticipantID, arg3 livekit.ParticipantIdentity, arg4 *livekit.TrackInfo, arg5 bool) { +func (fake *FakeTelemetryService) TrackUnpublished(arg1 context.Context, arg2 livekit.RoomID, arg3 livekit.RoomName, arg4 livekit.ParticipantID, arg5 livekit.ParticipantIdentity, arg6 *livekit.TrackInfo, arg7 bool) { fake.trackUnpublishedMutex.Lock() fake.trackUnpublishedArgsForCall = append(fake.trackUnpublishedArgsForCall, struct { arg1 context.Context - arg2 livekit.ParticipantID - arg3 livekit.ParticipantIdentity - arg4 *livekit.TrackInfo - arg5 bool - }{arg1, arg2, arg3, arg4, arg5}) + arg2 livekit.RoomID + arg3 livekit.RoomName + arg4 livekit.ParticipantID + arg5 livekit.ParticipantIdentity + arg6 *livekit.TrackInfo + arg7 bool + }{arg1, arg2, arg3, arg4, arg5, arg6, arg7}) stub := fake.TrackUnpublishedStub - fake.recordInvocation("TrackUnpublished", []interface{}{arg1, arg2, arg3, arg4, arg5}) + fake.recordInvocation("TrackUnpublished", []interface{}{arg1, arg2, arg3, arg4, arg5, arg6, arg7}) fake.trackUnpublishedMutex.Unlock() if stub != nil { - fake.TrackUnpublishedStub(arg1, arg2, arg3, arg4, arg5) + fake.TrackUnpublishedStub(arg1, arg2, arg3, arg4, arg5, arg6, arg7) } } @@ -1453,32 +1590,34 @@ func (fake *FakeTelemetryService) TrackUnpublishedCallCount() int { return len(fake.trackUnpublishedArgsForCall) } -func (fake *FakeTelemetryService) TrackUnpublishedCalls(stub func(context.Context, livekit.ParticipantID, livekit.ParticipantIdentity, *livekit.TrackInfo, bool)) { +func (fake *FakeTelemetryService) TrackUnpublishedCalls(stub func(context.Context, livekit.RoomID, livekit.RoomName, livekit.ParticipantID, livekit.ParticipantIdentity, *livekit.TrackInfo, bool)) { fake.trackUnpublishedMutex.Lock() defer fake.trackUnpublishedMutex.Unlock() fake.TrackUnpublishedStub = stub } -func (fake *FakeTelemetryService) TrackUnpublishedArgsForCall(i int) (context.Context, livekit.ParticipantID, livekit.ParticipantIdentity, *livekit.TrackInfo, bool) { +func (fake *FakeTelemetryService) TrackUnpublishedArgsForCall(i int) (context.Context, livekit.RoomID, livekit.RoomName, livekit.ParticipantID, livekit.ParticipantIdentity, *livekit.TrackInfo, bool) { fake.trackUnpublishedMutex.RLock() defer fake.trackUnpublishedMutex.RUnlock() argsForCall := fake.trackUnpublishedArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4, argsForCall.arg5 + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4, argsForCall.arg5, argsForCall.arg6, argsForCall.arg7 } -func (fake *FakeTelemetryService) TrackUnsubscribed(arg1 context.Context, arg2 livekit.ParticipantID, arg3 *livekit.TrackInfo, arg4 bool) { +func (fake *FakeTelemetryService) TrackUnsubscribed(arg1 context.Context, arg2 livekit.RoomID, arg3 livekit.RoomName, arg4 livekit.ParticipantID, arg5 *livekit.TrackInfo, arg6 bool) { fake.trackUnsubscribedMutex.Lock() fake.trackUnsubscribedArgsForCall = append(fake.trackUnsubscribedArgsForCall, struct { arg1 context.Context - arg2 livekit.ParticipantID - arg3 *livekit.TrackInfo - arg4 bool - }{arg1, arg2, arg3, arg4}) + arg2 livekit.RoomID + arg3 livekit.RoomName + arg4 livekit.ParticipantID + arg5 *livekit.TrackInfo + arg6 bool + }{arg1, arg2, arg3, arg4, arg5, arg6}) stub := fake.TrackUnsubscribedStub - fake.recordInvocation("TrackUnsubscribed", []interface{}{arg1, arg2, arg3, arg4}) + fake.recordInvocation("TrackUnsubscribed", []interface{}{arg1, arg2, arg3, arg4, arg5, arg6}) fake.trackUnsubscribedMutex.Unlock() if stub != nil { - fake.TrackUnsubscribedStub(arg1, arg2, arg3, arg4) + fake.TrackUnsubscribedStub(arg1, arg2, arg3, arg4, arg5, arg6) } } @@ -1488,17 +1627,17 @@ func (fake *FakeTelemetryService) TrackUnsubscribedCallCount() int { return len(fake.trackUnsubscribedArgsForCall) } -func (fake *FakeTelemetryService) TrackUnsubscribedCalls(stub func(context.Context, livekit.ParticipantID, *livekit.TrackInfo, bool)) { +func (fake *FakeTelemetryService) TrackUnsubscribedCalls(stub func(context.Context, livekit.RoomID, livekit.RoomName, livekit.ParticipantID, *livekit.TrackInfo, bool)) { fake.trackUnsubscribedMutex.Lock() defer fake.trackUnsubscribedMutex.Unlock() fake.TrackUnsubscribedStub = stub } -func (fake *FakeTelemetryService) TrackUnsubscribedArgsForCall(i int) (context.Context, livekit.ParticipantID, *livekit.TrackInfo, bool) { +func (fake *FakeTelemetryService) TrackUnsubscribedArgsForCall(i int) (context.Context, livekit.RoomID, livekit.RoomName, livekit.ParticipantID, *livekit.TrackInfo, bool) { fake.trackUnsubscribedMutex.RLock() defer fake.trackUnsubscribedMutex.RUnlock() argsForCall := fake.trackUnsubscribedArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4 + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4, argsForCall.arg5, argsForCall.arg6 } func (fake *FakeTelemetryService) Webhook(arg1 context.Context, arg2 *livekit.WebhookInfo) { @@ -1537,80 +1676,6 @@ func (fake *FakeTelemetryService) WebhookArgsForCall(i int) (context.Context, *l func (fake *FakeTelemetryService) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() - fake.aPICallMutex.RLock() - defer fake.aPICallMutex.RUnlock() - fake.egressEndedMutex.RLock() - defer fake.egressEndedMutex.RUnlock() - fake.egressStartedMutex.RLock() - defer fake.egressStartedMutex.RUnlock() - fake.egressUpdatedMutex.RLock() - defer fake.egressUpdatedMutex.RUnlock() - fake.flushStatsMutex.RLock() - defer fake.flushStatsMutex.RUnlock() - fake.ingressCreatedMutex.RLock() - defer fake.ingressCreatedMutex.RUnlock() - fake.ingressDeletedMutex.RLock() - defer fake.ingressDeletedMutex.RUnlock() - fake.ingressEndedMutex.RLock() - defer fake.ingressEndedMutex.RUnlock() - fake.ingressStartedMutex.RLock() - defer fake.ingressStartedMutex.RUnlock() - fake.ingressUpdatedMutex.RLock() - defer fake.ingressUpdatedMutex.RUnlock() - fake.localRoomStateMutex.RLock() - defer fake.localRoomStateMutex.RUnlock() - fake.notifyEventMutex.RLock() - defer fake.notifyEventMutex.RUnlock() - fake.participantActiveMutex.RLock() - defer fake.participantActiveMutex.RUnlock() - fake.participantJoinedMutex.RLock() - defer fake.participantJoinedMutex.RUnlock() - fake.participantLeftMutex.RLock() - defer fake.participantLeftMutex.RUnlock() - fake.participantResumedMutex.RLock() - defer fake.participantResumedMutex.RUnlock() - fake.reportMutex.RLock() - defer fake.reportMutex.RUnlock() - fake.roomEndedMutex.RLock() - defer fake.roomEndedMutex.RUnlock() - fake.roomStartedMutex.RLock() - defer fake.roomStartedMutex.RUnlock() - fake.sendEventMutex.RLock() - defer fake.sendEventMutex.RUnlock() - fake.sendNodeRoomStatesMutex.RLock() - defer fake.sendNodeRoomStatesMutex.RUnlock() - fake.sendStatsMutex.RLock() - defer fake.sendStatsMutex.RUnlock() - fake.trackMaxSubscribedVideoQualityMutex.RLock() - defer fake.trackMaxSubscribedVideoQualityMutex.RUnlock() - fake.trackMutedMutex.RLock() - defer fake.trackMutedMutex.RUnlock() - fake.trackPublishRTPStatsMutex.RLock() - defer fake.trackPublishRTPStatsMutex.RUnlock() - fake.trackPublishRequestedMutex.RLock() - defer fake.trackPublishRequestedMutex.RUnlock() - fake.trackPublishedMutex.RLock() - defer fake.trackPublishedMutex.RUnlock() - fake.trackPublishedUpdateMutex.RLock() - defer fake.trackPublishedUpdateMutex.RUnlock() - fake.trackStatsMutex.RLock() - defer fake.trackStatsMutex.RUnlock() - fake.trackSubscribeFailedMutex.RLock() - defer fake.trackSubscribeFailedMutex.RUnlock() - fake.trackSubscribeRTPStatsMutex.RLock() - defer fake.trackSubscribeRTPStatsMutex.RUnlock() - fake.trackSubscribeRequestedMutex.RLock() - defer fake.trackSubscribeRequestedMutex.RUnlock() - fake.trackSubscribedMutex.RLock() - defer fake.trackSubscribedMutex.RUnlock() - fake.trackUnmutedMutex.RLock() - defer fake.trackUnmutedMutex.RUnlock() - fake.trackUnpublishedMutex.RLock() - defer fake.trackUnpublishedMutex.RUnlock() - fake.trackUnsubscribedMutex.RLock() - defer fake.trackUnsubscribedMutex.RUnlock() - fake.webhookMutex.RLock() - defer fake.webhookMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} for key, value := range fake.invocations { copiedInvocations[key] = value diff --git a/pkg/telemetry/telemetryservice.go b/pkg/telemetry/telemetryservice.go index d17099d..2ab77f5 100644 --- a/pkg/telemetry/telemetryservice.go +++ b/pkg/telemetry/telemetryservice.go @@ -19,8 +19,8 @@ import ( "sync" "time" - "github.com/livekit/livekit-server/pkg/sfu/mime" "github.com/livekit/livekit-server/pkg/utils" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" "github.com/livekit/protocol/webhook" @@ -31,70 +31,145 @@ import ( //counterfeiter:generate . TelemetryService type TelemetryService interface { // TrackStats is called periodically for each track in both directions (published/subscribed) - TrackStats(key StatsKey, stat *livekit.AnalyticsStat) + TrackStats(roomID livekit.RoomID, roomName livekit.RoomName, key StatsKey, stat *livekit.AnalyticsStat) // events RoomStarted(ctx context.Context, room *livekit.Room) RoomEnded(ctx context.Context, room *livekit.Room) + // ParticipantJoined - a participant establishes signal connection to a room - ParticipantJoined(ctx context.Context, room *livekit.Room, participant *livekit.ParticipantInfo, clientInfo *livekit.ClientInfo, clientMeta *livekit.AnalyticsClientMeta, shouldSendEvent bool) + ParticipantJoined(ctx context.Context, room *livekit.Room, participant *livekit.ParticipantInfo, clientInfo *livekit.ClientInfo, clientMeta *livekit.AnalyticsClientMeta, shouldSendEvent bool, guard *ReferenceGuard) // ParticipantActive - a participant establishes media connection - ParticipantActive(ctx context.Context, room *livekit.Room, participant *livekit.ParticipantInfo, clientMeta *livekit.AnalyticsClientMeta, isMigration bool) + ParticipantActive(ctx context.Context, room *livekit.Room, participant *livekit.ParticipantInfo, clientMeta *livekit.AnalyticsClientMeta, isMigration bool, guard *ReferenceGuard) // ParticipantResumed - there has been an ICE restart or connection resume attempt, and we've received their signal connection ParticipantResumed(ctx context.Context, room *livekit.Room, participant *livekit.ParticipantInfo, nodeID livekit.NodeID, reason livekit.ReconnectReason) // ParticipantLeft - the participant leaves the room, only sent if ParticipantActive has been called before - ParticipantLeft(ctx context.Context, room *livekit.Room, participant *livekit.ParticipantInfo, shouldSendEvent bool) + ParticipantLeft(ctx context.Context, room *livekit.Room, participant *livekit.ParticipantInfo, shouldSendEvent bool, guard *ReferenceGuard) // TrackPublishRequested - a publication attempt has been received - TrackPublishRequested(ctx context.Context, participantID livekit.ParticipantID, identity livekit.ParticipantIdentity, track *livekit.TrackInfo) + TrackPublishRequested(ctx context.Context, roomID livekit.RoomID, roomName livekit.RoomName, participantID livekit.ParticipantID, identity livekit.ParticipantIdentity, track *livekit.TrackInfo) // TrackPublished - a publication attempt has been successful - TrackPublished(ctx context.Context, participantID livekit.ParticipantID, identity livekit.ParticipantIdentity, track *livekit.TrackInfo) + TrackPublished(ctx context.Context, roomID livekit.RoomID, roomName livekit.RoomName, participantID livekit.ParticipantID, identity livekit.ParticipantIdentity, track *livekit.TrackInfo, shouldSendEvent bool) // TrackUnpublished - a participant unpublished a track - TrackUnpublished(ctx context.Context, participantID livekit.ParticipantID, identity livekit.ParticipantIdentity, track *livekit.TrackInfo, shouldSendEvent bool) + TrackUnpublished(ctx context.Context, roomID livekit.RoomID, roomName livekit.RoomName, participantID livekit.ParticipantID, identity livekit.ParticipantIdentity, track *livekit.TrackInfo, shouldSendEvent bool) // TrackSubscribeRequested - a participant requested to subscribe to a track - TrackSubscribeRequested(ctx context.Context, participantID livekit.ParticipantID, track *livekit.TrackInfo) + TrackSubscribeRequested(ctx context.Context, roomID livekit.RoomID, roomName livekit.RoomName, participantID livekit.ParticipantID, track *livekit.TrackInfo) // TrackSubscribed - a participant subscribed to a track successfully - TrackSubscribed(ctx context.Context, participantID livekit.ParticipantID, track *livekit.TrackInfo, publisher *livekit.ParticipantInfo, shouldSendEvent bool) + TrackSubscribed(ctx context.Context, roomID livekit.RoomID, roomName livekit.RoomName, participantID livekit.ParticipantID, track *livekit.TrackInfo, publisher *livekit.ParticipantInfo, shouldSendEvent bool) // TrackUnsubscribed - a participant unsubscribed from a track successfully - TrackUnsubscribed(ctx context.Context, participantID livekit.ParticipantID, track *livekit.TrackInfo, shouldSendEvent bool) + TrackUnsubscribed(ctx context.Context, roomID livekit.RoomID, roomName livekit.RoomName, participantID livekit.ParticipantID, track *livekit.TrackInfo, shouldSendEvent bool) // TrackSubscribeFailed - failure to subscribe to a track - TrackSubscribeFailed(ctx context.Context, participantID livekit.ParticipantID, trackID livekit.TrackID, err error, isUserError bool) + TrackSubscribeFailed(ctx context.Context, roomID livekit.RoomID, roomName livekit.RoomName, participantID livekit.ParticipantID, trackID livekit.TrackID, err error, isUserError bool) // TrackMuted - the publisher has muted the Track - TrackMuted(ctx context.Context, participantID livekit.ParticipantID, track *livekit.TrackInfo) + TrackMuted(ctx context.Context, roomID livekit.RoomID, roomName livekit.RoomName, participantID livekit.ParticipantID, track *livekit.TrackInfo) // TrackUnmuted - the publisher has muted the Track - TrackUnmuted(ctx context.Context, participantID livekit.ParticipantID, track *livekit.TrackInfo) + TrackUnmuted(ctx context.Context, roomID livekit.RoomID, roomName livekit.RoomName, participantID livekit.ParticipantID, track *livekit.TrackInfo) // TrackPublishedUpdate - track metadata has been updated - TrackPublishedUpdate(ctx context.Context, participantID livekit.ParticipantID, track *livekit.TrackInfo) + TrackPublishedUpdate(ctx context.Context, roomID livekit.RoomID, roomName livekit.RoomName, participantID livekit.ParticipantID, track *livekit.TrackInfo) // TrackMaxSubscribedVideoQuality - publisher is notified of the max quality subscribers desire - TrackMaxSubscribedVideoQuality(ctx context.Context, participantID livekit.ParticipantID, track *livekit.TrackInfo, mime mime.MimeType, maxQuality livekit.VideoQuality) - TrackPublishRTPStats(ctx context.Context, participantID livekit.ParticipantID, trackID livekit.TrackID, mimeType mime.MimeType, layer int, stats *livekit.RTPStats) - TrackSubscribeRTPStats(ctx context.Context, participantID livekit.ParticipantID, trackID livekit.TrackID, mimeType mime.MimeType, stats *livekit.RTPStats) + TrackMaxSubscribedVideoQuality(ctx context.Context, roomID livekit.RoomID, roomName livekit.RoomName, participantID livekit.ParticipantID, track *livekit.TrackInfo, mime mime.MimeType, maxQuality livekit.VideoQuality) + TrackPublishRTPStats(ctx context.Context, roomID livekit.RoomID, roomName livekit.RoomName, participantID livekit.ParticipantID, trackID livekit.TrackID, mimeType mime.MimeType, layer int, stats *livekit.RTPStats) + TrackSubscribeRTPStats(ctx context.Context, roomID livekit.RoomID, roomName livekit.RoomName, participantID livekit.ParticipantID, trackID livekit.TrackID, mimeType mime.MimeType, stats *livekit.RTPStats) + EgressStarted(ctx context.Context, info *livekit.EgressInfo) EgressUpdated(ctx context.Context, info *livekit.EgressInfo) EgressEnded(ctx context.Context, info *livekit.EgressInfo) + IngressCreated(ctx context.Context, info *livekit.IngressInfo) IngressDeleted(ctx context.Context, info *livekit.IngressInfo) IngressStarted(ctx context.Context, info *livekit.IngressInfo) IngressUpdated(ctx context.Context, info *livekit.IngressInfo) IngressEnded(ctx context.Context, info *livekit.IngressInfo) + LocalRoomState(ctx context.Context, info *livekit.AnalyticsNodeRooms) + Report(ctx context.Context, reportInfo *livekit.ReportInfo) + APICall(ctx context.Context, apiCallInfo *livekit.APICallInfo) + Webhook(ctx context.Context, webhookInfo *livekit.WebhookInfo) // helpers AnalyticsService - NotifyEvent(ctx context.Context, event *livekit.WebhookEvent) + NotifyEgressEvent(ctx context.Context, event string, info *livekit.EgressInfo) FlushStats() } +// ----------------------------- + +var _ TelemetryService = (*NullTelemetryService)(nil) + +type NullTelemetryService struct { + NullAnalyticService +} + +func (n NullTelemetryService) TrackStats(roomID livekit.RoomID, roomName livekit.RoomName, key StatsKey, stat *livekit.AnalyticsStat) { +} +func (n NullTelemetryService) RoomStarted(ctx context.Context, room *livekit.Room) {} +func (n NullTelemetryService) RoomEnded(ctx context.Context, room *livekit.Room) {} +func (n NullTelemetryService) ParticipantJoined(ctx context.Context, room *livekit.Room, participant *livekit.ParticipantInfo, clientInfo *livekit.ClientInfo, clientMeta *livekit.AnalyticsClientMeta, shouldSendEvent bool, guard *ReferenceGuard) { +} +func (n NullTelemetryService) ParticipantActive(ctx context.Context, room *livekit.Room, participant *livekit.ParticipantInfo, clientMeta *livekit.AnalyticsClientMeta, isMigration bool, guard *ReferenceGuard) { +} +func (n NullTelemetryService) ParticipantResumed(ctx context.Context, room *livekit.Room, participant *livekit.ParticipantInfo, nodeID livekit.NodeID, reason livekit.ReconnectReason) { +} +func (n NullTelemetryService) ParticipantLeft(ctx context.Context, room *livekit.Room, participant *livekit.ParticipantInfo, shouldSendEvent bool, guard *ReferenceGuard) { +} +func (n NullTelemetryService) TrackPublishRequested(ctx context.Context, roomID livekit.RoomID, roomName livekit.RoomName, participantID livekit.ParticipantID, identity livekit.ParticipantIdentity, track *livekit.TrackInfo) { +} +func (n NullTelemetryService) TrackPublished(ctx context.Context, roomID livekit.RoomID, roomName livekit.RoomName, participantID livekit.ParticipantID, identity livekit.ParticipantIdentity, track *livekit.TrackInfo, shouldSendEvent bool) { +} +func (n NullTelemetryService) TrackUnpublished(ctx context.Context, roomID livekit.RoomID, roomName livekit.RoomName, participantID livekit.ParticipantID, identity livekit.ParticipantIdentity, track *livekit.TrackInfo, shouldSendEvent bool) { +} +func (n NullTelemetryService) TrackSubscribeRequested(ctx context.Context, roomID livekit.RoomID, roomName livekit.RoomName, participantID livekit.ParticipantID, track *livekit.TrackInfo) { +} +func (n NullTelemetryService) TrackSubscribed(ctx context.Context, roomID livekit.RoomID, roomName livekit.RoomName, participantID livekit.ParticipantID, track *livekit.TrackInfo, publisher *livekit.ParticipantInfo, shouldSendEvent bool) { +} +func (n NullTelemetryService) TrackUnsubscribed(ctx context.Context, roomID livekit.RoomID, roomName livekit.RoomName, participantID livekit.ParticipantID, track *livekit.TrackInfo, shouldSendEvent bool) { +} +func (n NullTelemetryService) TrackSubscribeFailed(ctx context.Context, roomID livekit.RoomID, roomName livekit.RoomName, participantID livekit.ParticipantID, trackID livekit.TrackID, err error, isUserError bool) { +} +func (n NullTelemetryService) TrackMuted(ctx context.Context, roomID livekit.RoomID, roomName livekit.RoomName, participantID livekit.ParticipantID, track *livekit.TrackInfo) { +} +func (n NullTelemetryService) TrackUnmuted(ctx context.Context, roomID livekit.RoomID, roomName livekit.RoomName, participantID livekit.ParticipantID, track *livekit.TrackInfo) { +} +func (n NullTelemetryService) TrackPublishedUpdate(ctx context.Context, roomID livekit.RoomID, roomName livekit.RoomName, participantID livekit.ParticipantID, track *livekit.TrackInfo) { +} +func (n NullTelemetryService) TrackMaxSubscribedVideoQuality(ctx context.Context, roomID livekit.RoomID, roomName livekit.RoomName, participantID livekit.ParticipantID, track *livekit.TrackInfo, mime mime.MimeType, maxQuality livekit.VideoQuality) { +} +func (n NullTelemetryService) TrackPublishRTPStats(ctx context.Context, roomID livekit.RoomID, roomName livekit.RoomName, participantID livekit.ParticipantID, trackID livekit.TrackID, mimeType mime.MimeType, layer int, stats *livekit.RTPStats) { +} +func (n NullTelemetryService) TrackSubscribeRTPStats(ctx context.Context, roomID livekit.RoomID, roomName livekit.RoomName, participantID livekit.ParticipantID, trackID livekit.TrackID, mimeType mime.MimeType, stats *livekit.RTPStats) { +} +func (n NullTelemetryService) EgressStarted(ctx context.Context, info *livekit.EgressInfo) {} +func (n NullTelemetryService) EgressUpdated(ctx context.Context, info *livekit.EgressInfo) {} +func (n NullTelemetryService) EgressEnded(ctx context.Context, info *livekit.EgressInfo) {} +func (n NullTelemetryService) IngressCreated(ctx context.Context, info *livekit.IngressInfo) {} +func (n NullTelemetryService) IngressDeleted(ctx context.Context, info *livekit.IngressInfo) {} +func (n NullTelemetryService) IngressStarted(ctx context.Context, info *livekit.IngressInfo) {} +func (n NullTelemetryService) IngressUpdated(ctx context.Context, info *livekit.IngressInfo) {} +func (n NullTelemetryService) IngressEnded(ctx context.Context, info *livekit.IngressInfo) {} +func (n NullTelemetryService) LocalRoomState(ctx context.Context, info *livekit.AnalyticsNodeRooms) {} +func (n NullTelemetryService) Report(ctx context.Context, reportInfo *livekit.ReportInfo) {} +func (n NullTelemetryService) APICall(ctx context.Context, apiCallInfo *livekit.APICallInfo) {} +func (n NullTelemetryService) Webhook(ctx context.Context, webhookInfo *livekit.WebhookInfo) {} +func (n NullTelemetryService) NotifyEgressEvent(ctx context.Context, event string, info *livekit.EgressInfo) { +} +func (n NullTelemetryService) FlushStats() {} + +// ----------------------------- + const ( workerCleanupWait = 3 * time.Minute jobsQueueMinSize = 2048 - telemetryStatsUpdateInterval = time.Second * 30 - telemetryNonMediaStatsUpdateInterval = time.Second * 30 + telemetryStatsUpdateInterval = time.Second * 30 ) +type statsWorkerKey struct { + roomID livekit.RoomID + participantID livekit.ParticipantID +} + type telemetryService struct { AnalyticsService @@ -102,7 +177,7 @@ type telemetryService struct { jobsQueue *utils.OpsQueue workersMu sync.RWMutex - workers map[livekit.ParticipantID]*StatsWorker + workers map[statsWorkerKey]*StatsWorker workerList *StatsWorker flushMu sync.Mutex @@ -118,12 +193,7 @@ func NewTelemetryService(notifier webhook.QueuedNotifier, analytics AnalyticsSer FlushOnStop: true, Logger: logger.GetLogger(), }), - workers: make(map[livekit.ParticipantID]*StatsWorker), - } - if t.notifier != nil { - t.notifier.RegisterProcessedHook(func(ctx context.Context, whi *livekit.WebhookInfo) { - t.Webhook(ctx, whi) - }) + workers: make(map[statsWorkerKey]*StatsWorker), } t.jobsQueue.Start() @@ -144,7 +214,7 @@ func (t *telemetryService) FlushStats() { var prev, reap *StatsWorker for worker != nil { next := worker.next - if closed := worker.Flush(now); closed { + if closed := worker.Flush(now, workerCleanupWait); closed { if prev == nil { // this worker was at the head of the list t.workersMu.Lock() @@ -172,8 +242,9 @@ func (t *telemetryService) FlushStats() { if reap != nil { t.workersMu.Lock() for reap != nil { - if reap == t.workers[reap.participantID] { - delete(t.workers, reap.participantID) + key := statsWorkerKey{reap.roomID, reap.participantID} + if reap == t.workers[key] { + delete(t.workers, key) } reap = reap.next } @@ -191,11 +262,11 @@ func (t *telemetryService) enqueue(op func()) { t.jobsQueue.Enqueue(op) } -func (t *telemetryService) getWorker(participantID livekit.ParticipantID) (worker *StatsWorker, ok bool) { +func (t *telemetryService) getWorker(roomID livekit.RoomID, participantID livekit.ParticipantID) (worker *StatsWorker, ok bool) { t.workersMu.RLock() defer t.workersMu.RUnlock() - worker, ok = t.workers[participantID] + worker, ok = t.workers[statsWorkerKey{roomID, participantID}] return } @@ -205,12 +276,14 @@ func (t *telemetryService) getOrCreateWorker( roomName livekit.RoomName, participantID livekit.ParticipantID, participantIdentity livekit.ParticipantIdentity, + guard *ReferenceGuard, ) (*StatsWorker, bool) { t.workersMu.Lock() defer t.workersMu.Unlock() - worker, ok := t.workers[participantID] - if ok && !worker.Closed() { + key := statsWorkerKey{roomID, participantID} + worker, ok := t.workers[key] + if ok && !worker.Closed(guard) { return worker, true } @@ -226,12 +299,13 @@ func (t *telemetryService) getOrCreateWorker( roomName, participantID, participantIdentity, + guard, ) if existingIsConnected { worker.SetConnected() } - t.workers[participantID] = worker + t.workers[key] = worker worker.next = t.workerList t.workerList = worker diff --git a/pkg/utils/incrementaldispatcher.go b/pkg/utils/incrementaldispatcher.go index 2aaaf9f..845ed3e 100644 --- a/pkg/utils/incrementaldispatcher.go +++ b/pkg/utils/incrementaldispatcher.go @@ -43,8 +43,8 @@ func (d *IncrementalDispatcher[T]) Add(item T) { } d.lock.Lock() d.items = append(d.items, item) - d.lock.Unlock() d.cond.Broadcast() + d.lock.Unlock() } func (d *IncrementalDispatcher[T]) Done() { diff --git a/pkg/utils/incrementaldispatcher_test.go b/pkg/utils/incrementaldispatcher_test.go index 3e2d40e..4b756b9 100644 --- a/pkg/utils/incrementaldispatcher_test.go +++ b/pkg/utils/incrementaldispatcher_test.go @@ -53,7 +53,7 @@ func TestConcurrentConsumption(t *testing.T) { sums := make([]atomic.Int32, numConsumers) var wg sync.WaitGroup - for i := 0; i < numConsumers; i++ { + for i := range numConsumers { wg.Add(1) i := i go func() { @@ -66,12 +66,12 @@ func TestConcurrentConsumption(t *testing.T) { // Add items expectedSum := 0 - for i := 0; i < 20; i++ { + for i := range 20 { expectedSum += i producer.Add(i) } - for i := 0; i < numConsumers; i++ { + for i := range numConsumers { testutils.WithTimeout(t, func() string { if sums[i].Load() != int32(expectedSum) { return fmt.Sprintf("consumer %d did not consume all the items. expected %d, actual: %d", @@ -91,7 +91,7 @@ func TestConcurrentConsumption(t *testing.T) { producer.Done() wg.Wait() - for i := 0; i < numConsumers; i++ { + for i := range numConsumers { require.Equal(t, int32(expectedSum), sums[i].Load(), "consumer %d did not match", i) } } diff --git a/pkg/utils/math.go b/pkg/utils/math.go index 9e75c1a..3a419db 100644 --- a/pkg/utils/math.go +++ b/pkg/utils/math.go @@ -14,19 +14,18 @@ package utils -import "sort" +import "slices" -// MedianFloat32 gets median value for an array of float32 -func MedianFloat32(input []float32) float32 { +// Median gets median value for an array +func Median[T float32](input []T) T { num := len(input) - if num == 0 { + switch num { + case 0: return 0 - } else if num == 1 { + case 1: return input[0] } - sort.Slice(input, func(i, j int) bool { - return input[i] < input[j] - }) + slices.Sort(input) if num%2 != 0 { return input[num/2] } @@ -34,3 +33,16 @@ func MedianFloat32(input []float32) float32 { right := input[num/2] return (left + right) / 2 } + +func Signum[T int | int8 | int16 | int32 | int64 | float32 | float64](val T) int { + switch { + case val < 0: + return -1 + + case val > 0: + return 1 + + default: + return 0 + } +} diff --git a/pkg/utils/slice.go b/pkg/utils/slice.go new file mode 100644 index 0000000..1d692ef --- /dev/null +++ b/pkg/utils/slice.go @@ -0,0 +1,11 @@ +package utils + +import ( + "cmp" + "slices" +) + +func DedupeSlice[T cmp.Ordered](s []T) []T { + slices.Sort(s) + return slices.Compact(s) +} diff --git a/renovate.json b/renovate.json index b472aef..213b791 100644 --- a/renovate.json +++ b/renovate.json @@ -1,6 +1,7 @@ { "$schema": "https://docs.renovatebot.com/renovate-schema.json", - "extends": ["config:base"], + "extends": ["config:base", "helpers:pinGitHubActionDigests"], + "minimumReleaseAge": "2 weeks", "commitBody": "Generated by renovateBot", "packageRules": [ { diff --git a/test/agent_test.go b/test/agent_test.go index 795fa11..9b31452 100644 --- a/test/agent_test.go +++ b/test/agent_test.go @@ -32,194 +32,205 @@ var ( ) func TestAgents(t *testing.T) { - _, finish := setupSingleNodeTest("TestAgents") - defer finish() + for _, testRTCServicePath := range testRTCServicePaths { + t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) { + _, finish := setupSingleNodeTest("TestAgents") + defer finish() - ac1, err := newAgentClient(agentToken(), defaultServerPort) - require.NoError(t, err) - ac2, err := newAgentClient(agentToken(), defaultServerPort) - require.NoError(t, err) - ac3, err := newAgentClient(agentToken(), defaultServerPort) - require.NoError(t, err) - ac4, err := newAgentClient(agentToken(), defaultServerPort) - require.NoError(t, err) - ac5, err := newAgentClient(agentToken(), defaultServerPort) - require.NoError(t, err) - ac6, err := newAgentClient(agentToken(), defaultServerPort) - require.NoError(t, err) - defer ac1.close() - defer ac2.close() - defer ac3.close() - defer ac4.close() - defer ac5.close() - defer ac6.close() - ac1.Run(livekit.JobType_JT_ROOM, "default") - ac2.Run(livekit.JobType_JT_ROOM, "default") - ac3.Run(livekit.JobType_JT_PUBLISHER, "default") - ac4.Run(livekit.JobType_JT_PUBLISHER, "default") - ac5.Run(livekit.JobType_JT_PARTICIPANT, "default") - ac6.Run(livekit.JobType_JT_PARTICIPANT, "default") + ac1, err := newAgentClient(agentToken(), defaultServerPort) + require.NoError(t, err) + ac2, err := newAgentClient(agentToken(), defaultServerPort) + require.NoError(t, err) + ac3, err := newAgentClient(agentToken(), defaultServerPort) + require.NoError(t, err) + ac4, err := newAgentClient(agentToken(), defaultServerPort) + require.NoError(t, err) + ac5, err := newAgentClient(agentToken(), defaultServerPort) + require.NoError(t, err) + ac6, err := newAgentClient(agentToken(), defaultServerPort) + require.NoError(t, err) + defer ac1.close() + defer ac2.close() + defer ac3.close() + defer ac4.close() + defer ac5.close() + defer ac6.close() + ac1.Run(livekit.JobType_JT_ROOM, "default") + ac2.Run(livekit.JobType_JT_ROOM, "default") + ac3.Run(livekit.JobType_JT_PUBLISHER, "default") + ac4.Run(livekit.JobType_JT_PUBLISHER, "default") + ac5.Run(livekit.JobType_JT_PARTICIPANT, "default") + ac6.Run(livekit.JobType_JT_PARTICIPANT, "default") - testutils.WithTimeout(t, func() string { - if ac1.registered.Load() != 1 || ac2.registered.Load() != 1 || ac3.registered.Load() != 1 || ac4.registered.Load() != 1 || ac5.registered.Load() != 1 || ac6.registered.Load() != 1 { - return "worker not registered" - } + testutils.WithTimeout(t, func() string { + if ac1.registered.Load() != 1 || ac2.registered.Load() != 1 || ac3.registered.Load() != 1 || ac4.registered.Load() != 1 || ac5.registered.Load() != 1 || ac6.registered.Load() != 1 { + return "worker not registered" + } - return "" - }, RegisterTimeout) + return "" + }, RegisterTimeout) - c1 := createRTCClient("c1", defaultServerPort, nil) - c2 := createRTCClient("c2", defaultServerPort, nil) - waitUntilConnected(t, c1, c2) + c1 := createRTCClient("c1", defaultServerPort, testRTCServicePath, nil) + c2 := createRTCClient("c2", defaultServerPort, testRTCServicePath, nil) + waitUntilConnected(t, c1, c2) - // publish 2 tracks - t1, err := c1.AddStaticTrack("audio/opus", "audio", "micro") - require.NoError(t, err) - defer t1.Stop() - t2, err := c1.AddStaticTrack("video/vp8", "video", "webcam") - require.NoError(t, err) - defer t2.Stop() + // publish 2 tracks + t1, err := c1.AddStaticTrack("audio/opus", "audio", "micro") + require.NoError(t, err) + defer t1.Stop() + t2, err := c1.AddStaticTrack("video/vp8", "video", "webcam") + require.NoError(t, err) + defer t2.Stop() - testutils.WithTimeout(t, func() string { - if ac1.roomJobs.Load()+ac2.roomJobs.Load() != 1 { - return "room job not assigned" - } + testutils.WithTimeout(t, func() string { + if ac1.roomJobs.Load()+ac2.roomJobs.Load() != 1 { + return "room job not assigned" + } - if ac3.publisherJobs.Load()+ac4.publisherJobs.Load() != 1 { - return fmt.Sprintf("publisher jobs not assigned, ac3: %d, ac4: %d", ac3.publisherJobs.Load(), ac4.publisherJobs.Load()) - } + if ac3.publisherJobs.Load()+ac4.publisherJobs.Load() != 1 { + return fmt.Sprintf("publisher jobs not assigned, ac3: %d, ac4: %d", ac3.publisherJobs.Load(), ac4.publisherJobs.Load()) + } - if ac5.participantJobs.Load()+ac6.participantJobs.Load() != 2 { - return fmt.Sprintf("participant jobs not assigned, ac5: %d, ac6: %d", ac5.participantJobs.Load(), ac6.participantJobs.Load()) - } + if ac5.participantJobs.Load()+ac6.participantJobs.Load() != 2 { + return fmt.Sprintf("participant jobs not assigned, ac5: %d, ac6: %d", ac5.participantJobs.Load(), ac6.participantJobs.Load()) + } - return "" - }, 6*time.Second) + return "" + }, 6*time.Second) - // publish 2 tracks - t3, err := c2.AddStaticTrack("audio/opus", "audio", "micro") - require.NoError(t, err) - defer t3.Stop() - t4, err := c2.AddStaticTrack("video/vp8", "video", "webcam") - require.NoError(t, err) - defer t4.Stop() + // publish 2 tracks + t3, err := c2.AddStaticTrack("audio/opus", "audio", "micro") + require.NoError(t, err) + defer t3.Stop() + t4, err := c2.AddStaticTrack("video/vp8", "video", "webcam") + require.NoError(t, err) + defer t4.Stop() - testutils.WithTimeout(t, func() string { - if ac1.roomJobs.Load()+ac2.roomJobs.Load() != 1 { - return "room job must be assigned 1 time" - } + testutils.WithTimeout(t, func() string { + if ac1.roomJobs.Load()+ac2.roomJobs.Load() != 1 { + return "room job must be assigned 1 time" + } - if ac3.publisherJobs.Load()+ac4.publisherJobs.Load() != 2 { - return "2 publisher jobs must assigned" - } + if ac3.publisherJobs.Load()+ac4.publisherJobs.Load() != 2 { + return "2 publisher jobs must assigned" + } - if ac5.participantJobs.Load()+ac6.participantJobs.Load() != 2 { - return "2 participant jobs must assigned" - } + if ac5.participantJobs.Load()+ac6.participantJobs.Load() != 2 { + return "2 participant jobs must assigned" + } - return "" - }, AssignJobTimeout) + return "" + }, AssignJobTimeout) + }) + } } func TestAgentNamespaces(t *testing.T) { - _, finish := setupSingleNodeTest("TestAgentNamespaces") - defer finish() + for _, testRTCServicePath := range testRTCServicePaths { + t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) { + _, finish := setupSingleNodeTest("TestAgentNamespaces") + defer finish() - ac1, err := newAgentClient(agentToken(), defaultServerPort) - require.NoError(t, err) - ac2, err := newAgentClient(agentToken(), defaultServerPort) - require.NoError(t, err) - defer ac1.close() - defer ac2.close() - ac1.Run(livekit.JobType_JT_ROOM, "namespace1") - ac2.Run(livekit.JobType_JT_ROOM, "namespace2") + ac1, err := newAgentClient(agentToken(), defaultServerPort) + require.NoError(t, err) + ac2, err := newAgentClient(agentToken(), defaultServerPort) + require.NoError(t, err) + defer ac1.close() + defer ac2.close() + ac1.Run(livekit.JobType_JT_ROOM, "namespace1") + ac2.Run(livekit.JobType_JT_ROOM, "namespace2") - _, err = roomClient.CreateRoom(contextWithToken(createRoomToken()), &livekit.CreateRoomRequest{ - Name: testRoom, - Agents: []*livekit.RoomAgentDispatch{ - {}, - { - AgentName: "ag", - }, - }, - }) - require.NoError(t, err) + _, err = roomClient.CreateRoom(contextWithToken(createRoomToken()), &livekit.CreateRoomRequest{ + Name: testRoom, + Agents: []*livekit.RoomAgentDispatch{ + {}, + { + AgentName: "ag", + }, + }, + }) + require.NoError(t, err) - testutils.WithTimeout(t, func() string { - if ac1.registered.Load() != 1 || ac2.registered.Load() != 1 { - return "worker not registered" - } - return "" - }, RegisterTimeout) + testutils.WithTimeout(t, func() string { + if ac1.registered.Load() != 1 || ac2.registered.Load() != 1 { + return "worker not registered" + } + return "" + }, RegisterTimeout) - c1 := createRTCClient("c1", defaultServerPort, nil) - waitUntilConnected(t, c1) + c1 := createRTCClient("c1", defaultServerPort, testRTCServicePath, nil) + waitUntilConnected(t, c1) - testutils.WithTimeout(t, func() string { - if ac1.roomJobs.Load() != 1 || ac2.roomJobs.Load() != 1 { - return "room job not assigned" - } + testutils.WithTimeout(t, func() string { + if ac1.roomJobs.Load() != 1 || ac2.roomJobs.Load() != 1 { + return "room job not assigned" + } - job1 := <-ac1.requestedJobs - job2 := <-ac2.requestedJobs + job1 := <-ac1.requestedJobs + job2 := <-ac2.requestedJobs - if job1.Namespace != "namespace1" { - return "namespace is not 'namespace'" - } + if job1.Namespace != "namespace1" { + return "namespace is not 'namespace'" + } - if job2.Namespace != "namespace2" { - return "namespace is not 'namespace2'" - } + if job2.Namespace != "namespace2" { + return "namespace is not 'namespace2'" + } - if job1.Id == job2.Id { - return "job ids are the same" - } - - return "" - }, AssignJobTimeout) + if job1.Id == job2.Id { + return "job ids are the same" + } + return "" + }, AssignJobTimeout) + }) + } } func TestAgentMultiNode(t *testing.T) { - _, _, finish := setupMultiNodeTest("TestAgentMultiNode") - defer finish() + for _, testRTCServicePath := range testRTCServicePaths { + t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) { + _, _, finish := setupMultiNodeTest("TestAgentMultiNode") + defer finish() - ac1, err := newAgentClient(agentToken(), defaultServerPort) - require.NoError(t, err) - ac2, err := newAgentClient(agentToken(), defaultServerPort) - require.NoError(t, err) - defer ac1.close() - defer ac2.close() - ac1.Run(livekit.JobType_JT_ROOM, "default") - ac2.Run(livekit.JobType_JT_PUBLISHER, "default") + ac1, err := newAgentClient(agentToken(), defaultServerPort) + require.NoError(t, err) + ac2, err := newAgentClient(agentToken(), defaultServerPort) + require.NoError(t, err) + defer ac1.close() + defer ac2.close() + ac1.Run(livekit.JobType_JT_ROOM, "default") + ac2.Run(livekit.JobType_JT_PUBLISHER, "default") - testutils.WithTimeout(t, func() string { - if ac1.registered.Load() != 1 || ac2.registered.Load() != 1 { - return "worker not registered" - } - return "" - }, RegisterTimeout) + testutils.WithTimeout(t, func() string { + if ac1.registered.Load() != 1 || ac2.registered.Load() != 1 { + return "worker not registered" + } + return "" + }, RegisterTimeout) - c1 := createRTCClient("c1", secondServerPort, nil) // Create a room on the second node - waitUntilConnected(t, c1) + c1 := createRTCClient("c1", secondServerPort, testRTCServicePath, nil) // Create a room on the second node + waitUntilConnected(t, c1) - t1, err := c1.AddStaticTrack("audio/opus", "audio", "micro") - require.NoError(t, err) - defer t1.Stop() + t1, err := c1.AddStaticTrack("audio/opus", "audio", "micro") + require.NoError(t, err) + defer t1.Stop() - time.Sleep(time.Second * 10) + time.Sleep(time.Second * 10) - testutils.WithTimeout(t, func() string { - if ac1.roomJobs.Load() != 1 { - return "room job not assigned" - } + testutils.WithTimeout(t, func() string { + if ac1.roomJobs.Load() != 1 { + return "room job not assigned" + } - if ac2.publisherJobs.Load() != 1 { - return "participant job not assigned" - } + if ac2.publisherJobs.Load() != 1 { + return "participant job not assigned" + } - return "" - }, AssignJobTimeout) + return "" + }, AssignJobTimeout) + }) + } } func agentToken() string { diff --git a/test/client/client.go b/test/client/client.go index 777c5a1..bb38cda 100644 --- a/test/client/client.go +++ b/test/client/client.go @@ -16,12 +16,17 @@ package client import ( "context" + "encoding/base64" + "encoding/json" "errors" "fmt" "io" + "maps" + "math/rand/v2" "net/http" "net/url" "path/filepath" + "runtime" "sync" "time" @@ -35,14 +40,16 @@ import ( "github.com/livekit/mediatransportutil/pkg/rtcconfig" "github.com/livekit/protocol/auth" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" + "github.com/livekit/protocol/signalling" "github.com/livekit/livekit-server/pkg/rtc" + "github.com/livekit/livekit-server/pkg/rtc/datatrack" "github.com/livekit/livekit-server/pkg/rtc/transport/transportfakes" "github.com/livekit/livekit-server/pkg/rtc/types" "github.com/livekit/livekit-server/pkg/sfu/buffer" - "github.com/livekit/livekit-server/pkg/sfu/mime" ) type SignalRequestHandler func(msg *livekit.SignalRequest) error @@ -51,10 +58,14 @@ type SignalResponseHandler func(msg *livekit.SignalResponse) error type SignalResponseInterceptor func(msg *livekit.SignalResponse, next SignalResponseHandler) error type RTCClient struct { - id livekit.ParticipantID - conn *websocket.Conn - publisher *rtc.PCTransport - subscriber *rtc.PCTransport + useSinglePeerConnection bool + id livekit.ParticipantID + conn *websocket.Conn + publisher *rtc.PCTransport + subscriber *rtc.PCTransport + enabledCodecs []*livekit.Codec + forceRelay bool + transportReady chan struct{} // sid => track localTracks map[string]webrtc.TrackLocal trackSenders map[string]*webrtc.RTPSender @@ -76,21 +87,29 @@ type RTCClient struct { publisherFullyEstablished atomic.Bool subscriberFullyEstablished atomic.Bool pongReceivedAt atomic.Int64 - lastAnswer atomic.Pointer[webrtc.SessionDescription] // tracks waiting to be acked, cid => trackInfo pendingPublishedTracks map[string]*livekit.TrackInfo - pendingTrackWriters []*TrackWriter - OnConnected func() - OnDataReceived func(data []byte, sid string) - refreshToken string + // remote tracks waiting to be processed + pendingRemoteTracks []*webrtc.TrackRemote + + pendingTrackWriters []TrackWriter + OnConnected func() + OnDataReceived func(data []byte, sid string) + OnDataUnlabeledReceived func(data []byte) + refreshToken string // map of livekit.ParticipantID and last packet lastPackets map[livekit.ParticipantID]*rtp.Packet bytesReceived map[livekit.ParticipantID]uint64 subscriptionResponse atomic.Pointer[livekit.SubscriptionResponse] + + nextDataTrackHandle atomic.Uint32 + pendingPublishedDataTracks map[uint16]*livekit.DataTrackInfo + pendingDataTrackWriters []TrackWriter + subscribedDataTracks map[livekit.ParticipantID]map[uint16]*DataTrackRemote } var ( @@ -111,42 +130,94 @@ var ( type Options struct { AutoSubscribe bool + AutoSubscribeDataTrack bool Publish string + Attributes map[string]string ClientInfo *livekit.ClientInfo DisabledCodecs []webrtc.RTPCodecCapability TokenCustomizer func(token *auth.AccessToken, grants *auth.VideoGrant) SignalRequestInterceptor SignalRequestInterceptor SignalResponseInterceptor SignalResponseInterceptor + UseJoinRequestQueryParam bool + RTCServicePath string + ForceRelay bool } func NewWebSocketConn(host, token string, opts *Options) (*websocket.Conn, error) { - u, err := url.Parse(host + fmt.Sprintf("/rtc?protocol=%d", types.CurrentProtocol)) + rtcServicePath := "/rtc" + if opts != nil && opts.RTCServicePath != "" { + rtcServicePath = opts.RTCServicePath + } + parsedURL, err := url.Parse(host + rtcServicePath) if err != nil { return nil, err } requestHeader := make(http.Header) SetAuthorizationToken(requestHeader, token) - connectUrl := u.String() - sdk := "go" - if opts != nil { - connectUrl = fmt.Sprintf("%s&auto_subscribe=%t", connectUrl, opts.AutoSubscribe) - if opts.Publish != "" { - connectUrl += encodeQueryParam("publish", opts.Publish) + connectUrl := parsedURL.String() + if opts != nil && opts.UseJoinRequestQueryParam { + clientInfo := &livekit.ClientInfo{ + Os: runtime.GOOS, + Sdk: livekit.ClientInfo_GO, + Protocol: int32(types.CurrentProtocol), } if opts.ClientInfo != nil { - if opts.ClientInfo.DeviceModel != "" { - connectUrl += encodeQueryParam("device_model", opts.ClientInfo.DeviceModel) + clientInfo = opts.ClientInfo + } + + connectionSettings := &livekit.ConnectionSettings{ + AutoSubscribe: opts.AutoSubscribe, + AutoSubscribeDataTrack: &opts.AutoSubscribeDataTrack, + } + + joinRequest := &livekit.JoinRequest{ + ClientInfo: clientInfo, + ConnectionSettings: connectionSettings, + ParticipantAttributes: opts.Attributes, + } + + if marshalled, err := proto.Marshal(joinRequest); err == nil { + wrapped := &livekit.WrappedJoinRequest{ + JoinRequest: marshalled, } - if opts.ClientInfo.Os != "" { - connectUrl += encodeQueryParam("os", opts.ClientInfo.Os) - } - if opts.ClientInfo.Sdk != livekit.ClientInfo_UNKNOWN { - sdk = opts.ClientInfo.Sdk.String() + if marshalled, err := proto.Marshal(wrapped); err == nil { + connectUrl += fmt.Sprintf("?join_request=%s", base64.URLEncoding.EncodeToString(marshalled)) } } + } else { + connectUrl += fmt.Sprintf("?protocol=%d", types.CurrentProtocol) + + sdk := "go" + if opts != nil { + connectUrl += fmt.Sprintf("&auto_subscribe=%t", opts.AutoSubscribe) + connectUrl += fmt.Sprintf("&auto_subscribe_data_track=%t", opts.AutoSubscribeDataTrack) + if opts.Publish != "" { + connectUrl += encodeQueryParam("publish", opts.Publish) + } + if len(opts.Attributes) != 0 { + data, err := json.Marshal(opts.Attributes) + if err != nil { + return nil, err + } + connectUrl += encodeQueryParam("attributes", base64.URLEncoding.EncodeToString(data)) + } + if opts.ClientInfo != nil { + if opts.ClientInfo.DeviceModel != "" { + connectUrl += encodeQueryParam("device_model", opts.ClientInfo.DeviceModel) + } + if opts.ClientInfo.Os != "" { + connectUrl += encodeQueryParam("os", opts.ClientInfo.Os) + } + if opts.ClientInfo.Sdk != livekit.ClientInfo_UNKNOWN { + sdk = opts.ClientInfo.Sdk.String() + } + } + } + connectUrl += encodeQueryParam("sdk", sdk) } - connectUrl += encodeQueryParam("sdk", sdk) + + logger.Infow("connecting to", "url", parsedURL.String()) conn, _, err := websocket.DefaultDialer.Dial(connectUrl, requestHeader) return conn, err } @@ -155,31 +226,25 @@ func SetAuthorizationToken(header http.Header, token string) { header.Set("Authorization", "Bearer "+token) } -func NewRTCClient(conn *websocket.Conn, opts *Options) (*RTCClient, error) { - var err error - +func NewRTCClient(conn *websocket.Conn, useSinglePeerConnection bool, opts *Options) (*RTCClient, error) { c := &RTCClient{ - conn: conn, - localTracks: make(map[string]webrtc.TrackLocal), - trackSenders: make(map[string]*webrtc.RTPSender), - pendingPublishedTracks: make(map[string]*livekit.TrackInfo), - subscribedTracks: make(map[livekit.ParticipantID][]*webrtc.TrackRemote), - remoteParticipants: make(map[livekit.ParticipantID]*livekit.ParticipantInfo), - me: &webrtc.MediaEngine{}, - lastPackets: make(map[livekit.ParticipantID]*rtp.Packet), - bytesReceived: make(map[livekit.ParticipantID]uint64), + useSinglePeerConnection: useSinglePeerConnection, + conn: conn, + localTracks: make(map[string]webrtc.TrackLocal), + trackSenders: make(map[string]*webrtc.RTPSender), + pendingPublishedTracks: make(map[string]*livekit.TrackInfo), + subscribedTracks: make(map[livekit.ParticipantID][]*webrtc.TrackRemote), + remoteParticipants: make(map[livekit.ParticipantID]*livekit.ParticipantInfo), + me: &webrtc.MediaEngine{}, + lastPackets: make(map[livekit.ParticipantID]*rtp.Packet), + bytesReceived: make(map[livekit.ParticipantID]uint64), + pendingPublishedDataTracks: make(map[uint16]*livekit.DataTrackInfo), + subscribedDataTracks: make(map[livekit.ParticipantID]map[uint16]*DataTrackRemote), + transportReady: make(chan struct{}), } + c.nextDataTrackHandle.Store(uint32(rand.IntN(8192))) c.ctx, c.cancel = context.WithCancel(context.Background()) - conf := rtc.WebRTCConfig{ - WebRTCConfig: rtcconfig.WebRTCConfig{ - Configuration: rtcConf, - }, - } - conf.SettingEngine.SetLite(false) - conf.SettingEngine.SetAnsweringDTLSRole(webrtc.DTLSRoleClient) - ff := buffer.NewFactoryOfBufferFactory(500, 200) - conf.SetBufferFactory(ff.CreateBufferFactory()) var codecs []*livekit.Codec for _, codec := range []*livekit.Codec{ { @@ -205,6 +270,29 @@ func NewRTCClient(conn *websocket.Conn, opts *Options) (*RTCClient, error) { codecs = append(codecs, codec) } } + c.enabledCodecs = codecs + + if opts != nil { + c.signalRequestInterceptor = opts.SignalRequestInterceptor + c.signalResponseInterceptor = opts.SignalResponseInterceptor + c.forceRelay = opts.ForceRelay + } + + return c, nil +} + +func (c *RTCClient) createTransport(rtcconf webrtc.Configuration) error { + var err error + + conf := rtc.WebRTCConfig{ + WebRTCConfig: rtcconfig.WebRTCConfig{ + Configuration: rtcconf, + }, + } + conf.SettingEngine.SetLite(false) + conf.SettingEngine.SetAnsweringDTLSRole(webrtc.DTLSRoleClient) + ff := buffer.NewFactoryOfBufferFactory(500, 200) + conf.SetBufferFactory(ff.CreateBufferFactory()) // // The signal targets are from point of view of server. @@ -214,66 +302,33 @@ func NewRTCClient(conn *websocket.Conn, opts *Options) (*RTCClient, error) { // publisherHandler := &transportfakes.FakeHandler{} c.publisher, err = rtc.NewPCTransport(rtc.TransportParams{ - Config: &conf, - DirectionConfig: conf.Subscriber, - EnabledCodecs: codecs, - IsOfferer: true, - IsSendSide: true, - Handler: publisherHandler, - DatachannelSlowThreshold: 1024 * 1024 * 1024, - }) - if err != nil { - return nil, err - } - subscriberHandler := &transportfakes.FakeHandler{} - c.subscriber, err = rtc.NewPCTransport(rtc.TransportParams{ Config: &conf, - DirectionConfig: conf.Publisher, - EnabledCodecs: codecs, - Handler: subscriberHandler, + DirectionConfig: conf.Subscriber, + EnabledPublishCodecs: c.enabledCodecs, + EnabledSubscribeCodecs: c.enabledCodecs, + IsOfferer: true, + IsSendSide: true, + Handler: publisherHandler, DatachannelMaxReceiverBufferSize: 1500, + DatachannelSlowThreshold: 1024 * 1024 * 1024, FireOnTrackBySdp: true, + EnableDataTracks: true, }) if err != nil { - return nil, err + return err } publisherHandler.OnICECandidateCalls(func(ic *webrtc.ICECandidate, t livekit.SignalTarget) error { return c.SendIceCandidate(ic, livekit.SignalTarget_PUBLISHER) }) - publisherHandler.OnOfferCalls(c.onOffer) - publisherHandler.OnFullyEstablishedCalls(func() { - logger.Debugw("publisher fully established", "participant", c.localParticipant.Identity, "pID", c.localParticipant.Sid) - c.publisherFullyEstablished.Store(true) + publisherHandler.OnTrackCalls(func(track *webrtc.TrackRemote, rtpReceiver *webrtc.RTPReceiver) { + go c.processRemoteTrack(track) }) - - ordered := true - if err := c.publisher.CreateDataChannel(rtc.ReliableDataChannel, &webrtc.DataChannelInit{ - Ordered: &ordered, - }); err != nil { - return nil, err - } - - maxRetransmits := uint16(0) - if err := c.publisher.CreateDataChannel(rtc.LossyDataChannel, &webrtc.DataChannelInit{ - Ordered: &ordered, - MaxRetransmits: &maxRetransmits, - }); err != nil { - return nil, err - } - - subscriberHandler.OnICECandidateCalls(func(ic *webrtc.ICECandidate, t livekit.SignalTarget) error { - if ic == nil { - return nil - } - return c.SendIceCandidate(ic, livekit.SignalTarget_SUBSCRIBER) - }) - subscriberHandler.OnTrackCalls(func(track *webrtc.TrackRemote, rtpReceiver *webrtc.RTPReceiver) { - go c.processTrack(track) - }) - subscriberHandler.OnDataPacketCalls(c.handleDataMessage) - subscriberHandler.OnInitialConnectedCalls(func() { - logger.Debugw("subscriber initial connected", "participant", c.localParticipant.Identity) + publisherHandler.OnDataMessageCalls(c.handleDataMessage) + publisherHandler.OnDataMessageUnlabeledCalls(c.handleDataMessageUnlabeled) + publisherHandler.OnDataTrackMessageCalls(c.handleDataTrackMessage) + publisherHandler.OnInitialConnectedCalls(func() { + logger.Debugw("publisher initial connected", "participant", c.localParticipant.Identity) c.lock.Lock() defer c.lock.Unlock() @@ -282,36 +337,135 @@ func NewRTCClient(conn *websocket.Conn, opts *Options) (*RTCClient, error) { logger.Errorw("track writer error", err) } } - c.pendingTrackWriters = nil + for _, dtw := range c.pendingDataTrackWriters { + if err := dtw.Start(); err != nil { + logger.Errorw("data track writer error", err) + } + } + c.pendingDataTrackWriters = nil + if c.OnConnected != nil { go c.OnConnected() } }) - subscriberHandler.OnFullyEstablishedCalls(func() { - logger.Debugw("subscriber fully established", "participant", c.localParticipant.Identity, "pID", c.localParticipant.Sid) - c.subscriberFullyEstablished.Store(true) - }) - subscriberHandler.OnAnswerCalls(func(answer webrtc.SessionDescription) error { - // send remote an answer - logger.Infow("sending subscriber answer", - "participant", c.localParticipant.Identity, - // "sdp", answer, - ) - return c.SendRequest(&livekit.SignalRequest{ - Message: &livekit.SignalRequest_Answer{ - Answer: rtc.ToProtoSessionDescription(answer), - }, - }) + publisherHandler.OnOfferCalls(c.onOffer) + publisherHandler.OnFullyEstablishedCalls(func() { + logger.Debugw("publisher fully established", "participant", c.localParticipant.Identity, "participantID", c.localParticipant.Sid) + c.publisherFullyEstablished.Store(true) }) - if opts != nil { - c.signalRequestInterceptor = opts.SignalRequestInterceptor - c.signalResponseInterceptor = opts.SignalResponseInterceptor + ordered := true + if err := c.publisher.CreateDataChannel(rtc.ReliableDataChannel, &webrtc.DataChannelInit{ + Ordered: &ordered, + }); err != nil { + return err } - return c, nil + if err := c.publisher.CreateDataChannel("pubraw", &webrtc.DataChannelInit{ + Ordered: &ordered, + }); err != nil { + return err + } + + ordered = false + maxRetransmits := uint16(0) + if err := c.publisher.CreateDataChannel(rtc.LossyDataChannel, &webrtc.DataChannelInit{ + Ordered: &ordered, + MaxRetransmits: &maxRetransmits, + }); err != nil { + return err + } + + if err := c.publisher.CreateDataChannel(rtc.DataTrackDataChannel, &webrtc.DataChannelInit{ + Ordered: &ordered, + MaxRetransmits: &maxRetransmits, + }); err != nil { + return err + } + + if !c.useSinglePeerConnection { + subscriberHandler := &transportfakes.FakeHandler{} + c.subscriber, err = rtc.NewPCTransport(rtc.TransportParams{ + Config: &conf, + DirectionConfig: conf.Publisher, + EnabledPublishCodecs: c.enabledCodecs, + EnabledSubscribeCodecs: c.enabledCodecs, + Handler: subscriberHandler, + DatachannelMaxReceiverBufferSize: 1500, + DatachannelSlowThreshold: 1024 * 1024 * 1024, + FireOnTrackBySdp: true, + EnableDataTracks: true, + }) + if err != nil { + return err + } + + ordered := true + if err := c.subscriber.CreateReadableDataChannel("subraw", &webrtc.DataChannelInit{ + Ordered: &ordered, + }); err != nil { + return err + } + + subscriberHandler.OnICECandidateCalls(func(ic *webrtc.ICECandidate, t livekit.SignalTarget) error { + if ic == nil { + return nil + } + return c.SendIceCandidate(ic, livekit.SignalTarget_SUBSCRIBER) + }) + subscriberHandler.OnTrackCalls(func(track *webrtc.TrackRemote, rtpReceiver *webrtc.RTPReceiver) { + go c.processRemoteTrack(track) + }) + subscriberHandler.OnDataMessageCalls(c.handleDataMessage) + subscriberHandler.OnDataMessageUnlabeledCalls(c.handleDataMessageUnlabeled) + subscriberHandler.OnDataTrackMessageCalls(c.handleDataTrackMessage) + subscriberHandler.OnInitialConnectedCalls(func() { + logger.Debugw("subscriber initial connected", "participant", c.localParticipant.Identity) + + c.lock.Lock() + defer c.lock.Unlock() + for _, tw := range c.pendingTrackWriters { + if err := tw.Start(); err != nil { + logger.Errorw("track writer error", err) + } + } + c.pendingTrackWriters = nil + + for _, dtw := range c.pendingDataTrackWriters { + if err := dtw.Start(); err != nil { + logger.Errorw("data track writer error", err) + } + } + c.pendingDataTrackWriters = nil + + if c.OnConnected != nil { + go c.OnConnected() + } + }) + subscriberHandler.OnFullyEstablishedCalls(func() { + logger.Debugw("subscriber fully established", "participant", c.localParticipant.Identity, "participantID", c.localParticipant.Sid) + c.subscriberFullyEstablished.Store(true) + }) + subscriberHandler.OnAnswerCalls(func(answer webrtc.SessionDescription, answerId uint32, _midToTrackID map[string]string) error { + // send remote an answer + logger.Infow( + "sending subscriber answer", + "participant", c.localParticipant.Identity, + "sdp", answer, + ) + return c.SendRequest(&livekit.SignalRequest{ + Message: &livekit.SignalRequest_Answer{ + Answer: signalling.ToProtoSessionDescription(answer, answerId, nil), + }, + }) + }) + } else { + go c.ensurePublisherConnected() + } + + return nil } func (c *RTCClient) ID() livekit.ParticipantID { @@ -330,7 +484,7 @@ func (c *RTCClient) Run() error { // run the session for { res, err := c.ReadResponse() - if errors.Is(io.EOF, err) { + if errors.Is(err, io.EOF) { return nil } else if err != nil { logger.Errorw("error while reading", err) @@ -357,6 +511,29 @@ func (c *RTCClient) handleSignalResponse(res *livekit.SignalResponse) error { c.remoteParticipants[livekit.ParticipantID(p.Sid)] = p } c.lock.Unlock() + + var iceServers []webrtc.ICEServer + for _, is := range msg.Join.IceServers { + iceServers = append(iceServers, webrtc.ICEServer{ + URLs: is.Urls, + Username: is.Username, + Credential: is.Credential, + }) + } + if len(iceServers) == 0 { + iceServers = rtcConf.ICEServers + } + rtcconf := rtcConf + rtcconf.ICEServers = iceServers + if c.forceRelay { + rtcconf.ICETransportPolicy = webrtc.ICETransportPolicyRelay + } + + if err := c.createTransport(rtcconf); err != nil { + return err + } + close(c.transportReady) + // if publish only, negotiate if !msg.Join.SubscriberPrimary { c.subscriberAsPrimary.Store(false) @@ -365,20 +542,33 @@ func (c *RTCClient) handleSignalResponse(res *livekit.SignalResponse) error { c.subscriberAsPrimary.Store(true) } - logger.Infow("join accepted, awaiting offer", "participant", msg.Join.Participant.Identity) + if c.subscriber != nil { + logger.Infow("join accepted, awaiting offer", "participant", msg.Join.Participant.Identity) + } else { + logger.Infow("join accepted", "participant", msg.Join.Participant.Identity) + } + case *livekit.SignalResponse_Answer: - // logger.Debugw("received server answer", - // "participant", c.localParticipant.Identity, - // "answer", msg.Answer.Sdp) - c.handleAnswer(rtc.FromProtoSessionDescription(msg.Answer)) - case *livekit.SignalResponse_Offer: - logger.Infow("received server offer", + logger.Infow( + "received server answer", "participant", c.localParticipant.Identity, + "answer", msg.Answer.Sdp, ) - desc := rtc.FromProtoSessionDescription(msg.Offer) - c.handleOffer(desc) + c.handleAnswer(signalling.FromProtoSessionDescription(msg.Answer)) + + case *livekit.SignalResponse_Offer: + desc, offerId, midToTrackID := signalling.FromProtoSessionDescription(msg.Offer) + logger.Infow( + "received server offer", + "participant", c.localParticipant.Identity, + "sdp", desc, + "offerId", offerId, + "midToTrackID", midToTrackID, + ) + c.handleOffer(desc, offerId, midToTrackID) + case *livekit.SignalResponse_Trickle: - candidateInit, err := rtc.FromProtoTrickle(msg.Trickle) + candidateInit, err := signalling.FromProtoTrickle(msg.Trickle) if err != nil { return err } @@ -387,6 +577,7 @@ func (c *RTCClient) handleSignalResponse(res *livekit.SignalResponse) error { } else { c.subscriber.AddICECandidate(candidateInit) } + case *livekit.SignalResponse_Update: c.lock.Lock() for _, p := range msg.Update.Participants { @@ -401,20 +592,26 @@ func (c *RTCClient) handleSignalResponse(res *livekit.SignalResponse) error { c.lock.Unlock() case *livekit.SignalResponse_TrackPublished: - logger.Debugw("track published", "trackID", msg.TrackPublished.Track.Name, "participant", c.localParticipant.Sid, - "cid", msg.TrackPublished.Cid, "trackSid", msg.TrackPublished.Track.Sid) + logger.Debugw( + "track published", + "participant", c.localParticipant.Identity, + "cid", msg.TrackPublished.Cid, + "trackID", msg.TrackPublished.Track.Sid, + "trackName", msg.TrackPublished.Track.Name, + ) c.lock.Lock() c.pendingPublishedTracks[msg.TrackPublished.Cid] = msg.TrackPublished.Track c.lock.Unlock() + case *livekit.SignalResponse_RefreshToken: c.lock.Lock() c.refreshToken = msg.RefreshToken c.lock.Unlock() + case *livekit.SignalResponse_TrackUnpublished: sid := msg.TrackUnpublished.TrackSid c.lock.Lock() - sender := c.trackSenders[sid] - if sender != nil { + if sender := c.trackSenders[sid]; sender != nil { if err := c.publisher.RemoveTrack(sender); err != nil { logger.Errorw("Could not unpublish track", err) } @@ -423,16 +620,82 @@ func (c *RTCClient) handleSignalResponse(res *livekit.SignalResponse) error { delete(c.trackSenders, sid) delete(c.localTracks, sid) c.lock.Unlock() + case *livekit.SignalResponse_Pong: c.pongReceivedAt.Store(msg.Pong) + case *livekit.SignalResponse_SubscriptionResponse: c.subscriptionResponse.Store(msg.SubscriptionResponse) + + case *livekit.SignalResponse_MediaSectionsRequirement: + logger.Infow( + "received media sections requirement", + "participant", c.localParticipant.Identity, + "numAudios", msg.MediaSectionsRequirement.NumAudios, + "numVideos", msg.MediaSectionsRequirement.NumVideos, + ) + c.handleMediaSectionsRequirement(msg.MediaSectionsRequirement) + + case *livekit.SignalResponse_PublishDataTrackResponse: + logger.Debugw( + "data track published", + "participant", c.localParticipant.Identity, + "trackID", msg.PublishDataTrackResponse.Info.Sid, + "trackHandle", msg.PublishDataTrackResponse.Info.PubHandle, + "trackName", msg.PublishDataTrackResponse.Info.Name, + ) + c.lock.Lock() + c.pendingPublishedDataTracks[uint16(msg.PublishDataTrackResponse.Info.PubHandle)] = msg.PublishDataTrackResponse.Info + c.lock.Unlock() + + case *livekit.SignalResponse_DataTrackSubscriberHandles: + logger.Infow( + "received data track subscriber handles", + "participant", c.localParticipant.Identity, + "handles", msg.DataTrackSubscriberHandles.SubHandles, + ) + c.lock.Lock() + // create new remote data tracks if one does not exist for a handle + for handle, publishedDataTrack := range msg.DataTrackSubscriberHandles.SubHandles { + publisherID := livekit.ParticipantID(publishedDataTrack.PublisherSid) + tracks := c.subscribedDataTracks[publisherID] + if tracks == nil { + c.subscribedDataTracks[publisherID] = make(map[uint16]*DataTrackRemote) + tracks = c.subscribedDataTracks[publisherID] + } + if tracks[uint16(handle)] == nil { + tracks[uint16(handle)] = NewDataTrackRemote( + livekit.ParticipantIdentity(publishedDataTrack.PublisherIdentity), + livekit.ParticipantID(publishedDataTrack.PublisherSid), + uint16(handle), + livekit.TrackID(publishedDataTrack.TrackSid), + logger.GetLogger().WithValues("participant", c.localParticipant.Identity, "participantID", c.localParticipant.Sid), + ) + } + } + + // delete remote data tracks that have gone away + for publisherID, tracks := range c.subscribedDataTracks { + for handle, dataTrackRemote := range tracks { + if msg.DataTrackSubscriberHandles.SubHandles[uint32(handle)] == nil { + dataTrackRemote.Close() + delete(tracks, handle) + if len(tracks) == 0 { + delete(c.subscribedDataTracks, publisherID) + } + } + } + } + c.lock.Unlock() } return nil } -func (c *RTCClient) WaitUntilConnected() error { - ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) +func (c *RTCClient) WaitUntilConnected(timeout time.Duration) error { + if timeout == 0 { + timeout = 20 * time.Second + } + ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() for { select { @@ -488,8 +751,18 @@ func (c *RTCClient) SubscribedTracks() map[livekit.ParticipantID][]*webrtc.Track c.lock.Lock() defer c.lock.Unlock() tracks := make(map[livekit.ParticipantID][]*webrtc.TrackRemote, len(c.subscribedTracks)) - for key, val := range c.subscribedTracks { - tracks[key] = val + maps.Copy(tracks, c.subscribedTracks) + return tracks +} + +func (c *RTCClient) SubscribedDataTracks() map[livekit.ParticipantID]map[uint16]*DataTrackRemote { + // create a copy of this + c.lock.Lock() + defer c.lock.Unlock() + tracks := make(map[livekit.ParticipantID]map[uint16]*DataTrackRemote, len(c.subscribedDataTracks)) + for publisherID, sts := range c.subscribedDataTracks { + tracks[publisherID] = make(map[uint16]*DataTrackRemote) + maps.Copy(tracks[publisherID], sts) } return tracks } @@ -519,8 +792,12 @@ func (c *RTCClient) Stop() { c.publisherFullyEstablished.Store(false) c.subscriberFullyEstablished.Store(false) _ = c.conn.Close() - c.publisher.Close() - c.subscriber.Close() + if c.publisher != nil { + c.publisher.Close() + } + if c.subscriber != nil { + c.subscriber.Close() + } c.cancel() } @@ -573,7 +850,7 @@ func (c *RTCClient) SendIceCandidate(ic *webrtc.ICECandidate, target livekit.Sig return c.SendRequest(&livekit.SignalRequest{ Message: &livekit.SignalRequest_Trickle{ - Trickle: rtc.ToProtoTrickle(prevIC.ToJSON(), target, ic == nil), + Trickle: signalling.ToProtoTrickle(prevIC.ToJSON(), target, ic == nil), }, }) } @@ -608,7 +885,12 @@ func AddTrackNoWriter() AddTrackOption { } } -func (c *RTCClient) AddTrack(track *webrtc.TrackLocalStaticSample, path string, opts ...AddTrackOption) (writer *TrackWriter, err error) { +func (c *RTCClient) AddTrack(track *webrtc.TrackLocalStaticSample, path string, opts ...AddTrackOption) (writer TrackWriter, err error) { + select { + case <-c.transportReady: + case <-c.ctx.Done(): + return nil, c.ctx.Err() + } var params AddTrackParams for _, opt := range opts { opt(¶ms) @@ -618,7 +900,18 @@ func (c *RTCClient) AddTrack(track *webrtc.TrackLocalStaticSample, path string, trackType = livekit.TrackType_VIDEO } - if err = c.SendAddTrack(track.ID(), track.StreamID(), trackType); err != nil { + sender, _, err := c.publisher.AddTrack(track, types.AddTrackParams{}, nil, rtc.RTCPFeedbackConfig{}) + if err != nil { + logger.Errorw( + "add track failed", err, + "participant", c.localParticipant.Identity, + "participantID", c.localParticipant.Sid, + "trackID", track.ID(), + ) + return + } + + if err = c.SendAddTrack(track.ID(), track.Codec().MimeType, track.StreamID(), trackType); err != nil { return } @@ -632,10 +925,12 @@ func (c *RTCClient) AddTrack(track *webrtc.TrackLocalStaticSample, path string, default: c.lock.Lock() ti = c.pendingPublishedTracks[track.ID()] - c.lock.Unlock() if ti != nil { + delete(c.pendingPublishedTracks, track.ID()) + c.lock.Unlock() break } + c.lock.Unlock() time.Sleep(50 * time.Millisecond) } if ti != nil { @@ -646,11 +941,6 @@ func (c *RTCClient) AddTrack(track *webrtc.TrackLocalStaticSample, path string, c.lock.Lock() defer c.lock.Unlock() - sender, _, err := c.publisher.AddTrack(track, types.AddTrackParams{}) - if err != nil { - logger.Errorw("add track failed", err, "trackID", ti.Sid, "participant", c.localParticipant.Identity, "pID", c.localParticipant.Sid) - return - } c.localTracks[ti.Sid] = track c.trackSenders[ti.Sid] = sender c.publisher.Negotiate(false) @@ -669,11 +959,11 @@ func (c *RTCClient) AddTrack(track *webrtc.TrackLocalStaticSample, path string, return } -func (c *RTCClient) AddStaticTrack(mime string, id string, label string, opts ...AddTrackOption) (writer *TrackWriter, err error) { +func (c *RTCClient) AddStaticTrack(mime string, id string, label string, opts ...AddTrackOption) (writer TrackWriter, err error) { return c.AddStaticTrackWithCodec(webrtc.RTPCodecCapability{MimeType: mime}, id, label, opts...) } -func (c *RTCClient) AddStaticTrackWithCodec(codec webrtc.RTPCodecCapability, id string, label string, opts ...AddTrackOption) (writer *TrackWriter, err error) { +func (c *RTCClient) AddStaticTrackWithCodec(codec webrtc.RTPCodecCapability, id string, label string, opts ...AddTrackOption) (writer TrackWriter, err error) { track, err := webrtc.NewTrackLocalStaticSample(codec, id, label) if err != nil { return @@ -682,16 +972,14 @@ func (c *RTCClient) AddStaticTrackWithCodec(codec webrtc.RTPCodecCapability, id return c.AddTrack(track, "", opts...) } -func (c *RTCClient) AddFileTrack(path string, id string, label string) (writer *TrackWriter, err error) { +func (c *RTCClient) AddFileTrack(path string, id string, label string) (writer TrackWriter, err error) { // determine file mime mime, ok := extMimeMapping[filepath.Ext(path)] if !ok { return nil, fmt.Errorf("%s has an unsupported extension", filepath.Base(path)) } - logger.Debugw("adding file track", - "mime", mime, - ) + logger.Debugw("adding file track", "mime", mime) track, err := webrtc.NewTrackLocalStaticSample( webrtc.RTPCodecCapability{MimeType: mime}, @@ -706,13 +994,19 @@ func (c *RTCClient) AddFileTrack(path string, id string, label string) (writer * } // send AddTrack command to server to initiate server-side negotiation -func (c *RTCClient) SendAddTrack(cid string, name string, trackType livekit.TrackType) error { +func (c *RTCClient) SendAddTrack(cid string, mimeType string, name string, trackType livekit.TrackType) error { return c.SendRequest(&livekit.SignalRequest{ Message: &livekit.SignalRequest_AddTrack{ AddTrack: &livekit.AddTrackRequest{ Cid: cid, Name: name, Type: trackType, + SimulcastCodecs: []*livekit.SimulcastCodec{ + { + Cid: cid, + Codec: mimeType, + }, + }, }, }, }) @@ -732,7 +1026,69 @@ func (c *RTCClient) PublishData(data []byte, kind livekit.DataPacket_Kind) error return err } - return c.publisher.SendDataPacket(kind, dpData) + return c.publisher.SendDataMessage(kind, dpData) +} + +func (c *RTCClient) PublishDataUnlabeled(data []byte) error { + if err := c.ensurePublisherConnected(); err != nil { + return err + } + + return c.publisher.SendDataMessageUnlabeled(data, true, "test") +} + +func (c *RTCClient) PublishDataTrack() (writer TrackWriter, err error) { + if err = c.ensurePublisherConnected(); err != nil { + return + } + + dataTrackHandle := uint16(c.nextDataTrackHandle.Inc()) + if err = c.SendRequest(&livekit.SignalRequest{ + Message: &livekit.SignalRequest_PublishDataTrackRequest{ + PublishDataTrackRequest: &livekit.PublishDataTrackRequest{ + PubHandle: uint32(dataTrackHandle), + Name: fmt.Sprintf("data_track_%d", dataTrackHandle), + }, + }, + }); err != nil { + return + } + + // wait till data track published message is received + timeout := time.After(5 * time.Second) + var dti *livekit.DataTrackInfo + for { + select { + case <-timeout: + return nil, errors.New("could not publish data track after timeout") + default: + c.lock.Lock() + dti = c.pendingPublishedDataTracks[dataTrackHandle] + if dti != nil { + delete(c.pendingPublishedDataTracks, dataTrackHandle) + c.lock.Unlock() + break + } + c.lock.Unlock() + time.Sleep(50 * time.Millisecond) + } + if dti != nil { + break + } + } + + c.lock.Lock() + defer c.lock.Unlock() + + writer = NewDataTrackWriter(c.ctx, dataTrackHandle, c.publisher) + + // write data tracks only after connection established + if c.hasPrimaryEverConnected() { + err = writer.Start() + } else { + c.pendingDataTrackWriters = append(c.pendingDataTrackWriters, writer) + } + return } func (c *RTCClient) GetPublishedTrackIDs() []string { @@ -747,10 +1103,16 @@ func (c *RTCClient) GetPublishedTrackIDs() []string { // LastAnswer return SDP of the last answer for the publisher connection func (c *RTCClient) LastAnswer() *webrtc.SessionDescription { - return c.lastAnswer.Load() + return c.publisher.CurrentRemoteDescription() } func (c *RTCClient) ensurePublisherConnected() error { + select { + case <-c.transportReady: + case <-c.ctx.Done(): + return c.ctx.Err() + } + if c.publisher.HasEverConnected() { return nil } @@ -787,50 +1149,148 @@ func (c *RTCClient) handleDataMessage(kind livekit.DataPacket_Kind, data []byte) } } +func (c *RTCClient) handleDataMessageUnlabeled(data []byte) { + if c.OnDataUnlabeledReceived != nil { + c.OnDataUnlabeledReceived(data) + } +} + +func (c *RTCClient) handleDataTrackMessage(data []byte, _arrivalTime int64) { + var packet datatrack.Packet + if err := packet.Unmarshal(data); err != nil { + return + } + + var dataTrackRemote *DataTrackRemote + c.lock.Lock() + for _, tracks := range c.subscribedDataTracks { + if tracks[packet.Handle] != nil { + dataTrackRemote = tracks[packet.Handle] + break + } + } + c.lock.Unlock() + + if dataTrackRemote != nil { + dataTrackRemote.PacketReceived(&packet) + } +} + // handles a server initiated offer, handle on subscriber PC -func (c *RTCClient) handleOffer(desc webrtc.SessionDescription) { - c.subscriber.HandleRemoteDescription(desc) +func (c *RTCClient) handleOffer(desc webrtc.SessionDescription, offerId uint32, _midToTrackID map[string]string) { + logger.Infow("handling server offer", "participant", c.localParticipant.Identity) + c.subscriber.HandleRemoteDescription(desc, offerId) + c.processPendingRemoteTracks() } // the client handles answer on the publisher PC -func (c *RTCClient) handleAnswer(desc webrtc.SessionDescription) { +func (c *RTCClient) handleAnswer(desc webrtc.SessionDescription, answerId uint32, _midToTrackID map[string]string) { logger.Infow("handling server answer", "participant", c.localParticipant.Identity) - c.lastAnswer.Store(&desc) // remote answered the offer, establish connection - c.publisher.HandleRemoteDescription(desc) + c.publisher.HandleRemoteDescription(desc, answerId) + c.processPendingRemoteTracks() } -func (c *RTCClient) onOffer(offer webrtc.SessionDescription) error { +// the client handles media sections requirement on the publisher PC +func (c *RTCClient) handleMediaSectionsRequirement(mediaSectionsRequirement *livekit.MediaSectionsRequirement) { + addTransceivers := func(kind webrtc.RTPCodecType, count uint32) { + for range count { + if _, err := c.publisher.AddTransceiverFromKind( + kind, + webrtc.RTPTransceiverInit{ + Direction: webrtc.RTPTransceiverDirectionRecvonly, + }, + ); err != nil { + logger.Warnw( + "could not add transceiver", err, + "participant", c.localParticipant.Identity, + "kind", kind, + ) + } else { + logger.Infow( + "added transceiver of kind", + "participant", c.localParticipant.Identity, + "kind", kind, + ) + } + } + } + + addTransceivers(webrtc.RTPCodecTypeAudio, mediaSectionsRequirement.NumAudios) + addTransceivers(webrtc.RTPCodecTypeVideo, mediaSectionsRequirement.NumVideos) + c.publisher.Negotiate(false) +} + +func (c *RTCClient) onOffer(offer webrtc.SessionDescription, offerId uint32, midToTrackID map[string]string) error { if c.localParticipant != nil { logger.Infow("starting negotiation", "participant", c.localParticipant.Identity) + logger.Infow( + "sending publisher offer", + "participant", c.localParticipant.Identity, + "offer", offer, + "midToTrackID", midToTrackID, + ) } return c.SendRequest(&livekit.SignalRequest{ Message: &livekit.SignalRequest_Offer{ - Offer: rtc.ToProtoSessionDescription(offer), + Offer: signalling.ToProtoSessionDescription(offer, offerId, nil), }, }) } -func (c *RTCClient) processTrack(track *webrtc.TrackRemote) { - lastUpdate := time.Time{} - pId, trackId := rtc.UnpackStreamID(track.StreamID()) - if trackId == "" { - trackId = livekit.TrackID(track.ID()) - } +func (c *RTCClient) processPendingRemoteTracks() { c.lock.Lock() - c.subscribedTracks[pId] = append(c.subscribedTracks[pId], track) + pendingRemoteTracks := c.pendingRemoteTracks + c.pendingRemoteTracks = nil c.lock.Unlock() - logger.Infow("client added track", "participant", c.localParticipant.Identity, - "pID", pId, - "trackID", trackId, + for _, pendingRemoteTrack := range pendingRemoteTracks { + go c.processRemoteTrack(pendingRemoteTrack) + } +} + +func (c *RTCClient) processRemoteTrack(track *webrtc.TrackRemote) { + lastUpdate := time.Time{} + + // because of FireOnTrackBySdp, it is possible get an empty streamID + // if media comes before SDP, cache and try later + streamID := track.StreamID() + if streamID == "" { + logger.Infow( + "client caching track", + "participant", c.localParticipant.Identity, + "participantID", c.ID(), + "codec", track.Codec(), + "ssrc", track.SSRC(), + ) + c.lock.Lock() + c.pendingRemoteTracks = append(c.pendingRemoteTracks, track) + c.lock.Unlock() + return + } + + publisherID, trackID := rtc.UnpackStreamID(streamID) + if trackID == "" { + trackID = livekit.TrackID(track.ID()) + } + c.lock.Lock() + c.subscribedTracks[publisherID] = append(c.subscribedTracks[publisherID], track) + c.lock.Unlock() + + logger.Infow( + "client added track", + "participant", c.localParticipant.Identity, + "participantID", c.ID(), + "publisherID", publisherID, + "trackID", trackID, "codec", track.Codec(), + "ssrc", track.SSRC(), ) defer func() { c.lock.Lock() - c.subscribedTracks[pId] = funk.Without(c.subscribedTracks[pId], track).([]*webrtc.TrackRemote) + c.subscribedTracks[publisherID] = funk.Without(c.subscribedTracks[publisherID], track).([]*webrtc.TrackRemote) c.lock.Unlock() }() @@ -841,6 +1301,15 @@ func (c *RTCClient) processTrack(track *webrtc.TrackRemote) { break } if rtc.IsEOF(err) { + logger.Infow( + "client track removed", + "participant", c.localParticipant.Identity, + "participantID", c.ID(), + "publisherID", publisherID, + "trackID", trackID, + "codec", track.Codec(), + "ssrc", track.SSRC(), + ) break } if err != nil { @@ -848,14 +1317,19 @@ func (c *RTCClient) processTrack(track *webrtc.TrackRemote) { continue } c.lock.Lock() - c.lastPackets[pId] = pkt - c.bytesReceived[pId] += uint64(pkt.MarshalSize()) + c.lastPackets[publisherID] = pkt + c.bytesReceived[publisherID] += uint64(pkt.MarshalSize()) c.lock.Unlock() numBytes += pkt.MarshalSize() if time.Since(lastUpdate) > 30*time.Second { - logger.Infow("consumed from participant", - "trackID", trackId, "pID", pId, - "size", numBytes) + logger.Infow( + "consumed from participant", + "participant", c.localParticipant.Identity, + "participantID", c.ID(), + "publisherID", publisherID, + "trackID", trackID, + "size", numBytes, + ) lastUpdate = time.Now() } } @@ -871,12 +1345,34 @@ func (c *RTCClient) BytesReceived() uint64 { return total } +func (c *RTCClient) IsLocalCandidateRelaySelected() bool { + var info *types.ICEConnectionInfo + if c.subscriberAsPrimary.Load() { + if c.subscriber != nil { + info = c.subscriber.GetICEConnectionInfo() + } + } else { + if c.publisher != nil { + info = c.publisher.GetICEConnectionInfo() + } + } + if info == nil { + return false + } + for _, local := range info.Local { + if local.SelectedOrder > 0 && local.Candidate != nil && local.Candidate.Typ == webrtc.ICECandidateTypeRelay { + return true + } + } + return false +} + func (c *RTCClient) SendNacks(count int) { var packets []rtcp.Packet c.lock.Lock() for _, pkt := range c.lastPackets { seqs := make([]uint16, 0, count) - for i := 0; i < count; i++ { + for i := range count { seqs = append(seqs, pkt.SequenceNumber-uint16(i)) } packets = append(packets, &rtcp.TransportLayerNack{ diff --git a/test/client/datatrack_remote.go b/test/client/datatrack_remote.go new file mode 100644 index 0000000..2d2f663 --- /dev/null +++ b/test/client/datatrack_remote.go @@ -0,0 +1,84 @@ +package client + +import ( + "github.com/frostbyte73/core" + "github.com/livekit/livekit-server/pkg/rtc/datatrack" + "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/logger" + "go.uber.org/atomic" +) + +type DataTrackRemote struct { + publisherIdentity livekit.ParticipantIdentity + publisherID livekit.ParticipantID + handle uint16 + trackID livekit.TrackID + logger logger.Logger + numReceivedPackets atomic.Uint32 + + closed core.Fuse +} + +func NewDataTrackRemote( + publisherIdentity livekit.ParticipantIdentity, + publisherID livekit.ParticipantID, + handle uint16, + trackID livekit.TrackID, + logger logger.Logger, +) *DataTrackRemote { + logger.Infow( + "creating data track remote", + "publisherIdentity", publisherIdentity, + "publisherID", publisherID, + "handle", handle, + "trackID", trackID, + ) + return &DataTrackRemote{ + publisherIdentity: publisherIdentity, + publisherID: publisherID, + handle: handle, + trackID: trackID, + logger: logger, + } +} + +func (d *DataTrackRemote) Close() { + d.logger.Infow( + "closing data track remote", + "publisherIdentity", d.publisherIdentity, + "publisherID", d.publisherID, + "handle", d.handle, + "trackID", d.trackID, + ) + d.closed.Break() +} + +func (d *DataTrackRemote) Handle() uint16 { + return d.handle +} + +func (d *DataTrackRemote) ID() livekit.TrackID { + return d.trackID +} + +func (d *DataTrackRemote) PacketReceived(packet *datatrack.Packet) { + if d.closed.IsBroken() { + return + } + + valid := len(packet.Payload) != 0 + + for i := range packet.Payload { + if packet.Payload[i] != byte(255-i) { + valid = false + break + } + } + if valid { + d.numReceivedPackets.Inc() + } +} + +func (d *DataTrackRemote) NumReceivedPackets() uint32 { + return d.numReceivedPackets.Load() +} diff --git a/test/client/datatrack_writer.go b/test/client/datatrack_writer.go new file mode 100644 index 0000000..82c6a18 --- /dev/null +++ b/test/client/datatrack_writer.go @@ -0,0 +1,79 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package client + +import ( + "context" + "math/rand" + "time" + + "github.com/livekit/livekit-server/pkg/rtc/datatrack" + "github.com/livekit/livekit-server/pkg/rtc/types" + "github.com/livekit/protocol/logger" +) + +type dataTrackWriter struct { + ctx context.Context + cancel context.CancelFunc + handle uint16 + transport types.DataTrackTransport +} + +func NewDataTrackWriter(ctx context.Context, handle uint16, transport types.DataTrackTransport) TrackWriter { + ctx, cancel := context.WithCancel(ctx) + return &dataTrackWriter{ + ctx: ctx, + cancel: cancel, + handle: handle, + transport: transport, + } +} + +func (d *dataTrackWriter) Start() error { + go d.writeFrames() + return nil +} + +func (d *dataTrackWriter) Stop() { + d.cancel() +} + +func (d *dataTrackWriter) writeFrames() { + seqNum := uint16(0) + frameNum := uint16(0) + for { + select { + case <-d.ctx.Done(): + return + + default: + packets := datatrack.GenerateRawDataPackets(d.handle, seqNum, frameNum, 1, rand.Intn(2048)+1, 100*time.Millisecond) + for _, packet := range packets { + if err := d.transport.SendDataTrackMessage(packet); err != nil { + logger.Errorw("could not send data track packet", err) + } + } + + if len(packets) != 0 { + var lastPacket datatrack.Packet + if err := lastPacket.Unmarshal(packets[len(packets)-1]); err == nil { + seqNum = lastPacket.SequenceNumber + 1 + frameNum = lastPacket.FrameNumber + 1 + } + } + time.Sleep(100 * time.Millisecond) + } + } +} diff --git a/test/client/trackwriter.go b/test/client/trackwriter.go index 803bf33..09dad45 100644 --- a/test/client/trackwriter.go +++ b/test/client/trackwriter.go @@ -26,13 +26,18 @@ import ( "github.com/pion/webrtc/v4/pkg/media/ivfreader" "github.com/pion/webrtc/v4/pkg/media/oggreader" - "github.com/livekit/livekit-server/pkg/sfu/mime" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/logger" ) +type TrackWriter interface { + Start() error + Stop() +} + // Writes a file to an RTP track. // makes it easier to debug and create RTP streams -type TrackWriter struct { +type trackWriter struct { ctx context.Context cancel context.CancelFunc track *webrtc.TrackLocalStaticSample @@ -45,9 +50,9 @@ type TrackWriter struct { h264 *h264reader.H264Reader } -func NewTrackWriter(ctx context.Context, track *webrtc.TrackLocalStaticSample, filePath string) *TrackWriter { +func NewTrackWriter(ctx context.Context, track *webrtc.TrackLocalStaticSample, filePath string) TrackWriter { ctx, cancel := context.WithCancel(ctx) - return &TrackWriter{ + return &trackWriter{ ctx: ctx, cancel: cancel, track: track, @@ -56,7 +61,7 @@ func NewTrackWriter(ctx context.Context, track *webrtc.TrackLocalStaticSample, f } } -func (w *TrackWriter) Start() error { +func (w *trackWriter) Start() error { if w.filePath == "" { go w.writeNull() return nil @@ -95,11 +100,11 @@ func (w *TrackWriter) Start() error { return nil } -func (w *TrackWriter) Stop() { +func (w *trackWriter) Stop() { w.cancel() } -func (w *TrackWriter) writeNull() { +func (w *trackWriter) writeNull() { defer w.onWriteComplete() sample := media.Sample{Data: []byte{0x0, 0xff, 0xff, 0xff, 0xff}, Duration: 30 * time.Millisecond} h264Sample := media.Sample{Data: []byte{0x00, 0x00, 0x00, 0x01, 0x7, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, 0x8, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, 0x5, 0xff, 0xff, 0xff, 0xff}, Duration: 30 * time.Millisecond} @@ -117,7 +122,7 @@ func (w *TrackWriter) writeNull() { } } -func (w *TrackWriter) writeOgg() { +func (w *trackWriter) writeOgg() { // Keep track of last granule, the difference is the amount of samples in the buffer var lastGranule uint64 for { @@ -150,7 +155,7 @@ func (w *TrackWriter) writeOgg() { } } -func (w *TrackWriter) writeVP8() { +func (w *trackWriter) writeVP8() { // Send our video file frame at a time. Pace our sending such that we send it at the same speed it should be played back as. // This isn't required since the video is timestamped, but we will such much higher loss if we send all at once. sleepTime := time.Millisecond * time.Duration((float32(w.ivfheader.TimebaseNumerator)/float32(w.ivfheader.TimebaseDenominator))*1000) @@ -178,10 +183,9 @@ func (w *TrackWriter) writeVP8() { } } -func (w *TrackWriter) writeH264() { +func (w *trackWriter) writeH264() { // TODO: this is harder } -func (w *TrackWriter) onWriteComplete() { - +func (w *trackWriter) onWriteComplete() { } diff --git a/test/integration_helpers.go b/test/integration_helpers.go index 66bb351..3ccde58 100644 --- a/test/integration_helpers.go +++ b/test/integration_helpers.go @@ -25,17 +25,18 @@ import ( "github.com/redis/go-redis/v9" "github.com/twitchtv/twirp" + "github.com/livekit/mediatransportutil/pkg/rtcconfig" + "github.com/livekit/protocol/auth" + "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/logger" + "github.com/livekit/protocol/utils/guid" + "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/livekit-server/pkg/routing" "github.com/livekit/livekit-server/pkg/service" "github.com/livekit/livekit-server/pkg/telemetry/prometheus" "github.com/livekit/livekit-server/pkg/testutils" testclient "github.com/livekit/livekit-server/test/client" - "github.com/livekit/mediatransportutil/pkg/rtcconfig" - "github.com/livekit/protocol/auth" - "github.com/livekit/protocol/livekit" - "github.com/livekit/protocol/logger" - "github.com/livekit/protocol/utils/guid" ) const ( @@ -131,14 +132,30 @@ func waitUntilConnected(t *testing.T, clients ...*testclient.RTCClient) { wg := sync.WaitGroup{} for i := range clients { c := clients[i] - wg.Add(1) - go func() { - defer wg.Done() - err := c.WaitUntilConnected() + wg.Go(func() { + err := c.WaitUntilConnected(5 * time.Second) if err != nil { t.Error(err) } - }() + }) + } + wg.Wait() + if t.Failed() { + t.FailNow() + } +} + +func ensureNotConnected(t *testing.T, clients ...*testclient.RTCClient) { + logger.Infow("checking if clients connect") + wg := sync.WaitGroup{} + for i := range clients { + c := clients[i] + wg.Go(func() { + err := c.WaitUntilConnected(5 * time.Second) + if err == nil { + t.Error(fmt.Errorf("expected client to not connect: %s", c.ID())) + } + }) } wg.Wait() if t.Failed() { @@ -153,6 +170,7 @@ func createSingleNodeServer(configUpdater func(*config.Config)) *service.Livekit panic(fmt.Sprintf("could not create config: %v", err)) } conf.Keys = map[string]string{testApiKey: testApiSecret} + conf.EnableDataTracks = true if configUpdater != nil { configUpdater(conf) } @@ -183,6 +201,7 @@ func createMultiNodeServer(nodeID string, port uint32) *service.LivekitServer { conf.RTC.TCPPort = port + 2 conf.Redis.Address = "localhost:6379" conf.Keys = map[string]string{testApiKey: testApiSecret} + conf.EnableDataTracks = true currentNode, err := routing.NewLocalNode(conf) if err != nil { @@ -200,36 +219,77 @@ func createMultiNodeServer(nodeID string, port uint32) *service.LivekitServer { return s } +type testRTCServicePath int + +const ( + testRTCServicePathv0 testRTCServicePath = iota + testRTCServicePathv0SinglePeerConnection + testRTCServicePathv1 +) + +func (t testRTCServicePath) String() string { + switch t { + case testRTCServicePathv0: + return "v0" + case testRTCServicePathv0SinglePeerConnection: + return "v0-single-peer-connection" + case testRTCServicePathv1: + return "v1" + default: + return fmt.Sprintf("unknown: %d", t) + } +} + +var testRTCServicePaths = []testRTCServicePath{ + testRTCServicePathv0, + testRTCServicePathv0SinglePeerConnection, + testRTCServicePathv1, +} + +func testRTCServicePathToTestClientOptions(testRTCServicePath testRTCServicePath, opts *testclient.Options) { + if opts == nil { + return + } + + switch testRTCServicePath { + case testRTCServicePathv0: + opts.RTCServicePath = "/rtc" + case testRTCServicePathv0SinglePeerConnection: + opts.RTCServicePath = "/rtc" + opts.UseJoinRequestQueryParam = true + case testRTCServicePathv1: + opts.RTCServicePath = "/rtc/v1" + opts.UseJoinRequestQueryParam = true + default: + opts.RTCServicePath = "/rtc" + } +} + // creates a client and runs against server -func createRTCClient(name string, port int, opts *testclient.Options) *testclient.RTCClient { +func createRTCClient(name string, port int, testRTCServicePath testRTCServicePath, opts *testclient.Options) *testclient.RTCClient { var customizer func(token *auth.AccessToken, grants *auth.VideoGrant) if opts != nil { customizer = opts.TokenCustomizer } token := joinToken(testRoom, name, customizer) - ws, err := testclient.NewWebSocketConn(fmt.Sprintf("ws://localhost:%d", port), token, opts) - if err != nil { - panic(err) - } - c, err := testclient.NewRTCClient(ws, opts) - if err != nil { - panic(err) - } - - go c.Run() - - return c + return createRTCClientWithToken(token, port, testRTCServicePath, opts) } // creates a client and runs against server -func createRTCClientWithToken(token string, port int, opts *testclient.Options) *testclient.RTCClient { +func createRTCClientWithToken(token string, port int, testRTCServicePath testRTCServicePath, opts *testclient.Options) *testclient.RTCClient { + if opts == nil { + opts = &testclient.Options{ + AutoSubscribe: true, + } + } + testRTCServicePathToTestClientOptions(testRTCServicePath, opts) ws, err := testclient.NewWebSocketConn(fmt.Sprintf("ws://localhost:%d", port), token, opts) if err != nil { panic(err) } - c, err := testclient.NewRTCClient(ws, opts) + c, err := testclient.NewRTCClient(ws, opts.UseJoinRequestQueryParam, opts) if err != nil { panic(err) } @@ -304,7 +364,7 @@ func listRoomToken() string { return t } -func stopWriters(writers ...*testclient.TrackWriter) { +func stopWriters(writers ...testclient.TrackWriter) { for _, w := range writers { w.Stop() } diff --git a/test/multinode_roomservice_test.go b/test/multinode_roomservice_test.go index a1ec1cb..de74572 100644 --- a/test/multinode_roomservice_test.go +++ b/test/multinode_roomservice_test.go @@ -61,24 +61,28 @@ func TestMultiNodeUpdateRoomMetadata(t *testing.T) { }) t.Run("when room has a participant", func(t *testing.T) { - _, _, finish := setupMultiNodeTest("TestMultiNodeUpdateRoomMetadata_with_participant") - defer finish() + for _, testRTCServicePath := range testRTCServicePaths { + t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) { + _, _, finish := setupMultiNodeTest("TestMultiNodeUpdateRoomMetadata_with_participant") + defer finish() - c1 := createRTCClient("c1", defaultServerPort, nil) - waitUntilConnected(t, c1) - defer c1.Stop() + c1 := createRTCClient("c1", defaultServerPort, testRTCServicePath, nil) + waitUntilConnected(t, c1) + defer c1.Stop() - _, err := roomClient.CreateRoom(contextWithToken(createRoomToken()), &livekit.CreateRoomRequest{ - Name: "emptyRoom", - }) - require.NoError(t, err) + _, err := roomClient.CreateRoom(contextWithToken(createRoomToken()), &livekit.CreateRoomRequest{ + Name: "emptyRoom", + }) + require.NoError(t, err) - rm, err := roomClient.UpdateRoomMetadata(contextWithToken(adminRoomToken("emptyRoom")), &livekit.UpdateRoomMetadataRequest{ - Room: "emptyRoom", - Metadata: "updated metadata", - }) - require.NoError(t, err) - require.Equal(t, "updated metadata", rm.Metadata) + rm, err := roomClient.UpdateRoomMetadata(contextWithToken(adminRoomToken("emptyRoom")), &livekit.UpdateRoomMetadataRequest{ + Room: "emptyRoom", + Metadata: "updated metadata", + }) + require.NoError(t, err) + require.Equal(t, "updated metadata", rm.Metadata) + }) + } }) } @@ -89,85 +93,97 @@ func TestMultiNodeRemoveParticipant(t *testing.T) { return } - _, _, finish := setupMultiNodeTest("TestMultiNodeRemoveParticipant") - defer finish() + for _, testRTCServicePath := range testRTCServicePaths { + t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) { + _, _, finish := setupMultiNodeTest("TestMultiNodeRemoveParticipant") + defer finish() - c1 := createRTCClient("mn_remove_participant", defaultServerPort, nil) - defer c1.Stop() - waitUntilConnected(t, c1) + c1 := createRTCClient("mn_remove_participant", defaultServerPort, testRTCServicePath, nil) + defer c1.Stop() + waitUntilConnected(t, c1) - ctx := contextWithToken(adminRoomToken(testRoom)) - _, err := roomClient.RemoveParticipant(ctx, &livekit.RoomParticipantIdentity{ - Room: testRoom, - Identity: "mn_remove_participant", - }) - require.NoError(t, err) + ctx := contextWithToken(adminRoomToken(testRoom)) + _, err := roomClient.RemoveParticipant(ctx, &livekit.RoomParticipantIdentity{ + Room: testRoom, + Identity: "mn_remove_participant", + }) + require.NoError(t, err) - // participant list doesn't show the participant - listRes, err := roomClient.ListParticipants(ctx, &livekit.ListParticipantsRequest{ - Room: testRoom, - }) - require.NoError(t, err) - require.Len(t, listRes.Participants, 0) + // participant list doesn't show the participant + listRes, err := roomClient.ListParticipants(ctx, &livekit.ListParticipantsRequest{ + Room: testRoom, + }) + require.NoError(t, err) + require.Len(t, listRes.Participants, 0) + }) + } } // update participant metadata func TestMultiNodeUpdateParticipantMetadata(t *testing.T) { - _, _, finish := setupMultiNodeTest("TestMultiNodeUpdateParticipantMetadata") - defer finish() + for _, testRTCServicePath := range testRTCServicePaths { + t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) { + _, _, finish := setupMultiNodeTest("TestMultiNodeUpdateParticipantMetadata") + defer finish() - c1 := createRTCClient("update_participant_metadata", defaultServerPort, nil) - defer c1.Stop() - waitUntilConnected(t, c1) + c1 := createRTCClient("update_participant_metadata", defaultServerPort, testRTCServicePath, nil) + defer c1.Stop() + waitUntilConnected(t, c1) - ctx := contextWithToken(adminRoomToken(testRoom)) - res, err := roomClient.UpdateParticipant(ctx, &livekit.UpdateParticipantRequest{ - Room: testRoom, - Identity: "update_participant_metadata", - Metadata: "the new metadata", - }) - require.NoError(t, err) - require.Equal(t, "the new metadata", res.Metadata) + ctx := contextWithToken(adminRoomToken(testRoom)) + res, err := roomClient.UpdateParticipant(ctx, &livekit.UpdateParticipantRequest{ + Room: testRoom, + Identity: "update_participant_metadata", + Metadata: "the new metadata", + }) + require.NoError(t, err) + require.Equal(t, "the new metadata", res.Metadata) + }) + } } // admin mute published track func TestMultiNodeMutePublishedTrack(t *testing.T) { - _, _, finish := setupMultiNodeTest("TestMultiNodeMutePublishedTrack") - defer finish() + for _, testRTCServicePath := range testRTCServicePaths { + t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) { + _, _, finish := setupMultiNodeTest("TestMultiNodeMutePublishedTrack") + defer finish() - identity := "mute_published_track" - c1 := createRTCClient(identity, defaultServerPort, nil) - defer c1.Stop() - waitUntilConnected(t, c1) + identity := "mute_published_track" + c1 := createRTCClient(identity, defaultServerPort, testRTCServicePath, nil) + defer c1.Stop() + waitUntilConnected(t, c1) - writers := publishTracksForClients(t, c1) - defer stopWriters(writers...) + writers := publishTracksForClients(t, c1) + defer stopWriters(writers...) - trackIDs := c1.GetPublishedTrackIDs() - require.NotEmpty(t, trackIDs) + trackIDs := c1.GetPublishedTrackIDs() + require.NotEmpty(t, trackIDs) - ctx := contextWithToken(adminRoomToken(testRoom)) - // wait for it to be published before - testutils.WithTimeout(t, func() string { - res, err := roomClient.GetParticipant(ctx, &livekit.RoomParticipantIdentity{ - Room: testRoom, - Identity: identity, + ctx := contextWithToken(adminRoomToken(testRoom)) + // wait for it to be published before + testutils.WithTimeout(t, func() string { + res, err := roomClient.GetParticipant(ctx, &livekit.RoomParticipantIdentity{ + Room: testRoom, + Identity: identity, + }) + require.NoError(t, err) + if len(res.Tracks) == 2 { + return "" + } else { + return fmt.Sprintf("expected 2 tracks to be published, actual: %d", len(res.Tracks)) + } + }) + + res, err := roomClient.MutePublishedTrack(ctx, &livekit.MuteRoomTrackRequest{ + Room: testRoom, + Identity: identity, + TrackSid: trackIDs[0], + Muted: true, + }) + require.NoError(t, err) + require.Equal(t, trackIDs[0], res.Track.Sid) + require.True(t, res.Track.Muted) }) - require.NoError(t, err) - if len(res.Tracks) == 2 { - return "" - } else { - return fmt.Sprintf("expected 2 tracks to be published, actual: %d", len(res.Tracks)) - } - }) - - res, err := roomClient.MutePublishedTrack(ctx, &livekit.MuteRoomTrackRequest{ - Room: testRoom, - Identity: identity, - TrackSid: trackIDs[0], - Muted: true, - }) - require.NoError(t, err) - require.Equal(t, trackIDs[0], res.Track.Sid) - require.True(t, res.Track.Muted) + } } diff --git a/test/multinode_test.go b/test/multinode_test.go index e964f30..9b4d5ac 100644 --- a/test/multinode_test.go +++ b/test/multinode_test.go @@ -15,6 +15,7 @@ package test import ( + "fmt" "testing" "time" @@ -33,6 +34,7 @@ func TestMultiNodeRouting(t *testing.T) { t.SkipNow() return } + _, _, finish := setupMultiNodeTest("TestMultiNodeRouting") defer finish() @@ -42,35 +44,39 @@ func TestMultiNodeRouting(t *testing.T) { }) require.NoError(t, err) - // one node connecting to node 1, and another connecting to node 2 - c1 := createRTCClient("c1", defaultServerPort, nil) - c2 := createRTCClient("c2", secondServerPort, nil) - waitUntilConnected(t, c1, c2) - defer stopClients(c1, c2) + for _, testRTCServicePath := range testRTCServicePaths { + t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) { + // one node connecting to node 1, and another connecting to node 2 + c1 := createRTCClient("c1", defaultServerPort, testRTCServicePath, nil) + c2 := createRTCClient("c2", secondServerPort, testRTCServicePath, nil) + waitUntilConnected(t, c1, c2) + defer stopClients(c1, c2) - // c1 publishing, and c2 receiving - t1, err := c1.AddStaticTrack("audio/opus", "audio", "webcam") - require.NoError(t, err) - if t1 != nil { - defer t1.Stop() + // c1 publishing, and c2 receiving + t1, err := c1.AddStaticTrack("audio/opus", "audio", "webcam") + require.NoError(t, err) + if t1 != nil { + defer t1.Stop() + } + + testutils.WithTimeout(t, func() string { + if len(c2.SubscribedTracks()) == 0 { + return "c2 received no tracks" + } + if len(c2.SubscribedTracks()[c1.ID()]) != 1 { + return "c2 didn't receive track published by c1" + } + tr1 := c2.SubscribedTracks()[c1.ID()][0] + streamID, _ := rtc.UnpackStreamID(tr1.StreamID()) + require.Equal(t, c1.ID(), streamID) + return "" + }) + + remoteC1 := c2.GetRemoteParticipant(c1.ID()) + require.Equal(t, "c1", remoteC1.Name) + require.Equal(t, "metadatac1", remoteC1.Metadata) + }) } - - testutils.WithTimeout(t, func() string { - if len(c2.SubscribedTracks()) == 0 { - return "c2 received no tracks" - } - if len(c2.SubscribedTracks()[c1.ID()]) != 1 { - return "c2 didn't receive track published by c1" - } - tr1 := c2.SubscribedTracks()[c1.ID()][0] - streamID, _ := rtc.UnpackStreamID(tr1.StreamID()) - require.Equal(t, c1.ID(), streamID) - return "" - }) - - remoteC1 := c2.GetRemoteParticipant(c1.ID()) - require.Equal(t, "c1", remoteC1.Name) - require.Equal(t, "metadatac1", remoteC1.Metadata) } func TestConnectWithoutCreation(t *testing.T) { @@ -82,13 +88,16 @@ func TestConnectWithoutCreation(t *testing.T) { _, _, finish := setupMultiNodeTest("TestConnectWithoutCreation") defer finish() - c1 := createRTCClient("c1", defaultServerPort, nil) - waitUntilConnected(t, c1) + for _, testRTCServicePath := range testRTCServicePaths { + t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) { + c1 := createRTCClient("c1", defaultServerPort, testRTCServicePath, nil) + waitUntilConnected(t, c1) - c1.Stop() + c1.Stop() + }) + } } -// testing multiple scenarios rooms func TestMultinodePublishingUponJoining(t *testing.T) { if testing.Short() { t.SkipNow() @@ -118,30 +127,34 @@ func TestMultinodeReconnectAfterNodeShutdown(t *testing.T) { return } - _, s2, finish := setupMultiNodeTest("TestMultinodeReconnectAfterNodeShutdown") - defer finish() + for _, testRTCServicePath := range testRTCServicePaths { + t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) { + _, s2, finish := setupMultiNodeTest("TestMultinodeReconnectAfterNodeShutdown") + defer finish() - // creating room on node 1 - _, err := roomClient.CreateRoom(contextWithToken(createRoomToken()), &livekit.CreateRoomRequest{ - Name: testRoom, - NodeId: s2.Node().Id, - }) - require.NoError(t, err) + // creating room on node 1 + _, err := roomClient.CreateRoom(contextWithToken(createRoomToken()), &livekit.CreateRoomRequest{ + Name: testRoom, + NodeId: s2.Node().Id, + }) + require.NoError(t, err) - // one node connecting to node 1, and another connecting to node 2 - c1 := createRTCClient("c1", defaultServerPort, nil) - c2 := createRTCClient("c2", secondServerPort, nil) + // one node connecting to node 1, and another connecting to node 2 + c1 := createRTCClient("c1", defaultServerPort, testRTCServicePath, nil) + c2 := createRTCClient("c2", secondServerPort, testRTCServicePath, nil) - waitUntilConnected(t, c1, c2) - stopClients(c1, c2) + waitUntilConnected(t, c1, c2) + stopClients(c1, c2) - // stop s2, and connect to room again - s2.Stop(true) + // stop s2, and connect to room again + s2.Stop(true) - time.Sleep(syncDelay) + time.Sleep(syncDelay) - c3 := createRTCClient("c3", defaultServerPort, nil) - waitUntilConnected(t, c3) + c3 := createRTCClient("c3", defaultServerPort, testRTCServicePath, nil) + waitUntilConnected(t, c3) + }) + } } func TestMultinodeDataPublishing(t *testing.T) { @@ -154,6 +167,8 @@ func TestMultinodeDataPublishing(t *testing.T) { defer finish() scenarioDataPublish(t) + scenarioDataUnlabeledPublish(t) + scenarioDataTracksPublishingUponJoining(t) } func TestMultiNodeJoinAfterClose(t *testing.T) { @@ -185,48 +200,52 @@ func TestMultiNodeRefreshToken(t *testing.T) { _, _, finish := setupMultiNodeTest("TestMultiNodeJoinAfterClose") defer finish() - // a participant joining with full permissions - c1 := createRTCClient("c1", defaultServerPort, nil) - waitUntilConnected(t, c1) + for _, testRTCServicePath := range testRTCServicePaths { + t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) { + // a participant joining with full permissions + c1 := createRTCClient("c1", defaultServerPort, testRTCServicePath, nil) + waitUntilConnected(t, c1) - // update permissions and metadata - ctx := contextWithToken(adminRoomToken(testRoom)) - _, err := roomClient.UpdateParticipant(ctx, &livekit.UpdateParticipantRequest{ - Room: testRoom, - Identity: "c1", - Permission: &livekit.ParticipantPermission{ - CanPublish: false, - CanSubscribe: true, - }, - Metadata: "metadata", - }) - require.NoError(t, err) + // update permissions and metadata + ctx := contextWithToken(adminRoomToken(testRoom)) + _, err := roomClient.UpdateParticipant(ctx, &livekit.UpdateParticipantRequest{ + Room: testRoom, + Identity: "c1", + Permission: &livekit.ParticipantPermission{ + CanPublish: false, + CanSubscribe: true, + }, + Metadata: "metadata", + }) + require.NoError(t, err) - testutils.WithTimeout(t, func() string { - if c1.RefreshToken() == "" { - return "did not receive refresh token" - } - // parse token to ensure it's correct - verifier, err := auth.ParseAPIToken(c1.RefreshToken()) - require.NoError(t, err) + testutils.WithTimeout(t, func() string { + if c1.RefreshToken() == "" { + return "did not receive refresh token" + } + // parse token to ensure it's correct + verifier, err := auth.ParseAPIToken(c1.RefreshToken()) + require.NoError(t, err) - grants, err := verifier.Verify(testApiSecret) - require.NoError(t, err) + _, grants, err := verifier.Verify(testApiSecret) + require.NoError(t, err) - if grants.Metadata != "metadata" { - return "metadata did not match" - } - if *grants.Video.CanPublish { - return "canPublish should be false" - } - if *grants.Video.CanPublishData { - return "canPublishData should be false" - } - if !*grants.Video.CanSubscribe { - return "canSubscribe should be true" - } - return "" - }) + if grants.Metadata != "metadata" { + return "metadata did not match" + } + if *grants.Video.CanPublish { + return "canPublish should be false" + } + if *grants.Video.CanPublishData { + return "canPublishData should be false" + } + if !*grants.Video.CanSubscribe { + return "canSubscribe should be true" + } + return "" + }) + }) + } } // ensure that token accurately reflects out of band updates @@ -239,158 +258,170 @@ func TestMultiNodeUpdateAttributes(t *testing.T) { _, _, finish := setupMultiNodeTest("TestMultiNodeUpdateAttributes") defer finish() - c1 := createRTCClient("au1", defaultServerPort, &client.Options{ - TokenCustomizer: func(token *auth.AccessToken, grants *auth.VideoGrant) { - token.SetAttributes(map[string]string{ - "mykey": "au1", + for _, testRTCServicePath := range testRTCServicePaths { + t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) { + c1 := createRTCClient("au1", defaultServerPort, testRTCServicePath, &client.Options{ + TokenCustomizer: func(token *auth.AccessToken, grants *auth.VideoGrant) { + token.SetAttributes(map[string]string{ + "mykey": "au1", + }) + }, }) - }, - }) - c2 := createRTCClient("au2", secondServerPort, &client.Options{ - TokenCustomizer: func(token *auth.AccessToken, grants *auth.VideoGrant) { - token.SetAttributes(map[string]string{ - "mykey": "au2", + c2 := createRTCClient("au2", secondServerPort, testRTCServicePath, &client.Options{ + TokenCustomizer: func(token *auth.AccessToken, grants *auth.VideoGrant) { + token.SetAttributes(map[string]string{ + "mykey": "au2", + }) + grants.SetCanUpdateOwnMetadata(true) + }, }) - grants.SetCanUpdateOwnMetadata(true) - }, - }) - waitUntilConnected(t, c1, c2) + waitUntilConnected(t, c1, c2) - testutils.WithTimeout(t, func() string { - rc2 := c1.GetRemoteParticipant(c2.ID()) - rc1 := c2.GetRemoteParticipant(c1.ID()) - if rc2 == nil || rc1 == nil { - return "participants could not see each other" - } - if rc1.Attributes == nil || rc1.Attributes["mykey"] != "au1" { - return "rc1's initial attributes are incorrect" - } - if rc2.Attributes == nil || rc2.Attributes["mykey"] != "au2" { - return "rc2's initial attributes are incorrect" - } - return "" - }) + testutils.WithTimeout(t, func() string { + rc2 := c1.GetRemoteParticipant(c2.ID()) + rc1 := c2.GetRemoteParticipant(c1.ID()) + if rc2 == nil || rc1 == nil { + return "participants could not see each other" + } + if rc1.Attributes == nil || rc1.Attributes["mykey"] != "au1" { + return "rc1's initial attributes are incorrect" + } + if rc2.Attributes == nil || rc2.Attributes["mykey"] != "au2" { + return "rc2's initial attributes are incorrect" + } + return "" + }) - // this one should not go through - _ = c1.SetAttributes(map[string]string{"mykey": "shouldnotchange"}) - _ = c2.SetAttributes(map[string]string{"secondkey": "au2"}) + // this one should not go through + _ = c1.SetAttributes(map[string]string{"mykey": "shouldnotchange"}) + _ = c2.SetAttributes(map[string]string{"secondkey": "au2"}) - // updates using room API should succeed - _, err := roomClient.UpdateParticipant(contextWithToken(adminRoomToken(testRoom)), &livekit.UpdateParticipantRequest{ - Room: testRoom, - Identity: "au1", - Attributes: map[string]string{ - "secondkey": "au1", - }, - }) - require.NoError(t, err) + // updates using room API should succeed + _, err := roomClient.UpdateParticipant(contextWithToken(adminRoomToken(testRoom)), &livekit.UpdateParticipantRequest{ + Room: testRoom, + Identity: "au1", + Attributes: map[string]string{ + "secondkey": "au1", + }, + }) + require.NoError(t, err) - testutils.WithTimeout(t, func() string { - rc1 := c2.GetRemoteParticipant(c1.ID()) - rc2 := c1.GetRemoteParticipant(c2.ID()) - if rc1.Attributes["secondkey"] != "au1" { - return "au1's attribute update failed" - } - if rc2.Attributes["secondkey"] != "au2" { - return "au2's attribute update failed" - } - if rc1.Attributes["mykey"] != "au1" { - return "au1's mykey should not change" - } - if rc2.Attributes["mykey"] != "au2" { - return "au2's mykey should not change" - } - return "" - }) + testutils.WithTimeout(t, func() string { + rc1 := c2.GetRemoteParticipant(c1.ID()) + rc2 := c1.GetRemoteParticipant(c2.ID()) + if rc1.Attributes["secondkey"] != "au1" { + return "au1's attribute update failed" + } + if rc2.Attributes["secondkey"] != "au2" { + return "au2's attribute update failed" + } + if rc1.Attributes["mykey"] != "au1" { + return "au1's mykey should not change" + } + if rc2.Attributes["mykey"] != "au2" { + return "au2's mykey should not change" + } + return "" + }) + }) + } } func TestMultiNodeRevokePublishPermission(t *testing.T) { _, _, finish := setupMultiNodeTest("TestMultiNodeRevokePublishPermission") defer finish() - c1 := createRTCClient("c1", defaultServerPort, nil) - c2 := createRTCClient("c2", secondServerPort, nil) - waitUntilConnected(t, c1, c2) + for _, testRTCServicePath := range testRTCServicePaths { + t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) { + c1 := createRTCClient("c1", defaultServerPort, testRTCServicePath, nil) + c2 := createRTCClient("c2", secondServerPort, testRTCServicePath, nil) + waitUntilConnected(t, c1, c2) - // c1 publishes a track for c2 - writers := publishTracksForClients(t, c1) - defer stopWriters(writers...) + // c1 publishes a track for c2 + writers := publishTracksForClients(t, c1) + defer stopWriters(writers...) - testutils.WithTimeout(t, func() string { - if len(c2.SubscribedTracks()[c1.ID()]) != 2 { - return "c2 did not receive c1's tracks" - } - return "" - }) + testutils.WithTimeout(t, func() string { + if len(c2.SubscribedTracks()[c1.ID()]) != 2 { + return "c2 did not receive c1's tracks" + } + return "" + }) - // revoke permission - ctx := contextWithToken(adminRoomToken(testRoom)) - _, err := roomClient.UpdateParticipant(ctx, &livekit.UpdateParticipantRequest{ - Room: testRoom, - Identity: "c1", - Permission: &livekit.ParticipantPermission{ - CanPublish: false, - CanPublishData: true, - CanSubscribe: true, - }, - }) - require.NoError(t, err) + // revoke permission + ctx := contextWithToken(adminRoomToken(testRoom)) + _, err := roomClient.UpdateParticipant(ctx, &livekit.UpdateParticipantRequest{ + Room: testRoom, + Identity: "c1", + Permission: &livekit.ParticipantPermission{ + CanPublish: false, + CanPublishData: true, + CanSubscribe: true, + }, + }) + require.NoError(t, err) - // ensure c1 no longer has track published, c2 no longer see track under C1 - testutils.WithTimeout(t, func() string { - if len(c1.GetPublishedTrackIDs()) != 0 { - return "c1 did not unpublish tracks" - } - remoteC1 := c2.GetRemoteParticipant(c1.ID()) - if remoteC1 == nil { - return "c2 doesn't know about c1" - } - if len(remoteC1.Tracks) != 0 { - return "c2 still has c1's tracks" - } - return "" - }) + // ensure c1 no longer has track published, c2 no longer see track under C1 + testutils.WithTimeout(t, func() string { + if len(c1.GetPublishedTrackIDs()) != 0 { + return "c1 did not unpublish tracks" + } + remoteC1 := c2.GetRemoteParticipant(c1.ID()) + if remoteC1 == nil { + return "c2 doesn't know about c1" + } + if len(remoteC1.Tracks) != 0 { + return "c2 still has c1's tracks" + } + return "" + }) + }) + } } func TestCloseDisconnectedParticipantOnSignalClose(t *testing.T) { _, _, finish := setupMultiNodeTest("TestCloseDisconnectedParticipantOnSignalClose") defer finish() - c1 := createRTCClient("c1", secondServerPort, nil) - waitUntilConnected(t, c1) + for _, testRTCServicePath := range testRTCServicePaths { + t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) { + c1 := createRTCClient("c1", secondServerPort, testRTCServicePath, nil) + waitUntilConnected(t, c1) - c2 := createRTCClient("c2", defaultServerPort, &client.Options{ - SignalRequestInterceptor: func(msg *livekit.SignalRequest, next client.SignalRequestHandler) error { - switch msg.Message.(type) { - case *livekit.SignalRequest_Offer, *livekit.SignalRequest_Answer, *livekit.SignalRequest_Leave: - return nil - default: - return next(msg) - } - }, - SignalResponseInterceptor: func(msg *livekit.SignalResponse, next client.SignalResponseHandler) error { - switch msg.Message.(type) { - case *livekit.SignalResponse_Offer, *livekit.SignalResponse_Answer: - return nil - default: - return next(msg) - } - }, - }) + c2 := createRTCClient("c2", defaultServerPort, testRTCServicePath, &client.Options{ + SignalRequestInterceptor: func(msg *livekit.SignalRequest, next client.SignalRequestHandler) error { + switch msg.Message.(type) { + case *livekit.SignalRequest_Offer, *livekit.SignalRequest_Answer, *livekit.SignalRequest_Leave: + return nil + default: + return next(msg) + } + }, + SignalResponseInterceptor: func(msg *livekit.SignalResponse, next client.SignalResponseHandler) error { + switch msg.Message.(type) { + case *livekit.SignalResponse_Offer, *livekit.SignalResponse_Answer: + return nil + default: + return next(msg) + } + }, + }) - testutils.WithTimeout(t, func() string { - if len(c1.RemoteParticipants()) != 1 { - return "c1 did not see c2 join" - } - return "" - }) + testutils.WithTimeout(t, func() string { + if len(c1.RemoteParticipants()) != 1 { + return "c1 did not see c2 join" + } + return "" + }) - c2.Stop() + c2.Stop() - testutils.WithTimeout(t, func() string { - if len(c1.RemoteParticipants()) != 0 { - return "c1 did not see c2 removed" - } - return "" - }) + testutils.WithTimeout(t, func() string { + if len(c1.RemoteParticipants()) != 0 { + return "c1 did not see c2 removed" + } + return "" + }) + }) + } } diff --git a/test/scenarios.go b/test/scenarios.go index d725786..aee71fe 100644 --- a/test/scenarios.go +++ b/test/scenarios.go @@ -31,153 +31,286 @@ import ( // a scenario with lots of clients connecting, publishing, and leaving at random periods func scenarioPublishingUponJoining(t *testing.T) { - c1 := createRTCClient("puj_1", defaultServerPort, nil) - c2 := createRTCClient("puj_2", secondServerPort, &testclient.Options{AutoSubscribe: true}) - c3 := createRTCClient("puj_3", defaultServerPort, &testclient.Options{AutoSubscribe: true}) - defer stopClients(c1, c2, c3) + for _, testRTCServicePath := range testRTCServicePaths { + t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) { + c1 := createRTCClient("puj_1", defaultServerPort, testRTCServicePath, nil) + c2 := createRTCClient("puj_2", secondServerPort, testRTCServicePath, &testclient.Options{AutoSubscribe: true}) + c3 := createRTCClient("puj_3", defaultServerPort, testRTCServicePath, &testclient.Options{AutoSubscribe: true}) + defer stopClients(c1, c2, c3) - waitUntilConnected(t, c1, c2, c3) + waitUntilConnected(t, c1, c2, c3) - // c1 and c2 publishing, c3 just receiving - writers := publishTracksForClients(t, c1, c2) - defer stopWriters(writers...) + // c1 and c2 publishing, c3 just receiving + writers := publishTracksForClients(t, c1, c2) + defer stopWriters(writers...) - logger.Infow("waiting to receive tracks from c1 and c2") - testutils.WithTimeout(t, func() string { - tracks := c3.SubscribedTracks() - if len(tracks[c1.ID()]) != 2 { - return "did not receive tracks from c1" - } - if len(tracks[c2.ID()]) != 2 { - return "did not receive tracks from c2" - } - return "" - }) + logger.Infow("waiting to receive tracks from c1 and c2") + testutils.WithTimeout(t, func() string { + tracks := c3.SubscribedTracks() + if len(tracks[c1.ID()]) != 2 { + return "did not receive tracks from c1" + } + if len(tracks[c2.ID()]) != 2 { + return "did not receive tracks from c2" + } + return "" + }) - // after a delay, c2 reconnects, then publishing - time.Sleep(syncDelay) - c2.Stop() + // after a delay, c2 reconnects, then publishing + time.Sleep(syncDelay) + c2.Stop() - logger.Infow("waiting for c2 tracks to be gone") - testutils.WithTimeout(t, func() string { - tracks := c3.SubscribedTracks() + logger.Infow("waiting for c2 tracks to be gone") + testutils.WithTimeout(t, func() string { + tracks := c3.SubscribedTracks() - if len(tracks[c1.ID()]) != 2 { - return fmt.Sprintf("c3 should be subscribed to 2 tracks from c1, actual: %d", len(tracks[c1.ID()])) - } - if len(tracks[c2.ID()]) != 0 { - return fmt.Sprintf("c3 should be subscribed to 0 tracks from c2, actual: %d", len(tracks[c2.ID()])) - } - if len(c1.SubscribedTracks()[c2.ID()]) != 0 { - return fmt.Sprintf("c3 should be subscribed to 0 tracks from c2, actual: %d", len(c1.SubscribedTracks()[c2.ID()])) - } - return "" - }) + if len(tracks[c1.ID()]) != 2 { + return fmt.Sprintf("c3 should be subscribed to 2 tracks from c1, actual: %d", len(tracks[c1.ID()])) + } + if len(tracks[c2.ID()]) != 0 { + return fmt.Sprintf("c3 should be subscribed to 0 tracks from c2, actual: %d", len(tracks[c2.ID()])) + } + if len(c1.SubscribedTracks()[c2.ID()]) != 0 { + return fmt.Sprintf("c3 should be subscribed to 0 tracks from c2, actual: %d", len(c1.SubscribedTracks()[c2.ID()])) + } + return "" + }) - logger.Infow("c2 reconnecting") - // connect to a diff port - c2 = createRTCClient("puj_2", defaultServerPort, nil) - defer c2.Stop() - waitUntilConnected(t, c2) - writers = publishTracksForClients(t, c2) - defer stopWriters(writers...) + logger.Infow("c2 reconnecting") + // connect to a diff port + c2 = createRTCClient("puj_2", defaultServerPort, testRTCServicePath, nil) + defer c2.Stop() + waitUntilConnected(t, c2) + writers = publishTracksForClients(t, c2) + defer stopWriters(writers...) - testutils.WithTimeout(t, func() string { - tracks := c3.SubscribedTracks() - // "new c2 tracks should be published again", - if len(tracks[c2.ID()]) != 2 { - return fmt.Sprintf("c3 should be subscribed to 2 tracks from c2, actual: %d", len(tracks[c2.ID()])) - } - if len(c1.SubscribedTracks()[c2.ID()]) != 2 { - return fmt.Sprintf("c1 should be subscribed to 2 tracks from c2, actual: %d", len(c1.SubscribedTracks()[c2.ID()])) - } - return "" - }) + testutils.WithTimeout(t, func() string { + tracks := c3.SubscribedTracks() + // "new c2 tracks should be published again", + if len(tracks[c2.ID()]) != 2 { + return fmt.Sprintf("c3 should be subscribed to 2 tracks from c2, actual: %d", len(tracks[c2.ID()])) + } + if len(c1.SubscribedTracks()[c2.ID()]) != 2 { + return fmt.Sprintf("c1 should be subscribed to 2 tracks from c2, actual: %d", len(c1.SubscribedTracks()[c2.ID()])) + } + return "" + }) + }) + } } func scenarioReceiveBeforePublish(t *testing.T) { - c1 := createRTCClient("rbp_1", defaultServerPort, nil) - c2 := createRTCClient("rbp_2", defaultServerPort, nil) + for _, testRTCServicePath := range testRTCServicePaths { + t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) { + c1 := createRTCClient("rbp_1", defaultServerPort, testRTCServicePath, nil) + c2 := createRTCClient("rbp_2", defaultServerPort, testRTCServicePath, nil) - waitUntilConnected(t, c1, c2) - defer stopClients(c1, c2) + waitUntilConnected(t, c1, c2) + defer stopClients(c1, c2) - // c1 publishes - writers := publishTracksForClients(t, c1) - defer stopWriters(writers...) + // c1 publishes + writers := publishTracksForClients(t, c1) + defer stopWriters(writers...) - // c2 should see some bytes flowing through - testutils.WithTimeout(t, func() string { - if c2.BytesReceived() > 20 { - return "" - } else { - return fmt.Sprintf("c2 only received %d bytes", c2.BytesReceived()) - } - }) + // c2 should see some bytes flowing through + testutils.WithTimeout(t, func() string { + if c2.BytesReceived() > 20 { + return "" + } else { + return fmt.Sprintf("c2 only received %d bytes", c2.BytesReceived()) + } + }) - // now publish on C2 - writers = publishTracksForClients(t, c2) - defer stopWriters(writers...) + // now publish on C2 + writers = publishTracksForClients(t, c2) + defer stopWriters(writers...) - testutils.WithTimeout(t, func() string { - if len(c1.SubscribedTracks()[c2.ID()]) == 2 { - return "" - } else { - return fmt.Sprintf("expected c1 to receive 2 tracks from c2, actual: %d", len(c1.SubscribedTracks()[c2.ID()])) - } - }) + testutils.WithTimeout(t, func() string { + if len(c1.SubscribedTracks()[c2.ID()]) == 2 { + return "" + } else { + return fmt.Sprintf("expected c1 to receive 2 tracks from c2, actual: %d", len(c1.SubscribedTracks()[c2.ID()])) + } + }) - // now leave, and ensure that it's immediate - c2.Stop() + // now leave, and ensure that it's immediate + c2.Stop() - testutils.WithTimeout(t, func() string { - if len(c1.RemoteParticipants()) > 0 { - return fmt.Sprintf("expected no remote participants, actual: %v", c1.RemoteParticipants()) - } - return "" - }) + testutils.WithTimeout(t, func() string { + if len(c1.RemoteParticipants()) > 0 { + return fmt.Sprintf("expected no remote participants, actual: %v", c1.RemoteParticipants()) + } + return "" + }) + }) + } } func scenarioDataPublish(t *testing.T) { - c1 := createRTCClient("dp1", defaultServerPort, nil) - c2 := createRTCClient("dp2", secondServerPort, nil) - waitUntilConnected(t, c1, c2) - defer stopClients(c1, c2) + for _, testRTCServicePath := range testRTCServicePaths { + t.Run(fmt.Sprintf("scenarioDataPublish/testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) { + c1 := createRTCClient("dp1", defaultServerPort, testRTCServicePath, nil) + c2 := createRTCClient("dp2", secondServerPort, testRTCServicePath, nil) + waitUntilConnected(t, c1, c2) + defer stopClients(c1, c2) - payload := "test bytes" + payload := "test bytes" - received := atomic.NewBool(false) - c2.OnDataReceived = func(data []byte, sid string) { - if string(data) == payload && livekit.ParticipantID(sid) == c1.ID() { - received.Store(true) - } + received := atomic.NewBool(false) + c2.OnDataReceived = func(data []byte, sid string) { + if string(data) == payload && livekit.ParticipantID(sid) == c1.ID() { + received.Store(true) + } + } + + require.NoError(t, c1.PublishData([]byte(payload), livekit.DataPacket_RELIABLE)) + + testutils.WithTimeout(t, func() string { + if received.Load() { + return "" + } else { + return "c2 did not receive published data" + } + }) + }) } +} - require.NoError(t, c1.PublishData([]byte(payload), livekit.DataPacket_RELIABLE)) +func scenarioDataUnlabeledPublish(t *testing.T) { + for _, testRTCServicePath := range testRTCServicePaths { + t.Run(fmt.Sprintf("scenarioDataUnlabeledPublish/testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) { + c1 := createRTCClient("dp1", defaultServerPort, testRTCServicePath, nil) + c2 := createRTCClient("dp2", secondServerPort, testRTCServicePath, nil) + waitUntilConnected(t, c1, c2) + defer stopClients(c1, c2) - testutils.WithTimeout(t, func() string { - if received.Load() { - return "" - } else { - return "c2 did not receive published data" - } - }) + payload := "test unlabeled bytes" + + received := atomic.NewBool(false) + c2.OnDataReceived = func(data []byte, _sid string) { + if string(data) == payload { + received.Store(true) + } + } + + require.NoError(t, c1.PublishDataUnlabeled([]byte(payload))) + + testutils.WithTimeout(t, func() string { + if received.Load() { + return "" + } else { + return "c2 did not receive published data unlabeled" + } + }) + }) + } +} + +func scenarioDataTracksPublishingUponJoining(t *testing.T) { + for _, testRTCServicePath := range testRTCServicePaths { + t.Run(fmt.Sprintf("scenarioDataTracksPublishingUponJoining/testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) { + c1 := createRTCClient("dtpuj_1", defaultServerPort, testRTCServicePath, &testclient.Options{AutoSubscribeDataTrack: true}) + c2 := createRTCClient("dtpuj_2", secondServerPort, testRTCServicePath, &testclient.Options{AutoSubscribeDataTrack: true}) + c3 := createRTCClient("dtpuj_3", defaultServerPort, testRTCServicePath, &testclient.Options{AutoSubscribeDataTrack: true}) + defer stopClients(c1, c2, c3) + + waitUntilConnected(t, c1, c2, c3) + + // c1 and c2 publishing, c3 just receiving + writers := publishDataTracksForClients(t, c1, c2) + defer stopWriters(writers...) + + logger.Infow("waiting to receive data tracks from c1 and c2") + testutils.WithTimeout(t, func() string { + tracks := c3.SubscribedDataTracks() + if len(tracks[c1.ID()]) != 2 { + return "did not receive data tracks from c1" + } + if len(tracks[c2.ID()]) != 2 { + return "did not receive data tracks from c2" + } + for _, dts := range tracks { + for _, dt := range dts { + if dt.NumReceivedPackets() == 0 { + return fmt.Sprintf("no packets received from %s", dt.ID()) + } + } + } + return "" + }) + + // after a delay, c2 reconnects, then publishing + time.Sleep(syncDelay) + c2.Stop() + + logger.Infow("waiting for c2 data tracks to be gone") + testutils.WithTimeout(t, func() string { + tracks := c3.SubscribedDataTracks() + + if len(tracks[c1.ID()]) != 2 { + return fmt.Sprintf("c3 should be subscribed to 2 data tracks from c1, actual: %d", len(tracks[c1.ID()])) + } + if len(tracks[c2.ID()]) != 0 { + return fmt.Sprintf("c3 should be subscribed to 0 data tracks from c2, actual: %d", len(tracks[c2.ID()])) + } + if len(c1.SubscribedDataTracks()[c2.ID()]) != 0 { + return fmt.Sprintf("c3 should be subscribed to 0 data tracks from c2, actual: %d", len(c1.SubscribedTracks()[c2.ID()])) + } + return "" + }) + + logger.Infow("c2 reconnecting") + // connect to a diff port + c2 = createRTCClient("dtpuj_2", defaultServerPort, testRTCServicePath, &testclient.Options{AutoSubscribeDataTrack: true}) + defer c2.Stop() + waitUntilConnected(t, c2) + writers = publishDataTracksForClients(t, c2) + defer stopWriters(writers...) + + testutils.WithTimeout(t, func() string { + tracks := c3.SubscribedDataTracks() + // new c2 data tracks should be published again + if len(tracks[c2.ID()]) != 2 { + return fmt.Sprintf("c3 should be subscribed to 2 data tracks from c2, actual: %d", len(tracks[c2.ID()])) + } + for _, dt := range tracks[c2.ID()] { + if dt.NumReceivedPackets() == 0 { + return fmt.Sprintf("c3 did not receive packets from c2 data track after reconnecting %s", dt.ID()) + } + } + + if len(c1.SubscribedDataTracks()[c2.ID()]) != 2 { + return fmt.Sprintf("c1 should be subscribed to 2 data tracks from c2, actual: %d", len(c1.SubscribedTracks()[c2.ID()])) + } + for _, dt := range c1.SubscribedDataTracks()[c2.ID()] { + if dt.NumReceivedPackets() == 0 { + return fmt.Sprintf("c1 did not receive packets from c2 data track after reconnecting %s", dt.ID()) + } + } + return "" + }) + }) + } } func scenarioJoinClosedRoom(t *testing.T) { - c1 := createRTCClient("jcr1", defaultServerPort, nil) - waitUntilConnected(t, c1) + for _, testRTCServicePath := range testRTCServicePaths { + t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) { + c1 := createRTCClient("jcr1", defaultServerPort, testRTCServicePath, nil) + waitUntilConnected(t, c1) - // close room with room client - _, err := roomClient.DeleteRoom(contextWithToken(createRoomToken()), &livekit.DeleteRoomRequest{ - Room: testRoom, - }) - require.NoError(t, err) + // close room with room client + _, err := roomClient.DeleteRoom(contextWithToken(createRoomToken()), &livekit.DeleteRoomRequest{ + Room: testRoom, + }) + require.NoError(t, err) - // now join again - c2 := createRTCClient("jcr2", defaultServerPort, nil) - waitUntilConnected(t, c2) - stopClients(c2) + // now join again + c2 := createRTCClient("jcr2", defaultServerPort, testRTCServicePath, nil) + waitUntilConnected(t, c2) + stopClients(c2) + }) + } } // close a room that has been created, but no participant has joined @@ -194,15 +327,15 @@ func closeNonRTCRoom(t *testing.T) { require.NoError(t, err) } -func publishTracksForClients(t *testing.T, clients ...*testclient.RTCClient) []*testclient.TrackWriter { +func publishTracksForClients(t *testing.T, clients ...*testclient.RTCClient) []testclient.TrackWriter { logger.Infow("publishing tracks for clients") - var writers []*testclient.TrackWriter + var writers []testclient.TrackWriter for i := range clients { c := clients[i] tw, err := c.AddStaticTrack("audio/opus", "audio", "webcam") require.NoError(t, err) - writers = append(writers, tw) + tw, err = c.AddStaticTrack("video/vp8", "video", "webcam") require.NoError(t, err) writers = append(writers, tw) @@ -210,6 +343,20 @@ func publishTracksForClients(t *testing.T, clients ...*testclient.RTCClient) []* return writers } +func publishDataTracksForClients(t *testing.T, clients ...*testclient.RTCClient) []testclient.TrackWriter { + logger.Infow("publishing data tracks for clients") + var writers []testclient.TrackWriter + for i := range clients { + c := clients[i] + for range 2 { + dtw, err := c.PublishDataTrack() + require.NoError(t, err) + writers = append(writers, dtw) + } + } + return writers +} + // Room service tests func roomServiceListRoom(t *testing.T) { diff --git a/test/singlenode_test.go b/test/singlenode_test.go index b11e5da..6d571da 100644 --- a/test/singlenode_test.go +++ b/test/singlenode_test.go @@ -19,12 +19,18 @@ import ( "encoding/binary" "errors" "fmt" + "net" "net/http" + "reflect" "strings" + "sync" "testing" "time" + "github.com/jxskiss/base62" "github.com/pion/sdp/v3" + "github.com/pion/stun/v3" + "github.com/pion/turn/v5" "github.com/pion/webrtc/v4" "github.com/stretchr/testify/require" "github.com/thoas/go-funk" @@ -32,13 +38,16 @@ import ( "go.uber.org/atomic" "github.com/livekit/protocol/auth" + "github.com/livekit/protocol/codecs/mime" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" "github.com/livekit/livekit-server/pkg/config" "github.com/livekit/livekit-server/pkg/rtc" + "github.com/livekit/livekit-server/pkg/rtc/types" + "github.com/livekit/livekit-server/pkg/service" + "github.com/livekit/livekit-server/pkg/sfu" "github.com/livekit/livekit-server/pkg/sfu/datachannel" - "github.com/livekit/livekit-server/pkg/sfu/mime" "github.com/livekit/livekit-server/pkg/testutils" testclient "github.com/livekit/livekit-server/test/client" ) @@ -57,20 +66,24 @@ func TestClientCouldConnect(t *testing.T) { _, finish := setupSingleNodeTest("TestClientCouldConnect") defer finish() - c1 := createRTCClient("c1", defaultServerPort, nil) - c2 := createRTCClient("c2", defaultServerPort, nil) - waitUntilConnected(t, c1, c2) + for _, testRTCServicePath := range testRTCServicePaths { + t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) { + c1 := createRTCClient("c1", defaultServerPort, testRTCServicePath, nil) + c2 := createRTCClient("c2", defaultServerPort, testRTCServicePath, nil) + waitUntilConnected(t, c1, c2) - // ensure they both see each other - testutils.WithTimeout(t, func() string { - if len(c1.RemoteParticipants()) == 0 { - return "c1 did not see c2" - } - if len(c2.RemoteParticipants()) == 0 { - return "c2 did not see c1" - } - return "" - }) + // ensure they both see each other + testutils.WithTimeout(t, func() string { + if len(c1.RemoteParticipants()) == 0 { + return "c1 did not see c2" + } + if len(c2.RemoteParticipants()) == 0 { + return "c2 did not see c1" + } + return "" + }) + }) + } } func TestClientConnectDuplicate(t *testing.T) { @@ -79,68 +92,71 @@ func TestClientConnectDuplicate(t *testing.T) { return } - _, finish := setupSingleNodeTest("TestClientCouldConnect") + _, finish := setupSingleNodeTest("TestClientConnectDuplicate") defer finish() - grant := &auth.VideoGrant{RoomJoin: true, Room: testRoom} - grant.SetCanPublish(true) - grant.SetCanSubscribe(true) - token := joinTokenWithGrant("c1", grant) + for _, testRTCServicePath := range testRTCServicePaths { + t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) { + grant := &auth.VideoGrant{RoomJoin: true, Room: testRoom} + grant.SetCanPublish(true) + grant.SetCanSubscribe(true) + token := joinTokenWithGrant("c1", grant) + c1 := createRTCClientWithToken(token, defaultServerPort, testRTCServicePath, nil) - c1 := createRTCClientWithToken(token, defaultServerPort, nil) + // publish 2 tracks + t1, err := c1.AddStaticTrack("audio/opus", "audio", "webcam") + require.NoError(t, err) + defer t1.Stop() + t2, err := c1.AddStaticTrack("video/vp8", "video", "webcam") + require.NoError(t, err) + defer t2.Stop() - // publish 2 tracks - t1, err := c1.AddStaticTrack("audio/opus", "audio", "webcam") - require.NoError(t, err) - defer t1.Stop() - t2, err := c1.AddStaticTrack("video/vp8", "video", "webcam") - require.NoError(t, err) - defer t2.Stop() + c2 := createRTCClient("c2", defaultServerPort, testRTCServicePath, nil) + waitUntilConnected(t, c1, c2) - c2 := createRTCClient("c2", defaultServerPort, nil) - waitUntilConnected(t, c1, c2) + opts := &testclient.Options{ + Publish: "duplicate_connection", + } + testutils.WithTimeout(t, func() string { + if len(c2.SubscribedTracks()) == 0 { + return "c2 didn't subscribe to anything" + } + // should have received two tracks + if len(c2.SubscribedTracks()[c1.ID()]) != 2 { + return "c2 didn't subscribe to both tracks from c1" + } - opts := &testclient.Options{ - Publish: "duplicate_connection", + // participant ID can be appended with '#..' . but should contain orig id as prefix + tr1 := c2.SubscribedTracks()[c1.ID()][0] + participantId1, _ := rtc.UnpackStreamID(tr1.StreamID()) + require.Equal(t, c1.ID(), participantId1) + tr2 := c2.SubscribedTracks()[c1.ID()][1] + participantId2, _ := rtc.UnpackStreamID(tr2.StreamID()) + require.Equal(t, c1.ID(), participantId2) + return "" + }) + + c1Dup := createRTCClientWithToken(token, defaultServerPort, testRTCServicePath, opts) + + waitUntilConnected(t, c1Dup) + + t3, err := c1Dup.AddStaticTrack("video/vp8", "video", "webcam") + require.NoError(t, err) + defer t3.Stop() + + testutils.WithTimeout(t, func() string { + if len(c2.SubscribedTracks()[c1Dup.ID()]) != 1 { + return "c2 was not subscribed to track from duplicated c1" + } + + tr3 := c2.SubscribedTracks()[c1Dup.ID()][0] + participantId3, _ := rtc.UnpackStreamID(tr3.StreamID()) + require.Contains(t, c1Dup.ID(), participantId3) + + return "" + }) + }) } - testutils.WithTimeout(t, func() string { - if len(c2.SubscribedTracks()) == 0 { - return "c2 didn't subscribe to anything" - } - // should have received three tracks - if len(c2.SubscribedTracks()[c1.ID()]) != 2 { - return "c2 didn't subscribe to both tracks from c1" - } - - // participant ID can be appended with '#..' . but should contain orig id as prefix - tr1 := c2.SubscribedTracks()[c1.ID()][0] - participantId1, _ := rtc.UnpackStreamID(tr1.StreamID()) - require.Equal(t, c1.ID(), participantId1) - tr2 := c2.SubscribedTracks()[c1.ID()][1] - participantId2, _ := rtc.UnpackStreamID(tr2.StreamID()) - require.Equal(t, c1.ID(), participantId2) - return "" - }) - - c1Dup := createRTCClientWithToken(token, defaultServerPort, opts) - - waitUntilConnected(t, c1Dup) - - t3, err := c1Dup.AddStaticTrack("video/vp8", "video", "webcam") - require.NoError(t, err) - defer t3.Stop() - - testutils.WithTimeout(t, func() string { - if len(c2.SubscribedTracks()[c1Dup.ID()]) != 1 { - return "c2 was not subscribed to track from duplicated c1" - } - - tr3 := c2.SubscribedTracks()[c1Dup.ID()][0] - participantId3, _ := rtc.UnpackStreamID(tr3.StreamID()) - require.Contains(t, c1Dup.ID(), participantId3) - - return "" - }) } func TestSinglePublisher(t *testing.T) { @@ -152,76 +168,291 @@ func TestSinglePublisher(t *testing.T) { s, finish := setupSingleNodeTest("TestSinglePublisher") defer finish() - c1 := createRTCClient("c1", defaultServerPort, nil) - c2 := createRTCClient("c2", defaultServerPort, nil) - waitUntilConnected(t, c1, c2) + for _, testRTCServicePath := range testRTCServicePaths { + t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) { + c1 := createRTCClient("c1", defaultServerPort, testRTCServicePath, nil) + c2 := createRTCClient("c2", defaultServerPort, testRTCServicePath, nil) + waitUntilConnected(t, c1, c2) - // publish a track and ensure clients receive it ok - t1, err := c1.AddStaticTrack("audio/OPUS", "audio", "webcamaudio") - require.NoError(t, err) - defer t1.Stop() - t2, err := c1.AddStaticTrack("video/vp8", "video", "webcamvideo") - require.NoError(t, err) - defer t2.Stop() + // publish an audio and video track and ensure clients receive it ok + t1, err := c1.AddStaticTrack("audio/OPUS", "audio", "webcamaudio") + require.NoError(t, err) + defer t1.Stop() + t2, err := c1.AddStaticTrack("video/vp8", "video", "webcamvideo") + require.NoError(t, err) + defer t2.Stop() - testutils.WithTimeout(t, func() string { - if len(c2.SubscribedTracks()) == 0 { - return "c2 was not subscribed to anything" - } - // should have received two tracks - if len(c2.SubscribedTracks()[c1.ID()]) != 2 { - return "c2 didn't subscribe to both tracks from c1" - } + testutils.WithTimeout(t, func() string { + if len(c2.SubscribedTracks()) == 0 { + return "c2 was not subscribed to anything" + } + // should have received two tracks + if len(c2.SubscribedTracks()[c1.ID()]) != 2 { + return "c2 didn't subscribe to both tracks from c1" + } - tr1 := c2.SubscribedTracks()[c1.ID()][0] - participantId, _ := rtc.UnpackStreamID(tr1.StreamID()) - require.Equal(t, c1.ID(), participantId) - return "" - }) - // ensure mime type is received - remoteC1 := c2.GetRemoteParticipant(c1.ID()) - audioTrack := funk.Find(remoteC1.Tracks, func(ti *livekit.TrackInfo) bool { - return ti.Name == "webcamaudio" - }).(*livekit.TrackInfo) - require.Equal(t, "audio/opus", audioTrack.MimeType) + tr1 := c2.SubscribedTracks()[c1.ID()][0] + participantId, _ := rtc.UnpackStreamID(tr1.StreamID()) + require.Equal(t, c1.ID(), participantId) + return "" + }) + // ensure mime type is received + remoteC1 := c2.GetRemoteParticipant(c1.ID()) + audioTrack := funk.Find(remoteC1.Tracks, func(ti *livekit.TrackInfo) bool { + return ti.Name == "webcamaudio" + }).(*livekit.TrackInfo) + require.Equal(t, "audio/opus", audioTrack.MimeType) - // a new client joins and should get the initial stream - c3 := createRTCClient("c3", defaultServerPort, nil) + // a new client joins and should get the initial stream + c3 := createRTCClient("c3", defaultServerPort, testRTCServicePath, nil) - // ensure that new client that has joined also received tracks - waitUntilConnected(t, c3) - testutils.WithTimeout(t, func() string { - if len(c3.SubscribedTracks()) == 0 { - return "c3 didn't subscribe to anything" - } - // should have received two tracks - if len(c3.SubscribedTracks()[c1.ID()]) != 2 { - return "c3 didn't subscribe to tracks from c1" - } - return "" - }) + // ensure that new client that has joined also received tracks + waitUntilConnected(t, c3) + testutils.WithTimeout(t, func() string { + if len(c3.SubscribedTracks()) == 0 { + return "c3 didn't subscribe to anything" + } + // should have received two tracks + if len(c3.SubscribedTracks()[c1.ID()]) != 2 { + return "c3 didn't subscribe to tracks from c1" + } + return "" + }) - // ensure that the track ids are generated by server - tracks := c3.SubscribedTracks()[c1.ID()] - for _, tr := range tracks { - require.True(t, strings.HasPrefix(tr.ID(), "TR_"), "track should begin with TR") + // ensure that the track ids are generated by server + tracks := c3.SubscribedTracks()[c1.ID()] + for _, tr := range tracks { + require.True(t, strings.HasPrefix(tr.ID(), "TR_"), "track should begin with TR") + } + + // when c3 disconnects, ensure subscriber is cleaned up correctly + c3.Stop() + + testutils.WithTimeout(t, func() string { + room := s.RoomManager().GetRoom(context.Background(), testRoom) + p := room.GetParticipant("c1") + require.NotNil(t, p) + + for _, t := range p.GetPublishedTracks() { + if t.IsSubscriber(c3.ID()) { + return "c3 was not a subscriber of c1's tracks" + } + } + return "" + }) + }) + } +} + +func TestConnectionStats(t *testing.T) { + if testing.Short() { + t.SkipNow() + return } - // when c3 disconnects, ensure subscriber is cleaned up correctly - c3.Stop() + s, finish := setupSingleNodeTest("TestConnectionStats") + defer finish() - testutils.WithTimeout(t, func() string { - room := s.RoomManager().GetRoom(context.Background(), testRoom) - p := room.GetParticipant("c1") - require.NotNil(t, p) + for _, testRTCServicePath := range testRTCServicePaths { + t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) { + c1 := createRTCClient("c1", defaultServerPort, testRTCServicePath, nil) + c2 := createRTCClient("c2", defaultServerPort, testRTCServicePath, nil) + waitUntilConnected(t, c1, c2) + defer func() { + c1.Stop() + c2.Stop() + }() - for _, t := range p.GetPublishedTracks() { - if t.IsSubscriber(c3.ID()) { - return "c3 was not a subscriber of c1's tracks" + // both clients publish audio + video + t1, err := c1.AddStaticTrack("audio/opus", "audio", "c1audio") + require.NoError(t, err) + defer t1.Stop() + t2, err := c1.AddStaticTrack("video/vp8", "video", "c1video") + require.NoError(t, err) + defer t2.Stop() + + t3, err := c2.AddStaticTrack("audio/opus", "audio", "c2audio") + require.NoError(t, err) + defer t3.Stop() + t4, err := c2.AddStaticTrack("video/vp8", "video", "c2video") + require.NoError(t, err) + defer t4.Stop() + + // wait for cross-subscriptions: each client should receive 2 tracks from the other + testutils.WithTimeout(t, func() string { + if len(c1.SubscribedTracks()[c2.ID()]) != 2 { + return "c1 did not subscribe to both tracks from c2" + } + if len(c2.SubscribedTracks()[c1.ID()]) != 2 { + return "c2 did not subscribe to both tracks from c1" + } + return "" + }) + + room := s.RoomManager().GetRoom(context.Background(), testRoom) + require.NotNil(t, room) + + // hook the upstream WebRTCReceiver.OnStatsUpdate and downstream DownTrack.OnStatsUpdate + // callbacks so we can verify the AnalyticsStat delivered through each carries valid + // delta data. MediaTrack.Receivers() returns one entry per potential codec; only those + // matching the actually published codec are *sfu.WebRTCReceiver, the rest are + // placeholder *rtc.DummyReceiver instances that we skip. + type statCapture struct { + lock sync.Mutex + stat *livekit.AnalyticsStat } - } - return "" - }) + receiverCaptures := make(map[livekit.TrackID]*statCapture) + downTrackCaptures := make(map[livekit.ParticipantIdentity]map[livekit.TrackID]*statCapture) + for _, identity := range []livekit.ParticipantIdentity{"c1", "c2"} { + p := room.GetParticipant(identity) + require.NotNil(t, p, "participant %s not found", identity) + for _, mt := range p.GetPublishedTracks() { + rc := &statCapture{} + receiverCaptures[mt.ID()] = rc + var hooked int + for _, r := range mt.Receivers() { + if dr, ok := r.(*rtc.DummyReceiver); ok { + underlying := dr.Receiver() + if underlying == nil { + continue + } + r = underlying + } + wr, ok := r.(*sfu.WebRTCReceiver) + if !ok { + continue + } + wr.OnStatsUpdate(func(_ *sfu.WebRTCReceiver, stat *livekit.AnalyticsStat) { + rc.lock.Lock() + rc.stat = stat + rc.lock.Unlock() + }) + hooked++ + } + require.Greater(t, hooked, 0, "no live WebRTCReceiver found for published track %s", mt.ID()) + } + + dtCaps := make(map[livekit.TrackID]*statCapture) + downTrackCaptures[identity] = dtCaps + for _, st := range p.GetSubscribedTracks() { + dt := st.DownTrack() + require.NotNil(t, dt, "subscribed track %s has no DownTrack", st.ID()) + dc := &statCapture{} + dtCaps[st.ID()] = dc + dt.OnStatsUpdate(func(_ *sfu.DownTrack, stat *livekit.AnalyticsStat) { + dc.lock.Lock() + dc.stat = stat + dc.lock.Unlock() + }) + } + } + + validateAnalyticsStat := func(stat *livekit.AnalyticsStat) string { + if stat == nil { + return "stat nil" + } + if len(stat.Streams) == 0 { + return "stat has no streams" + } + var totalPackets uint32 + var totalBytes uint64 + for _, s := range stat.Streams { + totalPackets += s.PrimaryPackets + totalBytes += s.PrimaryBytes + } + if totalPackets == 0 { + return "stat has no packets across streams" + } + if totalBytes == 0 { + return "stat has no bytes across streams" + } + return "" + } + + // wait for cumulative + delta + OnStatsUpdate-derived stats. the + // connection-quality update interval is 5s, so allow plenty of time for + // the receiver OnStatsUpdate callback to fire at least once and for + // the downstream connection-quality scorer to compute a real score. + testutils.WithTimeout(t, func() string { + for _, identity := range []livekit.ParticipantIdentity{"c1", "c2"} { + p := room.GetParticipant(identity) + if p == nil { + return fmt.Sprintf("participant %s not found", identity) + } + + // upstream (publisher) cumulative stats + published := p.GetPublishedTracks() + if len(published) != 2 { + return fmt.Sprintf("%s expected 2 published tracks, got %d", identity, len(published)) + } + for _, mt := range published { + lmt, ok := mt.(types.LocalMediaTrack) + if !ok { + return fmt.Sprintf("%s published track %s is not a LocalMediaTrack", identity, mt.ID()) + } + stats := lmt.GetTrackStats() + if stats == nil { + return fmt.Sprintf("%s upstream cumulative stats nil for track %s", identity, mt.ID()) + } + if stats.Packets == 0 { + return fmt.Sprintf("%s upstream cumulative stats has no packets for track %s", identity, mt.ID()) + } + if stats.Bytes == 0 { + return fmt.Sprintf("%s upstream cumulative stats has no bytes for track %s", identity, mt.ID()) + } + + // upstream delta stats fed into the receiver OnStatsUpdate path + rc, ok := receiverCaptures[mt.ID()] + if !ok { + return fmt.Sprintf("%s missing receiver capture for track %s", identity, mt.ID()) + } + rc.lock.Lock() + stat := rc.stat + rc.lock.Unlock() + if msg := validateAnalyticsStat(stat); msg != "" { + return fmt.Sprintf("%s upstream OnStatsUpdate %s for track %s", identity, msg, mt.ID()) + } + } + + // downstream (subscriber) cumulative stats and DownTrack OnStatsUpdate + // delta stats captured from the listener path + subscribed := p.GetSubscribedTracks() + if len(subscribed) != 2 { + return fmt.Sprintf("%s expected 2 subscribed tracks, got %d", identity, len(subscribed)) + } + for _, st := range subscribed { + dt := st.DownTrack() + if dt == nil { + return fmt.Sprintf("%s subscribed track %s has no DownTrack", identity, st.ID()) + } + stats := dt.GetTrackStats() + if stats == nil { + return fmt.Sprintf("%s downstream cumulative stats nil for track %s", identity, st.ID()) + } + if stats.Packets == 0 { + return fmt.Sprintf("%s downstream cumulative stats has no packets for track %s", identity, st.ID()) + } + if stats.Bytes == 0 { + return fmt.Sprintf("%s downstream cumulative stats has no bytes for track %s", identity, st.ID()) + } + + // downstream delta stats fed into the DownTrack OnStatsUpdate path + dc, ok := downTrackCaptures[identity][st.ID()] + if !ok { + return fmt.Sprintf("%s missing DownTrack capture for track %s", identity, st.ID()) + } + dc.lock.Lock() + stat := dc.stat + dc.lock.Unlock() + if msg := validateAnalyticsStat(stat); msg != "" { + return fmt.Sprintf("%s downstream OnStatsUpdate %s for track %s", identity, msg, st.ID()) + } + } + } + return "" + }, 15*time.Second) + }) + } } func Test_WhenAutoSubscriptionDisabled_ClientShouldNotReceiveAnyPublishedTracks(t *testing.T) { @@ -233,20 +464,24 @@ func Test_WhenAutoSubscriptionDisabled_ClientShouldNotReceiveAnyPublishedTracks( _, finish := setupSingleNodeTest("Test_WhenAutoSubscriptionDisabled_ClientShouldNotReceiveAnyPublishedTracks") defer finish() - opts := testclient.Options{AutoSubscribe: false} - publisher := createRTCClient("publisher", defaultServerPort, &opts) - client := createRTCClient("client", defaultServerPort, &opts) - defer publisher.Stop() - defer client.Stop() - waitUntilConnected(t, publisher, client) + for _, testRTCServicePath := range testRTCServicePaths { + t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) { + opts := testclient.Options{AutoSubscribe: false} + publisher := createRTCClient("publisher", defaultServerPort, testRTCServicePath, &opts) + client := createRTCClient("client", defaultServerPort, testRTCServicePath, &opts) + defer publisher.Stop() + defer client.Stop() + waitUntilConnected(t, publisher, client) - track, err := publisher.AddStaticTrack("audio/opus", "audio", "webcam") - require.NoError(t, err) - defer track.Stop() + track, err := publisher.AddStaticTrack("audio/opus", "audio", "webcam") + require.NoError(t, err) + defer track.Stop() - time.Sleep(syncDelay) + time.Sleep(syncDelay) - require.Empty(t, client.SubscribedTracks()[publisher.ID()]) + require.Empty(t, client.SubscribedTracks()[publisher.ID()]) + }) + } } func Test_RenegotiationWithDifferentCodecs(t *testing.T) { @@ -258,71 +493,75 @@ func Test_RenegotiationWithDifferentCodecs(t *testing.T) { _, finish := setupSingleNodeTest("TestRenegotiationWithDifferentCodecs") defer finish() - c1 := createRTCClient("c1", defaultServerPort, nil) - c2 := createRTCClient("c2", defaultServerPort, nil) - waitUntilConnected(t, c1, c2) + for _, testRTCServicePath := range testRTCServicePaths { + t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) { + c1 := createRTCClient("c1", defaultServerPort, testRTCServicePath, nil) + c2 := createRTCClient("c2", defaultServerPort, testRTCServicePath, nil) + waitUntilConnected(t, c1, c2) - // publish a vp8 video track and ensure clients receive it ok - t1, err := c1.AddStaticTrack("audio/opus", "audio", "webcam") - require.NoError(t, err) - defer t1.Stop() - t2, err := c1.AddStaticTrack("video/vp8", "video", "webcam") - require.NoError(t, err) - defer t2.Stop() + // publish a vp8 video track and ensure clients receive it ok + t1, err := c1.AddStaticTrack("audio/opus", "audio", "webcam") + require.NoError(t, err) + defer t1.Stop() + t2, err := c1.AddStaticTrack("video/vp8", "video", "webcam") + require.NoError(t, err) + defer t2.Stop() - testutils.WithTimeout(t, func() string { - if len(c2.SubscribedTracks()) == 0 { - return "c2 was not subscribed to anything" - } - // should have received two tracks - if len(c2.SubscribedTracks()[c1.ID()]) != 2 { - return "c2 was not subscribed to tracks from c1" - } + testutils.WithTimeout(t, func() string { + if len(c2.SubscribedTracks()) == 0 { + return "c2 was not subscribed to anything" + } + // should have received two tracks + if len(c2.SubscribedTracks()[c1.ID()]) != 2 { + return "c2 was not subscribed to tracks from c1" + } - tracks := c2.SubscribedTracks()[c1.ID()] - for _, t := range tracks { - if mime.IsMimeTypeStringVP8(t.Codec().MimeType) { + tracks := c2.SubscribedTracks()[c1.ID()] + for _, t := range tracks { + if mime.IsMimeTypeStringVP8(t.Codec().MimeType) { + return "" + + } + } + return "did not receive track with vp8" + }) + + t3, err := c1.AddStaticTrackWithCodec(webrtc.RTPCodecCapability{ + MimeType: "video/h264", + ClockRate: 90000, + SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f", + }, "videoscreen", "screen") + defer t3.Stop() + require.NoError(t, err) + + testutils.WithTimeout(t, func() string { + if len(c2.SubscribedTracks()) == 0 { + return "c2's not subscribed to anything" + } + // should have received three tracks + if len(c2.SubscribedTracks()[c1.ID()]) != 3 { + return "c2's not subscribed to 3 tracks from c1" + } + + var vp8Found, h264Found bool + tracks := c2.SubscribedTracks()[c1.ID()] + for _, t := range tracks { + if mime.IsMimeTypeStringVP8(t.Codec().MimeType) { + vp8Found = true + } else if mime.IsMimeTypeStringH264(t.Codec().MimeType) { + h264Found = true + } + } + if !vp8Found { + return "did not receive track with vp8" + } + if !h264Found { + return "did not receive track with h264" + } return "" - - } - } - return "did not receive track with vp8" - }) - - t3, err := c1.AddStaticTrackWithCodec(webrtc.RTPCodecCapability{ - MimeType: "video/h264", - ClockRate: 90000, - SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f", - }, "videoscreen", "screen") - defer t3.Stop() - require.NoError(t, err) - - testutils.WithTimeout(t, func() string { - if len(c2.SubscribedTracks()) == 0 { - return "c2's not subscribed to anything" - } - // should have received three tracks - if len(c2.SubscribedTracks()[c1.ID()]) != 3 { - return "c2's not subscribed to 3 tracks from c1" - } - - var vp8Found, h264Found bool - tracks := c2.SubscribedTracks()[c1.ID()] - for _, t := range tracks { - if mime.IsMimeTypeStringVP8(t.Codec().MimeType) { - vp8Found = true - } else if mime.IsMimeTypeStringH264(t.Codec().MimeType) { - h264Found = true - } - } - if !vp8Found { - return "did not receive track with vp8" - } - if !h264Found { - return "did not receive track with h264" - } - return "" - }) + }) + }) + } } func TestSingleNodeRoomList(t *testing.T) { @@ -356,9 +595,7 @@ func TestSingleNodeUpdateParticipant(t *testing.T) { require.Error(t, err) var twErr twirp.Error require.True(t, errors.As(err, &twErr)) - // Note: for Cloud this would return 404, currently we are not able to differentiate between - // non-existent participant vs server being unavailable in OSS - require.Equal(t, twirp.Unavailable, twErr.Code()) + require.Equal(t, twirp.NotFound, twErr.Code()) }) } @@ -402,13 +639,17 @@ func TestPingPong(t *testing.T) { _, finish := setupSingleNodeTest("TestPingPong") defer finish() - c1 := createRTCClient("c1", defaultServerPort, nil) - waitUntilConnected(t, c1) + for _, testRTCServicePath := range testRTCServicePaths { + t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) { + c1 := createRTCClient("c1", defaultServerPort, testRTCServicePath, nil) + waitUntilConnected(t, c1) - require.NoError(t, c1.SendPing()) - require.Eventually(t, func() bool { - return c1.PongReceivedAt() > 0 - }, time.Second, 10*time.Millisecond) + require.NoError(t, c1.SendPing()) + require.Eventually(t, func() bool { + return c1.PongReceivedAt() > 0 + }, time.Second, 10*time.Millisecond) + }) + } } func TestSingleNodeJoinAfterClose(t *testing.T) { @@ -454,14 +695,28 @@ func TestAutoCreate(t *testing.T) { waitForServerToStart(s) - token := joinToken(testRoom, "start-before-create", nil) - _, err := testclient.NewWebSocketConn(fmt.Sprintf("ws://localhost:%d", defaultServerPort), token, nil) - require.Error(t, err) + for _, testRTCServicePath := range testRTCServicePaths { + t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) { + token := joinToken(testRoom, "start-before-create", nil) + opts := &testclient.Options{} + testRTCServicePathToTestClientOptions(testRTCServicePath, opts) + _, err := testclient.NewWebSocketConn( + fmt.Sprintf("ws://localhost:%d", defaultServerPort), + token, + opts, + ) + require.Error(t, err) - // second join should also fail - token = joinToken(testRoom, "start-before-create-2", nil) - _, err = testclient.NewWebSocketConn(fmt.Sprintf("ws://localhost:%d", defaultServerPort), token, nil) - require.Error(t, err) + // second join should also fail + token = joinToken(testRoom, "start-before-create-2", nil) + _, err = testclient.NewWebSocketConn( + fmt.Sprintf("ws://localhost:%d", defaultServerPort), + token, + opts, + ) + require.Error(t, err) + }) + } }) t.Run("join with explicit createRoom", func(t *testing.T) { @@ -479,10 +734,14 @@ func TestAutoCreate(t *testing.T) { _, err := roomClient.CreateRoom(contextWithToken(createRoomToken()), &livekit.CreateRoomRequest{Name: testRoom}) require.NoError(t, err) - c1 := createRTCClient("join-after-create", defaultServerPort, nil) - waitUntilConnected(t, c1) + for _, testRTCServicePath := range testRTCServicePaths { + t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) { + c1 := createRTCClient("join-after-create", defaultServerPort, testRTCServicePath, nil) + waitUntilConnected(t, c1) - c1.Stop() + c1.Stop() + }) + } }) } @@ -495,53 +754,114 @@ func TestSingleNodeUpdateSubscriptionPermissions(t *testing.T) { _, finish := setupSingleNodeTest("TestSingleNodeUpdateSubscriptionPermissions") defer finish() - pub := createRTCClient("pub", defaultServerPort, nil) - grant := &auth.VideoGrant{RoomJoin: true, Room: testRoom} - grant.SetCanSubscribe(false) - at := auth.NewAccessToken(testApiKey, testApiSecret). - AddGrant(grant). - SetIdentity("sub") - token, err := at.ToJWT() - require.NoError(t, err) - sub := createRTCClientWithToken(token, defaultServerPort, nil) + for _, testRTCServicePath := range testRTCServicePaths { + t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) { + pub := createRTCClient("pub", defaultServerPort, testRTCServicePath, nil) - waitUntilConnected(t, pub, sub) + grant := &auth.VideoGrant{RoomJoin: true, Room: testRoom} + grant.SetCanSubscribe(false) + at := auth.NewAccessToken(testApiKey, testApiSecret). + AddGrant(grant). + SetIdentity("sub") + token, err := at.ToJWT() + require.NoError(t, err) + sub := createRTCClientWithToken(token, defaultServerPort, testRTCServicePath, nil) - writers := publishTracksForClients(t, pub) - defer stopWriters(writers...) + waitUntilConnected(t, pub, sub) - // wait sub receives tracks - testutils.WithTimeout(t, func() string { - pubRemote := sub.GetRemoteParticipant(pub.ID()) - if pubRemote == nil { - return "could not find remote publisher" - } - if len(pubRemote.Tracks) != 2 { - return "did not receive metadata for published tracks" - } - return "" - }) + writers := publishTracksForClients(t, pub) + defer stopWriters(writers...) - // set permissions out of band - ctx := contextWithToken(adminRoomToken(testRoom)) - _, err = roomClient.UpdateParticipant(ctx, &livekit.UpdateParticipantRequest{ - Room: testRoom, - Identity: "sub", - Permission: &livekit.ParticipantPermission{ - CanSubscribe: true, - CanPublish: true, - }, - }) - require.NoError(t, err) + // wait sub receives tracks + testutils.WithTimeout(t, func() string { + pubRemote := sub.GetRemoteParticipant(pub.ID()) + if pubRemote == nil { + return "could not find remote publisher" + } + if len(pubRemote.Tracks) != 2 { + return "did not receive metadata for published tracks" + } + return "" + }) - testutils.WithTimeout(t, func() string { - tracks := sub.SubscribedTracks()[pub.ID()] - if len(tracks) == 2 { - return "" - } else { - return fmt.Sprintf("expected 2 tracks subscribed, actual: %d", len(tracks)) - } - }) + // set permissions out of band + ctx := contextWithToken(adminRoomToken(testRoom)) + _, err = roomClient.UpdateParticipant(ctx, &livekit.UpdateParticipantRequest{ + Room: testRoom, + Identity: "sub", + Permission: &livekit.ParticipantPermission{ + CanSubscribe: true, + CanPublish: true, + }, + }) + require.NoError(t, err) + + testutils.WithTimeout(t, func() string { + tracks := sub.SubscribedTracks()[pub.ID()] + if len(tracks) == 2 { + return "" + } else { + return fmt.Sprintf("expected 2 tracks subscribed, actual: %d", len(tracks)) + } + }) + }) + } +} + +func TestSingleNodeAttributes(t *testing.T) { + if testing.Short() { + t.SkipNow() + return + } + _, finish := setupSingleNodeTest("TestSingleNodeAttributes") + defer finish() + + for _, testRTCServicePath := range testRTCServicePaths { + t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) { + pub := createRTCClient("pub", defaultServerPort, testRTCServicePath, &testclient.Options{ + Attributes: map[string]string{ + "b": "2", + "c": "3", + }, + TokenCustomizer: func(token *auth.AccessToken, grants *auth.VideoGrant) { + T := true + grants.CanUpdateOwnMetadata = &T + token.SetAttributes(map[string]string{ + "a": "0", + "b": "1", + }) + }, + }) + + grant := &auth.VideoGrant{RoomJoin: true, Room: testRoom} + grant.SetCanSubscribe(false) + at := auth.NewAccessToken(testApiKey, testApiSecret). + SetVideoGrant(grant). + SetIdentity("sub") + token, err := at.ToJWT() + require.NoError(t, err) + sub := createRTCClientWithToken(token, defaultServerPort, testRTCServicePath, nil) + + waitUntilConnected(t, pub, sub) + + // wait sub receives initial attributes + testutils.WithTimeout(t, func() string { + pubRemote := sub.GetRemoteParticipant(pub.ID()) + if pubRemote == nil { + return "could not find remote publisher" + } + attrs := pubRemote.Attributes + if !reflect.DeepEqual(attrs, map[string]string{ + "a": "0", + "b": "2", + "c": "3", + }) { + return fmt.Sprintf("did not receive expected attributes: %v", attrs) + } + return "" + }) + }) + } } // TestDeviceCodecOverride checks that codecs that are incompatible with a device is not @@ -555,53 +875,63 @@ func TestDeviceCodecOverride(t *testing.T) { _, finish := setupSingleNodeTest("TestDeviceCodecOverride") defer finish() - // simulate device that isn't compatible with H.264 - c1 := createRTCClient("c1", defaultServerPort, &testclient.Options{ - ClientInfo: &livekit.ClientInfo{ - Os: "android", - DeviceModel: "Xiaomi 2201117TI", - }, - }) - defer c1.Stop() - waitUntilConnected(t, c1) + for _, testRTCServicePath := range testRTCServicePaths { + t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) { + // simulate device that isn't compatible with H.264 + c1 := createRTCClient("c1", defaultServerPort, testRTCServicePath, &testclient.Options{ + ClientInfo: &livekit.ClientInfo{ + Os: "android", + DeviceModel: "Xiaomi 2201117TI", + }, + }) + defer c1.Stop() + waitUntilConnected(t, c1) - // it doesn't really matter what the codec set here is, uses default Pion MediaEngine codecs - tw, err := c1.AddStaticTrack("video/h264", "video", "webcam") - require.NoError(t, err) - defer stopWriters(tw) + // it doesn't really matter what the codec set here is, uses default Pion MediaEngine codecs + tw, err := c1.AddStaticTrack("video/h264", "video", "webcam") + require.NoError(t, err) + defer stopWriters(tw) - // wait for server to receive track - require.Eventually(t, func() bool { - return c1.LastAnswer() != nil - }, waitTimeout, waitTick, "did not receive answer") + var desc *sdp.MediaDescription + require.Eventually(t, func() bool { + lastAnswer := c1.LastAnswer() + if lastAnswer == nil { + return false + } - sd := webrtc.SessionDescription{ - Type: webrtc.SDPTypeAnswer, - SDP: c1.LastAnswer().SDP, - } - answer, err := sd.Unmarshal() - require.NoError(t, err) + sd := webrtc.SessionDescription{ + Type: webrtc.SDPTypeAnswer, + SDP: lastAnswer.SDP, + } + answer, err := sd.Unmarshal() + require.NoError(t, err) - // video and data channel - require.Len(t, answer.MediaDescriptions, 2) - var desc *sdp.MediaDescription - for _, md := range answer.MediaDescriptions { - if md.MediaName.Media == "video" { - desc = md - break - } - } - require.NotNil(t, desc) - hasSeenVP8 := false - for _, a := range desc.Attributes { - if a.Key == "rtpmap" { - require.NotContains(t, a.Value, mime.MimeTypeCodecH264.String(), "should not contain H264 codec") - if strings.Contains(a.Value, mime.MimeTypeCodecVP8.String()) { - hasSeenVP8 = true + // video and data channel + if len(answer.MediaDescriptions) < 2 { + return false + } + + for _, md := range answer.MediaDescriptions { + if md.MediaName.Media == "video" { + desc = md + break + } + } + return desc != nil + }, waitTimeout, waitTick, "did not receive answer") + + hasSeenVP8 := false + for _, a := range desc.Attributes { + if a.Key == "rtpmap" { + require.NotContains(t, a.Value, mime.MimeTypeCodecH264.String(), "should not contain H264 codec") + if strings.Contains(a.Value, mime.MimeTypeCodecVP8.String()) { + hasSeenVP8 = true + } + } } - } + require.True(t, hasSeenVP8, "should have seen VP8 codec in SDP") + }) } - require.True(t, hasSeenVP8, "should have seen VP8 codec in SDP") } func TestSubscribeToCodecUnsupported(t *testing.T) { @@ -613,102 +943,106 @@ func TestSubscribeToCodecUnsupported(t *testing.T) { _, finish := setupSingleNodeTest("TestSubscribeToCodecUnsupported") defer finish() - c1 := createRTCClient("c1", defaultServerPort, nil) - // create a client that doesn't support H264 - c2 := createRTCClient("c2", defaultServerPort, &testclient.Options{ - AutoSubscribe: true, - DisabledCodecs: []webrtc.RTPCodecCapability{ - {MimeType: "video/H264"}, - }, - }) - waitUntilConnected(t, c1, c2) + for _, testRTCServicePath := range testRTCServicePaths { + t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) { + c1 := createRTCClient("c1", defaultServerPort, testRTCServicePath, nil) + // create a client that doesn't support H264 + c2 := createRTCClient("c2", defaultServerPort, testRTCServicePath, &testclient.Options{ + AutoSubscribe: true, + DisabledCodecs: []webrtc.RTPCodecCapability{ + {MimeType: "video/H264"}, + }, + }) + waitUntilConnected(t, c1, c2) - // publish a vp8 video track and ensure c2 receives it ok - t1, err := c1.AddStaticTrack("audio/opus", "audio", "webcam") - require.NoError(t, err) - defer t1.Stop() - t2, err := c1.AddStaticTrack("video/vp8", "video", "webcam") - require.NoError(t, err) - defer t2.Stop() + // publish a vp8 video track and ensure c2 receives it ok + t1, err := c1.AddStaticTrack("audio/opus", "audio", "webcam") + require.NoError(t, err) + defer t1.Stop() + t2, err := c1.AddStaticTrack("video/vp8", "video", "webcam") + require.NoError(t, err) + defer t2.Stop() - testutils.WithTimeout(t, func() string { - if len(c2.SubscribedTracks()) == 0 { - return "c2 was not subscribed to anything" - } - // should have received two tracks - if len(c2.SubscribedTracks()[c1.ID()]) != 2 { - return "c2 was not subscribed to tracks from c1" - } + testutils.WithTimeout(t, func() string { + if len(c2.SubscribedTracks()) == 0 { + return "c2 was not subscribed to anything" + } + // should have received two tracks + if len(c2.SubscribedTracks()[c1.ID()]) != 2 { + return "c2 was not subscribed to tracks from c1" + } - tracks := c2.SubscribedTracks()[c1.ID()] - for _, t := range tracks { - if mime.IsMimeTypeStringVP8(t.Codec().MimeType) { - return "" - } - } - return "did not receive track with vp8" - }) - require.Nil(t, c2.GetSubscriptionResponseAndClear()) + tracks := c2.SubscribedTracks()[c1.ID()] + for _, t := range tracks { + if mime.IsMimeTypeStringVP8(t.Codec().MimeType) { + return "" + } + } + return "did not receive track with vp8" + }) + require.Nil(t, c2.GetSubscriptionResponseAndClear()) - // publish a h264 track and ensure c2 got subscription error - t3, err := c1.AddStaticTrackWithCodec(webrtc.RTPCodecCapability{ - MimeType: "video/h264", - ClockRate: 90000, - SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f", - }, "videoscreen", "screen") - defer t3.Stop() - require.NoError(t, err) + // publish a h264 track and ensure c2 got subscription error + t3, err := c1.AddStaticTrackWithCodec(webrtc.RTPCodecCapability{ + MimeType: "video/h264", + ClockRate: 90000, + SDPFmtpLine: "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f", + }, "videoscreen", "screen") + defer t3.Stop() + require.NoError(t, err) - var h264TrackID string - require.Eventually(t, func() bool { - remoteC1 := c2.GetRemoteParticipant(c1.ID()) - require.NotNil(t, remoteC1) - for _, track := range remoteC1.Tracks { - if mime.IsMimeTypeStringH264(track.MimeType) { - h264TrackID = track.Sid + var h264TrackID string + require.Eventually(t, func() bool { + remoteC1 := c2.GetRemoteParticipant(c1.ID()) + require.NotNil(t, remoteC1) + for _, track := range remoteC1.Tracks { + if mime.IsMimeTypeStringH264(track.MimeType) { + h264TrackID = track.Sid + return true + } + } + return false + }, time.Second, 10*time.Millisecond, "did not receive track info with h264") + + require.Eventually(t, func() bool { + sr := c2.GetSubscriptionResponseAndClear() + if sr == nil { + return false + } + require.Equal(t, h264TrackID, sr.TrackSid) + require.Equal(t, livekit.SubscriptionError_SE_CODEC_UNSUPPORTED, sr.Err) return true - } - } - return false - }, time.Second, 10*time.Millisecond, "did not receive track info with h264") + }, 5*time.Second, 10*time.Millisecond, "did not receive subscription response") - require.Eventually(t, func() bool { - sr := c2.GetSubscriptionResponseAndClear() - if sr == nil { - return false - } - require.Equal(t, h264TrackID, sr.TrackSid) - require.Equal(t, livekit.SubscriptionError_SE_CODEC_UNSUPPORTED, sr.Err) - return true - }, 5*time.Second, 10*time.Millisecond, "did not receive subscription response") + // publish another vp8 track again, ensure the transport recovered by sfu and c2 can receive it + t4, err := c1.AddStaticTrack("video/vp8", "video2", "webcam2") + require.NoError(t, err) + defer t4.Stop() - // publish another vp8 track again, ensure the transport recovered by sfu and c2 can receive it - t4, err := c1.AddStaticTrack("video/vp8", "video2", "webcam2") - require.NoError(t, err) - defer t4.Stop() + testutils.WithTimeout(t, func() string { + if len(c2.SubscribedTracks()) == 0 { + return "c2 was not subscribed to anything" + } + // should have received two tracks + if len(c2.SubscribedTracks()[c1.ID()]) != 3 { + return "c2 was not subscribed to tracks from c1" + } - testutils.WithTimeout(t, func() string { - if len(c2.SubscribedTracks()) == 0 { - return "c2 was not subscribed to anything" - } - // should have received two tracks - if len(c2.SubscribedTracks()[c1.ID()]) != 3 { - return "c2 was not subscribed to tracks from c1" - } - - var vp8Count int - tracks := c2.SubscribedTracks()[c1.ID()] - for _, t := range tracks { - if mime.IsMimeTypeStringVP8(t.Codec().MimeType) { - vp8Count++ - } - } - if vp8Count == 2 { - return "" - } - return "did not 2 receive track with vp8" - }) - require.Nil(t, c2.GetSubscriptionResponseAndClear()) + var vp8Count int + tracks := c2.SubscribedTracks()[c1.ID()] + for _, t := range tracks { + if mime.IsMimeTypeStringVP8(t.Codec().MimeType) { + vp8Count++ + } + } + if vp8Count == 2 { + return "" + } + return "did not 2 receive track with vp8" + }) + require.Nil(t, c2.GetSubscriptionResponseAndClear()) + }) + } } func TestDataPublishSlowSubscriber(t *testing.T) { @@ -736,97 +1070,101 @@ func TestDataPublishSlowSubscriber(t *testing.T) { logger.Infow("----------------FINISHING TEST----------------", "test", t.Name()) }() - pub := createRTCClient("pub", defaultServerPort, nil) - fastSub := createRTCClient("fastSub", defaultServerPort, nil) - slowSubNotDrop := createRTCClient("slowSubNotDrop", defaultServerPort, nil) - slowSubDrop := createRTCClient("slowSubDrop", defaultServerPort, nil) - waitUntilConnected(t, pub, fastSub, slowSubDrop, slowSubNotDrop) - defer func() { - pub.Stop() - fastSub.Stop() - slowSubNotDrop.Stop() - slowSubDrop.Stop() - }() + for _, testRTCServicePath := range testRTCServicePaths { + t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) { + pub := createRTCClient("pub", defaultServerPort, testRTCServicePath, nil) + fastSub := createRTCClient("fastSub", defaultServerPort, testRTCServicePath, nil) + slowSubNotDrop := createRTCClient("slowSubNotDrop", defaultServerPort, testRTCServicePath, nil) + slowSubDrop := createRTCClient("slowSubDrop", defaultServerPort, testRTCServicePath, nil) + waitUntilConnected(t, pub, fastSub, slowSubDrop, slowSubNotDrop) + defer func() { + pub.Stop() + fastSub.Stop() + slowSubNotDrop.Stop() + slowSubDrop.Stop() + }() - // no data should be dropped for fast subscriber - var fastDataIndex atomic.Uint64 - fastSub.OnDataReceived = func(data []byte, sid string) { - idx := binary.BigEndian.Uint64(data[len(data)-8:]) - require.Equal(t, fastDataIndex.Load()+1, idx) - fastDataIndex.Store(idx) - } + // no data should be dropped for fast subscriber + var fastDataIndex atomic.Uint64 + fastSub.OnDataReceived = func(data []byte, sid string) { + idx := binary.BigEndian.Uint64(data[len(data)-8:]) + require.Equal(t, fastDataIndex.Load()+1, idx) + fastDataIndex.Store(idx) + } - // no data should be dropped for slow subscriber that is above threshold - var slowNoDropDataIndex atomic.Uint64 - var drainSlowSubNotDrop atomic.Bool - slowNoDropReader := testclient.NewDataChannelReader(dataChannelSlowThreshold * 2) - slowSubNotDrop.OnDataReceived = func(data []byte, sid string) { - idx := binary.BigEndian.Uint64(data[len(data)-8:]) - require.Equal(t, slowNoDropDataIndex.Load()+1, idx) - slowNoDropDataIndex.Store(idx) - if !drainSlowSubNotDrop.Load() { - slowNoDropReader.Read(data, sid) - } - } - - // data should be dropped for slow subscriber that is below threshold - var slowDropDataIndex atomic.Uint64 - dropped := make(chan struct{}) - slowDropReader := testclient.NewDataChannelReader(dataChannelSlowThreshold / 2) - slowSubDrop.OnDataReceived = func(data []byte, sid string) { - select { - case <-dropped: - return - default: - } - idx := binary.BigEndian.Uint64(data[len(data)-8:]) - if idx != slowDropDataIndex.Load()+1 { - close(dropped) - } - slowDropDataIndex.Store(idx) - slowDropReader.Read(data, sid) - } - - // publisher sends data as fast as possible, it will block by the slowest subscriber above the slow threshold - var ( - blocked atomic.Bool - stopWrite atomic.Bool - writeIdx atomic.Uint64 - ) - writeStopped := make(chan struct{}) - go func() { - defer close(writeStopped) - var i int - buf := make([]byte, 100) - for !stopWrite.Load() { - i++ - binary.BigEndian.PutUint64(buf[len(buf)-8:], uint64(i)) - if err := pub.PublishData(buf, livekit.DataPacket_RELIABLE); err != nil { - if errors.Is(err, datachannel.ErrDataDroppedBySlowReader) { - blocked.Store(true) - i-- - continue - } else { - t.Log("error writing", err) - break + // no data should be dropped for slow subscriber that is above threshold + var slowNoDropDataIndex atomic.Uint64 + var drainSlowSubNotDrop atomic.Bool + slowNoDropReader := testclient.NewDataChannelReader(dataChannelSlowThreshold * 2) + slowSubNotDrop.OnDataReceived = func(data []byte, sid string) { + idx := binary.BigEndian.Uint64(data[len(data)-8:]) + require.Equal(t, slowNoDropDataIndex.Load()+1, idx) + slowNoDropDataIndex.Store(idx) + if !drainSlowSubNotDrop.Load() { + slowNoDropReader.Read(data, sid) } } - writeIdx.Store(uint64(i)) - } - }() - <-dropped + // data should be dropped for slow subscriber that is below threshold + var slowDropDataIndex atomic.Uint64 + dropped := make(chan struct{}) + slowDropReader := testclient.NewDataChannelReader(dataChannelSlowThreshold / 2) + slowSubDrop.OnDataReceived = func(data []byte, sid string) { + select { + case <-dropped: + return + default: + } + idx := binary.BigEndian.Uint64(data[len(data)-8:]) + if idx != slowDropDataIndex.Load()+1 { + close(dropped) + } + slowDropDataIndex.Store(idx) + slowDropReader.Read(data, sid) + } - time.Sleep(time.Second) - blocked.Store(false) - require.Eventually(t, func() bool { return blocked.Load() }, 30*time.Second, 100*time.Millisecond) - stopWrite.Store(true) - <-writeStopped - drainSlowSubNotDrop.Store(true) - require.Eventually(t, func() bool { - return writeIdx.Load() == fastDataIndex.Load() && - writeIdx.Load() == slowNoDropDataIndex.Load() - }, 10*time.Second, 50*time.Millisecond, "writeIdx %d, fast %d, slowNoDrop %d", writeIdx.Load(), fastDataIndex.Load(), slowNoDropDataIndex.Load()) + // publisher sends data as fast as possible, it will block by the slowest subscriber above the slow threshold + var ( + blocked atomic.Bool + stopWrite atomic.Bool + writeIdx atomic.Uint64 + ) + writeStopped := make(chan struct{}) + go func() { + defer close(writeStopped) + var i int + buf := make([]byte, 100) + for !stopWrite.Load() { + i++ + binary.BigEndian.PutUint64(buf[len(buf)-8:], uint64(i)) + if err := pub.PublishData(buf, livekit.DataPacket_RELIABLE); err != nil { + if errors.Is(err, datachannel.ErrDataDroppedBySlowReader) { + blocked.Store(true) + i-- + continue + } else { + t.Log("error writing", err) + break + } + } + writeIdx.Store(uint64(i)) + } + }() + + <-dropped + + time.Sleep(time.Second) + blocked.Store(false) + require.Eventually(t, func() bool { return blocked.Load() }, 30*time.Second, 100*time.Millisecond) + stopWrite.Store(true) + <-writeStopped + drainSlowSubNotDrop.Store(true) + require.Eventually(t, func() bool { + return writeIdx.Load() == fastDataIndex.Load() && + writeIdx.Load() == slowNoDropDataIndex.Load() + }, 10*time.Second, 50*time.Millisecond, "writeIdx %d, fast %d, slowNoDrop %d", writeIdx.Load(), fastDataIndex.Load(), slowNoDropDataIndex.Load()) + }) + } } func TestFireTrackBySdp(t *testing.T) { @@ -841,8 +1179,8 @@ func TestFireTrackBySdp(t *testing.T) { { name: "js client could pub a/v tracks", codecs: []webrtc.RTPCodecCapability{ - {MimeType: "video/H264"}, - {MimeType: "audio/opus"}, + {MimeType: mime.MimeTypeH264.String()}, + {MimeType: mime.MimeTypeOpus.String()}, }, pubSDK: livekit.ClientInfo_JS, }, @@ -858,46 +1196,319 @@ func TestFireTrackBySdp(t *testing.T) { for _, c := range cases { codecs, sdk := c.codecs, c.pubSDK t.Run(c.name, func(t *testing.T) { - c1 := createRTCClient(c.name+"_c1", defaultServerPort, &testclient.Options{ - ClientInfo: &livekit.ClientInfo{ - Sdk: sdk, - }, - }) - c2 := createRTCClient(c.name+"_c2", defaultServerPort, &testclient.Options{ - AutoSubscribe: true, - ClientInfo: &livekit.ClientInfo{ - Sdk: livekit.ClientInfo_JS, - }, - }) - waitUntilConnected(t, c1, c2) - defer func() { - c1.Stop() - c2.Stop() - }() + for _, testRTCServicePath := range testRTCServicePaths { + t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) { + c1 := createRTCClient(c.name+"_c1", defaultServerPort, testRTCServicePath, &testclient.Options{ + ClientInfo: &livekit.ClientInfo{ + Sdk: sdk, + }, + }) + c2 := createRTCClient(c.name+"_c2", defaultServerPort, testRTCServicePath, &testclient.Options{ + AutoSubscribe: true, + ClientInfo: &livekit.ClientInfo{ + Sdk: livekit.ClientInfo_JS, + }, + }) + waitUntilConnected(t, c1, c2) + defer func() { + c1.Stop() + c2.Stop() + }() - // publish tracks and don't write any packets - for _, codec := range codecs { - _, err := c1.AddStaticTrackWithCodec(codec, codec.MimeType, codec.MimeType, testclient.AddTrackNoWriter()) - require.NoError(t, err) - } - - require.Eventually(t, func() bool { - return len(c2.SubscribedTracks()[c1.ID()]) == len(codecs) - }, 5*time.Second, 10*time.Millisecond) - - var found int - for _, pubTrack := range c1.GetPublishedTrackIDs() { - t.Log("pub track", pubTrack) - tracks := c2.SubscribedTracks()[c1.ID()] - for _, track := range tracks { - t.Log("sub track", track.ID(), track.Codec()) - if track.Codec().PayloadType == 0 && track.ID() == pubTrack { - found++ - break + // publish tracks and don't write any packets + for _, codec := range codecs { + _, err := c1.AddStaticTrackWithCodec(codec, codec.MimeType, codec.MimeType, testclient.AddTrackNoWriter()) + require.NoError(t, err) } - } + + require.Eventually(t, func() bool { + return len(c2.SubscribedTracks()[c1.ID()]) == len(codecs) + }, 5*time.Second, 10*time.Millisecond) + + var found int + for _, pubTrack := range c1.GetPublishedTrackIDs() { + t.Log("pub track", pubTrack) + tracks := c2.SubscribedTracks()[c1.ID()] + for _, track := range tracks { + t.Log("sub track", track.ID(), track.Codec()) + if track.Codec().PayloadType == 0 && track.ID() == pubTrack { + found++ + break + } + } + } + require.Equal(t, len(codecs), found) + }) } - require.Equal(t, len(codecs), found) + }) + } +} + +func TestSinglePublisherDataTrack(t *testing.T) { + if testing.Short() { + t.SkipNow() + return + } + + s, finish := setupSingleNodeTest("TestSinglePublisherDataTrack") + defer finish() + + for _, testRTCServicePath := range testRTCServicePaths { + t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) { + c1 := createRTCClient("c1", defaultServerPort, testRTCServicePath, &testclient.Options{AutoSubscribeDataTrack: true}) + c2 := createRTCClient("c2", defaultServerPort, testRTCServicePath, &testclient.Options{AutoSubscribeDataTrack: true}) + waitUntilConnected(t, c1, c2) + + // publish a couple of data tracks and ensure clients receive it ok + dt1, err := c1.PublishDataTrack() + require.NoError(t, err) + defer dt1.Stop() + + dt2, err := c1.PublishDataTrack() + require.NoError(t, err) + defer dt2.Stop() + + testutils.WithTimeout(t, func() string { + if len(c2.SubscribedDataTracks()) == 0 { + return "c2 was not subscribed to any data tracks" + } + // should have received two data tracks + if len(c2.SubscribedDataTracks()[c1.ID()]) != 2 { + return "c2 didn't subscribe to both data tracks from c1" + } + return "" + }) + + // a new client joins and should get the initial stream + c3 := createRTCClient("c3", defaultServerPort, testRTCServicePath, &testclient.Options{AutoSubscribeDataTrack: true}) + + // ensure that new client that has joined also received data tracks + waitUntilConnected(t, c3) + testutils.WithTimeout(t, func() string { + if len(c3.SubscribedDataTracks()) == 0 { + return "c3 didn't subscribe to any data tracks" + } + // should have received two data tracks + if len(c3.SubscribedDataTracks()[c1.ID()]) != 2 { + return "c3 didn't subscribe to tracks from c1" + } + return "" + }) + + // ensure that the data track ids are generated by server + tracks := c3.SubscribedDataTracks()[c1.ID()] + for _, tr := range tracks { + require.True(t, strings.HasPrefix(string(tr.ID()), "DTR_"), "data track should begin with DTR") + } + + // when c3 disconnects, ensure subscriber is cleaned up correctly + c3.Stop() + + testutils.WithTimeout(t, func() string { + room := s.RoomManager().GetRoom(context.Background(), testRoom) + p := room.GetParticipant("c1") + require.NotNil(t, p) + + for _, t := range p.GetPublishedDataTracks() { + if t.IsSubscriber(c3.ID()) { + return "c3 was not a subscriber of c1's data tracks" + } + } + return "" + }) + }) + } +} + +func TestTurnRelay(t *testing.T) { + if testing.Short() { + t.SkipNow() + return + } + + testCases := []struct { + name string + allowRestrictedPeerCIDRs []string + denyPeerCIDRs []string + expectedToConnect bool + }{ + { + "allow", + []string{"10.0.0.0/8", "192.168.0.0/16"}, + nil, + true, + }, + { + "not-allowed", + nil, + nil, + false, + }, + { + "denied-overrides-allowed", + []string{"10.0.0.0/8", "192.168.0.0/16"}, + []string{"10.0.0.0/8", "192.168.0.0/16"}, + false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + s := createSingleNodeServer(func(c *config.Config) { + c.TURN.Enabled = true + c.TURN.UDPPort = 3478 + c.TURN.AllowRestrictedPeerCIDRs = tc.allowRestrictedPeerCIDRs + c.TURN.DenyPeerCIDRs = tc.denyPeerCIDRs + }) + go func() { + if err := s.Start(); err != nil { + logger.Errorw("server returned error", err) + } + }() + defer s.Stop(true) + + waitForServerToStart(s) + + c1 := createRTCClient("relay_c1", defaultServerPort, testRTCServicePathv0, &testclient.Options{ + AutoSubscribe: true, + ForceRelay: true, + }) + defer c1.Stop() + + if tc.expectedToConnect { + waitUntilConnected(t, c1) + + testutils.WithTimeout(t, func() string { + if !c1.IsLocalCandidateRelaySelected() { + return "expected local candidate to be relay" + } + return "" + }) + } else { + ensureNotConnected(t, c1) + } + }) + } +} + +func TestTurnAuthFailure(t *testing.T) { + if testing.Short() { + t.SkipNow() + return + } + + const turnUDPPort = 3478 + + s := createSingleNodeServer(func(c *config.Config) { + c.TURN.Enabled = true + c.TURN.UDPPort = turnUDPPort + }) + go func() { + if err := s.Start(); err != nil { + logger.Errorw("server returned error", err) + } + }() + defer s.Stop(true) + + waitForServerToStart(s) + + // build a known-good username/password pair so individual cases can mutate + // only the part they are exercising. + pID := livekit.ParticipantID("PA_authfail") + authHandler := service.NewTURNAuthHandler(auth.NewSimpleKeyProvider(testApiKey, testApiSecret)) + validUsername, validExpiry := authHandler.CreateUsername(testApiKey, pID, 300) + validPassword, err := authHandler.CreatePassword(testApiKey, pID, validExpiry) + require.NoError(t, err) + + // username encoded with an already-expired timestamp. + expiredUsername, _ := authHandler.CreateUsername(testApiKey, pID, -10) + + // username encoded with an api key the server does not know about. + unknownAPIKeyUsername, _ := authHandler.CreateUsername("unknown-api-key", pID, 300) + + // password whose hash was generated for an expiry that doesn't match the + // one encoded in the username. The server reconstructs the password using + // the username's expiry, so the integrity check fails. + mismatchedExpiryPassword, err := authHandler.CreatePassword(testApiKey, pID, validExpiry+60) + require.NoError(t, err) + require.NotEqual(t, validPassword, mismatchedExpiryPassword) + + // username carrying expiry=0 must be rejected outright; constructed + // directly because CreateUsername always stamps a real expiry. + zeroExpiryUsername := base62.EncodeToString(fmt.Appendf(nil, "%s|%s|%d", testApiKey, pID, 0)) + + // username with only apiKey|pID (no expiry component) is the legacy + // pre-expiry form and must be rejected. + twoPartUsername := base62.EncodeToString(fmt.Appendf(nil, "%s|%s", testApiKey, pID)) + + testCases := []struct { + name string + username string + password string + }{ + { + name: "unparseable-username", + username: "not-base62!!!", + password: validPassword, + }, + { + name: "wrong-password", + username: validUsername, + password: "wrongpassword", + }, + { + name: "expired-username", + username: expiredUsername, + password: validPassword, + }, + { + name: "unknown-api-key", + username: unknownAPIKeyUsername, + password: validPassword, + }, + { + name: "password-expiry-mismatch", + username: validUsername, + password: mismatchedExpiryPassword, + }, + { + name: "zero-expiry-username", + username: zeroExpiryUsername, + password: validPassword, + }, + { + name: "two-part-username", + username: twoPartUsername, + password: validPassword, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + conn, err := net.ListenPacket("udp4", "0.0.0.0:0") + require.NoError(t, err) + defer conn.Close() + + client, err := turn.NewClient(&turn.ClientConfig{ + TURNServerAddr: fmt.Sprintf("127.0.0.1:%d", turnUDPPort), + Username: tc.username, + Password: tc.password, + Realm: service.LivekitRealm, + Conn: conn, + }) + require.NoError(t, err) + defer client.Close() + require.NoError(t, client.Listen()) + + _, allocErr := client.Allocate() + require.Error(t, allocErr) + + // pion's TURN server replies with 400 Bad Request for any + // authenticated-allocate failure (unknown user or integrity check + // mismatch); the initial unauthenticated probe is what returns 401. + var turnErr *stun.TurnError + require.ErrorAs(t, allocErr, &turnErr) + require.Equal(t, stun.CodeBadRequest, turnErr.ErrorCodeAttr.Code) }) } } diff --git a/test/webhook_test.go b/test/webhook_test.go index e0469a4..739c471 100644 --- a/test/webhook_test.go +++ b/test/webhook_test.go @@ -25,12 +25,12 @@ import ( "time" "github.com/stretchr/testify/require" - "google.golang.org/protobuf/encoding/protojson" "github.com/livekit/protocol/auth" "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" "github.com/livekit/protocol/utils/guid" + "github.com/livekit/protocol/utils/protojson" "github.com/livekit/protocol/webhook" "github.com/livekit/livekit-server/pkg/config" @@ -45,78 +45,82 @@ func TestWebhooks(t *testing.T) { require.NoError(t, err) defer finish() - c1 := createRTCClient("c1", defaultServerPort, nil) - waitUntilConnected(t, c1) - testutils.WithTimeout(t, func() string { - if ts.GetEvent(webhook.EventRoomStarted) == nil { - return "did not receive RoomStarted" - } - if ts.GetEvent(webhook.EventParticipantJoined) == nil { - return "did not receive ParticipantJoined" - } - return "" - }) + for _, testRTCServicePath := range testRTCServicePaths { + t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) { + c1 := createRTCClient("c1", defaultServerPort, testRTCServicePath, nil) + waitUntilConnected(t, c1) + testutils.WithTimeout(t, func() string { + if ts.GetEvent(webhook.EventRoomStarted) == nil { + return "did not receive RoomStarted" + } + if ts.GetEvent(webhook.EventParticipantJoined) == nil { + return "did not receive ParticipantJoined" + } + return "" + }) - // first participant join should have started the room - started := ts.GetEvent(webhook.EventRoomStarted) - require.Equal(t, testRoom, started.Room.Name) - require.NotEmpty(t, started.Id) - require.Greater(t, started.CreatedAt, time.Now().Unix()-100) - require.GreaterOrEqual(t, time.Now().Unix(), started.CreatedAt) - joined := ts.GetEvent(webhook.EventParticipantJoined) - require.Equal(t, "c1", joined.Participant.Identity) - ts.ClearEvents() + // first participant join should have started the room + started := ts.GetEvent(webhook.EventRoomStarted) + require.Equal(t, testRoom, started.Room.Name) + require.NotEmpty(t, started.Id) + require.Greater(t, started.CreatedAt, time.Now().Unix()-100) + require.GreaterOrEqual(t, time.Now().Unix(), started.CreatedAt) + joined := ts.GetEvent(webhook.EventParticipantJoined) + require.Equal(t, "c1", joined.Participant.Identity) + ts.ClearEvents() - // another participant joins - c2 := createRTCClient("c2", defaultServerPort, nil) - waitUntilConnected(t, c2) - defer c2.Stop() - testutils.WithTimeout(t, func() string { - if ts.GetEvent(webhook.EventParticipantJoined) == nil { - return "did not receive ParticipantJoined" - } - return "" - }) - joined = ts.GetEvent(webhook.EventParticipantJoined) - require.Equal(t, "c2", joined.Participant.Identity) - ts.ClearEvents() + // another participant joins + c2 := createRTCClient("c2", defaultServerPort, testRTCServicePath, nil) + waitUntilConnected(t, c2) + defer c2.Stop() + testutils.WithTimeout(t, func() string { + if ts.GetEvent(webhook.EventParticipantJoined) == nil { + return "did not receive ParticipantJoined" + } + return "" + }) + joined = ts.GetEvent(webhook.EventParticipantJoined) + require.Equal(t, "c2", joined.Participant.Identity) + ts.ClearEvents() - // track published - writers := publishTracksForClients(t, c1) - defer stopWriters(writers...) - testutils.WithTimeout(t, func() string { - ev := ts.GetEvent(webhook.EventTrackPublished) - if ev == nil { - return "did not receive TrackPublished" - } - require.NotNil(t, ev.Track, "TrackPublished did not include trackInfo") - require.Equal(t, string(c1.ID()), ev.Participant.Sid) - return "" - }) - ts.ClearEvents() + // track published + writers := publishTracksForClients(t, c1) + defer stopWriters(writers...) + testutils.WithTimeout(t, func() string { + ev := ts.GetEvent(webhook.EventTrackPublished) + if ev == nil { + return "did not receive TrackPublished" + } + require.NotNil(t, ev.Track, "TrackPublished did not include trackInfo") + require.Equal(t, string(c1.ID()), ev.Participant.Sid) + return "" + }) + ts.ClearEvents() - // first participant leaves - c1.Stop() - testutils.WithTimeout(t, func() string { - if ts.GetEvent(webhook.EventParticipantLeft) == nil { - return "did not receive ParticipantLeft" - } - return "" - }) - left := ts.GetEvent(webhook.EventParticipantLeft) - require.Equal(t, "c1", left.Participant.Identity) - ts.ClearEvents() + // first participant leaves + c1.Stop() + testutils.WithTimeout(t, func() string { + if ts.GetEvent(webhook.EventParticipantLeft) == nil { + return "did not receive ParticipantLeft" + } + return "" + }) + left := ts.GetEvent(webhook.EventParticipantLeft) + require.Equal(t, "c1", left.Participant.Identity) + ts.ClearEvents() - // room closed - rm := server.RoomManager().GetRoom(context.Background(), testRoom) - rm.Close(types.ParticipantCloseReasonNone) - testutils.WithTimeout(t, func() string { - if ts.GetEvent(webhook.EventRoomFinished) == nil { - return "did not receive RoomFinished" - } - return "" - }) - require.Equal(t, testRoom, ts.GetEvent(webhook.EventRoomFinished).Room.Name) + // room closed + rm := server.RoomManager().GetRoom(context.Background(), testRoom) + rm.Close(types.ParticipantCloseReasonNone) + testutils.WithTimeout(t, func() string { + if ts.GetEvent(webhook.EventRoomFinished) == nil { + return "did not receive RoomFinished" + } + return "" + }) + require.Equal(t, testRoom, ts.GetEvent(webhook.EventRoomFinished).Room.Name) + }) + } } func setupServerWithWebhook() (server *service.LivekitServer, testServer *webhookTestServer, finishFunc func(), err error) { diff --git a/version/version.go b/version/version.go index 7d45212..fbc9dc1 100644 --- a/version/version.go +++ b/version/version.go @@ -14,4 +14,4 @@ package version -const Version = "1.8.4" +const Version = "1.13.1"