Merge commit '0da97ebd2149ec2a0412a273b17270f540431d81' as 'livekit-server'

livekit-server/pkg/rtc/clientinfo.go

@@ -0,0 +1,138 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package rtc
+
+import (
+	"strconv"
+	"strings"
+
+	"github.com/livekit/protocol/livekit"
+)
+
+type ClientInfo struct {
+	*livekit.ClientInfo
+}
+
+func (c ClientInfo) isFirefox() bool {
+	return c.ClientInfo != nil && (strings.EqualFold(c.ClientInfo.Browser, "firefox") || strings.EqualFold(c.ClientInfo.Browser, "firefox mobile"))
+}
+
+func (c ClientInfo) isSafari() bool {
+	return c.ClientInfo != nil && strings.EqualFold(c.ClientInfo.Browser, "safari")
+}
+
+func (c ClientInfo) isGo() bool {
+	return c.ClientInfo != nil && c.ClientInfo.Sdk == livekit.ClientInfo_GO
+}
+
+func (c ClientInfo) isLinux() bool {
+	return c.ClientInfo != nil && strings.EqualFold(c.ClientInfo.Os, "linux")
+}
+
+func (c ClientInfo) isAndroid() bool {
+	return c.ClientInfo != nil && strings.EqualFold(c.ClientInfo.Os, "android")
+}
+
+func (c ClientInfo) SupportsAudioRED() bool {
+	return !c.isFirefox() && !c.isSafari()
+}
+
+func (c ClientInfo) SupportPrflxOverRelay() bool {
+	return !c.isFirefox()
+}
+
+// GoSDK(pion) relies on rtp packets to fire ontrack event, browsers and native (libwebrtc) rely on sdp
+func (c ClientInfo) FireTrackByRTPPacket() bool {
+	return c.isGo()
+}
+
+func (c ClientInfo) SupportsCodecChange() bool {
+	return c.ClientInfo != nil && c.ClientInfo.Sdk != livekit.ClientInfo_GO && c.ClientInfo.Sdk != livekit.ClientInfo_UNKNOWN
+}
+
+func (c ClientInfo) CanHandleReconnectResponse() bool {
+	if c.Sdk == livekit.ClientInfo_JS {
+		// JS handles Reconnect explicitly in 1.6.3, prior to 1.6.4 it could not handle unknown responses
+		if c.compareVersion("1.6.3") < 0 {
+			return false
+		}
+	}
+	return true
+}
+
+func (c ClientInfo) SupportsICETCP() bool {
+	if c.ClientInfo == nil {
+		return false
+	}
+	if c.ClientInfo.Sdk == livekit.ClientInfo_GO {
+		// Go does not support active TCP
+		return false
+	}
+	if c.ClientInfo.Sdk == livekit.ClientInfo_SWIFT {
+		// ICE/TCP added in 1.0.5
+		return c.compareVersion("1.0.5") >= 0
+	}
+	// most SDKs support ICE/TCP
+	return true
+}
+
+func (c ClientInfo) SupportsChangeRTPSenderEncodingActive() bool {
+	return !c.isFirefox()
+}
+
+func (c ClientInfo) ComplyWithCodecOrderInSDPAnswer() bool {
+	return !((c.isLinux() || c.isAndroid()) && c.isFirefox())
+}
+
+// Rust SDK can't decode unknown signal message (TrackSubscribed and ErrorResponse)
+func (c ClientInfo) SupportTrackSubscribedEvent() bool {
+	return !(c.ClientInfo.GetSdk() == livekit.ClientInfo_RUST && c.ClientInfo.GetProtocol() < 10)
+}
+
+func (c ClientInfo) SupportErrorResponse() bool {
+	return c.SupportTrackSubscribedEvent()
+}
+
+func (c ClientInfo) SupportSctpZeroChecksum() bool {
+	return !(c.ClientInfo.GetSdk() == livekit.ClientInfo_UNKNOWN ||
+		(c.isGo() && c.compareVersion("2.4.0") < 0))
+}
+
+// compareVersion compares a semver against the current client SDK version
+// returning 1 if current version is greater than version
+// 0 if they are the same, and -1 if it's an earlier version
+func (c ClientInfo) compareVersion(version string) int {
+	if c.ClientInfo == nil {
+		return -1
+	}
+	parts0 := strings.Split(c.ClientInfo.Version, ".")
+	parts1 := strings.Split(version, ".")
+	ints0 := make([]int, 3)
+	ints1 := make([]int, 3)
+	for i := 0; i < 3; i++ {
+		if len(parts0) > i {
+			ints0[i], _ = strconv.Atoi(parts0[i])
+		}
+		if len(parts1) > i {
+			ints1[i], _ = strconv.Atoi(parts1[i])
+		}
+		if ints0[i] > ints1[i] {
+			return 1
+		} else if ints0[i] < ints1[i] {
+			return -1
+		}
+	}
+	return 0
+}

livekit-server/pkg/rtc/clientinfo_test.go

@@ -0,0 +1,59 @@
+/*
+ * Copyright 2022 LiveKit, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package rtc
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/livekit/protocol/livekit"
+)
+
+func TestClientInfo_CompareVersion(t *testing.T) {
+	c := ClientInfo{
+		ClientInfo: &livekit.ClientInfo{
+			Version: "1",
+		},
+	}
+	require.Equal(t, 1, c.compareVersion("0.1.0"))
+	require.Equal(t, 0, c.compareVersion("1.0.0"))
+	require.Equal(t, -1, c.compareVersion("1.0.5"))
+}
+
+func TestClientInfo_SupportsICETCP(t *testing.T) {
+	t.Run("GO SDK cannot support TCP", func(t *testing.T) {
+		c := ClientInfo{
+			ClientInfo: &livekit.ClientInfo{
+				Sdk: livekit.ClientInfo_GO,
+			},
+		}
+		require.False(t, c.SupportsICETCP())
+	})
+
+	t.Run("Swift SDK cannot support TCP before 1.0.5", func(t *testing.T) {
+		c := ClientInfo{
+			ClientInfo: &livekit.ClientInfo{
+				Sdk:     livekit.ClientInfo_SWIFT,
+				Version: "1.0.4",
+			},
+		}
+		require.False(t, c.SupportsICETCP())
+		c.Version = "1.0.5"
+		require.True(t, c.SupportsICETCP())
+	})
+}

livekit-server/pkg/rtc/config.go

@@ -0,0 +1,166 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package rtc
+
+import (
+	"github.com/pion/sdp/v3"
+	"github.com/pion/webrtc/v4"
+
+	"github.com/livekit/livekit-server/pkg/config"
+	"github.com/livekit/livekit-server/pkg/sfu/buffer"
+	dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor"
+	"github.com/livekit/mediatransportutil/pkg/rtcconfig"
+)
+
+const (
+	frameMarking        = "urn:ietf:params:rtp-hdrext:framemarking"
+	repairedRTPStreamID = "urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id"
+)
+
+type WebRTCConfig struct {
+	rtcconfig.WebRTCConfig
+
+	BufferFactory *buffer.Factory
+	Receiver      ReceiverConfig
+	Publisher     DirectionConfig
+	Subscriber    DirectionConfig
+}
+
+type ReceiverConfig struct {
+	PacketBufferSizeVideo int
+	PacketBufferSizeAudio int
+}
+
+type RTPHeaderExtensionConfig struct {
+	Audio []string
+	Video []string
+}
+
+type RTCPFeedbackConfig struct {
+	Audio []webrtc.RTCPFeedback
+	Video []webrtc.RTCPFeedback
+}
+
+type DirectionConfig struct {
+	RTPHeaderExtension RTPHeaderExtensionConfig
+	RTCPFeedback       RTCPFeedbackConfig
+}
+
+func NewWebRTCConfig(conf *config.Config) (*WebRTCConfig, error) {
+	rtcConf := conf.RTC
+
+	webRTCConfig, err := rtcconfig.NewWebRTCConfig(&rtcConf.RTCConfig, conf.Development)
+	if err != nil {
+		return nil, err
+	}
+
+	// we don't want to use active TCP on a server, clients should be dialing
+	webRTCConfig.SettingEngine.DisableActiveTCP(true)
+
+	if rtcConf.PacketBufferSize == 0 {
+		rtcConf.PacketBufferSize = 500
+	}
+	if rtcConf.PacketBufferSizeVideo == 0 {
+		rtcConf.PacketBufferSizeVideo = rtcConf.PacketBufferSize
+	}
+	if rtcConf.PacketBufferSizeAudio == 0 {
+		rtcConf.PacketBufferSizeAudio = rtcConf.PacketBufferSize
+	}
+
+	// publisher configuration
+	publisherConfig := DirectionConfig{
+		RTPHeaderExtension: RTPHeaderExtensionConfig{
+			Audio: []string{
+				sdp.SDESMidURI,
+				sdp.SDESRTPStreamIDURI,
+				sdp.AudioLevelURI,
+				//act.AbsCaptureTimeURI,
+			},
+			Video: []string{
+				sdp.SDESMidURI,
+				sdp.SDESRTPStreamIDURI,
+				sdp.TransportCCURI,
+				frameMarking,
+				dd.ExtensionURI,
+				repairedRTPStreamID,
+				//act.AbsCaptureTimeURI,
+			},
+		},
+		RTCPFeedback: RTCPFeedbackConfig{
+			Audio: []webrtc.RTCPFeedback{
+				{Type: webrtc.TypeRTCPFBNACK},
+			},
+			Video: []webrtc.RTCPFeedback{
+				{Type: webrtc.TypeRTCPFBTransportCC},
+				{Type: webrtc.TypeRTCPFBCCM, Parameter: "fir"},
+				{Type: webrtc.TypeRTCPFBNACK},
+				{Type: webrtc.TypeRTCPFBNACK, Parameter: "pli"},
+			},
+		},
+	}
+
+	return &WebRTCConfig{
+		WebRTCConfig: *webRTCConfig,
+		Receiver: ReceiverConfig{
+			PacketBufferSizeVideo: rtcConf.PacketBufferSizeVideo,
+			PacketBufferSizeAudio: rtcConf.PacketBufferSizeAudio,
+		},
+		Publisher:  publisherConfig,
+		Subscriber: getSubscriberConfig(rtcConf.CongestionControl.UseSendSideBWEInterceptor || rtcConf.CongestionControl.UseSendSideBWE),
+	}, nil
+}
+
+func (c *WebRTCConfig) UpdateCongestionControl(ccConf config.CongestionControlConfig) {
+	c.Subscriber = getSubscriberConfig(ccConf.UseSendSideBWEInterceptor || ccConf.UseSendSideBWE)
+}
+
+func (c *WebRTCConfig) SetBufferFactory(factory *buffer.Factory) {
+	c.BufferFactory = factory
+	c.SettingEngine.BufferFactory = factory.GetOrNew
+}
+
+func getSubscriberConfig(enableTWCC bool) DirectionConfig {
+	subscriberConfig := DirectionConfig{
+		RTPHeaderExtension: RTPHeaderExtensionConfig{
+			Video: []string{
+				dd.ExtensionURI,
+				//act.AbsCaptureTimeURI,
+			},
+			Audio: []string{
+				//act.AbsCaptureTimeURI,
+			},
+		},
+		RTCPFeedback: RTCPFeedbackConfig{
+			Audio: []webrtc.RTCPFeedback{
+				// always enable NACK for audio but disable it later for red enabled transceiver. https://github.com/pion/webrtc/pull/2972
+				{Type: webrtc.TypeRTCPFBNACK},
+			},
+			Video: []webrtc.RTCPFeedback{
+				{Type: webrtc.TypeRTCPFBCCM, Parameter: "fir"},
+				{Type: webrtc.TypeRTCPFBNACK},
+				{Type: webrtc.TypeRTCPFBNACK, Parameter: "pli"},
+			},
+		},
+	}
+	if enableTWCC {
+		subscriberConfig.RTPHeaderExtension.Video = append(subscriberConfig.RTPHeaderExtension.Video, sdp.TransportCCURI)
+		subscriberConfig.RTCPFeedback.Video = append(subscriberConfig.RTCPFeedback.Video, webrtc.RTCPFeedback{Type: webrtc.TypeRTCPFBTransportCC})
+	} else {
+		subscriberConfig.RTPHeaderExtension.Video = append(subscriberConfig.RTPHeaderExtension.Video, sdp.ABSSendTimeURI)
+		subscriberConfig.RTCPFeedback.Video = append(subscriberConfig.RTCPFeedback.Video, webrtc.RTCPFeedback{Type: webrtc.TypeRTCPFBGoogREMB})
+	}
+
+	return subscriberConfig
+}

livekit-server/pkg/rtc/dynacast/dynacastmanager.go

@@ -0,0 +1,349 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dynacast
+
+import (
+	"sync"
+	"time"
+
+	"github.com/bep/debounce"
+	"golang.org/x/exp/maps"
+
+	"github.com/livekit/protocol/livekit"
+	"github.com/livekit/protocol/logger"
+
+	"github.com/livekit/livekit-server/pkg/rtc/types"
+	"github.com/livekit/livekit-server/pkg/sfu/mime"
+	"github.com/livekit/livekit-server/pkg/utils"
+)
+
+type DynacastManagerParams struct {
+	DynacastPauseDelay time.Duration
+	Logger             logger.Logger
+}
+
+type DynacastManager struct {
+	params DynacastManagerParams
+
+	lock                          sync.RWMutex
+	regressedCodec                map[mime.MimeType]struct{}
+	dynacastQuality               map[mime.MimeType]*DynacastQuality
+	maxSubscribedQuality          map[mime.MimeType]livekit.VideoQuality
+	committedMaxSubscribedQuality map[mime.MimeType]livekit.VideoQuality
+
+	maxSubscribedQualityDebounce        func(func())
+	maxSubscribedQualityDebouncePending bool
+
+	qualityNotifyOpQueue *utils.OpsQueue
+
+	isClosed bool
+
+	onSubscribedMaxQualityChange func(subscribedQualities []*livekit.SubscribedCodec, maxSubscribedQualities []types.SubscribedCodecQuality)
+}
+
+func NewDynacastManager(params DynacastManagerParams) *DynacastManager {
+	if params.Logger == nil {
+		params.Logger = logger.GetLogger()
+	}
+	d := &DynacastManager{
+		params:                        params,
+		regressedCodec:                make(map[mime.MimeType]struct{}),
+		dynacastQuality:               make(map[mime.MimeType]*DynacastQuality),
+		maxSubscribedQuality:          make(map[mime.MimeType]livekit.VideoQuality),
+		committedMaxSubscribedQuality: make(map[mime.MimeType]livekit.VideoQuality),
+		qualityNotifyOpQueue: utils.NewOpsQueue(utils.OpsQueueParams{
+			Name:        "quality-notify",
+			MinSize:     64,
+			FlushOnStop: true,
+			Logger:      params.Logger,
+		}),
+	}
+	if params.DynacastPauseDelay > 0 {
+		d.maxSubscribedQualityDebounce = debounce.New(params.DynacastPauseDelay)
+	}
+	d.qualityNotifyOpQueue.Start()
+	return d
+}
+
+func (d *DynacastManager) OnSubscribedMaxQualityChange(f func(subscribedQualities []*livekit.SubscribedCodec, maxSubscribedQualities []types.SubscribedCodecQuality)) {
+	d.lock.Lock()
+	d.onSubscribedMaxQualityChange = f
+	d.lock.Unlock()
+}
+
+func (d *DynacastManager) AddCodec(mime mime.MimeType) {
+	d.getOrCreateDynacastQuality(mime)
+}
+
+func (d *DynacastManager) HandleCodecRegression(fromMime, toMime mime.MimeType) {
+	fromDq := d.getOrCreateDynacastQuality(fromMime)
+
+	d.lock.Lock()
+	if d.isClosed {
+		d.lock.Unlock()
+		return
+	}
+
+	if fromDq == nil {
+		// should not happen as we have added the codec on setup receiver
+		d.params.Logger.Warnw("regression from codec not found", nil, "mime", fromMime)
+		d.lock.Unlock()
+		return
+	}
+	d.regressedCodec[fromMime] = struct{}{}
+	d.maxSubscribedQuality[fromMime] = livekit.VideoQuality_OFF
+
+	// if the new codec is not added, notify the publisher to start publishing
+	if _, ok := d.maxSubscribedQuality[toMime]; !ok {
+		d.maxSubscribedQuality[toMime] = livekit.VideoQuality_HIGH
+	}
+
+	d.lock.Unlock()
+	d.update(false)
+
+	fromDq.Stop()
+	fromDq.RegressTo(d.getOrCreateDynacastQuality(toMime))
+}
+
+func (d *DynacastManager) Restart() {
+	d.lock.Lock()
+	d.committedMaxSubscribedQuality = make(map[mime.MimeType]livekit.VideoQuality)
+
+	dqs := d.getDynacastQualitiesLocked()
+	d.lock.Unlock()
+
+	for _, dq := range dqs {
+		dq.Restart()
+	}
+}
+
+func (d *DynacastManager) Close() {
+	d.qualityNotifyOpQueue.Stop()
+
+	d.lock.Lock()
+	dqs := d.getDynacastQualitiesLocked()
+	d.dynacastQuality = make(map[mime.MimeType]*DynacastQuality)
+
+	d.isClosed = true
+	d.lock.Unlock()
+
+	for _, dq := range dqs {
+		dq.Stop()
+	}
+}
+
+// THere are situations like track unmute or streaming from a different node
+// where subscribed quality needs to sent to the provider immediately.
+// This bypasses any debouncing and forces a subscribed quality update
+// with immediate effect.
+func (d *DynacastManager) ForceUpdate() {
+	d.update(true)
+}
+
+// It is possible for tracks to be in pending close state. When track
+// is waiting to be closed, a node is not streaming a track. This can
+// be used to force an update announcing that subscribed quality is OFF,
+// i.e. indicating not pulling track any more.
+func (d *DynacastManager) ForceQuality(quality livekit.VideoQuality) {
+	d.lock.Lock()
+	defer d.lock.Unlock()
+
+	for mime := range d.committedMaxSubscribedQuality {
+		d.committedMaxSubscribedQuality[mime] = quality
+	}
+
+	d.enqueueSubscribedQualityChange()
+}
+
+func (d *DynacastManager) NotifySubscriberMaxQuality(subscriberID livekit.ParticipantID, mime mime.MimeType, quality livekit.VideoQuality) {
+	dq := d.getOrCreateDynacastQuality(mime)
+	if dq != nil {
+		dq.NotifySubscriberMaxQuality(subscriberID, quality)
+	}
+}
+
+func (d *DynacastManager) NotifySubscriberNodeMaxQuality(nodeID livekit.NodeID, qualities []types.SubscribedCodecQuality) {
+	for _, quality := range qualities {
+		dq := d.getOrCreateDynacastQuality(quality.CodecMime)
+		if dq != nil {
+			dq.NotifySubscriberNodeMaxQuality(nodeID, quality.Quality)
+		}
+	}
+}
+
+func (d *DynacastManager) getOrCreateDynacastQuality(mimeType mime.MimeType) *DynacastQuality {
+	d.lock.Lock()
+	defer d.lock.Unlock()
+
+	if d.isClosed || mimeType == mime.MimeTypeUnknown {
+		return nil
+	}
+
+	if dq := d.dynacastQuality[mimeType]; dq != nil {
+		return dq
+	}
+
+	dq := NewDynacastQuality(DynacastQualityParams{
+		MimeType: mimeType,
+		Logger:   d.params.Logger,
+	})
+	dq.OnSubscribedMaxQualityChange(d.updateMaxQualityForMime)
+	dq.Start()
+
+	d.dynacastQuality[mimeType] = dq
+	return dq
+}
+
+func (d *DynacastManager) getDynacastQualitiesLocked() []*DynacastQuality {
+	return maps.Values(d.dynacastQuality)
+}
+
+func (d *DynacastManager) updateMaxQualityForMime(mime mime.MimeType, maxQuality livekit.VideoQuality) {
+	d.lock.Lock()
+	if _, ok := d.regressedCodec[mime]; !ok {
+		d.maxSubscribedQuality[mime] = maxQuality
+	}
+	d.lock.Unlock()
+
+	d.update(false)
+}
+
+func (d *DynacastManager) update(force bool) {
+	d.lock.Lock()
+
+	d.params.Logger.Debugw("processing quality change",
+		"force", force,
+		"committedMaxSubscribedQuality", d.committedMaxSubscribedQuality,
+		"maxSubscribedQuality", d.maxSubscribedQuality,
+	)
+
+	if len(d.maxSubscribedQuality) == 0 {
+		// no mime has been added, nothing to update
+		d.lock.Unlock()
+		return
+	}
+
+	// add or remove of a mime triggers an update
+	changed := len(d.maxSubscribedQuality) != len(d.committedMaxSubscribedQuality)
+	downgradesOnly := !changed
+	if !changed {
+		for mime, quality := range d.maxSubscribedQuality {
+			if cq, ok := d.committedMaxSubscribedQuality[mime]; ok {
+				if cq != quality {
+					changed = true
+				}
+
+				if (cq == livekit.VideoQuality_OFF && quality != livekit.VideoQuality_OFF) || (cq != livekit.VideoQuality_OFF && quality != livekit.VideoQuality_OFF && cq < quality) {
+					downgradesOnly = false
+				}
+			}
+		}
+	}
+
+	if !force {
+		if !changed {
+			d.lock.Unlock()
+			return
+		}
+
+		if downgradesOnly && d.maxSubscribedQualityDebounce != nil {
+			if !d.maxSubscribedQualityDebouncePending {
+				d.params.Logger.Debugw("debouncing quality downgrade",
+					"committedMaxSubscribedQuality", d.committedMaxSubscribedQuality,
+					"maxSubscribedQuality", d.maxSubscribedQuality,
+				)
+				d.maxSubscribedQualityDebounce(func() {
+					d.update(true)
+				})
+				d.maxSubscribedQualityDebouncePending = true
+			} else {
+				d.params.Logger.Debugw("quality downgrade waiting for debounce",
+					"committedMaxSubscribedQuality", d.committedMaxSubscribedQuality,
+					"maxSubscribedQuality", d.maxSubscribedQuality,
+				)
+			}
+			d.lock.Unlock()
+			return
+		}
+	}
+
+	// clear debounce on send
+	if d.maxSubscribedQualityDebounce != nil {
+		d.maxSubscribedQualityDebounce(func() {})
+		d.maxSubscribedQualityDebouncePending = false
+	}
+
+	d.params.Logger.Debugw("committing quality change",
+		"force", force,
+		"committedMaxSubscribedQuality", d.committedMaxSubscribedQuality,
+		"maxSubscribedQuality", d.maxSubscribedQuality,
+	)
+
+	// commit change
+	d.committedMaxSubscribedQuality = make(map[mime.MimeType]livekit.VideoQuality, len(d.maxSubscribedQuality))
+	for mime, quality := range d.maxSubscribedQuality {
+		d.committedMaxSubscribedQuality[mime] = quality
+	}
+
+	d.enqueueSubscribedQualityChange()
+	d.lock.Unlock()
+}
+
+func (d *DynacastManager) enqueueSubscribedQualityChange() {
+	if d.isClosed || d.onSubscribedMaxQualityChange == nil {
+		return
+	}
+
+	subscribedCodecs := make([]*livekit.SubscribedCodec, 0, len(d.committedMaxSubscribedQuality))
+	maxSubscribedQualities := make([]types.SubscribedCodecQuality, 0, len(d.committedMaxSubscribedQuality))
+	for mime, quality := range d.committedMaxSubscribedQuality {
+		maxSubscribedQualities = append(maxSubscribedQualities, types.SubscribedCodecQuality{
+			CodecMime: mime,
+			Quality:   quality,
+		})
+
+		if quality == livekit.VideoQuality_OFF {
+			subscribedCodecs = append(subscribedCodecs, &livekit.SubscribedCodec{
+				Codec: mime.String(),
+				Qualities: []*livekit.SubscribedQuality{
+					{Quality: livekit.VideoQuality_LOW, Enabled: false},
+					{Quality: livekit.VideoQuality_MEDIUM, Enabled: false},
+					{Quality: livekit.VideoQuality_HIGH, Enabled: false},
+				},
+			})
+		} else {
+			var subscribedQualities []*livekit.SubscribedQuality
+			for q := livekit.VideoQuality_LOW; q <= livekit.VideoQuality_HIGH; q++ {
+				subscribedQualities = append(subscribedQualities, &livekit.SubscribedQuality{
+					Quality: q,
+					Enabled: q <= quality,
+				})
+			}
+			subscribedCodecs = append(subscribedCodecs, &livekit.SubscribedCodec{
+				Codec:     mime.String(),
+				Qualities: subscribedQualities,
+			})
+		}
+	}
+
+	d.params.Logger.Debugw(
+		"subscribedMaxQualityChange",
+		"subscribedCodecs", subscribedCodecs,
+		"maxSubscribedQualities", maxSubscribedQualities,
+	)
+	d.qualityNotifyOpQueue.Enqueue(func() {
+		d.onSubscribedMaxQualityChange(subscribedCodecs, maxSubscribedQualities)
+	})
+}

livekit-server/pkg/rtc/dynacast/dynacastmanager_test.go

@@ -0,0 +1,376 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dynacast
+
+import (
+	"sort"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/livekit/livekit-server/pkg/rtc/types"
+	"github.com/livekit/livekit-server/pkg/sfu/mime"
+	"github.com/livekit/protocol/livekit"
+)
+
+func TestSubscribedMaxQuality(t *testing.T) {
+
+	t.Run("subscribers muted", func(t *testing.T) {
+		dm := NewDynacastManager(DynacastManagerParams{})
+		var lock sync.Mutex
+		actualSubscribedQualities := make([]*livekit.SubscribedCodec, 0)
+		dm.OnSubscribedMaxQualityChange(func(subscribedQualities []*livekit.SubscribedCodec, _maxSubscribedQualities []types.SubscribedCodecQuality) {
+			lock.Lock()
+			actualSubscribedQualities = subscribedQualities
+			lock.Unlock()
+		})
+
+		dm.NotifySubscriberMaxQuality("s1", mime.MimeTypeVP8, livekit.VideoQuality_HIGH)
+		dm.NotifySubscriberMaxQuality("s2", mime.MimeTypeAV1, livekit.VideoQuality_HIGH)
+
+		// mute all subscribers of vp8
+		dm.NotifySubscriberMaxQuality("s1", mime.MimeTypeVP8, livekit.VideoQuality_OFF)
+
+		expectedSubscribedQualities := []*livekit.SubscribedCodec{
+			{
+				Codec: mime.MimeTypeVP8.String(),
+				Qualities: []*livekit.SubscribedQuality{
+					{Quality: livekit.VideoQuality_LOW, Enabled: false},
+					{Quality: livekit.VideoQuality_MEDIUM, Enabled: false},
+					{Quality: livekit.VideoQuality_HIGH, Enabled: false},
+				},
+			},
+			{
+				Codec: mime.MimeTypeAV1.String(),
+				Qualities: []*livekit.SubscribedQuality{
+					{Quality: livekit.VideoQuality_LOW, Enabled: true},
+					{Quality: livekit.VideoQuality_MEDIUM, Enabled: true},
+					{Quality: livekit.VideoQuality_HIGH, Enabled: true},
+				},
+			},
+		}
+		require.Eventually(t, func() bool {
+			lock.Lock()
+			defer lock.Unlock()
+
+			return subscribedCodecsAsString(expectedSubscribedQualities) == subscribedCodecsAsString(actualSubscribedQualities)
+		}, 10*time.Second, 100*time.Millisecond)
+	})
+
+	t.Run("subscribers max quality", func(t *testing.T) {
+		dm := NewDynacastManager(DynacastManagerParams{
+			DynacastPauseDelay: 100 * time.Millisecond,
+		})
+
+		lock := sync.RWMutex{}
+		lock.Lock()
+		actualSubscribedQualities := make([]*livekit.SubscribedCodec, 0)
+		lock.Unlock()
+		dm.OnSubscribedMaxQualityChange(func(subscribedQualities []*livekit.SubscribedCodec, _maxSubscribedQualities []types.SubscribedCodecQuality) {
+			lock.Lock()
+			actualSubscribedQualities = subscribedQualities
+			lock.Unlock()
+		})
+
+		dm.maxSubscribedQuality = map[mime.MimeType]livekit.VideoQuality{
+			mime.MimeTypeVP8: livekit.VideoQuality_LOW,
+			mime.MimeTypeAV1: livekit.VideoQuality_LOW,
+		}
+		dm.NotifySubscriberMaxQuality("s1", mime.MimeTypeVP8, livekit.VideoQuality_HIGH)
+		dm.NotifySubscriberMaxQuality("s2", mime.MimeTypeVP8, livekit.VideoQuality_MEDIUM)
+		dm.NotifySubscriberMaxQuality("s3", mime.MimeTypeAV1, livekit.VideoQuality_MEDIUM)
+
+		expectedSubscribedQualities := []*livekit.SubscribedCodec{
+			{
+				Codec: mime.MimeTypeVP8.String(),
+				Qualities: []*livekit.SubscribedQuality{
+					{Quality: livekit.VideoQuality_LOW, Enabled: true},
+					{Quality: livekit.VideoQuality_MEDIUM, Enabled: true},
+					{Quality: livekit.VideoQuality_HIGH, Enabled: true},
+				},
+			},
+			{
+				Codec: mime.MimeTypeAV1.String(),
+				Qualities: []*livekit.SubscribedQuality{
+					{Quality: livekit.VideoQuality_LOW, Enabled: true},
+					{Quality: livekit.VideoQuality_MEDIUM, Enabled: true},
+					{Quality: livekit.VideoQuality_HIGH, Enabled: false},
+				},
+			},
+		}
+		require.Eventually(t, func() bool {
+			lock.Lock()
+			defer lock.Unlock()
+
+			return subscribedCodecsAsString(expectedSubscribedQualities) == subscribedCodecsAsString(actualSubscribedQualities)
+		}, 10*time.Second, 100*time.Millisecond)
+
+		// "s1" dropping to MEDIUM should disable HIGH layer
+		dm.NotifySubscriberMaxQuality("s1", mime.MimeTypeVP8, livekit.VideoQuality_MEDIUM)
+
+		expectedSubscribedQualities = []*livekit.SubscribedCodec{
+			{
+				Codec: mime.MimeTypeVP8.String(),
+				Qualities: []*livekit.SubscribedQuality{
+					{Quality: livekit.VideoQuality_LOW, Enabled: true},
+					{Quality: livekit.VideoQuality_MEDIUM, Enabled: true},
+					{Quality: livekit.VideoQuality_HIGH, Enabled: false},
+				},
+			},
+			{
+				Codec: mime.MimeTypeAV1.String(),
+				Qualities: []*livekit.SubscribedQuality{
+					{Quality: livekit.VideoQuality_LOW, Enabled: true},
+					{Quality: livekit.VideoQuality_MEDIUM, Enabled: true},
+					{Quality: livekit.VideoQuality_HIGH, Enabled: false},
+				},
+			},
+		}
+		require.Eventually(t, func() bool {
+			lock.Lock()
+			defer lock.Unlock()
+
+			return subscribedCodecsAsString(expectedSubscribedQualities) == subscribedCodecsAsString(actualSubscribedQualities)
+		}, 10*time.Second, 100*time.Millisecond)
+
+		// "s1" , "s2" , "s3" dropping to LOW should disable HIGH & MEDIUM
+		dm.NotifySubscriberMaxQuality("s1", mime.MimeTypeVP8, livekit.VideoQuality_LOW)
+		dm.NotifySubscriberMaxQuality("s2", mime.MimeTypeVP8, livekit.VideoQuality_LOW)
+		dm.NotifySubscriberMaxQuality("s3", mime.MimeTypeAV1, livekit.VideoQuality_LOW)
+
+		expectedSubscribedQualities = []*livekit.SubscribedCodec{
+			{
+				Codec: mime.MimeTypeVP8.String(),
+				Qualities: []*livekit.SubscribedQuality{
+					{Quality: livekit.VideoQuality_LOW, Enabled: true},
+					{Quality: livekit.VideoQuality_MEDIUM, Enabled: false},
+					{Quality: livekit.VideoQuality_HIGH, Enabled: false},
+				},
+			},
+			{
+				Codec: mime.MimeTypeAV1.String(),
+				Qualities: []*livekit.SubscribedQuality{
+					{Quality: livekit.VideoQuality_LOW, Enabled: true},
+					{Quality: livekit.VideoQuality_MEDIUM, Enabled: false},
+					{Quality: livekit.VideoQuality_HIGH, Enabled: false},
+				},
+			},
+		}
+		require.Eventually(t, func() bool {
+			lock.Lock()
+			defer lock.Unlock()
+
+			return subscribedCodecsAsString(expectedSubscribedQualities) == subscribedCodecsAsString(actualSubscribedQualities)
+		}, 10*time.Second, 100*time.Millisecond)
+
+		// muting "s2" only should not disable all qualities of vp8, no change of expected qualities
+		dm.NotifySubscriberMaxQuality("s2", mime.MimeTypeVP8, livekit.VideoQuality_OFF)
+
+		time.Sleep(100 * time.Millisecond)
+		require.Eventually(t, func() bool {
+			lock.Lock()
+			defer lock.Unlock()
+
+			return subscribedCodecsAsString(expectedSubscribedQualities) == subscribedCodecsAsString(actualSubscribedQualities)
+		}, 10*time.Second, 100*time.Millisecond)
+
+		// muting "s1" and s3 also should disable all qualities
+		dm.NotifySubscriberMaxQuality("s1", mime.MimeTypeVP8, livekit.VideoQuality_OFF)
+		dm.NotifySubscriberMaxQuality("s3", mime.MimeTypeAV1, livekit.VideoQuality_OFF)
+
+		expectedSubscribedQualities = []*livekit.SubscribedCodec{
+			{
+				Codec: mime.MimeTypeVP8.String(),
+				Qualities: []*livekit.SubscribedQuality{
+					{Quality: livekit.VideoQuality_LOW, Enabled: false},
+					{Quality: livekit.VideoQuality_MEDIUM, Enabled: false},
+					{Quality: livekit.VideoQuality_HIGH, Enabled: false},
+				},
+			},
+			{
+				Codec: mime.MimeTypeAV1.String(),
+				Qualities: []*livekit.SubscribedQuality{
+					{Quality: livekit.VideoQuality_LOW, Enabled: false},
+					{Quality: livekit.VideoQuality_MEDIUM, Enabled: false},
+					{Quality: livekit.VideoQuality_HIGH, Enabled: false},
+				},
+			},
+		}
+		require.Eventually(t, func() bool {
+			lock.Lock()
+			defer lock.Unlock()
+
+			return subscribedCodecsAsString(expectedSubscribedQualities) == subscribedCodecsAsString(actualSubscribedQualities)
+		}, 10*time.Second, 100*time.Millisecond)
+
+		// unmuting "s1" should enable vp8 previously set max quality
+		dm.NotifySubscriberMaxQuality("s1", mime.MimeTypeVP8, livekit.VideoQuality_LOW)
+
+		expectedSubscribedQualities = []*livekit.SubscribedCodec{
+			{
+				Codec: mime.MimeTypeVP8.String(),
+				Qualities: []*livekit.SubscribedQuality{
+					{Quality: livekit.VideoQuality_LOW, Enabled: true},
+					{Quality: livekit.VideoQuality_MEDIUM, Enabled: false},
+					{Quality: livekit.VideoQuality_HIGH, Enabled: false},
+				},
+			},
+			{
+				Codec: mime.MimeTypeAV1.String(),
+				Qualities: []*livekit.SubscribedQuality{
+					{Quality: livekit.VideoQuality_LOW, Enabled: false},
+					{Quality: livekit.VideoQuality_MEDIUM, Enabled: false},
+					{Quality: livekit.VideoQuality_HIGH, Enabled: false},
+				},
+			},
+		}
+		require.Eventually(t, func() bool {
+			lock.Lock()
+			defer lock.Unlock()
+
+			return subscribedCodecsAsString(expectedSubscribedQualities) == subscribedCodecsAsString(actualSubscribedQualities)
+		}, 10*time.Second, 100*time.Millisecond)
+
+		// a higher quality from a different node should trigger that quality
+		dm.NotifySubscriberNodeMaxQuality("n1", []types.SubscribedCodecQuality{
+			{CodecMime: mime.MimeTypeVP8, Quality: livekit.VideoQuality_HIGH},
+			{CodecMime: mime.MimeTypeAV1, Quality: livekit.VideoQuality_MEDIUM},
+		})
+
+		expectedSubscribedQualities = []*livekit.SubscribedCodec{
+			{
+				Codec: mime.MimeTypeVP8.String(),
+				Qualities: []*livekit.SubscribedQuality{
+					{Quality: livekit.VideoQuality_LOW, Enabled: true},
+					{Quality: livekit.VideoQuality_MEDIUM, Enabled: true},
+					{Quality: livekit.VideoQuality_HIGH, Enabled: true},
+				},
+			},
+			{
+				Codec: mime.MimeTypeAV1.String(),
+				Qualities: []*livekit.SubscribedQuality{
+					{Quality: livekit.VideoQuality_LOW, Enabled: true},
+					{Quality: livekit.VideoQuality_MEDIUM, Enabled: true},
+					{Quality: livekit.VideoQuality_HIGH, Enabled: false},
+				},
+			},
+		}
+		require.Eventually(t, func() bool {
+			lock.Lock()
+			defer lock.Unlock()
+
+			return subscribedCodecsAsString(expectedSubscribedQualities) == subscribedCodecsAsString(actualSubscribedQualities)
+		}, 10*time.Second, 100*time.Millisecond)
+	})
+}
+
+func TestCodecRegression(t *testing.T) {
+	dm := NewDynacastManager(DynacastManagerParams{})
+	var lock sync.Mutex
+	actualSubscribedQualities := make([]*livekit.SubscribedCodec, 0)
+	dm.OnSubscribedMaxQualityChange(func(subscribedQualities []*livekit.SubscribedCodec, _maxSubscribedQualities []types.SubscribedCodecQuality) {
+		lock.Lock()
+		actualSubscribedQualities = subscribedQualities
+		lock.Unlock()
+	})
+
+	dm.NotifySubscriberMaxQuality("s1", mime.MimeTypeAV1, livekit.VideoQuality_HIGH)
+
+	expectedSubscribedQualities := []*livekit.SubscribedCodec{
+		{
+			Codec: mime.MimeTypeAV1.String(),
+			Qualities: []*livekit.SubscribedQuality{
+				{Quality: livekit.VideoQuality_LOW, Enabled: true},
+				{Quality: livekit.VideoQuality_MEDIUM, Enabled: true},
+				{Quality: livekit.VideoQuality_HIGH, Enabled: true},
+			},
+		},
+	}
+	require.Eventually(t, func() bool {
+		lock.Lock()
+		defer lock.Unlock()
+
+		return subscribedCodecsAsString(expectedSubscribedQualities) == subscribedCodecsAsString(actualSubscribedQualities)
+	}, 10*time.Second, 100*time.Millisecond)
+
+	dm.HandleCodecRegression(mime.MimeTypeAV1, mime.MimeTypeVP8)
+
+	expectedSubscribedQualities = []*livekit.SubscribedCodec{
+		{
+			Codec: mime.MimeTypeAV1.String(),
+			Qualities: []*livekit.SubscribedQuality{
+				{Quality: livekit.VideoQuality_LOW, Enabled: false},
+				{Quality: livekit.VideoQuality_MEDIUM, Enabled: false},
+				{Quality: livekit.VideoQuality_HIGH, Enabled: false},
+			},
+		},
+		{
+			Codec: mime.MimeTypeVP8.String(),
+			Qualities: []*livekit.SubscribedQuality{
+				{Quality: livekit.VideoQuality_LOW, Enabled: true},
+				{Quality: livekit.VideoQuality_MEDIUM, Enabled: true},
+				{Quality: livekit.VideoQuality_HIGH, Enabled: true},
+			},
+		},
+	}
+	require.Eventually(t, func() bool {
+		lock.Lock()
+		defer lock.Unlock()
+
+		return subscribedCodecsAsString(expectedSubscribedQualities) == subscribedCodecsAsString(actualSubscribedQualities)
+	}, 10*time.Second, 100*time.Millisecond)
+
+	// av1 quality change should be forwarded to vp8
+	// av1 quality change of node should be ignored
+	dm.NotifySubscriberMaxQuality("s1", mime.MimeTypeAV1, livekit.VideoQuality_MEDIUM)
+	dm.NotifySubscriberNodeMaxQuality("n1", []types.SubscribedCodecQuality{
+		{CodecMime: mime.MimeTypeAV1, Quality: livekit.VideoQuality_HIGH},
+	})
+	expectedSubscribedQualities = []*livekit.SubscribedCodec{
+		{
+			Codec: mime.MimeTypeAV1.String(),
+			Qualities: []*livekit.SubscribedQuality{
+				{Quality: livekit.VideoQuality_LOW, Enabled: false},
+				{Quality: livekit.VideoQuality_MEDIUM, Enabled: false},
+				{Quality: livekit.VideoQuality_HIGH, Enabled: false},
+			},
+		},
+		{
+			Codec: mime.MimeTypeVP8.String(),
+			Qualities: []*livekit.SubscribedQuality{
+				{Quality: livekit.VideoQuality_LOW, Enabled: true},
+				{Quality: livekit.VideoQuality_MEDIUM, Enabled: true},
+				{Quality: livekit.VideoQuality_HIGH, Enabled: false},
+			},
+		},
+	}
+	require.Eventually(t, func() bool {
+		lock.Lock()
+		defer lock.Unlock()
+
+		return subscribedCodecsAsString(expectedSubscribedQualities) == subscribedCodecsAsString(actualSubscribedQualities)
+	}, 10*time.Second, 100*time.Millisecond)
+}
+
+func subscribedCodecsAsString(c1 []*livekit.SubscribedCodec) string {
+	sort.Slice(c1, func(i, j int) bool { return c1[i].Codec < c1[j].Codec })
+	var s1 string
+	for _, c := range c1 {
+		s1 += c.String()
+	}
+	return s1
+}

livekit-server/pkg/rtc/dynacast/dynacastquality.go

@@ -0,0 +1,230 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dynacast
+
+import (
+	"sync"
+	"time"
+
+	"github.com/livekit/livekit-server/pkg/sfu/mime"
+	"github.com/livekit/protocol/livekit"
+	"github.com/livekit/protocol/logger"
+)
+
+const (
+	initialQualityUpdateWait = 10 * time.Second
+)
+
+type DynacastQualityParams struct {
+	MimeType mime.MimeType
+	Logger   logger.Logger
+}
+
+// DynacastQuality manages max subscribed quality of a single receiver of a media track
+type DynacastQuality struct {
+	params DynacastQualityParams
+
+	// quality level enable/disable
+	lock                     sync.RWMutex
+	initialized              bool
+	maxSubscriberQuality     map[livekit.ParticipantID]livekit.VideoQuality
+	maxSubscriberNodeQuality map[livekit.NodeID]livekit.VideoQuality
+	maxSubscribedQuality     livekit.VideoQuality
+	maxQualityTimer          *time.Timer
+	regressTo                *DynacastQuality
+
+	onSubscribedMaxQualityChange func(mimeType mime.MimeType, maxSubscribedQuality livekit.VideoQuality)
+}
+
+func NewDynacastQuality(params DynacastQualityParams) *DynacastQuality {
+	return &DynacastQuality{
+		params:                   params,
+		maxSubscriberQuality:     make(map[livekit.ParticipantID]livekit.VideoQuality),
+		maxSubscriberNodeQuality: make(map[livekit.NodeID]livekit.VideoQuality),
+	}
+}
+
+func (d *DynacastQuality) Start() {
+	d.reset()
+}
+
+func (d *DynacastQuality) Restart() {
+	d.reset()
+}
+
+func (d *DynacastQuality) Stop() {
+	d.stopMaxQualityTimer()
+}
+
+func (d *DynacastQuality) OnSubscribedMaxQualityChange(f func(mimeType mime.MimeType, maxSubscribedQuality livekit.VideoQuality)) {
+	d.lock.Lock()
+	defer d.lock.Unlock()
+	d.onSubscribedMaxQualityChange = f
+}
+
+func (d *DynacastQuality) NotifySubscriberMaxQuality(subscriberID livekit.ParticipantID, quality livekit.VideoQuality) {
+	d.params.Logger.Debugw(
+		"setting subscriber max quality",
+		"mime", d.params.MimeType,
+		"subscriberID", subscriberID,
+		"quality", quality.String(),
+	)
+
+	d.lock.Lock()
+	if r := d.regressTo; r != nil {
+		d.lock.Unlock()
+		r.NotifySubscriberMaxQuality(subscriberID, quality)
+		return
+	}
+
+	if quality == livekit.VideoQuality_OFF {
+		delete(d.maxSubscriberQuality, subscriberID)
+	} else {
+		d.maxSubscriberQuality[subscriberID] = quality
+	}
+	d.lock.Unlock()
+
+	d.updateQualityChange(false)
+}
+
+func (d *DynacastQuality) NotifySubscriberNodeMaxQuality(nodeID livekit.NodeID, quality livekit.VideoQuality) {
+	d.params.Logger.Debugw(
+		"setting subscriber node max quality",
+		"mime", d.params.MimeType,
+		"subscriberNodeID", nodeID,
+		"quality", quality.String(),
+	)
+
+	d.lock.Lock()
+	if r := d.regressTo; r != nil {
+		// the downstream node will synthesize correct quality notify (its dynacast manager has codec regression), just ignore it
+		d.lock.Unlock()
+		r.params.Logger.Debugw("ignoring node quality change, regressed to another dynacast quality", "mime", d.params.MimeType)
+		return
+	}
+
+	if quality == livekit.VideoQuality_OFF {
+		delete(d.maxSubscriberNodeQuality, nodeID)
+	} else {
+		d.maxSubscriberNodeQuality[nodeID] = quality
+	}
+	d.lock.Unlock()
+
+	d.updateQualityChange(false)
+}
+
+func (d *DynacastQuality) RegressTo(other *DynacastQuality) {
+	d.lock.Lock()
+	d.regressTo = other
+	maxSubscriberQuality := d.maxSubscriberQuality
+	maxSubscriberNodeQuality := d.maxSubscriberNodeQuality
+	d.maxSubscriberQuality = make(map[livekit.ParticipantID]livekit.VideoQuality)
+	d.maxSubscriberNodeQuality = make(map[livekit.NodeID]livekit.VideoQuality)
+	d.lock.Unlock()
+
+	other.lock.Lock()
+	for subID, quality := range maxSubscriberQuality {
+		if otherQuality, ok := other.maxSubscriberQuality[subID]; ok {
+			// no QUALITY_OFF in the map
+			if quality > otherQuality {
+				other.maxSubscriberQuality[subID] = quality
+			}
+		} else {
+			other.maxSubscriberQuality[subID] = quality
+		}
+	}
+
+	for nodeID, quality := range maxSubscriberNodeQuality {
+		if otherQuality, ok := other.maxSubscriberNodeQuality[nodeID]; ok {
+			// no QUALITY_OFF in the map
+			if quality > otherQuality {
+				other.maxSubscriberNodeQuality[nodeID] = quality
+			}
+		} else {
+			other.maxSubscriberNodeQuality[nodeID] = quality
+		}
+	}
+	other.lock.Unlock()
+	other.Restart()
+}
+
+func (d *DynacastQuality) reset() {
+	d.lock.Lock()
+	d.initialized = false
+	d.lock.Unlock()
+
+	d.startMaxQualityTimer()
+}
+
+func (d *DynacastQuality) updateQualityChange(force bool) {
+	d.lock.Lock()
+	maxSubscribedQuality := livekit.VideoQuality_OFF
+	for _, subQuality := range d.maxSubscriberQuality {
+		if maxSubscribedQuality == livekit.VideoQuality_OFF || (subQuality != livekit.VideoQuality_OFF && subQuality > maxSubscribedQuality) {
+			maxSubscribedQuality = subQuality
+		}
+	}
+	for _, nodeQuality := range d.maxSubscriberNodeQuality {
+		if maxSubscribedQuality == livekit.VideoQuality_OFF || (nodeQuality != livekit.VideoQuality_OFF && nodeQuality > maxSubscribedQuality) {
+			maxSubscribedQuality = nodeQuality
+		}
+	}
+
+	if maxSubscribedQuality == d.maxSubscribedQuality && d.initialized && !force {
+		d.lock.Unlock()
+		return
+	}
+
+	d.initialized = true
+	d.maxSubscribedQuality = maxSubscribedQuality
+	d.params.Logger.Debugw("notifying quality change",
+		"mime", d.params.MimeType,
+		"maxSubscriberQuality", d.maxSubscriberQuality,
+		"maxSubscriberNodeQuality", d.maxSubscriberNodeQuality,
+		"maxSubscribedQuality", d.maxSubscribedQuality,
+		"force", force,
+	)
+	onSubscribedMaxQualityChange := d.onSubscribedMaxQualityChange
+	d.lock.Unlock()
+
+	if onSubscribedMaxQualityChange != nil {
+		onSubscribedMaxQualityChange(d.params.MimeType, maxSubscribedQuality)
+	}
+}
+
+func (d *DynacastQuality) startMaxQualityTimer() {
+	d.lock.Lock()
+	defer d.lock.Unlock()
+
+	if d.maxQualityTimer != nil {
+		d.maxQualityTimer.Stop()
+		d.maxQualityTimer = nil
+	}
+
+	d.maxQualityTimer = time.AfterFunc(initialQualityUpdateWait, func() {
+		d.stopMaxQualityTimer()
+		d.updateQualityChange(true)
+	})
+}
+
+func (d *DynacastQuality) stopMaxQualityTimer() {
+	d.lock.Lock()
+	defer d.lock.Unlock()
+
+	if d.maxQualityTimer != nil {
+		d.maxQualityTimer.Stop()
+		d.maxQualityTimer = nil
+	}
+}

livekit-server/pkg/rtc/egress.go

@@ -0,0 +1,174 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package rtc
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+
+	"github.com/livekit/livekit-server/pkg/rtc/types"
+	"github.com/livekit/livekit-server/pkg/telemetry"
+	"github.com/livekit/protocol/livekit"
+	"github.com/livekit/protocol/rpc"
+	"github.com/livekit/protocol/webhook"
+)
+
+type EgressLauncher interface {
+	StartEgress(context.Context, *rpc.StartEgressRequest) (*livekit.EgressInfo, error)
+}
+
+func StartParticipantEgress(
+	ctx context.Context,
+	launcher EgressLauncher,
+	ts telemetry.TelemetryService,
+	opts *livekit.AutoParticipantEgress,
+	identity livekit.ParticipantIdentity,
+	roomName livekit.RoomName,
+	roomID livekit.RoomID,
+) error {
+	if req, err := startParticipantEgress(ctx, launcher, opts, identity, roomName, roomID); err != nil {
+		// send egress failed webhook
+		ts.NotifyEvent(ctx, &livekit.WebhookEvent{
+			Event: webhook.EventEgressEnded,
+			EgressInfo: &livekit.EgressInfo{
+				RoomId:   string(roomID),
+				RoomName: string(roomName),
+				Status:   livekit.EgressStatus_EGRESS_FAILED,
+				Error:    err.Error(),
+				Request:  &livekit.EgressInfo_Participant{Participant: req},
+			},
+		})
+		return err
+	}
+	return nil
+}
+
+func startParticipantEgress(
+	ctx context.Context,
+	launcher EgressLauncher,
+	opts *livekit.AutoParticipantEgress,
+	identity livekit.ParticipantIdentity,
+	roomName livekit.RoomName,
+	roomID livekit.RoomID,
+) (*livekit.ParticipantEgressRequest, error) {
+	req := &livekit.ParticipantEgressRequest{
+		RoomName:       string(roomName),
+		Identity:       string(identity),
+		FileOutputs:    opts.FileOutputs,
+		SegmentOutputs: opts.SegmentOutputs,
+	}
+
+	switch o := opts.Options.(type) {
+	case *livekit.AutoParticipantEgress_Preset:
+		req.Options = &livekit.ParticipantEgressRequest_Preset{Preset: o.Preset}
+	case *livekit.AutoParticipantEgress_Advanced:
+		req.Options = &livekit.ParticipantEgressRequest_Advanced{Advanced: o.Advanced}
+	}
+
+	if launcher == nil {
+		return req, errors.New("egress launcher not found")
+	}
+
+	_, err := launcher.StartEgress(ctx, &rpc.StartEgressRequest{
+		Request: &rpc.StartEgressRequest_Participant{
+			Participant: req,
+		},
+		RoomId: string(roomID),
+	})
+	return req, err
+}
+
+func StartTrackEgress(
+	ctx context.Context,
+	launcher EgressLauncher,
+	ts telemetry.TelemetryService,
+	opts *livekit.AutoTrackEgress,
+	track types.MediaTrack,
+	roomName livekit.RoomName,
+	roomID livekit.RoomID,
+) error {
+	if req, err := startTrackEgress(ctx, launcher, opts, track, roomName, roomID); err != nil {
+		// send egress failed webhook
+		ts.NotifyEvent(ctx, &livekit.WebhookEvent{
+			Event: webhook.EventEgressEnded,
+			EgressInfo: &livekit.EgressInfo{
+				RoomId:   string(roomID),
+				RoomName: string(roomName),
+				Status:   livekit.EgressStatus_EGRESS_FAILED,
+				Error:    err.Error(),
+				Request:  &livekit.EgressInfo_Track{Track: req},
+			},
+		})
+		return err
+	}
+	return nil
+}
+
+func startTrackEgress(
+	ctx context.Context,
+	launcher EgressLauncher,
+	opts *livekit.AutoTrackEgress,
+	track types.MediaTrack,
+	roomName livekit.RoomName,
+	roomID livekit.RoomID,
+) (*livekit.TrackEgressRequest, error) {
+	output := &livekit.DirectFileOutput{
+		Filepath: getFilePath(opts.Filepath),
+	}
+
+	switch out := opts.Output.(type) {
+	case *livekit.AutoTrackEgress_Azure:
+		output.Output = &livekit.DirectFileOutput_Azure{Azure: out.Azure}
+	case *livekit.AutoTrackEgress_Gcp:
+		output.Output = &livekit.DirectFileOutput_Gcp{Gcp: out.Gcp}
+	case *livekit.AutoTrackEgress_S3:
+		output.Output = &livekit.DirectFileOutput_S3{S3: out.S3}
+	}
+
+	req := &livekit.TrackEgressRequest{
+		RoomName: string(roomName),
+		TrackId:  string(track.ID()),
+		Output: &livekit.TrackEgressRequest_File{
+			File: output,
+		},
+	}
+
+	if launcher == nil {
+		return req, errors.New("egress launcher not found")
+	}
+
+	_, err := launcher.StartEgress(ctx, &rpc.StartEgressRequest{
+		Request: &rpc.StartEgressRequest_Track{
+			Track: req,
+		},
+		RoomId: string(roomID),
+	})
+	return req, err
+}
+
+func getFilePath(filepath string) string {
+	if filepath == "" || strings.HasSuffix(filepath, "/") || strings.Contains(filepath, "{track_id}") {
+		return filepath
+	}
+
+	idx := strings.Index(filepath, ".")
+	if idx == -1 {
+		return fmt.Sprintf("%s-{track_id}", filepath)
+	} else {
+		return fmt.Sprintf("%s-%s%s", filepath[:idx], "{track_id}", filepath[idx:])
+	}
+}

livekit-server/pkg/rtc/errors.go

@@ -0,0 +1,47 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package rtc
+
+import (
+	"errors"
+)
+
+var (
+	ErrRoomClosed               = errors.New("room has already closed")
+	ErrParticipantSessionClosed = errors.New("participant session is already closed")
+	ErrPermissionDenied         = errors.New("no permissions to access the room")
+	ErrMaxParticipantsExceeded  = errors.New("room has exceeded its max participants")
+	ErrLimitExceeded            = errors.New("node has exceeded its configured limit")
+	ErrAlreadyJoined            = errors.New("a participant with the same identity is already in the room")
+	ErrDataChannelUnavailable   = errors.New("data channel is not available")
+	ErrDataChannelBufferFull    = errors.New("data channel buffer is full")
+	ErrTransportFailure         = errors.New("transport failure")
+	ErrEmptyIdentity            = errors.New("participant identity cannot be empty")
+	ErrEmptyParticipantID       = errors.New("participant ID cannot be empty")
+	ErrMissingGrants            = errors.New("VideoGrant is missing")
+	ErrInternalError            = errors.New("internal error")
+	ErrNameExceedsLimits        = errors.New("name length exceeds limits")
+	ErrMetadataExceedsLimits    = errors.New("metadata size exceeds limits")
+	ErrAttributesExceedsLimits  = errors.New("attributes size exceeds limits")
+
+	// Track subscription related
+	ErrNoTrackPermission         = errors.New("participant is not allowed to subscribe to this track")
+	ErrNoSubscribePermission     = errors.New("participant is not given permission to subscribe to tracks")
+	ErrTrackNotFound             = errors.New("track cannot be found")
+	ErrTrackNotBound             = errors.New("track not bound")
+	ErrSubscriptionLimitExceeded = errors.New("participant has exceeded its subscription limit")
+
+	ErrNoSubscribeMetricsPermission = errors.New("participant is not given permission to subscribe to metrics")
+)

livekit-server/pkg/rtc/mediaengine.go

@@ -0,0 +1,217 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package rtc
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/pion/webrtc/v4"
+
+	"github.com/livekit/livekit-server/pkg/sfu/mime"
+	"github.com/livekit/protocol/livekit"
+)
+
+var OpusCodecCapability = webrtc.RTPCodecCapability{
+	MimeType:    mime.MimeTypeOpus.String(),
+	ClockRate:   48000,
+	Channels:    2,
+	SDPFmtpLine: "minptime=10;useinbandfec=1",
+}
+var RedCodecCapability = webrtc.RTPCodecCapability{
+	MimeType:    mime.MimeTypeRED.String(),
+	ClockRate:   48000,
+	Channels:    2,
+	SDPFmtpLine: "111/111",
+}
+var videoRTX = webrtc.RTPCodecCapability{
+	MimeType:  mime.MimeTypeRTX.String(),
+	ClockRate: 90000,
+}
+
+func registerCodecs(me *webrtc.MediaEngine, codecs []*livekit.Codec, rtcpFeedback RTCPFeedbackConfig, filterOutH264HighProfile bool) error {
+	opusCodec := OpusCodecCapability
+	opusCodec.RTCPFeedback = rtcpFeedback.Audio
+	var opusPayload webrtc.PayloadType
+	if IsCodecEnabled(codecs, opusCodec) {
+		opusPayload = 111
+		if err := me.RegisterCodec(webrtc.RTPCodecParameters{
+			RTPCodecCapability: opusCodec,
+			PayloadType:        opusPayload,
+		}, webrtc.RTPCodecTypeAudio); err != nil {
+			return err
+		}
+
+		if IsCodecEnabled(codecs, RedCodecCapability) {
+			if err := me.RegisterCodec(webrtc.RTPCodecParameters{
+				RTPCodecCapability: RedCodecCapability,
+				PayloadType:        63,
+			}, webrtc.RTPCodecTypeAudio); err != nil {
+				return err
+			}
+		}
+	}
+
+	rtxEnabled := IsCodecEnabled(codecs, videoRTX)
+
+	h264HighProfileFmtp := "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640032"
+	for _, codec := range []webrtc.RTPCodecParameters{
+		{
+			RTPCodecCapability: webrtc.RTPCodecCapability{
+				MimeType:     mime.MimeTypeVP8.String(),
+				ClockRate:    90000,
+				RTCPFeedback: rtcpFeedback.Video,
+			},
+			PayloadType: 96,
+		},
+		{
+			RTPCodecCapability: webrtc.RTPCodecCapability{
+				MimeType:     mime.MimeTypeVP9.String(),
+				ClockRate:    90000,
+				SDPFmtpLine:  "profile-id=0",
+				RTCPFeedback: rtcpFeedback.Video,
+			},
+			PayloadType: 98,
+		},
+		{
+			RTPCodecCapability: webrtc.RTPCodecCapability{
+				MimeType:     mime.MimeTypeVP9.String(),
+				ClockRate:    90000,
+				SDPFmtpLine:  "profile-id=1",
+				RTCPFeedback: rtcpFeedback.Video,
+			},
+			PayloadType: 100,
+		},
+		{
+			RTPCodecCapability: webrtc.RTPCodecCapability{
+				MimeType:     mime.MimeTypeH264.String(),
+				ClockRate:    90000,
+				SDPFmtpLine:  "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f",
+				RTCPFeedback: rtcpFeedback.Video,
+			},
+			PayloadType: 125,
+		},
+		{
+			RTPCodecCapability: webrtc.RTPCodecCapability{
+				MimeType:     mime.MimeTypeH264.String(),
+				ClockRate:    90000,
+				SDPFmtpLine:  "level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42e01f",
+				RTCPFeedback: rtcpFeedback.Video,
+			},
+			PayloadType: 108,
+		},
+		{
+			RTPCodecCapability: webrtc.RTPCodecCapability{
+				MimeType:     mime.MimeTypeH264.String(),
+				ClockRate:    90000,
+				SDPFmtpLine:  h264HighProfileFmtp,
+				RTCPFeedback: rtcpFeedback.Video,
+			},
+			PayloadType: 123,
+		},
+		{
+			RTPCodecCapability: webrtc.RTPCodecCapability{
+				MimeType:     mime.MimeTypeAV1.String(),
+				ClockRate:    90000,
+				RTCPFeedback: rtcpFeedback.Video,
+			},
+			PayloadType: 35,
+		},
+		{
+			RTPCodecCapability: webrtc.RTPCodecCapability{
+				MimeType:     webrtc.MimeTypeH265,
+				ClockRate:    90000,
+				RTCPFeedback: rtcpFeedback.Video,
+			},
+			PayloadType: 116,
+		},
+	} {
+		if filterOutH264HighProfile && codec.RTPCodecCapability.SDPFmtpLine == h264HighProfileFmtp {
+			continue
+		}
+		if mime.IsMimeTypeStringRTX(codec.MimeType) {
+			continue
+		}
+		if IsCodecEnabled(codecs, codec.RTPCodecCapability) {
+			if err := me.RegisterCodec(codec, webrtc.RTPCodecTypeVideo); err != nil {
+				return err
+			}
+			if rtxEnabled {
+				if err := me.RegisterCodec(webrtc.RTPCodecParameters{
+					RTPCodecCapability: webrtc.RTPCodecCapability{
+						MimeType:    mime.MimeTypeRTX.String(),
+						ClockRate:   90000,
+						SDPFmtpLine: fmt.Sprintf("apt=%d", codec.PayloadType),
+					},
+					PayloadType: codec.PayloadType + 1,
+				}, webrtc.RTPCodecTypeVideo); err != nil {
+					return err
+				}
+			}
+		}
+	}
+	return nil
+}
+
+func registerHeaderExtensions(me *webrtc.MediaEngine, rtpHeaderExtension RTPHeaderExtensionConfig) error {
+	for _, extension := range rtpHeaderExtension.Video {
+		if err := me.RegisterHeaderExtension(webrtc.RTPHeaderExtensionCapability{URI: extension}, webrtc.RTPCodecTypeVideo); err != nil {
+			return err
+		}
+	}
+
+	for _, extension := range rtpHeaderExtension.Audio {
+		if err := me.RegisterHeaderExtension(webrtc.RTPHeaderExtensionCapability{URI: extension}, webrtc.RTPCodecTypeAudio); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func createMediaEngine(codecs []*livekit.Codec, config DirectionConfig, filterOutH264HighProfile bool) (*webrtc.MediaEngine, error) {
+	me := &webrtc.MediaEngine{}
+	if err := registerCodecs(me, codecs, config.RTCPFeedback, filterOutH264HighProfile); err != nil {
+		return nil, err
+	}
+
+	if err := registerHeaderExtensions(me, config.RTPHeaderExtension); err != nil {
+		return nil, err
+	}
+
+	return me, nil
+}
+
+func IsCodecEnabled(codecs []*livekit.Codec, cap webrtc.RTPCodecCapability) bool {
+	for _, codec := range codecs {
+		if !mime.IsMimeTypeStringEqual(codec.Mime, cap.MimeType) {
+			continue
+		}
+		if codec.FmtpLine == "" || strings.EqualFold(codec.FmtpLine, cap.SDPFmtpLine) {
+			return true
+		}
+	}
+	return false
+}
+
+func selectAlternativeVideoCodec(enabledCodecs []*livekit.Codec) string {
+	for _, c := range enabledCodecs {
+		if mime.IsMimeTypeStringVideo(c.Mime) {
+			return c.Mime
+		}
+	}
+	// no viable codec in the list of enabled codecs, fall back to the most widely supported codec
+	return mime.MimeTypeVP8.String()
+}

livekit-server/pkg/rtc/mediaengine_test.go

@@ -0,0 +1,41 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package rtc
+
+import (
+	"testing"
+
+	"github.com/pion/webrtc/v4"
+	"github.com/stretchr/testify/require"
+
+	"github.com/livekit/livekit-server/pkg/sfu/mime"
+	"github.com/livekit/protocol/livekit"
+)
+
+func TestIsCodecEnabled(t *testing.T) {
+	t.Run("empty fmtp requirement should match all", func(t *testing.T) {
+		enabledCodecs := []*livekit.Codec{{Mime: "video/h264"}}
+		require.True(t, IsCodecEnabled(enabledCodecs, webrtc.RTPCodecCapability{MimeType: mime.MimeTypeH264.String(), SDPFmtpLine: "special"}))
+		require.True(t, IsCodecEnabled(enabledCodecs, webrtc.RTPCodecCapability{MimeType: mime.MimeTypeH264.String()}))
+		require.False(t, IsCodecEnabled(enabledCodecs, webrtc.RTPCodecCapability{MimeType: mime.MimeTypeVP8.String()}))
+	})
+
+	t.Run("when fmtp is provided, require match", func(t *testing.T) {
+		enabledCodecs := []*livekit.Codec{{Mime: "video/h264", FmtpLine: "special"}}
+		require.True(t, IsCodecEnabled(enabledCodecs, webrtc.RTPCodecCapability{MimeType: mime.MimeTypeH264.String(), SDPFmtpLine: "special"}))
+		require.False(t, IsCodecEnabled(enabledCodecs, webrtc.RTPCodecCapability{MimeType: mime.MimeTypeH264.String()}))
+		require.False(t, IsCodecEnabled(enabledCodecs, webrtc.RTPCodecCapability{MimeType: mime.MimeTypeVP8.String()}))
+	})
+}

livekit-server/pkg/rtc/medialossproxy.go

@@ -0,0 +1,105 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package rtc
+
+import (
+	"sync"
+	"time"
+
+	"github.com/pion/rtcp"
+
+	"github.com/livekit/protocol/livekit"
+	"github.com/livekit/protocol/logger"
+
+	"github.com/livekit/livekit-server/pkg/sfu"
+)
+
+const (
+	downLostUpdateDelta = time.Second
+)
+
+type MediaLossProxyParams struct {
+	Logger logger.Logger
+}
+
+type MediaLossProxy struct {
+	params MediaLossProxyParams
+
+	lock                 sync.Mutex
+	maxDownFracLost      uint8
+	maxDownFracLostTs    time.Time
+	maxDownFracLostValid bool
+
+	onMediaLossUpdate func(fractionalLoss uint8)
+}
+
+func NewMediaLossProxy(params MediaLossProxyParams) *MediaLossProxy {
+	return &MediaLossProxy{params: params}
+}
+
+func (m *MediaLossProxy) OnMediaLossUpdate(f func(fractionalLoss uint8)) {
+	m.lock.Lock()
+	m.onMediaLossUpdate = f
+	m.lock.Unlock()
+}
+
+func (m *MediaLossProxy) HandleMaxLossFeedback(_ *sfu.DownTrack, report *rtcp.ReceiverReport) {
+	m.lock.Lock()
+	for _, rr := range report.Reports {
+		m.maxDownFracLostValid = true
+		if m.maxDownFracLost < rr.FractionLost {
+			m.maxDownFracLost = rr.FractionLost
+		}
+	}
+	m.lock.Unlock()
+
+	m.maybeUpdateLoss()
+}
+
+func (m *MediaLossProxy) NotifySubscriberNodeMediaLoss(_nodeID livekit.NodeID, fractionalLoss uint8) {
+	m.lock.Lock()
+	m.maxDownFracLostValid = true
+	if m.maxDownFracLost < fractionalLoss {
+		m.maxDownFracLost = fractionalLoss
+	}
+	m.lock.Unlock()
+
+	m.maybeUpdateLoss()
+}
+
+func (m *MediaLossProxy) maybeUpdateLoss() {
+	var (
+		shouldUpdate bool
+		maxLost      uint8
+	)
+
+	m.lock.Lock()
+	now := time.Now()
+	if now.Sub(m.maxDownFracLostTs) > downLostUpdateDelta && m.maxDownFracLostValid {
+		shouldUpdate = true
+		maxLost = m.maxDownFracLost
+		m.maxDownFracLost = 0
+		m.maxDownFracLostTs = now
+		m.maxDownFracLostValid = false
+	}
+	onMediaLossUpdate := m.onMediaLossUpdate
+	m.lock.Unlock()
+
+	if shouldUpdate {
+		if onMediaLossUpdate != nil {
+			onMediaLossUpdate(maxLost)
+		}
+	}
+}

livekit-server/pkg/rtc/mediatrack.go

@@ -0,0 +1,488 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package rtc
+
+import (
+	"context"
+	"math"
+	"sync"
+	"time"
+
+	"github.com/pion/rtcp"
+	"github.com/pion/webrtc/v4"
+	"go.uber.org/atomic"
+
+	"github.com/livekit/protocol/livekit"
+	"github.com/livekit/protocol/logger"
+
+	"github.com/livekit/livekit-server/pkg/config"
+	"github.com/livekit/livekit-server/pkg/rtc/dynacast"
+	"github.com/livekit/livekit-server/pkg/rtc/types"
+	"github.com/livekit/livekit-server/pkg/sfu"
+	"github.com/livekit/livekit-server/pkg/sfu/buffer"
+	"github.com/livekit/livekit-server/pkg/sfu/connectionquality"
+	"github.com/livekit/livekit-server/pkg/sfu/mime"
+	"github.com/livekit/livekit-server/pkg/telemetry"
+	util "github.com/livekit/mediatransportutil"
+)
+
+// MediaTrack represents a WebRTC track that needs to be forwarded
+// Implements MediaTrack and PublishedTrack interface
+type MediaTrack struct {
+	params         MediaTrackParams
+	numUpTracks    atomic.Uint32
+	buffer         *buffer.Buffer
+	everSubscribed atomic.Bool
+
+	*MediaTrackReceiver
+	*MediaLossProxy
+
+	dynacastManager *dynacast.DynacastManager
+
+	lock sync.RWMutex
+
+	rttFromXR atomic.Bool
+
+	enableRegression              bool
+	regressionTargetCodec         mime.MimeType
+	regressionTargetCodecReceived bool
+}
+
+type MediaTrackParams struct {
+	SignalCid             string
+	SdpCid                string
+	ParticipantID         livekit.ParticipantID
+	ParticipantIdentity   livekit.ParticipantIdentity
+	ParticipantVersion    uint32
+	BufferFactory         *buffer.Factory
+	ReceiverConfig        ReceiverConfig
+	SubscriberConfig      DirectionConfig
+	PLIThrottleConfig     sfu.PLIThrottleConfig
+	AudioConfig           sfu.AudioConfig
+	VideoConfig           config.VideoConfig
+	Telemetry             telemetry.TelemetryService
+	Logger                logger.Logger
+	SimTracks             map[uint32]SimulcastTrackInfo
+	OnRTCP                func([]rtcp.Packet)
+	ForwardStats          *sfu.ForwardStats
+	OnTrackEverSubscribed func(livekit.TrackID)
+}
+
+func NewMediaTrack(params MediaTrackParams, ti *livekit.TrackInfo) *MediaTrack {
+	t := &MediaTrack{
+		params: params,
+	}
+
+	// TODO: disable codec regression until simulcast-codec clients knows that
+	if ti.BackupCodecPolicy == livekit.BackupCodecPolicy_REGRESSION && len(ti.Codecs) > 1 {
+		t.enableRegression = true
+		t.regressionTargetCodec = mime.NormalizeMimeType(ti.Codecs[1].MimeType)
+		t.params.Logger.Debugw("track enabled codec regression", "regressionCodec", t.regressionTargetCodec)
+	}
+
+	t.MediaTrackReceiver = NewMediaTrackReceiver(MediaTrackReceiverParams{
+		MediaTrack:            t,
+		IsRelayed:             false,
+		ParticipantID:         params.ParticipantID,
+		ParticipantIdentity:   params.ParticipantIdentity,
+		ParticipantVersion:    params.ParticipantVersion,
+		ReceiverConfig:        params.ReceiverConfig,
+		SubscriberConfig:      params.SubscriberConfig,
+		AudioConfig:           params.AudioConfig,
+		Telemetry:             params.Telemetry,
+		Logger:                params.Logger,
+		RegressionTargetCodec: t.regressionTargetCodec,
+	}, ti)
+
+	if ti.Type == livekit.TrackType_AUDIO {
+		t.MediaLossProxy = NewMediaLossProxy(MediaLossProxyParams{
+			Logger: params.Logger,
+		})
+		t.MediaLossProxy.OnMediaLossUpdate(func(fractionalLoss uint8) {
+			if t.buffer != nil {
+				t.buffer.SetLastFractionLostReport(fractionalLoss)
+			}
+		})
+		t.MediaTrackReceiver.OnMediaLossFeedback(t.MediaLossProxy.HandleMaxLossFeedback)
+	}
+
+	if ti.Type == livekit.TrackType_VIDEO {
+		t.dynacastManager = dynacast.NewDynacastManager(dynacast.DynacastManagerParams{
+			DynacastPauseDelay: params.VideoConfig.DynacastPauseDelay,
+			Logger:             params.Logger,
+		})
+		t.MediaTrackReceiver.OnSetupReceiver(func(mime mime.MimeType) {
+			t.dynacastManager.AddCodec(mime)
+		})
+		t.MediaTrackReceiver.OnSubscriberMaxQualityChange(
+			func(subscriberID livekit.ParticipantID, mimeType mime.MimeType, layer int32) {
+				t.dynacastManager.NotifySubscriberMaxQuality(
+					subscriberID,
+					mimeType,
+					buffer.SpatialLayerToVideoQuality(layer, t.MediaTrackReceiver.TrackInfo()),
+				)
+			},
+		)
+		t.MediaTrackReceiver.OnCodecRegression(func(old, new webrtc.RTPCodecParameters) {
+			t.dynacastManager.HandleCodecRegression(mime.NormalizeMimeType(old.MimeType), mime.NormalizeMimeType(new.MimeType))
+		})
+	}
+
+	return t
+}
+
+func (t *MediaTrack) OnSubscribedMaxQualityChange(
+	f func(
+		trackID livekit.TrackID,
+		trackInfo *livekit.TrackInfo,
+		subscribedQualities []*livekit.SubscribedCodec,
+		maxSubscribedQualities []types.SubscribedCodecQuality,
+	) error,
+) {
+	if t.dynacastManager == nil {
+		return
+	}
+
+	handler := func(subscribedQualities []*livekit.SubscribedCodec, maxSubscribedQualities []types.SubscribedCodecQuality) {
+		if f != nil && !t.IsMuted() {
+			_ = f(t.ID(), t.ToProto(), subscribedQualities, maxSubscribedQualities)
+		}
+
+		for _, q := range maxSubscribedQualities {
+			receiver := t.Receiver(q.CodecMime)
+			if receiver != nil {
+				receiver.SetMaxExpectedSpatialLayer(buffer.VideoQualityToSpatialLayer(q.Quality, t.MediaTrackReceiver.TrackInfo()))
+			}
+		}
+	}
+
+	t.dynacastManager.OnSubscribedMaxQualityChange(handler)
+}
+
+func (t *MediaTrack) NotifySubscriberNodeMaxQuality(nodeID livekit.NodeID, qualities []types.SubscribedCodecQuality) {
+	if t.dynacastManager != nil {
+		t.dynacastManager.NotifySubscriberNodeMaxQuality(nodeID, qualities)
+	}
+}
+
+func (t *MediaTrack) SignalCid() string {
+	return t.params.SignalCid
+}
+
+func (t *MediaTrack) HasSdpCid(cid string) bool {
+	if t.params.SdpCid == cid {
+		return true
+	}
+
+	ti := t.MediaTrackReceiver.TrackInfoClone()
+	for _, c := range ti.Codecs {
+		if c.Cid == cid {
+			return true
+		}
+	}
+	return false
+}
+
+func (t *MediaTrack) ToProto() *livekit.TrackInfo {
+	return t.MediaTrackReceiver.TrackInfoClone()
+}
+
+func (t *MediaTrack) UpdateCodecCid(codecs []*livekit.SimulcastCodec) {
+	t.MediaTrackReceiver.UpdateCodecCid(codecs)
+}
+
+// AddReceiver adds a new RTP receiver to the track, returns true when receiver represents a new codec
+func (t *MediaTrack) AddReceiver(receiver *webrtc.RTPReceiver, track sfu.TrackRemote, mid string) bool {
+	var newCodec bool
+	ssrc := uint32(track.SSRC())
+	buff, rtcpReader := t.params.BufferFactory.GetBufferPair(ssrc)
+	if buff == nil || rtcpReader == nil {
+		t.params.Logger.Errorw("could not retrieve buffer pair", nil)
+		return newCodec
+	}
+
+	var lastRR uint32
+	rtcpReader.OnPacket(func(bytes []byte) {
+		pkts, err := rtcp.Unmarshal(bytes)
+		if err != nil {
+			t.params.Logger.Errorw("could not unmarshal RTCP", err)
+			return
+		}
+
+		for _, pkt := range pkts {
+			switch pkt := pkt.(type) {
+			case *rtcp.SourceDescription:
+			case *rtcp.SenderReport:
+				if pkt.SSRC == uint32(track.SSRC()) {
+					buff.SetSenderReportData(pkt.RTPTime, pkt.NTPTime, pkt.PacketCount, pkt.OctetCount)
+				}
+			case *rtcp.ExtendedReport:
+			rttFromXR:
+				for _, report := range pkt.Reports {
+					if rr, ok := report.(*rtcp.DLRRReportBlock); ok {
+						for _, dlrrReport := range rr.Reports {
+							if dlrrReport.LastRR <= lastRR {
+								continue
+							}
+							nowNTP := util.ToNtpTime(time.Now())
+							nowNTP32 := uint32(nowNTP >> 16)
+							ntpDiff := nowNTP32 - dlrrReport.LastRR - dlrrReport.DLRR
+							rtt := uint32(math.Ceil(float64(ntpDiff) * 1000.0 / 65536.0))
+							buff.SetRTT(rtt)
+							t.rttFromXR.Store(true)
+							lastRR = dlrrReport.LastRR
+							break rttFromXR
+						}
+					}
+				}
+			}
+		}
+	})
+
+	ti := t.MediaTrackReceiver.TrackInfoClone()
+	t.lock.Lock()
+	var regressCodec bool
+	mimeType := mime.NormalizeMimeType(track.Codec().MimeType)
+	layer := buffer.RidToSpatialLayer(track.RID(), ti)
+	t.params.Logger.Debugw(
+		"AddReceiver",
+		"rid", track.RID(),
+		"layer", layer,
+		"ssrc", track.SSRC(),
+		"codec", track.Codec(),
+	)
+	wr := t.MediaTrackReceiver.Receiver(mimeType)
+	if wr == nil {
+		priority := -1
+		for idx, c := range ti.Codecs {
+			if mime.IsMimeTypeStringEqual(track.Codec().MimeType, c.MimeType) {
+				priority = idx
+				break
+			}
+		}
+		if priority < 0 {
+			switch len(ti.Codecs) {
+			case 0:
+				// audio track
+				priority = 0
+			case 1:
+				// older clients or non simulcast-codec, mime type only set later
+				if ti.Codecs[0].MimeType == "" {
+					priority = 0
+				}
+			}
+		}
+		if priority < 0 {
+			t.params.Logger.Warnw("could not find codec for webrtc receiver", nil, "webrtcCodec", mimeType, "track", logger.Proto(ti))
+			t.lock.Unlock()
+			return false
+		}
+
+		newWR := sfu.NewWebRTCReceiver(
+			receiver,
+			track,
+			ti,
+			LoggerWithCodecMime(t.params.Logger, mimeType),
+			t.params.OnRTCP,
+			t.params.VideoConfig.StreamTrackerManager,
+			sfu.WithPliThrottleConfig(t.params.PLIThrottleConfig),
+			sfu.WithAudioConfig(t.params.AudioConfig),
+			sfu.WithLoadBalanceThreshold(20),
+			sfu.WithStreamTrackers(),
+			sfu.WithForwardStats(t.params.ForwardStats),
+		)
+		newWR.OnCloseHandler(func() {
+			t.MediaTrackReceiver.SetClosing()
+			t.MediaTrackReceiver.ClearReceiver(mimeType, false)
+			if t.MediaTrackReceiver.TryClose() {
+				if t.dynacastManager != nil {
+					t.dynacastManager.Close()
+				}
+			}
+		})
+		// SIMULCAST-CODEC-TODO: these need to be receiver/mime aware, setting it up only for primary now
+		if priority == 0 {
+			newWR.OnStatsUpdate(func(_ *sfu.WebRTCReceiver, stat *livekit.AnalyticsStat) {
+				key := telemetry.StatsKeyForTrack(livekit.StreamType_UPSTREAM, t.PublisherID(), t.ID(), ti.Source, ti.Type)
+				t.params.Telemetry.TrackStats(key, stat)
+			})
+
+			newWR.OnMaxLayerChange(t.onMaxLayerChange)
+		}
+		if t.PrimaryReceiver() == nil {
+			// primary codec published, set potential codecs
+			potentialCodecs := make([]webrtc.RTPCodecParameters, 0, len(ti.Codecs))
+			parameters := receiver.GetParameters()
+			for _, c := range ti.Codecs {
+				for _, nc := range parameters.Codecs {
+					if mime.IsMimeTypeStringEqual(nc.MimeType, c.MimeType) {
+						potentialCodecs = append(potentialCodecs, nc)
+						break
+					}
+				}
+			}
+
+			if len(potentialCodecs) > 0 {
+				t.params.Logger.Debugw("primary codec published, set potential codecs", "potential", potentialCodecs)
+				t.MediaTrackReceiver.SetPotentialCodecs(potentialCodecs, parameters.HeaderExtensions)
+			}
+		}
+
+		t.buffer = buff
+
+		t.MediaTrackReceiver.SetupReceiver(newWR, priority, mid)
+
+		for ssrc, info := range t.params.SimTracks {
+			if info.Mid == mid {
+				t.MediaTrackReceiver.SetLayerSsrc(mimeType, info.Rid, ssrc)
+			}
+		}
+		wr = newWR
+		newCodec = true
+
+		newWR.AddOnCodecStateChange(func(codec webrtc.RTPCodecParameters, state sfu.ReceiverCodecState) {
+			t.MediaTrackReceiver.HandleReceiverCodecChange(newWR, codec, state)
+		})
+	}
+
+	if newCodec && t.enableRegression {
+		if mimeType == t.regressionTargetCodec {
+			t.params.Logger.Infow("regression target codec received", "codec", mimeType)
+			t.regressionTargetCodecReceived = true
+			regressCodec = true
+		} else if t.regressionTargetCodecReceived {
+			regressCodec = true
+		}
+	}
+	t.lock.Unlock()
+
+	if err := wr.(*sfu.WebRTCReceiver).AddUpTrack(track, buff); err != nil {
+		t.params.Logger.Warnw(
+			"adding up track failed", err,
+			"rid", track.RID(),
+			"layer", layer,
+			"ssrc", track.SSRC(),
+			"newCodec", newCodec,
+		)
+		buff.Close()
+		return false
+	}
+
+	// LK-TODO: can remove this completely when VideoLayers protocol becomes the default as it has info from client or if we decide to use TrackInfo.Simulcast
+	if t.numUpTracks.Inc() > 1 || track.RID() != "" {
+		// cannot only rely on numUpTracks since we fire metadata events immediately after the first layer
+		t.SetSimulcast(true)
+	}
+
+	var bitrates int
+	if len(ti.Layers) > int(layer) {
+		bitrates = int(ti.Layers[layer].GetBitrate())
+	}
+
+	t.MediaTrackReceiver.SetLayerSsrc(mimeType, track.RID(), uint32(track.SSRC()))
+
+	if regressCodec {
+		for _, c := range ti.Codecs {
+			if mime.NormalizeMimeType(c.MimeType) == t.regressionTargetCodec {
+				continue
+			}
+
+			t.params.Logger.Debugw("suspending codec for codec regression", "codec", c.MimeType)
+			if r := t.MediaTrackReceiver.Receiver(mime.NormalizeMimeType(c.MimeType)); r != nil {
+				if rtcreceiver, ok := r.(*sfu.WebRTCReceiver); ok {
+					rtcreceiver.SetCodecState(sfu.ReceiverCodecStateSuspended)
+				}
+			}
+		}
+	}
+
+	buff.Bind(receiver.GetParameters(), track.Codec().RTPCodecCapability, bitrates)
+
+	// if subscriber request fps before fps calculated, update them after fps updated.
+	buff.OnFpsChanged(func() {
+		t.MediaTrackSubscriptions.UpdateVideoLayers()
+	})
+
+	buff.OnFinalRtpStats(func(stats *livekit.RTPStats) {
+		t.params.Telemetry.TrackPublishRTPStats(
+			context.Background(),
+			t.params.ParticipantID,
+			t.ID(),
+			mimeType,
+			int(layer),
+			stats,
+		)
+	})
+	return newCodec
+}
+
+func (t *MediaTrack) GetConnectionScoreAndQuality() (float32, livekit.ConnectionQuality) {
+	receiver := t.PrimaryReceiver()
+	if rtcReceiver, ok := receiver.(*sfu.WebRTCReceiver); ok {
+		return rtcReceiver.GetConnectionScoreAndQuality()
+	}
+
+	return connectionquality.MaxMOS, livekit.ConnectionQuality_EXCELLENT
+}
+
+func (t *MediaTrack) SetRTT(rtt uint32) {
+	if !t.rttFromXR.Load() {
+		t.MediaTrackReceiver.SetRTT(rtt)
+	}
+}
+
+func (t *MediaTrack) HasPendingCodec() bool {
+	return t.MediaTrackReceiver.PrimaryReceiver() == nil
+}
+
+func (t *MediaTrack) onMaxLayerChange(maxLayer int32) {
+	t.MediaTrackReceiver.NotifyMaxLayerChange(maxLayer)
+}
+
+func (t *MediaTrack) Restart() {
+	t.MediaTrackReceiver.Restart()
+
+	if t.dynacastManager != nil {
+		t.dynacastManager.Restart()
+	}
+}
+
+func (t *MediaTrack) Close(isExpectedToResume bool) {
+	t.MediaTrackReceiver.SetClosing()
+	if t.dynacastManager != nil {
+		t.dynacastManager.Close()
+	}
+	t.MediaTrackReceiver.ClearAllReceivers(isExpectedToResume)
+	t.MediaTrackReceiver.Close(isExpectedToResume)
+}
+
+func (t *MediaTrack) SetMuted(muted bool) {
+	// update quality based on subscription if unmuting.
+	// This will queue up the current state, but subscriber
+	// driven changes could update it.
+	if !muted && t.dynacastManager != nil {
+		t.dynacastManager.ForceUpdate()
+	}
+
+	t.MediaTrackReceiver.SetMuted(muted)
+}
+
+// OnTrackSubscribed is called when the track is subscribed by a non-hidden subscriber
+// this allows the publisher to know when they should start sending data
+func (t *MediaTrack) OnTrackSubscribed() {
+	if !t.everSubscribed.Swap(true) && t.params.OnTrackEverSubscribed != nil {
+		go t.params.OnTrackEverSubscribed(t.ID())
+	}
+}

livekit-server/pkg/rtc/mediatrack_test.go

@@ -0,0 +1,174 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package rtc
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/livekit/protocol/livekit"
+)
+
+func TestTrackInfo(t *testing.T) {
+	// ensures that persisted trackinfo is being returned
+	ti := livekit.TrackInfo{
+		Sid:       "testsid",
+		Name:      "testtrack",
+		Source:    livekit.TrackSource_SCREEN_SHARE,
+		Type:      livekit.TrackType_VIDEO,
+		Simulcast: false,
+		Width:     100,
+		Height:    80,
+		Muted:     true,
+	}
+
+	mt := NewMediaTrack(MediaTrackParams{}, &ti)
+	outInfo := mt.ToProto()
+	require.Equal(t, ti.Muted, outInfo.Muted)
+	require.Equal(t, ti.Name, outInfo.Name)
+	require.Equal(t, ti.Name, mt.Name())
+	require.Equal(t, livekit.TrackID(ti.Sid), mt.ID())
+	require.Equal(t, ti.Type, outInfo.Type)
+	require.Equal(t, ti.Type, mt.Kind())
+	require.Equal(t, ti.Source, outInfo.Source)
+	require.Equal(t, ti.Width, outInfo.Width)
+	require.Equal(t, ti.Height, outInfo.Height)
+	require.Equal(t, ti.Simulcast, outInfo.Simulcast)
+
+	// make it simulcasted
+	mt.SetSimulcast(true)
+	require.True(t, mt.ToProto().Simulcast)
+}
+
+func TestGetQualityForDimension(t *testing.T) {
+	t.Run("landscape source", func(t *testing.T) {
+		mt := NewMediaTrack(MediaTrackParams{}, &livekit.TrackInfo{
+			Type:   livekit.TrackType_VIDEO,
+			Width:  1080,
+			Height: 720,
+		})
+
+		require.Equal(t, livekit.VideoQuality_LOW, mt.GetQualityForDimension(120, 120))
+		require.Equal(t, livekit.VideoQuality_LOW, mt.GetQualityForDimension(300, 200))
+		require.Equal(t, livekit.VideoQuality_MEDIUM, mt.GetQualityForDimension(200, 250))
+		require.Equal(t, livekit.VideoQuality_HIGH, mt.GetQualityForDimension(700, 480))
+		require.Equal(t, livekit.VideoQuality_HIGH, mt.GetQualityForDimension(500, 1000))
+	})
+
+	t.Run("portrait source", func(t *testing.T) {
+		mt := NewMediaTrack(MediaTrackParams{}, &livekit.TrackInfo{
+			Type:   livekit.TrackType_VIDEO,
+			Width:  540,
+			Height: 960,
+		})
+
+		require.Equal(t, livekit.VideoQuality_LOW, mt.GetQualityForDimension(200, 400))
+		require.Equal(t, livekit.VideoQuality_MEDIUM, mt.GetQualityForDimension(400, 400))
+		require.Equal(t, livekit.VideoQuality_MEDIUM, mt.GetQualityForDimension(400, 700))
+		require.Equal(t, livekit.VideoQuality_HIGH, mt.GetQualityForDimension(600, 900))
+	})
+
+	t.Run("layers provided", func(t *testing.T) {
+		mt := NewMediaTrack(MediaTrackParams{}, &livekit.TrackInfo{
+			Type:   livekit.TrackType_VIDEO,
+			Width:  1080,
+			Height: 720,
+			Layers: []*livekit.VideoLayer{
+				{
+					Quality: livekit.VideoQuality_LOW,
+					Width:   480,
+					Height:  270,
+				},
+				{
+					Quality: livekit.VideoQuality_MEDIUM,
+					Width:   960,
+					Height:  540,
+				},
+				{
+					Quality: livekit.VideoQuality_HIGH,
+					Width:   1080,
+					Height:  720,
+				},
+			},
+		})
+
+		require.Equal(t, livekit.VideoQuality_LOW, mt.GetQualityForDimension(120, 120))
+		require.Equal(t, livekit.VideoQuality_LOW, mt.GetQualityForDimension(300, 300))
+		require.Equal(t, livekit.VideoQuality_MEDIUM, mt.GetQualityForDimension(800, 500))
+		require.Equal(t, livekit.VideoQuality_HIGH, mt.GetQualityForDimension(1000, 700))
+	})
+
+	t.Run("highest layer with smallest dimensions", func(t *testing.T) {
+		mt := NewMediaTrack(MediaTrackParams{}, &livekit.TrackInfo{
+			Type:   livekit.TrackType_VIDEO,
+			Width:  1080,
+			Height: 720,
+			Layers: []*livekit.VideoLayer{
+				{
+					Quality: livekit.VideoQuality_LOW,
+					Width:   480,
+					Height:  270,
+				},
+				{
+					Quality: livekit.VideoQuality_MEDIUM,
+					Width:   1080,
+					Height:  720,
+				},
+				{
+					Quality: livekit.VideoQuality_HIGH,
+					Width:   1080,
+					Height:  720,
+				},
+			},
+		})
+
+		require.Equal(t, livekit.VideoQuality_LOW, mt.GetQualityForDimension(120, 120))
+		require.Equal(t, livekit.VideoQuality_LOW, mt.GetQualityForDimension(300, 300))
+		require.Equal(t, livekit.VideoQuality_HIGH, mt.GetQualityForDimension(800, 500))
+		require.Equal(t, livekit.VideoQuality_HIGH, mt.GetQualityForDimension(1000, 700))
+		require.Equal(t, livekit.VideoQuality_HIGH, mt.GetQualityForDimension(1200, 800))
+
+		mt = NewMediaTrack(MediaTrackParams{}, &livekit.TrackInfo{
+			Type:   livekit.TrackType_VIDEO,
+			Width:  1080,
+			Height: 720,
+			Layers: []*livekit.VideoLayer{
+				{
+					Quality: livekit.VideoQuality_LOW,
+					Width:   480,
+					Height:  270,
+				},
+				{
+					Quality: livekit.VideoQuality_MEDIUM,
+					Width:   480,
+					Height:  270,
+				},
+				{
+					Quality: livekit.VideoQuality_HIGH,
+					Width:   1080,
+					Height:  720,
+				},
+			},
+		})
+
+		require.Equal(t, livekit.VideoQuality_MEDIUM, mt.GetQualityForDimension(120, 120))
+		require.Equal(t, livekit.VideoQuality_MEDIUM, mt.GetQualityForDimension(300, 300))
+		require.Equal(t, livekit.VideoQuality_HIGH, mt.GetQualityForDimension(800, 500))
+		require.Equal(t, livekit.VideoQuality_HIGH, mt.GetQualityForDimension(1000, 700))
+		require.Equal(t, livekit.VideoQuality_HIGH, mt.GetQualityForDimension(1200, 800))
+	})
+
+}

livekit-server/pkg/rtc/mediatracksubscriptions.go

@@ -0,0 +1,465 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package rtc
+
+import (
+	"errors"
+	"sync"
+
+	"github.com/pion/rtcp"
+	"github.com/pion/webrtc/v4"
+	"go.uber.org/atomic"
+
+	"github.com/livekit/livekit-server/pkg/sfu/buffer"
+	"github.com/livekit/livekit-server/pkg/sfu/mime"
+	sutils "github.com/livekit/livekit-server/pkg/utils"
+	"github.com/livekit/protocol/livekit"
+	"github.com/livekit/protocol/logger"
+
+	"github.com/livekit/livekit-server/pkg/rtc/types"
+	"github.com/livekit/livekit-server/pkg/sfu"
+	"github.com/livekit/livekit-server/pkg/telemetry"
+)
+
+var (
+	errAlreadySubscribed = errors.New("already subscribed")
+	errNotFound          = errors.New("not found")
+)
+
+// MediaTrackSubscriptions manages subscriptions of a media track
+type MediaTrackSubscriptions struct {
+	params MediaTrackSubscriptionsParams
+
+	subscribedTracksMu sync.RWMutex
+	subscribedTracks   map[livekit.ParticipantID]types.SubscribedTrack
+
+	onDownTrackCreated           func(downTrack *sfu.DownTrack)
+	onSubscriberMaxQualityChange func(subscriberID livekit.ParticipantID, mime mime.MimeType, layer int32)
+}
+
+type MediaTrackSubscriptionsParams struct {
+	MediaTrack types.MediaTrack
+	IsRelayed  bool
+
+	ReceiverConfig   ReceiverConfig
+	SubscriberConfig DirectionConfig
+
+	Telemetry telemetry.TelemetryService
+
+	Logger logger.Logger
+}
+
+func NewMediaTrackSubscriptions(params MediaTrackSubscriptionsParams) *MediaTrackSubscriptions {
+	return &MediaTrackSubscriptions{
+		params:           params,
+		subscribedTracks: make(map[livekit.ParticipantID]types.SubscribedTrack),
+	}
+}
+
+func (t *MediaTrackSubscriptions) OnDownTrackCreated(f func(downTrack *sfu.DownTrack)) {
+	t.onDownTrackCreated = f
+}
+
+func (t *MediaTrackSubscriptions) OnSubscriberMaxQualityChange(f func(subscriberID livekit.ParticipantID, mime mime.MimeType, layer int32)) {
+	t.onSubscriberMaxQualityChange = f
+}
+
+func (t *MediaTrackSubscriptions) SetMuted(muted bool) {
+	// update mute of all subscribed tracks
+	for _, st := range t.getAllSubscribedTracks() {
+		st.SetPublisherMuted(muted)
+	}
+}
+
+func (t *MediaTrackSubscriptions) IsSubscriber(subID livekit.ParticipantID) bool {
+	t.subscribedTracksMu.RLock()
+	defer t.subscribedTracksMu.RUnlock()
+
+	_, ok := t.subscribedTracks[subID]
+	return ok
+}
+
+// AddSubscriber subscribes sub to current mediaTrack
+func (t *MediaTrackSubscriptions) AddSubscriber(sub types.LocalParticipant, wr *WrappedReceiver) (types.SubscribedTrack, error) {
+	trackID := t.params.MediaTrack.ID()
+	subscriberID := sub.ID()
+
+	// don't subscribe to the same track multiple times
+	t.subscribedTracksMu.Lock()
+	if _, ok := t.subscribedTracks[subscriberID]; ok {
+		t.subscribedTracksMu.Unlock()
+		return nil, errAlreadySubscribed
+	}
+	t.subscribedTracksMu.Unlock()
+
+	var rtcpFeedback []webrtc.RTCPFeedback
+	var maxTrack int
+	switch t.params.MediaTrack.Kind() {
+	case livekit.TrackType_AUDIO:
+		rtcpFeedback = t.params.SubscriberConfig.RTCPFeedback.Audio
+		maxTrack = t.params.ReceiverConfig.PacketBufferSizeAudio
+	case livekit.TrackType_VIDEO:
+		rtcpFeedback = t.params.SubscriberConfig.RTCPFeedback.Video
+		maxTrack = t.params.ReceiverConfig.PacketBufferSizeVideo
+	}
+	codecs := wr.Codecs()
+	for _, c := range codecs {
+		c.RTCPFeedback = rtcpFeedback
+	}
+
+	streamID := wr.StreamID()
+	if sub.SupportsSyncStreamID() && t.params.MediaTrack.Stream() != "" {
+		streamID = PackSyncStreamID(t.params.MediaTrack.PublisherID(), t.params.MediaTrack.Stream())
+	}
+
+	var trailer []byte
+	if t.params.MediaTrack.IsEncrypted() {
+		trailer = sub.GetTrailer()
+	}
+
+	downTrack, err := sfu.NewDownTrack(sfu.DowntrackParams{
+		Codecs:                         codecs,
+		Source:                         t.params.MediaTrack.Source(),
+		Receiver:                       wr,
+		BufferFactory:                  sub.GetBufferFactory(),
+		SubID:                          subscriberID,
+		StreamID:                       streamID,
+		MaxTrack:                       maxTrack,
+		PlayoutDelayLimit:              sub.GetPlayoutDelayConfig(),
+		Pacer:                          sub.GetPacer(),
+		Trailer:                        trailer,
+		Logger:                         LoggerWithTrack(sub.GetLogger().WithComponent(sutils.ComponentSub), trackID, t.params.IsRelayed),
+		RTCPWriter:                     sub.WriteSubscriberRTCP,
+		DisableSenderReportPassThrough: sub.GetDisableSenderReportPassThrough(),
+		SupportsCodecChange:            sub.SupportsCodecChange(),
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	if t.onDownTrackCreated != nil {
+		t.onDownTrackCreated(downTrack)
+	}
+
+	subTrack := NewSubscribedTrack(SubscribedTrackParams{
+		PublisherID:       t.params.MediaTrack.PublisherID(),
+		PublisherIdentity: t.params.MediaTrack.PublisherIdentity(),
+		PublisherVersion:  t.params.MediaTrack.PublisherVersion(),
+		Subscriber:        sub,
+		MediaTrack:        t.params.MediaTrack,
+		DownTrack:         downTrack,
+		AdaptiveStream:    sub.GetAdaptiveStream(),
+	})
+
+	if !sub.Hidden() {
+		subTrack.AddOnBind(func(err error) {
+			if err == nil {
+				t.params.MediaTrack.OnTrackSubscribed()
+			}
+		})
+	}
+
+	// Bind callback can happen from replaceTrack, so set it up early
+	var reusingTransceiver atomic.Bool
+	var dtState sfu.DownTrackState
+	downTrack.OnCodecNegotiated(func(codec webrtc.RTPCodecCapability) {
+		if !wr.DetermineReceiver(codec) {
+			if t.onSubscriberMaxQualityChange != nil {
+				go func() {
+					spatial := buffer.VideoQualityToSpatialLayer(livekit.VideoQuality_HIGH, t.params.MediaTrack.ToProto())
+					t.onSubscriberMaxQualityChange(downTrack.SubscriberID(), mime.NormalizeMimeType(codec.MimeType), spatial)
+				}()
+			}
+		}
+	})
+	downTrack.OnBinding(func(err error) {
+		if err != nil {
+			go subTrack.Bound(err)
+			return
+		}
+		if reusingTransceiver.Load() {
+			sub.GetLogger().Debugw("seeding downtrack state", "trackID", trackID)
+			downTrack.SeedState(dtState)
+		}
+		if err = wr.AddDownTrack(downTrack); err != nil && err != sfu.ErrReceiverClosed {
+			sub.GetLogger().Errorw(
+				"could not add down track", err,
+				"publisher", subTrack.PublisherIdentity(),
+				"publisherID", subTrack.PublisherID(),
+				"trackID", trackID,
+			)
+		}
+
+		go subTrack.Bound(nil)
+
+		subTrack.SetPublisherMuted(t.params.MediaTrack.IsMuted())
+	})
+
+	downTrack.OnStatsUpdate(func(_ *sfu.DownTrack, stat *livekit.AnalyticsStat) {
+		key := telemetry.StatsKeyForTrack(livekit.StreamType_DOWNSTREAM, subscriberID, trackID, t.params.MediaTrack.Source(), t.params.MediaTrack.Kind())
+		t.params.Telemetry.TrackStats(key, stat)
+	})
+
+	downTrack.OnMaxLayerChanged(func(dt *sfu.DownTrack, layer int32) {
+		if t.onSubscriberMaxQualityChange != nil {
+			t.onSubscriberMaxQualityChange(dt.SubscriberID(), dt.Mime(), layer)
+		}
+	})
+
+	downTrack.OnRttUpdate(func(_ *sfu.DownTrack, rtt uint32) {
+		go sub.UpdateMediaRTT(rtt)
+	})
+
+	downTrack.AddReceiverReportListener(func(dt *sfu.DownTrack, report *rtcp.ReceiverReport) {
+		sub.HandleReceiverReport(dt, report)
+	})
+
+	var transceiver *webrtc.RTPTransceiver
+	var sender *webrtc.RTPSender
+
+	// try cached RTP senders for a chance to replace track
+	var existingTransceiver *webrtc.RTPTransceiver
+	replacedTrack := false
+	existingTransceiver, dtState = sub.GetCachedDownTrack(trackID)
+	if existingTransceiver != nil {
+		sub.GetLogger().Debugw(
+			"trying to use existing transceiver",
+			"publisher", subTrack.PublisherIdentity(),
+			"publisherID", subTrack.PublisherID(),
+			"trackID", trackID,
+		)
+		reusingTransceiver.Store(true)
+		rtpSender := existingTransceiver.Sender()
+		if rtpSender != nil {
+			// replaced track will bind immediately without negotiation, SetTransceiver first before bind
+			downTrack.SetTransceiver(existingTransceiver)
+			err := rtpSender.ReplaceTrack(downTrack)
+			if err == nil {
+				sender = rtpSender
+				transceiver = existingTransceiver
+				replacedTrack = true
+				sub.GetLogger().Debugw(
+					"track replaced",
+					"publisher", subTrack.PublisherIdentity(),
+					"publisherID", subTrack.PublisherID(),
+					"trackID", trackID,
+				)
+			}
+		}
+
+		if !replacedTrack {
+			// Could not re-use cached transceiver for this track.
+			// Stop the transceiver so that it is at least not active.
+			// It is not usable once stopped,
+			//
+			// Adding down track will create a new transceiver (or re-use
+			// an inactive existing one). In either case, a renegotiation
+			// will happen and that will notify remote of this stopped
+			// transceiver
+			existingTransceiver.Stop()
+			reusingTransceiver.Store(false)
+		}
+	}
+
+	// if cannot replace, find an unused transceiver or add new one
+	if transceiver == nil {
+		info := t.params.MediaTrack.ToProto()
+		addTrackParams := types.AddTrackParams{
+			Stereo: info.Stereo,
+			Red:    !info.DisableRed,
+		}
+		if addTrackParams.Red && (len(codecs) == 1 && mime.IsMimeTypeStringOpus(codecs[0].MimeType)) {
+			addTrackParams.Red = false
+		}
+
+		sub.VerifySubscribeParticipantInfo(subTrack.PublisherID(), subTrack.PublisherVersion())
+		if sub.SupportsTransceiverReuse() {
+			//
+			// AddTrack will create a new transceiver or re-use an unused one
+			// if the attributes match. This prevents SDP from bloating
+			// because of dormant transceivers building up.
+			//
+			sender, transceiver, err = sub.AddTrackLocal(downTrack, addTrackParams)
+			if err != nil {
+				return nil, err
+			}
+		} else {
+			sender, transceiver, err = sub.AddTransceiverFromTrackLocal(downTrack, addTrackParams)
+			if err != nil {
+				return nil, err
+			}
+		}
+	}
+
+	// whether re-using or stopping remove transceiver from cache
+	// NOTE: safety net, if somehow a cached transceiver is re-used by a different track
+	sub.UncacheDownTrack(transceiver)
+
+	// negotiation isn't required if we've replaced track
+	// ONE-SHOT-SIGNALLING-MODE: this should not be needed, but that mode information is not available here,
+	//   but it is not detrimental to set this, needs clean up when participants modes are separated out better.
+	subTrack.SetNeedsNegotiation(!replacedTrack)
+	subTrack.SetRTPSender(sender)
+
+	// it is possible that subscribed track is closed before subscription manager sets
+	// the `OnClose` callback. That handler in subscription manager removes the track
+	// from the peer connection.
+	//
+	// But, the subscription could be removed early if the published track is closed
+	// while adding subscription. In those cases, subscription manager would not have set
+	// the `OnClose` callback. So, set it here to handle cases of early close.
+	subTrack.OnClose(func(isExpectedToResume bool) {
+		if !isExpectedToResume {
+			if err := sub.RemoveTrackLocal(sender); err != nil {
+				t.params.Logger.Warnw("could not remove track from peer connection", err)
+			}
+		}
+	})
+
+	downTrack.SetTransceiver(transceiver)
+
+	downTrack.OnCloseHandler(func(isExpectedToResume bool) {
+		t.downTrackClosed(sub, subTrack, isExpectedToResume)
+	})
+
+	t.subscribedTracksMu.Lock()
+	t.subscribedTracks[subscriberID] = subTrack
+	t.subscribedTracksMu.Unlock()
+
+	return subTrack, nil
+}
+
+// RemoveSubscriber removes participant from subscription
+// stop all forwarders to the client
+func (t *MediaTrackSubscriptions) RemoveSubscriber(subscriberID livekit.ParticipantID, isExpectedToResume bool) error {
+	subTrack := t.getSubscribedTrack(subscriberID)
+	if subTrack == nil {
+		return errNotFound
+	}
+
+	t.params.Logger.Debugw("removing subscriber", "subscriberID", subscriberID, "isExpectedToResume", isExpectedToResume)
+	t.closeSubscribedTrack(subTrack, isExpectedToResume)
+	return nil
+}
+
+func (t *MediaTrackSubscriptions) closeSubscribedTrack(subTrack types.SubscribedTrack, isExpectedToResume bool) {
+	dt := subTrack.DownTrack()
+	if dt == nil {
+		return
+	}
+
+	if isExpectedToResume {
+		dt.CloseWithFlush(false)
+	} else {
+		// flushing blocks, avoid blocking when publisher removes all its subscribers
+		go dt.CloseWithFlush(true)
+	}
+}
+
+func (t *MediaTrackSubscriptions) GetAllSubscribers() []livekit.ParticipantID {
+	t.subscribedTracksMu.RLock()
+	defer t.subscribedTracksMu.RUnlock()
+
+	subs := make([]livekit.ParticipantID, 0, len(t.subscribedTracks))
+	for id := range t.subscribedTracks {
+		subs = append(subs, id)
+	}
+	return subs
+}
+
+func (t *MediaTrackSubscriptions) GetAllSubscribersForMime(mime mime.MimeType) []livekit.ParticipantID {
+	t.subscribedTracksMu.RLock()
+	defer t.subscribedTracksMu.RUnlock()
+
+	subs := make([]livekit.ParticipantID, 0, len(t.subscribedTracks))
+	for id, subTrack := range t.subscribedTracks {
+		if subTrack.DownTrack().Mime() != mime {
+			continue
+		}
+
+		subs = append(subs, id)
+	}
+	return subs
+}
+
+func (t *MediaTrackSubscriptions) GetNumSubscribers() int {
+	t.subscribedTracksMu.RLock()
+	defer t.subscribedTracksMu.RUnlock()
+
+	return len(t.subscribedTracks)
+}
+
+func (t *MediaTrackSubscriptions) UpdateVideoLayers() {
+	for _, st := range t.getAllSubscribedTracks() {
+		st.UpdateVideoLayer()
+	}
+}
+
+func (t *MediaTrackSubscriptions) getSubscribedTrack(subscriberID livekit.ParticipantID) types.SubscribedTrack {
+	t.subscribedTracksMu.RLock()
+	defer t.subscribedTracksMu.RUnlock()
+
+	return t.subscribedTracks[subscriberID]
+}
+
+func (t *MediaTrackSubscriptions) getAllSubscribedTracks() []types.SubscribedTrack {
+	t.subscribedTracksMu.RLock()
+	defer t.subscribedTracksMu.RUnlock()
+
+	return t.getAllSubscribedTracksLocked()
+}
+
+func (t *MediaTrackSubscriptions) getAllSubscribedTracksLocked() []types.SubscribedTrack {
+	subTracks := make([]types.SubscribedTrack, 0, len(t.subscribedTracks))
+	for _, subTrack := range t.subscribedTracks {
+		subTracks = append(subTracks, subTrack)
+	}
+	return subTracks
+}
+
+func (t *MediaTrackSubscriptions) DebugInfo() []map[string]interface{} {
+	subscribedTrackInfo := make([]map[string]interface{}, 0)
+	for _, val := range t.getAllSubscribedTracks() {
+		if st, ok := val.(*SubscribedTrack); ok {
+			subscribedTrackInfo = append(subscribedTrackInfo, st.DownTrack().DebugInfo())
+		}
+	}
+
+	return subscribedTrackInfo
+}
+
+func (t *MediaTrackSubscriptions) downTrackClosed(
+	sub types.LocalParticipant,
+	subTrack types.SubscribedTrack,
+	isExpectedToResume bool,
+) {
+	// Cache transceiver for potential re-use on resume.
+	// To ensure subscription manager does not re-subscribe before caching,
+	// delete the subscribed track only after caching.
+	if isExpectedToResume {
+		dt := subTrack.DownTrack()
+		tr := dt.GetTransceiver()
+		if tr != nil {
+			sub.CacheDownTrack(subTrack.ID(), tr, dt.GetState())
+		}
+	}
+
+	go func() {
+		t.subscribedTracksMu.Lock()
+		delete(t.subscribedTracks, sub.ID())
+		t.subscribedTracksMu.Unlock()
+		subTrack.Close(isExpectedToResume)
+	}()
+}

livekit-server/pkg/rtc/participant_internal_test.go

@@ -0,0 +1,716 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package rtc
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/pion/webrtc/v4"
+	"github.com/stretchr/testify/require"
+	"go.uber.org/atomic"
+	"google.golang.org/protobuf/proto"
+
+	"github.com/livekit/livekit-server/pkg/sfu/buffer"
+	"github.com/livekit/livekit-server/pkg/sfu/mime"
+	"github.com/livekit/livekit-server/pkg/telemetry/telemetryfakes"
+	"github.com/livekit/protocol/auth"
+	"github.com/livekit/protocol/livekit"
+	"github.com/livekit/protocol/logger"
+	"github.com/livekit/protocol/utils"
+	"github.com/livekit/protocol/utils/guid"
+
+	"github.com/livekit/livekit-server/pkg/config"
+	"github.com/livekit/livekit-server/pkg/routing"
+	"github.com/livekit/livekit-server/pkg/routing/routingfakes"
+	"github.com/livekit/livekit-server/pkg/rtc/types"
+	"github.com/livekit/livekit-server/pkg/rtc/types/typesfakes"
+	"github.com/livekit/livekit-server/pkg/testutils"
+)
+
+func TestIsReady(t *testing.T) {
+	tests := []struct {
+		state livekit.ParticipantInfo_State
+		ready bool
+	}{
+		{
+			state: livekit.ParticipantInfo_JOINING,
+			ready: false,
+		},
+		{
+			state: livekit.ParticipantInfo_JOINED,
+			ready: true,
+		},
+		{
+			state: livekit.ParticipantInfo_ACTIVE,
+			ready: true,
+		},
+		{
+			state: livekit.ParticipantInfo_DISCONNECTED,
+			ready: false,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.state.String(), func(t *testing.T) {
+			p := &ParticipantImpl{}
+			p.state.Store(test.state)
+			require.Equal(t, test.ready, p.IsReady())
+		})
+	}
+}
+
+func TestTrackPublishing(t *testing.T) {
+	t.Run("should send the correct events", func(t *testing.T) {
+		p := newParticipantForTest("test")
+		track := &typesfakes.FakeMediaTrack{}
+		track.IDReturns("id")
+		published := false
+		updated := false
+		p.OnTrackUpdated(func(p types.LocalParticipant, track types.MediaTrack) {
+			updated = true
+		})
+		p.OnTrackPublished(func(p types.LocalParticipant, track types.MediaTrack) {
+			published = true
+		})
+		p.UpTrackManager.AddPublishedTrack(track)
+		p.handleTrackPublished(track)
+		require.True(t, published)
+		require.False(t, updated)
+		require.Len(t, p.UpTrackManager.publishedTracks, 1)
+	})
+
+	t.Run("sends back trackPublished event", func(t *testing.T) {
+		p := newParticipantForTest("test")
+		sink := p.params.Sink.(*routingfakes.FakeMessageSink)
+		p.AddTrack(&livekit.AddTrackRequest{
+			Cid:    "cid",
+			Name:   "webcam",
+			Type:   livekit.TrackType_VIDEO,
+			Width:  1024,
+			Height: 768,
+		})
+		require.Equal(t, 1, sink.WriteMessageCallCount())
+		res := sink.WriteMessageArgsForCall(0).(*livekit.SignalResponse)
+		require.IsType(t, &livekit.SignalResponse_TrackPublished{}, res.Message)
+		published := res.Message.(*livekit.SignalResponse_TrackPublished).TrackPublished
+		require.Equal(t, "cid", published.Cid)
+		require.Equal(t, "webcam", published.Track.Name)
+		require.Equal(t, livekit.TrackType_VIDEO, published.Track.Type)
+		require.Equal(t, uint32(1024), published.Track.Width)
+		require.Equal(t, uint32(768), published.Track.Height)
+	})
+
+	t.Run("should not allow adding of duplicate tracks", func(t *testing.T) {
+		p := newParticipantForTest("test")
+		sink := p.params.Sink.(*routingfakes.FakeMessageSink)
+		p.AddTrack(&livekit.AddTrackRequest{
+			Cid:  "cid",
+			Name: "webcam",
+			Type: livekit.TrackType_VIDEO,
+		})
+		p.AddTrack(&livekit.AddTrackRequest{
+			Cid:  "cid",
+			Name: "duplicate",
+			Type: livekit.TrackType_AUDIO,
+		})
+
+		require.Equal(t, 1, sink.WriteMessageCallCount())
+	})
+
+	t.Run("should queue adding of duplicate tracks if already published by client id in signalling", func(t *testing.T) {
+		p := newParticipantForTest("test")
+		sink := p.params.Sink.(*routingfakes.FakeMessageSink)
+
+		track := &typesfakes.FakeLocalMediaTrack{}
+		track.SignalCidReturns("cid")
+		track.ToProtoReturns(&livekit.TrackInfo{})
+		// directly add to publishedTracks without lock - for testing purpose only
+		p.UpTrackManager.publishedTracks["cid"] = track
+
+		p.AddTrack(&livekit.AddTrackRequest{
+			Cid:  "cid",
+			Name: "webcam",
+			Type: livekit.TrackType_VIDEO,
+		})
+		require.Equal(t, 0, sink.WriteMessageCallCount())
+		require.Equal(t, 1, len(p.pendingTracks["cid"].trackInfos))
+
+		// add again - it should be added to the queue
+		p.AddTrack(&livekit.AddTrackRequest{
+			Cid:  "cid",
+			Name: "webcam",
+			Type: livekit.TrackType_VIDEO,
+		})
+		require.Equal(t, 0, sink.WriteMessageCallCount())
+		require.Equal(t, 2, len(p.pendingTracks["cid"].trackInfos))
+
+		// check SID is the same
+		require.Equal(t, p.pendingTracks["cid"].trackInfos[0].Sid, p.pendingTracks["cid"].trackInfos[1].Sid)
+	})
+
+	t.Run("should queue adding of duplicate tracks if already published by client id in sdp", func(t *testing.T) {
+		p := newParticipantForTest("test")
+		sink := p.params.Sink.(*routingfakes.FakeMessageSink)
+
+		track := &typesfakes.FakeLocalMediaTrack{}
+		track.ToProtoReturns(&livekit.TrackInfo{})
+		track.HasSdpCidCalls(func(s string) bool { return s == "cid" })
+		// directly add to publishedTracks without lock - for testing purpose only
+		p.UpTrackManager.publishedTracks["cid"] = track
+
+		p.AddTrack(&livekit.AddTrackRequest{
+			Cid:  "cid",
+			Name: "webcam",
+			Type: livekit.TrackType_VIDEO,
+		})
+		require.Equal(t, 0, sink.WriteMessageCallCount())
+		require.Equal(t, 1, len(p.pendingTracks["cid"].trackInfos))
+
+		// add again - it should be added to the queue
+		p.AddTrack(&livekit.AddTrackRequest{
+			Cid:  "cid",
+			Name: "webcam",
+			Type: livekit.TrackType_VIDEO,
+		})
+		require.Equal(t, 0, sink.WriteMessageCallCount())
+		require.Equal(t, 2, len(p.pendingTracks["cid"].trackInfos))
+
+		// check SID is the same
+		require.Equal(t, p.pendingTracks["cid"].trackInfos[0].Sid, p.pendingTracks["cid"].trackInfos[1].Sid)
+	})
+
+	t.Run("should not allow adding disallowed sources", func(t *testing.T) {
+		p := newParticipantForTest("test")
+		p.SetPermission(&livekit.ParticipantPermission{
+			CanPublish: true,
+			CanPublishSources: []livekit.TrackSource{
+				livekit.TrackSource_CAMERA,
+			},
+		})
+		sink := p.params.Sink.(*routingfakes.FakeMessageSink)
+		p.AddTrack(&livekit.AddTrackRequest{
+			Cid:    "cid",
+			Name:   "webcam",
+			Source: livekit.TrackSource_CAMERA,
+			Type:   livekit.TrackType_VIDEO,
+		})
+		require.Equal(t, 1, sink.WriteMessageCallCount())
+
+		p.AddTrack(&livekit.AddTrackRequest{
+			Cid:    "cid2",
+			Name:   "rejected source",
+			Type:   livekit.TrackType_AUDIO,
+			Source: livekit.TrackSource_MICROPHONE,
+		})
+		require.Equal(t, 1, sink.WriteMessageCallCount())
+	})
+}
+
+func TestOutOfOrderUpdates(t *testing.T) {
+	p := newParticipantForTest("test")
+	p.updateState(livekit.ParticipantInfo_JOINED)
+	p.SetMetadata("initial metadata")
+	sink := p.getResponseSink().(*routingfakes.FakeMessageSink)
+	pi1 := p.ToProto()
+	p.SetMetadata("second update")
+	pi2 := p.ToProto()
+
+	require.Greater(t, pi2.Version, pi1.Version)
+
+	// send the second update first
+	require.NoError(t, p.SendParticipantUpdate([]*livekit.ParticipantInfo{pi2}))
+	require.NoError(t, p.SendParticipantUpdate([]*livekit.ParticipantInfo{pi1}))
+
+	// only sent once, and it's the earlier message
+	require.Equal(t, 1, sink.WriteMessageCallCount())
+	sent := sink.WriteMessageArgsForCall(0).(*livekit.SignalResponse)
+	require.Equal(t, "second update", sent.GetUpdate().Participants[0].Metadata)
+}
+
+// after disconnection, things should continue to function and not panic
+func TestDisconnectTiming(t *testing.T) {
+	t.Run("Negotiate doesn't panic after channel closed", func(t *testing.T) {
+		p := newParticipantForTest("test")
+		msg := routing.NewMessageChannel(livekit.ConnectionID("test"), routing.DefaultMessageChannelSize)
+		p.params.Sink = msg
+		go func() {
+			for msg := range msg.ReadChan() {
+				t.Log("received message from chan", msg)
+			}
+		}()
+		track := &typesfakes.FakeMediaTrack{}
+		p.UpTrackManager.AddPublishedTrack(track)
+		p.handleTrackPublished(track)
+
+		// close channel and then try to Negotiate
+		msg.Close()
+	})
+}
+
+func TestCorrectJoinedAt(t *testing.T) {
+	p := newParticipantForTest("test")
+	info := p.ToProto()
+	require.NotZero(t, info.JoinedAt)
+	require.True(t, time.Now().Unix()-info.JoinedAt <= 1)
+}
+
+func TestMuteSetting(t *testing.T) {
+	t.Run("can set mute when track is pending", func(t *testing.T) {
+		p := newParticipantForTest("test")
+		ti := &livekit.TrackInfo{Sid: "testTrack"}
+		p.pendingTracks["cid"] = &pendingTrackInfo{trackInfos: []*livekit.TrackInfo{ti}}
+
+		p.SetTrackMuted(livekit.TrackID(ti.Sid), true, false)
+		require.True(t, p.pendingTracks["cid"].trackInfos[0].Muted)
+	})
+
+	t.Run("can publish a muted track", func(t *testing.T) {
+		p := newParticipantForTest("test")
+		p.AddTrack(&livekit.AddTrackRequest{
+			Cid:   "cid",
+			Type:  livekit.TrackType_AUDIO,
+			Muted: true,
+		})
+
+		_, ti, _, _ := p.getPendingTrack("cid", livekit.TrackType_AUDIO, false)
+		require.NotNil(t, ti)
+		require.True(t, ti.Muted)
+	})
+}
+
+func TestSubscriberAsPrimary(t *testing.T) {
+	t.Run("protocol 4 uses subs as primary", func(t *testing.T) {
+		p := newParticipantForTestWithOpts("test", &participantOpts{
+			permissions: &livekit.ParticipantPermission{
+				CanSubscribe: true,
+				CanPublish:   true,
+			},
+		})
+		require.True(t, p.SubscriberAsPrimary())
+	})
+
+	t.Run("protocol 2 uses pub as primary", func(t *testing.T) {
+		p := newParticipantForTestWithOpts("test", &participantOpts{
+			protocolVersion: 2,
+			permissions: &livekit.ParticipantPermission{
+				CanSubscribe: true,
+				CanPublish:   true,
+			},
+		})
+		require.False(t, p.SubscriberAsPrimary())
+	})
+
+	t.Run("publisher only uses pub as primary", func(t *testing.T) {
+		p := newParticipantForTestWithOpts("test", &participantOpts{
+			permissions: &livekit.ParticipantPermission{
+				CanSubscribe: false,
+				CanPublish:   true,
+			},
+		})
+		require.False(t, p.SubscriberAsPrimary())
+
+		// ensure that it doesn't change after perms
+		p.SetPermission(&livekit.ParticipantPermission{
+			CanSubscribe: true,
+			CanPublish:   true,
+		})
+		require.False(t, p.SubscriberAsPrimary())
+	})
+}
+
+func TestDisableCodecs(t *testing.T) {
+	participant := newParticipantForTestWithOpts("123", &participantOpts{
+		publisher: false,
+		clientConf: &livekit.ClientConfiguration{
+			DisabledCodecs: &livekit.DisabledCodecs{
+				Codecs: []*livekit.Codec{
+					{Mime: "video/h264"},
+				},
+			},
+		},
+	})
+
+	participant.SetMigrateState(types.MigrateStateComplete)
+
+	pc, err := webrtc.NewPeerConnection(webrtc.Configuration{})
+	require.NoError(t, err)
+	transceiver, err := pc.AddTransceiverFromKind(webrtc.RTPCodecTypeVideo, webrtc.RTPTransceiverInit{Direction: webrtc.RTPTransceiverDirectionSendrecv})
+	require.NoError(t, err)
+	sdp, err := pc.CreateOffer(nil)
+	require.NoError(t, err)
+	pc.SetLocalDescription(sdp)
+	codecs := transceiver.Receiver().GetParameters().Codecs
+	var found264 bool
+	for _, c := range codecs {
+		if mime.IsMimeTypeStringH264(c.MimeType) {
+			found264 = true
+		}
+	}
+	require.True(t, found264)
+
+	// negotiated codec should not contain h264
+	sink := &routingfakes.FakeMessageSink{}
+	participant.SetResponseSink(sink)
+	var answer webrtc.SessionDescription
+	var answerReceived atomic.Bool
+	sink.WriteMessageCalls(func(msg proto.Message) error {
+		if res, ok := msg.(*livekit.SignalResponse); ok {
+			if res.GetAnswer() != nil {
+				answer = FromProtoSessionDescription(res.GetAnswer())
+				answerReceived.Store(true)
+			}
+		}
+		return nil
+	})
+	participant.HandleOffer(sdp)
+
+	testutils.WithTimeout(t, func() string {
+		if answerReceived.Load() {
+			return ""
+		} else {
+			return "answer not received"
+		}
+	})
+	require.NoError(t, pc.SetRemoteDescription(answer), answer.SDP, sdp.SDP)
+
+	codecs = transceiver.Receiver().GetParameters().Codecs
+	found264 = false
+	for _, c := range codecs {
+		if mime.IsMimeTypeStringH264(c.MimeType) {
+			found264 = true
+		}
+	}
+	require.False(t, found264)
+}
+
+func TestDisablePublishCodec(t *testing.T) {
+	participant := newParticipantForTestWithOpts("123", &participantOpts{
+		publisher: true,
+		clientConf: &livekit.ClientConfiguration{
+			DisabledCodecs: &livekit.DisabledCodecs{
+				Publish: []*livekit.Codec{
+					{Mime: "video/h264"},
+				},
+			},
+		},
+	})
+
+	for _, codec := range participant.enabledPublishCodecs {
+		require.False(t, mime.IsMimeTypeStringH264(codec.Mime))
+	}
+
+	sink := &routingfakes.FakeMessageSink{}
+	participant.SetResponseSink(sink)
+	var publishReceived atomic.Bool
+	sink.WriteMessageCalls(func(msg proto.Message) error {
+		if res, ok := msg.(*livekit.SignalResponse); ok {
+			if published := res.GetTrackPublished(); published != nil {
+				publishReceived.Store(true)
+				require.NotEmpty(t, published.Track.Codecs)
+				require.True(t, mime.IsMimeTypeStringVP8(published.Track.Codecs[0].MimeType))
+			}
+		}
+		return nil
+	})
+
+	// simulcast codec response should pick an alternative
+	participant.AddTrack(&livekit.AddTrackRequest{
+		Cid:  "cid1",
+		Type: livekit.TrackType_VIDEO,
+		SimulcastCodecs: []*livekit.SimulcastCodec{{
+			Codec: "h264",
+			Cid:   "cid1",
+		}},
+	})
+
+	require.Eventually(t, func() bool { return publishReceived.Load() }, 5*time.Second, 10*time.Millisecond)
+
+	// publishing a supported codec should not change
+	publishReceived.Store(false)
+	sink.WriteMessageCalls(func(msg proto.Message) error {
+		if res, ok := msg.(*livekit.SignalResponse); ok {
+			if published := res.GetTrackPublished(); published != nil {
+				publishReceived.Store(true)
+				require.NotEmpty(t, published.Track.Codecs)
+				require.True(t, mime.IsMimeTypeStringVP8(published.Track.Codecs[0].MimeType))
+			}
+		}
+		return nil
+	})
+	participant.AddTrack(&livekit.AddTrackRequest{
+		Cid:  "cid2",
+		Type: livekit.TrackType_VIDEO,
+		SimulcastCodecs: []*livekit.SimulcastCodec{{
+			Codec: "vp8",
+			Cid:   "cid2",
+		}},
+	})
+	require.Eventually(t, func() bool { return publishReceived.Load() }, 5*time.Second, 10*time.Millisecond)
+}
+
+func TestPreferVideoCodecForPublisher(t *testing.T) {
+	participant := newParticipantForTestWithOpts("123", &participantOpts{
+		publisher: true,
+	})
+	participant.SetMigrateState(types.MigrateStateComplete)
+
+	pc, err := webrtc.NewPeerConnection(webrtc.Configuration{})
+	require.NoError(t, err)
+	defer pc.Close()
+
+	for i := 0; i < 2; i++ {
+		// publish h264 track without client preferred codec
+		trackCid := fmt.Sprintf("preferh264video%d", i)
+		participant.AddTrack(&livekit.AddTrackRequest{
+			Type:   livekit.TrackType_VIDEO,
+			Name:   "video",
+			Width:  1280,
+			Height: 720,
+			Source: livekit.TrackSource_CAMERA,
+			SimulcastCodecs: []*livekit.SimulcastCodec{
+				{
+					Codec: "h264",
+					Cid:   trackCid,
+				},
+			},
+		})
+
+		track, err := webrtc.NewTrackLocalStaticRTP(webrtc.RTPCodecCapability{MimeType: "video/vp8"}, trackCid, trackCid)
+		require.NoError(t, err)
+		transceiver, err := pc.AddTransceiverFromTrack(track, webrtc.RTPTransceiverInit{Direction: webrtc.RTPTransceiverDirectionSendrecv})
+		require.NoError(t, err)
+		codecs := transceiver.Receiver().GetParameters().Codecs
+
+		if i > 0 {
+			// the negotiated codecs order could be updated by first negotiation, reorder to make h264 not preferred
+			for mime.IsMimeTypeStringH264(codecs[0].MimeType) {
+				codecs = append(codecs[1:], codecs[0])
+			}
+		}
+		// h264 should not be preferred
+		require.False(t, mime.IsMimeTypeStringH264(codecs[0].MimeType), "codecs", codecs)
+
+		sdp, err := pc.CreateOffer(nil)
+		require.NoError(t, err)
+		require.NoError(t, pc.SetLocalDescription(sdp))
+
+		sink := &routingfakes.FakeMessageSink{}
+		participant.SetResponseSink(sink)
+		var answer webrtc.SessionDescription
+		var answerReceived atomic.Bool
+		sink.WriteMessageCalls(func(msg proto.Message) error {
+			if res, ok := msg.(*livekit.SignalResponse); ok {
+				if res.GetAnswer() != nil {
+					answer = FromProtoSessionDescription(res.GetAnswer())
+					pc.SetRemoteDescription(answer)
+					answerReceived.Store(true)
+				}
+			}
+			return nil
+		})
+		participant.HandleOffer(sdp)
+
+		require.Eventually(t, func() bool { return answerReceived.Load() }, 5*time.Second, 10*time.Millisecond)
+
+		var h264Preferred bool
+		parsed, err := answer.Unmarshal()
+		require.NoError(t, err)
+		var videoSectionIndex int
+		for _, m := range parsed.MediaDescriptions {
+			if m.MediaName.Media == "video" {
+				if videoSectionIndex == i {
+					codecs, err := codecsFromMediaDescription(m)
+					require.NoError(t, err)
+					if mime.IsMimeTypeCodecStringH264(codecs[0].Name) {
+						h264Preferred = true
+						break
+					}
+				}
+				videoSectionIndex++
+			}
+		}
+
+		require.Truef(t, h264Preferred, "h264 should be preferred for video section %d, answer sdp: \n%s", i, answer.SDP)
+	}
+}
+
+func TestPreferAudioCodecForRed(t *testing.T) {
+	participant := newParticipantForTestWithOpts("123", &participantOpts{
+		publisher: true,
+	})
+	participant.SetMigrateState(types.MigrateStateComplete)
+
+	me := webrtc.MediaEngine{}
+	me.RegisterDefaultCodecs()
+	require.NoError(t, me.RegisterCodec(webrtc.RTPCodecParameters{
+		RTPCodecCapability: RedCodecCapability,
+		PayloadType:        63,
+	}, webrtc.RTPCodecTypeAudio))
+
+	api := webrtc.NewAPI(webrtc.WithMediaEngine(&me))
+	pc, err := api.NewPeerConnection(webrtc.Configuration{})
+	require.NoError(t, err)
+	defer pc.Close()
+
+	for i, disableRed := range []bool{false, true} {
+		t.Run(fmt.Sprintf("disableRed=%v", disableRed), func(t *testing.T) {
+			trackCid := fmt.Sprintf("audiotrack%d", i)
+			participant.AddTrack(&livekit.AddTrackRequest{
+				Type:       livekit.TrackType_AUDIO,
+				DisableRed: disableRed,
+				Cid:        trackCid,
+			})
+			track, err := webrtc.NewTrackLocalStaticRTP(webrtc.RTPCodecCapability{MimeType: "audio/opus"}, trackCid, trackCid)
+			require.NoError(t, err)
+			transceiver, err := pc.AddTransceiverFromTrack(track, webrtc.RTPTransceiverInit{Direction: webrtc.RTPTransceiverDirectionSendrecv})
+			require.NoError(t, err)
+			codecs := transceiver.Sender().GetParameters().Codecs
+			for i, c := range codecs {
+				if c.MimeType == "audio/opus" && i != 0 {
+					codecs[0], codecs[i] = codecs[i], codecs[0]
+					break
+				}
+			}
+			transceiver.SetCodecPreferences(codecs)
+			sdp, err := pc.CreateOffer(nil)
+			require.NoError(t, err)
+			pc.SetLocalDescription(sdp)
+			// opus should be preferred
+			require.Equal(t, codecs[0].MimeType, "audio/opus", sdp)
+
+			sink := &routingfakes.FakeMessageSink{}
+			participant.SetResponseSink(sink)
+			var answer webrtc.SessionDescription
+			var answerReceived atomic.Bool
+			sink.WriteMessageCalls(func(msg proto.Message) error {
+				if res, ok := msg.(*livekit.SignalResponse); ok {
+					if res.GetAnswer() != nil {
+						answer = FromProtoSessionDescription(res.GetAnswer())
+						pc.SetRemoteDescription(answer)
+						answerReceived.Store(true)
+					}
+				}
+				return nil
+			})
+			participant.HandleOffer(sdp)
+
+			require.Eventually(t, func() bool { return answerReceived.Load() }, 5*time.Second, 10*time.Millisecond)
+
+			var redPreferred bool
+			parsed, err := answer.Unmarshal()
+			require.NoError(t, err)
+			var audioSectionIndex int
+			for _, m := range parsed.MediaDescriptions {
+				if m.MediaName.Media == "audio" {
+					if audioSectionIndex == i {
+						codecs, err := codecsFromMediaDescription(m)
+						require.NoError(t, err)
+						// nack is always enabled. if red is preferred, server will not generate nack request
+						var nackEnabled bool
+						for _, c := range codecs {
+							if c.Name == "opus" {
+								for _, fb := range c.RTCPFeedback {
+									if strings.Contains(fb, "nack") {
+										nackEnabled = true
+										break
+									}
+								}
+							}
+						}
+						require.True(t, nackEnabled, "nack should be enabled for opus")
+
+						if mime.IsMimeTypeCodecStringRED(codecs[0].Name) {
+							redPreferred = true
+							break
+						}
+					}
+					audioSectionIndex++
+				}
+			}
+			require.Equalf(t, !disableRed, redPreferred, "offer : \n%s\nanswer sdp: \n%s", sdp, answer.SDP)
+		})
+	}
+}
+
+type participantOpts struct {
+	permissions     *livekit.ParticipantPermission
+	protocolVersion types.ProtocolVersion
+	publisher       bool
+	clientConf      *livekit.ClientConfiguration
+	clientInfo      *livekit.ClientInfo
+}
+
+func newParticipantForTestWithOpts(identity livekit.ParticipantIdentity, opts *participantOpts) *ParticipantImpl {
+	if opts == nil {
+		opts = &participantOpts{}
+	}
+	if opts.protocolVersion == 0 {
+		opts.protocolVersion = 6
+	}
+	conf, _ := config.NewConfig("", true, nil, nil)
+	// disable mux, it doesn't play too well with unit test
+	conf.RTC.TCPPort = 0
+	rtcConf, err := NewWebRTCConfig(conf)
+	if err != nil {
+		panic(err)
+	}
+	ff := buffer.NewFactoryOfBufferFactory(500, 200)
+	rtcConf.SetBufferFactory(ff.CreateBufferFactory())
+	grants := &auth.ClaimGrants{
+		Video: &auth.VideoGrant{},
+	}
+	if opts.permissions != nil {
+		grants.Video.SetCanPublish(opts.permissions.CanPublish)
+		grants.Video.SetCanPublishData(opts.permissions.CanPublishData)
+		grants.Video.SetCanSubscribe(opts.permissions.CanSubscribe)
+	}
+
+	enabledCodecs := make([]*livekit.Codec, 0, len(conf.Room.EnabledCodecs))
+	for _, c := range conf.Room.EnabledCodecs {
+		enabledCodecs = append(enabledCodecs, &livekit.Codec{
+			Mime:     c.Mime,
+			FmtpLine: c.FmtpLine,
+		})
+	}
+	sid := livekit.ParticipantID(guid.New(utils.ParticipantPrefix))
+	p, _ := NewParticipant(ParticipantParams{
+		SID:                    sid,
+		Identity:               identity,
+		Config:                 rtcConf,
+		Sink:                   &routingfakes.FakeMessageSink{},
+		ProtocolVersion:        opts.protocolVersion,
+		SessionStartTime:       time.Now(),
+		PLIThrottleConfig:      conf.RTC.PLIThrottle,
+		Grants:                 grants,
+		PublishEnabledCodecs:   enabledCodecs,
+		SubscribeEnabledCodecs: enabledCodecs,
+		ClientConf:             opts.clientConf,
+		ClientInfo:             ClientInfo{ClientInfo: opts.clientInfo},
+		Logger:                 LoggerWithParticipant(logger.GetLogger(), identity, sid, false),
+		Telemetry:              &telemetryfakes.FakeTelemetryService{},
+		VersionGenerator:       utils.NewDefaultTimedVersionGenerator(),
+	})
+	p.isPublisher.Store(opts.publisher)
+	p.updateState(livekit.ParticipantInfo_ACTIVE)
+
+	return p
+}
+
+func newParticipantForTest(identity livekit.ParticipantIdentity) *ParticipantImpl {
+	return newParticipantForTestWithOpts(identity, nil)
+}

livekit-server/pkg/rtc/participant_sdp.go

@@ -0,0 +1,274 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package rtc
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+
+	"github.com/pion/sdp/v3"
+	"github.com/pion/webrtc/v4"
+
+	"github.com/livekit/livekit-server/pkg/sfu/mime"
+	"github.com/livekit/protocol/livekit"
+	lksdp "github.com/livekit/protocol/sdp"
+)
+
+func (p *ParticipantImpl) setCodecPreferencesForPublisher(offer webrtc.SessionDescription) webrtc.SessionDescription {
+	offer = p.setCodecPreferencesOpusRedForPublisher(offer)
+	offer = p.setCodecPreferencesVideoForPublisher(offer)
+	return offer
+}
+
+func (p *ParticipantImpl) setCodecPreferencesOpusRedForPublisher(offer webrtc.SessionDescription) webrtc.SessionDescription {
+	parsed, unmatchAudios, err := p.TransportManager.GetUnmatchMediaForOffer(offer, "audio")
+	if err != nil || len(unmatchAudios) == 0 {
+		return offer
+	}
+
+	for _, unmatchAudio := range unmatchAudios {
+		streamID, ok := lksdp.ExtractStreamID(unmatchAudio)
+		if !ok {
+			continue
+		}
+
+		p.pendingTracksLock.RLock()
+		_, info, _, _ := p.getPendingTrack(streamID, livekit.TrackType_AUDIO, false)
+		// if RED is disabled for this track, don't prefer RED codec in offer
+		disableRed := info != nil && info.DisableRed
+		p.pendingTracksLock.RUnlock()
+
+		codecs, err := codecsFromMediaDescription(unmatchAudio)
+		if err != nil {
+			p.pubLogger.Errorw("extract codecs from media section failed", err, "media", unmatchAudio)
+			continue
+		}
+
+		var opusPayload uint8
+		for _, codec := range codecs {
+			if mime.IsMimeTypeCodecStringOpus(codec.Name) {
+				opusPayload = codec.PayloadType
+				break
+			}
+		}
+		if opusPayload == 0 {
+			continue
+		}
+
+		var preferredCodecs, leftCodecs []string
+		for _, codec := range codecs {
+			// codec contain opus/red
+			if !disableRed && mime.IsMimeTypeCodecStringRED(codec.Name) && strings.Contains(codec.Fmtp, strconv.FormatInt(int64(opusPayload), 10)) {
+				preferredCodecs = append(preferredCodecs, strconv.FormatInt(int64(codec.PayloadType), 10))
+			} else {
+				leftCodecs = append(leftCodecs, strconv.FormatInt(int64(codec.PayloadType), 10))
+			}
+		}
+
+		// ensure nack enabled for audio in publisher offer
+		var nackFound bool
+		for _, attr := range unmatchAudio.Attributes {
+			if attr.Key == "rtcp-fb" && strings.Contains(attr.Value, fmt.Sprintf("%d nack", opusPayload)) {
+				nackFound = true
+				break
+			}
+		}
+		if !nackFound {
+			unmatchAudio.Attributes = append(unmatchAudio.Attributes, sdp.Attribute{
+				Key:   "rtcp-fb",
+				Value: fmt.Sprintf("%d nack", opusPayload),
+			})
+		}
+
+		// no opus/red found
+		if len(preferredCodecs) == 0 {
+			continue
+		}
+
+		unmatchAudio.MediaName.Formats = append(unmatchAudio.MediaName.Formats[:0], preferredCodecs...)
+		unmatchAudio.MediaName.Formats = append(unmatchAudio.MediaName.Formats, leftCodecs...)
+	}
+
+	bytes, err := parsed.Marshal()
+	if err != nil {
+		p.pubLogger.Errorw("failed to marshal offer", err)
+		return offer
+	}
+
+	return webrtc.SessionDescription{
+		Type: offer.Type,
+		SDP:  string(bytes),
+	}
+}
+
+func (p *ParticipantImpl) setCodecPreferencesVideoForPublisher(offer webrtc.SessionDescription) webrtc.SessionDescription {
+	parsed, unmatchVideos, err := p.TransportManager.GetUnmatchMediaForOffer(offer, "video")
+	if err != nil || len(unmatchVideos) == 0 {
+		return offer
+	}
+	// unmatched video is pending for publish, set codec preference
+	for _, unmatchVideo := range unmatchVideos {
+		streamID, ok := lksdp.ExtractStreamID(unmatchVideo)
+		if !ok {
+			continue
+		}
+
+		var info *livekit.TrackInfo
+		p.pendingTracksLock.RLock()
+		mt := p.getPublishedTrackBySdpCid(streamID)
+		if mt != nil {
+			info = mt.ToProto()
+		} else {
+			_, info, _, _ = p.getPendingTrack(streamID, livekit.TrackType_VIDEO, false)
+		}
+
+		if info == nil {
+			p.pendingTracksLock.RUnlock()
+			continue
+		}
+		var mimeType string
+		for _, c := range info.Codecs {
+			if c.Cid == streamID {
+				mimeType = c.MimeType
+				break
+			}
+		}
+		if mimeType == "" && len(info.Codecs) > 0 {
+			mimeType = info.Codecs[0].MimeType
+		}
+		p.pendingTracksLock.RUnlock()
+
+		if mimeType != "" {
+			codecs, err := codecsFromMediaDescription(unmatchVideo)
+			if err != nil {
+				p.pubLogger.Errorw("extract codecs from media section failed", err, "media", unmatchVideo)
+				continue
+			}
+
+			var preferredCodecs, leftCodecs []string
+			for _, c := range codecs {
+				if mime.GetMimeTypeCodec(mimeType) == mime.NormalizeMimeTypeCodec(c.Name) {
+					preferredCodecs = append(preferredCodecs, strconv.FormatInt(int64(c.PayloadType), 10))
+				} else {
+					leftCodecs = append(leftCodecs, strconv.FormatInt(int64(c.PayloadType), 10))
+				}
+			}
+
+			unmatchVideo.MediaName.Formats = append(unmatchVideo.MediaName.Formats[:0], preferredCodecs...)
+			// if the client don't comply with codec order in SDP answer, only keep preferred codecs to force client to use it
+			if p.params.ClientInfo.ComplyWithCodecOrderInSDPAnswer() {
+				unmatchVideo.MediaName.Formats = append(unmatchVideo.MediaName.Formats, leftCodecs...)
+			}
+		}
+	}
+
+	bytes, err := parsed.Marshal()
+	if err != nil {
+		p.pubLogger.Errorw("failed to marshal offer", err)
+		return offer
+	}
+
+	return webrtc.SessionDescription{
+		Type: offer.Type,
+		SDP:  string(bytes),
+	}
+}
+
+// configure publisher answer for audio track's dtx and stereo settings
+func (p *ParticipantImpl) configurePublisherAnswer(answer webrtc.SessionDescription) webrtc.SessionDescription {
+	offer := p.TransportManager.LastPublisherOffer()
+	parsedOffer, err := offer.Unmarshal()
+	if err != nil {
+		return answer
+	}
+
+	parsed, err := answer.Unmarshal()
+	if err != nil {
+		return answer
+	}
+
+	for _, m := range parsed.MediaDescriptions {
+		switch m.MediaName.Media {
+		case "audio":
+			_, ok := m.Attribute(sdp.AttrKeyInactive)
+			if ok {
+				continue
+			}
+			mid, ok := m.Attribute(sdp.AttrKeyMID)
+			if !ok {
+				continue
+			}
+			// find track info from offer's stream id
+			var ti *livekit.TrackInfo
+			for _, om := range parsedOffer.MediaDescriptions {
+				_, ok := om.Attribute(sdp.AttrKeyInactive)
+				if ok {
+					continue
+				}
+				omid, ok := om.Attribute(sdp.AttrKeyMID)
+				if ok && omid == mid {
+					streamID, ok := lksdp.ExtractStreamID(om)
+					if !ok {
+						continue
+					}
+					track, _ := p.getPublishedTrackBySdpCid(streamID).(*MediaTrack)
+					if track == nil {
+						p.pendingTracksLock.RLock()
+						_, ti, _, _ = p.getPendingTrack(streamID, livekit.TrackType_AUDIO, false)
+						p.pendingTracksLock.RUnlock()
+					} else {
+						ti = track.ToProto()
+					}
+					break
+				}
+			}
+
+			if ti == nil || (ti.DisableDtx && !ti.Stereo) {
+				// no need to configure
+				continue
+			}
+
+			opusPT, err := parsed.GetPayloadTypeForCodec(sdp.Codec{Name: mime.MimeTypeCodecOpus.String()})
+			if err != nil {
+				p.pubLogger.Infow("failed to get opus payload type", "error", err, "trackID", ti.Sid)
+				continue
+			}
+
+			for i, attr := range m.Attributes {
+				if strings.HasPrefix(attr.String(), fmt.Sprintf("fmtp:%d", opusPT)) {
+					if !ti.DisableDtx {
+						attr.Value += ";usedtx=1"
+					}
+					if ti.Stereo {
+						attr.Value += ";stereo=1;maxaveragebitrate=510000"
+					}
+					m.Attributes[i] = attr
+				}
+			}
+
+		default:
+			continue
+		}
+	}
+
+	bytes, err := parsed.Marshal()
+	if err != nil {
+		p.pubLogger.Infow("failed to marshal answer", "error", err)
+		return answer
+	}
+	answer.SDP = string(bytes)
+	return answer
+}

livekit-server/pkg/rtc/participant_signal.go

@@ -0,0 +1,401 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package rtc
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/pion/webrtc/v4"
+
+	"github.com/livekit/protocol/livekit"
+	"github.com/livekit/protocol/logger"
+	"github.com/livekit/protocol/utils"
+	"github.com/livekit/psrpc"
+
+	"github.com/livekit/livekit-server/pkg/routing"
+	"github.com/livekit/livekit-server/pkg/rtc/types"
+)
+
+func (p *ParticipantImpl) getResponseSink() routing.MessageSink {
+	p.resSinkMu.Lock()
+	defer p.resSinkMu.Unlock()
+	return p.resSink
+}
+
+func (p *ParticipantImpl) SetResponseSink(sink routing.MessageSink) {
+	p.resSinkMu.Lock()
+	defer p.resSinkMu.Unlock()
+	p.resSink = sink
+}
+
+func (p *ParticipantImpl) SendJoinResponse(joinResponse *livekit.JoinResponse) error {
+	// keep track of participant updates and versions
+	p.updateLock.Lock()
+	for _, op := range joinResponse.OtherParticipants {
+		p.updateCache.Add(livekit.ParticipantID(op.Sid), participantUpdateInfo{
+			identity:  livekit.ParticipantIdentity(op.Identity),
+			version:   op.Version,
+			state:     op.State,
+			updatedAt: time.Now(),
+		})
+	}
+	p.updateLock.Unlock()
+
+	// send Join response
+	err := p.writeMessage(&livekit.SignalResponse{
+		Message: &livekit.SignalResponse_Join{
+			Join: joinResponse,
+		},
+	})
+	if err != nil {
+		return err
+	}
+
+	// update state after sending message, so that no participant updates could slip through before JoinResponse is sent
+	p.updateLock.Lock()
+	if p.State() == livekit.ParticipantInfo_JOINING {
+		p.updateState(livekit.ParticipantInfo_JOINED)
+	}
+	queuedUpdates := p.queuedUpdates
+	p.queuedUpdates = nil
+	p.updateLock.Unlock()
+
+	if len(queuedUpdates) > 0 {
+		return p.SendParticipantUpdate(queuedUpdates)
+	}
+
+	return nil
+}
+
+func (p *ParticipantImpl) SendParticipantUpdate(participantsToUpdate []*livekit.ParticipantInfo) error {
+	p.updateLock.Lock()
+	if p.IsDisconnected() {
+		p.updateLock.Unlock()
+		return nil
+	}
+
+	if !p.IsReady() {
+		// queue up updates
+		p.queuedUpdates = append(p.queuedUpdates, participantsToUpdate...)
+		p.updateLock.Unlock()
+		return nil
+	}
+	validUpdates := make([]*livekit.ParticipantInfo, 0, len(participantsToUpdate))
+	for _, pi := range participantsToUpdate {
+		isValid := true
+		pID := livekit.ParticipantID(pi.Sid)
+		if lastVersion, ok := p.updateCache.Get(pID); ok {
+			// this is a message delivered out of order, a more recent version of the message had already been
+			// sent.
+			if pi.Version < lastVersion.version {
+				p.params.Logger.Debugw(
+					"skipping outdated participant update",
+					"otherParticipant", pi.Identity,
+					"otherPID", pi.Sid,
+					"version", pi.Version,
+					"lastVersion", lastVersion,
+				)
+				isValid = false
+			}
+		}
+		if pi.Permission != nil && pi.Permission.Hidden && pi.Sid != string(p.params.SID) {
+			p.params.Logger.Debugw("skipping hidden participant update", "otherParticipant", pi.Identity)
+			isValid = false
+		}
+		if isValid {
+			p.updateCache.Add(pID, participantUpdateInfo{
+				identity:  livekit.ParticipantIdentity(pi.Identity),
+				version:   pi.Version,
+				state:     pi.State,
+				updatedAt: time.Now(),
+			})
+			validUpdates = append(validUpdates, pi)
+		}
+	}
+	p.updateLock.Unlock()
+
+	if len(validUpdates) == 0 {
+		return nil
+	}
+
+	return p.writeMessage(&livekit.SignalResponse{
+		Message: &livekit.SignalResponse_Update{
+			Update: &livekit.ParticipantUpdate{
+				Participants: validUpdates,
+			},
+		},
+	})
+}
+
+// SendSpeakerUpdate notifies participant changes to speakers. only send members that have changed since last update
+func (p *ParticipantImpl) SendSpeakerUpdate(speakers []*livekit.SpeakerInfo, force bool) error {
+	if !p.IsReady() {
+		return nil
+	}
+
+	var scopedSpeakers []*livekit.SpeakerInfo
+	if force {
+		scopedSpeakers = speakers
+	} else {
+		for _, s := range speakers {
+			participantID := livekit.ParticipantID(s.Sid)
+			if p.IsSubscribedTo(participantID) || participantID == p.ID() {
+				scopedSpeakers = append(scopedSpeakers, s)
+			}
+		}
+	}
+
+	if len(scopedSpeakers) == 0 {
+		return nil
+	}
+
+	return p.writeMessage(&livekit.SignalResponse{
+		Message: &livekit.SignalResponse_SpeakersChanged{
+			SpeakersChanged: &livekit.SpeakersChanged{
+				Speakers: scopedSpeakers,
+			},
+		},
+	})
+}
+
+func (p *ParticipantImpl) SendRoomUpdate(room *livekit.Room) error {
+	return p.writeMessage(&livekit.SignalResponse{
+		Message: &livekit.SignalResponse_RoomUpdate{
+			RoomUpdate: &livekit.RoomUpdate{
+				Room: room,
+			},
+		},
+	})
+}
+
+func (p *ParticipantImpl) SendConnectionQualityUpdate(update *livekit.ConnectionQualityUpdate) error {
+	return p.writeMessage(&livekit.SignalResponse{
+		Message: &livekit.SignalResponse_ConnectionQuality{
+			ConnectionQuality: update,
+		},
+	})
+}
+
+func (p *ParticipantImpl) SendRefreshToken(token string) error {
+	return p.writeMessage(&livekit.SignalResponse{
+		Message: &livekit.SignalResponse_RefreshToken{
+			RefreshToken: token,
+		},
+	})
+}
+
+func (p *ParticipantImpl) SendRequestResponse(requestResponse *livekit.RequestResponse) error {
+	if requestResponse.RequestId == 0 || !p.params.ClientInfo.SupportErrorResponse() {
+		return nil
+	}
+
+	if requestResponse.Reason == livekit.RequestResponse_OK && !p.ProtocolVersion().SupportsNonErrorSignalResponse() {
+		return nil
+	}
+
+	return p.writeMessage(&livekit.SignalResponse{
+		Message: &livekit.SignalResponse_RequestResponse{
+			RequestResponse: requestResponse,
+		},
+	})
+}
+
+func (p *ParticipantImpl) HandleReconnectAndSendResponse(reconnectReason livekit.ReconnectReason, reconnectResponse *livekit.ReconnectResponse) error {
+	p.TransportManager.HandleClientReconnect(reconnectReason)
+
+	if !p.params.ClientInfo.CanHandleReconnectResponse() {
+		return nil
+	}
+	if err := p.writeMessage(&livekit.SignalResponse{
+		Message: &livekit.SignalResponse_Reconnect{
+			Reconnect: reconnectResponse,
+		},
+	}); err != nil {
+		return err
+	}
+
+	if p.params.ProtocolVersion.SupportHandlesDisconnectedUpdate() {
+		return p.sendDisconnectUpdatesForReconnect()
+	}
+
+	return nil
+}
+
+func (p *ParticipantImpl) sendDisconnectUpdatesForReconnect() error {
+	lastSignalAt := p.TransportManager.LastSeenSignalAt()
+	var disconnectedParticipants []*livekit.ParticipantInfo
+	p.updateLock.Lock()
+	keys := p.updateCache.Keys()
+	for i := len(keys) - 1; i >= 0; i-- {
+		if info, ok := p.updateCache.Get(keys[i]); ok {
+			if info.updatedAt.Before(lastSignalAt) {
+				break
+			} else if info.state == livekit.ParticipantInfo_DISCONNECTED {
+				disconnectedParticipants = append(disconnectedParticipants, &livekit.ParticipantInfo{
+					Sid:      string(keys[i]),
+					Identity: string(info.identity),
+					Version:  info.version,
+					State:    livekit.ParticipantInfo_DISCONNECTED,
+				})
+			}
+		}
+	}
+	p.updateLock.Unlock()
+
+	if len(disconnectedParticipants) == 0 {
+		return nil
+	}
+
+	return p.writeMessage(&livekit.SignalResponse{
+		Message: &livekit.SignalResponse_Update{
+			Update: &livekit.ParticipantUpdate{
+				Participants: disconnectedParticipants,
+			},
+		},
+	})
+}
+
+func (p *ParticipantImpl) sendICECandidate(ic *webrtc.ICECandidate, target livekit.SignalTarget) error {
+	prevIC := p.icQueue[target].Swap(ic)
+	if prevIC == nil {
+		return nil
+	}
+
+	trickle := ToProtoTrickle(prevIC.ToJSON(), target, ic == nil)
+	p.params.Logger.Debugw("sending ICE candidate", "transport", target, "trickle", logger.Proto(trickle))
+
+	return p.writeMessage(&livekit.SignalResponse{
+		Message: &livekit.SignalResponse_Trickle{
+			Trickle: trickle,
+		},
+	})
+}
+
+func (p *ParticipantImpl) sendTrackMuted(trackID livekit.TrackID, muted bool) {
+	_ = p.writeMessage(&livekit.SignalResponse{
+		Message: &livekit.SignalResponse_Mute{
+			Mute: &livekit.MuteTrackRequest{
+				Sid:   string(trackID),
+				Muted: muted,
+			},
+		},
+	})
+}
+
+func (p *ParticipantImpl) sendTrackUnpublished(trackID livekit.TrackID) {
+	_ = p.writeMessage(&livekit.SignalResponse{
+		Message: &livekit.SignalResponse_TrackUnpublished{
+			TrackUnpublished: &livekit.TrackUnpublishedResponse{
+				TrackSid: string(trackID),
+			},
+		},
+	})
+}
+
+func (p *ParticipantImpl) sendTrackHasBeenSubscribed(trackID livekit.TrackID) {
+	if !p.params.ClientInfo.SupportTrackSubscribedEvent() {
+		return
+	}
+	_ = p.writeMessage(&livekit.SignalResponse{
+		Message: &livekit.SignalResponse_TrackSubscribed{
+			TrackSubscribed: &livekit.TrackSubscribed{
+				TrackSid: string(trackID),
+			},
+		},
+	})
+	p.params.Logger.Debugw("track has been subscribed", "trackID", trackID)
+}
+
+func (p *ParticipantImpl) writeMessage(msg *livekit.SignalResponse) error {
+	if p.IsDisconnected() || (!p.IsReady() && msg.GetJoin() == nil) {
+		return nil
+	}
+
+	sink := p.getResponseSink()
+	if sink == nil {
+		p.params.Logger.Debugw("could not send message to participant", "messageType", fmt.Sprintf("%T", msg.Message))
+		return nil
+	}
+
+	err := sink.WriteMessage(msg)
+	if utils.ErrorIsOneOf(err, psrpc.Canceled, routing.ErrChannelClosed) {
+		p.params.Logger.Debugw(
+			"could not send message to participant",
+			"error", err,
+			"messageType", fmt.Sprintf("%T", msg.Message),
+		)
+		return nil
+	} else if err != nil {
+		p.params.Logger.Warnw(
+			"could not send message to participant", err,
+			"messageType", fmt.Sprintf("%T", msg.Message),
+		)
+		return err
+	}
+	return nil
+}
+
+// closes signal connection to notify client to resume/reconnect
+func (p *ParticipantImpl) CloseSignalConnection(reason types.SignallingCloseReason) {
+	sink := p.getResponseSink()
+	if sink != nil {
+		p.params.Logger.Debugw("closing signal connection", "reason", reason, "connID", sink.ConnectionID())
+		sink.Close()
+		p.SetResponseSink(nil)
+	}
+}
+
+func (p *ParticipantImpl) sendLeaveRequest(
+	reason types.ParticipantCloseReason,
+	isExpectedToResume bool,
+	isExpectedToReconnect bool,
+	sendOnlyIfSupportingLeaveRequestWithAction bool,
+) error {
+	var leave *livekit.LeaveRequest
+	if p.ProtocolVersion().SupportsRegionsInLeaveRequest() {
+		leave = &livekit.LeaveRequest{
+			Reason: reason.ToDisconnectReason(),
+		}
+		switch {
+		case isExpectedToResume:
+			leave.Action = livekit.LeaveRequest_RESUME
+		case isExpectedToReconnect:
+			leave.Action = livekit.LeaveRequest_RECONNECT
+		default:
+			leave.Action = livekit.LeaveRequest_DISCONNECT
+		}
+		if leave.Action != livekit.LeaveRequest_DISCONNECT && p.params.GetRegionSettings != nil {
+			// sending region settings even for RESUME just in case client wants to a full reconnect despite server saying RESUME
+			leave.Regions = p.params.GetRegionSettings(p.params.ClientInfo.Address)
+		}
+	} else {
+		if !sendOnlyIfSupportingLeaveRequestWithAction {
+			leave = &livekit.LeaveRequest{
+				CanReconnect: isExpectedToReconnect,
+				Reason:       reason.ToDisconnectReason(),
+			}
+		}
+	}
+	if leave != nil {
+		return p.writeMessage(&livekit.SignalResponse{
+			Message: &livekit.SignalResponse_Leave{
+				Leave: leave,
+			},
+		})
+	}
+
+	return nil
+}

livekit-server/pkg/rtc/room_test.go

@@ -0,0 +1,835 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package rtc
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+	"google.golang.org/protobuf/proto"
+
+	"github.com/livekit/protocol/livekit"
+	"github.com/livekit/protocol/utils"
+	"github.com/livekit/protocol/webhook"
+
+	"github.com/livekit/livekit-server/version"
+
+	"github.com/livekit/livekit-server/pkg/config"
+	"github.com/livekit/livekit-server/pkg/rtc/types"
+	"github.com/livekit/livekit-server/pkg/rtc/types/typesfakes"
+	"github.com/livekit/livekit-server/pkg/sfu"
+	"github.com/livekit/livekit-server/pkg/sfu/audio"
+	"github.com/livekit/livekit-server/pkg/telemetry"
+	"github.com/livekit/livekit-server/pkg/telemetry/prometheus"
+	"github.com/livekit/livekit-server/pkg/telemetry/telemetryfakes"
+	"github.com/livekit/livekit-server/pkg/testutils"
+)
+
+func init() {
+	prometheus.Init("test", livekit.NodeType_SERVER)
+}
+
+const (
+	numParticipants     = 3
+	defaultDelay        = 10 * time.Millisecond
+	audioUpdateInterval = 25
+)
+
+func init() {
+	config.InitLoggerFromConfig(&config.DefaultConfig.Logging)
+	roomUpdateInterval = defaultDelay
+}
+
+var iceServersForRoom = []*livekit.ICEServer{{Urls: []string{"stun:stun.l.google.com:19302"}}}
+
+func TestJoinedState(t *testing.T) {
+	t.Run("new room should return joinedAt 0", func(t *testing.T) {
+		rm := newRoomWithParticipants(t, testRoomOpts{num: 0})
+		require.Equal(t, int64(0), rm.FirstJoinedAt())
+		require.Equal(t, int64(0), rm.LastLeftAt())
+	})
+
+	t.Run("should be current time when a participant joins", func(t *testing.T) {
+		s := time.Now().Unix()
+		rm := newRoomWithParticipants(t, testRoomOpts{num: 1})
+		require.LessOrEqual(t, s, rm.FirstJoinedAt())
+		require.Equal(t, int64(0), rm.LastLeftAt())
+	})
+
+	t.Run("should be set when a participant leaves", func(t *testing.T) {
+		rm := newRoomWithParticipants(t, testRoomOpts{num: 1})
+		p0 := rm.GetParticipants()[0]
+		s := time.Now().Unix()
+		rm.RemoveParticipant(p0.Identity(), p0.ID(), types.ParticipantCloseReasonClientRequestLeave)
+		require.LessOrEqual(t, s, rm.LastLeftAt())
+	})
+
+	t.Run("LastLeftAt should be set when there are still participants in the room", func(t *testing.T) {
+		rm := newRoomWithParticipants(t, testRoomOpts{num: 2})
+		p0 := rm.GetParticipants()[0]
+		rm.RemoveParticipant(p0.Identity(), p0.ID(), types.ParticipantCloseReasonClientRequestLeave)
+		require.Greater(t, rm.LastLeftAt(), int64(0))
+	})
+}
+
+func TestRoomJoin(t *testing.T) {
+	t.Run("joining returns existing participant data", func(t *testing.T) {
+		rm := newRoomWithParticipants(t, testRoomOpts{num: numParticipants})
+		pNew := NewMockParticipant("new", types.CurrentProtocol, false, false)
+
+		_ = rm.Join(pNew, nil, nil, iceServersForRoom)
+
+		// expect new participant to get a JoinReply
+		res := pNew.SendJoinResponseArgsForCall(0)
+		require.Equal(t, livekit.RoomID(res.Room.Sid), rm.ID())
+		require.Len(t, res.OtherParticipants, numParticipants)
+		require.Len(t, rm.GetParticipants(), numParticipants+1)
+		require.NotEmpty(t, res.IceServers)
+	})
+
+	t.Run("subscribe to existing channels upon join", func(t *testing.T) {
+		numExisting := 3
+		rm := newRoomWithParticipants(t, testRoomOpts{num: numExisting})
+		p := NewMockParticipant("new", types.CurrentProtocol, false, false)
+
+		err := rm.Join(p, nil, &ParticipantOptions{AutoSubscribe: true}, iceServersForRoom)
+		require.NoError(t, err)
+
+		stateChangeCB := p.OnStateChangeArgsForCall(0)
+		require.NotNil(t, stateChangeCB)
+		stateChangeCB(p, livekit.ParticipantInfo_ACTIVE)
+
+		// it should become a subscriber when connectivity changes
+		numTracks := 0
+		for _, op := range rm.GetParticipants() {
+			if p == op {
+				continue
+			}
+
+			numTracks += len(op.GetPublishedTracks())
+		}
+		require.Equal(t, numTracks, p.SubscribeToTrackCallCount())
+	})
+
+	t.Run("participant state change is broadcasted to others", func(t *testing.T) {
+		rm := newRoomWithParticipants(t, testRoomOpts{num: numParticipants})
+		var changedParticipant types.Participant
+		rm.OnParticipantChanged(func(participant types.LocalParticipant) {
+			changedParticipant = participant
+		})
+		participants := rm.GetParticipants()
+		p := participants[0].(*typesfakes.FakeLocalParticipant)
+		disconnectedParticipant := participants[1].(*typesfakes.FakeLocalParticipant)
+		disconnectedParticipant.StateReturns(livekit.ParticipantInfo_DISCONNECTED)
+
+		rm.RemoveParticipant(p.Identity(), p.ID(), types.ParticipantCloseReasonClientRequestLeave)
+		time.Sleep(defaultDelay)
+
+		require.Equal(t, p, changedParticipant)
+
+		numUpdates := 0
+		for _, op := range participants {
+			if op == p || op == disconnectedParticipant {
+				require.Zero(t, p.SendParticipantUpdateCallCount())
+				continue
+			}
+			fakeP := op.(*typesfakes.FakeLocalParticipant)
+			require.Equal(t, 1, fakeP.SendParticipantUpdateCallCount())
+			numUpdates += 1
+		}
+		require.Equal(t, numParticipants-2, numUpdates)
+	})
+
+	t.Run("cannot exceed max participants", func(t *testing.T) {
+		rm := newRoomWithParticipants(t, testRoomOpts{num: 1})
+		rm.lock.Lock()
+		rm.protoRoom.MaxParticipants = 1
+		rm.lock.Unlock()
+		p := NewMockParticipant("second", types.ProtocolVersion(0), false, false)
+
+		err := rm.Join(p, nil, nil, iceServersForRoom)
+		require.Equal(t, ErrMaxParticipantsExceeded, err)
+	})
+}
+
+// various state changes to participant and that others are receiving update
+func TestParticipantUpdate(t *testing.T) {
+	tests := []struct {
+		name         string
+		sendToSender bool // should sender receive it
+		action       func(p types.LocalParticipant)
+	}{
+		{
+			"track mutes are sent to everyone",
+			true,
+			func(p types.LocalParticipant) {
+				p.SetTrackMuted("", true, false)
+			},
+		},
+		{
+			"track metadata updates are sent to everyone",
+			true,
+			func(p types.LocalParticipant) {
+				p.SetMetadata("")
+			},
+		},
+		{
+			"track publishes are sent to existing participants",
+			true,
+			func(p types.LocalParticipant) {
+				p.AddTrack(&livekit.AddTrackRequest{
+					Type: livekit.TrackType_VIDEO,
+				})
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			rm := newRoomWithParticipants(t, testRoomOpts{num: 3})
+			// remember how many times send has been called for each
+			callCounts := make(map[livekit.ParticipantID]int)
+			for _, p := range rm.GetParticipants() {
+				fp := p.(*typesfakes.FakeLocalParticipant)
+				callCounts[p.ID()] = fp.SendParticipantUpdateCallCount()
+			}
+
+			sender := rm.GetParticipants()[0]
+			test.action(sender)
+
+			// go through the other participants, make sure they've received update
+			for _, p := range rm.GetParticipants() {
+				expected := callCounts[p.ID()]
+				if p != sender || test.sendToSender {
+					expected += 1
+				}
+				fp := p.(*typesfakes.FakeLocalParticipant)
+				require.Equal(t, expected, fp.SendParticipantUpdateCallCount())
+			}
+		})
+	}
+}
+
+func TestPushAndDequeueUpdates(t *testing.T) {
+	identity := "test_user"
+	publisher1v1 := &livekit.ParticipantInfo{
+		Identity:    identity,
+		Sid:         "1",
+		IsPublisher: true,
+		Version:     1,
+		JoinedAt:    0,
+	}
+	publisher1v2 := &livekit.ParticipantInfo{
+		Identity:    identity,
+		Sid:         "1",
+		IsPublisher: true,
+		Version:     2,
+		JoinedAt:    1,
+	}
+	publisher2 := &livekit.ParticipantInfo{
+		Identity:    identity,
+		Sid:         "2",
+		IsPublisher: true,
+		Version:     1,
+		JoinedAt:    2,
+	}
+	subscriber1v1 := &livekit.ParticipantInfo{
+		Identity: identity,
+		Sid:      "1",
+		Version:  1,
+		JoinedAt: 0,
+	}
+	subscriber1v2 := &livekit.ParticipantInfo{
+		Identity: identity,
+		Sid:      "1",
+		Version:  2,
+		JoinedAt: 1,
+	}
+
+	requirePIEquals := func(t *testing.T, a, b *livekit.ParticipantInfo) {
+		require.Equal(t, a.Sid, b.Sid)
+		require.Equal(t, a.Identity, b.Identity)
+		require.Equal(t, a.Version, b.Version)
+	}
+	testCases := []struct {
+		name        string
+		pi          *livekit.ParticipantInfo
+		closeReason types.ParticipantCloseReason
+		immediate   bool
+		existing    *participantUpdate
+		expected    []*participantUpdate
+		validate    func(t *testing.T, rm *Room, updates []*participantUpdate)
+	}{
+		{
+			name:     "publisher updates are immediate",
+			pi:       publisher1v1,
+			expected: []*participantUpdate{{pi: publisher1v1}},
+		},
+		{
+			name: "subscriber updates are queued",
+			pi:   subscriber1v1,
+		},
+		{
+			name:     "last version is enqueued",
+			pi:       subscriber1v2,
+			existing: &participantUpdate{pi: utils.CloneProto(subscriber1v1)}, // clone the existing value since it can be modified when setting to disconnected
+			validate: func(t *testing.T, rm *Room, _ []*participantUpdate) {
+				queued := rm.batchedUpdates[livekit.ParticipantIdentity(identity)]
+				require.NotNil(t, queued)
+				requirePIEquals(t, subscriber1v2, queued.pi)
+			},
+		},
+		{
+			name:      "latest version when immediate",
+			pi:        subscriber1v2,
+			existing:  &participantUpdate{pi: utils.CloneProto(subscriber1v1)},
+			immediate: true,
+			expected:  []*participantUpdate{{pi: subscriber1v2}},
+			validate: func(t *testing.T, rm *Room, _ []*participantUpdate) {
+				queued := rm.batchedUpdates[livekit.ParticipantIdentity(identity)]
+				require.Nil(t, queued)
+			},
+		},
+		{
+			name:     "out of order updates are rejected",
+			pi:       subscriber1v1,
+			existing: &participantUpdate{pi: utils.CloneProto(subscriber1v2)},
+			validate: func(t *testing.T, rm *Room, updates []*participantUpdate) {
+				queued := rm.batchedUpdates[livekit.ParticipantIdentity(identity)]
+				requirePIEquals(t, subscriber1v2, queued.pi)
+			},
+		},
+		{
+			name:        "sid change is broadcasted immediately with synthsized disconnect",
+			pi:          publisher2,
+			closeReason: types.ParticipantCloseReasonServiceRequestRemoveParticipant, // just to test if update contain the close reason
+			existing:    &participantUpdate{pi: utils.CloneProto(subscriber1v2), closeReason: types.ParticipantCloseReasonStale},
+			expected: []*participantUpdate{
+				{
+					pi: &livekit.ParticipantInfo{
+						Identity: identity,
+						Sid:      "1",
+						Version:  2,
+						State:    livekit.ParticipantInfo_DISCONNECTED,
+					},
+					isSynthesizedDisconnect: true,
+					closeReason:             types.ParticipantCloseReasonStale,
+				},
+				{pi: publisher2, closeReason: types.ParticipantCloseReasonServiceRequestRemoveParticipant},
+			},
+		},
+		{
+			name:     "when switching to publisher, queue is cleared",
+			pi:       publisher1v2,
+			existing: &participantUpdate{pi: utils.CloneProto(subscriber1v1)},
+			expected: []*participantUpdate{{pi: publisher1v2}},
+			validate: func(t *testing.T, rm *Room, updates []*participantUpdate) {
+				require.Empty(t, rm.batchedUpdates)
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			rm := newRoomWithParticipants(t, testRoomOpts{num: 1})
+			if tc.existing != nil {
+				rm.batchedUpdates[livekit.ParticipantIdentity(tc.existing.pi.Identity)] = tc.existing
+			}
+			updates := rm.pushAndDequeueUpdates(tc.pi, tc.closeReason, tc.immediate)
+			require.Equal(t, len(tc.expected), len(updates))
+			for i, item := range tc.expected {
+				requirePIEquals(t, item.pi, updates[i].pi)
+				require.Equal(t, item.isSynthesizedDisconnect, updates[i].isSynthesizedDisconnect)
+				require.Equal(t, item.closeReason, updates[i].closeReason)
+			}
+
+			if tc.validate != nil {
+				tc.validate(t, rm, updates)
+			}
+		})
+	}
+}
+
+func TestRoomClosure(t *testing.T) {
+	t.Run("room closes after participant leaves", func(t *testing.T) {
+		rm := newRoomWithParticipants(t, testRoomOpts{num: 1})
+		isClosed := false
+		rm.OnClose(func() {
+			isClosed = true
+		})
+		p := rm.GetParticipants()[0]
+		rm.lock.Lock()
+		// allows immediate close after
+		rm.protoRoom.EmptyTimeout = 0
+		rm.lock.Unlock()
+		rm.RemoveParticipant(p.Identity(), p.ID(), types.ParticipantCloseReasonClientRequestLeave)
+
+		time.Sleep(time.Duration(rm.ToProto().DepartureTimeout)*time.Second + defaultDelay)
+
+		rm.CloseIfEmpty()
+		require.Len(t, rm.GetParticipants(), 0)
+		require.True(t, isClosed)
+
+		require.Equal(t, ErrRoomClosed, rm.Join(p, nil, nil, iceServersForRoom))
+	})
+
+	t.Run("room does not close before empty timeout", func(t *testing.T) {
+		rm := newRoomWithParticipants(t, testRoomOpts{num: 0})
+		isClosed := false
+		rm.OnClose(func() {
+			isClosed = true
+		})
+		require.NotZero(t, rm.protoRoom.EmptyTimeout)
+		rm.CloseIfEmpty()
+		require.False(t, isClosed)
+	})
+
+	t.Run("room closes after empty timeout", func(t *testing.T) {
+		rm := newRoomWithParticipants(t, testRoomOpts{num: 0})
+		isClosed := false
+		rm.OnClose(func() {
+			isClosed = true
+		})
+		rm.lock.Lock()
+		rm.protoRoom.EmptyTimeout = 1
+		rm.lock.Unlock()
+
+		time.Sleep(1010 * time.Millisecond)
+		rm.CloseIfEmpty()
+		require.True(t, isClosed)
+	})
+}
+
+func TestNewTrack(t *testing.T) {
+	t.Run("new track should be added to ready participants", func(t *testing.T) {
+		rm := newRoomWithParticipants(t, testRoomOpts{num: 3})
+		participants := rm.GetParticipants()
+		p0 := participants[0].(*typesfakes.FakeLocalParticipant)
+		p0.StateReturns(livekit.ParticipantInfo_JOINED)
+		p1 := participants[1].(*typesfakes.FakeLocalParticipant)
+		p1.StateReturns(livekit.ParticipantInfo_ACTIVE)
+
+		pub := participants[2].(*typesfakes.FakeLocalParticipant)
+
+		// pub adds track
+		track := NewMockTrack(livekit.TrackType_VIDEO, "webcam")
+		trackCB := pub.OnTrackPublishedArgsForCall(0)
+		require.NotNil(t, trackCB)
+		trackCB(pub, track)
+		// only p1 should've been subscribed to
+		require.Equal(t, 0, p0.SubscribeToTrackCallCount())
+		require.Equal(t, 1, p1.SubscribeToTrackCallCount())
+	})
+}
+
+func TestActiveSpeakers(t *testing.T) {
+	t.Parallel()
+	getActiveSpeakerUpdates := func(p *typesfakes.FakeLocalParticipant) [][]*livekit.SpeakerInfo {
+		var updates [][]*livekit.SpeakerInfo
+		numCalls := p.SendSpeakerUpdateCallCount()
+		for i := 0; i < numCalls; i++ {
+			infos, _ := p.SendSpeakerUpdateArgsForCall(i)
+			updates = append(updates, infos)
+		}
+		return updates
+	}
+
+	audioUpdateDuration := (audioUpdateInterval + 10) * time.Millisecond
+	t.Run("participant should not be getting audio updates (protocol 2)", func(t *testing.T) {
+		rm := newRoomWithParticipants(t, testRoomOpts{num: 1, protocol: 2})
+		defer rm.Close(types.ParticipantCloseReasonNone)
+		p := rm.GetParticipants()[0].(*typesfakes.FakeLocalParticipant)
+		require.Empty(t, rm.GetActiveSpeakers())
+
+		time.Sleep(audioUpdateDuration)
+
+		updates := getActiveSpeakerUpdates(p)
+		require.Empty(t, updates)
+	})
+
+	t.Run("speakers should be sorted by loudness", func(t *testing.T) {
+		rm := newRoomWithParticipants(t, testRoomOpts{num: 2})
+		defer rm.Close(types.ParticipantCloseReasonNone)
+		participants := rm.GetParticipants()
+		p := participants[0].(*typesfakes.FakeLocalParticipant)
+		p2 := participants[1].(*typesfakes.FakeLocalParticipant)
+		p.GetAudioLevelReturns(20, true)
+		p2.GetAudioLevelReturns(10, true)
+
+		speakers := rm.GetActiveSpeakers()
+		require.Len(t, speakers, 2)
+		require.Equal(t, string(p.ID()), speakers[0].Sid)
+		require.Equal(t, string(p2.ID()), speakers[1].Sid)
+	})
+
+	t.Run("participants are getting audio updates (protocol 3+)", func(t *testing.T) {
+		rm := newRoomWithParticipants(t, testRoomOpts{num: 2, protocol: 3})
+		defer rm.Close(types.ParticipantCloseReasonNone)
+		participants := rm.GetParticipants()
+		p := participants[0].(*typesfakes.FakeLocalParticipant)
+		time.Sleep(time.Millisecond) // let the first update cycle run
+		p.GetAudioLevelReturns(30, true)
+
+		speakers := rm.GetActiveSpeakers()
+		require.NotEmpty(t, speakers)
+		require.Equal(t, string(p.ID()), speakers[0].Sid)
+
+		testutils.WithTimeout(t, func() string {
+			for _, op := range participants {
+				op := op.(*typesfakes.FakeLocalParticipant)
+				updates := getActiveSpeakerUpdates(op)
+				if len(updates) == 0 {
+					return fmt.Sprintf("%s did not get any audio updates", op.Identity())
+				}
+			}
+			return ""
+		})
+
+		// no longer speaking, send update with empty items
+		p.GetAudioLevelReturns(127, false)
+
+		testutils.WithTimeout(t, func() string {
+			updates := getActiveSpeakerUpdates(p)
+			lastUpdate := updates[len(updates)-1]
+			if len(lastUpdate) == 0 {
+				return "did not get updates of speaker going quiet"
+			}
+			if lastUpdate[0].Active {
+				return "speaker should not have been active"
+			}
+			return ""
+		})
+	})
+
+	t.Run("audio level is smoothed", func(t *testing.T) {
+		rm := newRoomWithParticipants(t, testRoomOpts{num: 2, protocol: 3, audioSmoothIntervals: 3})
+		defer rm.Close(types.ParticipantCloseReasonNone)
+		participants := rm.GetParticipants()
+		p := participants[0].(*typesfakes.FakeLocalParticipant)
+		op := participants[1].(*typesfakes.FakeLocalParticipant)
+		p.GetAudioLevelReturns(30, true)
+		convertedLevel := float32(audio.ConvertAudioLevel(30))
+
+		testutils.WithTimeout(t, func() string {
+			updates := getActiveSpeakerUpdates(op)
+			if len(updates) == 0 {
+				return "no speaker updates received"
+			}
+			lastSpeakers := updates[len(updates)-1]
+			if len(lastSpeakers) == 0 {
+				return "no speakers in the update"
+			}
+			if lastSpeakers[0].Level > convertedLevel {
+				return ""
+			}
+			return "level mismatch"
+		})
+
+		testutils.WithTimeout(t, func() string {
+			updates := getActiveSpeakerUpdates(op)
+			if len(updates) == 0 {
+				return "no updates received"
+			}
+			lastSpeakers := updates[len(updates)-1]
+			if len(lastSpeakers) == 0 {
+				return "no speakers found"
+			}
+			if lastSpeakers[0].Level > convertedLevel {
+				return ""
+			}
+			return "did not match expected levels"
+		})
+
+		p.GetAudioLevelReturns(127, false)
+		testutils.WithTimeout(t, func() string {
+			updates := getActiveSpeakerUpdates(op)
+			if len(updates) == 0 {
+				return "no speaker updates received"
+			}
+			lastSpeakers := updates[len(updates)-1]
+			if len(lastSpeakers) == 1 && !lastSpeakers[0].Active {
+				return ""
+			}
+			return "speakers didn't go back to zero"
+		})
+	})
+}
+
+func TestDataChannel(t *testing.T) {
+	t.Parallel()
+
+	const (
+		curAPI = iota
+		legacySID
+		legacyIdentity
+	)
+	modes := []int{
+		curAPI, legacySID, legacyIdentity,
+	}
+	modeNames := []string{
+		"cur", "legacy sid", "legacy identity",
+	}
+
+	setSource := func(mode int, dp *livekit.DataPacket, p types.LocalParticipant) {
+		switch mode {
+		case curAPI:
+			dp.ParticipantIdentity = string(p.Identity())
+		case legacySID:
+			dp.GetUser().ParticipantSid = string(p.ID())
+		case legacyIdentity:
+			dp.GetUser().ParticipantIdentity = string(p.Identity())
+		}
+	}
+	setDest := func(mode int, dp *livekit.DataPacket, p types.LocalParticipant) {
+		switch mode {
+		case curAPI:
+			dp.DestinationIdentities = []string{string(p.Identity())}
+		case legacySID:
+			dp.GetUser().DestinationSids = []string{string(p.ID())}
+		case legacyIdentity:
+			dp.GetUser().DestinationIdentities = []string{string(p.Identity())}
+		}
+	}
+
+	t.Run("participants should receive data", func(t *testing.T) {
+		for _, mode := range modes {
+			mode := mode
+			t.Run(modeNames[mode], func(t *testing.T) {
+				rm := newRoomWithParticipants(t, testRoomOpts{num: 3})
+				defer rm.Close(types.ParticipantCloseReasonNone)
+				participants := rm.GetParticipants()
+				p := participants[0].(*typesfakes.FakeLocalParticipant)
+
+				packet := &livekit.DataPacket{
+					Kind: livekit.DataPacket_RELIABLE,
+					Value: &livekit.DataPacket_User{
+						User: &livekit.UserPacket{
+							Payload: []byte("message.."),
+						},
+					},
+				}
+				setSource(mode, packet, p)
+
+				packetExp := utils.CloneProto(packet)
+				if mode != legacySID {
+					packetExp.ParticipantIdentity = string(p.Identity())
+					packetExp.GetUser().ParticipantIdentity = string(p.Identity())
+				}
+
+				encoded, _ := proto.Marshal(packetExp)
+				p.OnDataPacketArgsForCall(0)(p, packet.Kind, packet)
+
+				// ensure everyone has received the packet
+				for _, op := range participants {
+					fp := op.(*typesfakes.FakeLocalParticipant)
+					if fp == p {
+						require.Zero(t, fp.SendDataPacketCallCount())
+						continue
+					}
+					require.Equal(t, 1, fp.SendDataPacketCallCount())
+					_, got := fp.SendDataPacketArgsForCall(0)
+					require.Equal(t, encoded, got)
+				}
+			})
+		}
+	})
+
+	t.Run("only one participant should receive the data", func(t *testing.T) {
+		for _, mode := range modes {
+			mode := mode
+			t.Run(modeNames[mode], func(t *testing.T) {
+				rm := newRoomWithParticipants(t, testRoomOpts{num: 4})
+				defer rm.Close(types.ParticipantCloseReasonNone)
+				participants := rm.GetParticipants()
+				p := participants[0].(*typesfakes.FakeLocalParticipant)
+				p1 := participants[1].(*typesfakes.FakeLocalParticipant)
+
+				packet := &livekit.DataPacket{
+					Kind: livekit.DataPacket_RELIABLE,
+					Value: &livekit.DataPacket_User{
+						User: &livekit.UserPacket{
+							Payload: []byte("message to p1.."),
+						},
+					},
+				}
+				setSource(mode, packet, p)
+				setDest(mode, packet, p1)
+
+				packetExp := utils.CloneProto(packet)
+				if mode != legacySID {
+					packetExp.ParticipantIdentity = string(p.Identity())
+					packetExp.GetUser().ParticipantIdentity = string(p.Identity())
+					packetExp.DestinationIdentities = []string{string(p1.Identity())}
+					packetExp.GetUser().DestinationIdentities = []string{string(p1.Identity())}
+				}
+
+				encoded, _ := proto.Marshal(packetExp)
+				p.OnDataPacketArgsForCall(0)(p, packet.Kind, packet)
+
+				// only p1 should receive the data
+				for _, op := range participants {
+					fp := op.(*typesfakes.FakeLocalParticipant)
+					if fp != p1 {
+						require.Zero(t, fp.SendDataPacketCallCount())
+					}
+				}
+				require.Equal(t, 1, p1.SendDataPacketCallCount())
+				_, got := p1.SendDataPacketArgsForCall(0)
+				require.Equal(t, encoded, got)
+			})
+		}
+	})
+
+	t.Run("publishing disallowed", func(t *testing.T) {
+		rm := newRoomWithParticipants(t, testRoomOpts{num: 2})
+		defer rm.Close(types.ParticipantCloseReasonNone)
+		participants := rm.GetParticipants()
+		p := participants[0].(*typesfakes.FakeLocalParticipant)
+		p.CanPublishDataReturns(false)
+
+		packet := livekit.DataPacket{
+			Kind: livekit.DataPacket_RELIABLE,
+			Value: &livekit.DataPacket_User{
+				User: &livekit.UserPacket{
+					Payload: []byte{},
+				},
+			},
+		}
+		if p.CanPublishData() {
+			p.OnDataPacketArgsForCall(0)(p, packet.Kind, &packet)
+		}
+
+		// no one should've been sent packet
+		for _, op := range participants {
+			fp := op.(*typesfakes.FakeLocalParticipant)
+			require.Zero(t, fp.SendDataPacketCallCount())
+		}
+	})
+}
+
+func TestHiddenParticipants(t *testing.T) {
+	t.Run("other participants don't receive hidden updates", func(t *testing.T) {
+		rm := newRoomWithParticipants(t, testRoomOpts{num: 2, numHidden: 1})
+		defer rm.Close(types.ParticipantCloseReasonNone)
+
+		pNew := NewMockParticipant("new", types.CurrentProtocol, false, false)
+		rm.Join(pNew, nil, nil, iceServersForRoom)
+
+		// expect new participant to get a JoinReply
+		res := pNew.SendJoinResponseArgsForCall(0)
+		require.Equal(t, livekit.RoomID(res.Room.Sid), rm.ID())
+		require.Len(t, res.OtherParticipants, 2)
+		require.Len(t, rm.GetParticipants(), 4)
+		require.NotEmpty(t, res.IceServers)
+		require.Equal(t, "testregion", res.ServerInfo.Region)
+	})
+
+	t.Run("hidden participant subscribes to tracks", func(t *testing.T) {
+		rm := newRoomWithParticipants(t, testRoomOpts{num: 2})
+		hidden := NewMockParticipant("hidden", types.CurrentProtocol, true, false)
+
+		err := rm.Join(hidden, nil, &ParticipantOptions{AutoSubscribe: true}, iceServersForRoom)
+		require.NoError(t, err)
+
+		stateChangeCB := hidden.OnStateChangeArgsForCall(0)
+		require.NotNil(t, stateChangeCB)
+		stateChangeCB(hidden, livekit.ParticipantInfo_ACTIVE)
+
+		require.Eventually(t, func() bool { return hidden.SubscribeToTrackCallCount() == 2 }, 5*time.Second, 10*time.Millisecond)
+	})
+}
+
+func TestRoomUpdate(t *testing.T) {
+	t.Run("updates are sent when participant joined", func(t *testing.T) {
+		rm := newRoomWithParticipants(t, testRoomOpts{num: 1})
+		defer rm.Close(types.ParticipantCloseReasonNone)
+
+		p1 := rm.GetParticipants()[0].(*typesfakes.FakeLocalParticipant)
+		require.Equal(t, 0, p1.SendRoomUpdateCallCount())
+
+		p2 := NewMockParticipant("p2", types.CurrentProtocol, false, false)
+		require.NoError(t, rm.Join(p2, nil, nil, iceServersForRoom))
+
+		// p1 should have received an update
+		time.Sleep(2 * defaultDelay)
+		require.LessOrEqual(t, 1, p1.SendRoomUpdateCallCount())
+		require.EqualValues(t, 2, p1.SendRoomUpdateArgsForCall(p1.SendRoomUpdateCallCount()-1).NumParticipants)
+	})
+
+	t.Run("participants should receive metadata update", func(t *testing.T) {
+		rm := newRoomWithParticipants(t, testRoomOpts{num: 2})
+		defer rm.Close(types.ParticipantCloseReasonNone)
+
+		rm.SetMetadata("test metadata...")
+
+		// callbacks are updated from goroutine
+		time.Sleep(2 * defaultDelay)
+
+		for _, op := range rm.GetParticipants() {
+			fp := op.(*typesfakes.FakeLocalParticipant)
+			// room updates are now sent for both participant joining and room metadata
+			require.GreaterOrEqual(t, fp.SendRoomUpdateCallCount(), 1)
+		}
+	})
+}
+
+type testRoomOpts struct {
+	num                  int
+	numHidden            int
+	protocol             types.ProtocolVersion
+	audioSmoothIntervals uint32
+}
+
+func newRoomWithParticipants(t *testing.T, opts testRoomOpts) *Room {
+	rm := NewRoom(
+		&livekit.Room{Name: "room"},
+		nil,
+		WebRTCConfig{},
+		config.RoomConfig{
+			EmptyTimeout:     5 * 60,
+			DepartureTimeout: 1,
+		},
+		&sfu.AudioConfig{
+			AudioLevelConfig: audio.AudioLevelConfig{
+				UpdateInterval:  audioUpdateInterval,
+				SmoothIntervals: opts.audioSmoothIntervals,
+			},
+		},
+		&livekit.ServerInfo{
+			Edition:  livekit.ServerInfo_Standard,
+			Version:  version.Version,
+			Protocol: types.CurrentProtocol,
+			NodeId:   "testnode",
+			Region:   "testregion",
+		},
+		telemetry.NewTelemetryService(webhook.NewDefaultNotifier("", "", nil), &telemetryfakes.FakeAnalyticsService{}),
+		nil, nil, nil,
+	)
+	for i := 0; i < opts.num+opts.numHidden; i++ {
+		identity := livekit.ParticipantIdentity(fmt.Sprintf("p%d", i))
+		participant := NewMockParticipant(identity, opts.protocol, i >= opts.num, true)
+		err := rm.Join(participant, nil, &ParticipantOptions{AutoSubscribe: true}, iceServersForRoom)
+		require.NoError(t, err)
+		participant.StateReturns(livekit.ParticipantInfo_ACTIVE)
+		participant.IsReadyReturns(true)
+		// each participant has a track
+		participant.GetPublishedTracksReturns([]types.MediaTrack{
+			&typesfakes.FakeMediaTrack{},
+		})
+	}
+	return rm
+}

livekit-server/pkg/rtc/roomtrackmanager.go

@@ -0,0 +1,125 @@
+/*
+ * Copyright 2023 LiveKit, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package rtc
+
+import (
+	"sync"
+
+	"github.com/livekit/livekit-server/pkg/rtc/types"
+	"github.com/livekit/livekit-server/pkg/utils"
+	"github.com/livekit/protocol/livekit"
+)
+
+// RoomTrackManager holds tracks that are published to the room
+type RoomTrackManager struct {
+	lock            sync.RWMutex
+	changedNotifier *utils.ChangeNotifierManager
+	removedNotifier *utils.ChangeNotifierManager
+	tracks          map[livekit.TrackID]*TrackInfo
+}
+
+type TrackInfo struct {
+	Track             types.MediaTrack
+	PublisherIdentity livekit.ParticipantIdentity
+	PublisherID       livekit.ParticipantID
+}
+
+func NewRoomTrackManager() *RoomTrackManager {
+	return &RoomTrackManager{
+		tracks:          make(map[livekit.TrackID]*TrackInfo),
+		changedNotifier: utils.NewChangeNotifierManager(),
+		removedNotifier: utils.NewChangeNotifierManager(),
+	}
+}
+
+func (r *RoomTrackManager) AddTrack(track types.MediaTrack, publisherIdentity livekit.ParticipantIdentity, publisherID livekit.ParticipantID) {
+	r.lock.Lock()
+	r.tracks[track.ID()] = &TrackInfo{
+		Track:             track,
+		PublisherIdentity: publisherIdentity,
+		PublisherID:       publisherID,
+	}
+	r.lock.Unlock()
+
+	r.NotifyTrackChanged(track.ID())
+}
+
+func (r *RoomTrackManager) RemoveTrack(track types.MediaTrack) {
+	r.lock.Lock()
+	// ensure we are removing the same track as added
+	info, ok := r.tracks[track.ID()]
+	if !ok || info.Track != track {
+		r.lock.Unlock()
+		return
+	}
+	delete(r.tracks, track.ID())
+	r.lock.Unlock()
+
+	n := r.removedNotifier.GetNotifier(string(track.ID()))
+	if n != nil {
+		n.NotifyChanged()
+	}
+
+	r.changedNotifier.RemoveNotifier(string(track.ID()), true)
+	r.removedNotifier.RemoveNotifier(string(track.ID()), true)
+}
+
+func (r *RoomTrackManager) GetTrackInfo(trackID livekit.TrackID) *TrackInfo {
+	r.lock.RLock()
+	defer r.lock.RUnlock()
+
+	info := r.tracks[trackID]
+	if info == nil {
+		return nil
+	}
+	// when track is about to close, do not resolve
+	if info.Track != nil && !info.Track.IsOpen() {
+		return nil
+	}
+	return info
+}
+
+func (r *RoomTrackManager) NotifyTrackChanged(trackID livekit.TrackID) {
+	n := r.changedNotifier.GetNotifier(string(trackID))
+	if n != nil {
+		n.NotifyChanged()
+	}
+}
+
+// HasObservers lets caller know if the current media track has any observers
+// this is used to signal interest in the track. when another MediaTrack with the same
+// trackID is being used, track is not considered to be observed.
+func (r *RoomTrackManager) HasObservers(track types.MediaTrack) bool {
+	n := r.changedNotifier.GetNotifier(string(track.ID()))
+	if n == nil || !n.HasObservers() {
+		return false
+	}
+
+	info := r.GetTrackInfo(track.ID())
+	if info == nil || info.Track != track {
+		return false
+	}
+	return true
+}
+
+func (r *RoomTrackManager) GetOrCreateTrackChangeNotifier(trackID livekit.TrackID) *utils.ChangeNotifier {
+	return r.changedNotifier.GetOrCreateNotifier(string(trackID))
+}
+
+func (r *RoomTrackManager) GetOrCreateTrackRemoveNotifier(trackID livekit.TrackID) *utils.ChangeNotifier {
+	return r.removedNotifier.GetOrCreateNotifier(string(trackID))
+}

livekit-server/pkg/rtc/signalhandler.go

@@ -0,0 +1,159 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package rtc
+
+import (
+	"github.com/livekit/protocol/livekit"
+	"github.com/livekit/protocol/logger"
+
+	"github.com/livekit/livekit-server/pkg/rtc/types"
+)
+
+func HandleParticipantSignal(room types.Room, participant types.LocalParticipant, req *livekit.SignalRequest, pLogger logger.Logger) error {
+	participant.UpdateLastSeenSignal()
+
+	switch msg := req.GetMessage().(type) {
+	case *livekit.SignalRequest_Offer:
+		participant.HandleOffer(FromProtoSessionDescription(msg.Offer))
+
+	case *livekit.SignalRequest_Answer:
+		participant.HandleAnswer(FromProtoSessionDescription(msg.Answer))
+
+	case *livekit.SignalRequest_Trickle:
+		candidateInit, err := FromProtoTrickle(msg.Trickle)
+		if err != nil {
+			pLogger.Warnw("could not decode trickle", err)
+			return nil
+		}
+		participant.AddICECandidate(candidateInit, msg.Trickle.Target)
+
+	case *livekit.SignalRequest_AddTrack:
+		pLogger.Debugw("add track request", "trackID", msg.AddTrack.Cid)
+		participant.AddTrack(msg.AddTrack)
+
+	case *livekit.SignalRequest_Mute:
+		participant.SetTrackMuted(livekit.TrackID(msg.Mute.Sid), msg.Mute.Muted, false)
+
+	case *livekit.SignalRequest_Subscription:
+		// allow participant to indicate their interest in the subscription
+		// permission check happens later in SubscriptionManager
+		room.UpdateSubscriptions(
+			participant,
+			livekit.StringsAsIDs[livekit.TrackID](msg.Subscription.TrackSids),
+			msg.Subscription.ParticipantTracks,
+			msg.Subscription.Subscribe,
+		)
+
+	case *livekit.SignalRequest_TrackSetting:
+		for _, sid := range livekit.StringsAsIDs[livekit.TrackID](msg.TrackSetting.TrackSids) {
+			participant.UpdateSubscribedTrackSettings(sid, msg.TrackSetting)
+		}
+
+	case *livekit.SignalRequest_Leave:
+		reason := types.ParticipantCloseReasonClientRequestLeave
+		switch msg.Leave.Reason {
+		case livekit.DisconnectReason_CLIENT_INITIATED:
+			reason = types.ParticipantCloseReasonClientRequestLeave
+		case livekit.DisconnectReason_USER_UNAVAILABLE:
+			reason = types.ParticipantCloseReasonUserUnavailable
+		case livekit.DisconnectReason_USER_REJECTED:
+			reason = types.ParticipantCloseReasonUserRejected
+		}
+		pLogger.Debugw("client leaving room", "reason", reason)
+		room.RemoveParticipant(participant.Identity(), participant.ID(), reason)
+
+	case *livekit.SignalRequest_SubscriptionPermission:
+		err := room.UpdateSubscriptionPermission(participant, msg.SubscriptionPermission)
+		if err != nil {
+			pLogger.Warnw("could not update subscription permission", err,
+				"permissions", msg.SubscriptionPermission)
+		}
+
+	case *livekit.SignalRequest_SyncState:
+		err := room.SyncState(participant, msg.SyncState)
+		if err != nil {
+			pLogger.Warnw("could not sync state", err,
+				"state", msg.SyncState)
+		}
+
+	case *livekit.SignalRequest_Simulate:
+		err := room.SimulateScenario(participant, msg.Simulate)
+		if err != nil {
+			pLogger.Warnw("could not simulate scenario", err,
+				"simulate", msg.Simulate)
+		}
+
+	case *livekit.SignalRequest_PingReq:
+		if msg.PingReq.Rtt > 0 {
+			participant.UpdateSignalingRTT(uint32(msg.PingReq.Rtt))
+		}
+
+	case *livekit.SignalRequest_UpdateMetadata:
+		requestResponse := &livekit.RequestResponse{
+			RequestId: msg.UpdateMetadata.RequestId,
+			Reason:    livekit.RequestResponse_OK,
+		}
+		if participant.ClaimGrants().Video.GetCanUpdateOwnMetadata() {
+			if err := participant.CheckMetadataLimits(
+				msg.UpdateMetadata.Name,
+				msg.UpdateMetadata.Metadata,
+				msg.UpdateMetadata.Attributes,
+			); err == nil {
+				if msg.UpdateMetadata.Name != "" {
+					participant.SetName(msg.UpdateMetadata.Name)
+				}
+				if msg.UpdateMetadata.Metadata != "" {
+					participant.SetMetadata(msg.UpdateMetadata.Metadata)
+				}
+				if msg.UpdateMetadata.Attributes != nil {
+					participant.SetAttributes(msg.UpdateMetadata.Attributes)
+				}
+			} else {
+				pLogger.Warnw("could not update metadata", err)
+
+				switch err {
+				case ErrNameExceedsLimits:
+					requestResponse.Reason = livekit.RequestResponse_LIMIT_EXCEEDED
+					requestResponse.Message = "exceeds name length limit"
+
+				case ErrMetadataExceedsLimits:
+					requestResponse.Reason = livekit.RequestResponse_LIMIT_EXCEEDED
+					requestResponse.Message = "exceeds metadata size limit"
+
+				case ErrAttributesExceedsLimits:
+					requestResponse.Reason = livekit.RequestResponse_LIMIT_EXCEEDED
+					requestResponse.Message = "exceeds attributes size limit"
+				}
+
+			}
+		} else {
+			requestResponse.Reason = livekit.RequestResponse_NOT_ALLOWED
+			requestResponse.Message = "does not have permission to update own metadata"
+		}
+		participant.SendRequestResponse(requestResponse)
+
+	case *livekit.SignalRequest_UpdateAudioTrack:
+		if err := participant.UpdateAudioTrack(msg.UpdateAudioTrack); err != nil {
+			pLogger.Warnw("could not update audio track", err, "update", msg.UpdateAudioTrack)
+		}
+
+	case *livekit.SignalRequest_UpdateVideoTrack:
+		if err := participant.UpdateVideoTrack(msg.UpdateVideoTrack); err != nil {
+			pLogger.Warnw("could not update video track", err, "update", msg.UpdateVideoTrack)
+		}
+	}
+
+	return nil
+}

livekit-server/pkg/rtc/subscribedtrack.go

@@ -0,0 +1,304 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package rtc
+
+import (
+	"sync"
+	"time"
+
+	"github.com/bep/debounce"
+	"github.com/pion/webrtc/v4"
+	"go.uber.org/atomic"
+	"google.golang.org/protobuf/proto"
+
+	sutils "github.com/livekit/livekit-server/pkg/utils"
+	"github.com/livekit/protocol/livekit"
+	"github.com/livekit/protocol/logger"
+	"github.com/livekit/protocol/utils"
+
+	"github.com/livekit/livekit-server/pkg/rtc/types"
+	"github.com/livekit/livekit-server/pkg/sfu"
+	"github.com/livekit/livekit-server/pkg/sfu/buffer"
+)
+
+const (
+	subscriptionDebounceInterval = 100 * time.Millisecond
+)
+
+type SubscribedTrackParams struct {
+	PublisherID       livekit.ParticipantID
+	PublisherIdentity livekit.ParticipantIdentity
+	PublisherVersion  uint32
+	Subscriber        types.LocalParticipant
+	MediaTrack        types.MediaTrack
+	DownTrack         *sfu.DownTrack
+	AdaptiveStream    bool
+}
+
+type SubscribedTrack struct {
+	params           SubscribedTrackParams
+	logger           logger.Logger
+	sender           atomic.Pointer[webrtc.RTPSender]
+	needsNegotiation atomic.Bool
+
+	versionGenerator utils.TimedVersionGenerator
+	settingsLock     sync.Mutex
+	settings         *livekit.UpdateTrackSettings
+	settingsVersion  utils.TimedVersion
+
+	bindLock        sync.Mutex
+	bound           bool
+	onBindCallbacks []func(error)
+
+	onClose atomic.Value // func(bool)
+
+	debouncer func(func())
+}
+
+func NewSubscribedTrack(params SubscribedTrackParams) *SubscribedTrack {
+	s := &SubscribedTrack{
+		params: params,
+		logger: params.Subscriber.GetLogger().WithComponent(sutils.ComponentSub).WithValues(
+			"trackID", params.DownTrack.ID(),
+			"publisherID", params.PublisherID,
+			"publisher", params.PublisherIdentity,
+		),
+		versionGenerator: utils.NewDefaultTimedVersionGenerator(),
+		debouncer:        debounce.New(subscriptionDebounceInterval),
+	}
+
+	return s
+}
+
+func (t *SubscribedTrack) AddOnBind(f func(error)) {
+	t.bindLock.Lock()
+	bound := t.bound
+	if !bound {
+		t.onBindCallbacks = append(t.onBindCallbacks, f)
+	}
+	t.bindLock.Unlock()
+
+	if bound {
+		// fire immediately, do not need to persist since bind is a one time event
+		go f(nil)
+	}
+}
+
+// for DownTrack callback to notify us that it's bound
+func (t *SubscribedTrack) Bound(err error) {
+	t.bindLock.Lock()
+	if err == nil {
+		t.bound = true
+	}
+	callbacks := t.onBindCallbacks
+	t.onBindCallbacks = nil
+	t.bindLock.Unlock()
+
+	if err == nil && t.MediaTrack().Kind() == livekit.TrackType_VIDEO {
+		// When AdaptiveStream is enabled, default the subscriber to LOW quality stream
+		// we would want LOW instead of OFF for a couple of reasons
+		// 1. when a subscriber unsubscribes from a track, we would forget their previously defined settings
+		//    depending on client implementation, subscription on/off is kept separately from adaptive stream
+		//    So when there are no changes to desired resolution, but the user re-subscribes, we may leave stream at OFF
+		// 2. when interacting with dynacast *and* adaptive stream. If the publisher was not publishing at the
+		//    time of subscription, we might not be able to trigger adaptive stream updates on the client side
+		//    (since there isn't any video frames coming through). this will leave the stream "stuck" on off, without
+		//    a trigger to re-enable it
+		t.settingsLock.Lock()
+		if t.settings != nil {
+			if t.params.AdaptiveStream {
+				// remove `disabled` flag to force a visibility update
+				t.settings.Disabled = false
+			}
+		} else {
+			if t.params.AdaptiveStream {
+				t.settings = &livekit.UpdateTrackSettings{Quality: livekit.VideoQuality_LOW}
+			} else {
+				t.settings = &livekit.UpdateTrackSettings{Quality: livekit.VideoQuality_HIGH}
+			}
+		}
+		t.settingsLock.Unlock()
+		t.applySettings()
+	}
+
+	for _, cb := range callbacks {
+		go cb(err)
+	}
+}
+
+// for DownTrack callback to notify us that it's closed
+func (t *SubscribedTrack) Close(isExpectedToResume bool) {
+	if onClose := t.onClose.Load(); onClose != nil {
+		go onClose.(func(bool))(isExpectedToResume)
+	}
+}
+
+func (t *SubscribedTrack) OnClose(f func(bool)) {
+	t.onClose.Store(f)
+}
+
+func (t *SubscribedTrack) IsBound() bool {
+	t.bindLock.Lock()
+	defer t.bindLock.Unlock()
+
+	return t.bound
+}
+
+func (t *SubscribedTrack) ID() livekit.TrackID {
+	return livekit.TrackID(t.params.DownTrack.ID())
+}
+
+func (t *SubscribedTrack) PublisherID() livekit.ParticipantID {
+	return t.params.PublisherID
+}
+
+func (t *SubscribedTrack) PublisherIdentity() livekit.ParticipantIdentity {
+	return t.params.PublisherIdentity
+}
+
+func (t *SubscribedTrack) PublisherVersion() uint32 {
+	return t.params.PublisherVersion
+}
+
+func (t *SubscribedTrack) SubscriberID() livekit.ParticipantID {
+	return t.params.Subscriber.ID()
+}
+
+func (t *SubscribedTrack) SubscriberIdentity() livekit.ParticipantIdentity {
+	return t.params.Subscriber.Identity()
+}
+
+func (t *SubscribedTrack) Subscriber() types.LocalParticipant {
+	return t.params.Subscriber
+}
+
+func (t *SubscribedTrack) DownTrack() *sfu.DownTrack {
+	return t.params.DownTrack
+}
+
+func (t *SubscribedTrack) MediaTrack() types.MediaTrack {
+	return t.params.MediaTrack
+}
+
+// has subscriber indicated it wants to mute this track
+func (t *SubscribedTrack) IsMuted() bool {
+	t.settingsLock.Lock()
+	defer t.settingsLock.Unlock()
+
+	return t.isMutedLocked()
+}
+
+func (t *SubscribedTrack) isMutedLocked() bool {
+	if t.settings == nil {
+		return false
+	}
+
+	return t.settings.Disabled
+}
+
+func (t *SubscribedTrack) SetPublisherMuted(muted bool) {
+	t.DownTrack().PubMute(muted)
+}
+
+func (t *SubscribedTrack) UpdateSubscriberSettings(settings *livekit.UpdateTrackSettings, isImmediate bool) {
+	t.settingsLock.Lock()
+	if proto.Equal(t.settings, settings) {
+		t.settingsLock.Unlock()
+		return
+	}
+
+	isImmediate = isImmediate || (!settings.Disabled && settings.Disabled != t.isMutedLocked())
+	t.settings = utils.CloneProto(settings)
+	t.settingsLock.Unlock()
+
+	if isImmediate {
+		t.applySettings()
+	} else {
+		// avoid frequent changes to mute & video layers, unless it became visible
+		t.debouncer(t.applySettings)
+	}
+}
+
+func (t *SubscribedTrack) UpdateVideoLayer() {
+	t.applySettings()
+}
+
+func (t *SubscribedTrack) applySettings() {
+	t.settingsLock.Lock()
+	if t.settings == nil {
+		t.settingsLock.Unlock()
+		return
+	}
+
+	t.logger.Debugw("updating subscriber track settings", "settings", logger.Proto(t.settings))
+	t.settingsVersion = t.versionGenerator.Next()
+	settingsVersion := t.settingsVersion
+	t.settingsLock.Unlock()
+
+	dt := t.DownTrack()
+	spatial := buffer.InvalidLayerSpatial
+	temporal := buffer.InvalidLayerTemporal
+	if dt.Kind() == webrtc.RTPCodecTypeVideo {
+		mt := t.MediaTrack()
+		quality := t.settings.Quality
+		if t.settings.Width > 0 {
+			quality = mt.GetQualityForDimension(t.settings.Width, t.settings.Height)
+		}
+
+		spatial = buffer.VideoQualityToSpatialLayer(quality, mt.ToProto())
+		if t.settings.Fps > 0 {
+			temporal = mt.GetTemporalLayerForSpatialFps(spatial, t.settings.Fps, dt.Mime())
+		}
+	}
+
+	t.settingsLock.Lock()
+	if settingsVersion != t.settingsVersion {
+		// a newer settings has superceded this one
+		t.settingsLock.Unlock()
+		return
+	}
+
+	if t.settings.Disabled {
+		dt.Mute(true)
+		t.settingsLock.Unlock()
+		return
+	} else {
+		dt.Mute(false)
+	}
+
+	if dt.Kind() == webrtc.RTPCodecTypeVideo {
+		dt.SetMaxSpatialLayer(spatial)
+		if temporal != buffer.InvalidLayerTemporal {
+			dt.SetMaxTemporalLayer(temporal)
+		}
+	}
+	t.settingsLock.Unlock()
+}
+
+func (t *SubscribedTrack) NeedsNegotiation() bool {
+	return t.needsNegotiation.Load()
+}
+
+func (t *SubscribedTrack) SetNeedsNegotiation(needs bool) {
+	t.needsNegotiation.Store(needs)
+}
+
+func (t *SubscribedTrack) RTPSender() *webrtc.RTPSender {
+	return t.sender.Load()
+}
+
+func (t *SubscribedTrack) SetRTPSender(sender *webrtc.RTPSender) {
+	t.sender.Store(sender)
+}

livekit-server/pkg/rtc/subscriptionmanager_test.go

@@ -0,0 +1,545 @@
+/*
+ * Copyright 2023 LiveKit, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package rtc
+
+import (
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+	"go.uber.org/atomic"
+
+	"github.com/livekit/livekit-server/pkg/rtc/types"
+	"github.com/livekit/livekit-server/pkg/rtc/types/typesfakes"
+	"github.com/livekit/livekit-server/pkg/telemetry/telemetryfakes"
+	"github.com/livekit/livekit-server/pkg/utils"
+	"github.com/livekit/protocol/livekit"
+	"github.com/livekit/protocol/logger"
+)
+
+func init() {
+	reconcileInterval = 50 * time.Millisecond
+	notFoundTimeout = 200 * time.Millisecond
+	subscriptionTimeout = 200 * time.Millisecond
+}
+
+const (
+	subSettleTimeout = 600 * time.Millisecond
+	subCheckInterval = 10 * time.Millisecond
+)
+
+func TestSubscribe(t *testing.T) {
+	t.Run("happy path subscribe", func(t *testing.T) {
+		sm := newTestSubscriptionManager()
+		defer sm.Close(false)
+		resolver := newTestResolver(true, true, "pub", "pubID")
+		sm.params.TrackResolver = resolver.Resolve
+		subCount := atomic.Int32{}
+		failed := atomic.Bool{}
+		sm.params.OnTrackSubscribed = func(subTrack types.SubscribedTrack) {
+			subCount.Add(1)
+		}
+		sm.params.OnSubscriptionError = func(trackID livekit.TrackID, fatal bool, err error) {
+			failed.Store(true)
+		}
+		numParticipantSubscribed := atomic.Int32{}
+		numParticipantUnsubscribed := atomic.Int32{}
+		sm.OnSubscribeStatusChanged(func(pubID livekit.ParticipantID, subscribed bool) {
+			if subscribed {
+				numParticipantSubscribed.Add(1)
+			} else {
+				numParticipantUnsubscribed.Add(1)
+			}
+		})
+
+		sm.SubscribeToTrack("track")
+		s := sm.subscriptions["track"]
+		require.True(t, s.isDesired())
+		require.Eventually(t, func() bool {
+			return subCount.Load() == 1
+		}, subSettleTimeout, subCheckInterval, "track was not subscribed")
+
+		require.NotNil(t, s.getSubscribedTrack())
+		require.Len(t, sm.GetSubscribedTracks(), 1)
+
+		require.Eventually(t, func() bool {
+			return len(sm.GetSubscribedParticipants()) == 1
+		}, subSettleTimeout, subCheckInterval, "GetSubscribedParticipants should have returned one item")
+		require.Equal(t, "pubID", string(sm.GetSubscribedParticipants()[0]))
+
+		// ensure telemetry events are sent
+		tm := sm.params.Telemetry.(*telemetryfakes.FakeTelemetryService)
+		require.Equal(t, 1, tm.TrackSubscribeRequestedCallCount())
+
+		// ensure bound
+		setTestSubscribedTrackBound(t, s.getSubscribedTrack())
+		require.Eventually(t, func() bool {
+			return !s.needsBind()
+		}, subSettleTimeout, subCheckInterval, "track was not bound")
+
+		// telemetry event should have been sent
+		require.Equal(t, 1, tm.TrackSubscribedCallCount())
+
+		time.Sleep(notFoundTimeout)
+		require.False(t, failed.Load())
+
+		resolver.SetPause(true)
+		// ensure its resilience after being closed
+		setTestSubscribedTrackClosed(t, s.getSubscribedTrack(), false)
+		require.Eventually(t, func() bool {
+			return s.needsSubscribe()
+		}, subSettleTimeout, subCheckInterval, "needs subscribe did not persist across track close")
+		resolver.SetPause(false)
+
+		require.Eventually(t, func() bool {
+			return s.isDesired() && !s.needsSubscribe()
+		}, subSettleTimeout, subCheckInterval, "track was not resubscribed")
+
+		// was subscribed twice, unsubscribed once (due to close)
+		require.Eventually(t, func() bool {
+			return numParticipantSubscribed.Load() == 2
+		}, subSettleTimeout, subCheckInterval, "participant subscribe status was not updated twice")
+		require.Equal(t, int32(1), numParticipantUnsubscribed.Load())
+	})
+
+	t.Run("no track permission", func(t *testing.T) {
+		sm := newTestSubscriptionManager()
+		defer sm.Close(false)
+		resolver := newTestResolver(false, true, "pub", "pubID")
+		sm.params.TrackResolver = resolver.Resolve
+		failed := atomic.Bool{}
+		sm.params.OnSubscriptionError = func(trackID livekit.TrackID, fatal bool, err error) {
+			failed.Store(true)
+		}
+
+		sm.SubscribeToTrack("track")
+		s := sm.subscriptions["track"]
+		require.Eventually(t, func() bool {
+			return !s.getHasPermission()
+		}, subSettleTimeout, subCheckInterval, "should not have permission to subscribe")
+
+		time.Sleep(subscriptionTimeout)
+
+		// should not have called failed callbacks, isDesired remains unchanged
+		require.True(t, s.isDesired())
+		require.False(t, failed.Load())
+		require.True(t, s.needsSubscribe())
+		require.Len(t, sm.GetSubscribedTracks(), 0)
+
+		// trackSubscribed telemetry not sent
+		tm := sm.params.Telemetry.(*telemetryfakes.FakeTelemetryService)
+		require.Equal(t, 1, tm.TrackSubscribeRequestedCallCount())
+		require.Equal(t, 0, tm.TrackSubscribedCallCount())
+
+		// give permissions now
+		resolver.lock.Lock()
+		resolver.hasPermission = true
+		resolver.lock.Unlock()
+
+		require.Eventually(t, func() bool {
+			return !s.needsSubscribe()
+		}, subSettleTimeout, subCheckInterval, "should be subscribed")
+
+		require.Len(t, sm.GetSubscribedTracks(), 1)
+	})
+
+	t.Run("publisher left", func(t *testing.T) {
+		sm := newTestSubscriptionManager()
+		defer sm.Close(false)
+		resolver := newTestResolver(true, true, "pub", "pubID")
+		sm.params.TrackResolver = resolver.Resolve
+		failed := atomic.Bool{}
+		sm.params.OnSubscriptionError = func(trackID livekit.TrackID, fatal bool, err error) {
+			failed.Store(true)
+		}
+
+		sm.SubscribeToTrack("track")
+		s := sm.subscriptions["track"]
+		require.Eventually(t, func() bool {
+			return !s.needsSubscribe()
+		}, subSettleTimeout, subCheckInterval, "should be subscribed")
+
+		resolver.lock.Lock()
+		resolver.hasTrack = false
+		resolver.lock.Unlock()
+
+		// publisher triggers close
+		setTestSubscribedTrackClosed(t, s.getSubscribedTrack(), false)
+
+		require.Eventually(t, func() bool {
+			return !s.isDesired()
+		}, subSettleTimeout, subCheckInterval, "isDesired not set to false")
+	})
+}
+
+func TestUnsubscribe(t *testing.T) {
+	sm := newTestSubscriptionManager()
+	defer sm.Close(false)
+	unsubCount := atomic.Int32{}
+	sm.params.OnTrackUnsubscribed = func(subTrack types.SubscribedTrack) {
+		unsubCount.Add(1)
+	}
+
+	resolver := newTestResolver(true, true, "pub", "pubID")
+
+	s := &trackSubscription{
+		trackID:           "track",
+		desired:           true,
+		subscriberID:      sm.params.Participant.ID(),
+		publisherID:       "pubID",
+		publisherIdentity: "pub",
+		hasPermission:     true,
+		bound:             true,
+		logger:            logger.GetLogger(),
+	}
+	// a bunch of unfortunate manual wiring
+	res := resolver.Resolve(nil, s.trackID)
+	res.TrackChangedNotifier.AddObserver(string(sm.params.Participant.ID()), func() {})
+	s.changedNotifier = res.TrackChangedNotifier
+	st, err := res.Track.AddSubscriber(sm.params.Participant)
+	require.NoError(t, err)
+	s.subscribedTrack = st
+	st.OnClose(func(isExpectedToResume bool) {
+		sm.handleSubscribedTrackClose(s, isExpectedToResume)
+	})
+	res.Track.(*typesfakes.FakeMediaTrack).RemoveSubscriberCalls(func(pID livekit.ParticipantID, isExpectedToResume bool) {
+		setTestSubscribedTrackClosed(t, st, isExpectedToResume)
+	})
+
+	sm.lock.Lock()
+	sm.subscriptions["track"] = s
+	sm.lock.Unlock()
+
+	require.False(t, s.needsSubscribe())
+	require.False(t, s.needsUnsubscribe())
+
+	// unsubscribe
+	sm.UnsubscribeFromTrack("track")
+	require.False(t, s.isDesired())
+
+	require.Eventually(t, func() bool {
+		if s.needsUnsubscribe() {
+			return false
+		}
+		if sm.pendingUnsubscribes.Load() != 0 {
+			return false
+		}
+		sm.lock.RLock()
+		subLen := len(sm.subscriptions)
+		sm.lock.RUnlock()
+		if subLen != 0 {
+			return false
+		}
+		return true
+	}, subSettleTimeout, subCheckInterval, "Track was not unsubscribed")
+
+	// no traces should be left
+	require.Len(t, sm.GetSubscribedTracks(), 0)
+	require.False(t, res.TrackChangedNotifier.HasObservers())
+
+	tm := sm.params.Telemetry.(*telemetryfakes.FakeTelemetryService)
+	require.Equal(t, 1, tm.TrackUnsubscribedCallCount())
+}
+
+func TestSubscribeStatusChanged(t *testing.T) {
+	sm := newTestSubscriptionManager()
+	defer sm.Close(false)
+	resolver := newTestResolver(true, true, "pub", "pubID")
+	sm.params.TrackResolver = resolver.Resolve
+	numParticipantSubscribed := atomic.Int32{}
+	numParticipantUnsubscribed := atomic.Int32{}
+	sm.OnSubscribeStatusChanged(func(pubID livekit.ParticipantID, subscribed bool) {
+		if subscribed {
+			numParticipantSubscribed.Add(1)
+		} else {
+			numParticipantUnsubscribed.Add(1)
+		}
+	})
+
+	sm.SubscribeToTrack("track1")
+	sm.SubscribeToTrack("track2")
+	s1 := sm.subscriptions["track1"]
+	s2 := sm.subscriptions["track2"]
+	require.Eventually(t, func() bool {
+		return !s1.needsSubscribe() && !s2.needsSubscribe()
+	}, subSettleTimeout, subCheckInterval, "track1 and track2 should be subscribed")
+	st1 := s1.getSubscribedTrack()
+	st1.OnClose(func(isExpectedToResume bool) {
+		sm.handleSubscribedTrackClose(s1, isExpectedToResume)
+	})
+	st2 := s2.getSubscribedTrack()
+	st2.OnClose(func(isExpectedToResume bool) {
+		sm.handleSubscribedTrackClose(s2, isExpectedToResume)
+	})
+	st1.MediaTrack().(*typesfakes.FakeMediaTrack).RemoveSubscriberCalls(func(pID livekit.ParticipantID, isExpectedToResume bool) {
+		setTestSubscribedTrackClosed(t, st1, isExpectedToResume)
+	})
+	st2.MediaTrack().(*typesfakes.FakeMediaTrack).RemoveSubscriberCalls(func(pID livekit.ParticipantID, isExpectedToResume bool) {
+		setTestSubscribedTrackClosed(t, st2, isExpectedToResume)
+	})
+
+	require.Eventually(t, func() bool {
+		return numParticipantSubscribed.Load() == 1
+	}, subSettleTimeout, subCheckInterval, "should be subscribed to publisher")
+	require.Equal(t, int32(0), numParticipantUnsubscribed.Load())
+	require.True(t, sm.IsSubscribedTo("pubID"))
+
+	// now unsubscribe track2, no event should be fired
+	sm.UnsubscribeFromTrack("track2")
+	require.Eventually(t, func() bool {
+		return !s2.needsUnsubscribe()
+	}, subSettleTimeout, subCheckInterval, "track2 should be unsubscribed")
+	require.Equal(t, int32(0), numParticipantUnsubscribed.Load())
+
+	// unsubscribe track1, expect event
+	sm.UnsubscribeFromTrack("track1")
+	require.Eventually(t, func() bool {
+		return !s1.needsUnsubscribe()
+	}, subSettleTimeout, subCheckInterval, "track1 should be unsubscribed")
+	require.Eventually(t, func() bool {
+		return numParticipantUnsubscribed.Load() == 1
+	}, subSettleTimeout, subCheckInterval, "should be subscribed to publisher")
+	require.False(t, sm.IsSubscribedTo("pubID"))
+}
+
+// clients may send update subscribed settings prior to subscription events coming through
+// settings should be persisted and used when the subscription does take place.
+func TestUpdateSettingsBeforeSubscription(t *testing.T) {
+	sm := newTestSubscriptionManager()
+	defer sm.Close(false)
+	resolver := newTestResolver(true, true, "pub", "pubID")
+	sm.params.TrackResolver = resolver.Resolve
+
+	settings := &livekit.UpdateTrackSettings{
+		Disabled: true,
+		Width:    100,
+		Height:   100,
+	}
+	sm.UpdateSubscribedTrackSettings("track", settings)
+
+	sm.SubscribeToTrack("track")
+
+	s := sm.subscriptions["track"]
+	require.Eventually(t, func() bool {
+		return !s.needsSubscribe()
+	}, subSettleTimeout, subCheckInterval, "Track should be subscribed")
+
+	st := s.getSubscribedTrack().(*typesfakes.FakeSubscribedTrack)
+	require.Eventually(t, func() bool {
+		return st.UpdateSubscriberSettingsCallCount() == 1
+	}, subSettleTimeout, subCheckInterval, "UpdateSubscriberSettings should be called once")
+
+	applied, _ := st.UpdateSubscriberSettingsArgsForCall(0)
+	require.Equal(t, settings.Disabled, applied.Disabled)
+	require.Equal(t, settings.Width, applied.Width)
+	require.Equal(t, settings.Height, applied.Height)
+}
+
+func TestSubscriptionLimits(t *testing.T) {
+	sm := newTestSubscriptionManagerWithParams(testSubscriptionParams{
+		SubscriptionLimitAudio: 1,
+		SubscriptionLimitVideo: 1,
+	})
+	defer sm.Close(false)
+	resolver := newTestResolver(true, true, "pub", "pubID")
+	sm.params.TrackResolver = resolver.Resolve
+	subCount := atomic.Int32{}
+	failed := atomic.Bool{}
+	sm.params.OnTrackSubscribed = func(subTrack types.SubscribedTrack) {
+		subCount.Add(1)
+	}
+	sm.params.OnSubscriptionError = func(trackID livekit.TrackID, fatal bool, err error) {
+		failed.Store(true)
+	}
+	numParticipantSubscribed := atomic.Int32{}
+	numParticipantUnsubscribed := atomic.Int32{}
+	sm.OnSubscribeStatusChanged(func(pubID livekit.ParticipantID, subscribed bool) {
+		if subscribed {
+			numParticipantSubscribed.Add(1)
+		} else {
+			numParticipantUnsubscribed.Add(1)
+		}
+	})
+
+	sm.SubscribeToTrack("track")
+	s := sm.subscriptions["track"]
+	require.True(t, s.isDesired())
+	require.Eventually(t, func() bool {
+		return subCount.Load() == 1
+	}, subSettleTimeout, subCheckInterval, "track was not subscribed")
+
+	require.NotNil(t, s.getSubscribedTrack())
+	require.Len(t, sm.GetSubscribedTracks(), 1)
+
+	require.Eventually(t, func() bool {
+		return len(sm.GetSubscribedParticipants()) == 1
+	}, subSettleTimeout, subCheckInterval, "GetSubscribedParticipants should have returned one item")
+	require.Equal(t, "pubID", string(sm.GetSubscribedParticipants()[0]))
+
+	// ensure telemetry events are sent
+	tm := sm.params.Telemetry.(*telemetryfakes.FakeTelemetryService)
+	require.Equal(t, 1, tm.TrackSubscribeRequestedCallCount())
+
+	// ensure bound
+	setTestSubscribedTrackBound(t, s.getSubscribedTrack())
+	require.Eventually(t, func() bool {
+		return !s.needsBind()
+	}, subSettleTimeout, subCheckInterval, "track was not bound")
+
+	// telemetry event should have been sent
+	require.Equal(t, 1, tm.TrackSubscribedCallCount())
+
+	// reach subscription limit, subscribe pending
+	sm.SubscribeToTrack("track2")
+	s2 := sm.subscriptions["track2"]
+	time.Sleep(subscriptionTimeout * 2)
+	require.True(t, s2.needsSubscribe())
+	require.Equal(t, 2, tm.TrackSubscribeRequestedCallCount())
+	require.Equal(t, 1, tm.TrackSubscribeFailedCallCount())
+	require.Len(t, sm.GetSubscribedTracks(), 1)
+
+	// unsubscribe track1, then track2 should be subscribed
+	sm.UnsubscribeFromTrack("track")
+	require.False(t, s.isDesired())
+	require.True(t, s.needsUnsubscribe())
+	// wait for unsubscribe to take effect
+	time.Sleep(reconcileInterval)
+	setTestSubscribedTrackClosed(t, s.getSubscribedTrack(), false)
+	require.Nil(t, s.getSubscribedTrack())
+
+	time.Sleep(reconcileInterval)
+	require.True(t, s2.isDesired())
+	require.False(t, s2.needsSubscribe())
+	require.EqualValues(t, 2, subCount.Load())
+	require.NotNil(t, s2.getSubscribedTrack())
+	require.Equal(t, 2, tm.TrackSubscribeRequestedCallCount())
+	require.Len(t, sm.GetSubscribedTracks(), 1)
+
+	// ensure bound
+	setTestSubscribedTrackBound(t, s2.getSubscribedTrack())
+	require.Eventually(t, func() bool {
+		return !s2.needsBind()
+	}, subSettleTimeout, subCheckInterval, "track was not bound")
+
+	// subscribe to track1 again, which should pending
+	sm.SubscribeToTrack("track")
+	s = sm.subscriptions["track"]
+	require.True(t, s.isDesired())
+	time.Sleep(subscriptionTimeout * 2)
+	require.True(t, s.needsSubscribe())
+	require.Equal(t, 3, tm.TrackSubscribeRequestedCallCount())
+	require.Equal(t, 2, tm.TrackSubscribeFailedCallCount())
+	require.Len(t, sm.GetSubscribedTracks(), 1)
+}
+
+type testSubscriptionParams struct {
+	SubscriptionLimitAudio int32
+	SubscriptionLimitVideo int32
+}
+
+func newTestSubscriptionManager() *SubscriptionManager {
+	return newTestSubscriptionManagerWithParams(testSubscriptionParams{})
+}
+
+func newTestSubscriptionManagerWithParams(params testSubscriptionParams) *SubscriptionManager {
+	p := &typesfakes.FakeLocalParticipant{}
+	p.CanSubscribeReturns(true)
+	p.IDReturns("subID")
+	p.IdentityReturns("sub")
+	p.KindReturns(livekit.ParticipantInfo_STANDARD)
+	return NewSubscriptionManager(SubscriptionManagerParams{
+		Participant:         p,
+		Logger:              logger.GetLogger(),
+		OnTrackSubscribed:   func(subTrack types.SubscribedTrack) {},
+		OnTrackUnsubscribed: func(subTrack types.SubscribedTrack) {},
+		OnSubscriptionError: func(trackID livekit.TrackID, fatal bool, err error) {},
+		TrackResolver: func(sub types.LocalParticipant, trackID livekit.TrackID) types.MediaResolverResult {
+			return types.MediaResolverResult{}
+		},
+		Telemetry:              &telemetryfakes.FakeTelemetryService{},
+		SubscriptionLimitAudio: params.SubscriptionLimitAudio,
+		SubscriptionLimitVideo: params.SubscriptionLimitVideo,
+	})
+}
+
+type testResolver struct {
+	lock          sync.Mutex
+	hasPermission bool
+	hasTrack      bool
+	pubIdentity   livekit.ParticipantIdentity
+	pubID         livekit.ParticipantID
+
+	paused bool
+}
+
+func newTestResolver(hasPermission bool, hasTrack bool, pubIdentity livekit.ParticipantIdentity, pubID livekit.ParticipantID) *testResolver {
+	return &testResolver{
+		hasPermission: hasPermission,
+		hasTrack:      hasTrack,
+		pubIdentity:   pubIdentity,
+		pubID:         pubID,
+	}
+}
+
+func (t *testResolver) SetPause(paused bool) {
+	t.lock.Lock()
+	defer t.lock.Unlock()
+	t.paused = paused
+}
+
+func (t *testResolver) Resolve(_subscriber types.LocalParticipant, trackID livekit.TrackID) types.MediaResolverResult {
+	t.lock.Lock()
+	defer t.lock.Unlock()
+	res := types.MediaResolverResult{
+		TrackChangedNotifier: utils.NewChangeNotifier(),
+		TrackRemovedNotifier: utils.NewChangeNotifier(),
+		HasPermission:        t.hasPermission,
+		PublisherID:          t.pubID,
+		PublisherIdentity:    t.pubIdentity,
+	}
+	if t.hasTrack && !t.paused {
+		mt := &typesfakes.FakeMediaTrack{}
+		st := &typesfakes.FakeSubscribedTrack{}
+		st.IDReturns(trackID)
+		st.PublisherIDReturns(t.pubID)
+		st.PublisherIdentityReturns(t.pubIdentity)
+		mt.AddSubscriberCalls(func(sub types.LocalParticipant) (types.SubscribedTrack, error) {
+			st.SubscriberReturns(sub)
+			return st, nil
+		})
+		st.MediaTrackReturns(mt)
+		res.Track = mt
+	}
+	return res
+}
+
+func setTestSubscribedTrackBound(t *testing.T, st types.SubscribedTrack) {
+	fst, ok := st.(*typesfakes.FakeSubscribedTrack)
+	require.True(t, ok)
+
+	for i := 0; i < fst.AddOnBindCallCount(); i++ {
+		fst.AddOnBindArgsForCall(i)(nil)
+	}
+}
+
+func setTestSubscribedTrackClosed(t *testing.T, st types.SubscribedTrack, isExpectedToResume bool) {
+	fst, ok := st.(*typesfakes.FakeSubscribedTrack)
+	require.True(t, ok)
+
+	fst.OnCloseArgsForCall(0)(isExpectedToResume)
+}

livekit-server/pkg/rtc/supervisor/participant_supervisor.go

@@ -0,0 +1,183 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package supervisor
+
+import (
+	"sync"
+	"time"
+
+	"go.uber.org/atomic"
+
+	"github.com/livekit/livekit-server/pkg/rtc/types"
+	"github.com/livekit/protocol/livekit"
+	"github.com/livekit/protocol/logger"
+)
+
+const (
+	monitorInterval = 1 * time.Second
+)
+
+type ParticipantSupervisorParams struct {
+	Logger logger.Logger
+}
+
+type trackMonitor struct {
+	opMon types.OperationMonitor
+	err   error
+}
+
+type ParticipantSupervisor struct {
+	params ParticipantSupervisorParams
+
+	lock                 sync.RWMutex
+	isPublisherConnected bool
+	publications         map[livekit.TrackID]*trackMonitor
+
+	isStopped atomic.Bool
+
+	onPublicationError func(trackID livekit.TrackID)
+}
+
+func NewParticipantSupervisor(params ParticipantSupervisorParams) *ParticipantSupervisor {
+	p := &ParticipantSupervisor{
+		params:       params,
+		publications: make(map[livekit.TrackID]*trackMonitor),
+	}
+
+	go p.checkState()
+
+	return p
+}
+
+func (p *ParticipantSupervisor) Stop() {
+	p.isStopped.Store(true)
+}
+
+func (p *ParticipantSupervisor) OnPublicationError(f func(trackID livekit.TrackID)) {
+	p.lock.Lock()
+	defer p.lock.Unlock()
+
+	p.onPublicationError = f
+}
+
+func (p *ParticipantSupervisor) getOnPublicationError() func(trackID livekit.TrackID) {
+	p.lock.RLock()
+	defer p.lock.RUnlock()
+
+	return p.onPublicationError
+}
+
+func (p *ParticipantSupervisor) SetPublisherPeerConnectionConnected(isConnected bool) {
+	p.lock.Lock()
+	p.isPublisherConnected = isConnected
+
+	for _, pm := range p.publications {
+		pm.opMon.PostEvent(types.OperationMonitorEventPublisherPeerConnectionConnected, p.isPublisherConnected)
+	}
+	p.lock.Unlock()
+}
+
+func (p *ParticipantSupervisor) AddPublication(trackID livekit.TrackID) {
+	p.lock.Lock()
+	pm, ok := p.publications[trackID]
+	if !ok {
+		pm = &trackMonitor{
+			opMon: NewPublicationMonitor(
+				PublicationMonitorParams{
+					TrackID:                   trackID,
+					IsPeerConnectionConnected: p.isPublisherConnected,
+					Logger:                    p.params.Logger,
+				},
+			),
+		}
+		p.publications[trackID] = pm
+	}
+	pm.opMon.PostEvent(types.OperationMonitorEventAddPendingPublication, nil)
+	p.lock.Unlock()
+}
+
+func (p *ParticipantSupervisor) SetPublicationMute(trackID livekit.TrackID, isMuted bool) {
+	p.lock.Lock()
+	pm, ok := p.publications[trackID]
+	if ok {
+		pm.opMon.PostEvent(types.OperationMonitorEventSetPublicationMute, isMuted)
+	}
+	p.lock.Unlock()
+}
+
+func (p *ParticipantSupervisor) SetPublishedTrack(trackID livekit.TrackID, pubTrack types.LocalMediaTrack) {
+	p.lock.RLock()
+	pm, ok := p.publications[trackID]
+	if ok {
+		pm.opMon.PostEvent(types.OperationMonitorEventSetPublishedTrack, pubTrack)
+	}
+	p.lock.RUnlock()
+}
+
+func (p *ParticipantSupervisor) ClearPublishedTrack(trackID livekit.TrackID, pubTrack types.LocalMediaTrack) {
+	p.lock.RLock()
+	pm, ok := p.publications[trackID]
+	if ok {
+		pm.opMon.PostEvent(types.OperationMonitorEventClearPublishedTrack, pubTrack)
+	}
+	p.lock.RUnlock()
+}
+
+func (p *ParticipantSupervisor) checkState() {
+	ticker := time.NewTicker(monitorInterval)
+	defer ticker.Stop()
+
+	for !p.isStopped.Load() {
+		<-ticker.C
+
+		p.checkPublications()
+	}
+}
+
+func (p *ParticipantSupervisor) checkPublications() {
+	var erroredPublications []livekit.TrackID
+	var removablePublications []livekit.TrackID
+	p.lock.RLock()
+	for trackID, pm := range p.publications {
+		if err := pm.opMon.Check(); err != nil {
+			if pm.err == nil {
+				p.params.Logger.Errorw("supervisor error on publication", err, "trackID", trackID)
+				pm.err = err
+				erroredPublications = append(erroredPublications, trackID)
+			}
+		} else {
+			if pm.err != nil {
+				p.params.Logger.Infow("supervisor publication recovered", "trackID", trackID)
+				pm.err = err
+			}
+			if pm.opMon.IsIdle() {
+				removablePublications = append(removablePublications, trackID)
+			}
+		}
+	}
+	p.lock.RUnlock()
+
+	p.lock.Lock()
+	for _, trackID := range removablePublications {
+		delete(p.publications, trackID)
+	}
+	p.lock.Unlock()
+
+	if onPublicationError := p.getOnPublicationError(); onPublicationError != nil {
+		for _, trackID := range erroredPublications {
+			onPublicationError(trackID)
+		}
+	}
+}

livekit-server/pkg/rtc/supervisor/publication_monitor.go

@@ -0,0 +1,190 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package supervisor
+
+import (
+	"errors"
+	"sync"
+	"time"
+
+	"github.com/gammazero/deque"
+
+	"github.com/livekit/livekit-server/pkg/rtc/types"
+	"github.com/livekit/protocol/livekit"
+	"github.com/livekit/protocol/logger"
+)
+
+const (
+	publishWaitDuration = 30 * time.Second
+)
+
+var (
+	errPublishTimeout = errors.New("publish time out")
+)
+
+type publish struct {
+	isStart bool
+}
+
+type PublicationMonitorParams struct {
+	TrackID                   livekit.TrackID
+	IsPeerConnectionConnected bool
+	Logger                    logger.Logger
+}
+
+type PublicationMonitor struct {
+	params PublicationMonitorParams
+
+	lock             sync.RWMutex
+	desiredPublishes deque.Deque[*publish]
+
+	isConnected bool
+
+	publishedTrack types.LocalMediaTrack
+	isMuted        bool
+	unmutedAt      time.Time
+}
+
+func NewPublicationMonitor(params PublicationMonitorParams) *PublicationMonitor {
+	p := &PublicationMonitor{
+		params:      params,
+		isConnected: params.IsPeerConnectionConnected,
+	}
+	p.desiredPublishes.SetBaseCap(4)
+	return p
+}
+
+func (p *PublicationMonitor) PostEvent(ome types.OperationMonitorEvent, omd types.OperationMonitorData) {
+	switch ome {
+	case types.OperationMonitorEventPublisherPeerConnectionConnected:
+		p.setConnected(omd.(bool))
+	case types.OperationMonitorEventAddPendingPublication:
+		p.addPending()
+	case types.OperationMonitorEventSetPublicationMute:
+		p.setMute(omd.(bool))
+	case types.OperationMonitorEventSetPublishedTrack:
+		p.setPublishedTrack(omd.(types.LocalMediaTrack))
+	case types.OperationMonitorEventClearPublishedTrack:
+		p.clearPublishedTrack(omd.(types.LocalMediaTrack))
+	}
+}
+
+func (p *PublicationMonitor) addPending() {
+	p.lock.Lock()
+	p.desiredPublishes.PushBack(
+		&publish{
+			isStart: true,
+		},
+	)
+
+	// synthesize an end
+	p.desiredPublishes.PushBack(
+		&publish{
+			isStart: false,
+		},
+	)
+	p.update()
+	p.lock.Unlock()
+}
+
+func (p *PublicationMonitor) maybeStartMonitor() {
+	if p.isConnected && !p.isMuted {
+		p.unmutedAt = time.Now()
+	}
+}
+
+func (p *PublicationMonitor) setConnected(isConnected bool) {
+	p.lock.Lock()
+	p.isConnected = isConnected
+	p.maybeStartMonitor()
+	p.lock.Unlock()
+}
+
+func (p *PublicationMonitor) setMute(isMuted bool) {
+	p.lock.Lock()
+	p.isMuted = isMuted
+	p.maybeStartMonitor()
+	p.lock.Unlock()
+}
+
+func (p *PublicationMonitor) setPublishedTrack(pubTrack types.LocalMediaTrack) {
+	p.lock.Lock()
+	p.publishedTrack = pubTrack
+	p.update()
+	p.lock.Unlock()
+}
+
+func (p *PublicationMonitor) clearPublishedTrack(pubTrack types.LocalMediaTrack) {
+	p.lock.Lock()
+	if p.publishedTrack == pubTrack {
+		p.publishedTrack = nil
+	} else {
+		p.params.Logger.Errorw("supervisor: mismatched published track on clear", nil, "trackID", p.params.TrackID)
+	}
+
+	p.update()
+	p.lock.Unlock()
+}
+
+func (p *PublicationMonitor) Check() error {
+	p.lock.RLock()
+	var pub *publish
+	if p.desiredPublishes.Len() > 0 {
+		pub = p.desiredPublishes.Front()
+	}
+
+	isMuted := p.isMuted
+	unmutedAt := p.unmutedAt
+	p.lock.RUnlock()
+
+	if pub == nil {
+		return nil
+	}
+
+	if pub.isStart && !isMuted && !unmutedAt.IsZero() && time.Since(unmutedAt) > publishWaitDuration {
+		// timed out waiting for publish
+		return errPublishTimeout
+	}
+
+	// give more time for publish to happen
+	// NOTE: synthesized end events do not have a start time, so do not check them for time out
+	return nil
+}
+
+func (p *PublicationMonitor) IsIdle() bool {
+	p.lock.RLock()
+	defer p.lock.RUnlock()
+
+	return p.desiredPublishes.Len() == 0 && p.publishedTrack == nil
+}
+
+func (p *PublicationMonitor) update() {
+	for {
+		var pub *publish
+		if p.desiredPublishes.Len() > 0 {
+			pub = p.desiredPublishes.PopFront()
+		}
+
+		if pub == nil {
+			return
+		}
+
+		if (pub.isStart && p.publishedTrack == nil) || (!pub.isStart && p.publishedTrack != nil) {
+			// put it back as the condition is not satisfied
+			p.desiredPublishes.PushFront(pub)
+			return
+		}
+	}
+}

livekit-server/pkg/rtc/testutils.go

@@ -0,0 +1,92 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package rtc
+
+import (
+	"github.com/livekit/protocol/livekit"
+	"github.com/livekit/protocol/logger"
+	"github.com/livekit/protocol/utils"
+	"github.com/livekit/protocol/utils/guid"
+
+	"github.com/livekit/livekit-server/pkg/rtc/types"
+	"github.com/livekit/livekit-server/pkg/rtc/types/typesfakes"
+)
+
+func NewMockParticipant(identity livekit.ParticipantIdentity, protocol types.ProtocolVersion, hidden bool, publisher bool) *typesfakes.FakeLocalParticipant {
+	p := &typesfakes.FakeLocalParticipant{}
+	sid := guid.New(utils.ParticipantPrefix)
+	p.IDReturns(livekit.ParticipantID(sid))
+	p.IdentityReturns(identity)
+	p.StateReturns(livekit.ParticipantInfo_JOINED)
+	p.ProtocolVersionReturns(protocol)
+	p.CanSubscribeReturns(true)
+	p.CanPublishSourceReturns(!hidden)
+	p.CanPublishDataReturns(!hidden)
+	p.HiddenReturns(hidden)
+	p.ToProtoReturns(&livekit.ParticipantInfo{
+		Sid:         sid,
+		Identity:    string(identity),
+		State:       livekit.ParticipantInfo_JOINED,
+		IsPublisher: publisher,
+	})
+	p.ToProtoWithVersionReturns(&livekit.ParticipantInfo{
+		Sid:         sid,
+		Identity:    string(identity),
+		State:       livekit.ParticipantInfo_JOINED,
+		IsPublisher: publisher,
+	}, utils.TimedVersion(0))
+
+	p.SetMetadataCalls(func(m string) {
+		var f func(participant types.LocalParticipant)
+		if p.OnParticipantUpdateCallCount() > 0 {
+			f = p.OnParticipantUpdateArgsForCall(p.OnParticipantUpdateCallCount() - 1)
+		}
+		if f != nil {
+			f(p)
+		}
+	})
+	updateTrack := func() {
+		var f func(participant types.LocalParticipant, track types.MediaTrack)
+		if p.OnTrackUpdatedCallCount() > 0 {
+			f = p.OnTrackUpdatedArgsForCall(p.OnTrackUpdatedCallCount() - 1)
+		}
+		if f != nil {
+			f(p, NewMockTrack(livekit.TrackType_VIDEO, "testcam"))
+		}
+	}
+
+	p.SetTrackMutedCalls(func(sid livekit.TrackID, muted bool, fromServer bool) *livekit.TrackInfo {
+		updateTrack()
+		return nil
+	})
+	p.AddTrackCalls(func(req *livekit.AddTrackRequest) {
+		updateTrack()
+	})
+	p.GetLoggerReturns(logger.GetLogger())
+
+	return p
+}
+
+func NewMockTrack(kind livekit.TrackType, name string) *typesfakes.FakeMediaTrack {
+	t := &typesfakes.FakeMediaTrack{}
+	t.IDReturns(livekit.TrackID(guid.New(utils.TrackPrefix)))
+	t.KindReturns(kind)
+	t.NameReturns(name)
+	t.ToProtoReturns(&livekit.TrackInfo{
+		Type: kind,
+		Name: name,
+	})
+	return t
+}

livekit-server/pkg/rtc/transport/handler.go

@@ -0,0 +1,72 @@
+// Copyright 2024 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package transport
+
+import (
+	"errors"
+
+	"github.com/pion/webrtc/v4"
+
+	"github.com/livekit/livekit-server/pkg/rtc/types"
+	"github.com/livekit/livekit-server/pkg/sfu/streamallocator"
+	"github.com/livekit/protocol/livekit"
+)
+
+//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -generate
+
+var (
+	ErrNoICECandidateHandler = errors.New("no ICE candidate handler")
+	ErrNoOfferHandler        = errors.New("no offer handler")
+	ErrNoAnswerHandler       = errors.New("no answer handler")
+)
+
+//counterfeiter:generate . Handler
+type Handler interface {
+	OnICECandidate(c *webrtc.ICECandidate, target livekit.SignalTarget) error
+	OnInitialConnected()
+	OnFullyEstablished()
+	OnFailed(isShortLived bool, iceConnectionInfo *types.ICEConnectionInfo)
+	OnTrack(track *webrtc.TrackRemote, rtpReceiver *webrtc.RTPReceiver)
+	OnDataPacket(kind livekit.DataPacket_Kind, data []byte)
+	OnDataSendError(err error)
+	OnOffer(sd webrtc.SessionDescription) error
+	OnAnswer(sd webrtc.SessionDescription) error
+	OnNegotiationStateChanged(state NegotiationState)
+	OnNegotiationFailed()
+	OnStreamStateChange(update *streamallocator.StreamStateUpdate) error
+}
+
+type UnimplementedHandler struct{}
+
+func (h UnimplementedHandler) OnICECandidate(c *webrtc.ICECandidate, target livekit.SignalTarget) error {
+	return ErrNoICECandidateHandler
+}
+func (h UnimplementedHandler) OnInitialConnected()                                                {}
+func (h UnimplementedHandler) OnFullyEstablished()                                                {}
+func (h UnimplementedHandler) OnFailed(isShortLived bool)                                         {}
+func (h UnimplementedHandler) OnTrack(track *webrtc.TrackRemote, rtpReceiver *webrtc.RTPReceiver) {}
+func (h UnimplementedHandler) OnDataPacket(kind livekit.DataPacket_Kind, data []byte)             {}
+func (h UnimplementedHandler) OnDataSendError(err error)                                          {}
+func (h UnimplementedHandler) OnOffer(sd webrtc.SessionDescription) error {
+	return ErrNoOfferHandler
+}
+func (h UnimplementedHandler) OnAnswer(sd webrtc.SessionDescription) error {
+	return ErrNoAnswerHandler
+}
+func (h UnimplementedHandler) OnNegotiationStateChanged(state NegotiationState) {}
+func (h UnimplementedHandler) OnNegotiationFailed()                             {}
+func (h UnimplementedHandler) OnStreamStateChange(update *streamallocator.StreamStateUpdate) error {
+	return nil
+}

livekit-server/pkg/rtc/transport/negotiationstate.go

@@ -0,0 +1,40 @@
+// Copyright 2024 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package transport
+
+import "fmt"
+
+type NegotiationState int
+
+const (
+	NegotiationStateNone NegotiationState = iota
+	// waiting for remote description
+	NegotiationStateRemote
+	// need to Negotiate again
+	NegotiationStateRetry
+)
+
+func (n NegotiationState) String() string {
+	switch n {
+	case NegotiationStateNone:
+		return "NONE"
+	case NegotiationStateRemote:
+		return "WAITING_FOR_REMOTE"
+	case NegotiationStateRetry:
+		return "RETRY"
+	default:
+		return fmt.Sprintf("%d", int(n))
+	}
+}

livekit-server/pkg/rtc/transport/transportfakes/fake_handler.go

@@ -0,0 +1,635 @@
+// Code generated by counterfeiter. DO NOT EDIT.
+package transportfakes
+
+import (
+	"sync"
+
+	"github.com/livekit/livekit-server/pkg/rtc/transport"
+	"github.com/livekit/livekit-server/pkg/rtc/types"
+	"github.com/livekit/livekit-server/pkg/sfu/streamallocator"
+	"github.com/livekit/protocol/livekit"
+	webrtc "github.com/pion/webrtc/v4"
+)
+
+type FakeHandler struct {
+	OnAnswerStub        func(webrtc.SessionDescription) error
+	onAnswerMutex       sync.RWMutex
+	onAnswerArgsForCall []struct {
+		arg1 webrtc.SessionDescription
+	}
+	onAnswerReturns struct {
+		result1 error
+	}
+	onAnswerReturnsOnCall map[int]struct {
+		result1 error
+	}
+	OnDataPacketStub        func(livekit.DataPacket_Kind, []byte)
+	onDataPacketMutex       sync.RWMutex
+	onDataPacketArgsForCall []struct {
+		arg1 livekit.DataPacket_Kind
+		arg2 []byte
+	}
+	OnDataSendErrorStub        func(error)
+	onDataSendErrorMutex       sync.RWMutex
+	onDataSendErrorArgsForCall []struct {
+		arg1 error
+	}
+	OnFailedStub        func(bool, *types.ICEConnectionInfo)
+	onFailedMutex       sync.RWMutex
+	onFailedArgsForCall []struct {
+		arg1 bool
+		arg2 *types.ICEConnectionInfo
+	}
+	OnFullyEstablishedStub        func()
+	onFullyEstablishedMutex       sync.RWMutex
+	onFullyEstablishedArgsForCall []struct {
+	}
+	OnICECandidateStub        func(*webrtc.ICECandidate, livekit.SignalTarget) error
+	onICECandidateMutex       sync.RWMutex
+	onICECandidateArgsForCall []struct {
+		arg1 *webrtc.ICECandidate
+		arg2 livekit.SignalTarget
+	}
+	onICECandidateReturns struct {
+		result1 error
+	}
+	onICECandidateReturnsOnCall map[int]struct {
+		result1 error
+	}
+	OnInitialConnectedStub        func()
+	onInitialConnectedMutex       sync.RWMutex
+	onInitialConnectedArgsForCall []struct {
+	}
+	OnNegotiationFailedStub        func()
+	onNegotiationFailedMutex       sync.RWMutex
+	onNegotiationFailedArgsForCall []struct {
+	}
+	OnNegotiationStateChangedStub        func(transport.NegotiationState)
+	onNegotiationStateChangedMutex       sync.RWMutex
+	onNegotiationStateChangedArgsForCall []struct {
+		arg1 transport.NegotiationState
+	}
+	OnOfferStub        func(webrtc.SessionDescription) error
+	onOfferMutex       sync.RWMutex
+	onOfferArgsForCall []struct {
+		arg1 webrtc.SessionDescription
+	}
+	onOfferReturns struct {
+		result1 error
+	}
+	onOfferReturnsOnCall map[int]struct {
+		result1 error
+	}
+	OnStreamStateChangeStub        func(*streamallocator.StreamStateUpdate) error
+	onStreamStateChangeMutex       sync.RWMutex
+	onStreamStateChangeArgsForCall []struct {
+		arg1 *streamallocator.StreamStateUpdate
+	}
+	onStreamStateChangeReturns struct {
+		result1 error
+	}
+	onStreamStateChangeReturnsOnCall map[int]struct {
+		result1 error
+	}
+	OnTrackStub        func(*webrtc.TrackRemote, *webrtc.RTPReceiver)
+	onTrackMutex       sync.RWMutex
+	onTrackArgsForCall []struct {
+		arg1 *webrtc.TrackRemote
+		arg2 *webrtc.RTPReceiver
+	}
+	invocations      map[string][][]interface{}
+	invocationsMutex sync.RWMutex
+}
+
+func (fake *FakeHandler) OnAnswer(arg1 webrtc.SessionDescription) error {
+	fake.onAnswerMutex.Lock()
+	ret, specificReturn := fake.onAnswerReturnsOnCall[len(fake.onAnswerArgsForCall)]
+	fake.onAnswerArgsForCall = append(fake.onAnswerArgsForCall, struct {
+		arg1 webrtc.SessionDescription
+	}{arg1})
+	stub := fake.OnAnswerStub
+	fakeReturns := fake.onAnswerReturns
+	fake.recordInvocation("OnAnswer", []interface{}{arg1})
+	fake.onAnswerMutex.Unlock()
+	if stub != nil {
+		return stub(arg1)
+	}
+	if specificReturn {
+		return ret.result1
+	}
+	return fakeReturns.result1
+}
+
+func (fake *FakeHandler) OnAnswerCallCount() int {
+	fake.onAnswerMutex.RLock()
+	defer fake.onAnswerMutex.RUnlock()
+	return len(fake.onAnswerArgsForCall)
+}
+
+func (fake *FakeHandler) OnAnswerCalls(stub func(webrtc.SessionDescription) error) {
+	fake.onAnswerMutex.Lock()
+	defer fake.onAnswerMutex.Unlock()
+	fake.OnAnswerStub = stub
+}
+
+func (fake *FakeHandler) OnAnswerArgsForCall(i int) webrtc.SessionDescription {
+	fake.onAnswerMutex.RLock()
+	defer fake.onAnswerMutex.RUnlock()
+	argsForCall := fake.onAnswerArgsForCall[i]
+	return argsForCall.arg1
+}
+
+func (fake *FakeHandler) OnAnswerReturns(result1 error) {
+	fake.onAnswerMutex.Lock()
+	defer fake.onAnswerMutex.Unlock()
+	fake.OnAnswerStub = nil
+	fake.onAnswerReturns = struct {
+		result1 error
+	}{result1}
+}
+
+func (fake *FakeHandler) OnAnswerReturnsOnCall(i int, result1 error) {
+	fake.onAnswerMutex.Lock()
+	defer fake.onAnswerMutex.Unlock()
+	fake.OnAnswerStub = nil
+	if fake.onAnswerReturnsOnCall == nil {
+		fake.onAnswerReturnsOnCall = make(map[int]struct {
+			result1 error
+		})
+	}
+	fake.onAnswerReturnsOnCall[i] = struct {
+		result1 error
+	}{result1}
+}
+
+func (fake *FakeHandler) OnDataPacket(arg1 livekit.DataPacket_Kind, arg2 []byte) {
+	var arg2Copy []byte
+	if arg2 != nil {
+		arg2Copy = make([]byte, len(arg2))
+		copy(arg2Copy, arg2)
+	}
+	fake.onDataPacketMutex.Lock()
+	fake.onDataPacketArgsForCall = append(fake.onDataPacketArgsForCall, struct {
+		arg1 livekit.DataPacket_Kind
+		arg2 []byte
+	}{arg1, arg2Copy})
+	stub := fake.OnDataPacketStub
+	fake.recordInvocation("OnDataPacket", []interface{}{arg1, arg2Copy})
+	fake.onDataPacketMutex.Unlock()
+	if stub != nil {
+		fake.OnDataPacketStub(arg1, arg2)
+	}
+}
+
+func (fake *FakeHandler) OnDataPacketCallCount() int {
+	fake.onDataPacketMutex.RLock()
+	defer fake.onDataPacketMutex.RUnlock()
+	return len(fake.onDataPacketArgsForCall)
+}
+
+func (fake *FakeHandler) OnDataPacketCalls(stub func(livekit.DataPacket_Kind, []byte)) {
+	fake.onDataPacketMutex.Lock()
+	defer fake.onDataPacketMutex.Unlock()
+	fake.OnDataPacketStub = stub
+}
+
+func (fake *FakeHandler) OnDataPacketArgsForCall(i int) (livekit.DataPacket_Kind, []byte) {
+	fake.onDataPacketMutex.RLock()
+	defer fake.onDataPacketMutex.RUnlock()
+	argsForCall := fake.onDataPacketArgsForCall[i]
+	return argsForCall.arg1, argsForCall.arg2
+}
+
+func (fake *FakeHandler) OnDataSendError(arg1 error) {
+	fake.onDataSendErrorMutex.Lock()
+	fake.onDataSendErrorArgsForCall = append(fake.onDataSendErrorArgsForCall, struct {
+		arg1 error
+	}{arg1})
+	stub := fake.OnDataSendErrorStub
+	fake.recordInvocation("OnDataSendError", []interface{}{arg1})
+	fake.onDataSendErrorMutex.Unlock()
+	if stub != nil {
+		fake.OnDataSendErrorStub(arg1)
+	}
+}
+
+func (fake *FakeHandler) OnDataSendErrorCallCount() int {
+	fake.onDataSendErrorMutex.RLock()
+	defer fake.onDataSendErrorMutex.RUnlock()
+	return len(fake.onDataSendErrorArgsForCall)
+}
+
+func (fake *FakeHandler) OnDataSendErrorCalls(stub func(error)) {
+	fake.onDataSendErrorMutex.Lock()
+	defer fake.onDataSendErrorMutex.Unlock()
+	fake.OnDataSendErrorStub = stub
+}
+
+func (fake *FakeHandler) OnDataSendErrorArgsForCall(i int) error {
+	fake.onDataSendErrorMutex.RLock()
+	defer fake.onDataSendErrorMutex.RUnlock()
+	argsForCall := fake.onDataSendErrorArgsForCall[i]
+	return argsForCall.arg1
+}
+
+func (fake *FakeHandler) OnFailed(arg1 bool, arg2 *types.ICEConnectionInfo) {
+	fake.onFailedMutex.Lock()
+	fake.onFailedArgsForCall = append(fake.onFailedArgsForCall, struct {
+		arg1 bool
+		arg2 *types.ICEConnectionInfo
+	}{arg1, arg2})
+	stub := fake.OnFailedStub
+	fake.recordInvocation("OnFailed", []interface{}{arg1, arg2})
+	fake.onFailedMutex.Unlock()
+	if stub != nil {
+		fake.OnFailedStub(arg1, arg2)
+	}
+}
+
+func (fake *FakeHandler) OnFailedCallCount() int {
+	fake.onFailedMutex.RLock()
+	defer fake.onFailedMutex.RUnlock()
+	return len(fake.onFailedArgsForCall)
+}
+
+func (fake *FakeHandler) OnFailedCalls(stub func(bool, *types.ICEConnectionInfo)) {
+	fake.onFailedMutex.Lock()
+	defer fake.onFailedMutex.Unlock()
+	fake.OnFailedStub = stub
+}
+
+func (fake *FakeHandler) OnFailedArgsForCall(i int) (bool, *types.ICEConnectionInfo) {
+	fake.onFailedMutex.RLock()
+	defer fake.onFailedMutex.RUnlock()
+	argsForCall := fake.onFailedArgsForCall[i]
+	return argsForCall.arg1, argsForCall.arg2
+}
+
+func (fake *FakeHandler) OnFullyEstablished() {
+	fake.onFullyEstablishedMutex.Lock()
+	fake.onFullyEstablishedArgsForCall = append(fake.onFullyEstablishedArgsForCall, struct {
+	}{})
+	stub := fake.OnFullyEstablishedStub
+	fake.recordInvocation("OnFullyEstablished", []interface{}{})
+	fake.onFullyEstablishedMutex.Unlock()
+	if stub != nil {
+		fake.OnFullyEstablishedStub()
+	}
+}
+
+func (fake *FakeHandler) OnFullyEstablishedCallCount() int {
+	fake.onFullyEstablishedMutex.RLock()
+	defer fake.onFullyEstablishedMutex.RUnlock()
+	return len(fake.onFullyEstablishedArgsForCall)
+}
+
+func (fake *FakeHandler) OnFullyEstablishedCalls(stub func()) {
+	fake.onFullyEstablishedMutex.Lock()
+	defer fake.onFullyEstablishedMutex.Unlock()
+	fake.OnFullyEstablishedStub = stub
+}
+
+func (fake *FakeHandler) OnICECandidate(arg1 *webrtc.ICECandidate, arg2 livekit.SignalTarget) error {
+	fake.onICECandidateMutex.Lock()
+	ret, specificReturn := fake.onICECandidateReturnsOnCall[len(fake.onICECandidateArgsForCall)]
+	fake.onICECandidateArgsForCall = append(fake.onICECandidateArgsForCall, struct {
+		arg1 *webrtc.ICECandidate
+		arg2 livekit.SignalTarget
+	}{arg1, arg2})
+	stub := fake.OnICECandidateStub
+	fakeReturns := fake.onICECandidateReturns
+	fake.recordInvocation("OnICECandidate", []interface{}{arg1, arg2})
+	fake.onICECandidateMutex.Unlock()
+	if stub != nil {
+		return stub(arg1, arg2)
+	}
+	if specificReturn {
+		return ret.result1
+	}
+	return fakeReturns.result1
+}
+
+func (fake *FakeHandler) OnICECandidateCallCount() int {
+	fake.onICECandidateMutex.RLock()
+	defer fake.onICECandidateMutex.RUnlock()
+	return len(fake.onICECandidateArgsForCall)
+}
+
+func (fake *FakeHandler) OnICECandidateCalls(stub func(*webrtc.ICECandidate, livekit.SignalTarget) error) {
+	fake.onICECandidateMutex.Lock()
+	defer fake.onICECandidateMutex.Unlock()
+	fake.OnICECandidateStub = stub
+}
+
+func (fake *FakeHandler) OnICECandidateArgsForCall(i int) (*webrtc.ICECandidate, livekit.SignalTarget) {
+	fake.onICECandidateMutex.RLock()
+	defer fake.onICECandidateMutex.RUnlock()
+	argsForCall := fake.onICECandidateArgsForCall[i]
+	return argsForCall.arg1, argsForCall.arg2
+}
+
+func (fake *FakeHandler) OnICECandidateReturns(result1 error) {
+	fake.onICECandidateMutex.Lock()
+	defer fake.onICECandidateMutex.Unlock()
+	fake.OnICECandidateStub = nil
+	fake.onICECandidateReturns = struct {
+		result1 error
+	}{result1}
+}
+
+func (fake *FakeHandler) OnICECandidateReturnsOnCall(i int, result1 error) {
+	fake.onICECandidateMutex.Lock()
+	defer fake.onICECandidateMutex.Unlock()
+	fake.OnICECandidateStub = nil
+	if fake.onICECandidateReturnsOnCall == nil {
+		fake.onICECandidateReturnsOnCall = make(map[int]struct {
+			result1 error
+		})
+	}
+	fake.onICECandidateReturnsOnCall[i] = struct {
+		result1 error
+	}{result1}
+}
+
+func (fake *FakeHandler) OnInitialConnected() {
+	fake.onInitialConnectedMutex.Lock()
+	fake.onInitialConnectedArgsForCall = append(fake.onInitialConnectedArgsForCall, struct {
+	}{})
+	stub := fake.OnInitialConnectedStub
+	fake.recordInvocation("OnInitialConnected", []interface{}{})
+	fake.onInitialConnectedMutex.Unlock()
+	if stub != nil {
+		fake.OnInitialConnectedStub()
+	}
+}
+
+func (fake *FakeHandler) OnInitialConnectedCallCount() int {
+	fake.onInitialConnectedMutex.RLock()
+	defer fake.onInitialConnectedMutex.RUnlock()
+	return len(fake.onInitialConnectedArgsForCall)
+}
+
+func (fake *FakeHandler) OnInitialConnectedCalls(stub func()) {
+	fake.onInitialConnectedMutex.Lock()
+	defer fake.onInitialConnectedMutex.Unlock()
+	fake.OnInitialConnectedStub = stub
+}
+
+func (fake *FakeHandler) OnNegotiationFailed() {
+	fake.onNegotiationFailedMutex.Lock()
+	fake.onNegotiationFailedArgsForCall = append(fake.onNegotiationFailedArgsForCall, struct {
+	}{})
+	stub := fake.OnNegotiationFailedStub
+	fake.recordInvocation("OnNegotiationFailed", []interface{}{})
+	fake.onNegotiationFailedMutex.Unlock()
+	if stub != nil {
+		fake.OnNegotiationFailedStub()
+	}
+}
+
+func (fake *FakeHandler) OnNegotiationFailedCallCount() int {
+	fake.onNegotiationFailedMutex.RLock()
+	defer fake.onNegotiationFailedMutex.RUnlock()
+	return len(fake.onNegotiationFailedArgsForCall)
+}
+
+func (fake *FakeHandler) OnNegotiationFailedCalls(stub func()) {
+	fake.onNegotiationFailedMutex.Lock()
+	defer fake.onNegotiationFailedMutex.Unlock()
+	fake.OnNegotiationFailedStub = stub
+}
+
+func (fake *FakeHandler) OnNegotiationStateChanged(arg1 transport.NegotiationState) {
+	fake.onNegotiationStateChangedMutex.Lock()
+	fake.onNegotiationStateChangedArgsForCall = append(fake.onNegotiationStateChangedArgsForCall, struct {
+		arg1 transport.NegotiationState
+	}{arg1})
+	stub := fake.OnNegotiationStateChangedStub
+	fake.recordInvocation("OnNegotiationStateChanged", []interface{}{arg1})
+	fake.onNegotiationStateChangedMutex.Unlock()
+	if stub != nil {
+		fake.OnNegotiationStateChangedStub(arg1)
+	}
+}
+
+func (fake *FakeHandler) OnNegotiationStateChangedCallCount() int {
+	fake.onNegotiationStateChangedMutex.RLock()
+	defer fake.onNegotiationStateChangedMutex.RUnlock()
+	return len(fake.onNegotiationStateChangedArgsForCall)
+}
+
+func (fake *FakeHandler) OnNegotiationStateChangedCalls(stub func(transport.NegotiationState)) {
+	fake.onNegotiationStateChangedMutex.Lock()
+	defer fake.onNegotiationStateChangedMutex.Unlock()
+	fake.OnNegotiationStateChangedStub = stub
+}
+
+func (fake *FakeHandler) OnNegotiationStateChangedArgsForCall(i int) transport.NegotiationState {
+	fake.onNegotiationStateChangedMutex.RLock()
+	defer fake.onNegotiationStateChangedMutex.RUnlock()
+	argsForCall := fake.onNegotiationStateChangedArgsForCall[i]
+	return argsForCall.arg1
+}
+
+func (fake *FakeHandler) OnOffer(arg1 webrtc.SessionDescription) error {
+	fake.onOfferMutex.Lock()
+	ret, specificReturn := fake.onOfferReturnsOnCall[len(fake.onOfferArgsForCall)]
+	fake.onOfferArgsForCall = append(fake.onOfferArgsForCall, struct {
+		arg1 webrtc.SessionDescription
+	}{arg1})
+	stub := fake.OnOfferStub
+	fakeReturns := fake.onOfferReturns
+	fake.recordInvocation("OnOffer", []interface{}{arg1})
+	fake.onOfferMutex.Unlock()
+	if stub != nil {
+		return stub(arg1)
+	}
+	if specificReturn {
+		return ret.result1
+	}
+	return fakeReturns.result1
+}
+
+func (fake *FakeHandler) OnOfferCallCount() int {
+	fake.onOfferMutex.RLock()
+	defer fake.onOfferMutex.RUnlock()
+	return len(fake.onOfferArgsForCall)
+}
+
+func (fake *FakeHandler) OnOfferCalls(stub func(webrtc.SessionDescription) error) {
+	fake.onOfferMutex.Lock()
+	defer fake.onOfferMutex.Unlock()
+	fake.OnOfferStub = stub
+}
+
+func (fake *FakeHandler) OnOfferArgsForCall(i int) webrtc.SessionDescription {
+	fake.onOfferMutex.RLock()
+	defer fake.onOfferMutex.RUnlock()
+	argsForCall := fake.onOfferArgsForCall[i]
+	return argsForCall.arg1
+}
+
+func (fake *FakeHandler) OnOfferReturns(result1 error) {
+	fake.onOfferMutex.Lock()
+	defer fake.onOfferMutex.Unlock()
+	fake.OnOfferStub = nil
+	fake.onOfferReturns = struct {
+		result1 error
+	}{result1}
+}
+
+func (fake *FakeHandler) OnOfferReturnsOnCall(i int, result1 error) {
+	fake.onOfferMutex.Lock()
+	defer fake.onOfferMutex.Unlock()
+	fake.OnOfferStub = nil
+	if fake.onOfferReturnsOnCall == nil {
+		fake.onOfferReturnsOnCall = make(map[int]struct {
+			result1 error
+		})
+	}
+	fake.onOfferReturnsOnCall[i] = struct {
+		result1 error
+	}{result1}
+}
+
+func (fake *FakeHandler) OnStreamStateChange(arg1 *streamallocator.StreamStateUpdate) error {
+	fake.onStreamStateChangeMutex.Lock()
+	ret, specificReturn := fake.onStreamStateChangeReturnsOnCall[len(fake.onStreamStateChangeArgsForCall)]
+	fake.onStreamStateChangeArgsForCall = append(fake.onStreamStateChangeArgsForCall, struct {
+		arg1 *streamallocator.StreamStateUpdate
+	}{arg1})
+	stub := fake.OnStreamStateChangeStub
+	fakeReturns := fake.onStreamStateChangeReturns
+	fake.recordInvocation("OnStreamStateChange", []interface{}{arg1})
+	fake.onStreamStateChangeMutex.Unlock()
+	if stub != nil {
+		return stub(arg1)
+	}
+	if specificReturn {
+		return ret.result1
+	}
+	return fakeReturns.result1
+}
+
+func (fake *FakeHandler) OnStreamStateChangeCallCount() int {
+	fake.onStreamStateChangeMutex.RLock()
+	defer fake.onStreamStateChangeMutex.RUnlock()
+	return len(fake.onStreamStateChangeArgsForCall)
+}
+
+func (fake *FakeHandler) OnStreamStateChangeCalls(stub func(*streamallocator.StreamStateUpdate) error) {
+	fake.onStreamStateChangeMutex.Lock()
+	defer fake.onStreamStateChangeMutex.Unlock()
+	fake.OnStreamStateChangeStub = stub
+}
+
+func (fake *FakeHandler) OnStreamStateChangeArgsForCall(i int) *streamallocator.StreamStateUpdate {
+	fake.onStreamStateChangeMutex.RLock()
+	defer fake.onStreamStateChangeMutex.RUnlock()
+	argsForCall := fake.onStreamStateChangeArgsForCall[i]
+	return argsForCall.arg1
+}
+
+func (fake *FakeHandler) OnStreamStateChangeReturns(result1 error) {
+	fake.onStreamStateChangeMutex.Lock()
+	defer fake.onStreamStateChangeMutex.Unlock()
+	fake.OnStreamStateChangeStub = nil
+	fake.onStreamStateChangeReturns = struct {
+		result1 error
+	}{result1}
+}
+
+func (fake *FakeHandler) OnStreamStateChangeReturnsOnCall(i int, result1 error) {
+	fake.onStreamStateChangeMutex.Lock()
+	defer fake.onStreamStateChangeMutex.Unlock()
+	fake.OnStreamStateChangeStub = nil
+	if fake.onStreamStateChangeReturnsOnCall == nil {
+		fake.onStreamStateChangeReturnsOnCall = make(map[int]struct {
+			result1 error
+		})
+	}
+	fake.onStreamStateChangeReturnsOnCall[i] = struct {
+		result1 error
+	}{result1}
+}
+
+func (fake *FakeHandler) OnTrack(arg1 *webrtc.TrackRemote, arg2 *webrtc.RTPReceiver) {
+	fake.onTrackMutex.Lock()
+	fake.onTrackArgsForCall = append(fake.onTrackArgsForCall, struct {
+		arg1 *webrtc.TrackRemote
+		arg2 *webrtc.RTPReceiver
+	}{arg1, arg2})
+	stub := fake.OnTrackStub
+	fake.recordInvocation("OnTrack", []interface{}{arg1, arg2})
+	fake.onTrackMutex.Unlock()
+	if stub != nil {
+		fake.OnTrackStub(arg1, arg2)
+	}
+}
+
+func (fake *FakeHandler) OnTrackCallCount() int {
+	fake.onTrackMutex.RLock()
+	defer fake.onTrackMutex.RUnlock()
+	return len(fake.onTrackArgsForCall)
+}
+
+func (fake *FakeHandler) OnTrackCalls(stub func(*webrtc.TrackRemote, *webrtc.RTPReceiver)) {
+	fake.onTrackMutex.Lock()
+	defer fake.onTrackMutex.Unlock()
+	fake.OnTrackStub = stub
+}
+
+func (fake *FakeHandler) OnTrackArgsForCall(i int) (*webrtc.TrackRemote, *webrtc.RTPReceiver) {
+	fake.onTrackMutex.RLock()
+	defer fake.onTrackMutex.RUnlock()
+	argsForCall := fake.onTrackArgsForCall[i]
+	return argsForCall.arg1, argsForCall.arg2
+}
+
+func (fake *FakeHandler) Invocations() map[string][][]interface{} {
+	fake.invocationsMutex.RLock()
+	defer fake.invocationsMutex.RUnlock()
+	fake.onAnswerMutex.RLock()
+	defer fake.onAnswerMutex.RUnlock()
+	fake.onDataPacketMutex.RLock()
+	defer fake.onDataPacketMutex.RUnlock()
+	fake.onDataSendErrorMutex.RLock()
+	defer fake.onDataSendErrorMutex.RUnlock()
+	fake.onFailedMutex.RLock()
+	defer fake.onFailedMutex.RUnlock()
+	fake.onFullyEstablishedMutex.RLock()
+	defer fake.onFullyEstablishedMutex.RUnlock()
+	fake.onICECandidateMutex.RLock()
+	defer fake.onICECandidateMutex.RUnlock()
+	fake.onInitialConnectedMutex.RLock()
+	defer fake.onInitialConnectedMutex.RUnlock()
+	fake.onNegotiationFailedMutex.RLock()
+	defer fake.onNegotiationFailedMutex.RUnlock()
+	fake.onNegotiationStateChangedMutex.RLock()
+	defer fake.onNegotiationStateChangedMutex.RUnlock()
+	fake.onOfferMutex.RLock()
+	defer fake.onOfferMutex.RUnlock()
+	fake.onStreamStateChangeMutex.RLock()
+	defer fake.onStreamStateChangeMutex.RUnlock()
+	fake.onTrackMutex.RLock()
+	defer fake.onTrackMutex.RUnlock()
+	copiedInvocations := map[string][][]interface{}{}
+	for key, value := range fake.invocations {
+		copiedInvocations[key] = value
+	}
+	return copiedInvocations
+}
+
+func (fake *FakeHandler) recordInvocation(key string, args []interface{}) {
+	fake.invocationsMutex.Lock()
+	defer fake.invocationsMutex.Unlock()
+	if fake.invocations == nil {
+		fake.invocations = map[string][][]interface{}{}
+	}
+	if fake.invocations[key] == nil {
+		fake.invocations[key] = [][]interface{}{}
+	}
+	fake.invocations[key] = append(fake.invocations[key], args)
+}
+
+var _ transport.Handler = new(FakeHandler)

livekit-server/pkg/rtc/transport_test.go

@@ -0,0 +1,626 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package rtc
+
+import (
+	"fmt"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/pion/sdp/v3"
+	"github.com/pion/webrtc/v4"
+	"github.com/stretchr/testify/require"
+	"go.uber.org/atomic"
+
+	"github.com/livekit/livekit-server/pkg/rtc/transport"
+	"github.com/livekit/livekit-server/pkg/rtc/transport/transportfakes"
+	"github.com/livekit/livekit-server/pkg/sfu/mime"
+	"github.com/livekit/livekit-server/pkg/testutils"
+	"github.com/livekit/protocol/livekit"
+)
+
+func TestMissingAnswerDuringICERestart(t *testing.T) {
+	params := TransportParams{
+		ParticipantID:       "id",
+		ParticipantIdentity: "identity",
+		Config:              &WebRTCConfig{},
+		IsOfferer:           true,
+	}
+
+	paramsA := params
+	handlerA := &transportfakes.FakeHandler{}
+	paramsA.Handler = handlerA
+	transportA, err := NewPCTransport(paramsA)
+	require.NoError(t, err)
+	_, err = transportA.pc.CreateDataChannel(ReliableDataChannel, nil)
+	require.NoError(t, err)
+
+	paramsB := params
+	handlerB := &transportfakes.FakeHandler{}
+	paramsB.Handler = handlerB
+	paramsB.IsOfferer = false
+	transportB, err := NewPCTransport(paramsB)
+	require.NoError(t, err)
+
+	// exchange ICE
+	handleICEExchange(t, transportA, transportB, handlerA, handlerB)
+
+	connectTransports(t, transportA, transportB, handlerA, handlerB, false, 1, 1)
+	require.Equal(t, webrtc.ICEConnectionStateConnected, transportA.pc.ICEConnectionState())
+	require.Equal(t, webrtc.ICEConnectionStateConnected, transportB.pc.ICEConnectionState())
+
+	var negotiationState atomic.Value
+	transportA.OnNegotiationStateChanged(func(state transport.NegotiationState) {
+		negotiationState.Store(state)
+	})
+
+	// offer again, but missed
+	var offerReceived atomic.Bool
+	handlerA.OnOfferCalls(func(sd webrtc.SessionDescription) error {
+		require.Equal(t, webrtc.SignalingStateHaveLocalOffer, transportA.pc.SignalingState())
+		require.Equal(t, transport.NegotiationStateRemote, negotiationState.Load().(transport.NegotiationState))
+		offerReceived.Store(true)
+		return nil
+	})
+	transportA.Negotiate(true)
+	require.Eventually(t, func() bool {
+		return offerReceived.Load()
+	}, 10*time.Second, time.Millisecond*10, "transportA offer not received")
+
+	connectTransports(t, transportA, transportB, handlerA, handlerB, true, 1, 1)
+	require.Equal(t, webrtc.ICEConnectionStateConnected, transportA.pc.ICEConnectionState())
+	require.Equal(t, webrtc.ICEConnectionStateConnected, transportB.pc.ICEConnectionState())
+
+	transportA.Close()
+	transportB.Close()
+}
+
+func TestNegotiationTiming(t *testing.T) {
+	params := TransportParams{
+		ParticipantID:       "id",
+		ParticipantIdentity: "identity",
+		Config:              &WebRTCConfig{},
+		IsOfferer:           true,
+	}
+
+	paramsA := params
+	handlerA := &transportfakes.FakeHandler{}
+	paramsA.Handler = handlerA
+	transportA, err := NewPCTransport(paramsA)
+	require.NoError(t, err)
+	_, err = transportA.pc.CreateDataChannel(LossyDataChannel, nil)
+	require.NoError(t, err)
+
+	paramsB := params
+	handlerB := &transportfakes.FakeHandler{}
+	paramsB.Handler = handlerB
+	paramsB.IsOfferer = false
+	transportB, err := NewPCTransport(paramsB)
+	require.NoError(t, err)
+
+	require.False(t, transportA.IsEstablished())
+	require.False(t, transportB.IsEstablished())
+
+	handleICEExchange(t, transportA, transportB, handlerA, handlerB)
+	offer := atomic.Value{}
+	handlerA.OnOfferCalls(func(sd webrtc.SessionDescription) error {
+		offer.Store(&sd)
+		return nil
+	})
+
+	var negotiationState atomic.Value
+	transportA.OnNegotiationStateChanged(func(state transport.NegotiationState) {
+		negotiationState.Store(state)
+	})
+
+	// initial offer
+	transportA.Negotiate(true)
+	require.Eventually(t, func() bool {
+		state, ok := negotiationState.Load().(transport.NegotiationState)
+		if !ok {
+			return false
+		}
+
+		return state == transport.NegotiationStateRemote
+	}, 10*time.Second, 10*time.Millisecond, "negotiation state does not match NegotiateStateRemote")
+
+	// second try, should've flipped transport status to retry
+	transportA.Negotiate(true)
+	require.Eventually(t, func() bool {
+		state, ok := negotiationState.Load().(transport.NegotiationState)
+		if !ok {
+			return false
+		}
+
+		return state == transport.NegotiationStateRetry
+	}, 10*time.Second, 10*time.Millisecond, "negotiation state does not match NegotiateStateRetry")
+
+	// third try, should've stayed at retry
+	transportA.Negotiate(true)
+	time.Sleep(100 * time.Millisecond) // some time to process the negotiate event
+	require.Eventually(t, func() bool {
+		state, ok := negotiationState.Load().(transport.NegotiationState)
+		if !ok {
+			return false
+		}
+
+		return state == transport.NegotiationStateRetry
+	}, 10*time.Second, 10*time.Millisecond, "negotiation state does not match NegotiateStateRetry")
+
+	time.Sleep(5 * time.Millisecond)
+	actualOffer, ok := offer.Load().(*webrtc.SessionDescription)
+	require.True(t, ok)
+
+	handlerB.OnAnswerCalls(func(answer webrtc.SessionDescription) error {
+		transportA.HandleRemoteDescription(answer)
+		return nil
+	})
+	transportB.HandleRemoteDescription(*actualOffer)
+
+	require.Eventually(t, func() bool {
+		return transportA.IsEstablished()
+	}, 10*time.Second, time.Millisecond*10, "transportA is not established")
+	require.Eventually(t, func() bool {
+		return transportB.IsEstablished()
+	}, 10*time.Second, time.Millisecond*10, "transportB is not established")
+
+	// it should still be negotiating again
+	require.Equal(t, transport.NegotiationStateRemote, negotiationState.Load().(transport.NegotiationState))
+	offer2, ok := offer.Load().(*webrtc.SessionDescription)
+	require.True(t, ok)
+	require.False(t, offer2 == actualOffer)
+
+	transportA.Close()
+	transportB.Close()
+}
+
+func TestFirstOfferMissedDuringICERestart(t *testing.T) {
+	params := TransportParams{
+		ParticipantID:       "id",
+		ParticipantIdentity: "identity",
+		Config:              &WebRTCConfig{},
+		IsOfferer:           true,
+	}
+
+	paramsA := params
+	handlerA := &transportfakes.FakeHandler{}
+	paramsA.Handler = handlerA
+	transportA, err := NewPCTransport(paramsA)
+	require.NoError(t, err)
+	_, err = transportA.pc.CreateDataChannel(ReliableDataChannel, nil)
+	require.NoError(t, err)
+
+	paramsB := params
+	handlerB := &transportfakes.FakeHandler{}
+	paramsB.Handler = handlerB
+	paramsB.IsOfferer = false
+	transportB, err := NewPCTransport(paramsB)
+	require.NoError(t, err)
+
+	// exchange ICE
+	handleICEExchange(t, transportA, transportB, handlerA, handlerB)
+
+	// first offer missed
+	var firstOfferReceived atomic.Bool
+	handlerA.OnOfferCalls(func(sd webrtc.SessionDescription) error {
+		firstOfferReceived.Store(true)
+		return nil
+	})
+	transportA.Negotiate(true)
+	require.Eventually(t, func() bool {
+		return firstOfferReceived.Load()
+	}, 10*time.Second, 10*time.Millisecond, "first offer not received")
+
+	// set offer/answer with restart ICE, will negotiate twice,
+	// first one is recover from missed offer
+	// second one is restartICE
+	handlerB.OnAnswerCalls(func(answer webrtc.SessionDescription) error {
+		transportA.HandleRemoteDescription(answer)
+		return nil
+	})
+
+	var offerCount atomic.Int32
+	handlerA.OnOfferCalls(func(sd webrtc.SessionDescription) error {
+		offerCount.Inc()
+
+		// the second offer is a ice restart offer, so we wait transportB complete the ice gathering
+		if transportB.pc.ICEGatheringState() == webrtc.ICEGatheringStateGathering {
+			require.Eventually(t, func() bool {
+				return transportB.pc.ICEGatheringState() == webrtc.ICEGatheringStateComplete
+			}, 10*time.Second, time.Millisecond*10)
+		}
+
+		transportB.HandleRemoteDescription(sd)
+		return nil
+	})
+
+	// first establish connection
+	transportA.ICERestart()
+
+	// ensure we are connected
+	require.Eventually(t, func() bool {
+		return transportA.pc.ICEConnectionState() == webrtc.ICEConnectionStateConnected &&
+			transportB.pc.ICEConnectionState() == webrtc.ICEConnectionStateConnected &&
+			offerCount.Load() == 2
+	}, testutils.ConnectTimeout, 10*time.Millisecond, "transport did not connect")
+
+	transportA.Close()
+	transportB.Close()
+}
+
+func TestFirstAnswerMissedDuringICERestart(t *testing.T) {
+	params := TransportParams{
+		ParticipantID:       "id",
+		ParticipantIdentity: "identity",
+		Config:              &WebRTCConfig{},
+		IsOfferer:           true,
+	}
+
+	paramsA := params
+	handlerA := &transportfakes.FakeHandler{}
+	paramsA.Handler = handlerA
+	transportA, err := NewPCTransport(paramsA)
+	require.NoError(t, err)
+	_, err = transportA.pc.CreateDataChannel(LossyDataChannel, nil)
+	require.NoError(t, err)
+
+	paramsB := params
+	handlerB := &transportfakes.FakeHandler{}
+	paramsB.Handler = handlerB
+	paramsB.IsOfferer = false
+	transportB, err := NewPCTransport(paramsB)
+	require.NoError(t, err)
+
+	// exchange ICE
+	handleICEExchange(t, transportA, transportB, handlerA, handlerB)
+
+	// first answer missed
+	var firstAnswerReceived atomic.Bool
+	handlerB.OnAnswerCalls(func(sd webrtc.SessionDescription) error {
+		if firstAnswerReceived.Load() {
+			transportA.HandleRemoteDescription(sd)
+		} else {
+			// do not send first answer so that remote misses the first answer
+			firstAnswerReceived.Store(true)
+		}
+		return nil
+	})
+	handlerA.OnOfferCalls(func(sd webrtc.SessionDescription) error {
+		transportB.HandleRemoteDescription(sd)
+		return nil
+	})
+
+	transportA.Negotiate(true)
+	require.Eventually(t, func() bool {
+		return transportB.pc.SignalingState() == webrtc.SignalingStateStable && firstAnswerReceived.Load()
+	}, time.Second, 10*time.Millisecond, "transportB signaling state did not go to stable")
+
+	// set offer/answer with restart ICE, will negotiate twice,
+	// first one is recover from missed offer
+	// second one is restartICE
+	var offerCount atomic.Int32
+	handlerA.OnOfferCalls(func(sd webrtc.SessionDescription) error {
+		offerCount.Inc()
+
+		// the second offer is a ice restart offer, so we wait for transportB to complete ICE gathering
+		if transportB.pc.ICEGatheringState() == webrtc.ICEGatheringStateGathering {
+			require.Eventually(t, func() bool {
+				return transportB.pc.ICEGatheringState() == webrtc.ICEGatheringStateComplete
+			}, 10*time.Second, time.Millisecond*10)
+		}
+
+		transportB.HandleRemoteDescription(sd)
+		return nil
+	})
+
+	// first establish connection
+	transportA.ICERestart()
+
+	// ensure we are connected
+	require.Eventually(t, func() bool {
+		return transportA.pc.ICEConnectionState() == webrtc.ICEConnectionStateConnected &&
+			transportB.pc.ICEConnectionState() == webrtc.ICEConnectionStateConnected &&
+			offerCount.Load() == 2
+	}, testutils.ConnectTimeout, 10*time.Millisecond, "transport did not connect")
+
+	transportA.Close()
+	transportB.Close()
+}
+
+func TestNegotiationFailed(t *testing.T) {
+	params := TransportParams{
+		ParticipantID:       "id",
+		ParticipantIdentity: "identity",
+		Config:              &WebRTCConfig{},
+		IsOfferer:           true,
+	}
+
+	paramsA := params
+	handlerA := &transportfakes.FakeHandler{}
+	paramsA.Handler = handlerA
+	transportA, err := NewPCTransport(paramsA)
+	require.NoError(t, err)
+	_, err = transportA.pc.CreateDataChannel(ReliableDataChannel, nil)
+	require.NoError(t, err)
+
+	paramsB := params
+	handlerB := &transportfakes.FakeHandler{}
+	paramsB.Handler = handlerB
+	paramsB.IsOfferer = false
+	transportB, err := NewPCTransport(paramsB)
+	require.NoError(t, err)
+
+	// exchange ICE
+	handleICEExchange(t, transportA, transportB, handlerA, handlerB)
+
+	// wait for transport to be connected before maiming the signalling channel
+	connectTransports(t, transportA, transportB, handlerA, handlerB, false, 1, 1)
+
+	// reset OnOffer to force a negotiation failure
+	handlerA.OnOfferCalls(func(sd webrtc.SessionDescription) error { return nil })
+	var failed atomic.Int32
+	handlerA.OnNegotiationFailedCalls(func() {
+		failed.Inc()
+	})
+	transportA.Negotiate(true)
+	require.Eventually(t, func() bool {
+		return failed.Load() == 1
+	}, negotiationFailedTimeout+time.Second, 10*time.Millisecond, "negotiation failed")
+
+	transportA.Close()
+}
+
+func TestFilteringCandidates(t *testing.T) {
+	params := TransportParams{
+		ParticipantID:       "id",
+		ParticipantIdentity: "identity",
+		Config:              &WebRTCConfig{},
+		EnabledCodecs: []*livekit.Codec{
+			{Mime: mime.MimeTypeOpus.String()},
+			{Mime: mime.MimeTypeVP8.String()},
+			{Mime: mime.MimeTypeH264.String()},
+		},
+		Handler: &transportfakes.FakeHandler{},
+	}
+	transport, err := NewPCTransport(params)
+	require.NoError(t, err)
+
+	_, err = transport.pc.CreateDataChannel(ReliableDataChannel, nil)
+	require.NoError(t, err)
+
+	_, err = transport.pc.AddTransceiverFromKind(webrtc.RTPCodecTypeAudio)
+	require.NoError(t, err)
+
+	_, err = transport.pc.AddTransceiverFromKind(webrtc.RTPCodecTypeVideo)
+	require.NoError(t, err)
+
+	offer, err := transport.pc.CreateOffer(nil)
+	require.NoError(t, err)
+
+	offerGatheringComplete := webrtc.GatheringCompletePromise(transport.pc)
+	require.NoError(t, transport.pc.SetLocalDescription(offer))
+	<-offerGatheringComplete
+
+	// should not filter out UDP candidates if TCP is not preferred
+	offer = *transport.pc.LocalDescription()
+	filteredOffer := transport.filterCandidates(offer, false, true)
+	require.EqualValues(t, offer.SDP, filteredOffer.SDP)
+
+	parsed, err := offer.Unmarshal()
+	require.NoError(t, err)
+
+	// add a couple of TCP candidates
+	done := false
+	for _, m := range parsed.MediaDescriptions {
+		for _, a := range m.Attributes {
+			if a.Key == sdp.AttrKeyCandidate {
+				for idx, aa := range m.Attributes {
+					if aa.Key == sdp.AttrKeyEndOfCandidates {
+						modifiedAttributes := make([]sdp.Attribute, idx)
+						copy(modifiedAttributes, m.Attributes[:idx])
+						modifiedAttributes = append(modifiedAttributes, []sdp.Attribute{
+							{
+								Key:   sdp.AttrKeyCandidate,
+								Value: "054225987 1 tcp 2124414975 159.203.70.248 7881 typ host tcptype passive",
+							},
+							{
+								Key:   sdp.AttrKeyCandidate,
+								Value: "054225987 2 tcp 2124414975 159.203.70.248 7881 typ host tcptype passive",
+							},
+						}...)
+						m.Attributes = append(modifiedAttributes, m.Attributes[idx:]...)
+						done = true
+						break
+					}
+				}
+			}
+			if done {
+				break
+			}
+		}
+		if done {
+			break
+		}
+	}
+	bytes, err := parsed.Marshal()
+	require.NoError(t, err)
+	offer.SDP = string(bytes)
+
+	parsed, err = offer.Unmarshal()
+	require.NoError(t, err)
+
+	getNumTransportTypeCandidates := func(sd *sdp.SessionDescription) (int, int) {
+		numUDPCandidates := 0
+		numTCPCandidates := 0
+		for _, a := range sd.Attributes {
+			if a.Key == sdp.AttrKeyCandidate {
+				if strings.Contains(a.Value, "udp") {
+					numUDPCandidates++
+				}
+				if strings.Contains(a.Value, "tcp") {
+					numTCPCandidates++
+				}
+			}
+		}
+		for _, m := range sd.MediaDescriptions {
+			for _, a := range m.Attributes {
+				if a.Key == sdp.AttrKeyCandidate {
+					if strings.Contains(a.Value, "udp") {
+						numUDPCandidates++
+					}
+					if strings.Contains(a.Value, "tcp") {
+						numTCPCandidates++
+					}
+				}
+			}
+		}
+		return numUDPCandidates, numTCPCandidates
+	}
+	udp, tcp := getNumTransportTypeCandidates(parsed)
+	require.NotZero(t, udp)
+	require.Equal(t, 2, tcp)
+
+	transport.SetPreferTCP(true)
+	filteredOffer = transport.filterCandidates(offer, true, true)
+	parsed, err = filteredOffer.Unmarshal()
+	require.NoError(t, err)
+	udp, tcp = getNumTransportTypeCandidates(parsed)
+	require.Zero(t, udp)
+	require.Equal(t, 2, tcp)
+
+	transport.Close()
+}
+
+func handleICEExchange(t *testing.T, a, b *PCTransport, ah, bh *transportfakes.FakeHandler) {
+	ah.OnICECandidateCalls(func(candidate *webrtc.ICECandidate, target livekit.SignalTarget) error {
+		if candidate == nil {
+			return nil
+		}
+		t.Logf("got ICE candidate from A: %v", candidate)
+		b.AddICECandidate(candidate.ToJSON())
+		return nil
+	})
+	bh.OnICECandidateCalls(func(candidate *webrtc.ICECandidate, target livekit.SignalTarget) error {
+		if candidate == nil {
+			return nil
+		}
+		t.Logf("got ICE candidate from B: %v", candidate)
+		a.AddICECandidate(candidate.ToJSON())
+		return nil
+	})
+}
+
+func connectTransports(t *testing.T, offerer, answerer *PCTransport, offererHandler, answererHandler *transportfakes.FakeHandler, isICERestart bool, expectedOfferCount int32, expectedAnswerCount int32) {
+	var offerCount atomic.Int32
+	var answerCount atomic.Int32
+	answererHandler.OnAnswerCalls(func(answer webrtc.SessionDescription) error {
+		answerCount.Inc()
+		offerer.HandleRemoteDescription(answer)
+		return nil
+	})
+
+	offererHandler.OnOfferCalls(func(offer webrtc.SessionDescription) error {
+		offerCount.Inc()
+		answerer.HandleRemoteDescription(offer)
+		return nil
+	})
+
+	if isICERestart {
+		offerer.ICERestart()
+	} else {
+		offerer.Negotiate(true)
+	}
+
+	require.Eventually(t, func() bool {
+		return offerCount.Load() == expectedOfferCount
+	}, 10*time.Second, time.Millisecond*10, fmt.Sprintf("offer count mismatch, expected: %d, actual: %d", expectedOfferCount, offerCount.Load()))
+
+	require.Eventually(t, func() bool {
+		return offerer.pc.ICEConnectionState() == webrtc.ICEConnectionStateConnected
+	}, 10*time.Second, time.Millisecond*10, "offerer did not become connected")
+
+	require.Eventually(t, func() bool {
+		return answerCount.Load() == expectedAnswerCount
+	}, 10*time.Second, time.Millisecond*10, fmt.Sprintf("answer count mismatch, expected: %d, actual: %d", expectedAnswerCount, answerCount.Load()))
+
+	require.Eventually(t, func() bool {
+		return answerer.pc.ICEConnectionState() == webrtc.ICEConnectionStateConnected
+	}, 10*time.Second, time.Millisecond*10, "answerer did not become connected")
+
+	transportsConnected := untilTransportsConnected(offererHandler, answererHandler)
+	transportsConnected.Wait()
+}
+
+func untilTransportsConnected(transports ...*transportfakes.FakeHandler) *sync.WaitGroup {
+	var triggered sync.WaitGroup
+	triggered.Add(len(transports))
+
+	for _, t := range transports {
+		var done atomic.Value
+		done.Store(false)
+		hdlr := func() {
+			if val, ok := done.Load().(bool); ok && !val {
+				done.Store(true)
+				triggered.Done()
+			}
+		}
+
+		if t.OnInitialConnectedCallCount() != 0 {
+			hdlr()
+		}
+		t.OnInitialConnectedCalls(hdlr)
+	}
+	return &triggered
+}
+
+func TestConfigureAudioTransceiver(t *testing.T) {
+	for _, testcase := range []struct {
+		nack   bool
+		stereo bool
+	}{
+		{false, false},
+		{true, false},
+		{false, true},
+		{true, true},
+	} {
+		t.Run(fmt.Sprintf("nack=%v,stereo=%v", testcase.nack, testcase.stereo), func(t *testing.T) {
+			var me webrtc.MediaEngine
+			registerCodecs(&me, []*livekit.Codec{{Mime: mime.MimeTypeOpus.String()}}, RTCPFeedbackConfig{Audio: []webrtc.RTCPFeedback{{Type: webrtc.TypeRTCPFBNACK}}}, false)
+			pc, err := webrtc.NewAPI(webrtc.WithMediaEngine(&me)).NewPeerConnection(webrtc.Configuration{})
+			require.NoError(t, err)
+			defer pc.Close()
+			tr, err := pc.AddTransceiverFromKind(webrtc.RTPCodecTypeAudio, webrtc.RTPTransceiverInit{Direction: webrtc.RTPTransceiverDirectionSendonly})
+			require.NoError(t, err)
+
+			configureAudioTransceiver(tr, testcase.stereo, testcase.nack)
+			codecs := tr.Sender().GetParameters().Codecs
+			for _, codec := range codecs {
+				if mime.IsMimeTypeStringOpus(codec.MimeType) {
+					require.Equal(t, testcase.stereo, strings.Contains(codec.SDPFmtpLine, "sprop-stereo=1"))
+					var nackEnabled bool
+					for _, fb := range codec.RTCPFeedback {
+						if fb.Type == webrtc.TypeRTCPFBNACK {
+							nackEnabled = true
+							break
+						}
+					}
+					require.Equal(t, testcase.nack, nackEnabled)
+				}
+			}
+		})
+	}
+}

livekit-server/pkg/rtc/transportmanager.go

@@ -0,0 +1,825 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package rtc
+
+import (
+	"context"
+	"io"
+	"math/bits"
+	"sync"
+	"time"
+
+	"github.com/pion/rtcp"
+	"github.com/pion/sctp"
+	"github.com/pion/sdp/v3"
+	"github.com/pion/webrtc/v4"
+	"github.com/pkg/errors"
+	"go.uber.org/atomic"
+	"google.golang.org/protobuf/proto"
+
+	"github.com/livekit/mediatransportutil/pkg/twcc"
+	"github.com/livekit/protocol/livekit"
+	"github.com/livekit/protocol/logger"
+	"github.com/livekit/protocol/utils"
+
+	"github.com/livekit/livekit-server/pkg/config"
+	"github.com/livekit/livekit-server/pkg/rtc/transport"
+	"github.com/livekit/livekit-server/pkg/rtc/types"
+	"github.com/livekit/livekit-server/pkg/sfu"
+	"github.com/livekit/livekit-server/pkg/sfu/datachannel"
+	"github.com/livekit/livekit-server/pkg/sfu/pacer"
+	"github.com/livekit/livekit-server/pkg/telemetry"
+)
+
+const (
+	failureCountThreshold     = 2
+	preferNextByFailureWindow = time.Minute
+
+	// when RR report loss percentage over this threshold, we consider it is a unstable event
+	udpLossFracUnstable = 25
+	// if in last 32 times RR, the unstable report count over this threshold, the connection is unstable
+	udpLossUnstableCountThreshold = 20
+)
+
+// -------------------------------
+
+type TransportManagerTransportHandler struct {
+	transport.Handler
+	t      *TransportManager
+	logger logger.Logger
+}
+
+func (h TransportManagerTransportHandler) OnFailed(isShortLived bool, iceConnectionInfo *types.ICEConnectionInfo) {
+	if isShortLived {
+		h.logger.Infow("short ice connection", connectionDetailsFields([]*types.ICEConnectionInfo{iceConnectionInfo})...)
+	}
+	h.t.handleConnectionFailed(isShortLived)
+	h.Handler.OnFailed(isShortLived, iceConnectionInfo)
+}
+
+// -------------------------------
+
+type TransportManagerPublisherTransportHandler struct {
+	TransportManagerTransportHandler
+}
+
+func (h TransportManagerPublisherTransportHandler) OnAnswer(sd webrtc.SessionDescription) error {
+	h.t.lastPublisherAnswer.Store(sd)
+	return h.Handler.OnAnswer(sd)
+}
+
+// -------------------------------
+
+type TransportManagerParams struct {
+	Identity                     livekit.ParticipantIdentity
+	SID                          livekit.ParticipantID
+	SubscriberAsPrimary          bool
+	Config                       *WebRTCConfig
+	Twcc                         *twcc.Responder
+	ProtocolVersion              types.ProtocolVersion
+	CongestionControlConfig      config.CongestionControlConfig
+	EnabledSubscribeCodecs       []*livekit.Codec
+	EnabledPublishCodecs         []*livekit.Codec
+	SimTracks                    map[uint32]SimulcastTrackInfo
+	ClientInfo                   ClientInfo
+	Migration                    bool
+	AllowTCPFallback             bool
+	TCPFallbackRTTThreshold      int
+	AllowUDPUnstableFallback     bool
+	TURNSEnabled                 bool
+	AllowPlayoutDelay            bool
+	DataChannelMaxBufferedAmount uint64
+	DatachannelSlowThreshold     int
+	Logger                       logger.Logger
+	PublisherHandler             transport.Handler
+	SubscriberHandler            transport.Handler
+	DataChannelStats             *telemetry.BytesTrackStats
+	UseOneShotSignallingMode     bool
+	FireOnTrackBySdp             bool
+}
+
+type TransportManager struct {
+	params TransportManagerParams
+
+	lock sync.RWMutex
+
+	publisher               *PCTransport
+	subscriber              *PCTransport
+	failureCount            int
+	isTransportReconfigured bool
+	lastFailure             time.Time
+	lastSignalAt            time.Time
+	signalSourceValid       atomic.Bool
+
+	pendingOfferPublisher        *webrtc.SessionDescription
+	pendingDataChannelsPublisher []*livekit.DataChannelInfo
+	lastPublisherAnswer          atomic.Value
+	lastPublisherOffer           atomic.Value
+	iceConfig                    *livekit.ICEConfig
+
+	mediaLossProxy       *MediaLossProxy
+	udpLossUnstableCount uint32
+	signalingRTT, udpRTT uint32
+
+	onICEConfigChanged func(iceConfig *livekit.ICEConfig)
+
+	droppedBySlowReaderCount atomic.Uint32
+}
+
+func NewTransportManager(params TransportManagerParams) (*TransportManager, error) {
+	if params.Logger == nil {
+		params.Logger = logger.GetLogger()
+	}
+	t := &TransportManager{
+		params:         params,
+		mediaLossProxy: NewMediaLossProxy(MediaLossProxyParams{Logger: params.Logger}),
+		iceConfig:      &livekit.ICEConfig{},
+	}
+	t.mediaLossProxy.OnMediaLossUpdate(t.onMediaLossUpdate)
+
+	lgr := LoggerWithPCTarget(params.Logger, livekit.SignalTarget_PUBLISHER)
+	publisher, err := NewPCTransport(TransportParams{
+		ParticipantID:                params.SID,
+		ParticipantIdentity:          params.Identity,
+		ProtocolVersion:              params.ProtocolVersion,
+		Config:                       params.Config,
+		Twcc:                         params.Twcc,
+		DirectionConfig:              params.Config.Publisher,
+		CongestionControlConfig:      params.CongestionControlConfig,
+		EnabledCodecs:                params.EnabledPublishCodecs,
+		Logger:                       lgr,
+		SimTracks:                    params.SimTracks,
+		ClientInfo:                   params.ClientInfo,
+		Transport:                    livekit.SignalTarget_PUBLISHER,
+		Handler:                      TransportManagerPublisherTransportHandler{TransportManagerTransportHandler{params.PublisherHandler, t, lgr}},
+		UseOneShotSignallingMode:     params.UseOneShotSignallingMode,
+		DataChannelMaxBufferedAmount: params.DataChannelMaxBufferedAmount,
+		DatachannelSlowThreshold:     params.DatachannelSlowThreshold,
+		FireOnTrackBySdp:             params.FireOnTrackBySdp,
+	})
+	if err != nil {
+		return nil, err
+	}
+	t.publisher = publisher
+
+	lgr = LoggerWithPCTarget(params.Logger, livekit.SignalTarget_SUBSCRIBER)
+	subscriber, err := NewPCTransport(TransportParams{
+		ParticipantID:            params.SID,
+		ParticipantIdentity:      params.Identity,
+		ProtocolVersion:          params.ProtocolVersion,
+		Config:                   params.Config,
+		DirectionConfig:          params.Config.Subscriber,
+		CongestionControlConfig:  params.CongestionControlConfig,
+		EnabledCodecs:            params.EnabledSubscribeCodecs,
+		Logger:                   lgr,
+		ClientInfo:               params.ClientInfo,
+		IsOfferer:                true,
+		IsSendSide:               true,
+		AllowPlayoutDelay:        params.AllowPlayoutDelay,
+		DatachannelSlowThreshold: params.DatachannelSlowThreshold,
+		Transport:                livekit.SignalTarget_SUBSCRIBER,
+		Handler:                  TransportManagerTransportHandler{params.SubscriberHandler, t, lgr},
+	})
+	if err != nil {
+		return nil, err
+	}
+	t.subscriber = subscriber
+	if !t.params.Migration {
+		if err := t.createDataChannelsForSubscriber(nil); err != nil {
+			return nil, err
+		}
+	}
+
+	t.signalSourceValid.Store(true)
+	return t, nil
+}
+
+func (t *TransportManager) Close() {
+	t.publisher.Close()
+	t.subscriber.Close()
+}
+
+func (t *TransportManager) SubscriberClose() {
+	t.subscriber.Close()
+}
+
+func (t *TransportManager) HasPublisherEverConnected() bool {
+	return t.publisher.HasEverConnected()
+}
+
+func (t *TransportManager) IsPublisherEstablished() bool {
+	return t.publisher.IsEstablished()
+}
+
+func (t *TransportManager) GetPublisherRTT() (float64, bool) {
+	return t.publisher.GetRTT()
+}
+
+func (t *TransportManager) GetPublisherMid(rtpReceiver *webrtc.RTPReceiver) string {
+	return t.publisher.GetMid(rtpReceiver)
+}
+
+func (t *TransportManager) GetPublisherRTPReceiver(mid string) *webrtc.RTPReceiver {
+	return t.publisher.GetRTPReceiver(mid)
+}
+
+func (t *TransportManager) WritePublisherRTCP(pkts []rtcp.Packet) error {
+	return t.publisher.WriteRTCP(pkts)
+}
+
+func (t *TransportManager) GetSubscriberRTT() (float64, bool) {
+	return t.subscriber.GetRTT()
+}
+
+func (t *TransportManager) HasSubscriberEverConnected() bool {
+	return t.subscriber.HasEverConnected()
+}
+
+func (t *TransportManager) AddTrackLocal(
+	trackLocal webrtc.TrackLocal,
+	params types.AddTrackParams,
+) (*webrtc.RTPSender, *webrtc.RTPTransceiver, error) {
+	if t.params.UseOneShotSignallingMode {
+		return t.publisher.AddTrack(trackLocal, params)
+	} else {
+		return t.subscriber.AddTrack(trackLocal, params)
+	}
+}
+
+func (t *TransportManager) AddTransceiverFromTrackLocal(
+	trackLocal webrtc.TrackLocal,
+	params types.AddTrackParams,
+) (*webrtc.RTPSender, *webrtc.RTPTransceiver, error) {
+	if t.params.UseOneShotSignallingMode {
+		return t.publisher.AddTransceiverFromTrack(trackLocal, params)
+	} else {
+		return t.subscriber.AddTransceiverFromTrack(trackLocal, params)
+	}
+}
+
+func (t *TransportManager) RemoveTrackLocal(sender *webrtc.RTPSender) error {
+	if t.params.UseOneShotSignallingMode {
+		return t.publisher.RemoveTrack(sender)
+	} else {
+		return t.subscriber.RemoveTrack(sender)
+	}
+}
+
+func (t *TransportManager) WriteSubscriberRTCP(pkts []rtcp.Packet) error {
+	if t.params.UseOneShotSignallingMode {
+		return t.publisher.WriteRTCP(pkts)
+	} else {
+		return t.subscriber.WriteRTCP(pkts)
+	}
+}
+
+func (t *TransportManager) GetSubscriberPacer() pacer.Pacer {
+	return t.subscriber.GetPacer()
+}
+
+func (t *TransportManager) AddSubscribedTrack(subTrack types.SubscribedTrack) {
+	t.subscriber.AddTrackToStreamAllocator(subTrack)
+}
+
+func (t *TransportManager) RemoveSubscribedTrack(subTrack types.SubscribedTrack) {
+	t.subscriber.RemoveTrackFromStreamAllocator(subTrack)
+}
+
+func (t *TransportManager) SendDataPacket(kind livekit.DataPacket_Kind, encoded []byte) error {
+	// downstream data is sent via primary peer connection
+	err := t.getTransport(true).SendDataPacket(kind, encoded)
+	if err != nil {
+		if !utils.ErrorIsOneOf(err, io.ErrClosedPipe, sctp.ErrStreamClosed, ErrTransportFailure, ErrDataChannelBufferFull, context.DeadlineExceeded) {
+			if errors.Is(err, datachannel.ErrDataDroppedBySlowReader) {
+				droppedBySlowReaderCount := t.droppedBySlowReaderCount.Inc()
+				if (droppedBySlowReaderCount-1)%100 == 0 {
+					t.params.Logger.Infow("drop data packet by slow reader", "error", err, "kind", kind, "count", droppedBySlowReaderCount)
+				}
+			} else {
+				t.params.Logger.Warnw("send data packet error", err)
+			}
+		}
+		if utils.ErrorIsOneOf(err, sctp.ErrStreamClosed, io.ErrClosedPipe) {
+			t.params.SubscriberHandler.OnDataSendError(err)
+		}
+	} else {
+		t.params.DataChannelStats.AddBytes(uint64(len(encoded)), true)
+	}
+
+	return err
+}
+
+func (t *TransportManager) createDataChannelsForSubscriber(pendingDataChannels []*livekit.DataChannelInfo) error {
+	var (
+		reliableID, lossyID       uint16
+		reliableIDPtr, lossyIDPtr *uint16
+	)
+
+	//
+	// For old version migration clients, they don't send subscriber data channel info
+	// so we need to create data channels with default ID and don't negotiate as client already has
+	// data channels with default ID.
+	//
+	// For new version migration clients, we create data channels with new ID and negotiate with client
+	//
+	for _, dc := range pendingDataChannels {
+		if dc.Label == ReliableDataChannel {
+			// pion use step 2 for auto generated ID, so we need to add 4 to avoid conflict
+			reliableID = uint16(dc.Id) + 4
+			reliableIDPtr = &reliableID
+		} else if dc.Label == LossyDataChannel {
+			lossyID = uint16(dc.Id) + 4
+			lossyIDPtr = &lossyID
+		}
+	}
+
+	ordered := true
+	negotiated := t.params.Migration && reliableIDPtr == nil
+	if err := t.subscriber.CreateDataChannel(ReliableDataChannel, &webrtc.DataChannelInit{
+		Ordered:    &ordered,
+		ID:         reliableIDPtr,
+		Negotiated: &negotiated,
+	}); err != nil {
+		return err
+	}
+
+	retransmits := uint16(0)
+	negotiated = t.params.Migration && lossyIDPtr == nil
+	if err := t.subscriber.CreateDataChannel(LossyDataChannel, &webrtc.DataChannelInit{
+		Ordered:        &ordered,
+		MaxRetransmits: &retransmits,
+		ID:             lossyIDPtr,
+		Negotiated:     &negotiated,
+	}); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (t *TransportManager) GetUnmatchMediaForOffer(offer webrtc.SessionDescription, mediaType string) (parsed *sdp.SessionDescription, unmatched []*sdp.MediaDescription, err error) {
+	// prefer codec from offer for clients that don't support setCodecPreferences
+	parsed, err = offer.Unmarshal()
+	if err != nil {
+		t.params.Logger.Errorw("failed to parse offer for codec preference", err)
+		return
+	}
+
+	var lastMatchedMid string
+	lastAnswer := t.lastPublisherAnswer.Load()
+	if lastAnswer != nil {
+		answer := lastAnswer.(webrtc.SessionDescription)
+		parsedAnswer, err1 := answer.Unmarshal()
+		if err1 != nil {
+			// should not happen
+			t.params.Logger.Errorw("failed to parse last answer", err)
+			return
+		}
+
+		for i := len(parsedAnswer.MediaDescriptions) - 1; i >= 0; i-- {
+			media := parsedAnswer.MediaDescriptions[i]
+			if media.MediaName.Media == mediaType {
+				lastMatchedMid, _ = media.Attribute(sdp.AttrKeyMID)
+				break
+			}
+		}
+	}
+
+	for i := len(parsed.MediaDescriptions) - 1; i >= 0; i-- {
+		media := parsed.MediaDescriptions[i]
+		if media.MediaName.Media == mediaType {
+			mid, _ := media.Attribute(sdp.AttrKeyMID)
+			if mid == lastMatchedMid {
+				break
+			}
+			unmatched = append(unmatched, media)
+		}
+	}
+
+	return
+}
+
+func (t *TransportManager) LastPublisherOffer() webrtc.SessionDescription {
+	if sd := t.lastPublisherOffer.Load(); sd != nil {
+		return sd.(webrtc.SessionDescription)
+	}
+	return webrtc.SessionDescription{}
+}
+
+func (t *TransportManager) HandleOffer(offer webrtc.SessionDescription, shouldPend bool) error {
+	t.lock.Lock()
+	if shouldPend {
+		t.pendingOfferPublisher = &offer
+		t.lock.Unlock()
+		return nil
+	}
+	t.lock.Unlock()
+	t.lastPublisherOffer.Store(offer)
+
+	return t.publisher.HandleRemoteDescription(offer)
+}
+
+func (t *TransportManager) GetAnswer() (webrtc.SessionDescription, error) {
+	answer, err := t.publisher.GetAnswer()
+	if err == nil {
+		t.lastPublisherAnswer.Store(answer)
+	}
+	return answer, err
+}
+
+func (t *TransportManager) ProcessPendingPublisherOffer() {
+	t.lock.Lock()
+	pendingOffer := t.pendingOfferPublisher
+	t.pendingOfferPublisher = nil
+	t.lock.Unlock()
+
+	if pendingOffer != nil {
+		t.HandleOffer(*pendingOffer, false)
+	}
+}
+
+func (t *TransportManager) HandleAnswer(answer webrtc.SessionDescription) {
+	t.subscriber.HandleRemoteDescription(answer)
+}
+
+// AddICECandidate adds candidates for remote peer
+func (t *TransportManager) AddICECandidate(candidate webrtc.ICECandidateInit, target livekit.SignalTarget) {
+	switch target {
+	case livekit.SignalTarget_PUBLISHER:
+		t.publisher.AddICECandidate(candidate)
+	case livekit.SignalTarget_SUBSCRIBER:
+		t.subscriber.AddICECandidate(candidate)
+	default:
+		err := errors.New("unknown signal target")
+		t.params.Logger.Errorw("ice candidate for unknown signal target", err, "target", target)
+	}
+}
+
+func (t *TransportManager) NegotiateSubscriber(force bool) {
+	t.subscriber.Negotiate(force)
+}
+
+func (t *TransportManager) HandleClientReconnect(reason livekit.ReconnectReason) {
+	var (
+		isShort              bool
+		duration             time.Duration
+		resetShortConnection bool
+	)
+	switch reason {
+	case livekit.ReconnectReason_RR_PUBLISHER_FAILED:
+		resetShortConnection = true
+		isShort, duration = t.publisher.IsShortConnection(time.Now())
+
+	case livekit.ReconnectReason_RR_SUBSCRIBER_FAILED:
+		resetShortConnection = true
+		isShort, duration = t.subscriber.IsShortConnection(time.Now())
+	}
+
+	if isShort {
+		t.lock.Lock()
+		t.resetTransportConfigureLocked(false)
+		t.lock.Unlock()
+		t.params.Logger.Infow("short connection by client ice restart", "duration", duration, "reason", reason)
+		t.handleConnectionFailed(isShort)
+	}
+
+	if resetShortConnection {
+		t.publisher.ResetShortConnOnICERestart()
+		t.subscriber.ResetShortConnOnICERestart()
+	}
+}
+
+func (t *TransportManager) ICERestart(iceConfig *livekit.ICEConfig) error {
+	t.SetICEConfig(iceConfig)
+
+	return t.subscriber.ICERestart()
+}
+
+func (t *TransportManager) OnICEConfigChanged(f func(iceConfig *livekit.ICEConfig)) {
+	t.lock.Lock()
+	t.onICEConfigChanged = f
+	t.lock.Unlock()
+}
+
+func (t *TransportManager) SetICEConfig(iceConfig *livekit.ICEConfig) {
+	if iceConfig != nil {
+		t.configureICE(iceConfig, true)
+	}
+}
+
+func (t *TransportManager) GetICEConfig() *livekit.ICEConfig {
+	t.lock.RLock()
+	defer t.lock.RUnlock()
+	if t.iceConfig == nil {
+		return nil
+	}
+	return utils.CloneProto(t.iceConfig)
+}
+
+func (t *TransportManager) resetTransportConfigureLocked(reconfigured bool) {
+	t.failureCount = 0
+	t.isTransportReconfigured = reconfigured
+	t.udpLossUnstableCount = 0
+	t.lastFailure = time.Time{}
+}
+
+func (t *TransportManager) configureICE(iceConfig *livekit.ICEConfig, reset bool) {
+	t.lock.Lock()
+	isEqual := proto.Equal(t.iceConfig, iceConfig)
+	if reset || !isEqual {
+		t.resetTransportConfigureLocked(!reset)
+	}
+
+	if isEqual {
+		t.lock.Unlock()
+		return
+	}
+
+	t.params.Logger.Infow("setting ICE config", "iceConfig", logger.Proto(iceConfig))
+	onICEConfigChanged := t.onICEConfigChanged
+	t.iceConfig = iceConfig
+	t.lock.Unlock()
+
+	if iceConfig.PreferenceSubscriber != livekit.ICECandidateType_ICT_NONE {
+		t.mediaLossProxy.OnMediaLossUpdate(nil)
+	}
+
+	t.publisher.SetPreferTCP(iceConfig.PreferencePublisher == livekit.ICECandidateType_ICT_TCP)
+	t.subscriber.SetPreferTCP(iceConfig.PreferenceSubscriber == livekit.ICECandidateType_ICT_TCP)
+
+	if onICEConfigChanged != nil {
+		onICEConfigChanged(iceConfig)
+	}
+}
+
+func (t *TransportManager) SubscriberAsPrimary() bool {
+	return t.params.SubscriberAsPrimary
+}
+
+func (t *TransportManager) GetICEConnectionInfo() []*types.ICEConnectionInfo {
+	infos := make([]*types.ICEConnectionInfo, 0, 2)
+	for _, pc := range []*PCTransport{t.publisher, t.subscriber} {
+		info := pc.GetICEConnectionInfo()
+		if info.HasCandidates() {
+			infos = append(infos, info)
+		}
+	}
+	return infos
+}
+
+func (t *TransportManager) getTransport(isPrimary bool) *PCTransport {
+	pcTransport := t.publisher
+	if (isPrimary && t.params.SubscriberAsPrimary) || (!isPrimary && !t.params.SubscriberAsPrimary) {
+		pcTransport = t.subscriber
+	}
+
+	return pcTransport
+}
+
+func (t *TransportManager) handleConnectionFailed(isShortLived bool) {
+	if !t.params.AllowTCPFallback || t.params.UseOneShotSignallingMode {
+		return
+	}
+
+	t.lock.Lock()
+	if t.isTransportReconfigured {
+		t.lock.Unlock()
+		return
+	}
+
+	lastSignalSince := time.Since(t.lastSignalAt)
+	signalValid := t.signalSourceValid.Load()
+	if !t.hasRecentSignalLocked() || !signalValid {
+		// the failed might cause by network interrupt because signal closed or we have not seen any signal in the time window,
+		// so don't switch to next candidate type
+		t.params.Logger.Debugw(
+			"ignoring prefer candidate check by ICE failure because signal connection interrupted",
+			"lastSignalSince", lastSignalSince,
+			"signalValid", signalValid,
+		)
+		t.failureCount = 0
+		t.lastFailure = time.Time{}
+		t.lock.Unlock()
+		return
+	}
+
+	//
+	// Checking only `PreferenceSubscriber` field although any connection failure (PUBLISHER OR SUBSCRIBER) will
+	// flow through here.
+	//
+	// As both transports are switched to the same type on any failure, checking just subscriber should be fine.
+	//
+	getNext := func(ic *livekit.ICEConfig) livekit.ICECandidateType {
+		if ic.PreferenceSubscriber == livekit.ICECandidateType_ICT_NONE && t.params.ClientInfo.SupportsICETCP() && t.canUseICETCP() {
+			return livekit.ICECandidateType_ICT_TCP
+		} else if ic.PreferenceSubscriber != livekit.ICECandidateType_ICT_TLS && t.params.TURNSEnabled {
+			return livekit.ICECandidateType_ICT_TLS
+		} else {
+			return livekit.ICECandidateType_ICT_NONE
+		}
+	}
+
+	var preferNext livekit.ICECandidateType
+	if isShortLived {
+		preferNext = getNext(t.iceConfig)
+	} else {
+		t.failureCount++
+		lastFailure := t.lastFailure
+		t.lastFailure = time.Now()
+		if t.failureCount < failureCountThreshold || time.Since(lastFailure) > preferNextByFailureWindow {
+			t.lock.Unlock()
+			return
+		}
+
+		preferNext = getNext(t.iceConfig)
+	}
+
+	if preferNext == t.iceConfig.PreferenceSubscriber {
+		t.lock.Unlock()
+		return
+	}
+
+	t.isTransportReconfigured = true
+	t.lock.Unlock()
+
+	switch preferNext {
+	case livekit.ICECandidateType_ICT_TCP:
+		t.params.Logger.Debugw("prefer TCP transport on both peer connections")
+
+	case livekit.ICECandidateType_ICT_TLS:
+		t.params.Logger.Debugw("prefer TLS transport both peer connections")
+
+	case livekit.ICECandidateType_ICT_NONE:
+		t.params.Logger.Debugw("allowing all transports on both peer connections")
+	}
+
+	// irrespective of which one fails, force prefer candidate on both as the other one might
+	// fail at a different time and cause another disruption
+	t.configureICE(&livekit.ICEConfig{
+		PreferenceSubscriber: preferNext,
+		PreferencePublisher:  preferNext,
+	}, false)
+}
+
+func (t *TransportManager) SetMigrateInfo(previousOffer, previousAnswer *webrtc.SessionDescription, dataChannels []*livekit.DataChannelInfo) {
+	t.lock.Lock()
+	t.pendingDataChannelsPublisher = make([]*livekit.DataChannelInfo, 0, len(dataChannels))
+	pendingDataChannelsSubscriber := make([]*livekit.DataChannelInfo, 0, len(dataChannels))
+	for _, dci := range dataChannels {
+		if dci.Target == livekit.SignalTarget_SUBSCRIBER {
+			pendingDataChannelsSubscriber = append(pendingDataChannelsSubscriber, dci)
+		} else {
+			t.pendingDataChannelsPublisher = append(t.pendingDataChannelsPublisher, dci)
+		}
+	}
+	t.lock.Unlock()
+
+	if t.params.SubscriberAsPrimary {
+		if err := t.createDataChannelsForSubscriber(pendingDataChannelsSubscriber); err != nil {
+			t.params.Logger.Errorw("create subscriber data channels during migration failed", err)
+		}
+	}
+
+	t.subscriber.SetPreviousSdp(previousOffer, previousAnswer)
+}
+
+func (t *TransportManager) ProcessPendingPublisherDataChannels() {
+	t.lock.Lock()
+	pendingDataChannels := t.pendingDataChannelsPublisher
+	t.pendingDataChannelsPublisher = nil
+	t.lock.Unlock()
+
+	ordered := true
+	negotiated := true
+
+	for _, ci := range pendingDataChannels {
+		var (
+			dcLabel    string
+			dcID       uint16
+			dcExisting bool
+			err        error
+		)
+		if ci.Label == LossyDataChannel {
+			retransmits := uint16(0)
+			id := uint16(ci.GetId())
+			dcLabel, dcID, dcExisting, err = t.publisher.CreateDataChannelIfEmpty(LossyDataChannel, &webrtc.DataChannelInit{
+				Ordered:        &ordered,
+				MaxRetransmits: &retransmits,
+				Negotiated:     &negotiated,
+				ID:             &id,
+			})
+		} else if ci.Label == ReliableDataChannel {
+			id := uint16(ci.GetId())
+			dcLabel, dcID, dcExisting, err = t.publisher.CreateDataChannelIfEmpty(ReliableDataChannel, &webrtc.DataChannelInit{
+				Ordered:    &ordered,
+				Negotiated: &negotiated,
+				ID:         &id,
+			})
+		}
+		if err != nil {
+			t.params.Logger.Errorw("create migrated data channel failed", err, "label", ci.Label)
+		} else if dcExisting {
+			t.params.Logger.Debugw("existing data channel during migration", "label", dcLabel, "id", dcID)
+		} else {
+			t.params.Logger.Debugw("create migrated data channel", "label", dcLabel, "id", dcID)
+		}
+	}
+}
+
+func (t *TransportManager) HandleReceiverReport(dt *sfu.DownTrack, report *rtcp.ReceiverReport) {
+	t.mediaLossProxy.HandleMaxLossFeedback(dt, report)
+}
+
+func (t *TransportManager) onMediaLossUpdate(loss uint8) {
+	if t.params.TCPFallbackRTTThreshold == 0 || !t.params.AllowUDPUnstableFallback {
+		return
+	}
+	t.lock.Lock()
+	t.udpLossUnstableCount <<= 1
+	if loss >= uint8(255*udpLossFracUnstable/100) {
+		t.udpLossUnstableCount |= 1
+		if bits.OnesCount32(t.udpLossUnstableCount) >= udpLossUnstableCountThreshold {
+			if t.udpRTT > 0 && t.signalingRTT < uint32(float32(t.udpRTT)*1.3) && int(t.signalingRTT) < t.params.TCPFallbackRTTThreshold && t.hasRecentSignalLocked() {
+				t.udpLossUnstableCount = 0
+				t.lock.Unlock()
+
+				t.params.Logger.Infow("udp connection unstable, switch to tcp", "signalingRTT", t.signalingRTT)
+				t.params.SubscriberHandler.OnFailed(true, t.subscriber.GetICEConnectionInfo())
+				return
+			}
+		}
+	}
+	t.lock.Unlock()
+}
+
+func (t *TransportManager) UpdateSignalingRTT(rtt uint32) {
+	t.lock.Lock()
+	t.signalingRTT = rtt
+	t.lock.Unlock()
+	t.publisher.SetSignalingRTT(rtt)
+	t.subscriber.SetSignalingRTT(rtt)
+
+	// TODO: considering using tcp rtt to calculate ice connection cost, if ice connection can't be established
+	// within 5 * tcp rtt(at least 5s), means udp traffic might be block/dropped, switch to tcp.
+	// Currently, most cases reported is that ice connected but subsequent connection, so left the thinking for now.
+}
+
+func (t *TransportManager) UpdateMediaRTT(rtt uint32) {
+	t.lock.Lock()
+	if t.udpRTT == 0 {
+		t.udpRTT = rtt
+	} else {
+		t.udpRTT = uint32(int(t.udpRTT) + (int(rtt)-int(t.udpRTT))/2)
+	}
+	t.lock.Unlock()
+}
+
+func (t *TransportManager) UpdateLastSeenSignal() {
+	t.lock.Lock()
+	t.lastSignalAt = time.Now()
+	t.lock.Unlock()
+}
+
+func (t *TransportManager) SinceLastSignal() time.Duration {
+	t.lock.RLock()
+	defer t.lock.RUnlock()
+	return time.Since(t.lastSignalAt)
+}
+
+func (t *TransportManager) LastSeenSignalAt() time.Time {
+	t.lock.RLock()
+	defer t.lock.RUnlock()
+	return t.lastSignalAt
+}
+
+func (t *TransportManager) canUseICETCP() bool {
+	return t.params.TCPFallbackRTTThreshold == 0 || int(t.signalingRTT) < t.params.TCPFallbackRTTThreshold
+}
+
+func (t *TransportManager) SetSignalSourceValid(valid bool) {
+	t.signalSourceValid.Store(valid)
+	t.params.Logger.Debugw("signal source valid", "valid", valid)
+}
+
+func (t *TransportManager) SetSubscriberAllowPause(allowPause bool) {
+	t.subscriber.SetAllowPauseOfStreamAllocator(allowPause)
+}
+
+func (t *TransportManager) SetSubscriberChannelCapacity(channelCapacity int64) {
+	t.subscriber.SetChannelCapacityOfStreamAllocator(channelCapacity)
+}
+
+func (t *TransportManager) hasRecentSignalLocked() bool {
+	return time.Since(t.lastSignalAt) < PingTimeoutSeconds*time.Second
+}

livekit-server/pkg/rtc/types/ice.go

@@ -0,0 +1,368 @@
+/*
+ * Copyright 2023 LiveKit, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package types
+
+import (
+	"fmt"
+	"strings"
+	"sync"
+
+	"github.com/pion/ice/v4"
+	"github.com/pion/webrtc/v4"
+	"golang.org/x/exp/slices"
+
+	"github.com/livekit/protocol/livekit"
+	"github.com/livekit/protocol/logger"
+)
+
+type ICEConnectionType string
+
+const (
+	ICEConnectionTypeUDP     ICEConnectionType = "udp"
+	ICEConnectionTypeTCP     ICEConnectionType = "tcp"
+	ICEConnectionTypeTURN    ICEConnectionType = "turn"
+	ICEConnectionTypeUnknown ICEConnectionType = "unknown"
+)
+
+type ICECandidateExtended struct {
+	// only one of local or remote is set. This is due to type foo in Pion
+	Local         *webrtc.ICECandidate
+	Remote        ice.Candidate
+	SelectedOrder int
+	Filtered      bool
+	Trickle       bool
+}
+
+// --------------------------------------------
+
+type ICEConnectionInfo struct {
+	Local     []*ICECandidateExtended
+	Remote    []*ICECandidateExtended
+	Transport livekit.SignalTarget
+	Type      ICEConnectionType
+}
+
+func (i *ICEConnectionInfo) HasCandidates() bool {
+	return len(i.Local) > 0 || len(i.Remote) > 0
+}
+
+// --------------------------------------------
+
+type ICEConnectionDetails struct {
+	ICEConnectionInfo
+	lock          sync.Mutex
+	selectedCount int
+	logger        logger.Logger
+}
+
+func NewICEConnectionDetails(transport livekit.SignalTarget, l logger.Logger) *ICEConnectionDetails {
+	d := &ICEConnectionDetails{
+		ICEConnectionInfo: ICEConnectionInfo{
+			Transport: transport,
+			Type:      ICEConnectionTypeUnknown,
+		},
+		logger: l,
+	}
+	return d
+}
+
+func (d *ICEConnectionDetails) GetInfo() *ICEConnectionInfo {
+	d.lock.Lock()
+	defer d.lock.Unlock()
+	info := &ICEConnectionInfo{
+		Transport: d.Transport,
+		Type:      d.Type,
+		Local:     make([]*ICECandidateExtended, 0, len(d.Local)),
+		Remote:    make([]*ICECandidateExtended, 0, len(d.Remote)),
+	}
+	for _, c := range d.Local {
+		info.Local = append(info.Local, &ICECandidateExtended{
+			Local:         c.Local,
+			Filtered:      c.Filtered,
+			SelectedOrder: c.SelectedOrder,
+			Trickle:       c.Trickle,
+		})
+	}
+	for _, c := range d.Remote {
+		info.Remote = append(info.Remote, &ICECandidateExtended{
+			Remote:        c.Remote,
+			Filtered:      c.Filtered,
+			SelectedOrder: c.SelectedOrder,
+			Trickle:       c.Trickle,
+		})
+	}
+	return info
+}
+
+func (d *ICEConnectionDetails) AddLocalCandidate(c *webrtc.ICECandidate, filtered, trickle bool) {
+	d.lock.Lock()
+	defer d.lock.Unlock()
+	compFn := func(e *ICECandidateExtended) bool {
+		return isCandidateEqualTo(e.Local, c)
+	}
+	if slices.ContainsFunc(d.Local, compFn) {
+		return
+	}
+	d.Local = append(d.Local, &ICECandidateExtended{
+		Local:    c,
+		Filtered: filtered,
+		Trickle:  trickle,
+	})
+}
+
+func (d *ICEConnectionDetails) AddLocalICECandidate(c ice.Candidate, filtered, trickle bool) {
+	candidate, err := unmarshalCandidate(c)
+	if err != nil {
+		d.logger.Errorw("could not unmarshal ice candidate", err, "candidate", c)
+		return
+	}
+
+	d.AddLocalCandidate(candidate, filtered, trickle)
+}
+
+func (d *ICEConnectionDetails) AddRemoteCandidate(c webrtc.ICECandidateInit, filtered, trickle, canUpdate bool) {
+	candidate, err := unmarshalICECandidate(c)
+	if err != nil {
+		d.logger.Errorw("could not unmarshal candidate", err, "candidate", c)
+		return
+	}
+	d.AddRemoteICECandidate(candidate, filtered, trickle, canUpdate)
+}
+
+func (d *ICEConnectionDetails) AddRemoteICECandidate(candidate ice.Candidate, filtered, trickle, canUpdate bool) {
+	if candidate == nil {
+		// end-of-candidates candidate
+		return
+	}
+
+	d.lock.Lock()
+	defer d.lock.Unlock()
+	indexFn := func(e *ICECandidateExtended) bool {
+		return isICECandidateEqualTo(e.Remote, candidate)
+	}
+	if idx := slices.IndexFunc(d.Remote, indexFn); idx != -1 {
+		if canUpdate {
+			d.Remote[idx].Filtered = filtered
+			d.Remote[idx].Trickle = trickle
+		}
+		return
+	}
+	d.Remote = append(d.Remote, &ICECandidateExtended{
+		Remote:   candidate,
+		Filtered: filtered,
+		Trickle:  trickle,
+	})
+}
+
+func (d *ICEConnectionDetails) Clear() {
+	d.lock.Lock()
+	defer d.lock.Unlock()
+	d.Local = nil
+	d.Remote = nil
+	d.Type = ICEConnectionTypeUnknown
+}
+
+func (d *ICEConnectionDetails) SetSelectedPair(pair *webrtc.ICECandidatePair) {
+	d.lock.Lock()
+	defer d.lock.Unlock()
+
+	d.selectedCount++
+
+	remoteIdx := slices.IndexFunc(d.Remote, func(e *ICECandidateExtended) bool {
+		return isICECandidateEqualToCandidate(e.Remote, pair.Remote)
+	})
+	if remoteIdx < 0 {
+		// it's possible for prflx candidates to be generated by Pion, we'll add them
+		candidate, err := unmarshalICECandidate(pair.Remote.ToJSON())
+		if err != nil {
+			d.logger.Errorw("could not unmarshal remote candidate", err, "candidate", pair.Remote)
+			return
+		}
+		if candidate == nil {
+			return
+		}
+		d.Remote = append(d.Remote, &ICECandidateExtended{
+			Remote:   candidate,
+			Filtered: false,
+			Trickle:  true,
+		})
+		remoteIdx = len(d.Remote) - 1
+	}
+	remote := d.Remote[remoteIdx]
+	remote.SelectedOrder = d.selectedCount
+
+	localIdx := slices.IndexFunc(d.Local, func(e *ICECandidateExtended) bool {
+		return isCandidateEqualTo(e.Local, pair.Local)
+	})
+	if localIdx < 0 {
+		d.logger.Errorw("could not match local candidate", nil, "local", pair.Local)
+		// should not happen
+		return
+	}
+	local := d.Local[localIdx]
+	local.SelectedOrder = d.selectedCount
+
+	d.Type = ICEConnectionTypeUDP
+	if pair.Remote.Protocol == webrtc.ICEProtocolTCP {
+		d.Type = ICEConnectionTypeTCP
+	}
+	if pair.Remote.Typ == webrtc.ICECandidateTypeRelay {
+		d.Type = ICEConnectionTypeTURN
+	} else if pair.Remote.Typ == webrtc.ICECandidateTypePrflx {
+		// if the remote relay candidate pings us *before* we get a relay candidate,
+		// Pion would have created a prflx candidate with the same address as the relay candidate.
+		// to report an accurate connection type, we'll compare to see if existing relay candidates match
+		for _, other := range d.Remote {
+			or := other.Remote
+			if or.Type() == ice.CandidateTypeRelay &&
+				pair.Remote.Address == or.Address() &&
+				pair.Remote.Port == uint16(or.Port()) &&
+				pair.Remote.Protocol.String() == or.NetworkType().NetworkShort() {
+				d.Type = ICEConnectionTypeTURN
+			}
+		}
+	}
+}
+
+// -------------------------------------------------------------
+
+func isCandidateEqualTo(c1, c2 *webrtc.ICECandidate) bool {
+	if c1 == nil && c2 == nil {
+		return true
+	}
+	if (c1 == nil && c2 != nil) || (c1 != nil && c2 == nil) {
+		return false
+	}
+	return c1.Typ == c2.Typ &&
+		c1.Protocol == c2.Protocol &&
+		c1.Address == c2.Address &&
+		c1.Port == c2.Port &&
+		c1.Foundation == c2.Foundation &&
+		c1.Priority == c2.Priority &&
+		c1.RelatedAddress == c2.RelatedAddress &&
+		c1.RelatedPort == c2.RelatedPort &&
+		c1.TCPType == c2.TCPType
+}
+
+func isICECandidateEqualTo(c1, c2 ice.Candidate) bool {
+	if c1 == nil && c2 == nil {
+		return true
+	}
+	if (c1 == nil && c2 != nil) || (c1 != nil && c2 == nil) {
+		return false
+	}
+	return c1.Type() == c2.Type() &&
+		c1.NetworkType() == c2.NetworkType() &&
+		c1.Address() == c2.Address() &&
+		c1.Port() == c2.Port() &&
+		c1.Foundation() == c2.Foundation() &&
+		c1.Priority() == c2.Priority() &&
+		c1.RelatedAddress().Equal(c2.RelatedAddress()) &&
+		c1.TCPType() == c2.TCPType()
+}
+
+func isICECandidateEqualToCandidate(c1 ice.Candidate, c2 *webrtc.ICECandidate) bool {
+	if c1 == nil && c2 == nil {
+		return true
+	}
+	if (c1 == nil && c2 != nil) || (c1 != nil && c2 == nil) {
+		return false
+	}
+	return c1.Type().String() == c2.Typ.String() &&
+		c1.NetworkType().NetworkShort() == c2.Protocol.String() &&
+		c1.Address() == c2.Address &&
+		c1.Port() == int(c2.Port) &&
+		c1.Foundation() == c2.Foundation &&
+		c1.Priority() == c2.Priority &&
+		c1.TCPType().String() == c2.TCPType
+}
+
+func unmarshalICECandidate(c webrtc.ICECandidateInit) (ice.Candidate, error) {
+	candidateValue := strings.TrimPrefix(c.Candidate, "candidate:")
+	if candidateValue == "" {
+		return nil, nil
+	}
+
+	candidate, err := ice.UnmarshalCandidate(candidateValue)
+	if err != nil {
+		return nil, err
+	}
+
+	return candidate, nil
+}
+
+func unmarshalCandidate(i ice.Candidate) (*webrtc.ICECandidate, error) {
+	var typ webrtc.ICECandidateType
+	switch i.Type() {
+	case ice.CandidateTypeHost:
+		typ = webrtc.ICECandidateTypeHost
+	case ice.CandidateTypeServerReflexive:
+		typ = webrtc.ICECandidateTypeSrflx
+	case ice.CandidateTypePeerReflexive:
+		typ = webrtc.ICECandidateTypePrflx
+	case ice.CandidateTypeRelay:
+		typ = webrtc.ICECandidateTypeRelay
+	default:
+		return nil, fmt.Errorf("unknown candidate type: %s", i.Type())
+	}
+
+	var protocol webrtc.ICEProtocol
+	switch strings.ToLower(i.NetworkType().NetworkShort()) {
+	case "udp":
+		protocol = webrtc.ICEProtocolUDP
+	case "tcp":
+		protocol = webrtc.ICEProtocolTCP
+	default:
+		return nil, fmt.Errorf("unknown network type: %s", i.NetworkType())
+	}
+
+	c := webrtc.ICECandidate{
+		Foundation: i.Foundation(),
+		Priority:   i.Priority(),
+		Address:    i.Address(),
+		Protocol:   protocol,
+		Port:       uint16(i.Port()),
+		Component:  i.Component(),
+		Typ:        typ,
+		TCPType:    i.TCPType().String(),
+	}
+
+	if i.RelatedAddress() != nil {
+		c.RelatedAddress = i.RelatedAddress().Address
+		c.RelatedPort = uint16(i.RelatedAddress().Port)
+	}
+
+	return &c, nil
+}
+
+func IsCandidateMDNS(candidate webrtc.ICECandidateInit) bool {
+	c, err := unmarshalICECandidate(candidate)
+	if err != nil {
+		return false
+	}
+
+	return IsICECandidateMDNS(c)
+}
+
+func IsICECandidateMDNS(candidate ice.Candidate) bool {
+	if candidate == nil {
+		// end-of-candidates candidate
+		return false
+	}
+
+	return strings.HasSuffix(candidate.Address(), ".local")
+}

livekit-server/pkg/rtc/types/interfaces.go

@@ -0,0 +1,635 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package types
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/pion/rtcp"
+	"github.com/pion/webrtc/v4"
+
+	"github.com/livekit/protocol/auth"
+	"github.com/livekit/protocol/livekit"
+	"github.com/livekit/protocol/logger"
+	"github.com/livekit/protocol/utils"
+
+	"github.com/livekit/livekit-server/pkg/routing"
+	"github.com/livekit/livekit-server/pkg/sfu"
+	"github.com/livekit/livekit-server/pkg/sfu/buffer"
+	"github.com/livekit/livekit-server/pkg/sfu/mime"
+	"github.com/livekit/livekit-server/pkg/sfu/pacer"
+)
+
+//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -generate
+
+//counterfeiter:generate . WebsocketClient
+type WebsocketClient interface {
+	ReadMessage() (messageType int, p []byte, err error)
+	WriteMessage(messageType int, data []byte) error
+	WriteControl(messageType int, data []byte, deadline time.Time) error
+	SetReadDeadline(deadline time.Time) error
+	Close() error
+}
+
+type AddSubscriberParams struct {
+	AllTracks bool
+	TrackIDs  []livekit.TrackID
+}
+
+// ---------------------------------------------
+
+type MigrateState int32
+
+const (
+	MigrateStateInit MigrateState = iota
+	MigrateStateSync
+	MigrateStateComplete
+)
+
+func (m MigrateState) String() string {
+	switch m {
+	case MigrateStateInit:
+		return "MIGRATE_STATE_INIT"
+	case MigrateStateSync:
+		return "MIGRATE_STATE_SYNC"
+	case MigrateStateComplete:
+		return "MIGRATE_STATE_COMPLETE"
+	default:
+		return fmt.Sprintf("%d", int(m))
+	}
+}
+
+// ---------------------------------------------
+
+type SubscribedCodecQuality struct {
+	CodecMime mime.MimeType
+	Quality   livekit.VideoQuality
+}
+
+// ---------------------------------------------
+
+type ParticipantCloseReason int
+
+const (
+	ParticipantCloseReasonNone ParticipantCloseReason = iota
+	ParticipantCloseReasonClientRequestLeave
+	ParticipantCloseReasonRoomManagerStop
+	ParticipantCloseReasonVerifyFailed
+	ParticipantCloseReasonJoinFailed
+	ParticipantCloseReasonJoinTimeout
+	ParticipantCloseReasonMessageBusFailed
+	ParticipantCloseReasonPeerConnectionDisconnected
+	ParticipantCloseReasonDuplicateIdentity
+	ParticipantCloseReasonMigrationComplete
+	ParticipantCloseReasonStale
+	ParticipantCloseReasonServiceRequestRemoveParticipant
+	ParticipantCloseReasonServiceRequestDeleteRoom
+	ParticipantCloseReasonSimulateMigration
+	ParticipantCloseReasonSimulateNodeFailure
+	ParticipantCloseReasonSimulateServerLeave
+	ParticipantCloseReasonSimulateLeaveRequest
+	ParticipantCloseReasonNegotiateFailed
+	ParticipantCloseReasonMigrationRequested
+	ParticipantCloseReasonPublicationError
+	ParticipantCloseReasonSubscriptionError
+	ParticipantCloseReasonDataChannelError
+	ParticipantCloseReasonMigrateCodecMismatch
+	ParticipantCloseReasonSignalSourceClose
+	ParticipantCloseReasonRoomClosed
+	ParticipantCloseReasonUserUnavailable
+	ParticipantCloseReasonUserRejected
+)
+
+func (p ParticipantCloseReason) String() string {
+	switch p {
+	case ParticipantCloseReasonNone:
+		return "NONE"
+	case ParticipantCloseReasonClientRequestLeave:
+		return "CLIENT_REQUEST_LEAVE"
+	case ParticipantCloseReasonRoomManagerStop:
+		return "ROOM_MANAGER_STOP"
+	case ParticipantCloseReasonVerifyFailed:
+		return "VERIFY_FAILED"
+	case ParticipantCloseReasonJoinFailed:
+		return "JOIN_FAILED"
+	case ParticipantCloseReasonJoinTimeout:
+		return "JOIN_TIMEOUT"
+	case ParticipantCloseReasonMessageBusFailed:
+		return "MESSAGE_BUS_FAILED"
+	case ParticipantCloseReasonPeerConnectionDisconnected:
+		return "PEER_CONNECTION_DISCONNECTED"
+	case ParticipantCloseReasonDuplicateIdentity:
+		return "DUPLICATE_IDENTITY"
+	case ParticipantCloseReasonMigrationComplete:
+		return "MIGRATION_COMPLETE"
+	case ParticipantCloseReasonStale:
+		return "STALE"
+	case ParticipantCloseReasonServiceRequestRemoveParticipant:
+		return "SERVICE_REQUEST_REMOVE_PARTICIPANT"
+	case ParticipantCloseReasonServiceRequestDeleteRoom:
+		return "SERVICE_REQUEST_DELETE_ROOM"
+	case ParticipantCloseReasonSimulateMigration:
+		return "SIMULATE_MIGRATION"
+	case ParticipantCloseReasonSimulateNodeFailure:
+		return "SIMULATE_NODE_FAILURE"
+	case ParticipantCloseReasonSimulateServerLeave:
+		return "SIMULATE_SERVER_LEAVE"
+	case ParticipantCloseReasonSimulateLeaveRequest:
+		return "SIMULATE_LEAVE_REQUEST"
+	case ParticipantCloseReasonNegotiateFailed:
+		return "NEGOTIATE_FAILED"
+	case ParticipantCloseReasonMigrationRequested:
+		return "MIGRATION_REQUESTED"
+	case ParticipantCloseReasonPublicationError:
+		return "PUBLICATION_ERROR"
+	case ParticipantCloseReasonSubscriptionError:
+		return "SUBSCRIPTION_ERROR"
+	case ParticipantCloseReasonDataChannelError:
+		return "DATA_CHANNEL_ERROR"
+	case ParticipantCloseReasonMigrateCodecMismatch:
+		return "MIGRATE_CODEC_MISMATCH"
+	case ParticipantCloseReasonSignalSourceClose:
+		return "SIGNAL_SOURCE_CLOSE"
+	case ParticipantCloseReasonRoomClosed:
+		return "ROOM_CLOSED"
+	case ParticipantCloseReasonUserUnavailable:
+		return "USER_UNAVAILABLE"
+	case ParticipantCloseReasonUserRejected:
+		return "USER_REJECTED"
+	default:
+		return fmt.Sprintf("%d", int(p))
+	}
+}
+
+func (p ParticipantCloseReason) ToDisconnectReason() livekit.DisconnectReason {
+	switch p {
+	case ParticipantCloseReasonClientRequestLeave, ParticipantCloseReasonSimulateLeaveRequest:
+		return livekit.DisconnectReason_CLIENT_INITIATED
+	case ParticipantCloseReasonRoomManagerStop:
+		return livekit.DisconnectReason_SERVER_SHUTDOWN
+	case ParticipantCloseReasonVerifyFailed, ParticipantCloseReasonJoinFailed, ParticipantCloseReasonJoinTimeout, ParticipantCloseReasonMessageBusFailed:
+		// expected to be connected but is not
+		return livekit.DisconnectReason_JOIN_FAILURE
+	case ParticipantCloseReasonPeerConnectionDisconnected:
+		return livekit.DisconnectReason_STATE_MISMATCH
+	case ParticipantCloseReasonDuplicateIdentity, ParticipantCloseReasonStale:
+		return livekit.DisconnectReason_DUPLICATE_IDENTITY
+	case ParticipantCloseReasonMigrationRequested, ParticipantCloseReasonMigrationComplete, ParticipantCloseReasonSimulateMigration:
+		return livekit.DisconnectReason_MIGRATION
+	case ParticipantCloseReasonServiceRequestRemoveParticipant:
+		return livekit.DisconnectReason_PARTICIPANT_REMOVED
+	case ParticipantCloseReasonServiceRequestDeleteRoom:
+		return livekit.DisconnectReason_ROOM_DELETED
+	case ParticipantCloseReasonSimulateNodeFailure, ParticipantCloseReasonSimulateServerLeave:
+		return livekit.DisconnectReason_SERVER_SHUTDOWN
+	case ParticipantCloseReasonNegotiateFailed, ParticipantCloseReasonPublicationError, ParticipantCloseReasonSubscriptionError, ParticipantCloseReasonDataChannelError, ParticipantCloseReasonMigrateCodecMismatch:
+		return livekit.DisconnectReason_STATE_MISMATCH
+	case ParticipantCloseReasonSignalSourceClose:
+		return livekit.DisconnectReason_SIGNAL_CLOSE
+	case ParticipantCloseReasonRoomClosed:
+		return livekit.DisconnectReason_ROOM_CLOSED
+	case ParticipantCloseReasonUserUnavailable:
+		return livekit.DisconnectReason_USER_UNAVAILABLE
+	case ParticipantCloseReasonUserRejected:
+		return livekit.DisconnectReason_USER_REJECTED
+	default:
+		// the other types will map to unknown reason
+		return livekit.DisconnectReason_UNKNOWN_REASON
+	}
+}
+
+// ---------------------------------------------
+
+type SignallingCloseReason int
+
+const (
+	SignallingCloseReasonUnknown SignallingCloseReason = iota
+	SignallingCloseReasonMigration
+	SignallingCloseReasonResume
+	SignallingCloseReasonTransportFailure
+	SignallingCloseReasonFullReconnectPublicationError
+	SignallingCloseReasonFullReconnectSubscriptionError
+	SignallingCloseReasonFullReconnectDataChannelError
+	SignallingCloseReasonFullReconnectNegotiateFailed
+	SignallingCloseReasonParticipantClose
+	SignallingCloseReasonDisconnectOnResume
+	SignallingCloseReasonDisconnectOnResumeNoMessages
+)
+
+func (s SignallingCloseReason) String() string {
+	switch s {
+	case SignallingCloseReasonUnknown:
+		return "UNKNOWN"
+	case SignallingCloseReasonMigration:
+		return "MIGRATION"
+	case SignallingCloseReasonResume:
+		return "RESUME"
+	case SignallingCloseReasonTransportFailure:
+		return "TRANSPORT_FAILURE"
+	case SignallingCloseReasonFullReconnectPublicationError:
+		return "FULL_RECONNECT_PUBLICATION_ERROR"
+	case SignallingCloseReasonFullReconnectSubscriptionError:
+		return "FULL_RECONNECT_SUBSCRIPTION_ERROR"
+	case SignallingCloseReasonFullReconnectDataChannelError:
+		return "FULL_RECONNECT_DATA_CHANNEL_ERROR"
+	case SignallingCloseReasonFullReconnectNegotiateFailed:
+		return "FULL_RECONNECT_NEGOTIATE_FAILED"
+	case SignallingCloseReasonParticipantClose:
+		return "PARTICIPANT_CLOSE"
+	case SignallingCloseReasonDisconnectOnResume:
+		return "DISCONNECT_ON_RESUME"
+	case SignallingCloseReasonDisconnectOnResumeNoMessages:
+		return "DISCONNECT_ON_RESUME_NO_MESSAGES"
+	default:
+		return fmt.Sprintf("%d", int(s))
+	}
+}
+
+// ---------------------------------------------
+
+//counterfeiter:generate . Participant
+type Participant interface {
+	ID() livekit.ParticipantID
+	Identity() livekit.ParticipantIdentity
+	State() livekit.ParticipantInfo_State
+	ConnectedAt() time.Time
+	CloseReason() ParticipantCloseReason
+	Kind() livekit.ParticipantInfo_Kind
+	IsRecorder() bool
+	IsDependent() bool
+	IsAgent() bool
+
+	CanSkipBroadcast() bool
+	Version() utils.TimedVersion
+	ToProto() *livekit.ParticipantInfo
+
+	IsPublisher() bool
+	GetPublishedTrack(trackID livekit.TrackID) MediaTrack
+	GetPublishedTracks() []MediaTrack
+	RemovePublishedTrack(track MediaTrack, isExpectedToResume bool, shouldClose bool)
+
+	GetAudioLevel() (smoothedLevel float64, active bool)
+
+	// HasPermission checks permission of the subscriber by identity. Returns true if subscriber is allowed to subscribe
+	// to the track with trackID
+	HasPermission(trackID livekit.TrackID, subIdentity livekit.ParticipantIdentity) bool
+
+	// permissions
+	Hidden() bool
+
+	Close(sendLeave bool, reason ParticipantCloseReason, isExpectedToResume bool) error
+
+	SubscriptionPermission() (*livekit.SubscriptionPermission, utils.TimedVersion)
+
+	// updates from remotes
+	UpdateSubscriptionPermission(
+		subscriptionPermission *livekit.SubscriptionPermission,
+		timedVersion utils.TimedVersion,
+		resolverBySid func(participantID livekit.ParticipantID) LocalParticipant,
+	) error
+
+	DebugInfo() map[string]interface{}
+
+	OnMetrics(callback func(Participant, *livekit.DataPacket))
+}
+
+// -------------------------------------------------------
+
+type AddTrackParams struct {
+	Stereo bool
+	Red    bool
+}
+
+//counterfeiter:generate . LocalParticipant
+type LocalParticipant interface {
+	Participant
+
+	ToProtoWithVersion() (*livekit.ParticipantInfo, utils.TimedVersion)
+
+	// getters
+	GetTrailer() []byte
+	GetLogger() logger.Logger
+	GetAdaptiveStream() bool
+	ProtocolVersion() ProtocolVersion
+	SupportsSyncStreamID() bool
+	SupportsTransceiverReuse() bool
+	IsClosed() bool
+	IsReady() bool
+	IsDisconnected() bool
+	Disconnected() <-chan struct{}
+	IsIdle() bool
+	SubscriberAsPrimary() bool
+	GetClientInfo() *livekit.ClientInfo
+	GetClientConfiguration() *livekit.ClientConfiguration
+	GetBufferFactory() *buffer.Factory
+	GetPlayoutDelayConfig() *livekit.PlayoutDelay
+	GetPendingTrack(trackID livekit.TrackID) *livekit.TrackInfo
+	GetICEConnectionInfo() []*ICEConnectionInfo
+	HasConnected() bool
+	GetEnabledPublishCodecs() []*livekit.Codec
+
+	SetResponseSink(sink routing.MessageSink)
+	CloseSignalConnection(reason SignallingCloseReason)
+	UpdateLastSeenSignal()
+	SetSignalSourceValid(valid bool)
+	HandleSignalSourceClose()
+
+	// updates
+	CheckMetadataLimits(name string, metadata string, attributes map[string]string) error
+	SetName(name string)
+	SetMetadata(metadata string)
+	SetAttributes(attributes map[string]string)
+	UpdateAudioTrack(update *livekit.UpdateLocalAudioTrack) error
+	UpdateVideoTrack(update *livekit.UpdateLocalVideoTrack) error
+
+	// permissions
+	ClaimGrants() *auth.ClaimGrants
+	SetPermission(permission *livekit.ParticipantPermission) bool
+	CanPublish() bool
+	CanPublishSource(source livekit.TrackSource) bool
+	CanSubscribe() bool
+	CanPublishData() bool
+
+	// PeerConnection
+	AddICECandidate(candidate webrtc.ICECandidateInit, target livekit.SignalTarget)
+	HandleOffer(sdp webrtc.SessionDescription) error
+	GetAnswer() (webrtc.SessionDescription, error)
+	AddTrack(req *livekit.AddTrackRequest)
+	SetTrackMuted(trackID livekit.TrackID, muted bool, fromAdmin bool) *livekit.TrackInfo
+
+	HandleAnswer(sdp webrtc.SessionDescription)
+	Negotiate(force bool)
+	ICERestart(iceConfig *livekit.ICEConfig)
+	AddTrackLocal(trackLocal webrtc.TrackLocal, params AddTrackParams) (*webrtc.RTPSender, *webrtc.RTPTransceiver, error)
+	AddTransceiverFromTrackLocal(trackLocal webrtc.TrackLocal, params AddTrackParams) (*webrtc.RTPSender, *webrtc.RTPTransceiver, error)
+	RemoveTrackLocal(sender *webrtc.RTPSender) error
+
+	WriteSubscriberRTCP(pkts []rtcp.Packet) error
+
+	// subscriptions
+	SubscribeToTrack(trackID livekit.TrackID)
+	UnsubscribeFromTrack(trackID livekit.TrackID)
+	UpdateSubscribedTrackSettings(trackID livekit.TrackID, settings *livekit.UpdateTrackSettings)
+	GetSubscribedTracks() []SubscribedTrack
+	IsTrackNameSubscribed(publisherIdentity livekit.ParticipantIdentity, trackName string) bool
+	Verify() bool
+	VerifySubscribeParticipantInfo(pID livekit.ParticipantID, version uint32)
+	// WaitUntilSubscribed waits until all subscriptions have been settled, or if the timeout
+	// has been reached. If the timeout expires, it will return an error.
+	WaitUntilSubscribed(timeout time.Duration) error
+	StopAndGetSubscribedTracksForwarderState() map[livekit.TrackID]*livekit.RTPForwarderState
+	SupportsCodecChange() bool
+
+	// returns list of participant identities that the current participant is subscribed to
+	GetSubscribedParticipants() []livekit.ParticipantID
+	IsSubscribedTo(sid livekit.ParticipantID) bool
+
+	GetConnectionQuality() *livekit.ConnectionQualityInfo
+
+	// server sent messages
+	SendJoinResponse(joinResponse *livekit.JoinResponse) error
+	SendParticipantUpdate(participants []*livekit.ParticipantInfo) error
+	SendSpeakerUpdate(speakers []*livekit.SpeakerInfo, force bool) error
+	SendDataPacket(kind livekit.DataPacket_Kind, encoded []byte) error
+	SendRoomUpdate(room *livekit.Room) error
+	SendConnectionQualityUpdate(update *livekit.ConnectionQualityUpdate) error
+	SubscriptionPermissionUpdate(publisherID livekit.ParticipantID, trackID livekit.TrackID, allowed bool)
+	SendRefreshToken(token string) error
+	SendRequestResponse(requestResponse *livekit.RequestResponse) error
+	HandleReconnectAndSendResponse(reconnectReason livekit.ReconnectReason, reconnectResponse *livekit.ReconnectResponse) error
+	IssueFullReconnect(reason ParticipantCloseReason)
+
+	// callbacks
+	OnStateChange(func(p LocalParticipant, state livekit.ParticipantInfo_State))
+	OnMigrateStateChange(func(p LocalParticipant, migrateState MigrateState))
+	// OnTrackPublished - remote added a track
+	OnTrackPublished(func(LocalParticipant, MediaTrack))
+	// OnTrackUpdated - one of its publishedTracks changed in status
+	OnTrackUpdated(callback func(LocalParticipant, MediaTrack))
+	// OnTrackUnpublished - a track was unpublished
+	OnTrackUnpublished(callback func(LocalParticipant, MediaTrack))
+	// OnParticipantUpdate - metadata or permission is updated
+	OnParticipantUpdate(callback func(LocalParticipant))
+	OnDataPacket(callback func(LocalParticipant, livekit.DataPacket_Kind, *livekit.DataPacket))
+	OnSubscribeStatusChanged(fn func(publisherID livekit.ParticipantID, subscribed bool))
+	OnClose(callback func(LocalParticipant))
+	OnClaimsChanged(callback func(LocalParticipant))
+
+	HandleReceiverReport(dt *sfu.DownTrack, report *rtcp.ReceiverReport)
+
+	// session migration
+	MaybeStartMigration(force bool, onStart func()) bool
+	NotifyMigration()
+	SetMigrateState(s MigrateState)
+	MigrateState() MigrateState
+	SetMigrateInfo(
+		previousOffer, previousAnswer *webrtc.SessionDescription,
+		mediaTracks []*livekit.TrackPublishedResponse,
+		dataChannels []*livekit.DataChannelInfo,
+	)
+	IsReconnect() bool
+
+	UpdateMediaRTT(rtt uint32)
+	UpdateSignalingRTT(rtt uint32)
+
+	CacheDownTrack(trackID livekit.TrackID, rtpTransceiver *webrtc.RTPTransceiver, downTrackState sfu.DownTrackState)
+	UncacheDownTrack(rtpTransceiver *webrtc.RTPTransceiver)
+	GetCachedDownTrack(trackID livekit.TrackID) (*webrtc.RTPTransceiver, sfu.DownTrackState)
+
+	SetICEConfig(iceConfig *livekit.ICEConfig)
+	GetICEConfig() *livekit.ICEConfig
+	OnICEConfigChanged(callback func(participant LocalParticipant, iceConfig *livekit.ICEConfig))
+
+	UpdateSubscribedQuality(nodeID livekit.NodeID, trackID livekit.TrackID, maxQualities []SubscribedCodecQuality) error
+	UpdateMediaLoss(nodeID livekit.NodeID, trackID livekit.TrackID, fractionalLoss uint32) error
+
+	// down stream bandwidth management
+	SetSubscriberAllowPause(allowPause bool)
+	SetSubscriberChannelCapacity(channelCapacity int64)
+
+	GetPacer() pacer.Pacer
+
+	GetDisableSenderReportPassThrough() bool
+
+	HandleMetrics(senderParticipantID livekit.ParticipantID, batch *livekit.MetricsBatch) error
+}
+
+// Room is a container of participants, and can provide room-level actions
+//
+//counterfeiter:generate . Room
+type Room interface {
+	Name() livekit.RoomName
+	ID() livekit.RoomID
+	RemoveParticipant(identity livekit.ParticipantIdentity, pID livekit.ParticipantID, reason ParticipantCloseReason)
+	UpdateSubscriptions(participant LocalParticipant, trackIDs []livekit.TrackID, participantTracks []*livekit.ParticipantTracks, subscribe bool)
+	UpdateSubscriptionPermission(participant LocalParticipant, permissions *livekit.SubscriptionPermission) error
+	SyncState(participant LocalParticipant, state *livekit.SyncState) error
+	SimulateScenario(participant LocalParticipant, scenario *livekit.SimulateScenario) error
+	ResolveMediaTrackForSubscriber(sub LocalParticipant, trackID livekit.TrackID) MediaResolverResult
+	GetLocalParticipants() []LocalParticipant
+	IsDataMessageUserPacketDuplicate(ip *livekit.UserPacket) bool
+}
+
+// MediaTrack represents a media track
+//
+//counterfeiter:generate . MediaTrack
+type MediaTrack interface {
+	ID() livekit.TrackID
+	Kind() livekit.TrackType
+	Name() string
+	Source() livekit.TrackSource
+	Stream() string
+
+	UpdateTrackInfo(ti *livekit.TrackInfo)
+	UpdateAudioTrack(update *livekit.UpdateLocalAudioTrack)
+	UpdateVideoTrack(update *livekit.UpdateLocalVideoTrack)
+	ToProto() *livekit.TrackInfo
+
+	PublisherID() livekit.ParticipantID
+	PublisherIdentity() livekit.ParticipantIdentity
+	PublisherVersion() uint32
+
+	IsMuted() bool
+	SetMuted(muted bool)
+
+	IsSimulcast() bool
+
+	GetAudioLevel() (level float64, active bool)
+
+	Close(isExpectedToResume bool)
+	IsOpen() bool
+
+	// callbacks
+	AddOnClose(func(isExpectedToResume bool))
+
+	// subscribers
+	AddSubscriber(participant LocalParticipant) (SubscribedTrack, error)
+	RemoveSubscriber(participantID livekit.ParticipantID, isExpectedToResume bool)
+	IsSubscriber(subID livekit.ParticipantID) bool
+	RevokeDisallowedSubscribers(allowedSubscriberIdentities []livekit.ParticipantIdentity) []livekit.ParticipantIdentity
+	GetAllSubscribers() []livekit.ParticipantID
+	GetNumSubscribers() int
+	OnTrackSubscribed()
+
+	// returns quality information that's appropriate for width & height
+	GetQualityForDimension(width, height uint32) livekit.VideoQuality
+
+	// returns temporal layer that's appropriate for fps
+	GetTemporalLayerForSpatialFps(spatial int32, fps uint32, mime mime.MimeType) int32
+
+	Receivers() []sfu.TrackReceiver
+	ClearAllReceivers(isExpectedToResume bool)
+
+	IsEncrypted() bool
+}
+
+//counterfeiter:generate . LocalMediaTrack
+type LocalMediaTrack interface {
+	MediaTrack
+
+	Restart()
+
+	SignalCid() string
+	HasSdpCid(cid string) bool
+
+	GetConnectionScoreAndQuality() (float32, livekit.ConnectionQuality)
+	GetTrackStats() *livekit.RTPStats
+
+	SetRTT(rtt uint32)
+
+	NotifySubscriberNodeMaxQuality(nodeID livekit.NodeID, qualities []SubscribedCodecQuality)
+	NotifySubscriberNodeMediaLoss(nodeID livekit.NodeID, fractionalLoss uint8)
+}
+
+//counterfeiter:generate . SubscribedTrack
+type SubscribedTrack interface {
+	AddOnBind(f func(error))
+	IsBound() bool
+	Close(isExpectedToResume bool)
+	OnClose(f func(isExpectedToResume bool))
+	ID() livekit.TrackID
+	PublisherID() livekit.ParticipantID
+	PublisherIdentity() livekit.ParticipantIdentity
+	PublisherVersion() uint32
+	SubscriberID() livekit.ParticipantID
+	SubscriberIdentity() livekit.ParticipantIdentity
+	Subscriber() LocalParticipant
+	DownTrack() *sfu.DownTrack
+	MediaTrack() MediaTrack
+	RTPSender() *webrtc.RTPSender
+	IsMuted() bool
+	SetPublisherMuted(muted bool)
+	UpdateSubscriberSettings(settings *livekit.UpdateTrackSettings, isImmediate bool)
+	// selects appropriate video layer according to subscriber preferences
+	UpdateVideoLayer()
+	NeedsNegotiation() bool
+}
+
+type ChangeNotifier interface {
+	AddObserver(key string, onChanged func())
+	RemoveObserver(key string)
+	HasObservers() bool
+	NotifyChanged()
+}
+
+type MediaResolverResult struct {
+	TrackChangedNotifier ChangeNotifier
+	TrackRemovedNotifier ChangeNotifier
+	Track                MediaTrack
+	// is permission given to the requesting participant
+	HasPermission     bool
+	PublisherID       livekit.ParticipantID
+	PublisherIdentity livekit.ParticipantIdentity
+}
+
+// MediaTrackResolver locates a specific media track for a subscriber
+type MediaTrackResolver func(LocalParticipant, livekit.TrackID) MediaResolverResult
+
+// Supervisor/operation monitor related definitions
+type OperationMonitorEvent int
+
+const (
+	OperationMonitorEventPublisherPeerConnectionConnected OperationMonitorEvent = iota
+	OperationMonitorEventAddPendingPublication
+	OperationMonitorEventSetPublicationMute
+	OperationMonitorEventSetPublishedTrack
+	OperationMonitorEventClearPublishedTrack
+)
+
+func (o OperationMonitorEvent) String() string {
+	switch o {
+	case OperationMonitorEventPublisherPeerConnectionConnected:
+		return "PUBLISHER_PEER_CONNECTION_CONNECTED"
+	case OperationMonitorEventAddPendingPublication:
+		return "ADD_PENDING_PUBLICATION"
+	case OperationMonitorEventSetPublicationMute:
+		return "SET_PUBLICATION_MUTE"
+	case OperationMonitorEventSetPublishedTrack:
+		return "SET_PUBLISHED_TRACK"
+	case OperationMonitorEventClearPublishedTrack:
+		return "CLEAR_PUBLISHED_TRACK"
+	default:
+		return fmt.Sprintf("%d", int(o))
+	}
+}
+
+type OperationMonitorData interface{}
+
+type OperationMonitor interface {
+	PostEvent(ome OperationMonitorEvent, omd OperationMonitorData)
+	Check() error
+	IsIdle() bool
+}

livekit-server/pkg/rtc/types/protocol_version.go

@@ -0,0 +1,97 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package types
+
+type ProtocolVersion int
+
+const CurrentProtocol = 15
+
+func (v ProtocolVersion) SupportsPackedStreamId() bool {
+	return v > 0
+}
+
+func (v ProtocolVersion) SupportsProtobuf() bool {
+	return v > 0
+}
+
+func (v ProtocolVersion) HandlesDataPackets() bool {
+	return v > 1
+}
+
+// SubscriberAsPrimary indicates clients initiate subscriber connection as primary
+func (v ProtocolVersion) SubscriberAsPrimary() bool {
+	return v > 2
+}
+
+// SupportsSpeakerChanged - if client handles speaker info deltas, instead of a comprehensive list
+func (v ProtocolVersion) SupportsSpeakerChanged() bool {
+	return v > 2
+}
+
+// SupportsTransceiverReuse - if transceiver reuse is supported, optimizes SDP size
+func (v ProtocolVersion) SupportsTransceiverReuse() bool {
+	return v > 3
+}
+
+// SupportsConnectionQuality - avoid sending frequent ConnectionQuality updates for lower protocol versions
+func (v ProtocolVersion) SupportsConnectionQuality() bool {
+	return v > 4
+}
+
+func (v ProtocolVersion) SupportsSessionMigrate() bool {
+	return v > 5
+}
+
+func (v ProtocolVersion) SupportsICELite() bool {
+	return v > 5
+}
+
+func (v ProtocolVersion) SupportsUnpublish() bool {
+	return v > 6
+}
+
+// SupportFastStart - if client supports fast start, server side will send media streams
+// in the first offer
+func (v ProtocolVersion) SupportFastStart() bool {
+	return v > 7
+}
+
+func (v ProtocolVersion) SupportHandlesDisconnectedUpdate() bool {
+	return v > 8
+}
+
+func (v ProtocolVersion) SupportSyncStreamID() bool {
+	return v > 9
+}
+
+func (v ProtocolVersion) SupportsConnectionQualityLost() bool {
+	return v > 10
+}
+
+func (v ProtocolVersion) SupportsAsyncRoomID() bool {
+	return v > 11
+}
+
+func (v ProtocolVersion) SupportsIdentityBasedReconnection() bool {
+	return v > 11
+}
+
+func (v ProtocolVersion) SupportsRegionsInLeaveRequest() bool {
+	return v > 12
+}
+
+func (v ProtocolVersion) SupportsNonErrorSignalResponse() bool {
+	return v > 14
+}

livekit-server/pkg/rtc/types/trafficstats.go

@@ -0,0 +1,161 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package types
+
+import (
+	"time"
+
+	"github.com/livekit/protocol/livekit"
+)
+
+type TrafficStats struct {
+	StartTime         time.Time
+	EndTime           time.Time
+	Packets           uint32
+	PacketsLost       uint32
+	PacketsPadding    uint32
+	PacketsOutOfOrder uint32
+	Bytes             uint64
+}
+
+type TrafficTypeStats struct {
+	TrackType    livekit.TrackType
+	StreamType   livekit.StreamType
+	TrafficStats *TrafficStats
+}
+
+type TrafficLoad struct {
+	TrafficTypeStats []*TrafficTypeStats
+}
+
+func RTPStatsDiffToTrafficStats(before, after *livekit.RTPStats) *TrafficStats {
+	if after == nil {
+		return nil
+	}
+
+	startTime := after.StartTime
+	if before != nil {
+		startTime = before.EndTime
+	}
+
+	getAfter := func() *TrafficStats {
+		return &TrafficStats{
+			StartTime:         startTime.AsTime(),
+			EndTime:           after.EndTime.AsTime(),
+			Packets:           after.Packets,
+			PacketsLost:       after.PacketsLost,
+			PacketsPadding:    after.PacketsPadding,
+			PacketsOutOfOrder: after.PacketsOutOfOrder,
+			Bytes:             after.Bytes + after.BytesDuplicate + after.BytesPadding,
+		}
+	}
+
+	if before == nil {
+		return getAfter()
+	}
+
+	if (after.Packets - before.Packets) > (1 << 31) {
+		// after packets < before packets, probably got reset, just return after
+		return getAfter()
+	}
+	if ((after.Bytes + after.BytesDuplicate + after.BytesPadding) - (before.Bytes + before.BytesDuplicate + before.BytesPadding)) > (1 << 63) {
+		// after bytes < before bytes, probably got reset, just return after
+		return getAfter()
+	}
+
+	packetsLost := uint32(0)
+	if after.PacketsLost >= before.PacketsLost {
+		packetsLost = after.PacketsLost - before.PacketsLost
+	}
+	return &TrafficStats{
+		StartTime:         startTime.AsTime(),
+		EndTime:           after.EndTime.AsTime(),
+		Packets:           after.Packets - before.Packets,
+		PacketsLost:       packetsLost,
+		PacketsPadding:    after.PacketsPadding - before.PacketsPadding,
+		PacketsOutOfOrder: after.PacketsOutOfOrder - before.PacketsOutOfOrder,
+		Bytes:             (after.Bytes + after.BytesDuplicate + after.BytesPadding) - (before.Bytes + before.BytesDuplicate + before.BytesPadding),
+	}
+}
+
+func AggregateTrafficStats(statsList ...*TrafficStats) *TrafficStats {
+	if len(statsList) == 0 {
+		return nil
+	}
+
+	startTime := time.Time{}
+	endTime := time.Time{}
+
+	packets := uint32(0)
+	packetsLost := uint32(0)
+	packetsPadding := uint32(0)
+	packetsOutOfOrder := uint32(0)
+	bytes := uint64(0)
+
+	for _, stats := range statsList {
+		if startTime.IsZero() || startTime.After(stats.StartTime) {
+			startTime = stats.StartTime
+		}
+
+		if endTime.IsZero() || endTime.Before(stats.EndTime) {
+			endTime = stats.EndTime
+		}
+
+		packets += stats.Packets
+		packetsLost += stats.PacketsLost
+		packetsPadding += stats.PacketsPadding
+		packetsOutOfOrder += stats.PacketsOutOfOrder
+		bytes += stats.Bytes
+	}
+
+	if endTime.IsZero() {
+		endTime = time.Now()
+	}
+	return &TrafficStats{
+		StartTime:         startTime,
+		EndTime:           endTime,
+		Packets:           packets,
+		PacketsLost:       packetsLost,
+		PacketsPadding:    packetsPadding,
+		PacketsOutOfOrder: packetsOutOfOrder,
+		Bytes:             bytes,
+	}
+}
+
+func TrafficLoadToTrafficRate(trafficLoad *TrafficLoad) (
+	packetRateIn float64,
+	byteRateIn float64,
+	packetRateOut float64,
+	byteRateOut float64,
+) {
+	if trafficLoad == nil {
+		return
+	}
+
+	for _, trafficTypeStat := range trafficLoad.TrafficTypeStats {
+		elapsed := trafficTypeStat.TrafficStats.EndTime.Sub(trafficTypeStat.TrafficStats.StartTime).Seconds()
+		packetRate := float64(trafficTypeStat.TrafficStats.Packets) / elapsed
+		byteRate := float64(trafficTypeStat.TrafficStats.Bytes) / elapsed
+		switch trafficTypeStat.StreamType {
+		case livekit.StreamType_UPSTREAM:
+			packetRateIn += packetRate
+			byteRateIn += byteRate
+		case livekit.StreamType_DOWNSTREAM:
+			packetRateOut += packetRate
+			byteRateOut += byteRate
+		}
+	}
+	return
+}

livekit-server/pkg/rtc/types/typesfakes/fake_room.go

@@ -0,0 +1,709 @@
+// Code generated by counterfeiter. DO NOT EDIT.
+package typesfakes
+
+import (
+	"sync"
+
+	"github.com/livekit/livekit-server/pkg/rtc/types"
+	"github.com/livekit/protocol/livekit"
+)
+
+type FakeRoom struct {
+	GetLocalParticipantsStub        func() []types.LocalParticipant
+	getLocalParticipantsMutex       sync.RWMutex
+	getLocalParticipantsArgsForCall []struct {
+	}
+	getLocalParticipantsReturns struct {
+		result1 []types.LocalParticipant
+	}
+	getLocalParticipantsReturnsOnCall map[int]struct {
+		result1 []types.LocalParticipant
+	}
+	IDStub        func() livekit.RoomID
+	iDMutex       sync.RWMutex
+	iDArgsForCall []struct {
+	}
+	iDReturns struct {
+		result1 livekit.RoomID
+	}
+	iDReturnsOnCall map[int]struct {
+		result1 livekit.RoomID
+	}
+	IsDataMessageUserPacketDuplicateStub        func(*livekit.UserPacket) bool
+	isDataMessageUserPacketDuplicateMutex       sync.RWMutex
+	isDataMessageUserPacketDuplicateArgsForCall []struct {
+		arg1 *livekit.UserPacket
+	}
+	isDataMessageUserPacketDuplicateReturns struct {
+		result1 bool
+	}
+	isDataMessageUserPacketDuplicateReturnsOnCall map[int]struct {
+		result1 bool
+	}
+	NameStub        func() livekit.RoomName
+	nameMutex       sync.RWMutex
+	nameArgsForCall []struct {
+	}
+	nameReturns struct {
+		result1 livekit.RoomName
+	}
+	nameReturnsOnCall map[int]struct {
+		result1 livekit.RoomName
+	}
+	RemoveParticipantStub        func(livekit.ParticipantIdentity, livekit.ParticipantID, types.ParticipantCloseReason)
+	removeParticipantMutex       sync.RWMutex
+	removeParticipantArgsForCall []struct {
+		arg1 livekit.ParticipantIdentity
+		arg2 livekit.ParticipantID
+		arg3 types.ParticipantCloseReason
+	}
+	ResolveMediaTrackForSubscriberStub        func(types.LocalParticipant, livekit.TrackID) types.MediaResolverResult
+	resolveMediaTrackForSubscriberMutex       sync.RWMutex
+	resolveMediaTrackForSubscriberArgsForCall []struct {
+		arg1 types.LocalParticipant
+		arg2 livekit.TrackID
+	}
+	resolveMediaTrackForSubscriberReturns struct {
+		result1 types.MediaResolverResult
+	}
+	resolveMediaTrackForSubscriberReturnsOnCall map[int]struct {
+		result1 types.MediaResolverResult
+	}
+	SimulateScenarioStub        func(types.LocalParticipant, *livekit.SimulateScenario) error
+	simulateScenarioMutex       sync.RWMutex
+	simulateScenarioArgsForCall []struct {
+		arg1 types.LocalParticipant
+		arg2 *livekit.SimulateScenario
+	}
+	simulateScenarioReturns struct {
+		result1 error
+	}
+	simulateScenarioReturnsOnCall map[int]struct {
+		result1 error
+	}
+	SyncStateStub        func(types.LocalParticipant, *livekit.SyncState) error
+	syncStateMutex       sync.RWMutex
+	syncStateArgsForCall []struct {
+		arg1 types.LocalParticipant
+		arg2 *livekit.SyncState
+	}
+	syncStateReturns struct {
+		result1 error
+	}
+	syncStateReturnsOnCall map[int]struct {
+		result1 error
+	}
+	UpdateSubscriptionPermissionStub        func(types.LocalParticipant, *livekit.SubscriptionPermission) error
+	updateSubscriptionPermissionMutex       sync.RWMutex
+	updateSubscriptionPermissionArgsForCall []struct {
+		arg1 types.LocalParticipant
+		arg2 *livekit.SubscriptionPermission
+	}
+	updateSubscriptionPermissionReturns struct {
+		result1 error
+	}
+	updateSubscriptionPermissionReturnsOnCall map[int]struct {
+		result1 error
+	}
+	UpdateSubscriptionsStub        func(types.LocalParticipant, []livekit.TrackID, []*livekit.ParticipantTracks, bool)
+	updateSubscriptionsMutex       sync.RWMutex
+	updateSubscriptionsArgsForCall []struct {
+		arg1 types.LocalParticipant
+		arg2 []livekit.TrackID
+		arg3 []*livekit.ParticipantTracks
+		arg4 bool
+	}
+	invocations      map[string][][]interface{}
+	invocationsMutex sync.RWMutex
+}
+
+func (fake *FakeRoom) GetLocalParticipants() []types.LocalParticipant {
+	fake.getLocalParticipantsMutex.Lock()
+	ret, specificReturn := fake.getLocalParticipantsReturnsOnCall[len(fake.getLocalParticipantsArgsForCall)]
+	fake.getLocalParticipantsArgsForCall = append(fake.getLocalParticipantsArgsForCall, struct {
+	}{})
+	stub := fake.GetLocalParticipantsStub
+	fakeReturns := fake.getLocalParticipantsReturns
+	fake.recordInvocation("GetLocalParticipants", []interface{}{})
+	fake.getLocalParticipantsMutex.Unlock()
+	if stub != nil {
+		return stub()
+	}
+	if specificReturn {
+		return ret.result1
+	}
+	return fakeReturns.result1
+}
+
+func (fake *FakeRoom) GetLocalParticipantsCallCount() int {
+	fake.getLocalParticipantsMutex.RLock()
+	defer fake.getLocalParticipantsMutex.RUnlock()
+	return len(fake.getLocalParticipantsArgsForCall)
+}
+
+func (fake *FakeRoom) GetLocalParticipantsCalls(stub func() []types.LocalParticipant) {
+	fake.getLocalParticipantsMutex.Lock()
+	defer fake.getLocalParticipantsMutex.Unlock()
+	fake.GetLocalParticipantsStub = stub
+}
+
+func (fake *FakeRoom) GetLocalParticipantsReturns(result1 []types.LocalParticipant) {
+	fake.getLocalParticipantsMutex.Lock()
+	defer fake.getLocalParticipantsMutex.Unlock()
+	fake.GetLocalParticipantsStub = nil
+	fake.getLocalParticipantsReturns = struct {
+		result1 []types.LocalParticipant
+	}{result1}
+}
+
+func (fake *FakeRoom) GetLocalParticipantsReturnsOnCall(i int, result1 []types.LocalParticipant) {
+	fake.getLocalParticipantsMutex.Lock()
+	defer fake.getLocalParticipantsMutex.Unlock()
+	fake.GetLocalParticipantsStub = nil
+	if fake.getLocalParticipantsReturnsOnCall == nil {
+		fake.getLocalParticipantsReturnsOnCall = make(map[int]struct {
+			result1 []types.LocalParticipant
+		})
+	}
+	fake.getLocalParticipantsReturnsOnCall[i] = struct {
+		result1 []types.LocalParticipant
+	}{result1}
+}
+
+func (fake *FakeRoom) ID() livekit.RoomID {
+	fake.iDMutex.Lock()
+	ret, specificReturn := fake.iDReturnsOnCall[len(fake.iDArgsForCall)]
+	fake.iDArgsForCall = append(fake.iDArgsForCall, struct {
+	}{})
+	stub := fake.IDStub
+	fakeReturns := fake.iDReturns
+	fake.recordInvocation("ID", []interface{}{})
+	fake.iDMutex.Unlock()
+	if stub != nil {
+		return stub()
+	}
+	if specificReturn {
+		return ret.result1
+	}
+	return fakeReturns.result1
+}
+
+func (fake *FakeRoom) IDCallCount() int {
+	fake.iDMutex.RLock()
+	defer fake.iDMutex.RUnlock()
+	return len(fake.iDArgsForCall)
+}
+
+func (fake *FakeRoom) IDCalls(stub func() livekit.RoomID) {
+	fake.iDMutex.Lock()
+	defer fake.iDMutex.Unlock()
+	fake.IDStub = stub
+}
+
+func (fake *FakeRoom) IDReturns(result1 livekit.RoomID) {
+	fake.iDMutex.Lock()
+	defer fake.iDMutex.Unlock()
+	fake.IDStub = nil
+	fake.iDReturns = struct {
+		result1 livekit.RoomID
+	}{result1}
+}
+
+func (fake *FakeRoom) IDReturnsOnCall(i int, result1 livekit.RoomID) {
+	fake.iDMutex.Lock()
+	defer fake.iDMutex.Unlock()
+	fake.IDStub = nil
+	if fake.iDReturnsOnCall == nil {
+		fake.iDReturnsOnCall = make(map[int]struct {
+			result1 livekit.RoomID
+		})
+	}
+	fake.iDReturnsOnCall[i] = struct {
+		result1 livekit.RoomID
+	}{result1}
+}
+
+func (fake *FakeRoom) IsDataMessageUserPacketDuplicate(arg1 *livekit.UserPacket) bool {
+	fake.isDataMessageUserPacketDuplicateMutex.Lock()
+	ret, specificReturn := fake.isDataMessageUserPacketDuplicateReturnsOnCall[len(fake.isDataMessageUserPacketDuplicateArgsForCall)]
+	fake.isDataMessageUserPacketDuplicateArgsForCall = append(fake.isDataMessageUserPacketDuplicateArgsForCall, struct {
+		arg1 *livekit.UserPacket
+	}{arg1})
+	stub := fake.IsDataMessageUserPacketDuplicateStub
+	fakeReturns := fake.isDataMessageUserPacketDuplicateReturns
+	fake.recordInvocation("IsDataMessageUserPacketDuplicate", []interface{}{arg1})
+	fake.isDataMessageUserPacketDuplicateMutex.Unlock()
+	if stub != nil {
+		return stub(arg1)
+	}
+	if specificReturn {
+		return ret.result1
+	}
+	return fakeReturns.result1
+}
+
+func (fake *FakeRoom) IsDataMessageUserPacketDuplicateCallCount() int {
+	fake.isDataMessageUserPacketDuplicateMutex.RLock()
+	defer fake.isDataMessageUserPacketDuplicateMutex.RUnlock()
+	return len(fake.isDataMessageUserPacketDuplicateArgsForCall)
+}
+
+func (fake *FakeRoom) IsDataMessageUserPacketDuplicateCalls(stub func(*livekit.UserPacket) bool) {
+	fake.isDataMessageUserPacketDuplicateMutex.Lock()
+	defer fake.isDataMessageUserPacketDuplicateMutex.Unlock()
+	fake.IsDataMessageUserPacketDuplicateStub = stub
+}
+
+func (fake *FakeRoom) IsDataMessageUserPacketDuplicateArgsForCall(i int) *livekit.UserPacket {
+	fake.isDataMessageUserPacketDuplicateMutex.RLock()
+	defer fake.isDataMessageUserPacketDuplicateMutex.RUnlock()
+	argsForCall := fake.isDataMessageUserPacketDuplicateArgsForCall[i]
+	return argsForCall.arg1
+}
+
+func (fake *FakeRoom) IsDataMessageUserPacketDuplicateReturns(result1 bool) {
+	fake.isDataMessageUserPacketDuplicateMutex.Lock()
+	defer fake.isDataMessageUserPacketDuplicateMutex.Unlock()
+	fake.IsDataMessageUserPacketDuplicateStub = nil
+	fake.isDataMessageUserPacketDuplicateReturns = struct {
+		result1 bool
+	}{result1}
+}
+
+func (fake *FakeRoom) IsDataMessageUserPacketDuplicateReturnsOnCall(i int, result1 bool) {
+	fake.isDataMessageUserPacketDuplicateMutex.Lock()
+	defer fake.isDataMessageUserPacketDuplicateMutex.Unlock()
+	fake.IsDataMessageUserPacketDuplicateStub = nil
+	if fake.isDataMessageUserPacketDuplicateReturnsOnCall == nil {
+		fake.isDataMessageUserPacketDuplicateReturnsOnCall = make(map[int]struct {
+			result1 bool
+		})
+	}
+	fake.isDataMessageUserPacketDuplicateReturnsOnCall[i] = struct {
+		result1 bool
+	}{result1}
+}
+
+func (fake *FakeRoom) Name() livekit.RoomName {
+	fake.nameMutex.Lock()
+	ret, specificReturn := fake.nameReturnsOnCall[len(fake.nameArgsForCall)]
+	fake.nameArgsForCall = append(fake.nameArgsForCall, struct {
+	}{})
+	stub := fake.NameStub
+	fakeReturns := fake.nameReturns
+	fake.recordInvocation("Name", []interface{}{})
+	fake.nameMutex.Unlock()
+	if stub != nil {
+		return stub()
+	}
+	if specificReturn {
+		return ret.result1
+	}
+	return fakeReturns.result1
+}
+
+func (fake *FakeRoom) NameCallCount() int {
+	fake.nameMutex.RLock()
+	defer fake.nameMutex.RUnlock()
+	return len(fake.nameArgsForCall)
+}
+
+func (fake *FakeRoom) NameCalls(stub func() livekit.RoomName) {
+	fake.nameMutex.Lock()
+	defer fake.nameMutex.Unlock()
+	fake.NameStub = stub
+}
+
+func (fake *FakeRoom) NameReturns(result1 livekit.RoomName) {
+	fake.nameMutex.Lock()
+	defer fake.nameMutex.Unlock()
+	fake.NameStub = nil
+	fake.nameReturns = struct {
+		result1 livekit.RoomName
+	}{result1}
+}
+
+func (fake *FakeRoom) NameReturnsOnCall(i int, result1 livekit.RoomName) {
+	fake.nameMutex.Lock()
+	defer fake.nameMutex.Unlock()
+	fake.NameStub = nil
+	if fake.nameReturnsOnCall == nil {
+		fake.nameReturnsOnCall = make(map[int]struct {
+			result1 livekit.RoomName
+		})
+	}
+	fake.nameReturnsOnCall[i] = struct {
+		result1 livekit.RoomName
+	}{result1}
+}
+
+func (fake *FakeRoom) RemoveParticipant(arg1 livekit.ParticipantIdentity, arg2 livekit.ParticipantID, arg3 types.ParticipantCloseReason) {
+	fake.removeParticipantMutex.Lock()
+	fake.removeParticipantArgsForCall = append(fake.removeParticipantArgsForCall, struct {
+		arg1 livekit.ParticipantIdentity
+		arg2 livekit.ParticipantID
+		arg3 types.ParticipantCloseReason
+	}{arg1, arg2, arg3})
+	stub := fake.RemoveParticipantStub
+	fake.recordInvocation("RemoveParticipant", []interface{}{arg1, arg2, arg3})
+	fake.removeParticipantMutex.Unlock()
+	if stub != nil {
+		fake.RemoveParticipantStub(arg1, arg2, arg3)
+	}
+}
+
+func (fake *FakeRoom) RemoveParticipantCallCount() int {
+	fake.removeParticipantMutex.RLock()
+	defer fake.removeParticipantMutex.RUnlock()
+	return len(fake.removeParticipantArgsForCall)
+}
+
+func (fake *FakeRoom) RemoveParticipantCalls(stub func(livekit.ParticipantIdentity, livekit.ParticipantID, types.ParticipantCloseReason)) {
+	fake.removeParticipantMutex.Lock()
+	defer fake.removeParticipantMutex.Unlock()
+	fake.RemoveParticipantStub = stub
+}
+
+func (fake *FakeRoom) RemoveParticipantArgsForCall(i int) (livekit.ParticipantIdentity, livekit.ParticipantID, types.ParticipantCloseReason) {
+	fake.removeParticipantMutex.RLock()
+	defer fake.removeParticipantMutex.RUnlock()
+	argsForCall := fake.removeParticipantArgsForCall[i]
+	return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3
+}
+
+func (fake *FakeRoom) ResolveMediaTrackForSubscriber(arg1 types.LocalParticipant, arg2 livekit.TrackID) types.MediaResolverResult {
+	fake.resolveMediaTrackForSubscriberMutex.Lock()
+	ret, specificReturn := fake.resolveMediaTrackForSubscriberReturnsOnCall[len(fake.resolveMediaTrackForSubscriberArgsForCall)]
+	fake.resolveMediaTrackForSubscriberArgsForCall = append(fake.resolveMediaTrackForSubscriberArgsForCall, struct {
+		arg1 types.LocalParticipant
+		arg2 livekit.TrackID
+	}{arg1, arg2})
+	stub := fake.ResolveMediaTrackForSubscriberStub
+	fakeReturns := fake.resolveMediaTrackForSubscriberReturns
+	fake.recordInvocation("ResolveMediaTrackForSubscriber", []interface{}{arg1, arg2})
+	fake.resolveMediaTrackForSubscriberMutex.Unlock()
+	if stub != nil {
+		return stub(arg1, arg2)
+	}
+	if specificReturn {
+		return ret.result1
+	}
+	return fakeReturns.result1
+}
+
+func (fake *FakeRoom) ResolveMediaTrackForSubscriberCallCount() int {
+	fake.resolveMediaTrackForSubscriberMutex.RLock()
+	defer fake.resolveMediaTrackForSubscriberMutex.RUnlock()
+	return len(fake.resolveMediaTrackForSubscriberArgsForCall)
+}
+
+func (fake *FakeRoom) ResolveMediaTrackForSubscriberCalls(stub func(types.LocalParticipant, livekit.TrackID) types.MediaResolverResult) {
+	fake.resolveMediaTrackForSubscriberMutex.Lock()
+	defer fake.resolveMediaTrackForSubscriberMutex.Unlock()
+	fake.ResolveMediaTrackForSubscriberStub = stub
+}
+
+func (fake *FakeRoom) ResolveMediaTrackForSubscriberArgsForCall(i int) (types.LocalParticipant, livekit.TrackID) {
+	fake.resolveMediaTrackForSubscriberMutex.RLock()
+	defer fake.resolveMediaTrackForSubscriberMutex.RUnlock()
+	argsForCall := fake.resolveMediaTrackForSubscriberArgsForCall[i]
+	return argsForCall.arg1, argsForCall.arg2
+}
+
+func (fake *FakeRoom) ResolveMediaTrackForSubscriberReturns(result1 types.MediaResolverResult) {
+	fake.resolveMediaTrackForSubscriberMutex.Lock()
+	defer fake.resolveMediaTrackForSubscriberMutex.Unlock()
+	fake.ResolveMediaTrackForSubscriberStub = nil
+	fake.resolveMediaTrackForSubscriberReturns = struct {
+		result1 types.MediaResolverResult
+	}{result1}
+}
+
+func (fake *FakeRoom) ResolveMediaTrackForSubscriberReturnsOnCall(i int, result1 types.MediaResolverResult) {
+	fake.resolveMediaTrackForSubscriberMutex.Lock()
+	defer fake.resolveMediaTrackForSubscriberMutex.Unlock()
+	fake.ResolveMediaTrackForSubscriberStub = nil
+	if fake.resolveMediaTrackForSubscriberReturnsOnCall == nil {
+		fake.resolveMediaTrackForSubscriberReturnsOnCall = make(map[int]struct {
+			result1 types.MediaResolverResult
+		})
+	}
+	fake.resolveMediaTrackForSubscriberReturnsOnCall[i] = struct {
+		result1 types.MediaResolverResult
+	}{result1}
+}
+
+func (fake *FakeRoom) SimulateScenario(arg1 types.LocalParticipant, arg2 *livekit.SimulateScenario) error {
+	fake.simulateScenarioMutex.Lock()
+	ret, specificReturn := fake.simulateScenarioReturnsOnCall[len(fake.simulateScenarioArgsForCall)]
+	fake.simulateScenarioArgsForCall = append(fake.simulateScenarioArgsForCall, struct {
+		arg1 types.LocalParticipant
+		arg2 *livekit.SimulateScenario
+	}{arg1, arg2})
+	stub := fake.SimulateScenarioStub
+	fakeReturns := fake.simulateScenarioReturns
+	fake.recordInvocation("SimulateScenario", []interface{}{arg1, arg2})
+	fake.simulateScenarioMutex.Unlock()
+	if stub != nil {
+		return stub(arg1, arg2)
+	}
+	if specificReturn {
+		return ret.result1
+	}
+	return fakeReturns.result1
+}
+
+func (fake *FakeRoom) SimulateScenarioCallCount() int {
+	fake.simulateScenarioMutex.RLock()
+	defer fake.simulateScenarioMutex.RUnlock()
+	return len(fake.simulateScenarioArgsForCall)
+}
+
+func (fake *FakeRoom) SimulateScenarioCalls(stub func(types.LocalParticipant, *livekit.SimulateScenario) error) {
+	fake.simulateScenarioMutex.Lock()
+	defer fake.simulateScenarioMutex.Unlock()
+	fake.SimulateScenarioStub = stub
+}
+
+func (fake *FakeRoom) SimulateScenarioArgsForCall(i int) (types.LocalParticipant, *livekit.SimulateScenario) {
+	fake.simulateScenarioMutex.RLock()
+	defer fake.simulateScenarioMutex.RUnlock()
+	argsForCall := fake.simulateScenarioArgsForCall[i]
+	return argsForCall.arg1, argsForCall.arg2
+}
+
+func (fake *FakeRoom) SimulateScenarioReturns(result1 error) {
+	fake.simulateScenarioMutex.Lock()
+	defer fake.simulateScenarioMutex.Unlock()
+	fake.SimulateScenarioStub = nil
+	fake.simulateScenarioReturns = struct {
+		result1 error
+	}{result1}
+}
+
+func (fake *FakeRoom) SimulateScenarioReturnsOnCall(i int, result1 error) {
+	fake.simulateScenarioMutex.Lock()
+	defer fake.simulateScenarioMutex.Unlock()
+	fake.SimulateScenarioStub = nil
+	if fake.simulateScenarioReturnsOnCall == nil {
+		fake.simulateScenarioReturnsOnCall = make(map[int]struct {
+			result1 error
+		})
+	}
+	fake.simulateScenarioReturnsOnCall[i] = struct {
+		result1 error
+	}{result1}
+}
+
+func (fake *FakeRoom) SyncState(arg1 types.LocalParticipant, arg2 *livekit.SyncState) error {
+	fake.syncStateMutex.Lock()
+	ret, specificReturn := fake.syncStateReturnsOnCall[len(fake.syncStateArgsForCall)]
+	fake.syncStateArgsForCall = append(fake.syncStateArgsForCall, struct {
+		arg1 types.LocalParticipant
+		arg2 *livekit.SyncState
+	}{arg1, arg2})
+	stub := fake.SyncStateStub
+	fakeReturns := fake.syncStateReturns
+	fake.recordInvocation("SyncState", []interface{}{arg1, arg2})
+	fake.syncStateMutex.Unlock()
+	if stub != nil {
+		return stub(arg1, arg2)
+	}
+	if specificReturn {
+		return ret.result1
+	}
+	return fakeReturns.result1
+}
+
+func (fake *FakeRoom) SyncStateCallCount() int {
+	fake.syncStateMutex.RLock()
+	defer fake.syncStateMutex.RUnlock()
+	return len(fake.syncStateArgsForCall)
+}
+
+func (fake *FakeRoom) SyncStateCalls(stub func(types.LocalParticipant, *livekit.SyncState) error) {
+	fake.syncStateMutex.Lock()
+	defer fake.syncStateMutex.Unlock()
+	fake.SyncStateStub = stub
+}
+
+func (fake *FakeRoom) SyncStateArgsForCall(i int) (types.LocalParticipant, *livekit.SyncState) {
+	fake.syncStateMutex.RLock()
+	defer fake.syncStateMutex.RUnlock()
+	argsForCall := fake.syncStateArgsForCall[i]
+	return argsForCall.arg1, argsForCall.arg2
+}
+
+func (fake *FakeRoom) SyncStateReturns(result1 error) {
+	fake.syncStateMutex.Lock()
+	defer fake.syncStateMutex.Unlock()
+	fake.SyncStateStub = nil
+	fake.syncStateReturns = struct {
+		result1 error
+	}{result1}
+}
+
+func (fake *FakeRoom) SyncStateReturnsOnCall(i int, result1 error) {
+	fake.syncStateMutex.Lock()
+	defer fake.syncStateMutex.Unlock()
+	fake.SyncStateStub = nil
+	if fake.syncStateReturnsOnCall == nil {
+		fake.syncStateReturnsOnCall = make(map[int]struct {
+			result1 error
+		})
+	}
+	fake.syncStateReturnsOnCall[i] = struct {
+		result1 error
+	}{result1}
+}
+
+func (fake *FakeRoom) UpdateSubscriptionPermission(arg1 types.LocalParticipant, arg2 *livekit.SubscriptionPermission) error {
+	fake.updateSubscriptionPermissionMutex.Lock()
+	ret, specificReturn := fake.updateSubscriptionPermissionReturnsOnCall[len(fake.updateSubscriptionPermissionArgsForCall)]
+	fake.updateSubscriptionPermissionArgsForCall = append(fake.updateSubscriptionPermissionArgsForCall, struct {
+		arg1 types.LocalParticipant
+		arg2 *livekit.SubscriptionPermission
+	}{arg1, arg2})
+	stub := fake.UpdateSubscriptionPermissionStub
+	fakeReturns := fake.updateSubscriptionPermissionReturns
+	fake.recordInvocation("UpdateSubscriptionPermission", []interface{}{arg1, arg2})
+	fake.updateSubscriptionPermissionMutex.Unlock()
+	if stub != nil {
+		return stub(arg1, arg2)
+	}
+	if specificReturn {
+		return ret.result1
+	}
+	return fakeReturns.result1
+}
+
+func (fake *FakeRoom) UpdateSubscriptionPermissionCallCount() int {
+	fake.updateSubscriptionPermissionMutex.RLock()
+	defer fake.updateSubscriptionPermissionMutex.RUnlock()
+	return len(fake.updateSubscriptionPermissionArgsForCall)
+}
+
+func (fake *FakeRoom) UpdateSubscriptionPermissionCalls(stub func(types.LocalParticipant, *livekit.SubscriptionPermission) error) {
+	fake.updateSubscriptionPermissionMutex.Lock()
+	defer fake.updateSubscriptionPermissionMutex.Unlock()
+	fake.UpdateSubscriptionPermissionStub = stub
+}
+
+func (fake *FakeRoom) UpdateSubscriptionPermissionArgsForCall(i int) (types.LocalParticipant, *livekit.SubscriptionPermission) {
+	fake.updateSubscriptionPermissionMutex.RLock()
+	defer fake.updateSubscriptionPermissionMutex.RUnlock()
+	argsForCall := fake.updateSubscriptionPermissionArgsForCall[i]
+	return argsForCall.arg1, argsForCall.arg2
+}
+
+func (fake *FakeRoom) UpdateSubscriptionPermissionReturns(result1 error) {
+	fake.updateSubscriptionPermissionMutex.Lock()
+	defer fake.updateSubscriptionPermissionMutex.Unlock()
+	fake.UpdateSubscriptionPermissionStub = nil
+	fake.updateSubscriptionPermissionReturns = struct {
+		result1 error
+	}{result1}
+}
+
+func (fake *FakeRoom) UpdateSubscriptionPermissionReturnsOnCall(i int, result1 error) {
+	fake.updateSubscriptionPermissionMutex.Lock()
+	defer fake.updateSubscriptionPermissionMutex.Unlock()
+	fake.UpdateSubscriptionPermissionStub = nil
+	if fake.updateSubscriptionPermissionReturnsOnCall == nil {
+		fake.updateSubscriptionPermissionReturnsOnCall = make(map[int]struct {
+			result1 error
+		})
+	}
+	fake.updateSubscriptionPermissionReturnsOnCall[i] = struct {
+		result1 error
+	}{result1}
+}
+
+func (fake *FakeRoom) UpdateSubscriptions(arg1 types.LocalParticipant, arg2 []livekit.TrackID, arg3 []*livekit.ParticipantTracks, arg4 bool) {
+	var arg2Copy []livekit.TrackID
+	if arg2 != nil {
+		arg2Copy = make([]livekit.TrackID, len(arg2))
+		copy(arg2Copy, arg2)
+	}
+	var arg3Copy []*livekit.ParticipantTracks
+	if arg3 != nil {
+		arg3Copy = make([]*livekit.ParticipantTracks, len(arg3))
+		copy(arg3Copy, arg3)
+	}
+	fake.updateSubscriptionsMutex.Lock()
+	fake.updateSubscriptionsArgsForCall = append(fake.updateSubscriptionsArgsForCall, struct {
+		arg1 types.LocalParticipant
+		arg2 []livekit.TrackID
+		arg3 []*livekit.ParticipantTracks
+		arg4 bool
+	}{arg1, arg2Copy, arg3Copy, arg4})
+	stub := fake.UpdateSubscriptionsStub
+	fake.recordInvocation("UpdateSubscriptions", []interface{}{arg1, arg2Copy, arg3Copy, arg4})
+	fake.updateSubscriptionsMutex.Unlock()
+	if stub != nil {
+		fake.UpdateSubscriptionsStub(arg1, arg2, arg3, arg4)
+	}
+}
+
+func (fake *FakeRoom) UpdateSubscriptionsCallCount() int {
+	fake.updateSubscriptionsMutex.RLock()
+	defer fake.updateSubscriptionsMutex.RUnlock()
+	return len(fake.updateSubscriptionsArgsForCall)
+}
+
+func (fake *FakeRoom) UpdateSubscriptionsCalls(stub func(types.LocalParticipant, []livekit.TrackID, []*livekit.ParticipantTracks, bool)) {
+	fake.updateSubscriptionsMutex.Lock()
+	defer fake.updateSubscriptionsMutex.Unlock()
+	fake.UpdateSubscriptionsStub = stub
+}
+
+func (fake *FakeRoom) UpdateSubscriptionsArgsForCall(i int) (types.LocalParticipant, []livekit.TrackID, []*livekit.ParticipantTracks, bool) {
+	fake.updateSubscriptionsMutex.RLock()
+	defer fake.updateSubscriptionsMutex.RUnlock()
+	argsForCall := fake.updateSubscriptionsArgsForCall[i]
+	return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4
+}
+
+func (fake *FakeRoom) Invocations() map[string][][]interface{} {
+	fake.invocationsMutex.RLock()
+	defer fake.invocationsMutex.RUnlock()
+	fake.getLocalParticipantsMutex.RLock()
+	defer fake.getLocalParticipantsMutex.RUnlock()
+	fake.iDMutex.RLock()
+	defer fake.iDMutex.RUnlock()
+	fake.isDataMessageUserPacketDuplicateMutex.RLock()
+	defer fake.isDataMessageUserPacketDuplicateMutex.RUnlock()
+	fake.nameMutex.RLock()
+	defer fake.nameMutex.RUnlock()
+	fake.removeParticipantMutex.RLock()
+	defer fake.removeParticipantMutex.RUnlock()
+	fake.resolveMediaTrackForSubscriberMutex.RLock()
+	defer fake.resolveMediaTrackForSubscriberMutex.RUnlock()
+	fake.simulateScenarioMutex.RLock()
+	defer fake.simulateScenarioMutex.RUnlock()
+	fake.syncStateMutex.RLock()
+	defer fake.syncStateMutex.RUnlock()
+	fake.updateSubscriptionPermissionMutex.RLock()
+	defer fake.updateSubscriptionPermissionMutex.RUnlock()
+	fake.updateSubscriptionsMutex.RLock()
+	defer fake.updateSubscriptionsMutex.RUnlock()
+	copiedInvocations := map[string][][]interface{}{}
+	for key, value := range fake.invocations {
+		copiedInvocations[key] = value
+	}
+	return copiedInvocations
+}
+
+func (fake *FakeRoom) recordInvocation(key string, args []interface{}) {
+	fake.invocationsMutex.Lock()
+	defer fake.invocationsMutex.Unlock()
+	if fake.invocations == nil {
+		fake.invocations = map[string][][]interface{}{}
+	}
+	if fake.invocations[key] == nil {
+		fake.invocations[key] = [][]interface{}{}
+	}
+	fake.invocations[key] = append(fake.invocations[key], args)
+}
+
+var _ types.Room = new(FakeRoom)

livekit-server/pkg/rtc/types/typesfakes/fake_websocket_client.go

@@ -0,0 +1,416 @@
+// Code generated by counterfeiter. DO NOT EDIT.
+package typesfakes
+
+import (
+	"sync"
+	"time"
+
+	"github.com/livekit/livekit-server/pkg/rtc/types"
+)
+
+type FakeWebsocketClient struct {
+	CloseStub        func() error
+	closeMutex       sync.RWMutex
+	closeArgsForCall []struct {
+	}
+	closeReturns struct {
+		result1 error
+	}
+	closeReturnsOnCall map[int]struct {
+		result1 error
+	}
+	ReadMessageStub        func() (int, []byte, error)
+	readMessageMutex       sync.RWMutex
+	readMessageArgsForCall []struct {
+	}
+	readMessageReturns struct {
+		result1 int
+		result2 []byte
+		result3 error
+	}
+	readMessageReturnsOnCall map[int]struct {
+		result1 int
+		result2 []byte
+		result3 error
+	}
+	SetReadDeadlineStub        func(time.Time) error
+	setReadDeadlineMutex       sync.RWMutex
+	setReadDeadlineArgsForCall []struct {
+		arg1 time.Time
+	}
+	setReadDeadlineReturns struct {
+		result1 error
+	}
+	setReadDeadlineReturnsOnCall map[int]struct {
+		result1 error
+	}
+	WriteControlStub        func(int, []byte, time.Time) error
+	writeControlMutex       sync.RWMutex
+	writeControlArgsForCall []struct {
+		arg1 int
+		arg2 []byte
+		arg3 time.Time
+	}
+	writeControlReturns struct {
+		result1 error
+	}
+	writeControlReturnsOnCall map[int]struct {
+		result1 error
+	}
+	WriteMessageStub        func(int, []byte) error
+	writeMessageMutex       sync.RWMutex
+	writeMessageArgsForCall []struct {
+		arg1 int
+		arg2 []byte
+	}
+	writeMessageReturns struct {
+		result1 error
+	}
+	writeMessageReturnsOnCall map[int]struct {
+		result1 error
+	}
+	invocations      map[string][][]interface{}
+	invocationsMutex sync.RWMutex
+}
+
+func (fake *FakeWebsocketClient) Close() error {
+	fake.closeMutex.Lock()
+	ret, specificReturn := fake.closeReturnsOnCall[len(fake.closeArgsForCall)]
+	fake.closeArgsForCall = append(fake.closeArgsForCall, struct {
+	}{})
+	stub := fake.CloseStub
+	fakeReturns := fake.closeReturns
+	fake.recordInvocation("Close", []interface{}{})
+	fake.closeMutex.Unlock()
+	if stub != nil {
+		return stub()
+	}
+	if specificReturn {
+		return ret.result1
+	}
+	return fakeReturns.result1
+}
+
+func (fake *FakeWebsocketClient) CloseCallCount() int {
+	fake.closeMutex.RLock()
+	defer fake.closeMutex.RUnlock()
+	return len(fake.closeArgsForCall)
+}
+
+func (fake *FakeWebsocketClient) CloseCalls(stub func() error) {
+	fake.closeMutex.Lock()
+	defer fake.closeMutex.Unlock()
+	fake.CloseStub = stub
+}
+
+func (fake *FakeWebsocketClient) CloseReturns(result1 error) {
+	fake.closeMutex.Lock()
+	defer fake.closeMutex.Unlock()
+	fake.CloseStub = nil
+	fake.closeReturns = struct {
+		result1 error
+	}{result1}
+}
+
+func (fake *FakeWebsocketClient) CloseReturnsOnCall(i int, result1 error) {
+	fake.closeMutex.Lock()
+	defer fake.closeMutex.Unlock()
+	fake.CloseStub = nil
+	if fake.closeReturnsOnCall == nil {
+		fake.closeReturnsOnCall = make(map[int]struct {
+			result1 error
+		})
+	}
+	fake.closeReturnsOnCall[i] = struct {
+		result1 error
+	}{result1}
+}
+
+func (fake *FakeWebsocketClient) ReadMessage() (int, []byte, error) {
+	fake.readMessageMutex.Lock()
+	ret, specificReturn := fake.readMessageReturnsOnCall[len(fake.readMessageArgsForCall)]
+	fake.readMessageArgsForCall = append(fake.readMessageArgsForCall, struct {
+	}{})
+	stub := fake.ReadMessageStub
+	fakeReturns := fake.readMessageReturns
+	fake.recordInvocation("ReadMessage", []interface{}{})
+	fake.readMessageMutex.Unlock()
+	if stub != nil {
+		return stub()
+	}
+	if specificReturn {
+		return ret.result1, ret.result2, ret.result3
+	}
+	return fakeReturns.result1, fakeReturns.result2, fakeReturns.result3
+}
+
+func (fake *FakeWebsocketClient) ReadMessageCallCount() int {
+	fake.readMessageMutex.RLock()
+	defer fake.readMessageMutex.RUnlock()
+	return len(fake.readMessageArgsForCall)
+}
+
+func (fake *FakeWebsocketClient) ReadMessageCalls(stub func() (int, []byte, error)) {
+	fake.readMessageMutex.Lock()
+	defer fake.readMessageMutex.Unlock()
+	fake.ReadMessageStub = stub
+}
+
+func (fake *FakeWebsocketClient) ReadMessageReturns(result1 int, result2 []byte, result3 error) {
+	fake.readMessageMutex.Lock()
+	defer fake.readMessageMutex.Unlock()
+	fake.ReadMessageStub = nil
+	fake.readMessageReturns = struct {
+		result1 int
+		result2 []byte
+		result3 error
+	}{result1, result2, result3}
+}
+
+func (fake *FakeWebsocketClient) ReadMessageReturnsOnCall(i int, result1 int, result2 []byte, result3 error) {
+	fake.readMessageMutex.Lock()
+	defer fake.readMessageMutex.Unlock()
+	fake.ReadMessageStub = nil
+	if fake.readMessageReturnsOnCall == nil {
+		fake.readMessageReturnsOnCall = make(map[int]struct {
+			result1 int
+			result2 []byte
+			result3 error
+		})
+	}
+	fake.readMessageReturnsOnCall[i] = struct {
+		result1 int
+		result2 []byte
+		result3 error
+	}{result1, result2, result3}
+}
+
+func (fake *FakeWebsocketClient) SetReadDeadline(arg1 time.Time) error {
+	fake.setReadDeadlineMutex.Lock()
+	ret, specificReturn := fake.setReadDeadlineReturnsOnCall[len(fake.setReadDeadlineArgsForCall)]
+	fake.setReadDeadlineArgsForCall = append(fake.setReadDeadlineArgsForCall, struct {
+		arg1 time.Time
+	}{arg1})
+	stub := fake.SetReadDeadlineStub
+	fakeReturns := fake.setReadDeadlineReturns
+	fake.recordInvocation("SetReadDeadline", []interface{}{arg1})
+	fake.setReadDeadlineMutex.Unlock()
+	if stub != nil {
+		return stub(arg1)
+	}
+	if specificReturn {
+		return ret.result1
+	}
+	return fakeReturns.result1
+}
+
+func (fake *FakeWebsocketClient) SetReadDeadlineCallCount() int {
+	fake.setReadDeadlineMutex.RLock()
+	defer fake.setReadDeadlineMutex.RUnlock()
+	return len(fake.setReadDeadlineArgsForCall)
+}
+
+func (fake *FakeWebsocketClient) SetReadDeadlineCalls(stub func(time.Time) error) {
+	fake.setReadDeadlineMutex.Lock()
+	defer fake.setReadDeadlineMutex.Unlock()
+	fake.SetReadDeadlineStub = stub
+}
+
+func (fake *FakeWebsocketClient) SetReadDeadlineArgsForCall(i int) time.Time {
+	fake.setReadDeadlineMutex.RLock()
+	defer fake.setReadDeadlineMutex.RUnlock()
+	argsForCall := fake.setReadDeadlineArgsForCall[i]
+	return argsForCall.arg1
+}
+
+func (fake *FakeWebsocketClient) SetReadDeadlineReturns(result1 error) {
+	fake.setReadDeadlineMutex.Lock()
+	defer fake.setReadDeadlineMutex.Unlock()
+	fake.SetReadDeadlineStub = nil
+	fake.setReadDeadlineReturns = struct {
+		result1 error
+	}{result1}
+}
+
+func (fake *FakeWebsocketClient) SetReadDeadlineReturnsOnCall(i int, result1 error) {
+	fake.setReadDeadlineMutex.Lock()
+	defer fake.setReadDeadlineMutex.Unlock()
+	fake.SetReadDeadlineStub = nil
+	if fake.setReadDeadlineReturnsOnCall == nil {
+		fake.setReadDeadlineReturnsOnCall = make(map[int]struct {
+			result1 error
+		})
+	}
+	fake.setReadDeadlineReturnsOnCall[i] = struct {
+		result1 error
+	}{result1}
+}
+
+func (fake *FakeWebsocketClient) WriteControl(arg1 int, arg2 []byte, arg3 time.Time) error {
+	var arg2Copy []byte
+	if arg2 != nil {
+		arg2Copy = make([]byte, len(arg2))
+		copy(arg2Copy, arg2)
+	}
+	fake.writeControlMutex.Lock()
+	ret, specificReturn := fake.writeControlReturnsOnCall[len(fake.writeControlArgsForCall)]
+	fake.writeControlArgsForCall = append(fake.writeControlArgsForCall, struct {
+		arg1 int
+		arg2 []byte
+		arg3 time.Time
+	}{arg1, arg2Copy, arg3})
+	stub := fake.WriteControlStub
+	fakeReturns := fake.writeControlReturns
+	fake.recordInvocation("WriteControl", []interface{}{arg1, arg2Copy, arg3})
+	fake.writeControlMutex.Unlock()
+	if stub != nil {
+		return stub(arg1, arg2, arg3)
+	}
+	if specificReturn {
+		return ret.result1
+	}
+	return fakeReturns.result1
+}
+
+func (fake *FakeWebsocketClient) WriteControlCallCount() int {
+	fake.writeControlMutex.RLock()
+	defer fake.writeControlMutex.RUnlock()
+	return len(fake.writeControlArgsForCall)
+}
+
+func (fake *FakeWebsocketClient) WriteControlCalls(stub func(int, []byte, time.Time) error) {
+	fake.writeControlMutex.Lock()
+	defer fake.writeControlMutex.Unlock()
+	fake.WriteControlStub = stub
+}
+
+func (fake *FakeWebsocketClient) WriteControlArgsForCall(i int) (int, []byte, time.Time) {
+	fake.writeControlMutex.RLock()
+	defer fake.writeControlMutex.RUnlock()
+	argsForCall := fake.writeControlArgsForCall[i]
+	return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3
+}
+
+func (fake *FakeWebsocketClient) WriteControlReturns(result1 error) {
+	fake.writeControlMutex.Lock()
+	defer fake.writeControlMutex.Unlock()
+	fake.WriteControlStub = nil
+	fake.writeControlReturns = struct {
+		result1 error
+	}{result1}
+}
+
+func (fake *FakeWebsocketClient) WriteControlReturnsOnCall(i int, result1 error) {
+	fake.writeControlMutex.Lock()
+	defer fake.writeControlMutex.Unlock()
+	fake.WriteControlStub = nil
+	if fake.writeControlReturnsOnCall == nil {
+		fake.writeControlReturnsOnCall = make(map[int]struct {
+			result1 error
+		})
+	}
+	fake.writeControlReturnsOnCall[i] = struct {
+		result1 error
+	}{result1}
+}
+
+func (fake *FakeWebsocketClient) WriteMessage(arg1 int, arg2 []byte) error {
+	var arg2Copy []byte
+	if arg2 != nil {
+		arg2Copy = make([]byte, len(arg2))
+		copy(arg2Copy, arg2)
+	}
+	fake.writeMessageMutex.Lock()
+	ret, specificReturn := fake.writeMessageReturnsOnCall[len(fake.writeMessageArgsForCall)]
+	fake.writeMessageArgsForCall = append(fake.writeMessageArgsForCall, struct {
+		arg1 int
+		arg2 []byte
+	}{arg1, arg2Copy})
+	stub := fake.WriteMessageStub
+	fakeReturns := fake.writeMessageReturns
+	fake.recordInvocation("WriteMessage", []interface{}{arg1, arg2Copy})
+	fake.writeMessageMutex.Unlock()
+	if stub != nil {
+		return stub(arg1, arg2)
+	}
+	if specificReturn {
+		return ret.result1
+	}
+	return fakeReturns.result1
+}
+
+func (fake *FakeWebsocketClient) WriteMessageCallCount() int {
+	fake.writeMessageMutex.RLock()
+	defer fake.writeMessageMutex.RUnlock()
+	return len(fake.writeMessageArgsForCall)
+}
+
+func (fake *FakeWebsocketClient) WriteMessageCalls(stub func(int, []byte) error) {
+	fake.writeMessageMutex.Lock()
+	defer fake.writeMessageMutex.Unlock()
+	fake.WriteMessageStub = stub
+}
+
+func (fake *FakeWebsocketClient) WriteMessageArgsForCall(i int) (int, []byte) {
+	fake.writeMessageMutex.RLock()
+	defer fake.writeMessageMutex.RUnlock()
+	argsForCall := fake.writeMessageArgsForCall[i]
+	return argsForCall.arg1, argsForCall.arg2
+}
+
+func (fake *FakeWebsocketClient) WriteMessageReturns(result1 error) {
+	fake.writeMessageMutex.Lock()
+	defer fake.writeMessageMutex.Unlock()
+	fake.WriteMessageStub = nil
+	fake.writeMessageReturns = struct {
+		result1 error
+	}{result1}
+}
+
+func (fake *FakeWebsocketClient) WriteMessageReturnsOnCall(i int, result1 error) {
+	fake.writeMessageMutex.Lock()
+	defer fake.writeMessageMutex.Unlock()
+	fake.WriteMessageStub = nil
+	if fake.writeMessageReturnsOnCall == nil {
+		fake.writeMessageReturnsOnCall = make(map[int]struct {
+			result1 error
+		})
+	}
+	fake.writeMessageReturnsOnCall[i] = struct {
+		result1 error
+	}{result1}
+}
+
+func (fake *FakeWebsocketClient) Invocations() map[string][][]interface{} {
+	fake.invocationsMutex.RLock()
+	defer fake.invocationsMutex.RUnlock()
+	fake.closeMutex.RLock()
+	defer fake.closeMutex.RUnlock()
+	fake.readMessageMutex.RLock()
+	defer fake.readMessageMutex.RUnlock()
+	fake.setReadDeadlineMutex.RLock()
+	defer fake.setReadDeadlineMutex.RUnlock()
+	fake.writeControlMutex.RLock()
+	defer fake.writeControlMutex.RUnlock()
+	fake.writeMessageMutex.RLock()
+	defer fake.writeMessageMutex.RUnlock()
+	copiedInvocations := map[string][][]interface{}{}
+	for key, value := range fake.invocations {
+		copiedInvocations[key] = value
+	}
+	return copiedInvocations
+}
+
+func (fake *FakeWebsocketClient) recordInvocation(key string, args []interface{}) {
+	fake.invocationsMutex.Lock()
+	defer fake.invocationsMutex.Unlock()
+	if fake.invocations == nil {
+		fake.invocations = map[string][][]interface{}{}
+	}
+	if fake.invocations[key] == nil {
+		fake.invocations[key] = [][]interface{}{}
+	}
+	fake.invocations[key] = append(fake.invocations[key], args)
+}
+
+var _ types.WebsocketClient = new(FakeWebsocketClient)

livekit-server/pkg/rtc/unhandlesimulcast.go

@@ -0,0 +1,132 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package rtc
+
+import (
+	"github.com/pion/interceptor"
+	"github.com/pion/rtp"
+	"github.com/pion/sdp/v3"
+	"github.com/pion/webrtc/v4"
+
+	"github.com/livekit/livekit-server/pkg/sfu/utils"
+)
+
+const (
+	simulcastProbeCount = 10
+)
+
+type UnhandleSimulcastOption func(r *UnhandleSimulcastInterceptor) error
+
+func UnhandleSimulcastTracks(tracks map[uint32]SimulcastTrackInfo) UnhandleSimulcastOption {
+	return func(r *UnhandleSimulcastInterceptor) error {
+		r.simTracks = tracks
+		return nil
+	}
+}
+
+type UnhandleSimulcastInterceptorFactory struct {
+	opts []UnhandleSimulcastOption
+}
+
+func (f *UnhandleSimulcastInterceptorFactory) NewInterceptor(id string) (interceptor.Interceptor, error) {
+	i := &UnhandleSimulcastInterceptor{simTracks: map[uint32]SimulcastTrackInfo{}}
+	for _, o := range f.opts {
+		if err := o(i); err != nil {
+			return nil, err
+		}
+	}
+	return i, nil
+}
+
+func NewUnhandleSimulcastInterceptorFactory(opts ...UnhandleSimulcastOption) (*UnhandleSimulcastInterceptorFactory, error) {
+	return &UnhandleSimulcastInterceptorFactory{opts: opts}, nil
+}
+
+type unhandleSimulcastRTPReader struct {
+	SimulcastTrackInfo
+	tryTimes            int
+	reader              interceptor.RTPReader
+	midExtensionID      uint8
+	streamIDExtensionID uint8
+}
+
+func (r *unhandleSimulcastRTPReader) Read(b []byte, a interceptor.Attributes) (int, interceptor.Attributes, error) {
+	n, a, err := r.reader.Read(b, a)
+	if r.tryTimes < 0 || err != nil {
+		return n, a, err
+	}
+
+	header := rtp.Header{}
+	hsize, err := header.Unmarshal(b[:n])
+	if err != nil {
+		return n, a, nil
+	}
+	var mid, rid string
+	if payload := header.GetExtension(r.midExtensionID); payload != nil {
+		mid = string(payload)
+	}
+
+	if payload := header.GetExtension(r.streamIDExtensionID); payload != nil {
+		rid = string(payload)
+	}
+
+	if mid != "" && rid != "" {
+		r.tryTimes = -1
+		return n, a, nil
+	}
+
+	r.tryTimes--
+
+	if mid == "" {
+		header.SetExtension(r.midExtensionID, []byte(r.Mid))
+	}
+	if rid == "" {
+		header.SetExtension(r.streamIDExtensionID, []byte(r.Rid))
+	}
+
+	hsize2 := header.MarshalSize()
+
+	if hsize2-hsize+n > len(b) { // no enough buf to set extension
+		return n, a, nil
+	}
+	copy(b[hsize2:], b[hsize:n])
+	header.MarshalTo(b)
+	return hsize2 - hsize + n, a, nil
+}
+
+type UnhandleSimulcastInterceptor struct {
+	interceptor.NoOp
+	simTracks map[uint32]SimulcastTrackInfo
+}
+
+func (u *UnhandleSimulcastInterceptor) BindRemoteStream(info *interceptor.StreamInfo, reader interceptor.RTPReader) interceptor.RTPReader {
+	if t, ok := u.simTracks[info.SSRC]; ok {
+		// if we support fec for simulcast streams at future, should get rsid extensions
+		midExtensionID := utils.GetHeaderExtensionID(info.RTPHeaderExtensions, webrtc.RTPHeaderExtensionCapability{URI: sdp.SDESMidURI})
+		streamIDExtensionID := utils.GetHeaderExtensionID(info.RTPHeaderExtensions, webrtc.RTPHeaderExtensionCapability{URI: sdp.SDESRTPStreamIDURI})
+		if midExtensionID == 0 || streamIDExtensionID == 0 {
+			return reader
+		}
+
+		return &unhandleSimulcastRTPReader{
+			SimulcastTrackInfo:  t,
+			reader:              reader,
+			tryTimes:            simulcastProbeCount,
+			midExtensionID:      uint8(midExtensionID),
+			streamIDExtensionID: uint8(streamIDExtensionID),
+		}
+	}
+	return reader
+}

livekit-server/pkg/rtc/uptrackmanager.go

@@ -0,0 +1,439 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package rtc
+
+import (
+	"errors"
+	"sync"
+
+	"github.com/livekit/protocol/livekit"
+	"github.com/livekit/protocol/logger"
+	"golang.org/x/exp/maps"
+
+	"github.com/livekit/livekit-server/pkg/rtc/types"
+	"github.com/livekit/protocol/utils"
+)
+
+var (
+	ErrSubscriptionPermissionNeedsId = errors.New("either participant identity or SID needed")
+)
+
+type UpTrackManagerParams struct {
+	Logger           logger.Logger
+	VersionGenerator utils.TimedVersionGenerator
+}
+
+// UpTrackManager manages all uptracks from a participant
+type UpTrackManager struct {
+	// utils.TimedVersion is a atomic. To be correctly aligned also on 32bit archs
+	// 64it atomics need to be at the front of a struct
+	subscriptionPermissionVersion utils.TimedVersion
+
+	params UpTrackManagerParams
+
+	closed bool
+
+	// publishedTracks that participant is publishing
+	publishedTracks        map[livekit.TrackID]types.MediaTrack
+	subscriptionPermission *livekit.SubscriptionPermission
+	// subscriber permission for published tracks
+	subscriberPermissions map[livekit.ParticipantIdentity]*livekit.TrackPermission // subscriberIdentity => *livekit.TrackPermission
+
+	lock sync.RWMutex
+
+	// callbacks & handlers
+	onClose        func()
+	onTrackUpdated func(track types.MediaTrack)
+}
+
+func NewUpTrackManager(params UpTrackManagerParams) *UpTrackManager {
+	return &UpTrackManager{
+		params:          params,
+		publishedTracks: make(map[livekit.TrackID]types.MediaTrack),
+	}
+}
+
+func (u *UpTrackManager) Close(isExpectedToResume bool) {
+	u.lock.Lock()
+	if u.closed {
+		u.lock.Unlock()
+		return
+	}
+
+	u.closed = true
+
+	publishedTracks := u.publishedTracks
+	u.publishedTracks = make(map[livekit.TrackID]types.MediaTrack)
+	u.lock.Unlock()
+
+	for _, t := range publishedTracks {
+		t.Close(isExpectedToResume)
+	}
+
+	if onClose := u.getOnUpTrackManagerClose(); onClose != nil {
+		onClose()
+	}
+}
+
+func (u *UpTrackManager) OnUpTrackManagerClose(f func()) {
+	u.lock.Lock()
+	u.onClose = f
+	u.lock.Unlock()
+}
+
+func (u *UpTrackManager) getOnUpTrackManagerClose() func() {
+	u.lock.RLock()
+	defer u.lock.RUnlock()
+
+	return u.onClose
+}
+
+func (u *UpTrackManager) ToProto() []*livekit.TrackInfo {
+	u.lock.RLock()
+	defer u.lock.RUnlock()
+
+	var trackInfos []*livekit.TrackInfo
+	for _, t := range u.publishedTracks {
+		trackInfos = append(trackInfos, t.ToProto())
+	}
+
+	return trackInfos
+}
+
+func (u *UpTrackManager) OnPublishedTrackUpdated(f func(track types.MediaTrack)) {
+	u.onTrackUpdated = f
+}
+
+func (u *UpTrackManager) SetPublishedTrackMuted(trackID livekit.TrackID, muted bool) types.MediaTrack {
+	track := u.GetPublishedTrack(trackID)
+	if track != nil {
+		currentMuted := track.IsMuted()
+		track.SetMuted(muted)
+
+		if currentMuted != track.IsMuted() {
+			u.params.Logger.Debugw("publisher mute status changed", "trackID", trackID, "muted", track.IsMuted())
+			if u.onTrackUpdated != nil {
+				u.onTrackUpdated(track)
+			}
+		}
+	}
+
+	return track
+}
+
+func (u *UpTrackManager) GetPublishedTrack(trackID livekit.TrackID) types.MediaTrack {
+	u.lock.RLock()
+	defer u.lock.RUnlock()
+
+	return u.getPublishedTrackLocked(trackID)
+}
+
+func (u *UpTrackManager) GetPublishedTracks() []types.MediaTrack {
+	u.lock.RLock()
+	defer u.lock.RUnlock()
+
+	return maps.Values(u.publishedTracks)
+}
+
+func (u *UpTrackManager) UpdateSubscriptionPermission(
+	subscriptionPermission *livekit.SubscriptionPermission,
+	timedVersion utils.TimedVersion,
+	resolverBySid func(participantID livekit.ParticipantID) types.LocalParticipant,
+) error {
+	u.lock.Lock()
+	if !timedVersion.IsZero() {
+		// it's possible for permission updates to come from another node. In that case
+		// they would be the authority for this participant's permissions
+		// we do not want to initialize subscriptionPermissionVersion too early since if another machine is the
+		// owner for the data, we'd prefer to use their TimedVersion
+		// ignore older version
+		if !timedVersion.After(u.subscriptionPermissionVersion) {
+			u.params.Logger.Debugw(
+				"skipping older subscription permission version",
+				"existingValue", logger.Proto(u.subscriptionPermission),
+				"existingVersion", &u.subscriptionPermissionVersion,
+				"requestingValue", logger.Proto(subscriptionPermission),
+				"requestingVersion", &timedVersion,
+			)
+			u.lock.Unlock()
+			return nil
+		}
+		u.subscriptionPermissionVersion.Update(timedVersion)
+	} else {
+		// for requests coming from the current node, use local versions
+		u.subscriptionPermissionVersion.Update(u.params.VersionGenerator.Next())
+	}
+
+	// store as is for use when migrating
+	u.subscriptionPermission = subscriptionPermission
+	if subscriptionPermission == nil {
+		u.params.Logger.Debugw(
+			"updating subscription permission, setting to nil",
+			"version", u.subscriptionPermissionVersion,
+		)
+		// possible to get a nil when migrating
+		u.lock.Unlock()
+		return nil
+	}
+
+	u.params.Logger.Debugw(
+		"updating subscription permission",
+		"permissions", logger.Proto(u.subscriptionPermission),
+		"version", u.subscriptionPermissionVersion,
+	)
+	if err := u.parseSubscriptionPermissionsLocked(subscriptionPermission, func(pID livekit.ParticipantID) types.LocalParticipant {
+		u.lock.Unlock()
+		var p types.LocalParticipant
+		if resolverBySid != nil {
+			p = resolverBySid(pID)
+		}
+		u.lock.Lock()
+		return p
+	}); err != nil {
+		// when failed, do not override previous permissions
+		u.params.Logger.Errorw("failed updating subscription permission", err)
+		u.lock.Unlock()
+		return err
+	}
+	u.lock.Unlock()
+
+	u.maybeRevokeSubscriptions()
+
+	return nil
+}
+
+func (u *UpTrackManager) SubscriptionPermission() (*livekit.SubscriptionPermission, utils.TimedVersion) {
+	u.lock.RLock()
+	defer u.lock.RUnlock()
+
+	if u.subscriptionPermissionVersion.IsZero() {
+		return nil, u.subscriptionPermissionVersion.Load()
+	}
+
+	return u.subscriptionPermission, u.subscriptionPermissionVersion.Load()
+}
+
+func (u *UpTrackManager) HasPermission(trackID livekit.TrackID, subIdentity livekit.ParticipantIdentity) bool {
+	u.lock.RLock()
+	defer u.lock.RUnlock()
+
+	return u.hasPermissionLocked(trackID, subIdentity)
+}
+
+func (u *UpTrackManager) UpdatePublishedAudioTrack(update *livekit.UpdateLocalAudioTrack) types.MediaTrack {
+	track := u.GetPublishedTrack(livekit.TrackID(update.TrackSid))
+	if track != nil {
+		track.UpdateAudioTrack(update)
+		if u.onTrackUpdated != nil {
+			u.onTrackUpdated(track)
+		}
+	}
+
+	return track
+}
+
+func (u *UpTrackManager) UpdatePublishedVideoTrack(update *livekit.UpdateLocalVideoTrack) types.MediaTrack {
+	track := u.GetPublishedTrack(livekit.TrackID(update.TrackSid))
+	if track != nil {
+		track.UpdateVideoTrack(update)
+		if u.onTrackUpdated != nil {
+			u.onTrackUpdated(track)
+		}
+	}
+
+	return track
+}
+
+func (u *UpTrackManager) AddPublishedTrack(track types.MediaTrack) {
+	u.lock.Lock()
+	if _, ok := u.publishedTracks[track.ID()]; !ok {
+		u.publishedTracks[track.ID()] = track
+	}
+	u.lock.Unlock()
+	u.params.Logger.Debugw("added published track", "trackID", track.ID(), "trackInfo", logger.Proto(track.ToProto()))
+
+	track.AddOnClose(func(_isExpectedToResume bool) {
+		u.lock.Lock()
+		delete(u.publishedTracks, track.ID())
+		// not modifying subscription permissions, will get reset on next update from participant
+		u.lock.Unlock()
+	})
+}
+
+func (u *UpTrackManager) RemovePublishedTrack(track types.MediaTrack, isExpectedToResume bool, shouldClose bool) {
+	if shouldClose {
+		track.Close(isExpectedToResume)
+	} else {
+		track.ClearAllReceivers(isExpectedToResume)
+	}
+	u.lock.Lock()
+	delete(u.publishedTracks, track.ID())
+	u.lock.Unlock()
+}
+
+func (u *UpTrackManager) getPublishedTrackLocked(trackID livekit.TrackID) types.MediaTrack {
+	return u.publishedTracks[trackID]
+}
+
+func (u *UpTrackManager) parseSubscriptionPermissionsLocked(
+	subscriptionPermission *livekit.SubscriptionPermission,
+	resolver func(participantID livekit.ParticipantID) types.LocalParticipant,
+) error {
+	// every update overrides the existing
+
+	// all_participants takes precedence
+	if subscriptionPermission.AllParticipants {
+		// everything is allowed, nothing else to do
+		u.subscriberPermissions = nil
+		return nil
+	}
+
+	// per participant permissions
+	subscriberPermissions := make(map[livekit.ParticipantIdentity]*livekit.TrackPermission)
+	for _, trackPerms := range subscriptionPermission.TrackPermissions {
+		subscriberIdentity := livekit.ParticipantIdentity(trackPerms.ParticipantIdentity)
+		if subscriberIdentity == "" {
+			if trackPerms.ParticipantSid == "" {
+				return ErrSubscriptionPermissionNeedsId
+			}
+
+			sub := resolver(livekit.ParticipantID(trackPerms.ParticipantSid))
+			if sub == nil {
+				u.params.Logger.Warnw("could not find subscriber for permissions update", nil, "subscriberID", trackPerms.ParticipantSid)
+				continue
+			}
+
+			subscriberIdentity = sub.Identity()
+		} else {
+			if trackPerms.ParticipantSid != "" {
+				sub := resolver(livekit.ParticipantID(trackPerms.ParticipantSid))
+				if sub != nil && sub.Identity() != subscriberIdentity {
+					u.params.Logger.Errorw("participant identity mismatch", nil, "expected", subscriberIdentity, "got", sub.Identity())
+				}
+				if sub == nil {
+					u.params.Logger.Warnw("could not find subscriber for permissions update", nil, "subscriberID", trackPerms.ParticipantSid)
+				}
+			}
+		}
+
+		subscriberPermissions[subscriberIdentity] = trackPerms
+	}
+
+	u.subscriberPermissions = subscriberPermissions
+
+	return nil
+}
+
+func (u *UpTrackManager) hasPermissionLocked(trackID livekit.TrackID, subscriberIdentity livekit.ParticipantIdentity) bool {
+	if u.subscriberPermissions == nil {
+		return true
+	}
+
+	perms, ok := u.subscriberPermissions[subscriberIdentity]
+	if !ok {
+		return false
+	}
+
+	if perms.AllTracks {
+		return true
+	}
+
+	for _, sid := range perms.TrackSids {
+		if livekit.TrackID(sid) == trackID {
+			return true
+		}
+	}
+
+	return false
+}
+
+// returns a list of participants that are allowed to subscribe to the track. if nil is returned, it means everyone is
+// allowed to subscribe to this track
+func (u *UpTrackManager) getAllowedSubscribersLocked(trackID livekit.TrackID) []livekit.ParticipantIdentity {
+	if u.subscriberPermissions == nil {
+		return nil
+	}
+
+	allowed := make([]livekit.ParticipantIdentity, 0)
+	for subscriberIdentity, perms := range u.subscriberPermissions {
+		if perms.AllTracks {
+			allowed = append(allowed, subscriberIdentity)
+			continue
+		}
+
+		for _, sid := range perms.TrackSids {
+			if livekit.TrackID(sid) == trackID {
+				allowed = append(allowed, subscriberIdentity)
+				break
+			}
+		}
+	}
+
+	return allowed
+}
+
+func (u *UpTrackManager) maybeRevokeSubscriptions() {
+	u.lock.Lock()
+	defer u.lock.Unlock()
+
+	for trackID, track := range u.publishedTracks {
+		allowed := u.getAllowedSubscribersLocked(trackID)
+		if allowed == nil {
+			// no restrictions
+			continue
+		}
+
+		track.RevokeDisallowedSubscribers(allowed)
+	}
+}
+
+func (u *UpTrackManager) DebugInfo() map[string]interface{} {
+	info := map[string]interface{}{}
+	publishedTrackInfo := make(map[livekit.TrackID]interface{})
+
+	u.lock.RLock()
+	for trackID, track := range u.publishedTracks {
+		if mt, ok := track.(*MediaTrack); ok {
+			publishedTrackInfo[trackID] = mt.DebugInfo()
+		} else {
+			publishedTrackInfo[trackID] = map[string]interface{}{
+				"ID":       track.ID(),
+				"Kind":     track.Kind().String(),
+				"PubMuted": track.IsMuted(),
+			}
+		}
+	}
+	u.lock.RUnlock()
+
+	info["PublishedTracks"] = publishedTrackInfo
+
+	return info
+}
+
+func (u *UpTrackManager) GetAudioLevel() (level float64, active bool) {
+	level = 0
+	for _, pt := range u.GetPublishedTracks() {
+		if pt.Source() == livekit.TrackSource_MICROPHONE {
+			tl, ta := pt.GetAudioLevel()
+			if ta {
+				active = true
+				if tl > level {
+					level = tl
+				}
+			}
+		}
+	}
+	return
+}

livekit-server/pkg/rtc/uptrackmanager_test.go

@@ -0,0 +1,329 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package rtc
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/livekit/protocol/livekit"
+	"github.com/livekit/protocol/logger"
+	"github.com/livekit/protocol/utils"
+
+	"github.com/livekit/livekit-server/pkg/rtc/types"
+	"github.com/livekit/livekit-server/pkg/rtc/types/typesfakes"
+)
+
+var defaultUptrackManagerParams = UpTrackManagerParams{
+	Logger:           logger.GetLogger(),
+	VersionGenerator: utils.NewDefaultTimedVersionGenerator(),
+}
+
+func TestUpdateSubscriptionPermission(t *testing.T) {
+	t.Run("updates subscription permission", func(t *testing.T) {
+		um := NewUpTrackManager(defaultUptrackManagerParams)
+		vg := utils.NewDefaultTimedVersionGenerator()
+
+		tra := &typesfakes.FakeMediaTrack{}
+		tra.IDReturns("audio")
+		um.publishedTracks["audio"] = tra
+
+		trv := &typesfakes.FakeMediaTrack{}
+		trv.IDReturns("video")
+		um.publishedTracks["video"] = trv
+
+		// no restrictive subscription permission
+		subscriptionPermission := &livekit.SubscriptionPermission{
+			AllParticipants: true,
+		}
+		um.UpdateSubscriptionPermission(subscriptionPermission, vg.Next(), nil)
+		require.Nil(t, um.subscriberPermissions)
+
+		// nobody is allowed to subscribe
+		subscriptionPermission = &livekit.SubscriptionPermission{
+			TrackPermissions: []*livekit.TrackPermission{},
+		}
+		um.UpdateSubscriptionPermission(subscriptionPermission, vg.Next(), nil)
+		require.NotNil(t, um.subscriberPermissions)
+		require.Equal(t, 0, len(um.subscriberPermissions))
+
+		lp1 := &typesfakes.FakeLocalParticipant{}
+		lp1.IdentityReturns("p1")
+		lp2 := &typesfakes.FakeLocalParticipant{}
+		lp2.IdentityReturns("p2")
+
+		sidResolver := func(sid livekit.ParticipantID) types.LocalParticipant {
+			if sid == "p1" {
+				return lp1
+			}
+
+			if sid == "p2" {
+				return lp2
+			}
+
+			return nil
+		}
+
+		// allow all tracks for participants
+		perms1 := &livekit.TrackPermission{
+			ParticipantSid: "p1",
+			AllTracks:      true,
+		}
+		perms2 := &livekit.TrackPermission{
+			ParticipantSid: "p2",
+			AllTracks:      true,
+		}
+		subscriptionPermission = &livekit.SubscriptionPermission{
+			TrackPermissions: []*livekit.TrackPermission{
+				perms1,
+				perms2,
+			},
+		}
+		um.UpdateSubscriptionPermission(subscriptionPermission, vg.Next(), sidResolver)
+		require.Equal(t, 2, len(um.subscriberPermissions))
+		require.EqualValues(t, perms1, um.subscriberPermissions["p1"])
+		require.EqualValues(t, perms2, um.subscriberPermissions["p2"])
+
+		// allow all tracks for some and restrictive for others
+		perms1 = &livekit.TrackPermission{
+			ParticipantIdentity: "p1",
+			AllTracks:           true,
+		}
+		perms2 = &livekit.TrackPermission{
+			ParticipantIdentity: "p2",
+			TrackSids:           []string{"audio"},
+		}
+		perms3 := &livekit.TrackPermission{
+			ParticipantIdentity: "p3",
+			TrackSids:           []string{"video"},
+		}
+		subscriptionPermission = &livekit.SubscriptionPermission{
+			TrackPermissions: []*livekit.TrackPermission{
+				perms1,
+				perms2,
+				perms3,
+			},
+		}
+		um.UpdateSubscriptionPermission(subscriptionPermission, vg.Next(), nil)
+		require.Equal(t, 3, len(um.subscriberPermissions))
+		require.EqualValues(t, perms1, um.subscriberPermissions["p1"])
+		require.EqualValues(t, perms2, um.subscriberPermissions["p2"])
+		require.EqualValues(t, perms3, um.subscriberPermissions["p3"])
+	})
+
+	t.Run("updates subscription permission using both", func(t *testing.T) {
+		um := NewUpTrackManager(defaultUptrackManagerParams)
+		vg := utils.NewDefaultTimedVersionGenerator()
+
+		tra := &typesfakes.FakeMediaTrack{}
+		tra.IDReturns("audio")
+		um.publishedTracks["audio"] = tra
+
+		trv := &typesfakes.FakeMediaTrack{}
+		trv.IDReturns("video")
+		um.publishedTracks["video"] = trv
+
+		lp1 := &typesfakes.FakeLocalParticipant{}
+		lp1.IdentityReturns("p1")
+		lp2 := &typesfakes.FakeLocalParticipant{}
+		lp2.IdentityReturns("p2")
+
+		sidResolver := func(sid livekit.ParticipantID) types.LocalParticipant {
+			if sid == "p1" {
+				return lp1
+			}
+
+			if sid == "p2" {
+				return lp2
+			}
+
+			return nil
+		}
+
+		// allow all tracks for participants
+		perms1 := &livekit.TrackPermission{
+			ParticipantSid:      "p1",
+			ParticipantIdentity: "p1",
+			AllTracks:           true,
+		}
+		perms2 := &livekit.TrackPermission{
+			ParticipantSid:      "p2",
+			ParticipantIdentity: "p2",
+			AllTracks:           true,
+		}
+		subscriptionPermission := &livekit.SubscriptionPermission{
+			TrackPermissions: []*livekit.TrackPermission{
+				perms1,
+				perms2,
+			},
+		}
+		err := um.UpdateSubscriptionPermission(subscriptionPermission, vg.Next(), sidResolver)
+		require.NoError(t, err)
+		require.Equal(t, 2, len(um.subscriberPermissions))
+		require.EqualValues(t, perms1, um.subscriberPermissions["p1"])
+		require.EqualValues(t, perms2, um.subscriberPermissions["p2"])
+
+		// mismatched identities should fail a permission update
+		badSidResolver := func(sid livekit.ParticipantID) types.LocalParticipant {
+			if sid == "p1" {
+				return lp2
+			}
+
+			if sid == "p2" {
+				return lp1
+			}
+
+			return nil
+		}
+
+		err = um.UpdateSubscriptionPermission(subscriptionPermission, vg.Next(), badSidResolver)
+		require.NoError(t, err)
+		require.Equal(t, 2, len(um.subscriberPermissions))
+		require.EqualValues(t, perms1, um.subscriberPermissions["p1"])
+		require.EqualValues(t, perms2, um.subscriberPermissions["p2"])
+	})
+
+	t.Run("update versions", func(t *testing.T) {
+		um := NewUpTrackManager(defaultUptrackManagerParams)
+		vg := utils.NewDefaultTimedVersionGenerator()
+
+		v0, v1, v2 := vg.Next(), vg.Next(), vg.Next()
+
+		um.UpdateSubscriptionPermission(&livekit.SubscriptionPermission{}, v1, nil)
+		require.Equal(t, v1.Load(), um.subscriptionPermissionVersion.Load(), "first update should be applied")
+
+		um.UpdateSubscriptionPermission(&livekit.SubscriptionPermission{}, v2, nil)
+		require.Equal(t, v2.Load(), um.subscriptionPermissionVersion.Load(), "ordered updates should be applied")
+
+		um.UpdateSubscriptionPermission(&livekit.SubscriptionPermission{}, v0, nil)
+		require.Equal(t, v2.Load(), um.subscriptionPermissionVersion.Load(), "out of order updates should be ignored")
+
+		um.UpdateSubscriptionPermission(&livekit.SubscriptionPermission{}, utils.TimedVersion(0), nil)
+		require.True(t, um.subscriptionPermissionVersion.After(v2), "zero version in updates should use next local version")
+	})
+}
+
+func TestSubscriptionPermission(t *testing.T) {
+	t.Run("checks subscription permission", func(t *testing.T) {
+		um := NewUpTrackManager(defaultUptrackManagerParams)
+		vg := utils.NewDefaultTimedVersionGenerator()
+
+		tra := &typesfakes.FakeMediaTrack{}
+		tra.IDReturns("audio")
+		um.publishedTracks["audio"] = tra
+
+		trv := &typesfakes.FakeMediaTrack{}
+		trv.IDReturns("video")
+		um.publishedTracks["video"] = trv
+
+		// no restrictive permission
+		subscriptionPermission := &livekit.SubscriptionPermission{
+			AllParticipants: true,
+		}
+		um.UpdateSubscriptionPermission(subscriptionPermission, vg.Next(), nil)
+		require.True(t, um.hasPermissionLocked("audio", "p1"))
+		require.True(t, um.hasPermissionLocked("audio", "p2"))
+
+		// nobody is allowed to subscribe
+		subscriptionPermission = &livekit.SubscriptionPermission{
+			TrackPermissions: []*livekit.TrackPermission{},
+		}
+		um.UpdateSubscriptionPermission(subscriptionPermission, vg.Next(), nil)
+		require.False(t, um.hasPermissionLocked("audio", "p1"))
+		require.False(t, um.hasPermissionLocked("audio", "p2"))
+
+		// allow all tracks for participants
+		subscriptionPermission = &livekit.SubscriptionPermission{
+			TrackPermissions: []*livekit.TrackPermission{
+				{
+					ParticipantIdentity: "p1",
+					AllTracks:           true,
+				},
+				{
+					ParticipantIdentity: "p2",
+					AllTracks:           true,
+				},
+			},
+		}
+		um.UpdateSubscriptionPermission(subscriptionPermission, vg.Next(), nil)
+		require.True(t, um.hasPermissionLocked("audio", "p1"))
+		require.True(t, um.hasPermissionLocked("video", "p1"))
+		require.True(t, um.hasPermissionLocked("audio", "p2"))
+		require.True(t, um.hasPermissionLocked("video", "p2"))
+
+		// add a new track after permissions are set
+		trs := &typesfakes.FakeMediaTrack{}
+		trs.IDReturns("screen")
+		um.publishedTracks["screen"] = trs
+
+		require.True(t, um.hasPermissionLocked("audio", "p1"))
+		require.True(t, um.hasPermissionLocked("video", "p1"))
+		require.True(t, um.hasPermissionLocked("screen", "p1"))
+		require.True(t, um.hasPermissionLocked("audio", "p2"))
+		require.True(t, um.hasPermissionLocked("video", "p2"))
+		require.True(t, um.hasPermissionLocked("screen", "p2"))
+
+		// allow all tracks for some and restrictive for others
+		subscriptionPermission = &livekit.SubscriptionPermission{
+			TrackPermissions: []*livekit.TrackPermission{
+				{
+					ParticipantIdentity: "p1",
+					AllTracks:           true,
+				},
+				{
+					ParticipantIdentity: "p2",
+					TrackSids:           []string{"audio"},
+				},
+				{
+					ParticipantIdentity: "p3",
+					TrackSids:           []string{"video"},
+				},
+			},
+		}
+		um.UpdateSubscriptionPermission(subscriptionPermission, vg.Next(), nil)
+		require.True(t, um.hasPermissionLocked("audio", "p1"))
+		require.True(t, um.hasPermissionLocked("video", "p1"))
+		require.True(t, um.hasPermissionLocked("screen", "p1"))
+
+		require.True(t, um.hasPermissionLocked("audio", "p2"))
+		require.False(t, um.hasPermissionLocked("video", "p2"))
+		require.False(t, um.hasPermissionLocked("screen", "p2"))
+
+		require.False(t, um.hasPermissionLocked("audio", "p3"))
+		require.True(t, um.hasPermissionLocked("video", "p3"))
+		require.False(t, um.hasPermissionLocked("screen", "p3"))
+
+		// add a new track after restrictive permissions are set
+		trw := &typesfakes.FakeMediaTrack{}
+		trw.IDReturns("watch")
+		um.publishedTracks["watch"] = trw
+
+		require.True(t, um.hasPermissionLocked("audio", "p1"))
+		require.True(t, um.hasPermissionLocked("video", "p1"))
+		require.True(t, um.hasPermissionLocked("screen", "p1"))
+		require.True(t, um.hasPermissionLocked("watch", "p1"))
+
+		require.True(t, um.hasPermissionLocked("audio", "p2"))
+		require.False(t, um.hasPermissionLocked("video", "p2"))
+		require.False(t, um.hasPermissionLocked("screen", "p2"))
+		require.False(t, um.hasPermissionLocked("watch", "p2"))
+
+		require.False(t, um.hasPermissionLocked("audio", "p3"))
+		require.True(t, um.hasPermissionLocked("video", "p3"))
+		require.False(t, um.hasPermissionLocked("screen", "p3"))
+		require.False(t, um.hasPermissionLocked("watch", "p3"))
+	})
+}

livekit-server/pkg/rtc/user_packet_deduper.go

@@ -0,0 +1,66 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package rtc
+
+import (
+	"sync"
+
+	"github.com/google/uuid"
+	"github.com/livekit/protocol/livekit"
+)
+
+const (
+	maxSize = 100
+)
+
+type UserPacketDeduper struct {
+	lock sync.Mutex
+	seen map[uuid.UUID]uuid.UUID
+	head uuid.UUID
+	tail uuid.UUID
+}
+
+func NewUserPacketDeduper() *UserPacketDeduper {
+	return &UserPacketDeduper{
+		seen: make(map[uuid.UUID]uuid.UUID),
+	}
+}
+
+func (u *UserPacketDeduper) IsDuplicate(up *livekit.UserPacket) bool {
+	id, err := uuid.FromBytes(up.Nonce)
+	if err != nil {
+		return false
+	}
+
+	u.lock.Lock()
+	defer u.lock.Unlock()
+
+	if u.head == id {
+		return true
+	}
+	if _, ok := u.seen[id]; ok {
+		return true
+	}
+
+	u.seen[u.head] = id
+	u.head = id
+
+	if len(u.seen) == maxSize {
+		tail := u.tail
+		u.tail = u.seen[tail]
+		delete(u.seen, tail)
+	}
+	return false
+}

livekit-server/pkg/rtc/utils.go

@@ -0,0 +1,217 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package rtc
+
+import (
+	"encoding/json"
+	"errors"
+	"io"
+	"net"
+	"strings"
+
+	"github.com/pion/webrtc/v4"
+
+	"github.com/livekit/livekit-server/pkg/sfu/mime"
+	"github.com/livekit/protocol/livekit"
+	"github.com/livekit/protocol/logger"
+)
+
+const (
+	trackIdSeparator = "|"
+
+	cMinIPTruncateLen = 8
+)
+
+func UnpackStreamID(packed string) (participantID livekit.ParticipantID, trackID livekit.TrackID) {
+	parts := strings.Split(packed, trackIdSeparator)
+	if len(parts) > 1 {
+		return livekit.ParticipantID(parts[0]), livekit.TrackID(packed[len(parts[0])+1:])
+	}
+	return livekit.ParticipantID(packed), ""
+}
+
+func PackStreamID(participantID livekit.ParticipantID, trackID livekit.TrackID) string {
+	return string(participantID) + trackIdSeparator + string(trackID)
+}
+
+func PackSyncStreamID(participantID livekit.ParticipantID, stream string) string {
+	return string(participantID) + trackIdSeparator + stream
+}
+
+func StreamFromTrackSource(source livekit.TrackSource) string {
+	// group camera/mic, screenshare/audio together
+	switch source {
+	case livekit.TrackSource_SCREEN_SHARE:
+		return "screen"
+	case livekit.TrackSource_SCREEN_SHARE_AUDIO:
+		return "screen"
+	case livekit.TrackSource_CAMERA:
+		return "camera"
+	case livekit.TrackSource_MICROPHONE:
+		return "camera"
+	}
+	return "unknown"
+}
+
+func PackDataTrackLabel(participantID livekit.ParticipantID, trackID livekit.TrackID, label string) string {
+	return string(participantID) + trackIdSeparator + string(trackID) + trackIdSeparator + label
+}
+
+func UnpackDataTrackLabel(packed string) (participantID livekit.ParticipantID, trackID livekit.TrackID, label string) {
+	parts := strings.Split(packed, trackIdSeparator)
+	if len(parts) != 3 {
+		return "", livekit.TrackID(packed), ""
+	}
+	participantID = livekit.ParticipantID(parts[0])
+	trackID = livekit.TrackID(parts[1])
+	label = parts[2]
+	return
+}
+
+func ToProtoSessionDescription(sd webrtc.SessionDescription) *livekit.SessionDescription {
+	return &livekit.SessionDescription{
+		Type: sd.Type.String(),
+		Sdp:  sd.SDP,
+	}
+}
+
+func FromProtoSessionDescription(sd *livekit.SessionDescription) webrtc.SessionDescription {
+	var sdType webrtc.SDPType
+	switch sd.Type {
+	case webrtc.SDPTypeOffer.String():
+		sdType = webrtc.SDPTypeOffer
+	case webrtc.SDPTypeAnswer.String():
+		sdType = webrtc.SDPTypeAnswer
+	case webrtc.SDPTypePranswer.String():
+		sdType = webrtc.SDPTypePranswer
+	case webrtc.SDPTypeRollback.String():
+		sdType = webrtc.SDPTypeRollback
+	}
+	return webrtc.SessionDescription{
+		Type: sdType,
+		SDP:  sd.Sdp,
+	}
+}
+
+func ToProtoTrickle(candidateInit webrtc.ICECandidateInit, target livekit.SignalTarget, final bool) *livekit.TrickleRequest {
+	data, _ := json.Marshal(candidateInit)
+	return &livekit.TrickleRequest{
+		CandidateInit: string(data),
+		Target:        target,
+		Final:         final,
+	}
+}
+
+func FromProtoTrickle(trickle *livekit.TrickleRequest) (webrtc.ICECandidateInit, error) {
+	ci := webrtc.ICECandidateInit{}
+	err := json.Unmarshal([]byte(trickle.CandidateInit), &ci)
+	if err != nil {
+		return webrtc.ICECandidateInit{}, err
+	}
+	return ci, nil
+}
+
+func ToProtoTrackKind(kind webrtc.RTPCodecType) livekit.TrackType {
+	switch kind {
+	case webrtc.RTPCodecTypeVideo:
+		return livekit.TrackType_VIDEO
+	case webrtc.RTPCodecTypeAudio:
+		return livekit.TrackType_AUDIO
+	}
+	panic("unsupported track direction")
+}
+
+func IsEOF(err error) bool {
+	return err == io.ErrClosedPipe || err == io.EOF
+}
+
+func Recover(l logger.Logger) any {
+	if l == nil {
+		l = logger.GetLogger()
+	}
+	r := recover()
+	if r != nil {
+		var err error
+		switch e := r.(type) {
+		case string:
+			err = errors.New(e)
+		case error:
+			err = e
+		default:
+			err = errors.New("unknown panic")
+		}
+		l.Errorw("recovered panic", err, "panic", r)
+	}
+
+	return r
+}
+
+// logger helpers
+func LoggerWithParticipant(l logger.Logger, identity livekit.ParticipantIdentity, sid livekit.ParticipantID, isRemote bool) logger.Logger {
+	values := make([]interface{}, 0, 4)
+	if identity != "" {
+		values = append(values, "participant", identity)
+	}
+	if sid != "" {
+		values = append(values, "pID", sid)
+	}
+	values = append(values, "remote", isRemote)
+	// enable sampling per participant
+	return l.WithValues(values...)
+}
+
+func LoggerWithRoom(l logger.Logger, name livekit.RoomName, roomID livekit.RoomID) logger.Logger {
+	values := make([]interface{}, 0, 2)
+	if name != "" {
+		values = append(values, "room", name)
+	}
+	if roomID != "" {
+		values = append(values, "roomID", roomID)
+	}
+	// also sample for the room
+	return l.WithItemSampler().WithValues(values...)
+}
+
+func LoggerWithTrack(l logger.Logger, trackID livekit.TrackID, isRelayed bool) logger.Logger {
+	// sampling not required because caller already passing in participant's logger
+	if trackID != "" {
+		return l.WithValues("trackID", trackID, "relayed", isRelayed)
+	}
+	return l
+}
+
+func LoggerWithPCTarget(l logger.Logger, target livekit.SignalTarget) logger.Logger {
+	return l.WithValues("transport", target)
+}
+
+func LoggerWithCodecMime(l logger.Logger, mimeType mime.MimeType) logger.Logger {
+	if mimeType != mime.MimeTypeUnknown {
+		return l.WithValues("mime", mimeType.String())
+	}
+	return l
+}
+
+func MaybeTruncateIP(addr string) string {
+	ipAddr := net.ParseIP(addr)
+	if ipAddr == nil {
+		return ""
+	}
+
+	if ipAddr.IsPrivate() || len(addr) <= cMinIPTruncateLen {
+		return addr
+	}
+
+	return addr[:len(addr)-3] + "..."
+}

livekit-server/pkg/rtc/utils_test.go

@@ -0,0 +1,45 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package rtc
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/livekit/protocol/livekit"
+)
+
+func TestPackStreamId(t *testing.T) {
+	packed := "PA_123abc|uuid-id"
+	pID, trackID := UnpackStreamID(packed)
+	require.Equal(t, livekit.ParticipantID("PA_123abc"), pID)
+	require.Equal(t, livekit.TrackID("uuid-id"), trackID)
+
+	require.Equal(t, packed, PackStreamID(pID, trackID))
+}
+
+func TestPackDataTrackLabel(t *testing.T) {
+	pID := livekit.ParticipantID("PA_123abc")
+	trackID := livekit.TrackID("TR_b3da25")
+	label := "trackLabel"
+	packed := "PA_123abc|TR_b3da25|trackLabel"
+	require.Equal(t, packed, PackDataTrackLabel(pID, trackID, label))
+
+	p, tr, l := UnpackDataTrackLabel(packed)
+	require.Equal(t, pID, p)
+	require.Equal(t, trackID, tr)
+	require.Equal(t, label, l)
+}

livekit-server/pkg/rtc/wrappedreceiver.go

@@ -0,0 +1,548 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package rtc
+
+import (
+	"errors"
+	"sync"
+
+	"github.com/pion/webrtc/v4"
+	"go.uber.org/atomic"
+	"golang.org/x/exp/maps"
+	"golang.org/x/exp/slices"
+
+	"github.com/livekit/protocol/livekit"
+	"github.com/livekit/protocol/logger"
+
+	"github.com/livekit/livekit-server/pkg/sfu"
+	"github.com/livekit/livekit-server/pkg/sfu/mime"
+)
+
+// wrapper around WebRTC receiver, overriding its ID
+
+type WrappedReceiverParams struct {
+	Receivers      []*simulcastReceiver
+	TrackID        livekit.TrackID
+	StreamId       string
+	UpstreamCodecs []webrtc.RTPCodecParameters
+	Logger         logger.Logger
+	DisableRed     bool
+}
+
+type WrappedReceiver struct {
+	lock sync.Mutex
+
+	sfu.TrackReceiver
+	params           WrappedReceiverParams
+	receivers        []sfu.TrackReceiver
+	codecs           []webrtc.RTPCodecParameters
+	onReadyCallbacks []func()
+}
+
+func NewWrappedReceiver(params WrappedReceiverParams) *WrappedReceiver {
+	sfuReceivers := make([]sfu.TrackReceiver, 0, len(params.Receivers))
+	for _, r := range params.Receivers {
+		sfuReceivers = append(sfuReceivers, r)
+	}
+
+	codecs := params.UpstreamCodecs
+	if len(codecs) == 1 {
+		normalizedMimeType := mime.NormalizeMimeType(codecs[0].MimeType)
+		if normalizedMimeType == mime.MimeTypeRED {
+			// if upstream is opus/red, then add opus to match clients that don't support red
+			codecs = append(codecs, webrtc.RTPCodecParameters{
+				RTPCodecCapability: OpusCodecCapability,
+				PayloadType:        111,
+			})
+		} else if !params.DisableRed && normalizedMimeType == mime.MimeTypeOpus {
+			// if upstream is opus only and red enabled, add red to match clients that support red
+			codecs = append(codecs, webrtc.RTPCodecParameters{
+				RTPCodecCapability: RedCodecCapability,
+				PayloadType:        63,
+			})
+			// prefer red codec
+			codecs[0], codecs[1] = codecs[1], codecs[0]
+		}
+	}
+
+	return &WrappedReceiver{
+		params:    params,
+		receivers: sfuReceivers,
+		codecs:    codecs,
+	}
+}
+
+func (r *WrappedReceiver) TrackID() livekit.TrackID {
+	return r.params.TrackID
+}
+
+func (r *WrappedReceiver) StreamID() string {
+	return r.params.StreamId
+}
+
+// DetermineReceiver determines the receiver of negotiated codec and return ready state of the receiver
+func (r *WrappedReceiver) DetermineReceiver(codec webrtc.RTPCodecCapability) bool {
+	r.lock.Lock()
+
+	codecMimeType := mime.NormalizeMimeType(codec.MimeType)
+	var trackReceiver sfu.TrackReceiver
+	for _, receiver := range r.receivers {
+		receiverMimeType := receiver.Mime()
+		if receiverMimeType == codecMimeType {
+			trackReceiver = receiver
+			break
+		} else if receiverMimeType == mime.MimeTypeRED && codecMimeType == mime.MimeTypeOpus {
+			// audio opus/red can match opus only
+			trackReceiver = receiver.GetPrimaryReceiverForRed()
+			break
+		} else if receiverMimeType == mime.MimeTypeOpus && codecMimeType == mime.MimeTypeRED {
+			trackReceiver = receiver.GetRedReceiver()
+			break
+		}
+	}
+	if trackReceiver == nil {
+		r.params.Logger.Errorw("can't determine receiver for codec", nil, "codec", codec.MimeType)
+		if len(r.receivers) > 0 {
+			trackReceiver = r.receivers[0]
+		}
+	}
+	r.TrackReceiver = trackReceiver
+
+	var onReadyCallbacks []func()
+	if trackReceiver != nil {
+		onReadyCallbacks = r.onReadyCallbacks
+		r.onReadyCallbacks = nil
+	}
+	r.lock.Unlock()
+
+	if trackReceiver != nil {
+		for _, f := range onReadyCallbacks {
+			trackReceiver.AddOnReady(f)
+		}
+
+		if s, ok := trackReceiver.(*simulcastReceiver); ok {
+			if d, ok := s.TrackReceiver.(*DummyReceiver); ok {
+				return d.IsReady()
+			}
+		}
+		return true
+	}
+	return false
+}
+
+func (r *WrappedReceiver) Codecs() []webrtc.RTPCodecParameters {
+	return slices.Clone(r.codecs)
+}
+
+func (r *WrappedReceiver) DeleteDownTrack(participantID livekit.ParticipantID) {
+	r.lock.Lock()
+	trackReceiver := r.TrackReceiver
+	r.lock.Unlock()
+
+	if trackReceiver != nil {
+		trackReceiver.DeleteDownTrack(participantID)
+	}
+}
+
+func (r *WrappedReceiver) AddOnReady(f func()) {
+	r.lock.Lock()
+	trackReceiver := r.TrackReceiver
+	if trackReceiver == nil {
+		r.onReadyCallbacks = append(r.onReadyCallbacks, f)
+		r.lock.Unlock()
+		return
+	}
+	r.lock.Unlock()
+
+	trackReceiver.AddOnReady(f)
+}
+
+// --------------------------------------------
+
+type DummyReceiver struct {
+	receiver         atomic.Value
+	trackID          livekit.TrackID
+	streamId         string
+	codec            webrtc.RTPCodecParameters
+	headerExtensions []webrtc.RTPHeaderExtensionParameter
+
+	downTrackLock      sync.Mutex
+	downTracks         map[livekit.ParticipantID]sfu.TrackSender
+	onReadyCallbacks   []func()
+	onCodecStateChange []func(webrtc.RTPCodecParameters, sfu.ReceiverCodecState)
+
+	settingsLock          sync.Mutex
+	maxExpectedLayerValid bool
+	maxExpectedLayer      int32
+	pausedValid           bool
+	paused                bool
+
+	redReceiver, primaryReceiver *DummyRedReceiver
+}
+
+func NewDummyReceiver(trackID livekit.TrackID, streamId string, codec webrtc.RTPCodecParameters, headerExtensions []webrtc.RTPHeaderExtensionParameter) *DummyReceiver {
+	return &DummyReceiver{
+		trackID:          trackID,
+		streamId:         streamId,
+		codec:            codec,
+		headerExtensions: headerExtensions,
+		downTracks:       make(map[livekit.ParticipantID]sfu.TrackSender),
+	}
+}
+
+func (d *DummyReceiver) Receiver() sfu.TrackReceiver {
+	r, _ := d.receiver.Load().(sfu.TrackReceiver)
+	return r
+}
+
+func (d *DummyReceiver) Upgrade(receiver sfu.TrackReceiver) {
+	if !d.receiver.CompareAndSwap(nil, receiver) {
+		return
+	}
+
+	d.downTrackLock.Lock()
+	for _, t := range d.downTracks {
+		receiver.AddDownTrack(t)
+	}
+	d.downTracks = make(map[livekit.ParticipantID]sfu.TrackSender)
+	onReadyCallbacks := d.onReadyCallbacks
+	d.onReadyCallbacks = nil
+	codecChange := d.onCodecStateChange
+	d.onCodecStateChange = nil
+	d.downTrackLock.Unlock()
+
+	for _, f := range onReadyCallbacks {
+		receiver.AddOnReady(f)
+	}
+
+	for _, f := range codecChange {
+		receiver.AddOnCodecStateChange(f)
+	}
+
+	d.settingsLock.Lock()
+	if d.maxExpectedLayerValid {
+		receiver.SetMaxExpectedSpatialLayer(d.maxExpectedLayer)
+	}
+	d.maxExpectedLayerValid = false
+
+	if d.pausedValid {
+		receiver.SetUpTrackPaused(d.paused)
+	}
+	d.pausedValid = false
+
+	if d.primaryReceiver != nil {
+		d.primaryReceiver.upgrade(receiver)
+	}
+	if d.redReceiver != nil {
+		d.redReceiver.upgrade(receiver)
+	}
+	d.settingsLock.Unlock()
+}
+
+func (d *DummyReceiver) TrackID() livekit.TrackID {
+	return d.trackID
+}
+
+func (d *DummyReceiver) StreamID() string {
+	return d.streamId
+}
+
+func (d *DummyReceiver) Codec() webrtc.RTPCodecParameters {
+	if r, ok := d.receiver.Load().(sfu.TrackReceiver); ok {
+		return r.Codec()
+	}
+	return d.codec
+}
+
+func (d *DummyReceiver) Mime() mime.MimeType {
+	if r, ok := d.receiver.Load().(sfu.TrackReceiver); ok {
+		return r.Mime()
+	}
+	return mime.NormalizeMimeType(d.codec.MimeType)
+}
+
+func (d *DummyReceiver) HeaderExtensions() []webrtc.RTPHeaderExtensionParameter {
+	if r, ok := d.receiver.Load().(sfu.TrackReceiver); ok {
+		return r.HeaderExtensions()
+	}
+	return d.headerExtensions
+}
+
+func (d *DummyReceiver) ReadRTP(buf []byte, layer uint8, esn uint64) (int, error) {
+	if r, ok := d.receiver.Load().(sfu.TrackReceiver); ok {
+		return r.ReadRTP(buf, layer, esn)
+	}
+	return 0, errors.New("no receiver")
+}
+
+func (d *DummyReceiver) GetLayeredBitrate() ([]int32, sfu.Bitrates) {
+	if r, ok := d.receiver.Load().(sfu.TrackReceiver); ok {
+		return r.GetLayeredBitrate()
+	}
+	return nil, sfu.Bitrates{}
+}
+
+func (d *DummyReceiver) GetAudioLevel() (float64, bool) {
+	if r, ok := d.receiver.Load().(sfu.TrackReceiver); ok {
+		return r.GetAudioLevel()
+	}
+	return 0, false
+}
+
+func (d *DummyReceiver) SendPLI(layer int32, force bool) {
+	if r, ok := d.receiver.Load().(sfu.TrackReceiver); ok {
+		r.SendPLI(layer, force)
+	}
+}
+
+func (d *DummyReceiver) SetUpTrackPaused(paused bool) {
+	d.settingsLock.Lock()
+	defer d.settingsLock.Unlock()
+	if r, ok := d.receiver.Load().(sfu.TrackReceiver); ok {
+		d.pausedValid = false
+		r.SetUpTrackPaused(paused)
+	} else {
+		d.pausedValid = true
+		d.paused = paused
+	}
+}
+
+func (d *DummyReceiver) SetMaxExpectedSpatialLayer(layer int32) {
+	d.settingsLock.Lock()
+	defer d.settingsLock.Unlock()
+	if r, ok := d.receiver.Load().(sfu.TrackReceiver); ok {
+		d.maxExpectedLayerValid = false
+		r.SetMaxExpectedSpatialLayer(layer)
+	} else {
+		d.maxExpectedLayerValid = true
+		d.maxExpectedLayer = layer
+	}
+}
+
+func (d *DummyReceiver) AddDownTrack(track sfu.TrackSender) error {
+	d.downTrackLock.Lock()
+	defer d.downTrackLock.Unlock()
+
+	if r, ok := d.receiver.Load().(sfu.TrackReceiver); ok {
+		r.AddDownTrack(track)
+	} else {
+		d.downTracks[track.SubscriberID()] = track
+	}
+	return nil
+}
+
+func (d *DummyReceiver) DeleteDownTrack(subscriberID livekit.ParticipantID) {
+	d.downTrackLock.Lock()
+	defer d.downTrackLock.Unlock()
+
+	if r, ok := d.receiver.Load().(sfu.TrackReceiver); ok {
+		r.DeleteDownTrack(subscriberID)
+	} else {
+		delete(d.downTracks, subscriberID)
+	}
+}
+
+func (d *DummyReceiver) GetDownTracks() []sfu.TrackSender {
+	d.downTrackLock.Lock()
+	defer d.downTrackLock.Unlock()
+
+	if r, ok := d.receiver.Load().(sfu.TrackReceiver); ok {
+		return r.GetDownTracks()
+	}
+	return maps.Values(d.downTracks)
+}
+
+func (d *DummyReceiver) DebugInfo() map[string]interface{} {
+	if r, ok := d.receiver.Load().(sfu.TrackReceiver); ok {
+		return r.DebugInfo()
+	}
+	return nil
+}
+
+func (d *DummyReceiver) GetTemporalLayerFpsForSpatial(spatial int32) []float32 {
+	if r, ok := d.receiver.Load().(sfu.TrackReceiver); ok {
+		return r.GetTemporalLayerFpsForSpatial(spatial)
+	}
+	return nil
+}
+
+func (d *DummyReceiver) TrackInfo() *livekit.TrackInfo {
+	if r, ok := d.receiver.Load().(sfu.TrackReceiver); ok {
+		return r.TrackInfo()
+	}
+	return nil
+}
+
+func (d *DummyReceiver) UpdateTrackInfo(ti *livekit.TrackInfo) {
+	if r, ok := d.receiver.Load().(sfu.TrackReceiver); ok {
+		r.UpdateTrackInfo(ti)
+	}
+}
+
+func (d *DummyReceiver) IsClosed() bool {
+	if r, ok := d.receiver.Load().(sfu.TrackReceiver); ok {
+		return r.IsClosed()
+	}
+	return false
+}
+
+func (d *DummyReceiver) GetPrimaryReceiverForRed() sfu.TrackReceiver {
+	d.settingsLock.Lock()
+	defer d.settingsLock.Unlock()
+
+	if d.primaryReceiver == nil {
+		d.primaryReceiver = NewDummyRedReceiver(d, false)
+		if r, ok := d.receiver.Load().(sfu.TrackReceiver); ok {
+			d.primaryReceiver.upgrade(r)
+		}
+	}
+	return d.primaryReceiver
+}
+
+func (d *DummyReceiver) GetRedReceiver() sfu.TrackReceiver {
+	d.settingsLock.Lock()
+	defer d.settingsLock.Unlock()
+	if d.redReceiver == nil {
+		d.redReceiver = NewDummyRedReceiver(d, true)
+		if r, ok := d.receiver.Load().(sfu.TrackReceiver); ok {
+			d.redReceiver.upgrade(r)
+		}
+	}
+	return d.redReceiver
+}
+
+func (d *DummyReceiver) GetTrackStats() *livekit.RTPStats {
+	if r, ok := d.receiver.Load().(sfu.TrackReceiver); ok {
+		return r.GetTrackStats()
+	}
+	return nil
+}
+
+func (d *DummyReceiver) AddOnReady(f func()) {
+	var receiver sfu.TrackReceiver
+	d.downTrackLock.Lock()
+	if r, ok := d.receiver.Load().(sfu.TrackReceiver); ok {
+		receiver = r
+	} else {
+		d.onReadyCallbacks = append(d.onReadyCallbacks, f)
+	}
+	d.downTrackLock.Unlock()
+	if receiver != nil {
+		receiver.AddOnReady(f)
+	}
+}
+
+func (d *DummyReceiver) IsReady() bool {
+	return d.receiver.Load() != nil
+}
+
+func (d *DummyReceiver) AddOnCodecStateChange(f func(codec webrtc.RTPCodecParameters, state sfu.ReceiverCodecState)) {
+	var receiver sfu.TrackReceiver
+	d.downTrackLock.Lock()
+	if r, ok := d.receiver.Load().(sfu.TrackReceiver); ok {
+		receiver = r
+	} else {
+		d.onCodecStateChange = append(d.onCodecStateChange, f)
+	}
+	d.downTrackLock.Unlock()
+	if receiver != nil {
+		receiver.AddOnCodecStateChange(f)
+	}
+}
+
+func (d *DummyReceiver) CodecState() sfu.ReceiverCodecState {
+	if r, ok := d.receiver.Load().(sfu.TrackReceiver); ok {
+		return r.CodecState()
+	}
+	return sfu.ReceiverCodecStateNormal
+}
+
+// --------------------------------------------
+
+type DummyRedReceiver struct {
+	*DummyReceiver
+	redReceiver atomic.Value // sfu.TrackReceiver
+	// indicates this receiver is for RED encoding receiver of primary codec OR
+	// primary decoding receiver of RED codec
+	isRedEncoding bool
+
+	downTrackLock sync.Mutex
+	downTracks    map[livekit.ParticipantID]sfu.TrackSender
+}
+
+func NewDummyRedReceiver(d *DummyReceiver, isRedEncoding bool) *DummyRedReceiver {
+	return &DummyRedReceiver{
+		DummyReceiver: d,
+		isRedEncoding: isRedEncoding,
+		downTracks:    make(map[livekit.ParticipantID]sfu.TrackSender),
+	}
+}
+
+func (d *DummyRedReceiver) AddDownTrack(track sfu.TrackSender) error {
+	d.downTrackLock.Lock()
+	defer d.downTrackLock.Unlock()
+
+	if r, ok := d.redReceiver.Load().(sfu.TrackReceiver); ok {
+		r.AddDownTrack(track)
+	} else {
+		d.downTracks[track.SubscriberID()] = track
+	}
+	return nil
+}
+
+func (d *DummyRedReceiver) DeleteDownTrack(subscriberID livekit.ParticipantID) {
+	d.downTrackLock.Lock()
+	defer d.downTrackLock.Unlock()
+
+	if r, ok := d.redReceiver.Load().(sfu.TrackReceiver); ok {
+		r.DeleteDownTrack(subscriberID)
+	} else {
+		delete(d.downTracks, subscriberID)
+	}
+}
+
+func (d *DummyRedReceiver) GetDownTracks() []sfu.TrackSender {
+	d.downTrackLock.Lock()
+	defer d.downTrackLock.Unlock()
+
+	if r, ok := d.redReceiver.Load().(sfu.TrackReceiver); ok {
+		return r.GetDownTracks()
+	}
+	return maps.Values(d.downTracks)
+}
+
+func (d *DummyRedReceiver) ReadRTP(buf []byte, layer uint8, esn uint64) (int, error) {
+	if r, ok := d.redReceiver.Load().(sfu.TrackReceiver); ok {
+		return r.ReadRTP(buf, layer, esn)
+	}
+	return 0, errors.New("no receiver")
+}
+
+func (d *DummyRedReceiver) upgrade(receiver sfu.TrackReceiver) {
+	var redReceiver sfu.TrackReceiver
+	if d.isRedEncoding {
+		redReceiver = receiver.GetRedReceiver()
+	} else {
+		redReceiver = receiver.GetPrimaryReceiverForRed()
+	}
+	d.redReceiver.Store(redReceiver)
+
+	d.downTrackLock.Lock()
+	for _, t := range d.downTracks {
+		redReceiver.AddDownTrack(t)
+	}
+	d.downTracks = make(map[livekit.ParticipantID]sfu.TrackSender)
+	d.downTrackLock.Unlock()
+}