Merge commit '6bd7fac875d9e9009915053d8a590abb372c5679' into feature/livekit-upgrade-1.13.1

livekit-server/pkg/sfu/audio/audiolevel.go

@@ -51,12 +51,15 @@ var (
 // --------------------------------------
 
 type AudioLevelParams struct {
-	Config AudioLevelConfig
+	ClockRate uint32
 }
 
 // keeps track of audio level for a participant
 type AudioLevel struct {
 	params AudioLevelParams
+
+	config AudioLevelConfig
+
 	// min duration within an observe duration window to be considered active
 	minActiveDuration uint32
 	smoothFactor      float64
@@ -69,23 +72,32 @@ type AudioLevel struct {
 	activeDuration       uint32 // ms
 	observedDuration     uint32 // ms
 	lastObservedAt       int64
+
+	highestRTPTimestamp            uint32
+	highestRTPTimestampInitialized bool
 }
 
 func NewAudioLevel(params AudioLevelParams) *AudioLevel {
-	l := &AudioLevel{
+	return &AudioLevel{
 		params:               params,
-		minActiveDuration:    uint32(params.Config.MinPercentile) * params.Config.UpdateInterval / 100,
 		smoothFactor:         1,
-		activeThreshold:      ConvertAudioLevel(float64(params.Config.ActiveLevel)),
 		loudestObservedLevel: silentAudioLevel,
 	}
+}
 
-	if l.params.Config.SmoothIntervals > 0 {
+func (l *AudioLevel) SetConfig(config AudioLevelConfig) {
+	l.lock.Lock()
+	defer l.lock.Unlock()
+
+	l.config = config
+	l.minActiveDuration = uint32(l.config.MinPercentile) * l.config.UpdateInterval / 100
+	if l.config.SmoothIntervals > 0 {
 		// exponential moving average (EMA), same center of mass with simple moving average (SMA)
-		l.smoothFactor = float64(2) / (float64(l.params.Config.SmoothIntervals + 1))
+		l.smoothFactor = float64(2) / (float64(l.config.SmoothIntervals + 1))
+	} else {
+		l.smoothFactor = 1
 	}
-
-	return l
+	l.activeThreshold = ConvertAudioLevel(float64(l.config.ActiveLevel))
 }
 
 // Observes a new frame
@@ -93,18 +105,26 @@ func (l *AudioLevel) Observe(level uint8, durationMs uint32, arrivalTime int64)
 	l.lock.Lock()
 	defer l.lock.Unlock()
 
+	l.observeLocked(level, durationMs, arrivalTime)
+}
+
+func (l *AudioLevel) observeLocked(level uint8, durationMs uint32, arrivalTime int64) {
+	if l.config.UpdateInterval == 0 {
+		return
+	}
+
 	l.lastObservedAt = arrivalTime
 
 	l.observedDuration += durationMs
 
-	if level <= l.params.Config.ActiveLevel {
+	if level <= l.config.ActiveLevel {
 		l.activeDuration += durationMs
 		if l.loudestObservedLevel > level {
 			l.loudestObservedLevel = level
 		}
 	}
 
-	if l.observedDuration >= l.params.Config.UpdateInterval {
+	if l.observedDuration >= l.config.UpdateInterval {
 		smoothedLevel := float64(0.0)
 		// compute and reset
 		if l.activeDuration >= l.minActiveDuration {
@@ -112,7 +132,7 @@ func (l *AudioLevel) Observe(level uint8, durationMs uint32, arrivalTime int64)
 			// Weight will be 0 if active the entire duration
 			// > 0 if active for longer than observe duration
 			// < 0 if active for less than observe duration
-			activityWeight := 20 * math.Log10(float64(l.activeDuration)/float64(l.params.Config.UpdateInterval))
+			activityWeight := 20 * math.Log10(float64(l.activeDuration)/float64(l.config.UpdateInterval))
 			adjustedLevel := float64(l.loudestObservedLevel) - activityWeight
 			linearLevel := ConvertAudioLevel(adjustedLevel)
 
@@ -123,18 +143,39 @@ func (l *AudioLevel) Observe(level uint8, durationMs uint32, arrivalTime int64)
 	}
 }
 
+func (l *AudioLevel) ObserveWithRTPTimestamp(level uint8, ts uint32, arrivalTime int64) {
+	l.lock.Lock()
+	defer l.lock.Unlock()
+
+	if !l.highestRTPTimestampInitialized {
+		l.highestRTPTimestampInitialized = true
+		l.highestRTPTimestamp = ts
+	}
+
+	if (ts - l.highestRTPTimestamp) < (1 << 31) {
+		durationMs := (ts - l.highestRTPTimestamp) * 1e3 / l.params.ClockRate
+		l.observeLocked(level, durationMs, arrivalTime)
+
+		l.highestRTPTimestamp = ts
+	}
+}
+
 // returns current smoothed audio level
 func (l *AudioLevel) GetLevel(now int64) (float64, bool) {
 	l.lock.Lock()
 	defer l.lock.Unlock()
 
+	if l.config.UpdateInterval == 0 {
+		return 0.0, false
+	}
+
 	l.resetIfStaleLocked(now)
 
 	return l.smoothedLevel, l.smoothedLevel >= l.activeThreshold
 }
 
 func (l *AudioLevel) resetIfStaleLocked(arrivalTime int64) {
-	if (arrivalTime-l.lastObservedAt)/1e6 < int64(2*l.params.Config.UpdateInterval) {
+	if (arrivalTime-l.lastObservedAt)/1e6 < int64(2*l.config.UpdateInterval) {
 		return
 	}
 

livekit-server/pkg/sfu/audio/audiolevel_test.go

@@ -15,6 +15,7 @@
 package audio
 
 import (
+	"math/rand"
 	"testing"
 	"time"
 
@@ -104,20 +105,54 @@ func TestAudioLevel(t *testing.T) {
 		require.Equal(t, float64(0.0), level)
 		require.False(t, noisy)
 	})
-}
 
-func createAudioLevel(activeLevel uint8, minPercentile uint8, observeDuration uint32) *AudioLevel {
-	return NewAudioLevel(AudioLevelParams{
-		Config: AudioLevelConfig{
-			ActiveLevel:    activeLevel,
-			MinPercentile:  minPercentile,
-			UpdateInterval: observeDuration,
-		},
+	t.Run("not noisy when samples are stale - with RTP timestamp", func(t *testing.T) {
+		clock := time.Now()
+		a := createAudioLevel(defaultActiveLevel, defaultPercentile, defaultObserveDuration)
+
+		observeSamplesWithRTPTimestamp(a, 25, 100, clock)
+		clock = clock.Add(100 * 20 * time.Millisecond)
+		level, noisy := a.GetLevel(clock.UnixNano())
+		require.True(t, noisy)
+		require.Greater(t, level, ConvertAudioLevel(float64(defaultActiveLevel)))
+		require.Less(t, level, ConvertAudioLevel(float64(20)))
+
+		// let enough time pass to make the samples stale
+		clock = clock.Add(1500 * time.Millisecond)
+		level, noisy = a.GetLevel(clock.UnixNano())
+		require.Equal(t, float64(0.0), level)
+		require.False(t, noisy)
 	})
 }
 
+func createAudioLevel(activeLevel uint8, minPercentile uint8, observeDuration uint32) *AudioLevel {
+	al := NewAudioLevel(AudioLevelParams{
+		ClockRate: 48000,
+	})
+	al.SetConfig(AudioLevelConfig{
+		ActiveLevel:    activeLevel,
+		MinPercentile:  minPercentile,
+		UpdateInterval: observeDuration,
+	})
+	return al
+}
+
 func observeSamples(a *AudioLevel, level uint8, count int, baseTime time.Time) {
-	for i := 0; i < count; i++ {
-		a.Observe(level, 20, baseTime.Add(+time.Duration(i*20)*time.Millisecond).UnixNano())
+	for i := range count {
+		a.Observe(level, 20, baseTime.Add(time.Duration(i*20)*time.Millisecond).UnixNano())
+	}
+}
+
+func observeSamplesWithRTPTimestamp(a *AudioLevel, level uint8, count int, baseTime time.Time) {
+	sampleTS := uint32(rand.Intn(1 << 20))
+	sampleTime := baseTime
+	for i := range count {
+		if (i % 5) == 0 {
+			// out-of-order sample
+			a.ObserveWithRTPTimestamp(level, sampleTS-1920, sampleTime.UnixNano())
+		}
+		a.ObserveWithRTPTimestamp(level, sampleTS, sampleTime.UnixNano())
+		sampleTS += 960 // 20 ms at 48 kHz
+		sampleTime = sampleTime.Add(20 * time.Millisecond)
 	}
 }

livekit-server/pkg/sfu/buffer/buffer_test.go

@@ -15,6 +15,7 @@
 package buffer
 
 import (
+	"fmt"
 	"math"
 	"sync"
 	"testing"
@@ -82,7 +83,7 @@ func TestNack(t *testing.T) {
 		}, vp8Codec.RTPCodecCapability, 0)
 		rtt := uint32(20)
 		buff.nacker.SetRTT(rtt)
-		for i := 0; i < 15; i++ {
+		for i := range 15 {
 			if i == 1 {
 				continue
 			}
@@ -147,7 +148,7 @@ func TestNack(t *testing.T) {
 		}, vp8Codec.RTPCodecCapability, 0)
 		rtt := uint32(30)
 		buff.nacker.SetRTT(rtt)
-		for i := 0; i < 15; i++ {
+		for i := range 15 {
 			if i > 0 && i < 5 {
 				continue
 			}
@@ -265,7 +266,7 @@ func TestFractionLostReport(t *testing.T) {
 		HeaderExtensions: nil,
 		Codecs:           []webrtc.RTPCodecParameters{opusCodec},
 	}, opusCodec.RTPCodecCapability, 0)
-	for i := 0; i < 15; i++ {
+	for i := range 15 {
 		pkt := rtp.Packet{
 			Header: rtp.Header{
 				Version:        2,
@@ -303,7 +304,7 @@ func TestFractionLostReport(t *testing.T) {
 		HeaderExtensions: nil,
 		Codecs:           []webrtc.RTPCodecParameters{opusCodec},
 	}, opusCodec.RTPCodecCapability, 0)
-	for i := 0; i < 15; i++ {
+	for i := range 15 {
 		pkt := rtp.Packet{
 			Header: rtp.Header{
 				Version:        2,
@@ -337,6 +338,15 @@ func TestCodecChange(t *testing.T) {
 			t.Fatalf("codec change not consumed")
 		}
 	})
+	buff.OnStreamRestart(func(reason string) {
+		require.Equal(t, "codec-change", reason)
+
+		// read once to clear pending restart
+		var buf [1500]byte
+		extPkt, err := buff.ReadExtended(buf[:])
+		require.NoError(t, err)
+		require.Nil(t, extPkt)
+	})
 
 	h265Pkt := rtp.Packet{
 		Header: rtp.Header{
@@ -359,20 +369,27 @@ func TestCodecChange(t *testing.T) {
 	case <-time.After(100 * time.Millisecond):
 	}
 
-	buff.Bind(webrtc.RTPParameters{
-		HeaderExtensions: nil,
-		Codecs:           []webrtc.RTPCodecParameters{vp8Codec, h265Codec},
-	}, vp8Codec.RTPCodecCapability, 0)
+	// Bind sets up VP8 as expected codec,
+	// packet written to the buffer above before Bind is H.265,
+	// that should trigger a codec change and a stream restart
+	// when the queued packets from Write before Bind get flushed
+	buff.Bind(
+		webrtc.RTPParameters{
+			HeaderExtensions: nil,
+			Codecs:           []webrtc.RTPCodecParameters{vp8Codec, h265Codec},
+		},
+		vp8Codec.RTPCodecCapability,
+		0,
+	)
 
 	select {
 	case c := <-changedCodec:
 		require.Equal(t, h265Codec, c)
-
 	case <-time.After(1 * time.Second):
 		t.Fatalf("expected codec change")
 	}
 
-	// codec change after bind
+	// second codec change - writing VP8 packet after Bind should trigger another codec change
 	vp8Pkt := rtp.Packet{
 		Header: rtp.Header{
 			Version:        2,
@@ -391,12 +408,16 @@ func TestCodecChange(t *testing.T) {
 	select {
 	case c := <-changedCodec:
 		require.Equal(t, vp8Codec, c)
-
 	case <-time.After(1 * time.Second):
 		t.Fatalf("expected codec change")
 	}
+	fmt.Printf("done second codec change\n") // REMOVE
 
 	// out of order pkts can't cause codec change
+	// rewrite the VP8 packet to start the sequence after a stream restart
+	_, err = buff.Write(buf)
+	require.NoError(t, err)
+
 	h265Pkt.SequenceNumber = 2
 	h265Pkt.Timestamp = 2
 	buf, err = h265Pkt.Marshal()
@@ -409,7 +430,7 @@ func TestCodecChange(t *testing.T) {
 	case <-time.After(100 * time.Millisecond):
 	}
 
-	// unknown codec should not cause change
+	// unknown codec should not cause change even if it is in order
 	h265Pkt.SequenceNumber = 4
 	h265Pkt.Timestamp = 4
 	h265Pkt.PayloadType = 117
@@ -422,14 +443,37 @@ func TestCodecChange(t *testing.T) {
 		t.Fatalf("unexpected codec change")
 	case <-time.After(100 * time.Millisecond):
 	}
+
+	// an in-order packet should change codec again
+	h265Pkt.SequenceNumber = 5
+	h265Pkt.Timestamp = 5
+	h265Pkt.PayloadType = 116
+	buf, err = h265Pkt.Marshal()
+	require.NoError(t, err)
+	_, err = buff.Write(buf)
+	require.NoError(t, err)
+	select {
+	case c := <-changedCodec:
+		require.Equal(t, h265Codec, c)
+	case <-time.After(1 * time.Second):
+		t.Fatalf("expected codec change")
+	}
 }
 
 func BenchmarkMemcpu(b *testing.B) {
 	buf := make([]byte, 1500*1500*10)
 	buf2 := make([]byte, 1500*1500*20)
 
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
+	for b.Loop() {
 		copy(buf2, buf)
 	}
 }
+
+func BenchmarkExtPacketFactory(b *testing.B) {
+
+	for b.Loop() {
+		extPkt := ExtPacketFactory.Get().(*ExtPacket)
+		*extPkt = ExtPacket{}
+		ExtPacketFactory.Put(extPkt)
+	}
+}

livekit-server/pkg/sfu/buffer/dependencydescriptorparser.go

@@ -16,20 +16,41 @@ package buffer
 
 import (
 	"fmt"
-	"sort"
+	"slices"
+	"sync"
+	"time"
 
 	"github.com/pion/rtp"
 	"go.uber.org/atomic"
 
 	dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor"
-	"github.com/livekit/livekit-server/pkg/sfu/utils"
-
+	"github.com/livekit/mediatransportutil/pkg/codec"
+	"github.com/livekit/mediatransportutil/pkg/utils"
 	"github.com/livekit/protocol/logger"
 )
 
+var (
+	ExtDependencyDescriptorFactory = &sync.Pool{
+		New: func() any {
+			return &ExtDependencyDescriptor{}
+		},
+	}
+)
+
+// --------------------------------------
+
+const (
+	ddRestartThreshold = 30 * time.Second
+
+	// frame integrity check 2 seconds for L3T3 30fps video
+	integrityCheckFrame = 180
+	integrityCheckPkt   = 1024
+)
+
 var (
 	ErrFrameEarlierThanKeyFrame            = fmt.Errorf("frame is earlier than current keyframe")
 	ErrDDStructureAttachedToNonFirstPacket = fmt.Errorf("dependency descriptor structure is attached to non-first packet of a frame")
+	ErrDDExtentionNotFound                 = fmt.Errorf("dependency descriptor extension not found")
 )
 
 type DependencyDescriptorParser struct {
@@ -47,16 +68,22 @@ type DependencyDescriptorParser struct {
 	frameChecker              *FrameIntegrityChecker
 
 	ddNotFoundCount atomic.Uint32
+
+	// restart detection
+	restartGeneration int
+	enableRestart     bool
+	lastPacketAt      time.Time
 }
 
-func NewDependencyDescriptorParser(ddExtID uint8, logger logger.Logger, onMaxLayerChanged func(int32, int32)) *DependencyDescriptorParser {
+func NewDependencyDescriptorParser(ddExtID uint8, logger logger.Logger, onMaxLayerChanged func(int32, int32), enableRestart bool) *DependencyDescriptorParser {
 	return &DependencyDescriptorParser{
 		ddExtID:           ddExtID,
 		logger:            logger,
 		onMaxLayerChanged: onMaxLayerChanged,
 		seqWrapAround:     utils.NewWrapAround[uint16, uint64](utils.WrapAroundParams{IsRestartAllowed: false}),
 		frameWrapAround:   utils.NewWrapAround[uint16, uint64](utils.WrapAroundParams{IsRestartAllowed: false}),
-		frameChecker:      NewFrameIntegrityChecker(180, 1024), // 2seconds for L3T3 30fps video
+		frameChecker:      NewFrameIntegrityChecker(integrityCheckFrame, integrityCheckPkt),
+		enableRestart:     enableRestart,
 	}
 }
 
@@ -70,6 +97,10 @@ type ExtDependencyDescriptor struct {
 	ExtFrameNum                uint64
 	// the frame number of the keyframe which the current frame depends on
 	ExtKeyFrameNum uint64
+
+	// increase when the stream restarts, clear and reinitialize all dd state includes
+	// attached structure, frame chain, decode target.
+	RestartGeneration int
 }
 
 func (r *DependencyDescriptorParser) Parse(pkt *rtp.Packet) (*ExtDependencyDescriptor, VideoLayer, error) {
@@ -80,7 +111,7 @@ func (r *DependencyDescriptorParser) Parse(pkt *rtp.Packet) (*ExtDependencyDescr
 		if ddNotFoundCount%100 == 0 {
 			r.logger.Warnw("dependency descriptor extension is not present", nil, "seq", pkt.SequenceNumber, "count", ddNotFoundCount)
 		}
-		return nil, videoLayer, nil
+		return nil, videoLayer, ErrDDExtentionNotFound
 	}
 
 	var ddVal dd.DependencyDescriptor
@@ -96,41 +127,92 @@ func (r *DependencyDescriptorParser) Parse(pkt *rtp.Packet) (*ExtDependencyDescr
 		return nil, videoLayer, err
 	}
 
+	var restart bool
+	if r.enableRestart {
+		if !r.lastPacketAt.IsZero() && time.Since(r.lastPacketAt) > ddRestartThreshold {
+			r.restart()
+			restart = true
+			r.logger.Debugw(
+				"dependency descriptor parser restart stream",
+				"generation", r.restartGeneration,
+				"lastPacketAt", r.lastPacketAt,
+				"sinceLast", time.Since(r.lastPacketAt),
+				"frameWrapAround", r.frameWrapAround,
+			)
+		}
+		r.lastPacketAt = time.Now()
+	}
+
 	extSeq := r.seqWrapAround.Update(pkt.SequenceNumber).ExtendedVal
 
 	if ddVal.FrameDependencies != nil {
 		videoLayer.Spatial, videoLayer.Temporal = int32(ddVal.FrameDependencies.SpatialId), int32(ddVal.FrameDependencies.TemporalId)
 	}
 
-	unwrapped := r.frameWrapAround.Update(ddVal.FrameNumber)
+	// assume the packet is in-order when stream restarting
+	unwrapped := r.frameWrapAround.UpdateWithOrderKnown(ddVal.FrameNumber, restart)
 	extFN := unwrapped.ExtendedVal
 
 	if extFN < r.structureExtFrameNum {
-		r.logger.Debugw("drop frame which is earlier than current structure", "frameNum", extFN, "structureFrameNum", r.structureExtFrameNum)
+		r.logger.Debugw(
+			"drop frame which is earlier than current structure",
+			"fn", ddVal.FrameNumber,
+			"extFN", extFN,
+			"structureExtFrameNum", r.structureExtFrameNum,
+			"unwrappedFN", unwrapped,
+			"frameWrapAround", r.frameWrapAround,
+		)
 		return nil, videoLayer, ErrFrameEarlierThanKeyFrame
 	}
 
 	r.frameChecker.AddPacket(extSeq, extFN, &ddVal)
 
-	extDD := &ExtDependencyDescriptor{
-		Descriptor:  &ddVal,
-		ExtFrameNum: extFN,
-		Integrity:   r.frameChecker.FrameIntegrity(extFN),
+	extDD := ExtDependencyDescriptorFactory.Get().(*ExtDependencyDescriptor)
+	*extDD = ExtDependencyDescriptor{
+		Descriptor:        &ddVal,
+		ExtFrameNum:       extFN,
+		Integrity:         r.frameChecker.FrameIntegrity(extFN),
+		RestartGeneration: r.restartGeneration,
 	}
 
 	if ddVal.AttachedStructure != nil {
 		if !ddVal.FirstPacketInFrame {
-			r.logger.Warnw("attached structure is not the first packet in frame", nil, "extSeq", extSeq, "extFN", extFN)
+			r.logger.Warnw(
+				"attached structure is not the first packet in frame", nil,
+				"sn", pkt.SequenceNumber,
+				"extSeq", extSeq,
+				"fn", ddVal.FrameNumber,
+				"extFN", extFN,
+			)
+			ReleaseExtDependencyDescriptor(extDD)
 			return nil, videoLayer, ErrDDStructureAttachedToNonFirstPacket
 		}
 
 		if r.structure == nil || ddVal.AttachedStructure.StructureId != r.structure.StructureId {
-			r.logger.Debugw("structure updated", "structureID", ddVal.AttachedStructure.StructureId, "extSeq", extSeq, "extFN", extFN, "descriptor", ddVal.String())
+			r.logger.Debugw(
+				"structure updated",
+				"structureID", ddVal.AttachedStructure.StructureId,
+				"sn", pkt.SequenceNumber,
+				"extSeq", extSeq,
+				"fn", ddVal.FrameNumber,
+				"extFN", extFN,
+				"descriptor", ddVal.String(),
+				"unwrappedFN", unwrapped,
+				"frameWrapAround", r.frameWrapAround,
+			)
 		}
 		r.structure = ddVal.AttachedStructure
 		r.decodeTargets = ProcessFrameDependencyStructure(ddVal.AttachedStructure)
 		if extFN > unwrapped.PreExtendedHighest && extFN-unwrapped.PreExtendedHighest > 1000 {
-			r.logger.Debugw("large frame number jump on structure updating", "extFN", extFN, "preExtendedHighest", unwrapped.PreExtendedHighest, "structureExtFrameNum", r.structureExtFrameNum)
+			r.logger.Debugw(
+				"large frame number jump on structure updating",
+				"fn", ddVal.FrameNumber,
+				"extFN", extFN,
+				"preExtendedHighest", unwrapped.PreExtendedHighest,
+				"structureExtFrameNum", r.structureExtFrameNum,
+				"unwrappedFN", unwrapped,
+				"frameWrapAround", r.frameWrapAround,
+			)
 		}
 		r.structureExtFrameNum = extFN
 		extDD.StructureUpdated = true
@@ -166,6 +248,16 @@ func (r *DependencyDescriptorParser) Parse(pkt *rtp.Packet) (*ExtDependencyDescr
 	return extDD, videoLayer, nil
 }
 
+func (r *DependencyDescriptorParser) restart() {
+	r.frameChecker = NewFrameIntegrityChecker(integrityCheckFrame, integrityCheckPkt)
+	r.structure = nil
+	r.structureExtFrameNum = 0
+	r.activeDecodeTargetsExtSeq = 0
+	r.activeDecodeTargetsMask = 0
+	r.decodeTargets = r.decodeTargets[:0]
+	r.restartGeneration++
+}
+
 // ------------------------------------------------------------------------------
 
 type DependencyDescriptorDecodeTarget struct {
@@ -197,8 +289,14 @@ func ProcessFrameDependencyStructure(structure *dd.FrameDependencyStructure) []D
 	}
 
 	// sort decode target layer by spatial and temporal from high to low
-	sort.Slice(decodeTargets, func(i, j int) bool {
-		return decodeTargets[i].Layer.GreaterThan(decodeTargets[j].Layer)
+	slices.SortFunc(decodeTargets, func(a, b DependencyDescriptorDecodeTarget) int {
+		if a.Layer.GreaterThan(b.Layer) {
+			return -1
+		}
+		if b.Layer.GreaterThan(a.Layer) {
+			return 1
+		}
+		return 0
 	})
 
 	return decodeTargets
@@ -216,3 +314,27 @@ func GetActiveDecodeTargetBitmask(layer VideoLayer, decodeTargets []DependencyDe
 }
 
 // ------------------------------------------------------------------------------
+
+func ExtractDependencyDescriptorVideoSize(dd *dd.DependencyDescriptor) []codec.VideoSize {
+	if dd.AttachedStructure == nil {
+		return nil
+	}
+
+	videoSizes := make([]codec.VideoSize, 0, len(dd.AttachedStructure.Resolutions))
+	for _, res := range dd.AttachedStructure.Resolutions {
+		videoSizes = append(videoSizes, codec.VideoSize{Width: uint32(res.Width), Height: uint32(res.Height)})
+	}
+
+	return videoSizes
+}
+
+// ------------------------------------------------------------------------------
+
+func ReleaseExtDependencyDescriptor(extDD *ExtDependencyDescriptor) {
+	if extDD == nil {
+		return
+	}
+
+	*extDD = ExtDependencyDescriptor{}
+	ExtDependencyDescriptorFactory.Put(extDD)
+}

livekit-server/pkg/sfu/buffer/factory.go

@@ -18,7 +18,7 @@ import (
 	"io"
 	"sync"
 
-	"github.com/pion/transport/v3/packetio"
+	"github.com/pion/transport/v4/packetio"
 )
 
 type FactoryOfBufferFactory struct {
@@ -118,7 +118,7 @@ func (f *Factory) GetRTCPReader(ssrc uint32) *RTCPReader {
 	return f.rtcpReaders[ssrc]
 }
 
-func (f *Factory) SetRTXPair(repair, base uint32) {
+func (f *Factory) SetRTXPair(repair, base uint32, rsid string) {
 	f.Lock()
 	repairBuffer, baseBuffer := f.rtpBuffers[repair], f.rtpBuffers[base]
 	if repairBuffer == nil || baseBuffer == nil {
@@ -127,5 +127,8 @@ func (f *Factory) SetRTXPair(repair, base uint32) {
 	f.Unlock()
 	if repairBuffer != nil && baseBuffer != nil {
 		repairBuffer.SetPrimaryBufferForRTX(baseBuffer)
+		if rsid != "" {
+			baseBuffer.NotifyRTX(base, repair, rsid)
+		}
 	}
 }

livekit-server/pkg/sfu/buffer/fps.go

@@ -19,6 +19,7 @@ import (
 
 	"github.com/pion/rtp/codecs"
 
+	"github.com/livekit/mediatransportutil/pkg/codec"
 	"github.com/livekit/protocol/logger"
 )
 
@@ -209,7 +210,7 @@ func (f *FrameRateCalculatorVP8) RecvPacket(ep *ExtPacket) bool {
 		return true
 	}
 
-	vp8, ok := ep.Payload.(VP8)
+	vp8, ok := ep.Payload.(codec.VP8)
 	if !ok {
 		f.logger.Debugw("no vp8 payload", "sn", ep.Packet.SequenceNumber)
 		return false

livekit-server/pkg/sfu/buffer/fps_test.go

@@ -21,6 +21,7 @@ import (
 	"github.com/stretchr/testify/require"
 
 	dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor"
+	"github.com/livekit/mediatransportutil/pkg/codec"
 	"github.com/livekit/protocol/logger"
 )
 
@@ -35,7 +36,7 @@ type testFrameInfo struct {
 func (f *testFrameInfo) toVP8() *ExtPacket {
 	return &ExtPacket{
 		Packet: &rtp.Packet{Header: f.header},
-		Payload: VP8{
+		Payload: codec.VP8{
 			PictureID: f.framenumber,
 		},
 		VideoLayer: VideoLayer{Spatial: InvalidLayerSpatial, Temporal: int32(f.temporal)},
@@ -75,7 +76,7 @@ func createFrames(startFrameNumber uint16, startTs uint32, startSeq uint16, tota
 
 	nextTs := make([][]uint32, spatials)
 	tsStep := make([][]uint32, spatials)
-	for i := 0; i < spatials; i++ {
+	for i := range spatials {
 		nextTs[i] = make([]uint32, temporals)
 		tsStep[i] = make([]uint32, temporals)
 		for j := 0; j < temporals; j++ {
@@ -85,11 +86,11 @@ func createFrames(startFrameNumber uint16, startTs uint32, startSeq uint16, tota
 	}
 
 	currentTs := make([]uint32, spatials)
-	for i := 0; i < spatials; i++ {
+	for i := range spatials {
 		currentTs[i] = startTs
 	}
-	for i := 0; i < totalFramesPerSpatial; i++ {
-		for s := 0; s < spatials; s++ {
+	for range totalFramesPerSpatial {
+		for s := range spatials {
 			frame := &testFrameInfo{
 				header:      rtp.Header{Timestamp: currentTs[s], SequenceNumber: startSeq},
 				framenumber: fn,
@@ -134,7 +135,7 @@ func createFrames(startFrameNumber uint16, startTs uint32, startSeq uint16, tota
 
 func verifyFps(t *testing.T, expect, got []float32) {
 	require.Equal(t, len(expect), len(got))
-	for i := 0; i < len(expect); i++ {
+	for i := range expect {
 		require.GreaterOrEqual(t, got[i], expect[i]*0.9, "expect %v, got %v", expect, got)
 		require.LessOrEqual(t, got[i], expect[i]*1.1, "expect %v, got %v", expect, got)
 	}

livekit-server/pkg/sfu/buffer/helpers.go

@@ -1,455 +0,0 @@
-// Copyright 2023 LiveKit, Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package buffer
-
-import (
-	"encoding/binary"
-	"errors"
-
-	"github.com/pion/rtp/codecs"
-
-	"github.com/livekit/protocol/logger"
-)
-
-var (
-	errShortPacket   = errors.New("packet is not large enough")
-	errNilPacket     = errors.New("invalid nil packet")
-	errInvalidPacket = errors.New("invalid packet")
-)
-
-// VP8 is a helper to get temporal data from VP8 packet header
-/*
-	VP8 Payload Descriptor
-			0 1 2 3 4 5 6 7                      0 1 2 3 4 5 6 7
-			+-+-+-+-+-+-+-+-+                   +-+-+-+-+-+-+-+-+
-			|X|R|N|S|R| PID | (REQUIRED)        |X|R|N|S|R| PID | (REQUIRED)
-			+-+-+-+-+-+-+-+-+                   +-+-+-+-+-+-+-+-+
-		X:  |I|L|T|K| RSV   | (OPTIONAL)   X:   |I|L|T|K| RSV   | (OPTIONAL)
-			+-+-+-+-+-+-+-+-+                   +-+-+-+-+-+-+-+-+
-		I:  |M| PictureID   | (OPTIONAL)   I:   |M| PictureID   | (OPTIONAL)
-			+-+-+-+-+-+-+-+-+                   +-+-+-+-+-+-+-+-+
-		L:  |   TL0PICIDX   | (OPTIONAL)        |   PictureID   |
-			+-+-+-+-+-+-+-+-+                   +-+-+-+-+-+-+-+-+
-		T/K:|TID|Y| KEYIDX  | (OPTIONAL)   L:   |   TL0PICIDX   | (OPTIONAL)
-			+-+-+-+-+-+-+-+-+                   +-+-+-+-+-+-+-+-+
-		T/K:|TID|Y| KEYIDX  | (OPTIONAL)
-			+-+-+-+-+-+-+-+-+
-*/
-type VP8 struct {
-	FirstByte byte
-	S         bool
-
-	I         bool
-	M         bool
-	PictureID uint16 /* 7 or 15 bits, picture ID */
-
-	L         bool
-	TL0PICIDX uint8 /* 8 bits temporal level zero index */
-
-	// Optional Header If either of the T or K bits are set to 1,
-	// the TID/Y/KEYIDX extension field MUST be present.
-	T   bool
-	TID uint8 /* 2 bits temporal layer idx */
-	Y   bool
-
-	K      bool
-	KEYIDX uint8 /* 5 bits of key frame idx */
-
-	HeaderSize int
-
-	// IsKeyFrame is a helper to detect if current packet is a keyframe
-	IsKeyFrame bool
-}
-
-// Unmarshal parses the passed byte slice and stores the result in the VP8 this method is called upon
-func (v *VP8) Unmarshal(payload []byte) error {
-	if payload == nil {
-		return errNilPacket
-	}
-
-	payloadLen := len(payload)
-	if payloadLen < 1 {
-		return errShortPacket
-	}
-
-	idx := 0
-	v.FirstByte = payload[idx]
-	v.S = payload[idx]&0x10 > 0
-	// Check for extended bit control
-	if payload[idx]&0x80 > 0 {
-		idx++
-		if payloadLen < idx+1 {
-			return errShortPacket
-		}
-		v.I = payload[idx]&0x80 > 0
-		v.L = payload[idx]&0x40 > 0
-		v.T = payload[idx]&0x20 > 0
-		v.K = payload[idx]&0x10 > 0
-		if v.L && !v.T {
-			return errInvalidPacket
-		}
-
-		if v.I {
-			idx++
-			if payloadLen < idx+1 {
-				return errShortPacket
-			}
-			pid := payload[idx] & 0x7f
-			// if m is 1, then Picture ID is 15 bits
-			v.M = payload[idx]&0x80 > 0
-			if v.M {
-				idx++
-				if payloadLen < idx+1 {
-					return errShortPacket
-				}
-				v.PictureID = binary.BigEndian.Uint16([]byte{pid, payload[idx]})
-			} else {
-				v.PictureID = uint16(pid)
-			}
-		}
-
-		if v.L {
-			idx++
-			if payloadLen < idx+1 {
-				return errShortPacket
-			}
-			v.TL0PICIDX = payload[idx]
-		}
-
-		if v.T || v.K {
-			idx++
-			if payloadLen < idx+1 {
-				return errShortPacket
-			}
-
-			if v.T {
-				v.TID = (payload[idx] & 0xc0) >> 6
-				v.Y = (payload[idx] & 0x20) > 0
-			}
-
-			if v.K {
-				v.KEYIDX = payload[idx] & 0x1f
-			}
-		}
-		idx++
-		if payloadLen < idx+1 {
-			return errShortPacket
-		}
-
-		// Check is packet is a keyframe by looking at P bit in vp8 payload
-		v.IsKeyFrame = payload[idx]&0x01 == 0 && v.S
-	} else {
-		idx++
-		if payloadLen < idx+1 {
-			return errShortPacket
-		}
-		// Check is packet is a keyframe by looking at P bit in vp8 payload
-		v.IsKeyFrame = payload[idx]&0x01 == 0 && v.S
-	}
-	v.HeaderSize = idx
-	return nil
-}
-
-func (v *VP8) Marshal() ([]byte, error) {
-	buf := make([]byte, v.HeaderSize)
-	n, err := v.MarshalTo(buf)
-	if err != nil {
-		return nil, err
-	}
-	return buf[:n], err
-}
-
-func (v *VP8) MarshalTo(buf []byte) (int, error) {
-	if len(buf) < v.HeaderSize {
-		return 0, errShortPacket
-	}
-
-	idx := 0
-	buf[idx] = v.FirstByte
-	if v.I || v.L || v.T || v.K {
-		buf[idx] |= 0x80 // X bit
-		idx++
-
-		xpos := idx
-		xval := byte(0)
-
-		idx++
-		if v.I {
-			xval |= (1 << 7)
-			if v.M {
-				buf[idx] = 0x80 | byte((v.PictureID>>8)&0x7f)
-				buf[idx+1] = byte(v.PictureID & 0xff)
-				idx += 2
-			} else {
-				buf[idx] = byte(v.PictureID)
-				idx++
-			}
-		}
-
-		if v.L {
-			xval |= (1 << 6)
-			buf[idx] = v.TL0PICIDX
-			idx++
-		}
-
-		if v.T || v.K {
-			buf[idx] = 0
-			if v.T {
-				xval |= (1 << 5)
-				buf[idx] = v.TID << 6
-				if v.Y {
-					buf[idx] |= (1 << 5)
-				}
-			}
-
-			if v.K {
-				xval |= (1 << 4)
-				buf[idx] |= v.KEYIDX & 0x1f
-			}
-			idx++
-		}
-
-		buf[xpos] = xval
-	} else {
-		buf[idx] &^= 0x80 // X bit
-		idx++
-	}
-
-	return idx, nil
-}
-
-// -------------------------------------
-
-func VPxPictureIdSizeDiff(mBit1 bool, mBit2 bool) int {
-	if mBit1 == mBit2 {
-		return 0
-	}
-
-	if mBit1 {
-		return 1
-	}
-
-	return -1
-}
-
-// -------------------------------------
-
-// IsH264KeyFrame detects if h264 payload is a keyframe
-// this code was taken from https://github.com/jech/galene/blob/codecs/rtpconn/rtpreader.go#L45
-// all credits belongs to Juliusz Chroboczek @jech and the awesome Galene SFU
-func IsH264KeyFrame(payload []byte) bool {
-	if len(payload) < 1 {
-		return false
-	}
-	nalu := payload[0] & 0x1F
-	if nalu == 0 {
-		// reserved
-		return false
-	} else if nalu <= 23 {
-		// simple NALU
-		return nalu == 7
-	} else if nalu == 24 || nalu == 25 || nalu == 26 || nalu == 27 {
-		// STAP-A, STAP-B, MTAP16 or MTAP24
-		i := 1
-		if nalu == 25 || nalu == 26 || nalu == 27 {
-			// skip DON
-			i += 2
-		}
-		for i < len(payload) {
-			if i+2 > len(payload) {
-				return false
-			}
-			length := uint16(payload[i])<<8 |
-				uint16(payload[i+1])
-			i += 2
-			if i+int(length) > len(payload) {
-				return false
-			}
-			offset := 0
-			if nalu == 26 {
-				offset = 3
-			} else if nalu == 27 {
-				offset = 4
-			}
-			if offset >= int(length) {
-				return false
-			}
-			n := payload[i+offset] & 0x1F
-			if n == 7 {
-				return true
-			} else if n >= 24 {
-				// is this legal?
-				logger.Debugw("Non-simple NALU within a STAP")
-			}
-			i += int(length)
-		}
-		if i == len(payload) {
-			return false
-		}
-		return false
-	} else if nalu == 28 || nalu == 29 {
-		// FU-A or FU-B
-		if len(payload) < 2 {
-			return false
-		}
-		if (payload[1] & 0x80) == 0 {
-			// not a starting fragment
-			return false
-		}
-		return payload[1]&0x1F == 7
-	}
-	return false
-}
-
-// -------------------------------------
-
-// IsVP9KeyFrame detects if vp9 payload is a keyframe
-// taken from https://github.com/jech/galene/blob/master/codecs/codecs.go
-// all credits belongs to Juliusz Chroboczek @jech and the awesome Galene SFU
-func IsVP9KeyFrame(payload []byte) bool {
-	var vp9 codecs.VP9Packet
-	_, err := vp9.Unmarshal(payload)
-	if err != nil || len(vp9.Payload) < 1 {
-		return false
-	}
-	if !vp9.B {
-		return false
-	}
-
-	if (vp9.Payload[0] & 0xc0) != 0x80 {
-		return false
-	}
-
-	profile := (vp9.Payload[0] >> 4) & 0x3
-	if profile != 3 {
-		return (vp9.Payload[0] & 0xC) == 0
-	}
-	return (vp9.Payload[0] & 0x6) == 0
-}
-
-// -------------------------------------
-
-// IsAV1KeyFrame detects if av1 payload is a keyframe
-// taken from https://github.com/jech/galene/blob/master/codecs/codecs.go
-// all credits belongs to Juliusz Chroboczek @jech and the awesome Galene SFU
-func IsAV1KeyFrame(payload []byte) bool {
-	if len(payload) < 2 {
-		return false
-	}
-	// Z=0, N=1
-	if (payload[0] & 0x88) != 0x08 {
-		return false
-	}
-	w := (payload[0] & 0x30) >> 4
-
-	getObu := func(data []byte, last bool) ([]byte, int, bool) {
-		if last {
-			return data, len(data), false
-		}
-		offset := 0
-		length := 0
-		for {
-			if len(data) <= offset {
-				return nil, offset, offset > 0
-			}
-			l := data[offset]
-			length |= int(l&0x7f) << (offset * 7)
-			offset++
-			if (l & 0x80) == 0 {
-				break
-			}
-		}
-		if len(data) < offset+length {
-			return data[offset:], len(data), true
-		}
-		return data[offset : offset+length],
-			offset + length, false
-	}
-	offset := 1
-	i := 0
-	for {
-		obu, length, truncated :=
-			getObu(payload[offset:], int(w) == i+1)
-		if len(obu) < 1 {
-			return false
-		}
-		tpe := (obu[0] & 0x38) >> 3
-		switch i {
-		case 0:
-			// OBU_SEQUENCE_HEADER
-			if tpe != 1 {
-				return false
-			}
-		default:
-			// OBU_FRAME_HEADER or OBU_FRAME
-			if tpe == 3 || tpe == 6 {
-				if len(obu) < 2 {
-					return false
-				}
-				// show_existing_frame == 0
-				if (obu[1] & 0x80) != 0 {
-					return false
-				}
-				// frame_type == KEY_FRAME
-				return (obu[1] & 0x60) == 0
-			}
-		}
-		if truncated || i >= int(w) {
-			// the first frame header is in a second
-			// packet, give up.
-			return false
-		}
-		offset += length
-		i++
-	}
-}
-
-func IsH265KeyFrame(payload []byte) (kf bool) {
-	if len(payload) < 2 {
-		return false
-	}
-	naluType := (payload[0] & 0x7E) >> 1
-	switch {
-	case naluType == 33 || naluType == 34:
-		return true
-	case naluType == 48: // AP
-		idx := 2
-		for idx < len(payload)-2 {
-			// TODO: check the DONL field (controled by sprop-max-don-diff)
-			size := binary.BigEndian.Uint16(payload[idx:])
-			idx += 2
-			if idx >= len(payload) {
-				return false
-			}
-			naluType = (payload[idx] & 0x7E) >> 1
-			if naluType == 33 || naluType == 34 {
-				return true
-			}
-			idx += int(size)
-		}
-		return false
-
-	case naluType == 49: // FU
-		if len(payload) < 3 {
-			return false
-		}
-		naluType = (payload[2] & 0x7E) >> 1
-		return naluType == 33 || naluType == 34
-	default:
-		return false
-	}
-}
-
-// -------------------------------------

livekit-server/pkg/sfu/buffer/helpers_test.go

@@ -1,110 +0,0 @@
-// Copyright 2023 LiveKit, Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package buffer
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/require"
-)
-
-func TestVP8Helper_Unmarshal(t *testing.T) {
-	type args struct {
-		payload []byte
-	}
-	tests := []struct {
-		name            string
-		args            args
-		wantErr         bool
-		checkTemporal   bool
-		temporalSupport bool
-		checkKeyFrame   bool
-		keyFrame        bool
-		checkPictureID  bool
-		pictureID       uint16
-		checkTlzIdx     bool
-		tlzIdx          uint8
-		checkTempID     bool
-		temporalID      uint8
-	}{
-		{
-			name:    "Empty or nil payload must return error",
-			args:    args{payload: []byte{}},
-			wantErr: true,
-		},
-		{
-			name:            "Temporal must be supported by setting T bit to 1",
-			args:            args{payload: []byte{0xff, 0x20, 0x1, 0x2, 0x3, 0x4}},
-			checkTemporal:   true,
-			temporalSupport: true,
-		},
-		{
-			name:           "Picture must be ID 7 bits by setting M bit to 0 and present by I bit set to 1",
-			args:           args{payload: []byte{0xff, 0xff, 0x11, 0x2, 0x3, 0x4}},
-			checkPictureID: true,
-			pictureID:      17,
-		},
-		{
-			name:           "Picture ID must be 15 bits by setting M bit to 1 and present by I bit set to 1",
-			args:           args{payload: []byte{0xff, 0xff, 0x92, 0x67, 0x3, 0x4, 0x5}},
-			checkPictureID: true,
-			pictureID:      4711,
-		},
-		{
-			name:        "Temporal level zero index must be present if L set to 1",
-			args:        args{payload: []byte{0xff, 0xff, 0xff, 0xfd, 0xb4, 0x4, 0x5}},
-			checkTlzIdx: true,
-			tlzIdx:      180,
-		},
-		{
-			name:        "Temporal index must be present and used if T bit set to 1",
-			args:        args{payload: []byte{0xff, 0xff, 0xff, 0xfd, 0xb4, 0x9f, 0x5, 0x6}},
-			checkTempID: true,
-			temporalID:  2,
-		},
-		{
-			name:          "Check if packet is a keyframe by looking at P bit set to 0",
-			args:          args{payload: []byte{0xff, 0xff, 0xff, 0xfd, 0xb4, 0x9f, 0x94, 0x1}},
-			checkKeyFrame: true,
-			keyFrame:      true,
-		},
-	}
-	for _, tt := range tests {
-		tt := tt
-		t.Run(tt.name, func(t *testing.T) {
-			p := &VP8{}
-			if err := p.Unmarshal(tt.args.payload); (err != nil) != tt.wantErr {
-				t.Errorf("Unmarshal() error = %v, wantErr %v", err, tt.wantErr)
-			}
-			if tt.checkTemporal {
-				require.Equal(t, tt.temporalSupport, p.T)
-			}
-			if tt.checkKeyFrame {
-				require.Equal(t, tt.keyFrame, p.IsKeyFrame)
-			}
-			if tt.checkPictureID {
-				require.Equal(t, tt.pictureID, p.PictureID)
-			}
-			if tt.checkTlzIdx {
-				require.Equal(t, tt.tlzIdx, p.TL0PICIDX)
-			}
-			if tt.checkTempID {
-				require.Equal(t, tt.temporalID, p.TID)
-			}
-		})
-	}
-}
-
-// ------------------------------------------

livekit-server/pkg/sfu/buffer/rtcpreader.go

@@ -31,6 +31,7 @@ func NewRTCPReader(ssrc uint32) *RTCPReader {
 	return &RTCPReader{ssrc: ssrc}
 }
 
+//go:noinline
 func (r *RTCPReader) Write(p []byte) (n int, err error) {
 	if r.closed.Load() {
 		err = io.EOF

livekit-server/pkg/sfu/buffer/videolayerutils.go

@@ -15,24 +15,43 @@
 package buffer
 
 import (
+	"slices"
+
+	"github.com/livekit/protocol/codecs/mime"
 	"github.com/livekit/protocol/livekit"
 	"github.com/livekit/protocol/logger"
 )
 
 const (
-	QuarterResolution = "q"
-	HalfResolution    = "h"
-	FullResolution    = "f"
+	quarterResolutionQ = "q"
+	halfResolutionH    = "h"
+	fullResolutionF    = "f"
+
+	quarterResolution2 = "2"
+	halfResolution1    = "1"
+	fullResolution0    = "0"
 )
 
-// SIMULCAST-CODEC-TODO: these need to be codec mime aware if and when each codec suppports different layers
-func LayerPresenceFromTrackInfo(trackInfo *livekit.TrackInfo) *[livekit.VideoQuality_HIGH + 1]bool {
-	if trackInfo == nil || len(trackInfo.Layers) == 0 {
+type VideoLayersRid [DefaultMaxLayerSpatial + 1]string
+
+var (
+	videoLayersRidQHF     = VideoLayersRid{quarterResolutionQ, halfResolutionH, fullResolutionF}
+	videoLayersRid210     = VideoLayersRid{quarterResolution2, halfResolution1, fullResolution0}
+	DefaultVideoLayersRid = videoLayersRidQHF
+)
+
+func LayerPresenceFromTrackInfo(mimeType mime.MimeType, trackInfo *livekit.TrackInfo) *[livekit.VideoQuality_HIGH + 1]bool {
+	if trackInfo == nil {
+		return nil
+	}
+
+	layers := GetVideoLayersForMimeType(mimeType, trackInfo)
+	if len(layers) == 0 {
 		return nil
 	}
 
 	var layerPresence [livekit.VideoQuality_HIGH + 1]bool
-	for _, layer := range trackInfo.Layers {
+	for _, layer := range layers {
 		// WARNING: comparing protobuf enum
 		if layer.Quality <= livekit.VideoQuality_HIGH {
 			layerPresence[layer.Quality] = true
@@ -44,15 +63,15 @@ func LayerPresenceFromTrackInfo(trackInfo *livekit.TrackInfo) *[livekit.VideoQua
 	return &layerPresence
 }
 
-func RidToSpatialLayer(rid string, trackInfo *livekit.TrackInfo) int32 {
-	lp := LayerPresenceFromTrackInfo(trackInfo)
+func RidToSpatialLayer(mimeType mime.MimeType, rid string, trackInfo *livekit.TrackInfo, ridSpace VideoLayersRid) int32 {
+	lp := LayerPresenceFromTrackInfo(mimeType, trackInfo)
 	if lp == nil {
 		switch rid {
-		case QuarterResolution:
+		case quarterResolutionQ:
 			return 0
-		case HalfResolution:
+		case halfResolutionH:
 			return 1
-		case FullResolution:
+		case fullResolutionF:
 			return 2
 		default:
 			return 0
@@ -60,7 +79,7 @@ func RidToSpatialLayer(rid string, trackInfo *livekit.TrackInfo) int32 {
 	}
 
 	switch rid {
-	case QuarterResolution:
+	case ridSpace[0]:
 		switch {
 		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
 			fallthrough
@@ -76,7 +95,7 @@ func RidToSpatialLayer(rid string, trackInfo *livekit.TrackInfo) int32 {
 			return 0
 		}
 
-	case HalfResolution:
+	case ridSpace[1]:
 		switch {
 		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
 			fallthrough
@@ -92,19 +111,19 @@ func RidToSpatialLayer(rid string, trackInfo *livekit.TrackInfo) int32 {
 			return 0
 		}
 
-	case FullResolution:
+	case ridSpace[2]:
 		switch {
 		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
 			return 2
 
 		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM]:
-			logger.Warnw("unexpected rid f with only two qualities, low and medium", nil, "trackID", trackInfo.Sid, "trackInfo", logger.Proto(trackInfo))
+			logger.Warnw("unexpected rid with only two qualities, low and medium", nil, "trackID", trackInfo.Sid, "trackInfo", logger.Proto(trackInfo), "rid", ridSpace[2])
 			return 1
 		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_HIGH]:
-			logger.Warnw("unexpected rid f with only two qualities, low and high", nil, "trackID", trackInfo.Sid, "trackInfo", logger.Proto(trackInfo))
+			logger.Warnw("unexpected rid with only two qualities, low and high", nil, "trackID", trackInfo.Sid, "trackInfo", logger.Proto(trackInfo), "rid", ridSpace[2])
 			return 1
 		case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
-			logger.Warnw("unexpected rid f with only two qualities, medium and high", nil, "trackID", trackInfo.Sid, "trackInfo", logger.Proto(trackInfo))
+			logger.Warnw("unexpected rid with only two qualities, medium and high", nil, "trackID", trackInfo.Sid, "trackInfo", logger.Proto(trackInfo), "rid", ridSpace[2])
 			return 1
 
 		default:
@@ -118,18 +137,18 @@ func RidToSpatialLayer(rid string, trackInfo *livekit.TrackInfo) int32 {
 	}
 }
 
-func SpatialLayerToRid(layer int32, trackInfo *livekit.TrackInfo) string {
-	lp := LayerPresenceFromTrackInfo(trackInfo)
+func SpatialLayerToRid(mimeType mime.MimeType, layer int32, trackInfo *livekit.TrackInfo, ridSpace VideoLayersRid) string {
+	lp := LayerPresenceFromTrackInfo(mimeType, trackInfo)
 	if lp == nil {
 		switch layer {
 		case 0:
-			return QuarterResolution
+			return quarterResolutionQ
 		case 1:
-			return HalfResolution
+			return halfResolutionH
 		case 2:
-			return FullResolution
+			return fullResolutionF
 		default:
-			return QuarterResolution
+			return quarterResolutionQ
 		}
 	}
 
@@ -143,10 +162,10 @@ func SpatialLayerToRid(layer int32, trackInfo *livekit.TrackInfo) string {
 		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_HIGH]:
 			fallthrough
 		case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
-			return QuarterResolution
+			return ridSpace[0]
 
 		default:
-			return QuarterResolution
+			return ridSpace[0]
 		}
 
 	case 1:
@@ -158,42 +177,42 @@ func SpatialLayerToRid(layer int32, trackInfo *livekit.TrackInfo) string {
 		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_HIGH]:
 			fallthrough
 		case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
-			return HalfResolution
+			return ridSpace[1]
 
 		default:
-			return QuarterResolution
+			return ridSpace[0]
 		}
 
 	case 2:
 		switch {
 		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
-			return FullResolution
+			return ridSpace[2]
 
 		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM]:
 			logger.Warnw("unexpected layer 2 with only two qualities, low and medium", nil, "trackID", trackInfo.Sid, "trackInfo", logger.Proto(trackInfo))
-			return HalfResolution
+			return ridSpace[1]
 		case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_HIGH]:
 			logger.Warnw("unexpected layer 2 with only two qualities, low and high", nil, "trackID", trackInfo.Sid, "trackInfo", logger.Proto(trackInfo))
-			return HalfResolution
+			return ridSpace[1]
 		case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
 			logger.Warnw("unexpected layer 2 with only two qualities, medium and high", nil, "trackID", trackInfo.Sid, "trackInfo", logger.Proto(trackInfo))
-			return HalfResolution
+			return ridSpace[1]
 
 		default:
-			return QuarterResolution
+			return ridSpace[0]
 		}
 
 	default:
-		return QuarterResolution
+		return ridSpace[0]
 	}
 }
 
-func VideoQualityToRid(quality livekit.VideoQuality, trackInfo *livekit.TrackInfo) string {
-	return SpatialLayerToRid(VideoQualityToSpatialLayer(quality, trackInfo), trackInfo)
+func VideoQualityToRid(mimeType mime.MimeType, quality livekit.VideoQuality, trackInfo *livekit.TrackInfo, ridSpace VideoLayersRid) string {
+	return SpatialLayerToRid(mimeType, VideoQualityToSpatialLayer(mimeType, quality, trackInfo), trackInfo, ridSpace)
 }
 
-func SpatialLayerToVideoQuality(layer int32, trackInfo *livekit.TrackInfo) livekit.VideoQuality {
-	lp := LayerPresenceFromTrackInfo(trackInfo)
+func SpatialLayerToVideoQuality(mimeType mime.MimeType, layer int32, trackInfo *livekit.TrackInfo) livekit.VideoQuality {
+	lp := LayerPresenceFromTrackInfo(mimeType, trackInfo)
 	if lp == nil {
 		switch layer {
 		case 0:
@@ -259,8 +278,8 @@ func SpatialLayerToVideoQuality(layer int32, trackInfo *livekit.TrackInfo) livek
 	return livekit.VideoQuality_OFF
 }
 
-func VideoQualityToSpatialLayer(quality livekit.VideoQuality, trackInfo *livekit.TrackInfo) int32 {
-	lp := LayerPresenceFromTrackInfo(trackInfo)
+func VideoQualityToSpatialLayer(mimeType mime.MimeType, quality livekit.VideoQuality, trackInfo *livekit.TrackInfo) int32 {
+	lp := LayerPresenceFromTrackInfo(mimeType, trackInfo)
 	if lp == nil {
 		switch quality {
 		case livekit.VideoQuality_LOW:
@@ -324,3 +343,178 @@ func VideoQualityToSpatialLayer(quality livekit.VideoQuality, trackInfo *livekit
 
 	return InvalidLayerSpatial
 }
+
+func GetVideoLayerModeForMimeType(mimeType mime.MimeType, ti *livekit.TrackInfo) livekit.VideoLayer_Mode {
+	if ti != nil {
+		for _, codec := range ti.Codecs {
+			if mime.NormalizeMimeType(codec.MimeType) == mimeType {
+				return codec.VideoLayerMode
+			}
+		}
+	}
+
+	return livekit.VideoLayer_MODE_UNUSED
+}
+
+func GetVideoLayersForMimeType(mimeType mime.MimeType, ti *livekit.TrackInfo) []*livekit.VideoLayer {
+	var layers []*livekit.VideoLayer
+	if ti != nil {
+		for _, codec := range ti.Codecs {
+			if mime.NormalizeMimeType(codec.MimeType) == mimeType {
+				layers = codec.Layers
+				break
+			}
+		}
+		if len(layers) == 0 {
+			layers = ti.Layers
+		}
+	}
+	return layers
+}
+
+func GetSpatialLayerForRid(mimeType mime.MimeType, rid string, ti *livekit.TrackInfo) int32 {
+	if ti == nil {
+		return InvalidLayerSpatial
+	}
+
+	if rid == "" {
+		// single layer without RID
+		return 0
+	}
+
+	layers := GetVideoLayersForMimeType(mimeType, ti)
+	for _, layer := range layers {
+		if layer.Rid == rid {
+			return layer.SpatialLayer
+		}
+	}
+
+	if len(layers) != 0 {
+		// RID present in codec, but may not be specified via signalling
+		// (happens with older browsers setting a rid for SVC codecs)
+		hasRid := false
+		for _, layer := range layers {
+			if layer.Rid != "" {
+				hasRid = true
+				break
+			}
+		}
+		if !hasRid {
+			return 0
+		}
+	}
+
+	// SIMULCAST-CODEC-TODO - ideally should return invalid, but there are
+	// VP9 publishers using rid = f, if there are only two layers
+	// in TrackInfo, that will be q;h and f will become invalid.
+	//
+	// Actually, there should be no rids for VP9 in SDP and hence
+	// the above check should take effect. However, as simulcast
+	// codec/back up codec does not update rids from SDP,
+	// the default rids are used when vp9 (primary codec)
+	// is published. Due to that the above check gets bypassed.
+	//
+	// The full proper sequence would be
+	// 1. For primary codec using SVC, there will be no rids.
+	//    The above check should take effect and it should
+	//    return 0 even if some publisher uses a rid like `f`.
+	// 2. When secondary codec is published, rids for the codec
+	//    corresponding to the back up codec mime type should
+	//    be updated in `TrackInfo`. This is a bit tricky
+	//    for a couple of cases
+	//    a. Browsers like Firefox use a different CID everytime.
+	//       So, it cannot be matched between `AddTrack` and SDP.
+	//       One option is to look for a published track with
+	//       back up codec and apply it there. But, that becomes
+	//       a challenge if there are multiple published tracks
+	//       with pending back up codec.
+	//    b. The back up codec publish SDP will have the full
+	//       codec list. It should be okay to assume that the
+	//       codec that will be published is the back up codec,
+	//       but just something to be aware of.
+	// 3. Use of this function with proper mime so that proper
+	//    codec section can be looked up in `TrackInfo`.
+	// return InvalidLayerSpatial
+	logger.Infow(
+		"invalid layer for rid, returning default",
+		"trackID", ti.Sid,
+		"rid", rid,
+		"mimeType", mimeType,
+		"trackInfo", logger.Proto(ti),
+	)
+	return 0
+}
+
+func GetSpatialLayerForVideoQuality(mimeType mime.MimeType, quality livekit.VideoQuality, ti *livekit.TrackInfo) int32 {
+	if ti == nil || quality == livekit.VideoQuality_OFF {
+		return InvalidLayerSpatial
+	}
+
+	layers := GetVideoLayersForMimeType(mimeType, ti)
+	for _, layer := range layers {
+		if layer.Quality == quality {
+			return layer.SpatialLayer
+		}
+	}
+
+	if len(layers) == 0 {
+		// single layer
+		return 0
+	}
+
+	// requested quality is higher than available layers, return the highest available layer
+	return VideoQualityToSpatialLayer(mimeType, quality, ti)
+}
+
+func GetVideoQualityForSpatialLayer(mimeType mime.MimeType, spatialLayer int32, ti *livekit.TrackInfo) livekit.VideoQuality {
+	if spatialLayer == InvalidLayerSpatial || ti == nil {
+		return livekit.VideoQuality_OFF
+	}
+
+	layers := GetVideoLayersForMimeType(mimeType, ti)
+	for _, layer := range layers {
+		if layer.SpatialLayer == spatialLayer {
+			return layer.Quality
+		}
+	}
+
+	return livekit.VideoQuality_OFF
+}
+
+func isVideoLayersRidKnown(rids VideoLayersRid, knownRids VideoLayersRid) bool {
+	for _, rid := range rids {
+		if rid == "" {
+			continue
+		}
+
+		if !slices.Contains(knownRids[:], rid) {
+			return false
+		}
+	}
+
+	return true
+}
+
+func NormalizeVideoLayersRid(rids VideoLayersRid) VideoLayersRid {
+	out := rids
+
+	normalize := func(knownRids VideoLayersRid) {
+		idx := 0
+		for _, known := range knownRids {
+			if slices.Contains(rids[:], known) {
+				out[idx] = known
+				idx++
+			}
+		}
+	}
+
+	if isVideoLayersRidKnown(rids, videoLayersRidQHF) {
+		normalize(videoLayersRidQHF)
+	}
+
+	if isVideoLayersRidKnown(rids, videoLayersRid210) {
+		normalize(videoLayersRid210)
+	}
+
+	return out
+}

livekit-server/pkg/sfu/buffer/videolayerutils_test.go

@@ -19,6 +19,7 @@ import (
 
 	"github.com/stretchr/testify/require"
 
+	"github.com/livekit/protocol/codecs/mime"
 	"github.com/livekit/protocol/livekit"
 )
 
@@ -30,129 +31,174 @@ func TestRidConversion(t *testing.T) {
 	tests := []struct {
 		name       string
 		trackInfo  *livekit.TrackInfo
+		mimeType   mime.MimeType
 		ridToLayer map[string]RidAndLayer
 	}{
 		{
 			"no track info",
 			nil,
+			mime.MimeTypeVP8,
 			map[string]RidAndLayer{
-				"":                {rid: QuarterResolution, layer: 0},
-				QuarterResolution: {rid: QuarterResolution, layer: 0},
-				HalfResolution:    {rid: HalfResolution, layer: 1},
-				FullResolution:    {rid: FullResolution, layer: 2},
+				"":                 {rid: quarterResolutionQ, layer: 0},
+				quarterResolutionQ: {rid: quarterResolutionQ, layer: 0},
+				halfResolutionH:    {rid: halfResolutionH, layer: 1},
+				fullResolutionF:    {rid: fullResolutionF, layer: 2},
 			},
 		},
 		{
 			"no layers",
 			&livekit.TrackInfo{},
+			mime.MimeTypeVP8,
 			map[string]RidAndLayer{
-				"":                {rid: QuarterResolution, layer: 0},
-				QuarterResolution: {rid: QuarterResolution, layer: 0},
-				HalfResolution:    {rid: HalfResolution, layer: 1},
-				FullResolution:    {rid: FullResolution, layer: 2},
+				"":                 {rid: quarterResolutionQ, layer: 0},
+				quarterResolutionQ: {rid: quarterResolutionQ, layer: 0},
+				halfResolutionH:    {rid: halfResolutionH, layer: 1},
+				fullResolutionF:    {rid: fullResolutionF, layer: 2},
 			},
 		},
 		{
 			"single layer, low",
 			&livekit.TrackInfo{
-				Layers: []*livekit.VideoLayer{
-					{Quality: livekit.VideoQuality_LOW},
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_LOW},
+						},
+					},
 				},
 			},
+			mime.MimeTypeVP8,
 			map[string]RidAndLayer{
-				"":                {rid: QuarterResolution, layer: 0},
-				QuarterResolution: {rid: QuarterResolution, layer: 0},
-				HalfResolution:    {rid: QuarterResolution, layer: 0},
-				FullResolution:    {rid: QuarterResolution, layer: 0},
+				"":                 {rid: quarterResolutionQ, layer: 0},
+				quarterResolutionQ: {rid: quarterResolutionQ, layer: 0},
+				halfResolutionH:    {rid: quarterResolutionQ, layer: 0},
+				fullResolutionF:    {rid: quarterResolutionQ, layer: 0},
 			},
 		},
 		{
 			"single layer, medium",
 			&livekit.TrackInfo{
-				Layers: []*livekit.VideoLayer{
-					{Quality: livekit.VideoQuality_MEDIUM},
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_MEDIUM},
+						},
+					},
 				},
 			},
+			mime.MimeTypeVP8,
 			map[string]RidAndLayer{
-				"":                {rid: QuarterResolution, layer: 0},
-				QuarterResolution: {rid: QuarterResolution, layer: 0},
-				HalfResolution:    {rid: QuarterResolution, layer: 0},
-				FullResolution:    {rid: QuarterResolution, layer: 0},
+				"":                 {rid: quarterResolutionQ, layer: 0},
+				quarterResolutionQ: {rid: quarterResolutionQ, layer: 0},
+				halfResolutionH:    {rid: quarterResolutionQ, layer: 0},
+				fullResolutionF:    {rid: quarterResolutionQ, layer: 0},
 			},
 		},
 		{
 			"single layer, high",
 			&livekit.TrackInfo{
-				Layers: []*livekit.VideoLayer{
-					{Quality: livekit.VideoQuality_HIGH},
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_MEDIUM},
+						},
+					},
 				},
 			},
+			mime.MimeTypeVP8,
 			map[string]RidAndLayer{
-				"":                {rid: QuarterResolution, layer: 0},
-				QuarterResolution: {rid: QuarterResolution, layer: 0},
-				HalfResolution:    {rid: QuarterResolution, layer: 0},
-				FullResolution:    {rid: QuarterResolution, layer: 0},
+				"":                 {rid: quarterResolutionQ, layer: 0},
+				quarterResolutionQ: {rid: quarterResolutionQ, layer: 0},
+				halfResolutionH:    {rid: quarterResolutionQ, layer: 0},
+				fullResolutionF:    {rid: quarterResolutionQ, layer: 0},
 			},
 		},
 		{
 			"two layers, low and medium",
 			&livekit.TrackInfo{
-				Layers: []*livekit.VideoLayer{
-					{Quality: livekit.VideoQuality_LOW},
-					{Quality: livekit.VideoQuality_MEDIUM},
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_LOW},
+							{Quality: livekit.VideoQuality_MEDIUM},
+						},
+					},
 				},
 			},
+			mime.MimeTypeVP8,
 			map[string]RidAndLayer{
-				"":                {rid: QuarterResolution, layer: 0},
-				QuarterResolution: {rid: QuarterResolution, layer: 0},
-				HalfResolution:    {rid: HalfResolution, layer: 1},
-				FullResolution:    {rid: HalfResolution, layer: 1},
+				"":                 {rid: quarterResolutionQ, layer: 0},
+				quarterResolutionQ: {rid: quarterResolutionQ, layer: 0},
+				halfResolutionH:    {rid: halfResolutionH, layer: 1},
+				fullResolutionF:    {rid: halfResolutionH, layer: 1},
 			},
 		},
 		{
 			"two layers, low and high",
 			&livekit.TrackInfo{
-				Layers: []*livekit.VideoLayer{
-					{Quality: livekit.VideoQuality_LOW},
-					{Quality: livekit.VideoQuality_HIGH},
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_LOW},
+							{Quality: livekit.VideoQuality_HIGH},
+						},
+					},
 				},
 			},
+			mime.MimeTypeVP8,
 			map[string]RidAndLayer{
-				"":                {rid: QuarterResolution, layer: 0},
-				QuarterResolution: {rid: QuarterResolution, layer: 0},
-				HalfResolution:    {rid: HalfResolution, layer: 1},
-				FullResolution:    {rid: HalfResolution, layer: 1},
+				"":                 {rid: quarterResolutionQ, layer: 0},
+				quarterResolutionQ: {rid: quarterResolutionQ, layer: 0},
+				halfResolutionH:    {rid: halfResolutionH, layer: 1},
+				fullResolutionF:    {rid: halfResolutionH, layer: 1},
 			},
 		},
 		{
 			"two layers, medium and high",
 			&livekit.TrackInfo{
-				Layers: []*livekit.VideoLayer{
-					{Quality: livekit.VideoQuality_MEDIUM},
-					{Quality: livekit.VideoQuality_HIGH},
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_MEDIUM},
+							{Quality: livekit.VideoQuality_HIGH},
+						},
+					},
 				},
 			},
+			mime.MimeTypeVP8,
 			map[string]RidAndLayer{
-				"":                {rid: QuarterResolution, layer: 0},
-				QuarterResolution: {rid: QuarterResolution, layer: 0},
-				HalfResolution:    {rid: HalfResolution, layer: 1},
-				FullResolution:    {rid: HalfResolution, layer: 1},
+				"":                 {rid: quarterResolutionQ, layer: 0},
+				quarterResolutionQ: {rid: quarterResolutionQ, layer: 0},
+				halfResolutionH:    {rid: halfResolutionH, layer: 1},
+				fullResolutionF:    {rid: halfResolutionH, layer: 1},
 			},
 		},
 		{
 			"three layers",
 			&livekit.TrackInfo{
-				Layers: []*livekit.VideoLayer{
-					{Quality: livekit.VideoQuality_LOW},
-					{Quality: livekit.VideoQuality_MEDIUM},
-					{Quality: livekit.VideoQuality_HIGH},
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_LOW},
+							{Quality: livekit.VideoQuality_MEDIUM},
+							{Quality: livekit.VideoQuality_HIGH},
+						},
+					},
 				},
 			},
+			mime.MimeTypeVP8,
 			map[string]RidAndLayer{
-				"":                {rid: QuarterResolution, layer: 0},
-				QuarterResolution: {rid: QuarterResolution, layer: 0},
-				HalfResolution:    {rid: HalfResolution, layer: 1},
-				FullResolution:    {rid: FullResolution, layer: 2},
+				"":                 {rid: quarterResolutionQ, layer: 0},
+				quarterResolutionQ: {rid: quarterResolutionQ, layer: 0},
+				halfResolutionH:    {rid: halfResolutionH, layer: 1},
+				fullResolutionF:    {rid: fullResolutionF, layer: 2},
 			},
 		},
 	}
@@ -160,10 +206,10 @@ func TestRidConversion(t *testing.T) {
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
 			for testRid, expectedResult := range test.ridToLayer {
-				actualLayer := RidToSpatialLayer(testRid, test.trackInfo)
+				actualLayer := RidToSpatialLayer(test.mimeType, testRid, test.trackInfo, DefaultVideoLayersRid)
 				require.Equal(t, expectedResult.layer, actualLayer)
 
-				actualRid := SpatialLayerToRid(actualLayer, test.trackInfo)
+				actualRid := SpatialLayerToRid(test.mimeType, actualLayer, test.trackInfo, DefaultVideoLayersRid)
 				require.Equal(t, expectedResult.rid, actualRid)
 			}
 		})
@@ -178,11 +224,13 @@ func TestQualityConversion(t *testing.T) {
 	tests := []struct {
 		name           string
 		trackInfo      *livekit.TrackInfo
+		mimeType       mime.MimeType
 		qualityToLayer map[livekit.VideoQuality]QualityAndLayer
 	}{
 		{
 			"no track info",
 			nil,
+			mime.MimeTypeVP8,
 			map[livekit.VideoQuality]QualityAndLayer{
 				livekit.VideoQuality_LOW:    {quality: livekit.VideoQuality_LOW, layer: 0},
 				livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_MEDIUM, layer: 1},
@@ -192,6 +240,7 @@ func TestQualityConversion(t *testing.T) {
 		{
 			"no layers",
 			&livekit.TrackInfo{},
+			mime.MimeTypeVP8,
 			map[livekit.VideoQuality]QualityAndLayer{
 				livekit.VideoQuality_LOW:    {quality: livekit.VideoQuality_LOW, layer: 0},
 				livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_MEDIUM, layer: 1},
@@ -201,10 +250,16 @@ func TestQualityConversion(t *testing.T) {
 		{
 			"single layer, low",
 			&livekit.TrackInfo{
-				Layers: []*livekit.VideoLayer{
-					{Quality: livekit.VideoQuality_LOW},
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_LOW},
+						},
+					},
 				},
 			},
+			mime.MimeTypeVP8,
 			map[livekit.VideoQuality]QualityAndLayer{
 				livekit.VideoQuality_LOW:    {quality: livekit.VideoQuality_LOW, layer: 0},
 				livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_LOW, layer: 0},
@@ -214,10 +269,16 @@ func TestQualityConversion(t *testing.T) {
 		{
 			"single layer, medium",
 			&livekit.TrackInfo{
-				Layers: []*livekit.VideoLayer{
-					{Quality: livekit.VideoQuality_MEDIUM},
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_MEDIUM},
+						},
+					},
 				},
 			},
+			mime.MimeTypeVP8,
 			map[livekit.VideoQuality]QualityAndLayer{
 				livekit.VideoQuality_LOW:    {quality: livekit.VideoQuality_MEDIUM, layer: 0},
 				livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_MEDIUM, layer: 0},
@@ -227,10 +288,16 @@ func TestQualityConversion(t *testing.T) {
 		{
 			"single layer, high",
 			&livekit.TrackInfo{
-				Layers: []*livekit.VideoLayer{
-					{Quality: livekit.VideoQuality_HIGH},
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_HIGH},
+						},
+					},
 				},
 			},
+			mime.MimeTypeVP8,
 			map[livekit.VideoQuality]QualityAndLayer{
 				livekit.VideoQuality_LOW:    {quality: livekit.VideoQuality_HIGH, layer: 0},
 				livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_HIGH, layer: 0},
@@ -240,11 +307,17 @@ func TestQualityConversion(t *testing.T) {
 		{
 			"two layers, low and medium",
 			&livekit.TrackInfo{
-				Layers: []*livekit.VideoLayer{
-					{Quality: livekit.VideoQuality_LOW},
-					{Quality: livekit.VideoQuality_MEDIUM},
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_LOW},
+							{Quality: livekit.VideoQuality_MEDIUM},
+						},
+					},
 				},
 			},
+			mime.MimeTypeVP8,
 			map[livekit.VideoQuality]QualityAndLayer{
 				livekit.VideoQuality_LOW:    {quality: livekit.VideoQuality_LOW, layer: 0},
 				livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_MEDIUM, layer: 1},
@@ -254,11 +327,17 @@ func TestQualityConversion(t *testing.T) {
 		{
 			"two layers, low and high",
 			&livekit.TrackInfo{
-				Layers: []*livekit.VideoLayer{
-					{Quality: livekit.VideoQuality_LOW},
-					{Quality: livekit.VideoQuality_HIGH},
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_LOW},
+							{Quality: livekit.VideoQuality_HIGH},
+						},
+					},
 				},
 			},
+			mime.MimeTypeVP8,
 			map[livekit.VideoQuality]QualityAndLayer{
 				livekit.VideoQuality_LOW:    {quality: livekit.VideoQuality_LOW, layer: 0},
 				livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_HIGH, layer: 1},
@@ -268,11 +347,17 @@ func TestQualityConversion(t *testing.T) {
 		{
 			"two layers, medium and high",
 			&livekit.TrackInfo{
-				Layers: []*livekit.VideoLayer{
-					{Quality: livekit.VideoQuality_MEDIUM},
-					{Quality: livekit.VideoQuality_HIGH},
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_MEDIUM},
+							{Quality: livekit.VideoQuality_HIGH},
+						},
+					},
 				},
 			},
+			mime.MimeTypeVP8,
 			map[livekit.VideoQuality]QualityAndLayer{
 				livekit.VideoQuality_LOW:    {quality: livekit.VideoQuality_MEDIUM, layer: 0},
 				livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_MEDIUM, layer: 0},
@@ -282,12 +367,18 @@ func TestQualityConversion(t *testing.T) {
 		{
 			"three layers",
 			&livekit.TrackInfo{
-				Layers: []*livekit.VideoLayer{
-					{Quality: livekit.VideoQuality_LOW},
-					{Quality: livekit.VideoQuality_MEDIUM},
-					{Quality: livekit.VideoQuality_HIGH},
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_LOW},
+							{Quality: livekit.VideoQuality_MEDIUM},
+							{Quality: livekit.VideoQuality_HIGH},
+						},
+					},
 				},
 			},
+			mime.MimeTypeVP8,
 			map[livekit.VideoQuality]QualityAndLayer{
 				livekit.VideoQuality_LOW:    {quality: livekit.VideoQuality_LOW, layer: 0},
 				livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_MEDIUM, layer: 1},
@@ -299,10 +390,10 @@ func TestQualityConversion(t *testing.T) {
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
 			for testQuality, expectedResult := range test.qualityToLayer {
-				actualLayer := VideoQualityToSpatialLayer(testQuality, test.trackInfo)
+				actualLayer := VideoQualityToSpatialLayer(test.mimeType, testQuality, test.trackInfo)
 				require.Equal(t, expectedResult.layer, actualLayer)
 
-				actualQuality := SpatialLayerToVideoQuality(actualLayer, test.trackInfo)
+				actualQuality := SpatialLayerToVideoQuality(test.mimeType, actualLayer, test.trackInfo)
 				require.Equal(t, expectedResult.quality, actualQuality)
 			}
 		})
@@ -313,120 +404,165 @@ func TestVideoQualityToRidConversion(t *testing.T) {
 	tests := []struct {
 		name         string
 		trackInfo    *livekit.TrackInfo
+		mimeTye      mime.MimeType
 		qualityToRid map[livekit.VideoQuality]string
 	}{
 		{
 			"no track info",
 			nil,
+			mime.MimeTypeVP8,
 			map[livekit.VideoQuality]string{
-				livekit.VideoQuality_LOW:    QuarterResolution,
-				livekit.VideoQuality_MEDIUM: HalfResolution,
-				livekit.VideoQuality_HIGH:   FullResolution,
+				livekit.VideoQuality_LOW:    quarterResolutionQ,
+				livekit.VideoQuality_MEDIUM: halfResolutionH,
+				livekit.VideoQuality_HIGH:   fullResolutionF,
 			},
 		},
 		{
 			"no layers",
 			&livekit.TrackInfo{},
+			mime.MimeTypeVP8,
 			map[livekit.VideoQuality]string{
-				livekit.VideoQuality_LOW:    QuarterResolution,
-				livekit.VideoQuality_MEDIUM: HalfResolution,
-				livekit.VideoQuality_HIGH:   FullResolution,
+				livekit.VideoQuality_LOW:    quarterResolutionQ,
+				livekit.VideoQuality_MEDIUM: halfResolutionH,
+				livekit.VideoQuality_HIGH:   fullResolutionF,
 			},
 		},
 		{
 			"single layer, low",
 			&livekit.TrackInfo{
-				Layers: []*livekit.VideoLayer{
-					{Quality: livekit.VideoQuality_LOW},
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_LOW},
+						},
+					},
 				},
 			},
+			mime.MimeTypeVP8,
 			map[livekit.VideoQuality]string{
-				livekit.VideoQuality_LOW:    QuarterResolution,
-				livekit.VideoQuality_MEDIUM: QuarterResolution,
-				livekit.VideoQuality_HIGH:   QuarterResolution,
+				livekit.VideoQuality_LOW:    quarterResolutionQ,
+				livekit.VideoQuality_MEDIUM: quarterResolutionQ,
+				livekit.VideoQuality_HIGH:   quarterResolutionQ,
 			},
 		},
 		{
 			"single layer, medium",
 			&livekit.TrackInfo{
-				Layers: []*livekit.VideoLayer{
-					{Quality: livekit.VideoQuality_MEDIUM},
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_MEDIUM},
+						},
+					},
 				},
 			},
+			mime.MimeTypeVP8,
 			map[livekit.VideoQuality]string{
-				livekit.VideoQuality_LOW:    QuarterResolution,
-				livekit.VideoQuality_MEDIUM: QuarterResolution,
-				livekit.VideoQuality_HIGH:   QuarterResolution,
+				livekit.VideoQuality_LOW:    quarterResolutionQ,
+				livekit.VideoQuality_MEDIUM: quarterResolutionQ,
+				livekit.VideoQuality_HIGH:   quarterResolutionQ,
 			},
 		},
 		{
 			"single layer, high",
 			&livekit.TrackInfo{
-				Layers: []*livekit.VideoLayer{
-					{Quality: livekit.VideoQuality_HIGH},
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_HIGH},
+						},
+					},
 				},
 			},
+			mime.MimeTypeVP8,
 			map[livekit.VideoQuality]string{
-				livekit.VideoQuality_LOW:    QuarterResolution,
-				livekit.VideoQuality_MEDIUM: QuarterResolution,
-				livekit.VideoQuality_HIGH:   QuarterResolution,
+				livekit.VideoQuality_LOW:    quarterResolutionQ,
+				livekit.VideoQuality_MEDIUM: quarterResolutionQ,
+				livekit.VideoQuality_HIGH:   quarterResolutionQ,
 			},
 		},
 		{
 			"two layers, low and medium",
 			&livekit.TrackInfo{
-				Layers: []*livekit.VideoLayer{
-					{Quality: livekit.VideoQuality_LOW},
-					{Quality: livekit.VideoQuality_MEDIUM},
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_LOW},
+							{Quality: livekit.VideoQuality_MEDIUM},
+						},
+					},
 				},
 			},
+			mime.MimeTypeVP8,
 			map[livekit.VideoQuality]string{
-				livekit.VideoQuality_LOW:    QuarterResolution,
-				livekit.VideoQuality_MEDIUM: HalfResolution,
-				livekit.VideoQuality_HIGH:   HalfResolution,
+				livekit.VideoQuality_LOW:    quarterResolutionQ,
+				livekit.VideoQuality_MEDIUM: halfResolutionH,
+				livekit.VideoQuality_HIGH:   halfResolutionH,
 			},
 		},
 		{
 			"two layers, low and high",
 			&livekit.TrackInfo{
-				Layers: []*livekit.VideoLayer{
-					{Quality: livekit.VideoQuality_LOW},
-					{Quality: livekit.VideoQuality_HIGH},
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_LOW},
+							{Quality: livekit.VideoQuality_HIGH},
+						},
+					},
 				},
 			},
+			mime.MimeTypeVP8,
 			map[livekit.VideoQuality]string{
-				livekit.VideoQuality_LOW:    QuarterResolution,
-				livekit.VideoQuality_MEDIUM: HalfResolution,
-				livekit.VideoQuality_HIGH:   HalfResolution,
+				livekit.VideoQuality_LOW:    quarterResolutionQ,
+				livekit.VideoQuality_MEDIUM: halfResolutionH,
+				livekit.VideoQuality_HIGH:   halfResolutionH,
 			},
 		},
 		{
 			"two layers, medium and high",
 			&livekit.TrackInfo{
-				Layers: []*livekit.VideoLayer{
-					{Quality: livekit.VideoQuality_MEDIUM},
-					{Quality: livekit.VideoQuality_HIGH},
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_MEDIUM},
+							{Quality: livekit.VideoQuality_HIGH},
+						},
+					},
 				},
 			},
+			mime.MimeTypeVP8,
 			map[livekit.VideoQuality]string{
-				livekit.VideoQuality_LOW:    QuarterResolution,
-				livekit.VideoQuality_MEDIUM: QuarterResolution,
-				livekit.VideoQuality_HIGH:   HalfResolution,
+				livekit.VideoQuality_LOW:    quarterResolutionQ,
+				livekit.VideoQuality_MEDIUM: quarterResolutionQ,
+				livekit.VideoQuality_HIGH:   halfResolutionH,
 			},
 		},
 		{
 			"three layers",
 			&livekit.TrackInfo{
-				Layers: []*livekit.VideoLayer{
-					{Quality: livekit.VideoQuality_LOW},
-					{Quality: livekit.VideoQuality_MEDIUM},
-					{Quality: livekit.VideoQuality_HIGH},
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_LOW},
+							{Quality: livekit.VideoQuality_MEDIUM},
+							{Quality: livekit.VideoQuality_HIGH},
+						},
+					},
 				},
 			},
+			mime.MimeTypeVP8,
 			map[livekit.VideoQuality]string{
-				livekit.VideoQuality_LOW:    QuarterResolution,
-				livekit.VideoQuality_MEDIUM: HalfResolution,
-				livekit.VideoQuality_HIGH:   FullResolution,
+				livekit.VideoQuality_LOW:    quarterResolutionQ,
+				livekit.VideoQuality_MEDIUM: halfResolutionH,
+				livekit.VideoQuality_HIGH:   fullResolutionF,
 			},
 		},
 	}
@@ -434,9 +570,323 @@ func TestVideoQualityToRidConversion(t *testing.T) {
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
 			for testQuality, expectedRid := range test.qualityToRid {
-				actualRid := VideoQualityToRid(testQuality, test.trackInfo)
+				actualRid := VideoQualityToRid(test.mimeTye, testQuality, test.trackInfo, DefaultVideoLayersRid)
 				require.Equal(t, expectedRid, actualRid)
 			}
 		})
 	}
 }
+
+func TestGetSpatialLayerForRid(t *testing.T) {
+	tests := []struct {
+		name              string
+		trackInfo         *livekit.TrackInfo
+		mimeType          mime.MimeType
+		ridToSpatialLayer map[string]int32
+	}{
+		{
+			"no track info",
+			nil,
+			mime.MimeTypeVP8,
+			map[string]int32{
+				quarterResolutionQ: InvalidLayerSpatial,
+				halfResolutionH:    InvalidLayerSpatial,
+				fullResolutionF:    InvalidLayerSpatial,
+			},
+		},
+		{
+			"no layers",
+			&livekit.TrackInfo{},
+			mime.MimeTypeVP8,
+			map[string]int32{
+				// SIMULCAST-CODEC-TODO
+				// quarterResolutionQ: InvalidLayerSpatial,
+				// halfResolutionH:    InvalidLayerSpatial,
+				// fullResolutionF:    InvalidLayerSpatial,
+				quarterResolutionQ: 0,
+				halfResolutionH:    0,
+				fullResolutionF:    0,
+			},
+		},
+		{
+			"no rid",
+			&livekit.TrackInfo{},
+			mime.MimeTypeVP8,
+			map[string]int32{
+				"": 0,
+			},
+		},
+		{
+			"single layer",
+			&livekit.TrackInfo{
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_LOW, SpatialLayer: 0},
+						},
+					},
+				},
+			},
+			mime.MimeTypeVP8,
+			map[string]int32{
+				quarterResolutionQ: 0,
+				halfResolutionH:    0,
+				fullResolutionF:    0,
+			},
+		},
+		{
+			"layers",
+			&livekit.TrackInfo{
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_LOW, SpatialLayer: 0, Rid: quarterResolutionQ},
+							{Quality: livekit.VideoQuality_MEDIUM, SpatialLayer: 1, Rid: halfResolutionH},
+						},
+					},
+				},
+			},
+			mime.MimeTypeVP8,
+			map[string]int32{
+				quarterResolutionQ: 0,
+				halfResolutionH:    1,
+				// SIMULCAST-CODEC-TODO
+				// fullResolutionF:    InvalidLayerSpatial,
+				fullResolutionF: 0,
+			},
+		},
+		{
+			"layers - no rid",
+			&livekit.TrackInfo{
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_LOW, SpatialLayer: 0},
+							{Quality: livekit.VideoQuality_MEDIUM, SpatialLayer: 1},
+						},
+					},
+				},
+			},
+			mime.MimeTypeVP8,
+			map[string]int32{
+				quarterResolutionQ: 0,
+				halfResolutionH:    0,
+				fullResolutionF:    0,
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			for testRid, expectedSpatialLayer := range test.ridToSpatialLayer {
+				actualSpatialLayer := GetSpatialLayerForRid(test.mimeType, testRid, test.trackInfo)
+				require.Equal(t, expectedSpatialLayer, actualSpatialLayer)
+			}
+		})
+	}
+}
+
+func TestGetSpatialLayerForVideoQuality(t *testing.T) {
+	tests := []struct {
+		name                       string
+		trackInfo                  *livekit.TrackInfo
+		mimeType                   mime.MimeType
+		videoQualityToSpatialLayer map[livekit.VideoQuality]int32
+	}{
+		{
+			"no track info",
+			nil,
+			mime.MimeTypeVP8,
+			map[livekit.VideoQuality]int32{
+				livekit.VideoQuality_LOW:    InvalidLayerSpatial,
+				livekit.VideoQuality_MEDIUM: InvalidLayerSpatial,
+				livekit.VideoQuality_HIGH:   InvalidLayerSpatial,
+				livekit.VideoQuality_OFF:    InvalidLayerSpatial,
+			},
+		},
+		{
+			"no layers",
+			&livekit.TrackInfo{},
+			mime.MimeTypeVP8,
+			map[livekit.VideoQuality]int32{
+				livekit.VideoQuality_LOW:    0,
+				livekit.VideoQuality_MEDIUM: 0,
+				livekit.VideoQuality_HIGH:   0,
+				livekit.VideoQuality_OFF:    InvalidLayerSpatial,
+			},
+		},
+		{
+			"not all layers",
+			&livekit.TrackInfo{
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_LOW, SpatialLayer: 0, Rid: quarterResolutionQ},
+							{Quality: livekit.VideoQuality_MEDIUM, SpatialLayer: 1, Rid: halfResolutionH},
+						},
+					},
+				},
+			},
+			mime.MimeTypeVP8,
+			map[livekit.VideoQuality]int32{
+				livekit.VideoQuality_LOW:    0,
+				livekit.VideoQuality_MEDIUM: 1,
+				livekit.VideoQuality_HIGH:   1,
+				livekit.VideoQuality_OFF:    InvalidLayerSpatial,
+			},
+		},
+		{
+			"all layers",
+			&livekit.TrackInfo{
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_LOW, SpatialLayer: 0, Rid: quarterResolutionQ},
+							{Quality: livekit.VideoQuality_MEDIUM, SpatialLayer: 1, Rid: halfResolutionH},
+							{Quality: livekit.VideoQuality_HIGH, SpatialLayer: 2, Rid: fullResolutionF},
+						},
+					},
+				},
+			},
+			mime.MimeTypeVP8,
+			map[livekit.VideoQuality]int32{
+				livekit.VideoQuality_LOW:    0,
+				livekit.VideoQuality_MEDIUM: 1,
+				livekit.VideoQuality_HIGH:   2,
+				livekit.VideoQuality_OFF:    InvalidLayerSpatial,
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			for testVideoQuality, expectedSpatialLayer := range test.videoQualityToSpatialLayer {
+				actualSpatialLayer := GetSpatialLayerForVideoQuality(test.mimeType, testVideoQuality, test.trackInfo)
+				require.Equal(t, expectedSpatialLayer, actualSpatialLayer)
+			}
+		})
+	}
+}
+
+func TestGetVideoQualityorSpatialLayer(t *testing.T) {
+	tests := []struct {
+		name                       string
+		trackInfo                  *livekit.TrackInfo
+		mimeType                   mime.MimeType
+		spatialLayerToVideoQuality map[int32]livekit.VideoQuality
+	}{
+		{
+			"no track info",
+			nil,
+			mime.MimeTypeVP8,
+			map[int32]livekit.VideoQuality{
+				InvalidLayerSpatial: livekit.VideoQuality_OFF,
+				0:                   livekit.VideoQuality_OFF,
+				1:                   livekit.VideoQuality_OFF,
+				2:                   livekit.VideoQuality_OFF,
+			},
+		},
+		{
+			"no layers",
+			&livekit.TrackInfo{},
+			mime.MimeTypeVP8,
+			map[int32]livekit.VideoQuality{
+				InvalidLayerSpatial: livekit.VideoQuality_OFF,
+				0:                   livekit.VideoQuality_OFF,
+				1:                   livekit.VideoQuality_OFF,
+				2:                   livekit.VideoQuality_OFF,
+			},
+		},
+		{
+			"layers",
+			&livekit.TrackInfo{
+				Codecs: []*livekit.SimulcastCodecInfo{
+					{
+						MimeType: mime.MimeTypeVP8.String(),
+						Layers: []*livekit.VideoLayer{
+							{Quality: livekit.VideoQuality_LOW, SpatialLayer: 0, Rid: quarterResolutionQ},
+							{Quality: livekit.VideoQuality_MEDIUM, SpatialLayer: 1, Rid: halfResolutionH},
+						},
+					},
+				},
+			},
+			mime.MimeTypeVP8,
+			map[int32]livekit.VideoQuality{
+				InvalidLayerSpatial: livekit.VideoQuality_OFF,
+				0:                   livekit.VideoQuality_LOW,
+				1:                   livekit.VideoQuality_MEDIUM,
+				2:                   livekit.VideoQuality_OFF,
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			for testSpatialLayer, expectedVideoQuality := range test.spatialLayerToVideoQuality {
+				actualVideoQuality := GetVideoQualityForSpatialLayer(test.mimeType, testSpatialLayer, test.trackInfo)
+				require.Equal(t, expectedVideoQuality, actualVideoQuality)
+			}
+		})
+	}
+}
+
+func TestNormalizeVideoLayersRid(t *testing.T) {
+	tests := []struct {
+		name       string
+		rids       VideoLayersRid
+		normalized VideoLayersRid
+	}{
+		{
+			"empty",
+			VideoLayersRid{},
+			VideoLayersRid{},
+		},
+		{
+			"unknown pattern",
+			VideoLayersRid{"3", "2", "1"},
+			VideoLayersRid{"3", "2", "1"},
+		},
+		{
+			"qhf",
+			videoLayersRidQHF,
+			videoLayersRidQHF,
+		},
+		{
+			"scrambled qhf",
+			VideoLayersRid{"f", "h", "q"},
+			videoLayersRidQHF,
+		},
+		{
+			"partial qhf",
+			VideoLayersRid{"h", "q"},
+			VideoLayersRid{"q", "h", ""},
+		},
+		{
+			"210",
+			videoLayersRid210,
+			videoLayersRid210,
+		},
+		{
+			"scrambled 210",
+			VideoLayersRid{"2", "0", "1"},
+			videoLayersRid210,
+		},
+		{
+			"partial 210",
+			VideoLayersRid{"1", "2"},
+			VideoLayersRid{"2", "1", ""},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			normalizedRids := NormalizeVideoLayersRid(test.rids)
+			require.Equal(t, test.normalized, normalizedRids)
+		})
+	}
+}

livekit-server/pkg/sfu/bwe/bwe.go

@@ -31,6 +31,29 @@ const (
 
 // ------------------------------------------------
 
+type BWEType int
+
+const (
+	BWETypeNone BWEType = iota
+	BWETypeRemote
+	BWETypeSendSide
+)
+
+func (b BWEType) String() string {
+	switch b {
+	case BWETypeNone:
+		return "NONE"
+	case BWETypeRemote:
+		return "REMOTE"
+	case BWETypeSendSide:
+		return "SEND_SIDE"
+	default:
+		return fmt.Sprintf("%d", int(b))
+	}
+}
+
+// ------------------------------------------------
+
 type CongestionState int
 
 const (
@@ -55,6 +78,8 @@ func (c CongestionState) String() string {
 // ------------------------------------------------
 
 type BWE interface {
+	Type() BWEType
+
 	SetBWEListener(bweListner BWEListener)
 
 	Reset()

livekit-server/pkg/sfu/bwe/remotebwe/remote_bwe.go

@@ -24,6 +24,8 @@ import (
 	"github.com/livekit/protocol/utils/mono"
 )
 
+var _ bwe.BWE = (*RemoteBWE)(nil)
+
 // ---------------------------------------------------------------------------
 
 type RemoteBWEConfig struct {
@@ -81,6 +83,10 @@ func NewRemoteBWE(params RemoteBWEParams) *RemoteBWE {
 	return r
 }
 
+func (r *RemoteBWE) Type() bwe.BWEType {
+	return bwe.BWETypeRemote
+}
+
 func (r *RemoteBWE) SetBWEListener(bweListener bwe.BWEListener) {
 	r.lock.Lock()
 	defer r.lock.Unlock()

livekit-server/pkg/sfu/bwe/sendsidebwe/congestion_detector.go

@@ -121,8 +121,8 @@ var (
 		MinBytesRatio:    0.5,
 		MinDurationRatio: 0.5,
 
-		JQRMinDelay: 40 * time.Millisecond,
-		DQRMaxDelay: 15 * time.Millisecond,
+		JQRMinDelay: 50 * time.Millisecond,
+		DQRMaxDelay: 20 * time.Millisecond,
 
 		WeightedLoss:       defaultWeightedLossConfig,
 		JQRMinWeightedLoss: 0.25,
@@ -177,17 +177,20 @@ func (c congestionReason) String() string {
 }
 
 // -------------------------------------------------------------------------------
-type qdMeasurement struct {
-	jqrConfig CongestionSignalConfig
-	dqrConfig CongestionSignalConfig
-	jqrMin    int64
-	dqrMax    int64
 
-	numGroups    int
-	numJQRGroups int
-	numDQRGroups int
-	minSendTime  int64
-	maxSendTime  int64
+type qdMeasurement struct {
+	jqrConfig              CongestionSignalConfig
+	dqrConfig              CongestionSignalConfig
+	jqrMinDelay            int64
+	jqrMinTrendCoefficient float64
+	dqrMaxDelay            int64
+
+	numGroups               int
+	numJQRGroups            int
+	numDQRGroups            int
+	minSendTime             int64
+	maxSendTime             int64
+	propagatedQueuingDelays []int64
 
 	isSealed    bool
 	minGroupIdx int
@@ -196,13 +199,14 @@ type qdMeasurement struct {
 	queuingRegion queuingRegion
 }
 
-func newQDMeasurement(jqrConfig CongestionSignalConfig, dqrConfig CongestionSignalConfig, jqrMin int64, dqrMax int64) *qdMeasurement {
+func newQDMeasurement(jqrConfig CongestionSignalConfig, dqrConfig CongestionSignalConfig, jqrMinDelay int64, jqrMinTrendCoefficient float64, dqrMaxDelay int64) *qdMeasurement {
 	return &qdMeasurement{
-		jqrConfig:     jqrConfig,
-		dqrConfig:     dqrConfig,
-		jqrMin:        jqrMin,
-		dqrMax:        dqrMax,
-		queuingRegion: queuingRegionIndeterminate,
+		jqrConfig:              jqrConfig,
+		dqrConfig:              dqrConfig,
+		jqrMinDelay:            jqrMinDelay,
+		jqrMinTrendCoefficient: jqrMinTrendCoefficient,
+		dqrMaxDelay:            dqrMaxDelay,
+		queuingRegion:          queuingRegionIndeterminate,
 	}
 }
 
@@ -228,8 +232,10 @@ func (q *qdMeasurement) ProcessPacketGroup(pg *packetGroup, groupIdx int) {
 	}
 	q.maxSendTime = max(q.maxSendTime, maxSendTime)
 
+	q.propagatedQueuingDelays = append(q.propagatedQueuingDelays, pqd)
+
 	switch {
-	case pqd < q.dqrMax:
+	case pqd < q.dqrMaxDelay:
 		q.numDQRGroups++
 		if q.numJQRGroups > 0 {
 			// broken continuity, seal
@@ -239,12 +245,12 @@ func (q *qdMeasurement) ProcessPacketGroup(pg *packetGroup, groupIdx int) {
 			q.queuingRegion = queuingRegionDQR
 		}
 
-	case pqd > q.jqrMin:
+	case pqd > q.jqrMinDelay:
 		q.numJQRGroups++
 		if q.numDQRGroups > 0 {
 			// broken continuity, seal
 			q.isSealed = true
-		} else if q.jqrConfig.IsTriggered(q.numJQRGroups, q.maxSendTime-q.minSendTime) {
+		} else if q.jqrConfig.IsTriggered(q.numJQRGroups, q.maxSendTime-q.minSendTime) && q.trendCoefficient() > q.jqrMinTrendCoefficient {
 			q.isSealed = true
 			q.queuingRegion = queuingRegionJQR
 		}
@@ -279,6 +285,8 @@ func (q *qdMeasurement) MarshalLogObject(e zapcore.ObjectEncoder) error {
 	e.AddInt("numDQRGroups", q.numDQRGroups)
 	e.AddInt64("minSendTime", q.minSendTime)
 	e.AddInt64("maxSendTime", q.maxSendTime)
+	e.AddArray("propagatedQueuingDelays", logger.Int64Slice(q.propagatedQueuingDelays))
+	e.AddFloat64("trendCoefficient", q.trendCoefficient())
 	e.AddDuration("duration", time.Duration((q.maxSendTime-q.minSendTime)*1000))
 	e.AddBool("isSealed", q.isSealed)
 	e.AddInt("minGroupIdx", q.minGroupIdx)
@@ -287,6 +295,35 @@ func (q *qdMeasurement) MarshalLogObject(e zapcore.ObjectEncoder) error {
 	return nil
 }
 
+func (q *qdMeasurement) trendCoefficient() float64 {
+	concordantPairs := 0
+	discordantPairs := 0
+
+	// the packet groups are processed from newest to oldest,
+	// so a concordant pair is when the value drops,
+	// i. e. the propagated queuing delay is increasing if newer (earlier entity in slice) is higher than older (later entity in slice)
+	for i := 0; i < len(q.propagatedQueuingDelays)-1; i++ {
+		for j := i + 1; j < len(q.propagatedQueuingDelays); j++ {
+			if q.propagatedQueuingDelays[i] > q.propagatedQueuingDelays[j] {
+				concordantPairs++
+			} else if q.propagatedQueuingDelays[i] < q.propagatedQueuingDelays[j] {
+				discordantPairs++
+			}
+		}
+	}
+
+	if (concordantPairs + discordantPairs) == 0 {
+		// if the min requirements is only one sample, trend calculation is not possible, declare highest trend value
+		if len(q.propagatedQueuingDelays) == 1 {
+			return 1.0
+		}
+
+		return 0.0
+	}
+
+	return (float64(concordantPairs) - float64(discordantPairs)) / (float64(concordantPairs) + float64(discordantPairs))
+}
+
 // -------------------------------------------------------------------------------
 
 type lossMeasurement struct {
@@ -405,8 +442,9 @@ type CongestionDetectorConfig struct {
 	ProbeRegulator   ccutils.ProbeRegulatorConfig `yaml:"probe_regulator,omitempty"`
 	ProbeSignal      ProbeSignalConfig            `yaml:"probe_signal,omitempty"`
 
-	JQRMinDelay time.Duration `yaml:"jqr_min_delay,omitempty"`
-	DQRMaxDelay time.Duration `yaml:"dqr_max_delay,omitempty"`
+	JQRMinDelay            time.Duration `yaml:"jqr_min_delay,omitempty"`
+	JQRMinTrendCoefficient float64       `yaml:"jqr_min_trend_coefficient,omitempty"`
+	DQRMaxDelay            time.Duration `yaml:"dqr_max_delay,omitempty"`
 
 	WeightedLoss       WeightedLossConfig `yaml:"weighted_loss,omitempty"`
 	JQRMinWeightedLoss float64            `yaml:"jqr_min_weighted_loss,omitempty"`
@@ -452,8 +490,9 @@ var (
 		ProbeRegulator:   ccutils.DefaultProbeRegulatorConfig,
 		ProbeSignal:      defaultProbeSignalConfig,
 
-		JQRMinDelay: 40 * time.Millisecond,
-		DQRMaxDelay: 15 * time.Millisecond,
+		JQRMinDelay:            50 * time.Millisecond,
+		JQRMinTrendCoefficient: 0.8,
+		DQRMaxDelay:            20 * time.Millisecond,
 
 		WeightedLoss:       defaultWeightedLossConfig,
 		JQRMinWeightedLoss: 0.25,
@@ -570,10 +609,7 @@ func (c *congestionDetector) getBWEListener() bwe.BWEListener {
 
 func (c *congestionDetector) HandleTWCCFeedback(report *rtcp.TransportLayerCC) {
 	c.lock.Lock()
-	recvRefTime, isOutOfOrder := c.twccFeedback.ProcessReport(report, mono.Now())
-	if isOutOfOrder {
-		c.params.Logger.Infow("send side bwe: received out-of-order feedback report")
-	}
+	recvRefTime, _ := c.twccFeedback.ProcessReport(report, mono.Now())
 
 	if len(c.packetGroups) == 0 {
 		c.packetGroups = append(
@@ -635,7 +671,7 @@ func (c *congestionDetector) HandleTWCCFeedback(report *rtcp.TransportLayerCC) {
 		}
 	}
 
-	// 1. go through the TWCC feedback report and record recive time as reported by remote
+	// 1. go through the TWCC feedback report and record receive time as reported by remote
 	// 2. process acknowledged packet and group them
 	//
 	// losses are not recorded if a feedback report is completely lost.
@@ -852,6 +888,7 @@ func (c *congestionDetector) updateCongestionSignal(
 		qdJQRConfig,
 		qdDQRConfig,
 		c.params.Config.JQRMinDelay.Microseconds(),
+		c.params.Config.JQRMinTrendCoefficient,
 		c.params.Config.DQRMaxDelay.Microseconds(),
 	)
 	c.lossMeasurement = newLossMeasurement(
@@ -1109,7 +1146,7 @@ func (c *congestionDetector) updateCongestionState(state bwe.CongestionState) (b
 	// when in congested state, monitor changes in captured traffic ratio (CTR)
 	// to ensure allocations are in line with latest estimates, it is possible that
 	// the estimate is incorrect when congestion starts and the allocation may be
-	// sub-optimal and not enough to reduce/relieve congestion, by monitoing CTR
+	// sub-optimal and not enough to reduce/relieve congestion, by monitoring CTR
 	// on a continuous basis allocations can be adjusted in the direction of
 	// reducing/relieving congestion
 	if state == bwe.CongestionStateCongested && fromState != bwe.CongestionStateCongested {

livekit-server/pkg/sfu/bwe/sendsidebwe/send_side_bwe.go

@@ -23,6 +23,8 @@ import (
 	"github.com/pion/rtcp"
 )
 
+var _ bwe.BWE = (*SendSideBWE)(nil)
+
 //
 // Based on a simplified/modified version of JitterPath paper
 // (https://homepage.iis.sinica.edu.tw/papers/lcs/2114-F.pdf)
@@ -88,6 +90,10 @@ func NewSendSideBWE(params SendSideBWEParams) *SendSideBWE {
 	}
 }
 
+func (s *SendSideBWE) Type() bwe.BWEType {
+	return bwe.BWETypeSendSide
+}
+
 func (s *SendSideBWE) SetBWEListener(bweListener bwe.BWEListener) {
 	s.congestionDetector.SetBWEListener(bweListener)
 }

livekit-server/pkg/sfu/bwe/sendsidebwe/twcc_feedback.go

@@ -15,7 +15,6 @@
 package sendsidebwe
 
 import (
-	"errors"
 	"time"
 
 	"github.com/livekit/protocol/logger"
@@ -35,12 +34,6 @@ const (
 
 // ------------------------------------------------------
 
-var (
-	errFeedbackReportOutOfOrder = errors.New("feedback report out-of-order")
-)
-
-// ------------------------------------------------------
-
 type twccFeedbackParams struct {
 	Logger logger.Logger
 }
@@ -78,6 +71,12 @@ func (t *twccFeedback) ProcessReport(report *rtcp.TransportLayerCC, at time.Time
 	if (report.FbPktCount - t.highestFeedbackCount) > (1 << 7) {
 		t.numReportsOutOfOrder++
 		isOutOfOrder = true
+		t.params.Logger.Infow(
+			"send side bwe: received out-of-order feedback report",
+			"highestFeedbackCount", t.highestFeedbackCount,
+			"fbPktCount", report.FbPktCount,
+			"numReportsOutOfOrder", t.numReportsOutOfOrder,
+		)
 	}
 
 	// reference time wrap around handling

livekit-server/pkg/sfu/codecmunger/codecmunger.go

@@ -27,8 +27,8 @@ var (
 )
 
 type CodecMunger interface {
-	GetState() interface{}
-	SeedState(state interface{})
+	GetState() any
+	SeedState(state any)
 
 	SetLast(extPkt *buffer.ExtPacket)
 	UpdateOffsets(extPkt *buffer.ExtPacket)

livekit-server/pkg/sfu/codecmunger/null.go

@@ -20,22 +20,22 @@ import (
 )
 
 type Null struct {
-	seededState interface{}
+	seededState any
 }
 
 func NewNull(_logger logger.Logger) *Null {
 	return &Null{}
 }
 
-func (n *Null) GetState() interface{} {
+func (n *Null) GetState() any {
 	return nil
 }
 
-func (n *Null) SeedState(state interface{}) {
+func (n *Null) SeedState(state any) {
 	n.seededState = state
 }
 
-func (n *Null) GetSeededState() interface{} {
+func (n *Null) GetSeededState() any {
 	return n.seededState
 }
 

livekit-server/pkg/sfu/codecmunger/vp8.go

@@ -15,8 +15,9 @@
 package codecmunger
 
 import (
-	"github.com/elliotchance/orderedmap/v2"
+	"github.com/elliotchance/orderedmap/v3"
 
+	"github.com/livekit/mediatransportutil/pkg/codec"
 	"github.com/livekit/protocol/livekit"
 	"github.com/livekit/protocol/logger"
 
@@ -60,13 +61,18 @@ func NewVP8(logger logger.Logger) *VP8 {
 	}
 }
 
-func NewVP8FromNull(cm CodecMunger, logger logger.Logger) *VP8 {
+func NewVP8FromOther(cm CodecMunger, logger logger.Logger) *VP8 {
 	v := NewVP8(logger)
-	v.SeedState(cm.(*Null).GetSeededState())
+	switch cm := cm.(type) {
+	case *Null:
+		v.SeedState(cm.GetSeededState())
+	case *VP8:
+		v.SeedState(cm.GetState())
+	}
 	return v
 }
 
-func (v *VP8) GetState() interface{} {
+func (v *VP8) GetState() any {
 	return &livekit.VP8MungerState{
 		ExtLastPictureId: v.extLastPictureId,
 		PictureIdUsed:    v.pictureIdUsed,
@@ -78,10 +84,15 @@ func (v *VP8) GetState() interface{} {
 	}
 }
 
-func (v *VP8) SeedState(seed interface{}) {
+func (v *VP8) SeedState(seed any) {
+	var state *livekit.VP8MungerState
 	switch cm := seed.(type) {
 	case *livekit.RTPForwarderState_Vp8Munger:
-		state := cm.Vp8Munger
+		state = cm.Vp8Munger
+	case *livekit.VP8MungerState:
+		state = cm
+	}
+	if state != nil {
 		v.extLastPictureId = state.ExtLastPictureId
 		v.pictureIdUsed = state.PictureIdUsed
 		v.lastTl0PicIdx = uint8(state.LastTl0PicIdx)
@@ -93,7 +104,7 @@ func (v *VP8) SeedState(seed interface{}) {
 }
 
 func (v *VP8) SetLast(extPkt *buffer.ExtPacket) {
-	vp8, ok := extPkt.Payload.(buffer.VP8)
+	vp8, ok := extPkt.Payload.(codec.VP8)
 	if !ok {
 		return
 	}
@@ -118,7 +129,7 @@ func (v *VP8) SetLast(extPkt *buffer.ExtPacket) {
 }
 
 func (v *VP8) UpdateOffsets(extPkt *buffer.ExtPacket) {
-	vp8, ok := extPkt.Payload.(buffer.VP8)
+	vp8, ok := extPkt.Payload.(codec.VP8)
 	if !ok {
 		return
 	}
@@ -143,7 +154,7 @@ func (v *VP8) UpdateOffsets(extPkt *buffer.ExtPacket) {
 }
 
 func (v *VP8) UpdateAndGet(extPkt *buffer.ExtPacket, snOutOfOrder bool, snHasGap bool, maxTemporalLayer int32) (int, []byte, error) {
-	vp8, ok := extPkt.Payload.(buffer.VP8)
+	vp8, ok := extPkt.Payload.(codec.VP8)
 	if !ok {
 		return 0, nil, ErrNotVP8
 	}
@@ -164,7 +175,7 @@ func (v *VP8) UpdateAndGet(extPkt *buffer.ExtPacket, snOutOfOrder bool, snHasGap
 		// when it reaches a certain size.
 
 		mungedPictureId := uint16((extPictureId - pictureIdOffset) & 0x7fff)
-		vp8Packet := &buffer.VP8{
+		vp8Packet := codec.VP8{
 			FirstByte:  vp8.FirstByte,
 			I:          vp8.I,
 			M:          mungedPictureId > 127,
@@ -177,7 +188,7 @@ func (v *VP8) UpdateAndGet(extPkt *buffer.ExtPacket, snOutOfOrder bool, snHasGap
 			K:          vp8.K,
 			KEYIDX:     vp8.KEYIDX - v.keyIdxOffset,
 			IsKeyFrame: vp8.IsKeyFrame,
-			HeaderSize: vp8.HeaderSize + buffer.VPxPictureIdSizeDiff(mungedPictureId > 127, vp8.M),
+			HeaderSize: vp8.HeaderSize + codec.VPxPictureIdSizeDiff(mungedPictureId > 127, vp8.M),
 		}
 		vp8HeaderBytes, err := vp8Packet.Marshal()
 		if err != nil {
@@ -271,7 +282,7 @@ func (v *VP8) UpdateAndGet(extPkt *buffer.ExtPacket, snOutOfOrder bool, snHasGap
 	v.lastTl0PicIdx = mungedTl0PicIdx
 	v.lastKeyIdx = mungedKeyIdx
 
-	vp8Packet := &buffer.VP8{
+	vp8Packet := codec.VP8{
 		FirstByte:  vp8.FirstByte,
 		I:          vp8.I,
 		M:          mungedPictureId > 127,
@@ -284,7 +295,7 @@ func (v *VP8) UpdateAndGet(extPkt *buffer.ExtPacket, snOutOfOrder bool, snHasGap
 		K:          vp8.K,
 		KEYIDX:     mungedKeyIdx,
 		IsKeyFrame: vp8.IsKeyFrame,
-		HeaderSize: vp8.HeaderSize + buffer.VPxPictureIdSizeDiff(mungedPictureId > 127, vp8.M),
+		HeaderSize: vp8.HeaderSize + codec.VPxPictureIdSizeDiff(mungedPictureId > 127, vp8.M),
 	}
 	vp8HeaderBytes, err := vp8Packet.Marshal()
 	if err != nil {
@@ -336,7 +347,7 @@ func (v *VP8) UpdateAndGetPadding(newPicture bool) ([]byte, error) {
 		v.keyIdxOffset -= uint8(offset)
 	}
 
-	vp8Packet := &buffer.VP8{
+	vp8Packet := &codec.VP8{
 		FirstByte:  0x10, // partition 0, start of VP8 Partition, reference frame
 		I:          v.pictureIdUsed,
 		M:          pictureId > 127,

livekit-server/pkg/sfu/codecmunger/vp8_test.go

@@ -20,9 +20,9 @@ import (
 
 	"github.com/stretchr/testify/require"
 
+	"github.com/livekit/mediatransportutil/pkg/codec"
 	"github.com/livekit/protocol/logger"
 
-	"github.com/livekit/livekit-server/pkg/sfu/buffer"
 	"github.com/livekit/livekit-server/pkg/sfu/testutils"
 )
 
@@ -52,7 +52,7 @@ func TestSetLast(t *testing.T) {
 		Timestamp:      0xabcdef,
 		SSRC:           0x12345678,
 	}
-	vp8 := &buffer.VP8{
+	vp8 := &codec.VP8{
 		FirstByte:  25,
 		I:          true,
 		M:          true,
@@ -102,7 +102,7 @@ func TestUpdateOffsets(t *testing.T) {
 		Timestamp:      0xabcdef,
 		SSRC:           0x12345678,
 	}
-	vp8 := &buffer.VP8{
+	vp8 := &codec.VP8{
 		FirstByte:  25,
 		I:          true,
 		M:          true,
@@ -125,7 +125,7 @@ func TestUpdateOffsets(t *testing.T) {
 		Timestamp:      0xabcdef,
 		SSRC:           0x87654321,
 	}
-	vp8 = &buffer.VP8{
+	vp8 = &codec.VP8{
 		FirstByte:  25,
 		I:          true,
 		M:          true,
@@ -172,7 +172,7 @@ func TestOutOfOrderPictureId(t *testing.T) {
 		Timestamp:      0xabcdef,
 		SSRC:           0x12345678,
 	}
-	vp8 := &buffer.VP8{
+	vp8 := &codec.VP8{
 		FirstByte:  25,
 		I:          true,
 		M:          true,
@@ -205,7 +205,7 @@ func TestOutOfOrderPictureId(t *testing.T) {
 	vp8.PictureID = 13469
 	extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8)
 
-	expectedVP8 := &buffer.VP8{
+	expectedVP8 := &codec.VP8{
 		FirstByte:  25,
 		I:          true,
 		M:          true,
@@ -244,7 +244,7 @@ func TestOutOfOrderPictureId(t *testing.T) {
 	vp8.PictureID = 13468
 	extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8)
 
-	expectedVP8 = &buffer.VP8{
+	expectedVP8 = &codec.VP8{
 		FirstByte:  25,
 		I:          true,
 		M:          true,
@@ -275,7 +275,7 @@ func TestTemporalLayerFiltering(t *testing.T) {
 		Timestamp:      0xabcdef,
 		SSRC:           0x12345678,
 	}
-	vp8 := &buffer.VP8{
+	vp8 := &codec.VP8{
 		FirstByte:  25,
 		I:          true,
 		M:          true,
@@ -341,7 +341,7 @@ func TestGapInSequenceNumberSamePicture(t *testing.T) {
 		SSRC:           0x12345678,
 		PayloadSize:    33,
 	}
-	vp8 := &buffer.VP8{
+	vp8 := &codec.VP8{
 		FirstByte:  25,
 		I:          true,
 		M:          true,
@@ -359,7 +359,7 @@ func TestGapInSequenceNumberSamePicture(t *testing.T) {
 	extPkt, _ := testutils.GetTestExtPacketVP8(params, vp8)
 	v.SetLast(extPkt)
 
-	expectedVP8 := &buffer.VP8{
+	expectedVP8 := &codec.VP8{
 		FirstByte:  25,
 		I:          true,
 		M:          true,
@@ -382,7 +382,7 @@ func TestGapInSequenceNumberSamePicture(t *testing.T) {
 	require.Equal(t, marshalledVP8, buf)
 
 	// telling there is a gap in sequence number will add pictures to missing picture cache
-	expectedVP8 = &buffer.VP8{
+	expectedVP8 = &codec.VP8{
 		FirstByte:  25,
 		I:          true,
 		M:          true,
@@ -418,7 +418,7 @@ func TestUpdateAndGetPadding(t *testing.T) {
 		SSRC:           0x12345678,
 		PayloadSize:    20,
 	}
-	vp8 := &buffer.VP8{
+	vp8 := &codec.VP8{
 		FirstByte:  25,
 		I:          true,
 		M:          true,
@@ -440,7 +440,7 @@ func TestUpdateAndGetPadding(t *testing.T) {
 	// getting padding with repeat of last picture
 	buf, err := v.UpdateAndGetPadding(false)
 	require.NoError(t, err)
-	expectedVP8 := buffer.VP8{
+	expectedVP8 := codec.VP8{
 		FirstByte:  16,
 		I:          true,
 		M:          true,
@@ -462,7 +462,7 @@ func TestUpdateAndGetPadding(t *testing.T) {
 	// getting padding with new picture
 	buf, err = v.UpdateAndGetPadding(true)
 	require.NoError(t, err)
-	expectedVP8 = buffer.VP8{
+	expectedVP8 = codec.VP8{
 		FirstByte:  16,
 		I:          true,
 		M:          true,

livekit-server/pkg/sfu/connectionquality/connectionstats.go

@@ -22,11 +22,11 @@ import (
 	"go.uber.org/atomic"
 	"google.golang.org/protobuf/types/known/timestamppb"
 
+	"github.com/livekit/protocol/codecs/mime"
 	"github.com/livekit/protocol/livekit"
 	"github.com/livekit/protocol/logger"
 
 	"github.com/livekit/livekit-server/pkg/sfu/buffer"
-	"github.com/livekit/livekit-server/pkg/sfu/mime"
 	"github.com/livekit/livekit-server/pkg/sfu/rtpstats"
 )
 
@@ -276,29 +276,30 @@ func (cs *ConnectionStats) updateScoreFromReceiverReport(at time.Time) (float32,
 	return cs.updateScoreWithAggregate(agg, time.Time{}, at), streams
 }
 
-func (cs *ConnectionStats) updateScoreAt(at time.Time) (float32, map[uint32]*buffer.StreamStatsWithLayers) {
+func (cs *ConnectionStats) updateScoreAt(at time.Time) (float32, map[uint32]*buffer.StreamStatsWithLayers, bool) {
 	if cs.params.SenderProvider != nil {
 		// receiver report based quality scoring, use stats from receiver report for scoring
-		return cs.updateScoreFromReceiverReport(at)
+		score, streams := cs.updateScoreFromReceiverReport(at)
+		return score, streams, true
 	}
 
 	if cs.params.ReceiverProvider == nil {
-		return MinMOS, nil
+		return MinMOS, nil, false
 	}
 
 	streams := cs.params.ReceiverProvider.GetDeltaStats()
 	if len(streams) == 0 {
 		mos, _ := cs.scorer.GetMOSAndQuality()
-		return mos, nil
+		return mos, nil, false
 	}
 
 	agg := toAggregateDeltaInfo(streams, false)
 	if agg == nil {
 		// no receiver report in the window
 		mos, _ := cs.scorer.GetMOSAndQuality()
-		return mos, streams
+		return mos, streams, false
 	}
-	return cs.updateScoreWithAggregate(agg, cs.params.ReceiverProvider.GetLastSenderReportTime(), at), streams
+	return cs.updateScoreWithAggregate(agg, cs.params.ReceiverProvider.GetLastSenderReportTime(), at), streams, false
 }
 
 func (cs *ConnectionStats) updateStreamingStart(at time.Time) time.Time {
@@ -324,12 +325,12 @@ func (cs *ConnectionStats) updateStreamingStart(at time.Time) time.Time {
 }
 
 func (cs *ConnectionStats) getStat() {
-	score, streams := cs.updateScoreAt(time.Time{})
+	score, streams, isSender := cs.updateScoreAt(time.Time{})
 
 	if cs.onStatsUpdate != nil && len(streams) != 0 {
 		analyticsStreams := make([]*livekit.AnalyticsStream, 0, len(streams))
 		for ssrc, stream := range streams {
-			as := toAnalyticsStream(ssrc, stream.RTPStats, stream.RTPStatsRemoteView)
+			as := toAnalyticsStream(ssrc, stream.RTPStats, stream.RTPStatsRemoteView, isSender)
 			if as == nil {
 				continue
 			}
@@ -373,6 +374,7 @@ func (cs *ConnectionStats) updateStatsWorker() {
 	for {
 		select {
 		case <-cs.done.Watch():
+			cs.getStat()
 			return
 
 		case <-tk.C:
@@ -429,6 +431,13 @@ func toAggregateDeltaInfo(streams map[uint32]*buffer.StreamStatsWithLayers, useR
 	for _, s := range streams {
 		if useRemoteView {
 			if s.RTPStatsRemoteView != nil {
+				// discount jitter from publisher side + internal processing while reporting downstream jitter
+				if s.RTPStats != nil {
+					s.RTPStatsRemoteView.JitterMax -= s.RTPStats.JitterMax
+					if s.RTPStatsRemoteView.JitterMax < 0.0 {
+						s.RTPStatsRemoteView.JitterMax = 0.0
+					}
+				}
 				deltaInfoList = append(deltaInfoList, s.RTPStatsRemoteView)
 			}
 		} else {
@@ -444,13 +453,17 @@ func toAnalyticsStream(
 	ssrc uint32,
 	deltaStats *rtpstats.RTPDeltaInfo,
 	deltaStatsRemoteView *rtpstats.RTPDeltaInfo,
+	isSender bool,
 ) *livekit.AnalyticsStream {
 	if deltaStats == nil {
 		return nil
 	}
 
 	// discount the feed side loss when reporting forwarded track stats,
+	// discount jitter from publisher side + internal processing while reporting downstream jitter
 	packetsLost := deltaStats.PacketsLost
+	rtt := deltaStats.RttMax
+	maxJitter := deltaStats.JitterMax
 	if deltaStatsRemoteView != nil {
 		packetsLost = deltaStatsRemoteView.PacketsLost
 		if deltaStatsRemoteView.PacketsMissing > packetsLost {
@@ -458,7 +471,14 @@ func toAnalyticsStream(
 		} else {
 			packetsLost -= deltaStatsRemoteView.PacketsMissing
 		}
+
+		rtt = deltaStatsRemoteView.RttMax
+		maxJitter = deltaStatsRemoteView.JitterMax
+	} else if isSender {
+		rtt = 0
+		maxJitter = 0
 	}
+
 	return &livekit.AnalyticsStream{
 		StartTime:         timestamppb.New(deltaStats.StartTime),
 		EndTime:           timestamppb.New(deltaStats.EndTime),
@@ -472,8 +492,8 @@ func toAnalyticsStream(
 		PacketsLost:       packetsLost,
 		PacketsOutOfOrder: deltaStats.PacketsOutOfOrder,
 		Frames:            deltaStats.Frames,
-		Rtt:               deltaStats.RttMax,
-		Jitter:            uint32(deltaStats.JitterMax),
+		Rtt:               rtt,
+		Jitter:            uint32(maxJitter),
 		Nacks:             deltaStats.Nacks,
 		Plis:              deltaStats.Plis,
 		Firs:              deltaStats.Firs,

livekit-server/pkg/sfu/connectionquality/connectionstats_test.go

@@ -22,8 +22,8 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/livekit/livekit-server/pkg/sfu/buffer"
-	"github.com/livekit/livekit-server/pkg/sfu/mime"
 	"github.com/livekit/livekit-server/pkg/sfu/rtpstats"
+	"github.com/livekit/protocol/codecs/mime"
 	"github.com/livekit/protocol/livekit"
 	"github.com/livekit/protocol/logger"
 )
@@ -319,7 +319,7 @@ func TestConnectionQuality(t *testing.T) {
 		// unmute and send packets to bring quality back up
 		now = now.Add(duration)
 		cs.UpdateMuteAt(false, now.Add(2*time.Second))
-		for i := 0; i < 3; i++ {
+		for range 3 {
 			trp.setStreams(map[uint32]*buffer.StreamStatsWithLayers{
 				1: {
 					RTPStats: &rtpstats.RTPDeltaInfo{

livekit-server/pkg/sfu/connectionquality/scorer.go

@@ -482,7 +482,7 @@ func (q *qualityScorer) updateAtLocked(stat *windowStat, at time.Time) {
 		ulgr.Debugw("quality rise")
 	default:
 		packets := stat.packets + stat.packetsPadding
-		if packets != 0 && (stat.packetsLost*100/packets) > 10 {
+		if packets != 0 && ((stat.packetsLost-stat.packetsMissing-stat.packetsOutOfOrder)*100/packets) > 10 {
 			ulgr.Debugw("quality hold - high loss")
 		}
 	}
@@ -525,10 +525,7 @@ func (q *qualityScorer) isUnmutedEnough(at time.Time) bool {
 		sinceLayerUnmute = at.Sub(q.layerUnmutedAt)
 	}
 
-	validDuration := sinceUnmute
-	if sinceLayerUnmute < validDuration {
-		validDuration = sinceLayerUnmute
-	}
+	validDuration := min(sinceLayerUnmute, sinceUnmute)
 
 	sinceLastUpdate := at.Sub(q.lastUpdateAt)
 
@@ -567,7 +564,7 @@ func (q *qualityScorer) getAdjustedPacketLossWeight(stat *windowStat) float64 {
 	//  2. enough time has elapsed since last calculation
 	if q.numPPSReadings > cPPSMinReadings && time.Since(q.modeCalculatedAt) > cModeCalculationInterval {
 		q.ppsMode = 0
-		for i := 0; i < len(q.ppsHistogram); i++ {
+		for i := range len(q.ppsHistogram) {
 			if q.ppsHistogram[i] > q.ppsMode {
 				q.ppsMode = i
 			}

livekit-server/pkg/sfu/datachannel/datachannel_writer.go

@@ -15,24 +15,30 @@ const (
 	singleWriteTimeout = 50 * time.Millisecond
 )
 
-var ErrDataDroppedBySlowReader = errors.New("data dropped by slow reader")
+var (
+	ErrDataDroppedBySlowReader         = errors.New("data dropped by slow reader")
+	ErrDataDroppedByHighBufferedAmount = errors.New("data dropped due to high buffered amount")
+)
 
 type BufferedAmountGetter interface {
 	BufferedAmount() uint64
 }
 
 type DataChannelWriter[T BufferedAmountGetter] struct {
-	bufferGetter  T
-	rawDC         datachannel.ReadWriteCloserDeadliner
-	slowThreshold int
-	rate          *BitrateCalculator
+	bufferGetter      T
+	rawDC             datachannel.ReadWriteCloserDeadliner
+	slowThreshold     int
+	rate              *BitrateCalculator
+	reliable          bool
+	targetLatency     time.Duration
+	minBufferedAmount uint64
 }
 
-// NewDataChannelWriter creates a new DataChannelWriter by detaching the data channel, when
-// writing to the datachanel times out, it will block and retry if the receiver's bitrate is
-// above the slowThreshold or drop the data if it's below the threshold. If the slowThreshold
-// is 0, it will never retry on write timeout.
-func NewDataChannelWriter[T BufferedAmountGetter](bufferGetter T, rawDC datachannel.ReadWriteCloserDeadliner, slowThreshold int) *DataChannelWriter[T] {
+// NewDataChannelWriterReliable creates a new DataChannelWriter for reliable data channel by
+// detaching it, when writing to the datachanel times out, it will block and retry if the
+// receiver's bitrate is above the slowThreshold or drop the data if it's below the threshold.
+// If the slowThreshold is 0, it will never retry on write timeout.
+func NewDataChannelWriterReliable[T BufferedAmountGetter](bufferGetter T, rawDC datachannel.ReadWriteCloserDeadliner, slowThreshold int) *DataChannelWriter[T] {
 	var rate *BitrateCalculator
 	if slowThreshold > 0 {
 		rate = NewBitrateCalculator(BitrateDuration, BitrateWindow)
@@ -42,6 +48,26 @@ func NewDataChannelWriter[T BufferedAmountGetter](bufferGetter T, rawDC datachan
 		rawDC:         rawDC,
 		slowThreshold: slowThreshold,
 		rate:          rate,
+		reliable:      true,
+	}
+}
+
+// NewDataChannelWriterUnreliable creates a new DataChannelWriter for unreliable data channel.
+// It will drop data when the buffered amount is too high to maintain the target latency.
+// The latency is estimated based on the bitrate in past 1 second. If targetLatency is 0, no
+// buffering control is applied.
+func NewDataChannelWriterUnreliable[T BufferedAmountGetter](bufferGetter T, rawDC datachannel.ReadWriteCloserDeadliner, targetLatency time.Duration, minBufferedAmount uint64) *DataChannelWriter[T] {
+	var rate *BitrateCalculator
+	if targetLatency > 0 {
+		rate = NewBitrateCalculator(BitrateDuration, BitrateWindow)
+	}
+	return &DataChannelWriter[T]{
+		bufferGetter:      bufferGetter,
+		rawDC:             rawDC,
+		rate:              rate,
+		targetLatency:     targetLatency,
+		minBufferedAmount: minBufferedAmount,
+		reliable:          false,
 	}
 }
 
@@ -50,6 +76,14 @@ func (w *DataChannelWriter[T]) BufferedAmountGetter() T {
 }
 
 func (w *DataChannelWriter[T]) Write(p []byte) (n int, err error) {
+	if w.reliable {
+		return w.writeReliable(p)
+	} else {
+		return w.writeUnreliable(p)
+	}
+}
+
+func (w *DataChannelWriter[T]) writeReliable(p []byte) (n int, err error) {
 	for {
 		err = w.rawDC.SetWriteDeadline(time.Now().Add(singleWriteTimeout))
 		if err != nil {
@@ -75,6 +109,34 @@ func (w *DataChannelWriter[T]) Write(p []byte) (n int, err error) {
 	}
 }
 
+func (w *DataChannelWriter[T]) writeUnreliable(p []byte) (n int, err error) {
+	if w.targetLatency == 0 {
+		err = w.rawDC.SetWriteDeadline(time.Now().Add(singleWriteTimeout))
+		if err != nil {
+			return 0, err
+		}
+		return w.rawDC.Write(p)
+	}
+
+	if bitrate, ok := w.rate.Bitrate(time.Now()); ok {
+		// control buffer latency to ~100ms
+		if w.bufferGetter.BufferedAmount() > uint64(time.Duration(bitrate)*w.targetLatency/8/time.Second) && w.bufferGetter.BufferedAmount() > w.minBufferedAmount {
+			return 0, ErrDataDroppedByHighBufferedAmount
+		}
+	}
+
+	err = w.rawDC.SetWriteDeadline(time.Now().Add(singleWriteTimeout))
+	if err != nil {
+		return 0, err
+	}
+	n, err = w.rawDC.Write(p)
+	if err != nil {
+		w.rate.AddBytes(n, int(w.bufferGetter.BufferedAmount()), mono.Now())
+	}
+
+	return
+}
+
 func (w *DataChannelWriter[T]) Close() error {
 	return w.rawDC.Close()
 }

livekit-server/pkg/sfu/datachannel/datachannel_writer_test.go

@@ -2,18 +2,19 @@ package datachannel
 
 import (
 	"context"
+	"sync/atomic"
 	"testing"
 	"time"
 
 	"github.com/pion/datachannel"
-	"github.com/pion/transport/v3/deadline"
+	"github.com/pion/transport/v4/deadline"
 	"github.com/stretchr/testify/require"
 )
 
 func TestDataChannelWriter(t *testing.T) {
 	mockDC := newMockDataChannelWriter()
 	// slow threshold is 1000B/s
-	w := NewDataChannelWriter(mockDC, mockDC, 8000)
+	w := NewDataChannelWriterReliable(mockDC, mockDC, 8000)
 	require.Equal(t, mockDC, w.BufferedAmountGetter())
 	buf := make([]byte, 2000)
 	// write 2000 bytes so it should not drop in 2 seconds
@@ -23,7 +24,7 @@ func TestDataChannelWriter(t *testing.T) {
 	require.Equal(t, 2000, n)
 
 	t1 := time.Now()
-	mockDC.SetNextWriteCompleteAt(t0.Add(time.Second))
+	mockDC.SetNextWriteCompleteAt(t0.Add(time.Second * 3 / 2))
 	n, err = w.Write(buf[:10])
 	require.NoError(t, err)
 	require.Equal(t, 10, n)
@@ -38,7 +39,7 @@ func TestDataChannelWriter(t *testing.T) {
 
 func TestDataChannelWriter_NoSlowThreshold(t *testing.T) {
 	mockDC := newMockDataChannelWriter()
-	w := NewDataChannelWriter(mockDC, mockDC, 0)
+	w := NewDataChannelWriterReliable(mockDC, mockDC, 0)
 	buf := make([]byte, 2000)
 	n, err := w.Write(buf)
 	require.NoError(t, err)
@@ -55,6 +56,24 @@ func TestDataChannelWriter_NoSlowThreshold(t *testing.T) {
 	require.Equal(t, 0, n)
 }
 
+func TestDataChannelWriter_Unreliable(t *testing.T) {
+	mockDC := newMockLossyDataChannelWriter(8192)
+	w := NewDataChannelWriterUnreliable(mockDC, mockDC, 100*time.Millisecond, 2000)
+	for range 10 {
+		buf := make([]byte, 128)
+		_, err := w.Write(buf)
+		require.NoError(t, err)
+		time.Sleep(100 * time.Millisecond)
+	}
+	buf := make([]byte, 4096)
+	_, err := w.Write(buf)
+	require.NoError(t, err)
+	// should drop due to high buffered amount
+	_, err = w.Write(buf)
+	require.ErrorIs(t, err, ErrDataDroppedByHighBufferedAmount)
+}
+
+// mockDataChannelWriter
 type mockDataChannelWriter struct {
 	datachannel.ReadWriteCloserDeadliner
 	nextWriteCompleteAt time.Time
@@ -92,3 +111,44 @@ func (m *mockDataChannelWriter) SetWriteDeadline(t time.Time) error {
 func (m *mockDataChannelWriter) SetNextWriteCompleteAt(t time.Time) {
 	m.nextWriteCompleteAt = t
 }
+
+// mockLossyDataChannelWriter
+type mockLossyDataChannelWriter struct {
+	datachannel.ReadWriteCloserDeadliner
+	bufferedAmount atomic.Int64
+	targetBitrate  int
+	lastWriteAt    time.Time
+}
+
+func newMockLossyDataChannelWriter(targetBitrate int) *mockLossyDataChannelWriter {
+	return &mockLossyDataChannelWriter{
+		targetBitrate: targetBitrate,
+		lastWriteAt:   time.Now(),
+	}
+}
+
+func (m *mockLossyDataChannelWriter) BufferedAmount() uint64 {
+	return uint64(m.bufferedAmount.Load())
+}
+
+func (m *mockLossyDataChannelWriter) Write(b []byte) (int, error) {
+	m.bufferedAmount.Add(int64(len(b)))
+	if time.Now().Before(m.lastWriteAt) {
+		return len(b), nil
+	}
+
+	// drain buffer based on target bitrate
+	canWriteBytes := time.Since(m.lastWriteAt) * time.Duration(m.targetBitrate) / time.Second / 8
+	if m.bufferedAmount.Load() <= int64(canWriteBytes) {
+		m.lastWriteAt = m.lastWriteAt.Add(time.Duration(int64(time.Second) * int64(m.BufferedAmount()) / (int64(m.targetBitrate) / 8)))
+		m.bufferedAmount.Store(0)
+	} else {
+		m.lastWriteAt = time.Now()
+		m.bufferedAmount.Add(-int64(canWriteBytes))
+	}
+	return len(b), nil
+}
+
+func (m *mockLossyDataChannelWriter) SetWriteDeadline(t time.Time) error {
+	return nil
+}

livekit-server/pkg/sfu/forwarder.go

@@ -27,6 +27,7 @@ import (
 	"go.uber.org/zap/zapcore"
 
 	"github.com/livekit/mediatransportutil"
+	"github.com/livekit/protocol/codecs/mime"
 	"github.com/livekit/protocol/livekit"
 	"github.com/livekit/protocol/logger"
 	"github.com/livekit/protocol/utils"
@@ -34,14 +35,13 @@ import (
 
 	"github.com/livekit/livekit-server/pkg/sfu/buffer"
 	"github.com/livekit/livekit-server/pkg/sfu/codecmunger"
-	"github.com/livekit/livekit-server/pkg/sfu/mime"
 	dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor"
 	"github.com/livekit/livekit-server/pkg/sfu/rtpstats"
+	sfuutils "github.com/livekit/livekit-server/pkg/sfu/utils"
 	"github.com/livekit/livekit-server/pkg/sfu/videolayerselector"
 	"github.com/livekit/livekit-server/pkg/sfu/videolayerselector/temporallayerselector"
 )
 
-// Forwarder
 const (
 	FlagPauseOnDowngrade  = true
 	FlagFilterRTX         = false
@@ -54,6 +54,11 @@ const (
 	SwitchAheadThresholdSeconds       = float64(0.025) // 25ms
 )
 
+var (
+	errSkipStartOnOutOfOrderPacket = errors.New("skip start on out-of-order packet")
+	errSwitchPointTooFarBehind     = errors.New("switch point too far behind")
+)
+
 // -------------------------------------------------------------------
 
 type VideoPauseReason int
@@ -180,6 +185,7 @@ func (v *VideoTransition) MarshalLogObject(e zapcore.ObjectEncoder) error {
 
 type TranslationParams struct {
 	shouldDrop         bool
+	isStarting         bool
 	isResuming         bool
 	isSwitching        bool
 	rtp                TranslationParamsRTP
@@ -209,13 +215,14 @@ func (r refInfo) MarshalLogObject(e zapcore.ObjectEncoder) error {
 // -------------------------------------------------------------------
 
 type Forwarder struct {
-	lock            sync.RWMutex
-	mime            mime.MimeType
-	clockRate       uint32
-	kind            webrtc.RTPCodecType
-	logger          logger.Logger
-	skipReferenceTS bool
-	rtpStats        *rtpstats.RTPStatsSender
+	lock                           sync.RWMutex
+	mime                           mime.MimeType
+	clockRate                      uint32
+	kind                           webrtc.RTPCodecType
+	logger                         logger.Logger
+	skipReferenceTS                bool
+	disableOpportunisticAllocation bool
+	rtpStats                       *rtpstats.RTPStatsSender
 
 	muted                 bool
 	pubMuted              bool
@@ -230,7 +237,8 @@ type Forwarder struct {
 	referenceLayerSpatial    int32
 	dummyStartTSOffset       uint64
 	refInfos                 [buffer.DefaultMaxLayerSpatial + 1]refInfo
-	refIsSVC                 bool
+	refVideoLayerMode        livekit.VideoLayer_Mode
+	isDDAvailable            bool
 
 	provisional *VideoAllocationProvisional
 
@@ -247,20 +255,22 @@ func NewForwarder(
 	kind webrtc.RTPCodecType,
 	logger logger.Logger,
 	skipReferenceTS bool,
+	disableOpportunisticAllocation bool,
 	rtpStats *rtpstats.RTPStatsSender,
 ) *Forwarder {
 	f := &Forwarder{
-		mime:                     mime.MimeTypeUnknown,
-		kind:                     kind,
-		logger:                   logger,
-		skipReferenceTS:          skipReferenceTS,
-		rtpStats:                 rtpStats,
-		referenceLayerSpatial:    buffer.InvalidLayerSpatial,
-		lastAllocation:           VideoAllocationDefault,
-		lastReferencePayloadType: -1,
-		rtpMunger:                NewRTPMunger(logger),
-		vls:                      videolayerselector.NewNull(logger),
-		codecMunger:              codecmunger.NewNull(logger),
+		mime:                           mime.MimeTypeUnknown,
+		kind:                           kind,
+		logger:                         logger,
+		skipReferenceTS:                skipReferenceTS,
+		disableOpportunisticAllocation: disableOpportunisticAllocation,
+		rtpStats:                       rtpStats,
+		referenceLayerSpatial:          buffer.InvalidLayerSpatial,
+		lastAllocation:                 VideoAllocationDefault,
+		lastReferencePayloadType:       -1,
+		rtpMunger:                      NewRTPMunger(logger),
+		vls:                            videolayerselector.NewNull(logger),
+		codecMunger:                    codecmunger.NewNull(logger),
 	}
 
 	if f.kind == webrtc.RTPCodecTypeVideo {
@@ -297,10 +307,14 @@ func (f *Forwarder) SetMaxTemporalLayerSeen(maxTemporalLayerSeen int32) bool {
 	return true
 }
 
-func (f *Forwarder) DetermineCodec(codec webrtc.RTPCodecCapability, extensions []webrtc.RTPHeaderExtensionParameter) {
+func (f *Forwarder) DetermineCodec(codec webrtc.RTPCodecCapability, extensions []webrtc.RTPHeaderExtensionParameter, videoLayerMode livekit.VideoLayer_Mode) {
 	f.lock.Lock()
 	defer f.lock.Unlock()
 
+	if videoLayerMode == livekit.VideoLayer_ONE_SPATIAL_LAYER_PER_STREAM_INCOMPLETE_RTCP_SR {
+		f.skipReferenceTS = true
+	}
+
 	toMimeType := mime.NormalizeMimeType(codec.MimeType)
 	codecChanged := f.mime != mime.MimeTypeUnknown && f.mime != toMimeType
 	if codecChanged {
@@ -308,6 +322,7 @@ func (f *Forwarder) DetermineCodec(codec webrtc.RTPCodecCapability, extensions [
 	}
 	f.mime = toMimeType
 	f.clockRate = codec.ClockRate
+	f.refVideoLayerMode = videoLayerMode
 
 	ddAvailable := func(exts []webrtc.RTPHeaderExtensionParameter) bool {
 		for _, ext := range exts {
@@ -320,7 +335,7 @@ func (f *Forwarder) DetermineCodec(codec webrtc.RTPCodecCapability, extensions [
 
 	switch f.mime {
 	case mime.MimeTypeVP8:
-		f.codecMunger = codecmunger.NewVP8FromNull(f.codecMunger, f.logger)
+		f.codecMunger = codecmunger.NewVP8FromOther(f.codecMunger, f.logger)
 		if f.vls != nil {
 			if vls := videolayerselector.NewSimulcastFromOther(f.vls); vls != nil {
 				f.vls = vls
@@ -333,47 +348,66 @@ func (f *Forwarder) DetermineCodec(codec webrtc.RTPCodecCapability, extensions [
 		f.vls.SetTemporalLayerSelector(temporallayerselector.NewVP8(f.logger))
 
 	case mime.MimeTypeH264, mime.MimeTypeH265:
+		f.codecMunger = codecmunger.NewNull(f.logger)
 		if f.vls != nil {
-			f.vls = videolayerselector.NewSimulcastFromOther(f.vls)
+			if vls := videolayerselector.NewSimulcastFromOther(f.vls); vls != nil {
+				f.vls = vls
+			} else {
+				f.logger.Errorw("failed to create simulcast on codec change", nil)
+			}
 		} else {
 			f.vls = videolayerselector.NewSimulcast(f.logger)
 		}
 
 	case mime.MimeTypeVP9:
-		// DD-TODO : we only enable dd layer selector for av1/vp9 now, in the future we can enable it for vp8 too
-		isDDAvailable := ddAvailable(extensions)
-		if isDDAvailable {
+		f.codecMunger = codecmunger.NewNull(f.logger)
+		if sfuutils.IsSimulcastMode(videoLayerMode) {
 			if f.vls != nil {
-				f.vls = videolayerselector.NewDependencyDescriptorFromOther(f.vls)
+				f.vls = videolayerselector.NewSimulcastFromOther(f.vls)
 			} else {
 				f.vls = videolayerselector.NewDependencyDescriptor(f.logger)
 			}
 		} else {
-			if f.vls != nil {
-				f.vls = videolayerselector.NewVP9FromOther(f.vls)
+			f.isDDAvailable = ddAvailable(extensions)
+			if f.isDDAvailable {
+				if f.vls != nil {
+					f.vls = videolayerselector.NewDependencyDescriptorFromOther(f.vls)
+				} else {
+					f.vls = videolayerselector.NewDependencyDescriptor(f.logger)
+				}
 			} else {
-				f.vls = videolayerselector.NewVP9(f.logger)
+				if f.vls != nil {
+					f.vls = videolayerselector.NewVP9FromOther(f.vls)
+				} else {
+					f.vls = videolayerselector.NewVP9(f.logger)
+				}
 			}
 		}
-		// SVC-TODO: Support for VP9 simulcast. When DD is not available, have to pick selector based on VP9 SVC or Simulcast
 
 	case mime.MimeTypeAV1:
-		// DD-TODO : we only enable dd layer selector for av1/vp9 now, in the future we can enable it for vp8 too
-		isDDAvailable := ddAvailable(extensions)
-		if isDDAvailable {
-			if f.vls != nil {
-				f.vls = videolayerselector.NewDependencyDescriptorFromOther(f.vls)
-			} else {
-				f.vls = videolayerselector.NewDependencyDescriptor(f.logger)
-			}
-		} else {
+		f.codecMunger = codecmunger.NewNull(f.logger)
+		if sfuutils.IsSimulcastMode(videoLayerMode) {
 			if f.vls != nil {
 				f.vls = videolayerselector.NewSimulcastFromOther(f.vls)
 			} else {
 				f.vls = videolayerselector.NewSimulcast(f.logger)
 			}
+		} else {
+			f.isDDAvailable = ddAvailable(extensions)
+			if f.isDDAvailable {
+				if f.vls != nil {
+					f.vls = videolayerselector.NewDependencyDescriptorFromOther(f.vls)
+				} else {
+					f.vls = videolayerselector.NewDependencyDescriptor(f.logger)
+				}
+			} else {
+				if f.vls != nil {
+					f.vls = videolayerselector.NewSimulcastFromOther(f.vls)
+				} else {
+					f.vls = videolayerselector.NewSimulcast(f.logger)
+				}
+			}
 		}
-		// SVC-TODO: Support for AV1 Simulcast
 	}
 }
 
@@ -460,7 +494,12 @@ func (f *Forwarder) Mute(muted bool, isSubscribeMutable bool) bool {
 	// It could result in some bandwidth consumed for stream without visibility in
 	// the case of intentional mute.
 	if muted && !isSubscribeMutable {
-		f.logger.Debugw("ignoring forwarder mute, paused due to congestion")
+		f.logger.Infow(
+			"ignoring forwarder mute, paused due to congestion",
+			"targetLayers", f.vls.GetTarget(),
+			"currentLayers", f.vls.GetCurrent(),
+			"lastAllocation", f.lastAllocation,
+		)
 		return false
 	}
 
@@ -529,6 +568,41 @@ func (f *Forwarder) SetMaxSpatialLayer(spatialLayer int32) (bool, buffer.VideoLa
 
 	f.logger.Debugw("setting max spatial layer", "layer", spatialLayer)
 	f.vls.SetMaxSpatial(spatialLayer)
+	if f.disableOpportunisticAllocation {
+		return true, f.vls.GetMax()
+	}
+
+	if f.vls.GetTarget().Spatial != buffer.InvalidLayerSpatial ||
+		f.isDeficientLocked() ||
+		f.lastAllocation.PauseReason == VideoPauseReasonMuted ||
+		f.lastAllocation.PauseReason == VideoPauseReasonPubMuted {
+		return true, f.vls.GetMax()
+	}
+
+	f.logger.Debugw("opportunistically setting target spatial layer", "layer", spatialLayer)
+
+	alloc := f.lastAllocation
+
+	// bitrates are not known
+	alloc.BandwidthRequested = 0
+	alloc.BandwidthDelta = 0
+	alloc.Bitrates = Bitrates{}
+
+	alloc.TargetLayer = f.vls.GetMax()
+	alloc.RequestLayerSpatial = f.vls.GetMax().Spatial
+	alloc.MaxLayer = f.vls.GetMax()
+
+	alloc.DistanceToDesired = getDistanceToDesired(
+		f.muted,
+		f.pubMuted,
+		f.vls.GetMaxSeen(),
+		nil,
+		alloc.Bitrates,
+		alloc.TargetLayer,
+		f.vls.GetMax(),
+	)
+
+	f.updateAllocation(alloc, "opportunistic")
 	return true, f.vls.GetMax()
 }
 
@@ -577,14 +651,10 @@ func (f *Forwarder) GetMaxSubscribedSpatial() int32 {
 
 	layer := buffer.InvalidLayerSpatial // covers muted case
 	if !f.muted {
-		layer = f.vls.GetMax().Spatial
-
 		// If current is higher, mark the current layer as max subscribed layer
 		// to prevent the current layer from stopping before forwarder switches
 		// to the new and lower max layer,
-		if layer < f.vls.GetCurrent().Spatial {
-			layer = f.vls.GetCurrent().Spatial
-		}
+		layer = max(f.vls.GetMax().Spatial, f.vls.GetCurrent().Spatial)
 
 		// if reference layer is higher, hold there until an RTCP Sender Report from
 		// publisher is available as that is used for reference time stamp between layers.
@@ -612,21 +682,24 @@ func (f *Forwarder) getRefLayer() (int32, int32) {
 		return buffer.InvalidLayerSpatial, buffer.InvalidLayerSpatial
 	}
 
-	if f.refIsSVC {
+	if f.refVideoLayerMode == livekit.VideoLayer_MULTIPLE_SPATIAL_LAYERS_PER_STREAM {
 		return 0, currentLayerSpatial
 	}
 
 	return currentLayerSpatial, currentLayerSpatial
 }
 
-func (f *Forwarder) SetRefSenderReport(isSVC bool, layer int32, srData *livekit.RTCPSenderReportState) {
+func (f *Forwarder) SetRefSenderReport(layer int32, srData *livekit.RTCPSenderReportState) {
 	f.lock.Lock()
 	defer f.lock.Unlock()
 
-	f.refIsSVC = isSVC
 	if layer >= 0 && int(layer) < len(f.refInfos) {
 		if layer == f.referenceLayerSpatial && f.refInfos[layer].senderReport == nil {
-			f.logger.Debugw("received RTCP sender report for reference layer spatial", "layer", layer)
+			f.logger.Debugw(
+				"received RTCP sender report for reference layer spatial",
+				"layer", layer,
+				"srData", rtpstats.WrappedRTCPSenderReportStateLogger{RTCPSenderReportState: srData},
+			)
 		}
 		f.refInfos[layer] = refInfo{srData, 0, false}
 
@@ -641,7 +714,7 @@ func (f *Forwarder) SetRefSenderReport(isSVC bool, layer int32, srData *livekit.
 		//
 		// It could happen like this
 		//   1. Normal operation: publisher sending sender reports and
-		//      suscribers use reports from publisher to calculate and send
+		//      subscribers use reports from publisher to calculate and send
 		//      RTCP sender report.
 		//   2. Publisher pauses: there are no more reports.
 		//   3. When paused, subscriber can still use the publisher side sender
@@ -665,7 +738,7 @@ func (f *Forwarder) SetRefSenderReport(isSVC bool, layer int32, srData *livekit.
 	}
 }
 
-func (f *Forwarder) GetSenderReportParams() (int32, uint64, *livekit.RTCPSenderReportState) {
+func (f *Forwarder) GetSenderReportParams() (int32, bool, uint64, *livekit.RTCPSenderReportState) {
 	f.lock.RLock()
 	defer f.lock.RUnlock()
 
@@ -673,10 +746,10 @@ func (f *Forwarder) GetSenderReportParams() (int32, uint64, *livekit.RTCPSenderR
 	if refLayer == buffer.InvalidLayerSpatial ||
 		f.refInfos[refLayer].senderReport == nil ||
 		!f.refInfos[refLayer].isTSOffsetValid {
-		return buffer.InvalidLayerSpatial, 0, nil
+		return buffer.InvalidLayerSpatial, false, 0, nil
 	}
 
-	return currentLayerSpatial, f.refInfos[refLayer].tsOffset, f.refInfos[refLayer].senderReport
+	return currentLayerSpatial, f.refVideoLayerMode == livekit.VideoLayer_MULTIPLE_SPATIAL_LAYERS_PER_STREAM, f.refInfos[refLayer].tsOffset, f.refInfos[refLayer].senderReport
 }
 
 func (f *Forwarder) isDeficientLocked() bool {
@@ -896,18 +969,24 @@ func (f *Forwarder) ProvisionalAllocatePrepare(availableLayers []int32, bitrates
 	f.lock.Lock()
 	defer f.lock.Unlock()
 
-	f.provisional = &VideoAllocationProvisional{
-		allocatedLayer: buffer.InvalidLayer,
-		muted:          f.muted,
-		pubMuted:       f.pubMuted,
-		maxSeenLayer:   f.vls.GetMaxSeen(),
-		bitrates:       bitrates,
-		maxLayer:       f.vls.GetMax(),
-		currentLayer:   f.vls.GetCurrent(),
+	if f.provisional == nil {
+		f.provisional = &VideoAllocationProvisional{}
 	}
 
-	f.provisional.availableLayers = make([]int32, len(availableLayers))
-	copy(f.provisional.availableLayers, availableLayers)
+	p := f.provisional
+	p.allocatedLayer = buffer.InvalidLayer
+	p.muted = f.muted
+	p.pubMuted = f.pubMuted
+	p.maxSeenLayer = f.vls.GetMaxSeen()
+	p.bitrates = bitrates
+	p.maxLayer = f.vls.GetMax()
+	p.currentLayer = f.vls.GetCurrent()
+	if cap(p.availableLayers) >= len(availableLayers) {
+		p.availableLayers = p.availableLayers[:len(availableLayers)]
+	} else {
+		p.availableLayers = make([]int32, len(availableLayers))
+	}
+	copy(p.availableLayers, availableLayers)
 }
 
 func (f *Forwarder) ProvisionalAllocateReset() {
@@ -1523,7 +1602,11 @@ func (f *Forwarder) updateAllocation(alloc VideoAllocation, reason string) Video
 		alloc.PauseReason != f.lastAllocation.PauseReason ||
 		alloc.TargetLayer != f.lastAllocation.TargetLayer ||
 		alloc.RequestLayerSpatial != f.lastAllocation.RequestLayerSpatial {
-		f.logger.Debugw(fmt.Sprintf("stream allocation: %s", reason), "allocation", &alloc)
+		f.logger.Debugw(
+			fmt.Sprintf("stream allocation: %s", reason),
+			"allocation", &alloc,
+			"lastAllocation", &f.lastAllocation,
+		)
 	}
 	f.lastAllocation = alloc
 
@@ -1563,18 +1646,7 @@ func (f *Forwarder) CheckSync() (bool, int32) {
 	f.lock.RLock()
 	defer f.lock.RUnlock()
 
-	locked, layer := f.vls.CheckSync()
-	if !locked {
-		return locked, layer
-	}
-
-	// max published layer (as seen by this forwarder) could be
-	// lower than max subscribed, mark de-synced if not deficient
-	if !f.isDeficientLocked() && f.vls.GetMax().Spatial > f.vls.GetTarget().Spatial {
-		return false, layer
-	}
-
-	return true, layer
+	return f.vls.CheckSync()
 }
 
 func (f *Forwarder) Restart() {
@@ -1586,11 +1658,11 @@ func (f *Forwarder) Restart() {
 	f.referenceLayerSpatial = buffer.InvalidLayerSpatial
 	f.lastReferencePayloadType = -1
 
-	for layer := 0; layer < len(f.refInfos); layer++ {
+	for layer := range len(f.refInfos) {
 		f.refInfos[layer] = refInfo{}
 	}
 	f.lastSwitchExtIncomingTS = 0
-	f.refIsSVC = false
+	f.refVideoLayerMode = livekit.VideoLayer_MODE_UNUSED
 }
 
 func (f *Forwarder) FilterRTX(nacks []uint16) (filtered []uint16, disallowedLayers [buffer.DefaultMaxLayerSpatial + 1]bool) {
@@ -1614,7 +1686,7 @@ func (f *Forwarder) FilterRTX(nacks []uint16) (filtered []uint16, disallowedLaye
 	if FlagFilterRTXLayers {
 		currentLayer := f.vls.GetCurrent()
 		targetLayer := f.vls.GetTarget()
-		for layer := int32(0); layer < buffer.DefaultMaxLayerSpatial+1; layer++ {
+		for layer := range buffer.DefaultMaxLayerSpatial + 1 {
 			if f.isDeficientLocked() && (targetLayer.Spatial < currentLayer.Spatial || layer > currentLayer.Spatial) {
 				disallowedLayers[layer] = true
 			}
@@ -1643,7 +1715,7 @@ func (f *Forwarder) GetTranslationParams(extPkt *buffer.ExtPacket, layer int32)
 
 	return TranslationParams{
 		shouldDrop: true,
-	}, ErrUnknownKind
+	}, errUnknownKind
 }
 
 func (f *Forwarder) getRefLayerRTPTimestamp(ts uint32, refLayer, targetLayer int32) (uint32, error) {
@@ -1651,7 +1723,7 @@ func (f *Forwarder) getRefLayerRTPTimestamp(ts uint32, refLayer, targetLayer int
 		return 0, fmt.Errorf("invalid layer(s), refLayer: %d, targetLayer: %d", refLayer, targetLayer)
 	}
 
-	if refLayer == targetLayer || f.refIsSVC {
+	if refLayer == targetLayer || f.refVideoLayerMode == livekit.VideoLayer_MULTIPLE_SPATIAL_LAYERS_PER_STREAM {
 		return ts, nil
 	}
 
@@ -1676,8 +1748,13 @@ func (f *Forwarder) getRefLayerRTPTimestamp(ts uint32, refLayer, targetLayer int
 	return ts + offset, nil
 }
 
-func (f *Forwarder) processSourceSwitch(extPkt *buffer.ExtPacket, layer int32) error {
+func (f *Forwarder) processSourceSwitch(extPkt *buffer.ExtPacket, layer int32) (bool, error) {
+	starting := false
 	if !f.started {
+		if extPkt.IsOutOfOrder {
+			return starting, errSkipStartOnOutOfOrderPacket
+		}
+
 		f.started = true
 		f.referenceLayerSpatial = layer
 		f.rtpMunger.SetLastSnTs(extPkt)
@@ -1691,8 +1768,13 @@ func (f *Forwarder) processSourceSwitch(extPkt *buffer.ExtPacket, layer int32) e
 			"layer", layer,
 			"referenceLayerSpatial", f.referenceLayerSpatial,
 		)
-		return nil
+		starting = true
+		return starting, nil
 	} else if f.referenceLayerSpatial == buffer.InvalidLayerSpatial {
+		if extPkt.IsOutOfOrder {
+			return starting, errSkipStartOnOutOfOrderPacket
+		}
+
 		f.referenceLayerSpatial = layer
 		f.codecMunger.SetLast(extPkt)
 		f.logger.Debugw(
@@ -1704,6 +1786,8 @@ func (f *Forwarder) processSourceSwitch(extPkt *buffer.ExtPacket, layer int32) e
 			"layer", layer,
 			"referenceLayerSpatial", f.referenceLayerSpatial,
 		)
+
+		starting = true
 	}
 
 	logTransition := func(message string, extExpectedTS, extRefTS, extLastTS uint64, diffSeconds float64) {
@@ -1722,23 +1806,6 @@ func (f *Forwarder) processSourceSwitch(extPkt *buffer.ExtPacket, layer int32) e
 			"rtpStats", f.rtpStats,
 		)
 	}
-	// TODO-REMOVE-AFTER-DATA-COLLECTION
-	logTransitionInfo := func(message string, extExpectedTS, extRefTS, extLastTS uint64, diffSeconds float64) {
-		f.logger.Infow(
-			message,
-			"layer", layer,
-			"referenceLayerSpatial", f.referenceLayerSpatial,
-			"extExpectedTS", extExpectedTS,
-			"incomingTS", extPkt.Packet.Timestamp,
-			"extIncomingTS", extPkt.ExtTimestamp,
-			"extRefTS", extRefTS,
-			"extLastTS", extLastTS,
-			"diffSeconds", math.Abs(diffSeconds),
-			"refInfos", logger.ObjectSlice(f.refInfos[:]),
-			"lastSwitchExtIncomingTS", f.lastSwitchExtIncomingTS,
-			"rtpStats", f.rtpStats,
-		)
-	}
 
 	// Compute how much time passed between the previous forwarded packet
 	// and the current incoming (to be forwarded) packet and calculate
@@ -1750,8 +1817,7 @@ func (f *Forwarder) processSourceSwitch(extPkt *buffer.ExtPacket, layer int32) e
 	//   3. extExpectedTS -> expected timestamp of this packet calculated based on elapsed time since first packet
 	// Ideally, extRefTS and extExpectedTS should be very close and extLastTS should be before both of those.
 	// But, cases like muting/unmuting, clock vagaries, pacing, etc. make them not satisfy those conditions always.
-	rtpMungerState := f.rtpMunger.GetState()
-	extLastTS := rtpMungerState.ExtLastTimestamp
+	extLastTS := f.rtpMunger.GetExtLastTimestamp()
 	extExpectedTS := extLastTS
 	extRefTS := extLastTS
 	refTS := uint32(extRefTS)
@@ -1771,7 +1837,7 @@ func (f *Forwarder) processSourceSwitch(extPkt *buffer.ExtPacket, layer int32) e
 				"layer", layer,
 				"error", err,
 			)
-			return err
+			return starting, err
 		}
 	}
 
@@ -1818,7 +1884,6 @@ func (f *Forwarder) processSourceSwitch(extPkt *buffer.ExtPacket, layer int32) e
 		}
 	}
 
-	bigJump := false
 	var extNextTS uint64
 	if f.lastSSRC == 0 {
 		// If resuming (e. g. on unmute), keep next timestamp close to expected timestamp.
@@ -1844,14 +1909,12 @@ func (f *Forwarder) processSourceSwitch(extPkt *buffer.ExtPacket, layer int32) e
 		diffSeconds := float64(int64(extExpectedTS-extRefTS)) / float64(f.clockRate)
 		if diffSeconds >= 0.0 {
 			if f.resumeBehindThreshold > 0 && diffSeconds > f.resumeBehindThreshold {
-				logTransitionInfo("resume, reference too far behind", extExpectedTS, extRefTS, extLastTS, diffSeconds)
+				logTransition("resume, reference too far behind", extExpectedTS, extRefTS, extLastTS, diffSeconds)
 				extNextTS = extExpectedTS
-				bigJump = true
 			} else if diffSeconds > ResumeBehindHighThresholdSeconds {
 				// could be due to incoming time stamp lagging a lot, like an unpause of the track
-				logTransitionInfo("resume, reference very far behind", extExpectedTS, extRefTS, extLastTS, diffSeconds)
+				logTransition("resume, reference very far behind", extExpectedTS, extRefTS, extLastTS, diffSeconds)
 				extNextTS = extExpectedTS
-				bigJump = true
 			} else {
 				extNextTS = extRefTS
 			}
@@ -1872,7 +1935,7 @@ func (f *Forwarder) processSourceSwitch(extPkt *buffer.ExtPacket, layer int32) e
 				// (like "have waited for too long for layer switch, nothing available, switch to whatever is available" kind of condition).
 				logTransition("layer switch, reference too far behind", extExpectedTS, extRefTS, extLastTS, diffSeconds)
 
-				return errors.New("switch point too far behind")
+				return starting, errSwitchPointTooFarBehind
 			}
 
 			// use a nominal increase to ensure that timestamp is always moving forward
@@ -1893,54 +1956,37 @@ func (f *Forwarder) processSourceSwitch(extPkt *buffer.ExtPacket, layer int32) e
 		// nominal increase
 		extNextTS = extLastTS + 1
 	}
-	if bigJump { // TODO-REMOVE-AFTER-DATA-COLLECTION
-		f.logger.Infow(
-			"next timestamp on switch",
-			"switchingAt", switchingAt,
-			"layer", layer,
-			"extLastTS", extLastTS,
-			"lastMarker", rtpMungerState.LastMarker,
-			"extRefTS", extRefTS,
-			"dummyStartTSOffset", f.dummyStartTSOffset,
-			"referenceLayerSpatial", f.referenceLayerSpatial,
-			"extExpectedTS", extExpectedTS,
-			"extNextTS", extNextTS,
-			"tsJump", extNextTS-extLastTS,
-			"nextSN", rtpMungerState.ExtLastSequenceNumber+1,
-			"extIncomingSN", extPkt.ExtSequenceNumber,
-			"incomingTS", extPkt.Packet.Timestamp,
-			"extIncomingTS", extPkt.ExtTimestamp,
-			"rtpStats", f.rtpStats,
-		)
-	} else {
-		f.logger.Debugw(
-			"next timestamp on switch",
-			"switchingAt", switchingAt,
-			"layer", layer,
-			"extLastTS", extLastTS,
-			"lastMarker", rtpMungerState.LastMarker,
-			"extRefTS", extRefTS,
-			"dummyStartTSOffset", f.dummyStartTSOffset,
-			"referenceLayerSpatial", f.referenceLayerSpatial,
-			"extExpectedTS", extExpectedTS,
-			"extNextTS", extNextTS,
-			"tsJump", extNextTS-extLastTS,
-			"nextSN", rtpMungerState.ExtLastSequenceNumber+1,
-			"extIncomingSN", extPkt.ExtSequenceNumber,
-			"extIncomingTS", extPkt.ExtTimestamp,
-			"rtpStats", f.rtpStats,
-		)
-	}
+	f.logger.Debugw(
+		"next timestamp on switch",
+		"switchingAt", switchingAt,
+		"layer", layer,
+		"extLastTS", extLastTS,
+		"lastMarker", f.rtpMunger.GetState().LastMarker,
+		"extRefTS", extRefTS,
+		"dummyStartTSOffset", f.dummyStartTSOffset,
+		"referenceLayerSpatial", f.referenceLayerSpatial,
+		"extExpectedTS", extExpectedTS,
+		"extNextTS", extNextTS,
+		"tsJump", extNextTS-extLastTS,
+		"nextSN", f.rtpMunger.GetState().ExtLastSequenceNumber+1,
+		"extIncomingSN", extPkt.ExtSequenceNumber,
+		"extIncomingTS", extPkt.ExtTimestamp,
+		"rtpStats", f.rtpStats,
+	)
 
 	f.rtpMunger.UpdateSnTsOffsets(extPkt, 1, extNextTS-extLastTS)
 	f.codecMunger.UpdateOffsets(extPkt)
-	return nil
+	return starting, nil
 }
 
 // should be called with lock held
-func (f *Forwarder) getTranslationParamsCommon(extPkt *buffer.ExtPacket, layer int32, tp *TranslationParams) error {
+func (f *Forwarder) getTranslationParamsCommon(extPkt *buffer.ExtPacket, layer int32, tp *TranslationParams) (bool, error) {
+	var (
+		starting bool
+		err      error
+	)
 	if f.lastSSRC != extPkt.Packet.SSRC {
-		if err := f.processSourceSwitch(extPkt, layer); err != nil {
+		if starting, err = f.processSourceSwitch(extPkt, layer); err != nil {
 			f.logger.Debugw(
 				"could not switch feed",
 				"error", err,
@@ -1954,7 +2000,7 @@ func (f *Forwarder) getTranslationParamsCommon(extPkt *buffer.ExtPacket, layer i
 			)
 			tp.shouldDrop = true
 			f.vls.Rollback()
-			return nil
+			return starting, nil
 		}
 		f.logger.Debugw(
 			"switching feed",
@@ -1977,29 +2023,30 @@ func (f *Forwarder) getTranslationParamsCommon(extPkt *buffer.ExtPacket, layer i
 	tpRTP, err := f.rtpMunger.UpdateAndGetSnTs(extPkt, tp.marker)
 	if err != nil {
 		tp.shouldDrop = true
-		if err == ErrPaddingOnlyPacket || err == ErrDuplicatePacket || err == ErrOutOfOrderSequenceNumberCacheMiss {
-			return nil
+		if err == errPaddingOnlyPacket || err == errDuplicatePacket || err == errOutOfOrderSequenceNumberCacheMiss {
+			return starting, nil
 		}
-		return err
+		return starting, err
 	}
 
 	tp.rtp = tpRTP
 
 	if len(extPkt.Packet.Payload) > 0 {
-		return f.translateCodecHeader(extPkt, tp)
+		return starting, f.translateCodecHeader(extPkt, tp)
 	}
 
-	return nil
+	return starting, nil
 }
 
 // should be called with lock held
 func (f *Forwarder) getTranslationParamsAudio(extPkt *buffer.ExtPacket, layer int32) (TranslationParams, error) {
 	tp := TranslationParams{}
-	if err := f.getTranslationParamsCommon(extPkt, layer, &tp); err != nil {
+	starting, err := f.getTranslationParamsCommon(extPkt, layer, &tp)
+	tp.isStarting = starting
+	if err != nil {
 		tp.shouldDrop = true
-		return tp, err
 	}
-	return tp, nil
+	return tp, err
 }
 
 // should be called with lock held
@@ -2013,6 +2060,24 @@ func (f *Forwarder) getTranslationParamsVideo(extPkt *buffer.ExtPacket, layer in
 
 	result := f.vls.Select(extPkt, layer)
 	if !result.IsSelected {
+		if f.isDDAvailable && extPkt.DependencyDescriptor == nil {
+			f.logger.Infow(
+				"turning off dependency descriptor",
+				"layer", layer,
+				"refInfos", logger.ObjectSlice(f.refInfos[:]),
+				"lastSwitchExtIncomingTS", f.lastSwitchExtIncomingTS,
+				"currentLayer", f.vls.GetCurrent(),
+				"targetLayer", f.vls.GetCurrent(),
+				"maxLayer", f.vls.GetMax(),
+			)
+			f.isDDAvailable = false
+			switch f.mime {
+			case mime.MimeTypeVP9:
+				f.vls = videolayerselector.NewVP9FromOther(f.vls)
+			case mime.MimeTypeAV1:
+				f.vls = videolayerselector.NewSimulcastFromOther(f.vls)
+			}
+		}
 		tp.shouldDrop = true
 		if f.started && result.IsRelevant {
 			// call to update highest incoming sequence number and other internal structures
@@ -2029,7 +2094,8 @@ func (f *Forwarder) getTranslationParamsVideo(extPkt *buffer.ExtPacket, layer in
 	tp.ddBytes = result.DependencyDescriptorExtension
 	tp.marker = result.RTPMarker
 
-	err := f.getTranslationParamsCommon(extPkt, layer, &tp)
+	starting, err := f.getTranslationParamsCommon(extPkt, layer, &tp)
+	tp.isStarting = starting
 	if tp.shouldDrop {
 		return tp, err
 	}
@@ -2117,7 +2183,7 @@ func (f *Forwarder) maybeStart() {
 	)
 }
 
-func (f *Forwarder) GetSnTsForPadding(num int, forceMarker bool) ([]SnTs, error) {
+func (f *Forwarder) GetSnTsForPadding(num int, frameRate uint32, forceMarker bool) ([]SnTs, error) {
 	f.lock.Lock()
 	defer f.lock.Unlock()
 
@@ -2131,7 +2197,13 @@ func (f *Forwarder) GetSnTsForPadding(num int, forceMarker bool) ([]SnTs, error)
 	if !f.vls.GetTarget().IsValid() {
 		forceMarker = true
 	}
-	return f.rtpMunger.UpdateAndGetPaddingSnTs(num, 0, 0, forceMarker, 0)
+	return f.rtpMunger.UpdateAndGetPaddingSnTs(
+		num,
+		f.clockRate,
+		frameRate,
+		forceMarker,
+		f.rtpMunger.GetExtLastTimestamp(),
+	)
 }
 
 func (f *Forwarder) GetSnTsForBlankFrames(frameRate uint32, numPackets int) ([]SnTs, bool, error) {
@@ -2145,7 +2217,7 @@ func (f *Forwarder) GetSnTsForBlankFrames(frameRate uint32, numPackets int) ([]S
 		numPackets++
 	}
 
-	extLastTS := f.rtpMunger.GetState().ExtLastTimestamp
+	extLastTS := f.rtpMunger.GetExtLastTimestamp()
 	extExpectedTS := extLastTS
 	if f.rtpStats != nil {
 		tsExt, err := f.rtpStats.GetExpectedRTPTimestamp(mono.Now())
@@ -2156,7 +2228,13 @@ func (f *Forwarder) GetSnTsForBlankFrames(frameRate uint32, numPackets int) ([]S
 	if int64(extExpectedTS-extLastTS) <= 0 {
 		extExpectedTS = extLastTS + 1
 	}
-	snts, err := f.rtpMunger.UpdateAndGetPaddingSnTs(numPackets, f.clockRate, frameRate, frameEndNeeded, extExpectedTS)
+	snts, err := f.rtpMunger.UpdateAndGetPaddingSnTs(
+		numPackets,
+		f.clockRate,
+		frameRate,
+		frameEndNeeded,
+		extExpectedTS,
+	)
 	return snts, frameEndNeeded, err
 }
 
@@ -2167,7 +2245,7 @@ func (f *Forwarder) GetPadding(frameEndNeeded bool) ([]byte, error) {
 	return f.codecMunger.UpdateAndGetPadding(!frameEndNeeded)
 }
 
-func (f *Forwarder) RTPMungerDebugInfo() map[string]interface{} {
+func (f *Forwarder) RTPMungerDebugInfo() map[string]any {
 	f.lock.RLock()
 	defer f.lock.RUnlock()
 

livekit-server/pkg/sfu/forwarder_test.go

@@ -20,6 +20,8 @@ import (
 	"github.com/pion/webrtc/v4"
 	"github.com/stretchr/testify/require"
 
+	"github.com/livekit/mediatransportutil/pkg/codec"
+	"github.com/livekit/protocol/livekit"
 	"github.com/livekit/protocol/logger"
 
 	"github.com/livekit/livekit-server/pkg/sfu/buffer"
@@ -32,8 +34,14 @@ func disable(f *Forwarder) {
 }
 
 func newForwarder(codec webrtc.RTPCodecCapability, kind webrtc.RTPCodecType) *Forwarder {
-	f := NewForwarder(kind, logger.GetLogger(), true, nil)
-	f.DetermineCodec(codec, nil)
+	f := NewForwarder(
+		kind,
+		logger.GetLogger(),
+		true, // skipReferenceTS
+		true, // disableOpportunisticAllocation
+		nil,
+	)
+	f.DetermineCodec(codec, nil, livekit.VideoLayer_MODE_UNUSED)
 	return f
 }
 
@@ -463,6 +471,9 @@ func TestForwarderProvisionalAllocate(t *testing.T) {
 	f.SetMaxPublishedLayer(buffer.DefaultMaxLayerSpatial)
 	f.SetMaxTemporalLayerSeen(buffer.DefaultMaxLayerTemporal)
 
+	// Reset to invalid layers for testing allocation from scratch
+	disable(f)
+
 	bitrates := Bitrates{
 		{1, 2, 3, 4},
 		{5, 6, 7, 8},
@@ -681,6 +692,9 @@ func TestForwarderProvisionalAllocateMute(t *testing.T) {
 	f.SetMaxSpatialLayer(buffer.DefaultMaxLayerSpatial)
 	f.SetMaxTemporalLayer(buffer.DefaultMaxLayerTemporal)
 
+	// Reset to invalid layers for testing muted state
+	disable(f)
+
 	bitrates := Bitrates{
 		{1, 2, 3, 4},
 		{5, 6, 7, 8},
@@ -722,6 +736,9 @@ func TestForwarderProvisionalAllocateGetCooperativeTransition(t *testing.T) {
 	f.SetMaxPublishedLayer(buffer.DefaultMaxLayerSpatial)
 	f.SetMaxTemporalLayerSeen(buffer.DefaultMaxLayerTemporal)
 
+	// Reset to invalid layers for testing cooperative transition from scratch
+	disable(f)
+
 	availableLayers := []int32{0, 1, 2}
 	bitrates := Bitrates{
 		{1, 2, 3, 4},
@@ -1272,22 +1289,42 @@ func TestForwarderGetTranslationParamsAudio(t *testing.T) {
 	f := newForwarder(testutils.TestOpusCodec, webrtc.RTPCodecTypeAudio)
 
 	params := &testutils.TestExtPacketParams{
+		SequenceNumber: 23332,
+		Timestamp:      0xabcdef,
+		SSRC:           0x12345678,
+		PayloadSize:    20,
+		IsOutOfOrder:   true,
+	}
+	extPkt, _ := testutils.GetTestExtPacket(params)
+
+	// should not start on an out-of-order packet
+	expectedTP := TranslationParams{
+		shouldDrop: true,
+	}
+	actualTP, err := f.GetTranslationParams(extPkt, 0)
+	require.NoError(t, err)
+	require.Equal(t, expectedTP, actualTP)
+	require.False(t, f.started)
+	require.Zero(t, f.lastSSRC)
+
+	params = &testutils.TestExtPacketParams{
 		SequenceNumber: 23333,
 		Timestamp:      0xabcdef,
 		SSRC:           0x12345678,
 		PayloadSize:    20,
 	}
-	extPkt, _ := testutils.GetTestExtPacket(params)
+	extPkt, _ = testutils.GetTestExtPacket(params)
 
-	// should lock onto the first packet
-	expectedTP := TranslationParams{
+	// should lock onto the first in-order packet
+	expectedTP = TranslationParams{
+		isStarting: true,
 		rtp: TranslationParamsRTP{
 			snOrdering:        SequenceNumberOrderingContiguous,
 			extSequenceNumber: 23333,
 			extTimestamp:      0xabcdef,
 		},
 	}
-	actualTP, err := f.GetTranslationParams(extPkt, 0)
+	actualTP, err = f.GetTranslationParams(extPkt, 0)
 	require.NoError(t, err)
 	require.Equal(t, expectedTP, actualTP)
 	require.True(t, f.started)
@@ -1436,13 +1473,14 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) {
 	f := newForwarder(testutils.TestVP8Codec, webrtc.RTPCodecTypeVideo)
 
 	params := &testutils.TestExtPacketParams{
-		SequenceNumber: 23333,
+		SequenceNumber: 23332,
 		Timestamp:      0xabcdef,
 		SSRC:           0x12345678,
 		PayloadSize:    20,
-		SetMarker:      true,
+		Marker:         true,
+		IsOutOfOrder:   true,
 	}
-	vp8 := &buffer.VP8{
+	vp8 := &codec.VP8{
 		FirstByte:  25,
 		I:          true,
 		M:          true,
@@ -1459,13 +1497,47 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) {
 	}
 	extPkt, _ := testutils.GetTestExtPacketVP8(params, vp8)
 
-	// no target layers, should drop
+	// should not start on an out-of-order packet
 	expectedTP := TranslationParams{
 		shouldDrop: true,
 	}
 	actualTP, err := f.GetTranslationParams(extPkt, 0)
 	require.NoError(t, err)
 	require.Equal(t, expectedTP, actualTP)
+	require.False(t, f.started)
+	require.Zero(t, f.lastSSRC)
+
+	params = &testutils.TestExtPacketParams{
+		SequenceNumber: 23333,
+		Timestamp:      0xabcdef,
+		SSRC:           0x12345678,
+		PayloadSize:    20,
+		Marker:         true,
+	}
+	vp8 = &codec.VP8{
+		FirstByte:  25,
+		I:          true,
+		M:          true,
+		PictureID:  13467,
+		L:          true,
+		TL0PICIDX:  233,
+		T:          true,
+		TID:        0,
+		Y:          true,
+		K:          true,
+		KEYIDX:     23,
+		HeaderSize: 6,
+		IsKeyFrame: false,
+	}
+	extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8)
+
+	// no target layers, should drop
+	expectedTP = TranslationParams{
+		shouldDrop: true,
+	}
+	actualTP, err = f.GetTranslationParams(extPkt, 0)
+	require.NoError(t, err)
+	require.Equal(t, expectedTP, actualTP)
 
 	// although target layer matches, not a key frame, so should drop
 	f.vls.SetTarget(buffer.VideoLayer{
@@ -1480,7 +1552,7 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) {
 	require.Equal(t, expectedTP, actualTP)
 
 	// should lock onto packet (key frame)
-	vp8 = &buffer.VP8{
+	vp8 = &codec.VP8{
 		FirstByte:  25,
 		I:          true,
 		M:          true,
@@ -1496,7 +1568,7 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) {
 		IsKeyFrame: true,
 	}
 	extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8)
-	expectedVP8 := &buffer.VP8{
+	expectedVP8 := &codec.VP8{
 		FirstByte:  25,
 		I:          true,
 		M:          true,
@@ -1511,8 +1583,9 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) {
 		HeaderSize: 6,
 		IsKeyFrame: true,
 	}
-	marshalledVP8, err := expectedVP8.Marshal()
+	marshalledVP8, _ := expectedVP8.Marshal()
 	expectedTP = TranslationParams{
+		isStarting:  true,
 		isSwitching: true,
 		isResuming:  true,
 		rtp: TranslationParamsRTP{
@@ -1576,7 +1649,7 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) {
 		PayloadSize:    20,
 	}
 	extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8)
-	expectedVP8 = &buffer.VP8{
+	expectedVP8 = &codec.VP8{
 		FirstByte:  25,
 		I:          true,
 		M:          true,
@@ -1613,7 +1686,7 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) {
 		SSRC:           0x12345678,
 		PayloadSize:    20,
 	}
-	vp8 = &buffer.VP8{
+	vp8 = &codec.VP8{
 		FirstByte:  25,
 		S:          true,
 		I:          true,
@@ -1630,7 +1703,7 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) {
 		IsKeyFrame: true,
 	}
 	extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8)
-	expectedVP8 = &buffer.VP8{
+	expectedVP8 = &codec.VP8{
 		FirstByte:  25,
 		I:          true,
 		M:          true,
@@ -1667,7 +1740,7 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) {
 		SSRC:           0x12345678,
 		PayloadSize:    20,
 	}
-	vp8 = &buffer.VP8{
+	vp8 = &codec.VP8{
 		FirstByte:  25,
 		I:          true,
 		M:          true,
@@ -1702,7 +1775,7 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) {
 		SSRC:           0x12345678,
 		PayloadSize:    20,
 	}
-	vp8 = &buffer.VP8{
+	vp8 = &codec.VP8{
 		FirstByte:  25,
 		I:          true,
 		M:          true,
@@ -1718,7 +1791,7 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) {
 		IsKeyFrame: false,
 	}
 	extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8)
-	expectedVP8 = &buffer.VP8{
+	expectedVP8 = &codec.VP8{
 		FirstByte:  25,
 		I:          true,
 		M:          true,
@@ -1799,7 +1872,7 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) {
 		SSRC:           0x87654321,
 		PayloadSize:    20,
 	}
-	vp8 = &buffer.VP8{
+	vp8 = &codec.VP8{
 		FirstByte:  25,
 		I:          true,
 		M:          false,
@@ -1816,7 +1889,7 @@ func TestForwarderGetTranslationParamsVideo(t *testing.T) {
 	}
 	extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8)
 
-	expectedVP8 = &buffer.VP8{
+	expectedVP8 = &codec.VP8{
 		FirstByte:  25,
 		I:          true,
 		M:          true,
@@ -1858,7 +1931,7 @@ func TestForwarderGetSnTsForPadding(t *testing.T) {
 		SSRC:           0x12345678,
 		PayloadSize:    20,
 	}
-	vp8 := &buffer.VP8{
+	vp8 := &codec.VP8{
 		FirstByte:  25,
 		I:          true,
 		M:          true,
@@ -1888,14 +1961,14 @@ func TestForwarderGetSnTsForPadding(t *testing.T) {
 	disable(f)
 
 	// should get back frame end needed as the last packet did not have RTP marker set
-	snts, err := f.GetSnTsForPadding(5, false)
+	snts, err := f.GetSnTsForPadding(5, 0, false)
 	require.NoError(t, err)
 
 	numPadding := 5
 	clockRate := uint32(0)
 	frameRate := uint32(5)
 	var sntsExpected = make([]SnTs, numPadding)
-	for i := 0; i < numPadding; i++ {
+	for i := range numPadding {
 		sntsExpected[i] = SnTs{
 			extSequenceNumber: 23333 + uint64(i) + 1,
 			extTimestamp:      0xabcdef + (uint64(i)*uint64(clockRate))/uint64(frameRate),
@@ -1904,10 +1977,10 @@ func TestForwarderGetSnTsForPadding(t *testing.T) {
 	require.Equal(t, sntsExpected, snts)
 
 	// now that there is a marker, timestamp should jump on first padding when asked again
-	snts, err = f.GetSnTsForPadding(numPadding, false)
+	snts, err = f.GetSnTsForPadding(numPadding, 0, false)
 	require.NoError(t, err)
 
-	for i := 0; i < numPadding; i++ {
+	for i := range numPadding {
 		sntsExpected[i] = SnTs{
 			extSequenceNumber: 23338 + uint64(i) + 1,
 			extTimestamp:      0xabcdef + (uint64(i+1)*uint64(clockRate))/uint64(frameRate),
@@ -1925,7 +1998,7 @@ func TestForwarderGetSnTsForBlankFrames(t *testing.T) {
 		SSRC:           0x12345678,
 		PayloadSize:    20,
 	}
-	vp8 := &buffer.VP8{
+	vp8 := &codec.VP8{
 		FirstByte:  25,
 		I:          true,
 		M:          true,
@@ -2002,7 +2075,7 @@ func TestForwarderGetPaddingVP8(t *testing.T) {
 		SSRC:           0x12345678,
 		PayloadSize:    20,
 	}
-	vp8 := &buffer.VP8{
+	vp8 := &codec.VP8{
 		FirstByte:  25,
 		I:          true,
 		M:          true,
@@ -2029,7 +2102,7 @@ func TestForwarderGetPaddingVP8(t *testing.T) {
 	_, _ = f.GetTranslationParams(extPkt, 0)
 
 	// getting padding with frame end needed, should repeat the last picture id
-	expectedVP8 := buffer.VP8{
+	expectedVP8 := codec.VP8{
 		FirstByte:  16,
 		I:          true,
 		M:          true,
@@ -2051,7 +2124,7 @@ func TestForwarderGetPaddingVP8(t *testing.T) {
 	require.Equal(t, marshalledVP8, buf)
 
 	// getting padding with no frame end needed, should get next picture id
-	expectedVP8 = buffer.VP8{
+	expectedVP8 = codec.VP8{
 		FirstByte:  16,
 		I:          true,
 		M:          true,

livekit-server/pkg/sfu/forwardstats.go

@@ -1,27 +1,33 @@
 package sfu
 
 import (
-	"fmt"
 	"sync"
 	"time"
 
-	"go.uber.org/atomic"
-
 	"github.com/livekit/livekit-server/pkg/telemetry/prometheus"
 	"github.com/livekit/protocol/logger"
 	"github.com/livekit/protocol/utils"
+	"github.com/livekit/protocol/utils/mono"
+)
+
+const (
+	cHighForwardingLatency = 20 * time.Millisecond
+	cSkewFactor            = 10
 )
 
 type ForwardStats struct {
 	lock         sync.Mutex
-	lastLeftNano atomic.Int64
 	latency      *utils.LatencyAggregate
+	lowest       int64
+	highest      int64
+	lastUpdateAt int64
 	closeCh      chan struct{}
 }
 
 func NewForwardStats(latencyUpdateInterval, reportInterval, latencyWindowLength time.Duration) *ForwardStats {
 	s := &ForwardStats{
 		latency: utils.NewLatencyAggregate(latencyUpdateInterval, latencyWindowLength),
+		lowest:  time.Second.Nanoseconds(),
 		closeCh: make(chan struct{}),
 	}
 
@@ -29,43 +35,58 @@ func NewForwardStats(latencyUpdateInterval, reportInterval, latencyWindowLength
 	return s
 }
 
-func (s *ForwardStats) Update(arrival, left int64) {
+func (s *ForwardStats) Update(arrival, left int64) (int64, bool) {
 	transit := left - arrival
-
-	// ignore if transit is too large or negative, this could happen if system time is adjusted
-	if transit < 0 || time.Duration(transit) > 5*time.Second {
-		return
-	}
-	lastLeftNano := s.lastLeftNano.Load()
-	if left < lastLeftNano || !s.lastLeftNano.CompareAndSwap(lastLeftNano, left) {
-		return
-	}
+	isHighForwardingLatency := time.Duration(transit) > cHighForwardingLatency
 
 	s.lock.Lock()
-	defer s.lock.Unlock()
 	s.latency.Update(time.Duration(arrival), float64(transit))
+	s.lowest = min(transit, s.lowest)
+	s.highest = max(transit, s.highest)
+	s.lastUpdateAt = arrival
+	s.lock.Unlock()
+
+	prometheus.RecordForwardLatencySample(transit)
+	return transit, isHighForwardingLatency
 }
 
-func (s *ForwardStats) GetStats() (latency, jitter time.Duration) {
+func (s *ForwardStats) GetStats(shortDuration time.Duration) (time.Duration, time.Duration) {
 	s.lock.Lock()
-	w := s.latency.Summarize()
+	// a dummy sample to flush the pipe to current time
+	now := mono.UnixNano()
+	if (now - s.lastUpdateAt) > shortDuration.Nanoseconds() {
+		s.latency.Update(time.Duration(now), 0)
+	}
+
+	wLong := s.latency.Summarize()
+
+	lowest := s.lowest
+	s.lowest = time.Second.Nanoseconds()
+
+	highest := s.highest
+	s.highest = 0
 	s.lock.Unlock()
-	latency, jitter = time.Duration(w.Mean()), time.Duration(w.StdDev())
-	// TODO: remove this check after debugging unexpected jitter issue
-	if jitter > 10*time.Second {
-		logger.Infow("unexpected forward jitter",
-			"jitter", jitter,
-			"stats", fmt.Sprintf("count %.2f, mean %.2f, stdDev %.2f", w.Count(), w.Mean(), w.StdDev()),
+
+	latencyLong, jitterLong := time.Duration(wLong.Mean()), time.Duration(wLong.StdDev())
+	if jitterLong > latencyLong*cSkewFactor {
+		logger.Infow(
+			"high jitter in forwarding path",
+			"lowest", time.Duration(lowest),
+			"highest", time.Duration(highest),
+			"countLong", wLong.Count(),
+			"latencyLong", latencyLong,
+			"jitterLong", jitterLong,
 		)
 	}
-	return
+	return latencyLong, jitterLong
 }
 
-func (s *ForwardStats) GetLastStats(duration time.Duration) (latency, jitter time.Duration) {
+func (s *ForwardStats) GetShortStats(shortDuration time.Duration) (time.Duration, time.Duration) {
 	s.lock.Lock()
-	defer s.lock.Unlock()
-	w := s.latency.SummarizeLast(duration)
-	return time.Duration(w.Mean()), time.Duration(w.StdDev())
+	wShort := s.latency.SummarizeLast(shortDuration)
+	s.lock.Unlock()
+
+	return time.Duration(wShort.Mean()), time.Duration(wShort.StdDev())
 }
 
 func (s *ForwardStats) Stop() {
@@ -75,15 +96,16 @@ func (s *ForwardStats) Stop() {
 func (s *ForwardStats) report(reportInterval time.Duration) {
 	ticker := time.NewTicker(reportInterval)
 	defer ticker.Stop()
+
 	for {
 		select {
 		case <-s.closeCh:
 			return
+
 		case <-ticker.C:
-			latency, jitter := s.GetLastStats(reportInterval)
-			latencySlow, jitterSlow := s.GetStats()
-			prometheus.RecordForwardJitter(uint32(jitter/time.Millisecond), uint32(jitterSlow/time.Millisecond))
-			prometheus.RecordForwardLatency(uint32(latency/time.Millisecond), uint32(latencySlow/time.Millisecond))
+			latencyLong, jitterLong := s.GetStats(reportInterval)
+			prometheus.RecordForwardJitter(uint32(jitterLong.Nanoseconds()))
+			prometheus.RecordForwardLatency(uint32(latencyLong.Nanoseconds()))
 		}
 	}
 }

livekit-server/pkg/sfu/interceptor/rtx.go

@@ -39,13 +39,17 @@ type streamInfo struct {
 
 type RTXInfoExtractorFactory struct {
 	onStreamFound  func(*interceptor.StreamInfo)
-	onRTXPairFound func(repair, base uint32)
+	onRTXPairFound func(repair, base uint32, rsid string)
 	lock           sync.Mutex
 	streams        map[uint32]streamInfo
 	logger         logger.Logger
 }
 
-func NewRTXInfoExtractorFactory(onStreamFound func(*interceptor.StreamInfo), onRTXPairFound func(repair, base uint32), logger logger.Logger) *RTXInfoExtractorFactory {
+func NewRTXInfoExtractorFactory(
+	onStreamFound func(*interceptor.StreamInfo),
+	onRTXPairFound func(repair, base uint32, rsid string),
+	logger logger.Logger,
+) *RTXInfoExtractorFactory {
 	return &RTXInfoExtractorFactory{
 		onStreamFound:  onStreamFound,
 		onRTXPairFound: onRTXPairFound,
@@ -61,16 +65,23 @@ func (f *RTXInfoExtractorFactory) NewInterceptor(id string) (interceptor.Interce
 	}, nil
 }
 
-func (f *RTXInfoExtractorFactory) setStreamInfo(ssrc uint32, mid, rid, rsid string) {
+func (f *RTXInfoExtractorFactory) SetStreamInfo(ssrc uint32, mid, rid, rsid string) {
 	var repairSsrc, baseSsrc uint32
+	var repairSid string
 	f.lock.Lock()
 
+	if mid == "" || (rid == "" && rsid == "") {
+		f.lock.Unlock()
+		return
+	}
+
 	if rsid != "" {
 		// repair stream found, find base stream
 		for base, info := range f.streams {
 			if info.mid == mid && info.rid == rsid {
 				repairSsrc = ssrc
 				baseSsrc = base
+				repairSid = rsid
 				delete(f.streams, base)
 				break
 			}
@@ -81,6 +92,7 @@ func (f *RTXInfoExtractorFactory) setStreamInfo(ssrc uint32, mid, rid, rsid stri
 			if info.mid == mid && info.rsid == rid {
 				repairSsrc = repair
 				baseSsrc = ssrc
+				repairSid = rid
 				delete(f.streams, repair)
 				break
 			}
@@ -99,10 +111,12 @@ func (f *RTXInfoExtractorFactory) setStreamInfo(ssrc uint32, mid, rid, rsid stri
 	f.lock.Unlock()
 
 	if repairSsrc != 0 && baseSsrc != 0 {
-		f.onRTXPairFound(repairSsrc, baseSsrc)
+		f.onRTXPairFound(repairSsrc, baseSsrc, repairSid)
 	}
 }
 
+// ------------------------------------------
+
 type RTXInfoExtractor struct {
 	interceptor.NoOp
 
@@ -131,6 +145,8 @@ func (u *RTXInfoExtractor) BindRemoteStream(info *interceptor.StreamInfo, reader
 	}
 }
 
+// ------------------------------------------
+
 type rtxInfoReader struct {
 	tryTimes  int
 	reader    interceptor.RTPReader
@@ -171,10 +187,10 @@ func (r *rtxInfoReader) Read(b []byte, a interceptor.Attributes) (int, intercept
 	if mid != "" && (rid != "" || rsid != "") {
 		r.logger.Debugw("stream found", "mid", mid, "rid", rid, "rsid", rsid, "ssrc", header.SSRC)
 		r.tryTimes = -1
-		go r.factory.setStreamInfo(header.SSRC, mid, rid, rsid)
+		go r.factory.SetStreamInfo(header.SSRC, mid, rid, rsid)
 	} else {
 		// ignore padding only packet for probe count
-		if !(header.Padding && n-header.MarshalSize()-int(b[n-1]) == 0) {
+		if !header.Padding || n-header.MarshalSize()-int(b[n-1]) != 0 {
 			r.tryTimes--
 		}
 	}

livekit-server/pkg/sfu/interceptor/unhandlesimulcast.go

@@ -0,0 +1,183 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package interceptor
+
+import (
+	"github.com/pion/interceptor"
+	"github.com/pion/rtp"
+	"github.com/pion/sdp/v3"
+	"github.com/pion/webrtc/v4"
+	"go.uber.org/zap/zapcore"
+
+	"github.com/livekit/livekit-server/pkg/sfu/utils"
+	"github.com/livekit/protocol/logger"
+)
+
+const (
+	simulcastProbeCount = 10
+)
+
+type SimulcastTrackInfo struct {
+	Mid            string
+	StreamID       string
+	RepairSSRC     uint32 // set only when `IsRepairStream: false`, i. e. RTX SSRC for the primary stream
+	IsRepairStream bool
+}
+
+func (s *SimulcastTrackInfo) MarshalLogObject(e zapcore.ObjectEncoder) error {
+	e.AddString("Mid", s.Mid)
+	e.AddString("StreamID", s.StreamID)
+	e.AddUint32("RepairSSRC", s.RepairSSRC)
+	e.AddBool("IsRepairStream", s.IsRepairStream)
+	return nil
+}
+
+// -------------------------------------------------------------------
+
+type UnhandleSimulcastOption func(u *UnhandleSimulcastInterceptor) error
+
+func UnhandleSimulcastTracks(logger logger.Logger, tracks map[uint32]SimulcastTrackInfo) UnhandleSimulcastOption {
+	return func(u *UnhandleSimulcastInterceptor) error {
+		u.logger = logger
+		u.simTracks = tracks
+		return nil
+	}
+}
+
+type UnhandleSimulcastInterceptorFactory struct {
+	opts []UnhandleSimulcastOption
+}
+
+func (f *UnhandleSimulcastInterceptorFactory) NewInterceptor(id string) (interceptor.Interceptor, error) {
+	i := &UnhandleSimulcastInterceptor{simTracks: map[uint32]SimulcastTrackInfo{}}
+	for _, o := range f.opts {
+		if err := o(i); err != nil {
+			return nil, err
+		}
+	}
+	return i, nil
+}
+
+func NewUnhandleSimulcastInterceptorFactory(opts ...UnhandleSimulcastOption) (*UnhandleSimulcastInterceptorFactory, error) {
+	return &UnhandleSimulcastInterceptorFactory{opts: opts}, nil
+}
+
+type unhandleSimulcastRTPReader struct {
+	SimulcastTrackInfo
+	logger    logger.Logger
+	tryTimes  int
+	reader    interceptor.RTPReader
+	midExtID  uint8
+	ridExtID  uint8
+	rsidExtID uint8
+}
+
+func (u *unhandleSimulcastRTPReader) Read(b []byte, a interceptor.Attributes) (int, interceptor.Attributes, error) {
+	n, a, err := u.reader.Read(b, a)
+	if u.tryTimes < 0 || err != nil {
+		return n, a, err
+	}
+
+	header := rtp.Header{}
+	hsize, err := header.Unmarshal(b[:n])
+	if err != nil {
+		return n, a, nil
+	}
+	var mid, rid, rsid string
+	if payload := header.GetExtension(u.midExtID); payload != nil {
+		mid = string(payload)
+	}
+
+	if payload := header.GetExtension(u.ridExtID); payload != nil {
+		rid = string(payload)
+	}
+
+	if payload := header.GetExtension(u.rsidExtID); payload != nil {
+		rid = string(payload)
+	}
+
+	if mid != "" && (rid != "" || rsid != "") {
+		u.logger.Debugw(
+			"unhandle stream found",
+			"mid", mid,
+			"rid", rid,
+			"rsid", rsid,
+			"ssrc", header.SSRC,
+			"simulcastTrackInfo", u.SimulcastTrackInfo,
+		)
+		u.tryTimes = -1
+		return n, a, nil
+	} else {
+		// ignore padding only packet for probe count
+		if !header.Padding || n-header.MarshalSize()-int(b[n-1]) != 0 {
+			u.tryTimes--
+		}
+	}
+
+	if mid == "" {
+		header.SetExtension(u.midExtID, []byte(u.Mid))
+	}
+	if rid == "" && !u.IsRepairStream {
+		header.SetExtension(u.ridExtID, []byte(u.StreamID))
+	}
+	if rsid == "" && u.IsRepairStream {
+		header.SetExtension(u.rsidExtID, []byte(u.StreamID))
+	}
+
+	hsize2 := header.MarshalSize()
+
+	if hsize2-hsize+n > len(b) { // no enough buf to set extension
+		return n, a, nil
+	}
+	copy(b[hsize2:], b[hsize:n])
+	header.MarshalTo(b)
+	u.logger.Debugw(
+		"unhandle stream injecting",
+		"mid", mid,
+		"rid", rid,
+		"rsid", rsid,
+		"ssrc", header.SSRC,
+		"simulcastTrackInfo", u.SimulcastTrackInfo,
+	)
+	return hsize2 - hsize + n, a, nil
+}
+
+type UnhandleSimulcastInterceptor struct {
+	interceptor.NoOp
+	logger    logger.Logger
+	simTracks map[uint32]SimulcastTrackInfo
+}
+
+func (u *UnhandleSimulcastInterceptor) BindRemoteStream(info *interceptor.StreamInfo, reader interceptor.RTPReader) interceptor.RTPReader {
+	if t, ok := u.simTracks[info.SSRC]; ok {
+		midExtensionID := utils.GetHeaderExtensionID(info.RTPHeaderExtensions, webrtc.RTPHeaderExtensionCapability{URI: sdp.SDESMidURI})
+		streamIDExtensionID := utils.GetHeaderExtensionID(info.RTPHeaderExtensions, webrtc.RTPHeaderExtensionCapability{URI: sdp.SDESRTPStreamIDURI})
+		repairStreamIDExtensionID := utils.GetHeaderExtensionID(info.RTPHeaderExtensions, webrtc.RTPHeaderExtensionCapability{URI: sdp.SDESRepairRTPStreamIDURI})
+		if midExtensionID == 0 || streamIDExtensionID == 0 || repairStreamIDExtensionID == 0 {
+			return reader
+		}
+
+		return &unhandleSimulcastRTPReader{
+			SimulcastTrackInfo: t,
+			logger:             u.logger,
+			reader:             reader,
+			tryTimes:           simulcastProbeCount,
+			midExtID:           uint8(midExtensionID),
+			ridExtID:           uint8(streamIDExtensionID),
+			rsidExtID:          uint8(repairStreamIDExtensionID),
+		}
+	}
+	return reader
+}

livekit-server/pkg/sfu/mime/mimetype.go

@@ -1,281 +0,0 @@
-// Copyright 2023 LiveKit, Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package mime
-
-import (
-	"strings"
-
-	"github.com/pion/webrtc/v4"
-)
-
-const (
-	MimeTypePrefixAudio = "audio/"
-	MimeTypePrefixVideo = "video/"
-)
-
-type MimeTypeCodec int
-
-const (
-	MimeTypeCodecUnknown MimeTypeCodec = iota
-	MimeTypeCodecH264
-	MimeTypeCodecH265
-	MimeTypeCodecOpus
-	MimeTypeCodecRED
-	MimeTypeCodecVP8
-	MimeTypeCodecVP9
-	MimeTypeCodecAV1
-	MimeTypeCodecG722
-	MimeTypeCodecPCMU
-	MimeTypeCodecPCMA
-	MimeTypeCodecRTX
-	MimeTypeCodecFlexFEC
-	MimeTypeCodecULPFEC
-)
-
-func (m MimeTypeCodec) String() string {
-	switch m {
-	case MimeTypeCodecUnknown:
-		return "MimeTypeCodecUnknown"
-	case MimeTypeCodecH264:
-		return "H264"
-	case MimeTypeCodecH265:
-		return "H265"
-	case MimeTypeCodecOpus:
-		return "opus"
-	case MimeTypeCodecRED:
-		return "red"
-	case MimeTypeCodecVP8:
-		return "VP8"
-	case MimeTypeCodecVP9:
-		return "VP9"
-	case MimeTypeCodecAV1:
-		return "AV1"
-	case MimeTypeCodecG722:
-		return "G722"
-	case MimeTypeCodecPCMU:
-		return "PCMU"
-	case MimeTypeCodecPCMA:
-		return "PCMA"
-	case MimeTypeCodecRTX:
-		return "rtx"
-	case MimeTypeCodecFlexFEC:
-		return "flexfec"
-	case MimeTypeCodecULPFEC:
-		return "ulpfec"
-	}
-
-	return "MimeTypeCodecUnknown"
-}
-
-func NormalizeMimeTypeCodec(codec string) MimeTypeCodec {
-	switch {
-	case strings.EqualFold(codec, "h264"):
-		return MimeTypeCodecH264
-	case strings.EqualFold(codec, "h265"):
-		return MimeTypeCodecH265
-	case strings.EqualFold(codec, "opus"):
-		return MimeTypeCodecOpus
-	case strings.EqualFold(codec, "red"):
-		return MimeTypeCodecRED
-	case strings.EqualFold(codec, "vp8"):
-		return MimeTypeCodecVP8
-	case strings.EqualFold(codec, "vp9"):
-		return MimeTypeCodecVP9
-	case strings.EqualFold(codec, "av1"):
-		return MimeTypeCodecAV1
-	case strings.EqualFold(codec, "g722"):
-		return MimeTypeCodecG722
-	case strings.EqualFold(codec, "pcmu"):
-		return MimeTypeCodecPCMU
-	case strings.EqualFold(codec, "pcma"):
-		return MimeTypeCodecPCMA
-	case strings.EqualFold(codec, "rtx"):
-		return MimeTypeCodecRTX
-	case strings.EqualFold(codec, "flexfec"):
-		return MimeTypeCodecFlexFEC
-	case strings.EqualFold(codec, "ulpfec"):
-		return MimeTypeCodecULPFEC
-	}
-
-	return MimeTypeCodecUnknown
-}
-
-func GetMimeTypeCodec(mime string) MimeTypeCodec {
-	i := strings.IndexByte(mime, '/')
-	if i == -1 {
-		return MimeTypeCodecUnknown
-	}
-
-	return NormalizeMimeTypeCodec(mime[i+1:])
-}
-
-func IsMimeTypeCodecStringOpus(codec string) bool {
-	return NormalizeMimeTypeCodec(codec) == MimeTypeCodecOpus
-}
-
-func IsMimeTypeCodecStringRED(codec string) bool {
-	return NormalizeMimeTypeCodec(codec) == MimeTypeCodecRED
-}
-
-func IsMimeTypeCodecStringH264(codec string) bool {
-	return NormalizeMimeTypeCodec(codec) == MimeTypeCodecH264
-}
-
-type MimeType int
-
-const (
-	MimeTypeUnknown MimeType = iota
-	MimeTypeH264
-	MimeTypeH265
-	MimeTypeOpus
-	MimeTypeRED
-	MimeTypeVP8
-	MimeTypeVP9
-	MimeTypeAV1
-	MimeTypeG722
-	MimeTypePCMU
-	MimeTypePCMA
-	MimeTypeRTX
-	MimeTypeFlexFEC
-	MimeTypeULPFEC
-)
-
-func (m MimeType) String() string {
-	switch m {
-	case MimeTypeUnknown:
-		return "MimeTypeUnknown"
-	case MimeTypeH264:
-		return webrtc.MimeTypeH264
-	case MimeTypeH265:
-		return webrtc.MimeTypeH265
-	case MimeTypeOpus:
-		return webrtc.MimeTypeOpus
-	case MimeTypeRED:
-		return "audio/red"
-	case MimeTypeVP8:
-		return webrtc.MimeTypeVP8
-	case MimeTypeVP9:
-		return webrtc.MimeTypeVP9
-	case MimeTypeAV1:
-		return webrtc.MimeTypeAV1
-	case MimeTypeG722:
-		return webrtc.MimeTypeG722
-	case MimeTypePCMU:
-		return webrtc.MimeTypePCMU
-	case MimeTypePCMA:
-		return webrtc.MimeTypePCMA
-	case MimeTypeRTX:
-		return webrtc.MimeTypeRTX
-	case MimeTypeFlexFEC:
-		return webrtc.MimeTypeFlexFEC
-	case MimeTypeULPFEC:
-		return "video/ulpfec"
-	}
-
-	return "MimeTypeUnknown"
-}
-
-func NormalizeMimeType(mime string) MimeType {
-	switch {
-	case strings.EqualFold(mime, webrtc.MimeTypeH264):
-		return MimeTypeH264
-	case strings.EqualFold(mime, webrtc.MimeTypeH265):
-		return MimeTypeH265
-	case strings.EqualFold(mime, webrtc.MimeTypeOpus):
-		return MimeTypeOpus
-	case strings.EqualFold(mime, "audio/red"):
-		return MimeTypeRED
-	case strings.EqualFold(mime, webrtc.MimeTypeVP8):
-		return MimeTypeVP8
-	case strings.EqualFold(mime, webrtc.MimeTypeVP9):
-		return MimeTypeVP9
-	case strings.EqualFold(mime, webrtc.MimeTypeAV1):
-		return MimeTypeAV1
-	case strings.EqualFold(mime, webrtc.MimeTypeG722):
-		return MimeTypeG722
-	case strings.EqualFold(mime, webrtc.MimeTypePCMU):
-		return MimeTypePCMU
-	case strings.EqualFold(mime, webrtc.MimeTypePCMA):
-		return MimeTypePCMA
-	case strings.EqualFold(mime, webrtc.MimeTypeRTX):
-		return MimeTypeRTX
-	case strings.EqualFold(mime, webrtc.MimeTypeFlexFEC):
-		return MimeTypeFlexFEC
-	case strings.EqualFold(mime, "video/ulpfec"):
-		return MimeTypeULPFEC
-	}
-
-	return MimeTypeUnknown
-}
-
-func IsMimeTypeStringEqual(mime1 string, mime2 string) bool {
-	return NormalizeMimeType(mime1) == NormalizeMimeType(mime2)
-}
-
-func IsMimeTypeStringAudio(mime string) bool {
-	return strings.HasPrefix(mime, MimeTypePrefixAudio)
-}
-
-func IsMimeTypeAudio(mimeType MimeType) bool {
-	return strings.HasPrefix(mimeType.String(), MimeTypePrefixAudio)
-}
-
-func IsMimeTypeStringVideo(mime string) bool {
-	return strings.HasPrefix(mime, MimeTypePrefixVideo)
-}
-
-func IsMimeTypeVideo(mimeType MimeType) bool {
-	return strings.HasPrefix(mimeType.String(), MimeTypePrefixVideo)
-}
-
-// SVC-TODO: Have to use more conditions to differentiate between
-// SVC-TODO: SVC and non-SVC (could be single layer or simulcast).
-// SVC-TODO: May only need to differentiate between simulcast and non-simulcast
-// SVC-TODO: i. e. may be possible to treat single layer as SVC to get proper/intended functionality.
-func IsMimeTypeSVC(mimeType MimeType) bool {
-	switch mimeType {
-	case MimeTypeAV1, MimeTypeVP9:
-		return true
-	}
-	return false
-}
-
-func IsMimeTypeStringSVC(mime string) bool {
-	return IsMimeTypeSVC(NormalizeMimeType(mime))
-}
-
-func IsMimeTypeStringRED(mime string) bool {
-	return NormalizeMimeType(mime) == MimeTypeRED
-}
-
-func IsMimeTypeStringOpus(mime string) bool {
-	return NormalizeMimeType(mime) == MimeTypeOpus
-}
-
-func IsMimeTypeStringRTX(mime string) bool {
-	return NormalizeMimeType(mime) == MimeTypeRTX
-}
-
-func IsMimeTypeStringVP8(mime string) bool {
-	return NormalizeMimeType(mime) == MimeTypeVP8
-}
-
-func IsMimeTypeStringVP9(mime string) bool {
-	return NormalizeMimeType(mime) == MimeTypeVP9
-}
-
-func IsMimeTypeStringH264(mime string) bool {
-	return NormalizeMimeType(mime) == MimeTypeH264
-}

livekit-server/pkg/sfu/pacer/base.go

@@ -20,6 +20,7 @@ import (
 	"time"
 
 	"github.com/livekit/livekit-server/pkg/sfu/bwe"
+	"github.com/livekit/mediatransportutil"
 	"github.com/livekit/protocol/logger"
 	"github.com/livekit/protocol/utils/mono"
 	"github.com/pion/rtp"
@@ -56,9 +57,17 @@ func (b *Base) TimeSinceLastSentPacket() time.Duration {
 
 func (b *Base) SendPacket(p *Packet) (int, error) {
 	defer func() {
+		if p.HeaderPool != nil && p.Header != nil {
+			*p.Header = rtp.Header{}
+			p.HeaderPool.Put(p.Header)
+		}
+
 		if p.Pool != nil && p.PoolEntity != nil {
 			p.Pool.Put(p.PoolEntity)
 		}
+
+		*p = Packet{}
+		PacketFactory.Put(p)
 	}()
 
 	err := b.patchRTPHeaderExtensions(p)
@@ -83,8 +92,10 @@ func (b *Base) SendPacket(p *Packet) (int, error) {
 func (b *Base) patchRTPHeaderExtensions(p *Packet) error {
 	sendingAt := mono.Now()
 	if p.AbsSendTimeExtID != 0 {
-		absSendTime := rtp.NewAbsSendTimeExtension(sendingAt)
-		absSendTimeBytes, err := absSendTime.Marshal()
+		absSendTimeExt := rtp.AbsSendTimeExtension{
+			Timestamp: uint64(mediatransportutil.ToNtpTime(sendingAt) >> 14),
+		}
+		absSendTimeBytes, err := absSendTimeExt.Marshal()
 		if err != nil {
 			return err
 		}

livekit-server/pkg/sfu/pacer/leaky_bucket.go

@@ -80,8 +80,8 @@ func (l *LeakyBucket) Enqueue(p *Packet) {
 func (l *LeakyBucket) sendWorker() {
 	l.lock.RLock()
 	interval := l.interval
-	bitrate := l.bitrate
 	l.lock.RUnlock()
+	var bitrate int
 
 	timer := time.NewTimer(interval)
 	overage := 0

livekit-server/pkg/sfu/pacer/pacer.go

@@ -23,8 +23,27 @@ import (
 	"github.com/pion/webrtc/v4"
 )
 
+var (
+	PacketFactory = &sync.Pool{
+		New: func() any {
+			return &Packet{}
+		},
+	}
+)
+
+// --------------------------------------
+
+type PacerBehavior string
+
+const (
+	PacerBehaviorPassThrough PacerBehavior = "pass-through"
+	PacerBehaviorNoQueue     PacerBehavior = "no-queue"
+	PacerBehaviorLeakybucket PacerBehavior = "leaky-bucket"
+)
+
 type Packet struct {
 	Header             *rtp.Header
+	HeaderPool         *sync.Pool
 	HeaderSize         int
 	Payload            []byte
 	IsRTX              bool

livekit-server/pkg/sfu/packettrailer/packet_trailer.go

@@ -0,0 +1,46 @@
+// Copyright 2026 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package packettrailer
+
+var Magic = [4]byte{'L', 'K', 'T', 'S'}
+
+const (
+	xorByte = 0xFF
+
+	envelopeSize = 5 // 1B trailer_len + 4B magic
+)
+
+// StripTrailer returns the number of bytes to strip from the end of an RTP
+// payload if it contains an LKTS trailer. The trailer is located by checking
+// for the "LKTS" magic suffix and then reading the XORed trailer_len byte
+// immediately before it. Returns 0 if absent or ineligible.
+func StripTrailer(payload []byte, marker bool) int {
+	if !marker || len(payload) < envelopeSize {
+		return 0
+	}
+
+	tail := payload[len(payload)-4:]
+	if tail[0] != Magic[0] || tail[1] != Magic[1] ||
+		tail[2] != Magic[2] || tail[3] != Magic[3] {
+		return 0
+	}
+
+	trailerLen := int(payload[len(payload)-5] ^ xorByte)
+	if trailerLen < envelopeSize || trailerLen > len(payload) {
+		return 0
+	}
+
+	return trailerLen
+}

livekit-server/pkg/sfu/packettrailer/packet_trailer_test.go

@@ -0,0 +1,182 @@
+// Copyright 2026 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package packettrailer
+
+import (
+	"encoding/binary"
+	"testing"
+)
+
+const (
+	tagTimestampUs = 0x01
+	tagFrameID     = 0x02
+)
+
+// appendTLV appends a single XORed TLV element to dst.
+func appendTLV(dst []byte, tag byte, value []byte) []byte {
+	dst = append(dst, tag^xorByte, byte(len(value))^xorByte)
+	for _, b := range value {
+		dst = append(dst, b^xorByte)
+	}
+	return dst
+}
+
+// appendEnvelope appends the 5-byte envelope (XORed trailer_len + magic).
+func appendEnvelope(dst []byte, trailerLen byte) []byte {
+	dst = append(dst, trailerLen^xorByte)
+	dst = append(dst, Magic[:]...)
+	return dst
+}
+
+// makeTrailer builds a complete LKTS trailer with both timestamp and frame_id TLVs.
+func makeTrailer(timestampUs int64, frameID uint32) []byte {
+	var trailer []byte
+
+	var tsBuf [8]byte
+	binary.BigEndian.PutUint64(tsBuf[:], uint64(timestampUs))
+	trailer = appendTLV(trailer, tagTimestampUs, tsBuf[:])
+
+	var fidBuf [4]byte
+	binary.BigEndian.PutUint32(fidBuf[:], frameID)
+	trailer = appendTLV(trailer, tagFrameID, fidBuf[:])
+
+	trailerLen := byte(len(trailer) + envelopeSize)
+	trailer = appendEnvelope(trailer, trailerLen)
+	return trailer
+}
+
+// makePayloadWithTrailer builds a fake video payload followed by a full LKTS trailer.
+func makePayloadWithTrailer(videoLen int, timestampUs int64, frameID uint32) []byte {
+	video := make([]byte, videoLen)
+	for i := range video {
+		video[i] = byte(i)
+	}
+	return append(video, makeTrailer(timestampUs, frameID)...)
+}
+
+// makeTimestampOnlyTrailer builds a trailer with only the timestamp TLV.
+func makeTimestampOnlyTrailer(timestampUs int64) []byte {
+	var trailer []byte
+	var tsBuf [8]byte
+	binary.BigEndian.PutUint64(tsBuf[:], uint64(timestampUs))
+	trailer = appendTLV(trailer, tagTimestampUs, tsBuf[:])
+	trailerLen := byte(len(trailer) + envelopeSize)
+	trailer = appendEnvelope(trailer, trailerLen)
+	return trailer
+}
+
+func TestStripTrailer(t *testing.T) {
+	fullTrailerSize := 21 // (1+1+8) + (1+1+4) + 5
+	tsOnlyTrailerSize := 15 // (1+1+8) + 5
+
+	tests := []struct {
+		name      string
+		payload   []byte
+		marker    bool
+		wantStrip int
+	}{
+		{
+			name:      "marker set with full trailer (timestamp + frame_id)",
+			payload:   makePayloadWithTrailer(20, 1700000000000000, 42),
+			marker:    true,
+			wantStrip: fullTrailerSize,
+		},
+		{
+			name: "marker set with timestamp-only trailer",
+			payload: func() []byte {
+				video := make([]byte, 20)
+				return append(video, makeTimestampOnlyTrailer(1700000000000000)...)
+			}(),
+			marker:    true,
+			wantStrip: tsOnlyTrailerSize,
+		},
+		{
+			name:      "marker not set with valid trailer",
+			payload:   makePayloadWithTrailer(20, 1700000000000000, 42),
+			marker:    false,
+			wantStrip: 0,
+		},
+		{
+			name:      "marker set without magic",
+			payload:   make([]byte, 32),
+			marker:    true,
+			wantStrip: 0,
+		},
+		{
+			name:      "marker set but payload too short for envelope",
+			payload:   []byte{0x4C, 0x4B, 0x54, 0x53},
+			marker:    true,
+			wantStrip: 0,
+		},
+		{
+			name: "marker set with partial magic mismatch",
+			payload: func() []byte {
+				p := makePayloadWithTrailer(20, 1700000000000000, 42)
+				p[len(p)-1] = 'x'
+				return p
+			}(),
+			marker:    true,
+			wantStrip: 0,
+		},
+		{
+			name: "trailer_len exceeds payload length",
+			payload: func() []byte {
+				var trailer []byte
+				var tsBuf [8]byte
+				binary.BigEndian.PutUint64(tsBuf[:], 42)
+				trailer = appendTLV(trailer, tagTimestampUs, tsBuf[:])
+				trailer = appendEnvelope(trailer, 200)
+				return trailer
+			}(),
+			marker:    true,
+			wantStrip: 0,
+		},
+		{
+			name: "trailer_len smaller than envelope (invalid)",
+			payload: func() []byte {
+				video := make([]byte, 20)
+				var trailer []byte
+				var tsBuf [8]byte
+				binary.BigEndian.PutUint64(tsBuf[:], 42)
+				trailer = appendTLV(trailer, tagTimestampUs, tsBuf[:])
+				trailer = appendEnvelope(trailer, 3)
+				return append(video, trailer...)
+			}(),
+			marker:    true,
+			wantStrip: 0,
+		},
+		{
+			name:      "exactly envelope-only trailer",
+			payload:   appendEnvelope(nil, byte(envelopeSize)),
+			marker:    true,
+			wantStrip: envelopeSize,
+		},
+		{
+			name:      "empty payload",
+			payload:   []byte{},
+			marker:    true,
+			wantStrip: 0,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := StripTrailer(tt.payload, tt.marker)
+			if got != tt.wantStrip {
+				t.Errorf("StripTrailer() = %d, want %d", got, tt.wantStrip)
+			}
+		})
+	}
+}

livekit-server/pkg/sfu/playoutdelay_test.go

@@ -26,7 +26,9 @@ import (
 )
 
 func TestPlayoutDelay(t *testing.T) {
-	stats := rtpstats.NewRTPStatsSender(rtpstats.RTPStatsParams{ClockRate: 900000, Logger: logger.GetLogger()}, 128)
+	stats := rtpstats.NewRTPStatsSender(rtpstats.RTPStatsParams{}, 128)
+	stats.SetClockRate(90000)
+
 	c, err := NewPlayoutDelayController(100, 120, logger.GetLogger(), stats)
 	require.NoError(t, err)
 

livekit-server/pkg/sfu/receiver.go

@@ -15,181 +15,38 @@
 package sfu
 
 import (
-	"errors"
-	"io"
 	"strings"
 	"sync"
 	"time"
 
 	"github.com/pion/rtcp"
 	"github.com/pion/webrtc/v4"
-	"go.uber.org/atomic"
 
-	"github.com/livekit/mediatransportutil/pkg/bucket"
+	"github.com/livekit/protocol/codecs/mime"
 	"github.com/livekit/protocol/livekit"
 	"github.com/livekit/protocol/logger"
-	"github.com/livekit/protocol/utils"
 
-	"github.com/livekit/livekit-server/pkg/sfu/audio"
 	"github.com/livekit/livekit-server/pkg/sfu/buffer"
 	"github.com/livekit/livekit-server/pkg/sfu/connectionquality"
-	"github.com/livekit/livekit-server/pkg/sfu/mime"
-	dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor"
 	"github.com/livekit/livekit-server/pkg/sfu/rtpstats"
-	"github.com/livekit/livekit-server/pkg/sfu/streamtracker"
 )
 
-var (
-	ErrReceiverClosed        = errors.New("receiver closed")
-	ErrDownTrackAlreadyExist = errors.New("DownTrack already exist")
-	ErrBufferNotFound        = errors.New("buffer not found")
-	ErrDuplicateLayer        = errors.New("duplicate layer")
-)
-
-// --------------------------------------
-
-type PLIThrottleConfig struct {
-	LowQuality  time.Duration `yaml:"low_quality,omitempty"`
-	MidQuality  time.Duration `yaml:"mid_quality,omitempty"`
-	HighQuality time.Duration `yaml:"high_quality,omitempty"`
-}
-
-var (
-	DefaultPLIThrottleConfig = PLIThrottleConfig{
-		LowQuality:  500 * time.Millisecond,
-		MidQuality:  time.Second,
-		HighQuality: time.Second,
-	}
-)
-
-// --------------------------------------
-
-type AudioConfig struct {
-	audio.AudioLevelConfig `yaml:",inline"`
-
-	// enable red encoding downtrack for opus only audio up track
-	ActiveREDEncoding bool `yaml:"active_red_encoding,omitempty"`
-	// enable proxying weakest subscriber loss to publisher in RTCP Receiver Report
-	EnableLossProxying bool `yaml:"enable_loss_proxying,omitempty"`
-}
-
-var (
-	DefaultAudioConfig = AudioConfig{
-		AudioLevelConfig: audio.DefaultAudioLevelConfig,
-	}
-)
-
-// --------------------------------------
-
-type AudioLevelHandle func(level uint8, duration uint32)
-
-type Bitrates [buffer.DefaultMaxLayerSpatial + 1][buffer.DefaultMaxLayerTemporal + 1]int64
-
-type ReceiverCodecState int
-
-const (
-	ReceiverCodecStateNormal ReceiverCodecState = iota
-	ReceiverCodecStateSuspended
-	ReceiverCodecStateInvalid
-)
-
-// TrackReceiver defines an interface receive media from remote peer
-type TrackReceiver interface {
-	TrackID() livekit.TrackID
-	StreamID() string
-
-	// returns the initial codec of the receiver, it is determined by the track's codec
-	// and will not change if the codec changes during the session (publisher changes codec)
-	Codec() webrtc.RTPCodecParameters
-	Mime() mime.MimeType
-	HeaderExtensions() []webrtc.RTPHeaderExtensionParameter
-	IsClosed() bool
-
-	ReadRTP(buf []byte, layer uint8, esn uint64) (int, error)
-	GetLayeredBitrate() ([]int32, Bitrates)
-
-	GetAudioLevel() (float64, bool)
-
-	SendPLI(layer int32, force bool)
-
-	SetUpTrackPaused(paused bool)
-	SetMaxExpectedSpatialLayer(layer int32)
-
-	AddDownTrack(track TrackSender) error
-	DeleteDownTrack(participantID livekit.ParticipantID)
-	GetDownTracks() []TrackSender
-
-	DebugInfo() map[string]interface{}
-
-	TrackInfo() *livekit.TrackInfo
-	UpdateTrackInfo(ti *livekit.TrackInfo)
-
-	// Get primary receiver if this receiver represents a RED codec; otherwise it will return itself
-	GetPrimaryReceiverForRed() TrackReceiver
-
-	// Get red receiver for primary codec, used by forward red encodings for opus only codec
-	GetRedReceiver() TrackReceiver
-
-	GetTemporalLayerFpsForSpatial(layer int32) []float32
-
-	GetTrackStats() *livekit.RTPStats
-
-	// AddOnReady adds a function to be called when the receiver is ready, the callback
-	// could be called immediately if the receiver is ready when the callback is added
-	AddOnReady(func())
-
-	AddOnCodecStateChange(func(webrtc.RTPCodecParameters, ReceiverCodecState))
-	CodecState() ReceiverCodecState
-}
-
-type redPktWriteFunc func(pkt *buffer.ExtPacket, spatialLayer int32) int
+var _ TrackReceiver = (*WebRTCReceiver)(nil)
 
 // WebRTCReceiver receives a media track
 type WebRTCReceiver struct {
-	logger logger.Logger
+	*ReceiverBase
 
-	pliThrottleConfig PLIThrottleConfig
-	audioConfig       AudioConfig
-
-	trackID            livekit.TrackID
-	streamID           string
-	kind               webrtc.RTPCodecType
-	receiver           *webrtc.RTPReceiver
-	codec              webrtc.RTPCodecParameters
-	codecState         ReceiverCodecState
-	codecStateLock     sync.Mutex
-	onCodecStateChange []func(webrtc.RTPCodecParameters, ReceiverCodecState)
-	isSVC              bool
-	isRED              bool
-	onCloseHandler     func()
-	closeOnce          sync.Once
-	closed             atomic.Bool
-	useTrackers        bool
-	trackInfo          atomic.Pointer[livekit.TrackInfo]
+	receiver       *webrtc.RTPReceiver
+	onCloseHandler func()
 
 	onRTCP func([]rtcp.Packet)
 
-	bufferMu sync.RWMutex
-	buffers  [buffer.DefaultMaxLayerSpatial + 1]*buffer.Buffer
-	upTracks [buffer.DefaultMaxLayerSpatial + 1]TrackRemote
-	rtt      uint32
-
-	lbThreshold int
-
-	streamTrackerManager *StreamTrackerManager
-
-	downTrackSpreader *DownTrackSpreader
+	upTracksMu sync.Mutex
+	upTracks   [buffer.DefaultMaxLayerSpatial + 1]TrackRemote
 
 	connectionStats *connectionquality.ConnectionStats
-
-	onStatsUpdate    func(w *WebRTCReceiver, stat *livekit.AnalyticsStat)
-	onMaxLayerChange func(maxLayer int32)
-
-	primaryReceiver atomic.Pointer[RedPrimaryReceiver]
-	redReceiver     atomic.Pointer[RedReceiver]
-	redPktWriter    atomic.Value // redPktWriteFunc
-
-	forwardStats *ForwardStats
+	onStatsUpdate   func(w *WebRTCReceiver, stat *livekit.AnalyticsStat)
 }
 
 type ReceiverOpts func(w *WebRTCReceiver) *WebRTCReceiver
@@ -197,7 +54,7 @@ type ReceiverOpts func(w *WebRTCReceiver) *WebRTCReceiver
 // WithPliThrottleConfig indicates minimum time(ms) between sending PLIs
 func WithPliThrottleConfig(pliThrottleConfig PLIThrottleConfig) ReceiverOpts {
 	return func(w *WebRTCReceiver) *WebRTCReceiver {
-		w.pliThrottleConfig = pliThrottleConfig
+		w.ReceiverBase.SetPLIThrottleConfig(pliThrottleConfig)
 		return w
 	}
 }
@@ -205,15 +62,14 @@ func WithPliThrottleConfig(pliThrottleConfig PLIThrottleConfig) ReceiverOpts {
 // WithAudioConfig sets up parameters for active speaker detection
 func WithAudioConfig(audioConfig AudioConfig) ReceiverOpts {
 	return func(w *WebRTCReceiver) *WebRTCReceiver {
-		w.audioConfig = audioConfig
+		w.ReceiverBase.SetAudioConfig(audioConfig)
 		return w
 	}
 }
 
-// WithStreamTrackers enables StreamTracker use for simulcast
-func WithStreamTrackers() ReceiverOpts {
+func WithEnableRTPStreamRestartDetection(enable bool) ReceiverOpts {
 	return func(w *WebRTCReceiver) *WebRTCReceiver {
-		w.useTrackers = true
+		w.ReceiverBase.SetEnableRTPStreamRestartDetection(enable)
 		return w
 	}
 }
@@ -225,14 +81,14 @@ func WithStreamTrackers() ReceiverOpts {
 // Set to 0 (disabled) by default.
 func WithLoadBalanceThreshold(downTracks int) ReceiverOpts {
 	return func(w *WebRTCReceiver) *WebRTCReceiver {
-		w.lbThreshold = downTracks
+		w.ReceiverBase.SetLBThreshold(downTracks)
 		return w
 	}
 }
 
 func WithForwardStats(forwardStats *ForwardStats) ReceiverOpts {
 	return func(w *WebRTCReceiver) *WebRTCReceiver {
-		w.forwardStats = forwardStats
+		w.ReceiverBase.SetForwardStats(forwardStats)
 		return w
 	}
 }
@@ -248,243 +104,119 @@ func NewWebRTCReceiver(
 	opts ...ReceiverOpts,
 ) *WebRTCReceiver {
 	w := &WebRTCReceiver{
-		logger:     logger,
-		receiver:   receiver,
-		trackID:    livekit.TrackID(track.ID()),
-		streamID:   track.StreamID(),
-		codec:      track.Codec(),
-		codecState: ReceiverCodecStateNormal,
-		kind:       track.Kind(),
-		onRTCP:     onRTCP,
-		isSVC:      mime.IsMimeTypeStringSVC(track.Codec().MimeType),
-		isRED:      mime.IsMimeTypeStringRED(track.Codec().MimeType),
+		receiver: receiver,
+		onRTCP:   onRTCP,
 	}
 
+	w.ReceiverBase = NewReceiverBase(
+		ReceiverBaseParams{
+			TrackID:                      livekit.TrackID(track.ID()),
+			StreamID:                     track.StreamID(),
+			Kind:                         track.Kind(),
+			Codec:                        track.Codec(),
+			HeaderExtensions:             receiver.GetParameters().HeaderExtensions,
+			Logger:                       logger,
+			StreamTrackerManagerConfig:   streamTrackerManagerConfig,
+			StreamTrackerManagerListener: w,
+			IsSelfClosing:                true,
+			OnClosed:                     w.onClosed,
+		},
+		trackInfo,
+		ReceiverCodecStateNormal,
+	)
+
 	for _, opt := range opts {
 		w = opt(w)
 	}
-	w.trackInfo.Store(utils.CloneProto(trackInfo))
-
-	w.downTrackSpreader = NewDownTrackSpreader(DownTrackSpreaderParams{
-		Threshold: w.lbThreshold,
-		Logger:    logger,
-	})
 
 	w.connectionStats = connectionquality.NewConnectionStats(connectionquality.ConnectionStatsParams{
 		ReceiverProvider: w,
-		Logger:           w.logger.WithValues("direction", "up"),
+		Logger:           logger.WithValues("direction", "up"),
 	})
 	w.connectionStats.OnStatsUpdate(func(_cs *connectionquality.ConnectionStats, stat *livekit.AnalyticsStat) {
 		if w.onStatsUpdate != nil {
 			w.onStatsUpdate(w, stat)
 		}
 	})
+	codec := track.Codec()
 	w.connectionStats.Start(
-		mime.NormalizeMimeType(w.codec.MimeType),
+		mime.NormalizeMimeType(codec.MimeType),
 		// TODO: technically not correct to declare FEC on when RED. Need the primary codec's fmtp line to check.
-		mime.IsMimeTypeStringRED(w.codec.MimeType) || strings.Contains(strings.ToLower(w.codec.SDPFmtpLine), "useinbandfec=1"),
+		mime.IsMimeTypeStringRED(codec.MimeType) || strings.Contains(strings.ToLower(codec.SDPFmtpLine), "useinbandfec=1"),
 	)
 
-	w.streamTrackerManager = NewStreamTrackerManager(logger, trackInfo, w.isSVC, w.codec.ClockRate, streamTrackerManagerConfig)
-	w.streamTrackerManager.SetListener(w)
-	// SVC-TODO: Handle DD for non-SVC cases???
-	if w.isSVC {
-		for _, ext := range receiver.GetParameters().HeaderExtensions {
-			if ext.URI == dd.ExtensionURI {
-				w.streamTrackerManager.AddDependencyDescriptorTrackers()
-				break
-			}
-		}
-	}
+	w.UpdateTrackInfo(trackInfo)
 
 	return w
 }
 
-func (w *WebRTCReceiver) TrackInfo() *livekit.TrackInfo {
-	return w.trackInfo.Load()
-}
-
-func (w *WebRTCReceiver) UpdateTrackInfo(ti *livekit.TrackInfo) {
-	w.trackInfo.Store(utils.CloneProto(ti))
-	w.streamTrackerManager.UpdateTrackInfo(ti)
-}
-
 func (w *WebRTCReceiver) OnStatsUpdate(fn func(w *WebRTCReceiver, stat *livekit.AnalyticsStat)) {
 	w.onStatsUpdate = fn
 }
 
-func (w *WebRTCReceiver) OnMaxLayerChange(fn func(maxLayer int32)) {
-	w.bufferMu.Lock()
-	w.onMaxLayerChange = fn
-	w.bufferMu.Unlock()
-}
-
-func (w *WebRTCReceiver) getOnMaxLayerChange() func(maxLayer int32) {
-	w.bufferMu.RLock()
-	defer w.bufferMu.RUnlock()
-
-	return w.onMaxLayerChange
-}
-
 func (w *WebRTCReceiver) GetConnectionScoreAndQuality() (float32, livekit.ConnectionQuality) {
 	return w.connectionStats.GetScoreAndQuality()
 }
 
-func (w *WebRTCReceiver) IsClosed() bool {
-	return w.closed.Load()
-}
-
-func (w *WebRTCReceiver) SetRTT(rtt uint32) {
-	w.bufferMu.Lock()
-	if w.rtt == rtt {
-		w.bufferMu.Unlock()
-		return
-	}
-
-	w.rtt = rtt
-	buffers := w.buffers
-	w.bufferMu.Unlock()
-
-	for _, buff := range buffers {
-		if buff == nil {
-			continue
-		}
-
-		buff.SetRTT(rtt)
-	}
-}
-
-func (w *WebRTCReceiver) StreamID() string {
-	return w.streamID
-}
-
-func (w *WebRTCReceiver) TrackID() livekit.TrackID {
-	return w.trackID
-}
-
 func (w *WebRTCReceiver) ssrc(layer int) uint32 {
+	w.upTracksMu.Lock()
+	defer w.upTracksMu.Unlock()
+
 	if track := w.upTracks[layer]; track != nil {
 		return uint32(track.SSRC())
 	}
 	return 0
 }
 
-func (w *WebRTCReceiver) Codec() webrtc.RTPCodecParameters {
-	return w.codec
-}
-
-func (w *WebRTCReceiver) Mime() mime.MimeType {
-	return mime.NormalizeMimeType(w.codec.MimeType)
-}
-
-func (w *WebRTCReceiver) HeaderExtensions() []webrtc.RTPHeaderExtensionParameter {
-	return w.receiver.GetParameters().HeaderExtensions
-}
-
-func (w *WebRTCReceiver) Kind() webrtc.RTPCodecType {
-	return w.kind
-}
-
 func (w *WebRTCReceiver) AddUpTrack(track TrackRemote, buff *buffer.Buffer) error {
-	if w.closed.Load() {
+	if w.isClosed.Load() {
 		return ErrReceiverClosed
 	}
 
 	layer := int32(0)
-	if w.Kind() == webrtc.RTPCodecTypeVideo && !w.isSVC {
-		layer = buffer.RidToSpatialLayer(track.RID(), w.trackInfo.Load())
+	if w.Kind() == webrtc.RTPCodecTypeVideo && w.videoLayerMode != livekit.VideoLayer_MULTIPLE_SPATIAL_LAYERS_PER_STREAM {
+		layer = buffer.GetSpatialLayerForRid(w.Mime(), track.RID(), w.ReceiverBase.TrackInfo())
 	}
-	buff.SetLogger(w.logger.WithValues("layer", layer))
-	buff.SetAudioLevelParams(audio.AudioLevelParams{
-		Config: w.audioConfig.AudioLevelConfig,
-	})
-	buff.SetAudioLossProxying(w.audioConfig.EnableLossProxying)
-	buff.OnRtcpFeedback(w.sendRTCP)
-	buff.OnRtcpSenderReport(func() {
-		srData := buff.GetSenderReportData()
-		w.downTrackSpreader.Broadcast(func(dt TrackSender) {
-			_ = dt.HandleRTCPSenderReportData(w.codec.PayloadType, w.isSVC, layer, srData)
-		})
-	})
-
-	if w.Kind() == webrtc.RTPCodecTypeVideo && layer == 0 {
-		buff.OnCodecChange(w.handleCodecChange)
+	if layer < 0 {
+		w.ReceiverBase.Logger().Warnw(
+			"invalid layer", nil,
+			"rid", track.RID(),
+			"trackInfo", logger.Proto(w.ReceiverBase.TrackInfo()),
+		)
+		return ErrInvalidLayer
 	}
 
-	var duration time.Duration
-	switch layer {
-	case 2:
-		duration = w.pliThrottleConfig.HighQuality
-	case 1:
-		duration = w.pliThrottleConfig.MidQuality
-	case 0:
-		duration = w.pliThrottleConfig.LowQuality
-	default:
-		duration = w.pliThrottleConfig.MidQuality
-	}
-	if duration != 0 {
-		buff.SetPLIThrottle(duration.Nanoseconds())
-	}
-
-	w.bufferMu.Lock()
+	w.upTracksMu.Lock()
 	if w.upTracks[layer] != nil {
-		w.bufferMu.Unlock()
+		w.upTracksMu.Unlock()
 		return ErrDuplicateLayer
 	}
 	w.upTracks[layer] = track
-	w.buffers[layer] = buff
-	rtt := w.rtt
-	w.bufferMu.Unlock()
+	w.upTracksMu.Unlock()
 
-	buff.SetRTT(rtt)
-	buff.SetPaused(w.streamTrackerManager.IsPaused())
-
-	if w.Kind() == webrtc.RTPCodecTypeVideo && w.useTrackers {
-		w.streamTrackerManager.AddTracker(layer)
-	}
-
-	go w.forwardRTP(layer, buff)
+	w.ReceiverBase.AddBuffer(buff, layer)
+	buff.OnRtcpFeedback(w.sendRTCP)
+	w.ReceiverBase.StartBuffer(buff, layer)
 	return nil
 }
 
-// SetUpTrackPaused indicates upstream will not be sending any data.
-// this will reflect the "muted" status and will pause streamtracker to ensure we don't turn off
-// the layer
-func (w *WebRTCReceiver) SetUpTrackPaused(paused bool) {
-	w.streamTrackerManager.SetPaused(paused)
+func (w *WebRTCReceiver) NumUpTracks() int {
+	numUpTracks := 0
 
-	w.bufferMu.RLock()
-	for _, buff := range w.buffers {
-		if buff == nil {
-			continue
+	w.upTracksMu.Lock()
+	for _, track := range w.upTracks {
+		if track != nil {
+			numUpTracks++
 		}
-
-		buff.SetPaused(paused)
 	}
-	w.bufferMu.RUnlock()
+	w.upTracksMu.Unlock()
 
-	w.connectionStats.UpdateMute(paused)
+	return numUpTracks
 }
 
-func (w *WebRTCReceiver) AddDownTrack(track TrackSender) error {
-	if w.closed.Load() {
-		return ErrReceiverClosed
-	}
-
-	if w.downTrackSpreader.HasDownTrack(track.SubscriberID()) {
-		w.logger.Infow("subscriberID already exists, replacing downtrack", "subscriberID", track.SubscriberID())
-	}
-
-	track.UpTrackMaxPublishedLayerChange(w.streamTrackerManager.GetMaxPublishedLayer())
-	track.UpTrackMaxTemporalLayerSeenChange(w.streamTrackerManager.GetMaxTemporalLayerSeen())
-
-	w.downTrackSpreader.Store(track)
-	w.logger.Debugw("downtrack added", "subscriberID", track.SubscriberID())
-	return nil
-}
-
-func (w *WebRTCReceiver) GetDownTracks() []TrackSender {
-	return w.downTrackSpreader.GetDownTracks()
+func (w *WebRTCReceiver) UpdateTrackInfo(ti *livekit.TrackInfo) {
+	w.ReceiverBase.UpdateTrackInfo(ti)
+	w.connectionStats.UpdateMute(ti.GetMuted())
 }
 
 func (w *WebRTCReceiver) notifyMaxExpectedLayer(layer int32) {
@@ -499,9 +231,8 @@ func (w *WebRTCReceiver) notifyMaxExpectedLayer(layer int32) {
 	}
 
 	expectedBitrate := int64(0)
-	for _, vl := range ti.Layers {
-		l := buffer.VideoQualityToSpatialLayer(vl.Quality, ti)
-		if l <= layer {
+	for _, vl := range buffer.GetVideoLayersForMimeType(w.Mime(), ti) {
+		if vl.SpatialLayer <= layer {
 			expectedBitrate += int64(vl.Bitrate)
 		}
 	}
@@ -510,89 +241,54 @@ func (w *WebRTCReceiver) notifyMaxExpectedLayer(layer int32) {
 }
 
 func (w *WebRTCReceiver) SetMaxExpectedSpatialLayer(layer int32) {
-	w.streamTrackerManager.SetMaxExpectedSpatialLayer(layer)
+	w.ReceiverBase.SetMaxExpectedSpatialLayer(layer)
+
 	w.notifyMaxExpectedLayer(layer)
 
 	if layer == buffer.InvalidLayerSpatial {
 		w.connectionStats.UpdateLayerMute(true)
 	} else {
 		w.connectionStats.UpdateLayerMute(false)
-		w.connectionStats.AddLayerTransition(w.streamTrackerManager.DistanceToDesired())
+		w.connectionStats.AddLayerTransition(w.ReceiverBase.StreamTrackerManager().DistanceToDesired())
 	}
 }
 
 // StreamTrackerManagerListener.OnAvailableLayersChanged
 func (w *WebRTCReceiver) OnAvailableLayersChanged() {
-	w.downTrackSpreader.Broadcast(func(dt TrackSender) {
-		dt.UpTrackLayersChange()
-	})
-
-	w.connectionStats.AddLayerTransition(w.streamTrackerManager.DistanceToDesired())
+	w.connectionStats.AddLayerTransition(w.ReceiverBase.StreamTrackerManager().DistanceToDesired())
 }
 
 // StreamTrackerManagerListener.OnBitrateAvailabilityChanged
 func (w *WebRTCReceiver) OnBitrateAvailabilityChanged() {
-	w.downTrackSpreader.Broadcast(func(dt TrackSender) {
-		dt.UpTrackBitrateAvailabilityChange()
-	})
 }
 
 // StreamTrackerManagerListener.OnMaxPublishedLayerChanged
 func (w *WebRTCReceiver) OnMaxPublishedLayerChanged(maxPublishedLayer int32) {
-	w.downTrackSpreader.Broadcast(func(dt TrackSender) {
-		dt.UpTrackMaxPublishedLayerChange(maxPublishedLayer)
-	})
-
 	w.notifyMaxExpectedLayer(maxPublishedLayer)
-	w.connectionStats.AddLayerTransition(w.streamTrackerManager.DistanceToDesired())
+	w.connectionStats.AddLayerTransition(w.ReceiverBase.StreamTrackerManager().DistanceToDesired())
 }
 
 // StreamTrackerManagerListener.OnMaxTemporalLayerSeenChanged
 func (w *WebRTCReceiver) OnMaxTemporalLayerSeenChanged(maxTemporalLayerSeen int32) {
-	w.downTrackSpreader.Broadcast(func(dt TrackSender) {
-		dt.UpTrackMaxTemporalLayerSeenChange(maxTemporalLayerSeen)
-	})
-
-	w.connectionStats.AddLayerTransition(w.streamTrackerManager.DistanceToDesired())
+	w.connectionStats.AddLayerTransition(w.ReceiverBase.StreamTrackerManager().DistanceToDesired())
 }
 
 // StreamTrackerManagerListener.OnMaxAvailableLayerChanged
 func (w *WebRTCReceiver) OnMaxAvailableLayerChanged(maxAvailableLayer int32) {
-	if onMaxLayerChange := w.getOnMaxLayerChange(); onMaxLayerChange != nil {
-		onMaxLayerChange(maxAvailableLayer)
-	}
 }
 
 // StreamTrackerManagerListener.OnBitrateReport
 func (w *WebRTCReceiver) OnBitrateReport(availableLayers []int32, bitrates Bitrates) {
-	w.downTrackSpreader.Broadcast(func(dt TrackSender) {
-		dt.UpTrackBitrateReport(availableLayers, bitrates)
-	})
-
-	w.connectionStats.AddLayerTransition(w.streamTrackerManager.DistanceToDesired())
+	w.connectionStats.AddLayerTransition(w.ReceiverBase.StreamTrackerManager().DistanceToDesired())
 }
 
-func (w *WebRTCReceiver) GetLayeredBitrate() ([]int32, Bitrates) {
-	return w.streamTrackerManager.GetLayeredBitrate()
-}
-
-// OnCloseHandler method to be called on remote tracked removed
+// OnCloseHandler method to be called on remote track removed
 func (w *WebRTCReceiver) OnCloseHandler(fn func()) {
 	w.onCloseHandler = fn
 }
 
-// DeleteDownTrack removes a DownTrack from a Receiver
-func (w *WebRTCReceiver) DeleteDownTrack(subscriberID livekit.ParticipantID) {
-	if w.closed.Load() {
-		return
-	}
-
-	w.downTrackSpreader.Free(subscriberID)
-	w.logger.Debugw("downtrack deleted", "subscriberID", subscriberID)
-}
-
 func (w *WebRTCReceiver) sendRTCP(packets []rtcp.Packet) {
-	if packets == nil || w.closed.Load() {
+	if packets == nil || w.isClosed.Load() {
 		return
 	}
 
@@ -601,93 +297,10 @@ func (w *WebRTCReceiver) sendRTCP(packets []rtcp.Packet) {
 	}
 }
 
-func (w *WebRTCReceiver) SendPLI(layer int32, force bool) {
-	// SVC-TODO :  should send LRR (Layer Refresh Request) instead of PLI
-	buff := w.getBuffer(layer)
-	if buff == nil {
-		return
-	}
-
-	buff.SendPLI(force)
-}
-
-func (w *WebRTCReceiver) getBuffer(layer int32) *buffer.Buffer {
-	w.bufferMu.RLock()
-	defer w.bufferMu.RUnlock()
-
-	return w.getBufferLocked(layer)
-}
-
-func (w *WebRTCReceiver) getBufferLocked(layer int32) *buffer.Buffer {
-	// for svc codecs, use layer = 0 always.
-	// spatial layers are in-built and handled by single buffer
-	if w.isSVC {
-		layer = 0
-	}
-
-	if layer < 0 || int(layer) >= len(w.buffers) {
-		return nil
-	}
-
-	return w.buffers[layer]
-}
-
-func (w *WebRTCReceiver) ReadRTP(buf []byte, layer uint8, esn uint64) (int, error) {
-	b := w.getBuffer(int32(layer))
-	if b == nil {
-		return 0, ErrBufferNotFound
-	}
-
-	return b.GetPacket(buf, esn)
-}
-
-func (w *WebRTCReceiver) GetTrackStats() *livekit.RTPStats {
-	w.bufferMu.RLock()
-	defer w.bufferMu.RUnlock()
-
-	stats := make([]*livekit.RTPStats, 0, len(w.buffers))
-	for _, buff := range w.buffers {
-		if buff == nil {
-			continue
-		}
-
-		sswl := buff.GetStats()
-		if sswl == nil {
-			continue
-		}
-
-		stats = append(stats, sswl)
-	}
-
-	return rtpstats.AggregateRTPStats(stats)
-}
-
-func (w *WebRTCReceiver) GetAudioLevel() (float64, bool) {
-	if w.Kind() == webrtc.RTPCodecTypeVideo {
-		return 0, false
-	}
-
-	w.bufferMu.RLock()
-	defer w.bufferMu.RUnlock()
-
-	for _, buff := range w.buffers {
-		if buff == nil {
-			continue
-		}
-
-		return buff.GetAudioLevel()
-	}
-
-	return 0, false
-}
-
 func (w *WebRTCReceiver) GetDeltaStats() map[uint32]*buffer.StreamStatsWithLayers {
-	w.bufferMu.RLock()
-	defer w.bufferMu.RUnlock()
-
-	deltaStats := make(map[uint32]*buffer.StreamStatsWithLayers, len(w.buffers))
-
-	for layer, buff := range w.buffers {
+	buffers := w.ReceiverBase.GetAllBuffers()
+	deltaStats := make(map[uint32]*buffer.StreamStatsWithLayers, len(buffers))
+	for layer, buff := range buffers {
 		if buff == nil {
 			continue
 		}
@@ -709,11 +322,9 @@ func (w *WebRTCReceiver) GetDeltaStats() map[uint32]*buffer.StreamStatsWithLayer
 }
 
 func (w *WebRTCReceiver) GetLastSenderReportTime() time.Time {
-	w.bufferMu.RLock()
-	defer w.bufferMu.RUnlock()
-
+	buffers := w.ReceiverBase.GetAllBuffers()
 	latestSRTime := time.Time{}
-	for _, buff := range w.buffers {
+	for _, buff := range buffers {
 		if buff == nil {
 			continue
 		}
@@ -727,120 +338,22 @@ func (w *WebRTCReceiver) GetLastSenderReportTime() time.Time {
 	return latestSRTime
 }
 
-func (w *WebRTCReceiver) forwardRTP(layer int32, buff *buffer.Buffer) {
-	defer func() {
-		w.closeOnce.Do(func() {
-			w.closed.Store(true)
-			w.closeTracks()
-			if pr := w.primaryReceiver.Load(); pr != nil {
-				pr.Close()
-			}
-			if pr := w.redReceiver.Load(); pr != nil {
-				pr.Close()
-			}
-		})
-
-		w.streamTrackerManager.RemoveTracker(layer)
-		if w.isSVC {
-			w.streamTrackerManager.RemoveAllTrackers()
-		}
-	}()
-
-	var spatialTrackers [buffer.DefaultMaxLayerSpatial + 1]streamtracker.StreamTrackerWorker
-	if layer < 0 || int(layer) >= len(spatialTrackers) {
-		w.logger.Errorw("invalid layer", nil, "layer", layer)
-		return
-	}
-	spatialTrackers[layer] = w.streamTrackerManager.GetTracker(layer)
-
-	pktBuf := make([]byte, bucket.MaxPktSize)
-	for {
-		pkt, err := buff.ReadExtended(pktBuf)
-		if err == io.EOF {
-			return
-		}
-
-		if pkt.Packet.PayloadType != uint8(w.codec.PayloadType) {
-			// drop packets as we don't support codec fallback directly
-			continue
-		}
-
-		spatialLayer := layer
-		if pkt.Spatial >= 0 {
-			// svc packet, take spatial layer info from packet
-			spatialLayer = pkt.Spatial
-		}
-		if int(spatialLayer) >= len(spatialTrackers) {
-			w.logger.Errorw(
-				"unexpected spatial layer", nil,
-				"spatialLayer", spatialLayer,
-				"pktSpatialLayer", pkt.Spatial,
-			)
-			continue
-		}
-
-		writeCount := w.downTrackSpreader.Broadcast(func(dt TrackSender) {
-			_ = dt.WriteRTP(pkt, spatialLayer)
-		})
-
-		if f := w.redPktWriter.Load(); f != nil {
-			writeCount += f.(redPktWriteFunc)(pkt, spatialLayer)
-		}
-
-		// track delay/jitter
-		if writeCount > 0 && w.forwardStats != nil {
-			w.forwardStats.Update(pkt.Arrival, time.Now().UnixNano())
-		}
-
-		// track video layers
-		if w.Kind() == webrtc.RTPCodecTypeVideo {
-			if spatialTrackers[spatialLayer] == nil {
-				spatialTrackers[spatialLayer] = w.streamTrackerManager.GetTracker(spatialLayer)
-				if spatialTrackers[spatialLayer] == nil {
-					spatialTrackers[spatialLayer] = w.streamTrackerManager.AddTracker(spatialLayer)
-				}
-			}
-			if spatialTrackers[spatialLayer] != nil {
-				spatialTrackers[spatialLayer].Observe(
-					pkt.Temporal,
-					len(pkt.RawPacket),
-					len(pkt.Packet.Payload),
-					pkt.Packet.Marker,
-					pkt.Packet.Timestamp,
-					pkt.DependencyDescriptor,
-				)
-			}
-		}
-	}
-}
-
-// closeTracks close all tracks from Receiver
-func (w *WebRTCReceiver) closeTracks() {
+func (w *WebRTCReceiver) onClosed() {
 	w.connectionStats.Close()
-	w.streamTrackerManager.Close()
-
-	closeTrackSenders(w.downTrackSpreader.ResetAndGetDownTracks())
 
 	if w.onCloseHandler != nil {
 		w.onCloseHandler()
 	}
 }
 
-func (w *WebRTCReceiver) DebugInfo() map[string]interface{} {
-	isSimulcast := !w.isSVC
-	if ti := w.trackInfo.Load(); ti != nil {
-		isSimulcast = isSimulcast && len(ti.Layers) > 1
-	}
-	info := map[string]interface{}{
-		"SVC":       w.isSVC,
-		"Simulcast": isSimulcast,
-	}
+func (w *WebRTCReceiver) DebugInfo() map[string]any {
+	info := w.ReceiverBase.DebugInfo()
 
-	w.bufferMu.RLock()
-	upTrackInfo := make([]map[string]interface{}, 0, len(w.upTracks))
+	w.upTracksMu.Lock()
+	upTrackInfo := make([]map[string]any, 0, len(w.upTracks))
 	for layer, ut := range w.upTracks {
 		if ut != nil {
-			upTrackInfo = append(upTrackInfo, map[string]interface{}{
+			upTrackInfo = append(upTrackInfo, map[string]any{
 				"Layer": layer,
 				"SSRC":  ut.SSRC(),
 				"Msid":  ut.Msid(),
@@ -848,110 +361,10 @@ func (w *WebRTCReceiver) DebugInfo() map[string]interface{} {
 			})
 		}
 	}
-	w.bufferMu.RUnlock()
+	w.upTracksMu.Unlock()
 	info["UpTracks"] = upTrackInfo
 
 	return info
 }
 
-func (w *WebRTCReceiver) GetPrimaryReceiverForRed() TrackReceiver {
-	if !w.isRED || w.closed.Load() {
-		return w
-	}
-
-	if w.primaryReceiver.Load() == nil {
-		pr := NewRedPrimaryReceiver(w, DownTrackSpreaderParams{
-			Threshold: w.lbThreshold,
-			Logger:    w.logger,
-		})
-		if w.primaryReceiver.CompareAndSwap(nil, pr) {
-			w.redPktWriter.Store(redPktWriteFunc(pr.ForwardRTP))
-		}
-	}
-	return w.primaryReceiver.Load()
-}
-
-func (w *WebRTCReceiver) GetRedReceiver() TrackReceiver {
-	if w.isRED || w.closed.Load() {
-		return w
-	}
-
-	if w.redReceiver.Load() == nil {
-		pr := NewRedReceiver(w, DownTrackSpreaderParams{
-			Threshold: w.lbThreshold,
-			Logger:    w.logger,
-		})
-		if w.redReceiver.CompareAndSwap(nil, pr) {
-			w.redPktWriter.Store(redPktWriteFunc(pr.ForwardRTP))
-		}
-	}
-	return w.redReceiver.Load()
-}
-
-func (w *WebRTCReceiver) GetTemporalLayerFpsForSpatial(layer int32) []float32 {
-	b := w.getBuffer(layer)
-	if b == nil {
-		return nil
-	}
-
-	if !w.isSVC {
-		return b.GetTemporalLayerFpsForSpatial(0)
-	}
-
-	return b.GetTemporalLayerFpsForSpatial(layer)
-}
-
-func (w *WebRTCReceiver) AddOnReady(fn func()) {
-	// webRTCReceiver is always ready after created
-	fn()
-}
-
-func (w *WebRTCReceiver) handleCodecChange(newCodec webrtc.RTPCodecParameters) {
-	// we don't support the codec fallback directly, set the codec state to invalid once it happens
-	w.SetCodecState(ReceiverCodecStateInvalid)
-}
-
-func (w *WebRTCReceiver) AddOnCodecStateChange(f func(webrtc.RTPCodecParameters, ReceiverCodecState)) {
-	w.codecStateLock.Lock()
-	w.onCodecStateChange = append(w.onCodecStateChange, f)
-	w.codecStateLock.Unlock()
-}
-
-func (w *WebRTCReceiver) CodecState() ReceiverCodecState {
-	w.codecStateLock.Lock()
-	defer w.codecStateLock.Unlock()
-
-	return w.codecState
-}
-
-func (w *WebRTCReceiver) SetCodecState(state ReceiverCodecState) {
-	w.codecStateLock.Lock()
-	if w.codecState == state || w.codecState == ReceiverCodecStateInvalid {
-		w.codecStateLock.Unlock()
-		return
-	}
-
-	w.codecState = state
-	fns := w.onCodecStateChange
-	w.codecStateLock.Unlock()
-
-	for _, f := range fns {
-		f(w.codec, state)
-	}
-}
-
 // -----------------------------------------------------------
-
-// closes all track senders in parallel, returns when all are closed
-func closeTrackSenders(senders []TrackSender) {
-	wg := sync.WaitGroup{}
-	for _, dt := range senders {
-		dt := dt
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			dt.Close()
-		}()
-	}
-	wg.Wait()
-}

livekit-server/pkg/sfu/receiver_test.go

@@ -102,15 +102,15 @@ func BenchmarkWriteRTP(b *testing.B) {
 }
 
 func benchmarkNoPool(b *testing.B, downTracks int) {
-	for i := 0; i < b.N; i++ {
-		for dt := 0; dt < downTracks; dt++ {
+	for b.Loop() {
+		for range downTracks {
 			writeRTP()
 		}
 	}
 }
 
 func benchmarkPool(b *testing.B, wp *workerpool.WorkerPool, buckets []int) {
-	for i := 0; i < b.N; i++ {
+	for b.Loop() {
 		var wg sync.WaitGroup
 		for j := range buckets {
 			downTracks := buckets[j]
@@ -130,7 +130,7 @@ func benchmarkPool(b *testing.B, wp *workerpool.WorkerPool, buckets []int) {
 }
 
 func benchmarkGoroutine(b *testing.B, buckets []int) {
-	for i := 0; i < b.N; i++ {
+	for b.Loop() {
 		var wg sync.WaitGroup
 		for j := range buckets {
 			downTracks := buckets[j]
@@ -150,7 +150,7 @@ func benchmarkGoroutine(b *testing.B, buckets []int) {
 }
 
 func benchmarkLoadBalanced(b *testing.B, numProcs, step, downTracks int) {
-	for i := 0; i < b.N; i++ {
+	for b.Loop() {
 		start := atomic.NewUint64(0)
 		step := uint64(step)
 		end := uint64(downTracks)
@@ -177,7 +177,7 @@ func benchmarkLoadBalanced(b *testing.B, numProcs, step, downTracks int) {
 }
 
 func benchmarkLoadBalancedPool(b *testing.B, wp *workerpool.WorkerPool, numProcs, step, downTracks int) {
-	for i := 0; i < b.N; i++ {
+	for b.Loop() {
 		start := atomic.NewUint64(0)
 		step := uint64(step)
 		end := uint64(downTracks)

livekit-server/pkg/sfu/redprimaryreceiver.go

@@ -21,12 +21,16 @@ import (
 	"go.uber.org/atomic"
 
 	"github.com/pion/rtp"
+	"github.com/pion/webrtc/v4"
 
 	"github.com/livekit/livekit-server/pkg/sfu/buffer"
+	"github.com/livekit/livekit-server/pkg/sfu/utils"
 	"github.com/livekit/protocol/livekit"
 	"github.com/livekit/protocol/logger"
 )
 
+var _ REDTransformer = (*RedPrimaryReceiver)(nil)
+
 var (
 	ErrIncompleteRedHeader = errors.New("incomplete red block header")
 	ErrIncompleteRedBlock  = errors.New("incomplete red block payload")
@@ -34,28 +38,30 @@ var (
 
 type RedPrimaryReceiver struct {
 	TrackReceiver
-	downTrackSpreader *DownTrackSpreader
+	downTrackSpreader *utils.DownTrackSpreader[TrackSender]
 	logger            logger.Logger
 	closed            atomic.Bool
 	redPT             uint8
 
-	firstPktReceived bool
-	lastSeq          uint16
+	firstPktReceived                  bool
+	lastSeq                           uint16
+	highestForwardedExtSequenceNumber uint64
+	highestForwardedExtTimestamp      uint64
 
 	// bitset for upstream packet receive history [lastSeq-8, lastSeq-1], bit 1 represents packet received
 	pktHistory byte
 }
 
-func NewRedPrimaryReceiver(receiver TrackReceiver, dsp DownTrackSpreaderParams) *RedPrimaryReceiver {
+func NewRedPrimaryReceiver(receiver TrackReceiver, dsp utils.DownTrackSpreaderParams) REDTransformer {
 	return &RedPrimaryReceiver{
 		TrackReceiver:     receiver,
-		downTrackSpreader: NewDownTrackSpreader(dsp),
+		downTrackSpreader: utils.NewDownTrackSpreader[TrackSender](dsp),
 		logger:            dsp.Logger,
 		redPT:             uint8(receiver.Codec().PayloadType),
 	}
 }
 
-func (r *RedPrimaryReceiver) ForwardRTP(pkt *buffer.ExtPacket, spatialLayer int32) int {
+func (r *RedPrimaryReceiver) ForwardRTP(pkt *buffer.ExtPacket, spatialLayer int32) int32 {
 	// extract primary payload from RED and forward to downtracks
 	if r.downTrackSpreader.DownTrackCount() == 0 {
 		return 0
@@ -63,9 +69,11 @@ func (r *RedPrimaryReceiver) ForwardRTP(pkt *buffer.ExtPacket, spatialLayer int3
 
 	if pkt.Packet.PayloadType != r.redPT {
 		// forward non-red packet directly
-		return r.downTrackSpreader.Broadcast(func(dt TrackSender) {
-			_ = dt.WriteRTP(pkt, spatialLayer)
+		var writeCount atomic.Int32
+		r.downTrackSpreader.Broadcast(func(dt TrackSender) {
+			writeCount.Add(dt.WriteRTP(pkt, spatialLayer))
 		})
+		return writeCount.Load()
 	}
 
 	pkts, err := r.getSendPktsFromRed(pkt.Packet)
@@ -74,7 +82,7 @@ func (r *RedPrimaryReceiver) ForwardRTP(pkt *buffer.ExtPacket, spatialLayer int3
 		return 0
 	}
 
-	var count int
+	var writeCount atomic.Int32
 	for i, sendPkt := range pkts {
 		pPkt := *pkt
 		if i != len(pkts)-1 {
@@ -85,13 +93,53 @@ func (r *RedPrimaryReceiver) ForwardRTP(pkt *buffer.ExtPacket, spatialLayer int3
 		}
 		pPkt.Packet = sendPkt
 
+		if r.highestForwardedExtTimestamp != 0 {
+			if (pPkt.ExtSequenceNumber > r.highestForwardedExtSequenceNumber && pPkt.ExtTimestamp < r.highestForwardedExtTimestamp) ||
+				(pPkt.ExtSequenceNumber < r.highestForwardedExtSequenceNumber && pPkt.ExtTimestamp > r.highestForwardedExtTimestamp) {
+				r.logger.Warnw(
+					"sequence number OR timestamp inversion, dropping", nil,
+					"numPackets", len(pkts),
+					"primaryIncomingSN", pkt.Packet.Header.SequenceNumber,
+					"primaryIncomingTS", pkt.Packet.Header.Timestamp,
+					"primaryExtractedSN", pkts[len(pkts)-1].SequenceNumber,
+					"primaryExtractedTS", pkts[len(pkts)-1].Timestamp,
+					"primaryESN", pkt.ExtSequenceNumber,
+					"primaryETS", pkt.ExtTimestamp,
+					"packetIndex", i,
+					"packetExtractedSN", pkts[i].SequenceNumber,
+					"packetESN", pPkt.ExtSequenceNumber,
+					"packetExtractedTS", pkts[i].Timestamp,
+					"packetETS", pPkt.ExtTimestamp,
+					"pktHistory", r.pktHistory,
+					"redHeader", pkt.Packet.Payload[:10],
+					"payloadSize", len(pkt.Packet.Payload),
+					"lastSeq", r.lastSeq,
+					"highestFowardedExtSequenceNumber", r.highestForwardedExtSequenceNumber,
+					"highestFowardedExtTimestamp", r.highestForwardedExtTimestamp,
+				)
+				continue // drop the packet which causes the inversion
+			}
+		}
+		r.highestForwardedExtSequenceNumber = max(r.highestForwardedExtSequenceNumber, pPkt.ExtSequenceNumber)
+		r.highestForwardedExtTimestamp = max(r.highestForwardedExtTimestamp, pPkt.ExtTimestamp)
+
 		// not modify the ExtPacket.RawPacket here for performance since it is not used by the DownTrack,
 		// otherwise it should be set to the correct value (marshal the primary rtp packet)
-		count += r.downTrackSpreader.Broadcast(func(dt TrackSender) {
-			_ = dt.WriteRTP(&pPkt, spatialLayer)
+		r.downTrackSpreader.Broadcast(func(dt TrackSender) {
+			writeCount.Add(dt.WriteRTP(&pPkt, spatialLayer))
 		})
 	}
-	return count
+	return writeCount.Load()
+}
+
+func (r *RedPrimaryReceiver) ForwardRTCPSenderReport(
+	payloadType webrtc.PayloadType,
+	layer int32,
+	publisherSRData *livekit.RTCPSenderReportState,
+) {
+	r.downTrackSpreader.Broadcast(func(dt TrackSender) {
+		_ = dt.HandleRTCPSenderReportData(payloadType, layer, publisherSRData)
+	})
 }
 
 func (r *RedPrimaryReceiver) AddDownTrack(track TrackSender) error {
@@ -127,6 +175,12 @@ func (r *RedPrimaryReceiver) ResyncDownTracks() {
 	})
 }
 
+func (r *RedPrimaryReceiver) OnStreamRestart() {
+	r.downTrackSpreader.Broadcast(func(dt TrackSender) {
+		dt.ReceiverRestart(r)
+	})
+}
+
 func (r *RedPrimaryReceiver) IsClosed() bool {
 	return r.closed.Load()
 }
@@ -191,7 +245,7 @@ func (r *RedPrimaryReceiver) getSendPktsFromRed(rtp *rtp.Packet) ([]*rtp.Packet,
 	var recoverBits byte
 	if needRecover {
 		bitIndex := r.lastSeq - rtp.SequenceNumber
-		for i := 0; i < maxRedCount; i++ {
+		for i := range maxRedCount {
 			if bitIndex > 7 {
 				break
 			}
@@ -205,6 +259,8 @@ func (r *RedPrimaryReceiver) getSendPktsFromRed(rtp *rtp.Packet) ([]*rtp.Packet,
 	return extractPktsFromRed(rtp, recoverBits)
 }
 
+// ---------------------------------------
+
 type block struct {
 	tsOffset uint32
 	length   int

livekit-server/pkg/sfu/redreceiver.go

@@ -21,13 +21,17 @@ import (
 	"go.uber.org/atomic"
 
 	"github.com/pion/rtp"
+	"github.com/pion/webrtc/v4"
 
 	"github.com/livekit/livekit-server/pkg/sfu/buffer"
+	"github.com/livekit/livekit-server/pkg/sfu/utils"
 	"github.com/livekit/mediatransportutil/pkg/bucket"
 	"github.com/livekit/protocol/livekit"
 	"github.com/livekit/protocol/logger"
 )
 
+var _ REDTransformer = (*RedReceiver)(nil)
+
 const (
 	maxRedCount   = 2
 	mtuSize       = 1500
@@ -42,22 +46,22 @@ const (
 
 type RedReceiver struct {
 	TrackReceiver
-	downTrackSpreader *DownTrackSpreader
+	downTrackSpreader *utils.DownTrackSpreader[TrackSender]
 	logger            logger.Logger
 	closed            atomic.Bool
 	pktBuff           [maxRedCount]*rtp.Packet
 	redPayloadBuf     [mtuSize]byte
 }
 
-func NewRedReceiver(receiver TrackReceiver, dsp DownTrackSpreaderParams) *RedReceiver {
+func NewRedReceiver(receiver TrackReceiver, dsp utils.DownTrackSpreaderParams) REDTransformer {
 	return &RedReceiver{
 		TrackReceiver:     receiver,
-		downTrackSpreader: NewDownTrackSpreader(dsp),
+		downTrackSpreader: utils.NewDownTrackSpreader[TrackSender](dsp),
 		logger:            dsp.Logger,
 	}
 }
 
-func (r *RedReceiver) ForwardRTP(pkt *buffer.ExtPacket, spatialLayer int32) int {
+func (r *RedReceiver) ForwardRTP(pkt *buffer.ExtPacket, spatialLayer int32) int32 {
 	// encode RED payload from primary payload and forward to downtracks
 	if r.downTrackSpreader.DownTrackCount() == 0 {
 		return 0
@@ -65,9 +69,11 @@ func (r *RedReceiver) ForwardRTP(pkt *buffer.ExtPacket, spatialLayer int32) int
 
 	// fallback to primary codec if payload size exceeds redundant block length
 	if len(pkt.Packet.Payload) >= maxRedPayload {
-		return r.downTrackSpreader.Broadcast(func(dt TrackSender) {
-			_ = dt.WriteRTP(pkt, spatialLayer)
+		var writeCount atomic.Int32
+		r.downTrackSpreader.Broadcast(func(dt TrackSender) {
+			writeCount.Add(dt.WriteRTP(pkt, spatialLayer))
 		})
+		return writeCount.Load()
 	}
 
 	redLen, err := r.encodeRedForPrimary(pkt.Packet, r.redPayloadBuf[:])
@@ -78,14 +84,26 @@ func (r *RedReceiver) ForwardRTP(pkt *buffer.ExtPacket, spatialLayer int32) int
 
 	pPkt := *pkt
 	redRtpPacket := *pkt.Packet
-	redRtpPacket.PayloadType = 63
+	redRtpPacket.PayloadType = opusRedPT
 	redRtpPacket.Payload = r.redPayloadBuf[:redLen]
 	pPkt.Packet = &redRtpPacket
 
 	// not modify the ExtPacket.RawPacket here for performance since it is not used by the DownTrack,
 	// otherwise it should be set to the correct value (marshal the primary rtp packet)
-	return r.downTrackSpreader.Broadcast(func(dt TrackSender) {
-		_ = dt.WriteRTP(&pPkt, spatialLayer)
+	var writeCount atomic.Int32
+	r.downTrackSpreader.Broadcast(func(dt TrackSender) {
+		writeCount.Add(dt.WriteRTP(&pPkt, spatialLayer))
+	})
+	return writeCount.Load()
+}
+
+func (r *RedReceiver) ForwardRTCPSenderReport(
+	payloadType webrtc.PayloadType,
+	layer int32,
+	publisherSRData *livekit.RTCPSenderReportState,
+) {
+	r.downTrackSpreader.Broadcast(func(dt TrackSender) {
+		_ = dt.HandleRTCPSenderReportData(payloadType, layer, publisherSRData)
 	})
 }
 
@@ -122,6 +140,12 @@ func (r *RedReceiver) ResyncDownTracks() {
 	})
 }
 
+func (r *RedReceiver) OnStreamRestart() {
+	r.downTrackSpreader.Broadcast(func(dt TrackSender) {
+		dt.ReceiverRestart(r)
+	})
+}
+
 func (r *RedReceiver) CanClose() bool {
 	return r.closed.Load() || r.downTrackSpreader.DownTrackCount() == 0
 }

livekit-server/pkg/sfu/redreceiver_test.go

@@ -26,8 +26,7 @@ import (
 )
 
 const (
-	tsStep    = uint32(48000 / 1000 * 10)
-	opusREDPT = 63
+	tsStep = uint32(48000 / 1000 * 10)
 )
 
 type dummyDowntrack struct {
@@ -36,24 +35,26 @@ type dummyDowntrack struct {
 	receivedPkts    []*rtp.Packet
 }
 
-func (dt *dummyDowntrack) WriteRTP(p *buffer.ExtPacket, _ int32) error {
+func (dt *dummyDowntrack) WriteRTP(p *buffer.ExtPacket, _ int32) int32 {
 	dt.lastReceivedPkt = p.Packet
 	dt.receivedPkts = append(dt.receivedPkts, p.Packet)
-	return nil
+	return 1
 }
 
-func (dt *dummyDowntrack) TrackInfoAvailable() {}
-
 func TestRedReceiver(t *testing.T) {
 	dt := &dummyDowntrack{TrackSender: &DownTrack{}}
 
 	t.Run("normal", func(t *testing.T) {
 		w := &WebRTCReceiver{
-			isRED:  true,
-			kind:   webrtc.RTPCodecTypeAudio,
-			logger: logger.GetLogger(),
+			ReceiverBase: &ReceiverBase{
+				params: ReceiverBaseParams{
+					Kind:   webrtc.RTPCodecTypeAudio,
+					Logger: logger.GetLogger(),
+				},
+				isRED: true,
+			},
 		}
-		require.Equal(t, w.GetRedReceiver(), w)
+		require.Equal(t, w.GetRedReceiver(), w.ReceiverBase)
 		w.isRED = false
 		red := w.GetRedReceiver().(*RedReceiver)
 		require.NotNil(t, red)
@@ -75,15 +76,19 @@ func TestRedReceiver(t *testing.T) {
 
 	t.Run("packet lost and jump", func(t *testing.T) {
 		w := &WebRTCReceiver{
-			kind:   webrtc.RTPCodecTypeAudio,
-			logger: logger.GetLogger(),
+			ReceiverBase: &ReceiverBase{
+				params: ReceiverBaseParams{
+					Kind:   webrtc.RTPCodecTypeAudio,
+					Logger: logger.GetLogger(),
+				},
+			},
 		}
 		red := w.GetRedReceiver().(*RedReceiver)
 		require.NoError(t, red.AddDownTrack(dt))
 
 		header := rtp.Header{SequenceNumber: 65534, Timestamp: (uint32(1) << 31) - 2*tsStep, PayloadType: 111}
 		expectPkt := make([]*rtp.Packet, 0, maxRedCount+1)
-		for i := 0; i < 10; i++ {
+		for i := range 10 {
 			if i%2 == 0 {
 				header.SequenceNumber++
 				header.Timestamp += tsStep
@@ -126,8 +131,12 @@ func TestRedReceiver(t *testing.T) {
 
 	t.Run("unorder and repeat", func(t *testing.T) {
 		w := &WebRTCReceiver{
-			kind:   webrtc.RTPCodecTypeAudio,
-			logger: logger.GetLogger(),
+			ReceiverBase: &ReceiverBase{
+				params: ReceiverBaseParams{
+					Kind:   webrtc.RTPCodecTypeAudio,
+					Logger: logger.GetLogger(),
+				},
+			},
 		}
 		red := w.GetRedReceiver().(*RedReceiver)
 		require.NoError(t, red.AddDownTrack(dt))
@@ -158,11 +167,15 @@ func TestRedReceiver(t *testing.T) {
 
 	t.Run("encoding exceed space", func(t *testing.T) {
 		w := &WebRTCReceiver{
-			isRED:  true,
-			kind:   webrtc.RTPCodecTypeAudio,
-			logger: logger.GetLogger(),
+			ReceiverBase: &ReceiverBase{
+				params: ReceiverBaseParams{
+					Kind:   webrtc.RTPCodecTypeAudio,
+					Logger: logger.GetLogger(),
+				},
+				isRED: true,
+			},
 		}
-		require.Equal(t, w.GetRedReceiver(), w)
+		require.Equal(t, w.GetRedReceiver(), w.ReceiverBase)
 		w.isRED = false
 		red := w.GetRedReceiver().(*RedReceiver)
 		require.NotNil(t, red)
@@ -183,11 +196,15 @@ func TestRedReceiver(t *testing.T) {
 
 	t.Run("large timestamp gap", func(t *testing.T) {
 		w := &WebRTCReceiver{
-			isRED:  true,
-			kind:   webrtc.RTPCodecTypeAudio,
-			logger: logger.GetLogger(),
+			ReceiverBase: &ReceiverBase{
+				params: ReceiverBaseParams{
+					Kind:   webrtc.RTPCodecTypeAudio,
+					Logger: logger.GetLogger(),
+				},
+				isRED: true,
+			},
 		}
-		require.Equal(t, w.GetRedReceiver(), w)
+		require.Equal(t, w.GetRedReceiver(), w.ReceiverBase)
 		w.isRED = false
 		red := w.GetRedReceiver().(*RedReceiver)
 		require.NotNil(t, red)
@@ -247,7 +264,7 @@ func verifyEncodingEqual(t *testing.T, p1, p2 *rtp.Packet) {
 
 func generatePkts(header rtp.Header, count int, tsStep uint32) []*rtp.Packet {
 	pkts := make([]*rtp.Packet, 0, count)
-	for i := 0; i < count; i++ {
+	for range count {
 		hbuf, _ := header.Marshal()
 		pkts = append(pkts, &rtp.Packet{
 			Header:  header,
@@ -271,7 +288,7 @@ func generateRedPkts(t *testing.T, pkts []*rtp.Packet, redCount int) []*rtp.Pack
 		}
 		buf := make([]byte, mtuSize)
 		redPkt := *pkt
-		redPkt.PayloadType = opusREDPT
+		redPkt.PayloadType = opusRedPT
 		encoded, err := encodeRedForPrimary(encodingPkts, pkt, buf)
 		require.NoError(t, err)
 		redPkt.Payload = buf[:encoded]
@@ -283,11 +300,15 @@ func generateRedPkts(t *testing.T, pkts []*rtp.Packet, redCount int) []*rtp.Pack
 func testRedRedPrimaryReceiver(t *testing.T, maxPktCount, redCount int, sendPktIdx, expectPktIdx []int) {
 	dt := &dummyDowntrack{TrackSender: &DownTrack{}}
 	w := &WebRTCReceiver{
-		kind:   webrtc.RTPCodecTypeAudio,
-		logger: logger.GetLogger(),
-		codec:  webrtc.RTPCodecParameters{PayloadType: opusREDPT, RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: "audio/red"}},
+		ReceiverBase: &ReceiverBase{
+			params: ReceiverBaseParams{
+				Kind:   webrtc.RTPCodecTypeAudio,
+				Logger: logger.GetLogger(),
+				Codec:  webrtc.RTPCodecParameters{PayloadType: opusRedPT, RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: "audio/red"}},
+			},
+		},
 	}
-	require.Equal(t, w.GetPrimaryReceiverForRed(), w)
+	require.Equal(t, w.GetPrimaryReceiverForRed(), w.ReceiverBase)
 	w.isRED = true
 	red := w.GetPrimaryReceiverForRed().(*RedPrimaryReceiver)
 	require.NotNil(t, red)
@@ -313,10 +334,14 @@ func testRedRedPrimaryReceiver(t *testing.T, maxPktCount, redCount int, sendPktI
 
 func TestRedPrimaryReceiver(t *testing.T) {
 	w := &WebRTCReceiver{
-		kind:   webrtc.RTPCodecTypeAudio,
-		logger: logger.GetLogger(),
+		ReceiverBase: &ReceiverBase{
+			params: ReceiverBaseParams{
+				Kind:   webrtc.RTPCodecTypeAudio,
+				Logger: logger.GetLogger(),
+			},
+		},
 	}
-	require.Equal(t, w.GetPrimaryReceiverForRed(), w)
+	require.Equal(t, w.GetPrimaryReceiverForRed(), w.ReceiverBase)
 	w.isRED = true
 	red := w.GetPrimaryReceiverForRed().(*RedPrimaryReceiver)
 	require.NotNil(t, red)
@@ -324,7 +349,7 @@ func TestRedPrimaryReceiver(t *testing.T) {
 	t.Run("packet should send only once", func(t *testing.T) {
 		maxPktCount := 19
 		var sendPktIndex []int
-		for i := 0; i < maxPktCount; i++ {
+		for i := range maxPktCount {
 			sendPktIndex = append(sendPktIndex, i)
 		}
 		testRedRedPrimaryReceiver(t, maxPktCount, maxRedCount, sendPktIndex, sendPktIndex)
@@ -333,7 +358,7 @@ func TestRedPrimaryReceiver(t *testing.T) {
 	t.Run("packet duplicate and unorder", func(t *testing.T) {
 		maxPktCount := 19
 		var sendPktIndex []int
-		for i := 0; i < maxPktCount; i++ {
+		for i := range maxPktCount {
 			sendPktIndex = append(sendPktIndex, i)
 			if i > 0 {
 				sendPktIndex = append(sendPktIndex, i-1)
@@ -346,7 +371,7 @@ func TestRedPrimaryReceiver(t *testing.T) {
 	t.Run("full recover", func(t *testing.T) {
 		maxPktCount := 19
 		var sendPktIndex, recvPktIndex []int
-		for i := 0; i < maxPktCount; i++ {
+		for i := range maxPktCount {
 			recvPktIndex = append(recvPktIndex, i)
 
 			// drop packets covered by red encoding
@@ -383,11 +408,15 @@ func TestRedPrimaryReceiver(t *testing.T) {
 	t.Run("mixed primary codec", func(t *testing.T) {
 		dt := &dummyDowntrack{TrackSender: &DownTrack{}}
 		w := &WebRTCReceiver{
-			kind:   webrtc.RTPCodecTypeAudio,
-			logger: logger.GetLogger(),
-			codec:  webrtc.RTPCodecParameters{PayloadType: opusREDPT, RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: "audio/red"}},
+			ReceiverBase: &ReceiverBase{
+				params: ReceiverBaseParams{
+					Kind:   webrtc.RTPCodecTypeAudio,
+					Logger: logger.GetLogger(),
+					Codec:  webrtc.RTPCodecParameters{PayloadType: opusRedPT, RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: "audio/red"}},
+				},
+			},
 		}
-		require.Equal(t, w.GetPrimaryReceiverForRed(), w)
+		require.Equal(t, w.GetPrimaryReceiverForRed(), w.ReceiverBase)
 		w.isRED = true
 		red := w.GetPrimaryReceiverForRed().(*RedPrimaryReceiver)
 		require.NotNil(t, red)

livekit-server/pkg/sfu/rtpextension/dependencydescriptor/bitstreamwriter.go

@@ -103,7 +103,7 @@ func (w *BitStreamWriter) writePartialByte(source uint8, sourceBitCount int, tar
 }
 
 func (w *BitStreamWriter) WriteNonSymmetric(val, numValues uint32) error {
-	if !(val < numValues && numValues <= 1<<31) {
+	if val >= numValues || numValues > 1<<31 {
 		return fmt.Errorf("invalid argument, val %d, numValues %d", val, numValues)
 	}
 	if numValues == 1 {

livekit-server/pkg/sfu/rtpextension/dependencydescriptor/dependencydescriptorreader.go

@@ -229,12 +229,13 @@ func (r *DependencyDescriptorReader) readTemplateLayers() error {
 		}
 		nextLayerIdc = nextLayerIdcType(idc)
 
-		if nextLayerIdc == nextTemporalLayer {
+		switch nextLayerIdc {
+		case nextTemporalLayer:
 			temporalId++
 			if temporalId >= MaxTemporalIds {
 				return ErrDDReaderTooManyTemporalLayers
 			}
-		} else if nextLayerIdc == nextSpatialLayer {
+		case nextSpatialLayer:
 			spatialId++
 			temporalId = 0
 			if spatialId >= MaxSpatialIds {
@@ -242,7 +243,7 @@ func (r *DependencyDescriptorReader) readTemplateLayers() error {
 			}
 		}
 
-		if !(nextLayerIdc != noMoreLayer && r.buffer.Ok()) {
+		if nextLayerIdc == noMoreLayer || !r.buffer.Ok() {
 			break
 		}
 	}

livekit-server/pkg/sfu/rtpextension/dependencydescriptor/dependencydescriptorwriter.go

@@ -16,8 +16,7 @@ package dependencydescriptor
 
 import (
 	"fmt"
-
-	"golang.org/x/exp/slices"
+	"slices"
 )
 
 type TemplateMatch struct {
@@ -246,8 +245,8 @@ func (w *DependencyDescriptorWriter) writeExtendedFields() error {
 }
 
 func (w *DependencyDescriptorWriter) writeTemplateDependencyStructure() error {
-	if !(w.structure.StructureId >= 0 && w.structure.StructureId < MaxTemplates &&
-		w.structure.NumDecodeTargets > 0 && w.structure.NumDecodeTargets <= MaxDecodeTargets) {
+	if w.structure.StructureId < 0 || w.structure.StructureId >= MaxTemplates ||
+		w.structure.NumDecodeTargets <= 0 || w.structure.NumDecodeTargets > MaxDecodeTargets {
 		return fmt.Errorf("invalid arguments, structureId: %d, numDecodeTargets: %d", w.structure.StructureId, w.structure.NumDecodeTargets)
 	}
 
@@ -283,8 +282,8 @@ func (w *DependencyDescriptorWriter) writeTemplateDependencyStructure() error {
 }
 
 func (w *DependencyDescriptorWriter) writeTemplateLayers() error {
-	if !(len(w.structure.Templates) > 0 && len(w.structure.Templates) <= MaxTemplates &&
-		w.structure.Templates[0].SpatialId == 0 && w.structure.Templates[0].TemporalId == 0) {
+	if len(w.structure.Templates) <= 0 || len(w.structure.Templates) > MaxTemplates ||
+		w.structure.Templates[0].SpatialId != 0 || w.structure.Templates[0].TemporalId != 0 {
 		return fmt.Errorf("invalid templates, len %d, templates[0]: spatialId %d, temporalId %d", len(w.structure.Templates), w.structure.Templates[0].SpatialId, w.structure.Templates[0].TemporalId)
 	}
 	for i := 1; i < len(w.structure.Templates); i++ {

livekit-server/pkg/sfu/rtpmunger.go

@@ -77,8 +77,8 @@ func NewRTPMunger(logger logger.Logger) *RTPMunger {
 	}
 }
 
-func (r *RTPMunger) DebugInfo() map[string]interface{} {
-	return map[string]interface{}{
+func (r *RTPMunger) DebugInfo() map[string]any {
+	return map[string]any{
 		"ExtHighestIncomingSN": r.extHighestIncomingSN,
 		"ExtLastSN":            r.extLastSN,
 		"ExtSecondLastSN":      r.extSecondLastSN,
@@ -102,6 +102,10 @@ func (r *RTPMunger) GetState() *livekit.RTPMungerState {
 	}
 }
 
+func (r *RTPMunger) GetExtLastTimestamp() uint64 {
+	return r.extLastTS
+}
+
 func (r *RTPMunger) GetTSOffset() uint64 {
 	return r.tsOffset
 }
@@ -185,7 +189,7 @@ func (r *RTPMunger) UpdateAndGetSnTs(extPkt *buffer.ExtPacket, marker bool) (Tra
 		r.secondLastMarker = r.lastMarker
 		r.lastMarker = marker
 
-		if extPkt.KeyFrame {
+		if extPkt.IsKeyFrame {
 			r.extRtxGateSn = extMungedSN
 			r.isInRtxGateRegion = true
 		}
@@ -207,7 +211,7 @@ func (r *RTPMunger) UpdateAndGetSnTs(extPkt *buffer.ExtPacket, marker bool) (Tra
 		if err != nil {
 			return TranslationParamsRTP{
 				snOrdering: SequenceNumberOrderingOutOfOrder,
-			}, ErrOutOfOrderSequenceNumberCacheMiss
+			}, errOutOfOrderSequenceNumberCacheMiss
 		}
 
 		extSequenceNumber := extPkt.ExtSequenceNumber - snOffset
@@ -223,7 +227,7 @@ func (r *RTPMunger) UpdateAndGetSnTs(extPkt *buffer.ExtPacket, marker bool) (Tra
 			)
 			return TranslationParamsRTP{
 				snOrdering: SequenceNumberOrderingOutOfOrder,
-			}, ErrOutOfOrderSequenceNumberCacheMiss
+			}, errOutOfOrderSequenceNumberCacheMiss
 		}
 
 		return TranslationParamsRTP{
@@ -245,13 +249,13 @@ func (r *RTPMunger) UpdateAndGetSnTs(extPkt *buffer.ExtPacket, marker bool) (Tra
 
 		return TranslationParamsRTP{
 			snOrdering: SequenceNumberOrderingContiguous,
-		}, ErrPaddingOnlyPacket
+		}, errPaddingOnlyPacket
 	}
 
 	// can get duplicate packet due to FEC
 	return TranslationParamsRTP{
 		snOrdering: SequenceNumberOrderingDuplicate,
-	}, ErrDuplicatePacket
+	}, errDuplicatePacket
 }
 
 func (r *RTPMunger) FilterRTX(nacks []uint16) []uint16 {
@@ -269,7 +273,13 @@ func (r *RTPMunger) FilterRTX(nacks []uint16) []uint16 {
 	return filtered
 }
 
-func (r *RTPMunger) UpdateAndGetPaddingSnTs(num int, clockRate uint32, frameRate uint32, forceMarker bool, extRtpTimestamp uint64) ([]SnTs, error) {
+func (r *RTPMunger) UpdateAndGetPaddingSnTs(
+	num int,
+	clockRate uint32,
+	frameRate uint32,
+	forceMarker bool,
+	extRtpTimestamp uint64,
+) ([]SnTs, error) {
 	if num == 0 {
 		return nil, nil
 	}
@@ -278,7 +288,7 @@ func (r *RTPMunger) UpdateAndGetPaddingSnTs(num int, clockRate uint32, frameRate
 	tsOffset := 0
 	if !r.lastMarker {
 		if !forceMarker {
-			return nil, ErrPaddingNotOnFrameBoundary
+			return nil, errPaddingNotOnFrameBoundary
 		}
 
 		// if forcing frame end, use timestamp of latest received frame for the first one
@@ -289,7 +299,7 @@ func (r *RTPMunger) UpdateAndGetPaddingSnTs(num int, clockRate uint32, frameRate
 	extLastSN := r.extLastSN
 	extLastTS := r.extLastTS
 	vals := make([]SnTs, num)
-	for i := 0; i < num; i++ {
+	for i := range num {
 		extLastSN++
 		vals[i].extSequenceNumber = extLastSN
 
@@ -322,6 +332,7 @@ func (r *RTPMunger) UpdateAndGetPaddingSnTs(num int, clockRate uint32, frameRate
 	r.tsOffset -= extLastTS - r.extLastTS
 	r.extLastTS = extLastTS
 
+	r.secondLastMarker = r.lastMarker
 	if forceMarker {
 		r.lastMarker = true
 	}

livekit-server/pkg/sfu/rtpmunger_test.go

@@ -44,9 +44,9 @@ func TestSetLastSnTs(t *testing.T) {
 	require.Equal(t, uint64(23332), r.extHighestIncomingSN)
 	require.Equal(t, uint64(23333), r.extLastSN)
 	require.Equal(t, uint64(0xabcdef), r.extLastTS)
-	snOffset, err := r.snRangeMap.GetValue(r.extHighestIncomingSN)
+	_, err = r.snRangeMap.GetValue(r.extHighestIncomingSN)
 	require.Error(t, err)
-	snOffset, err = r.snRangeMap.GetValue(r.extLastSN)
+	snOffset, err := r.snRangeMap.GetValue(r.extLastSN)
 	require.NoError(t, err)
 	require.Equal(t, uint64(0), snOffset)
 	require.Equal(t, uint64(0), r.snOffset)
@@ -96,9 +96,9 @@ func TestPacketDropped(t *testing.T) {
 	require.Equal(t, uint64(23332), r.extHighestIncomingSN)
 	require.Equal(t, uint64(23333), r.extLastSN)
 	require.Equal(t, uint64(0xabcdef), r.extLastTS)
-	snOffset, err := r.snRangeMap.GetValue(r.extHighestIncomingSN)
+	_, err := r.snRangeMap.GetValue(r.extHighestIncomingSN)
 	require.Error(t, err)
-	snOffset, err = r.snRangeMap.GetValue(r.extLastSN)
+	snOffset, err := r.snRangeMap.GetValue(r.extLastSN)
 	require.NoError(t, err)
 	require.Equal(t, uint64(0), snOffset)
 	require.Equal(t, uint64(0), r.tsOffset)
@@ -133,7 +133,7 @@ func TestPacketDropped(t *testing.T) {
 
 	r.PacketDropped(extPkt)
 	require.Equal(t, uint64(23333), r.extLastSN)
-	snOffset, err = r.snRangeMap.GetValue(r.extHighestIncomingSN)
+	_, err = r.snRangeMap.GetValue(r.extHighestIncomingSN)
 	require.Error(t, err)
 	snOffset, err = r.snRangeMap.GetValue(r.extHighestIncomingSN + 1)
 	require.NoError(t, err)
@@ -180,9 +180,10 @@ func TestOutOfOrderSequenceNumber(t *testing.T) {
 	}
 	extPkt, _ = testutils.GetTestExtPacket(params)
 
-	tp, err := r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker)
+	_, err = r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker)
 	require.Error(t, err)
 
+	var tp TranslationParamsRTP
 	//  add a missing sequence number to the cache
 	err = r.snRangeMap.ExcludeRange(23334, 23335)
 	require.NoError(t, err)
@@ -228,7 +229,7 @@ func TestOutOfOrderSequenceNumber(t *testing.T) {
 	}
 
 	tp, err = r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker)
-	require.Error(t, err, ErrOutOfOrderSequenceNumberCacheMiss)
+	require.Error(t, err, errOutOfOrderSequenceNumberCacheMiss)
 	require.Equal(t, tpExpected, tp)
 }
 
@@ -252,7 +253,7 @@ func TestDuplicateSequenceNumber(t *testing.T) {
 	}
 
 	tp, err := r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker)
-	require.ErrorIs(t, err, ErrDuplicatePacket)
+	require.ErrorIs(t, err, errDuplicatePacket)
 	require.Equal(t, tpExpected, tp)
 }
 
@@ -274,13 +275,14 @@ func TestPaddingOnlyPacket(t *testing.T) {
 
 	tp, err := r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker)
 	require.Error(t, err)
-	require.ErrorIs(t, err, ErrPaddingOnlyPacket)
+	require.ErrorIs(t, err, errPaddingOnlyPacket)
 	require.Equal(t, tpExpected, tp)
 	require.Equal(t, uint64(23333), r.extHighestIncomingSN)
 	require.Equal(t, uint64(23333), r.extLastSN)
-	snOffset, err := r.snRangeMap.GetValue(r.extHighestIncomingSN)
+	_, err = r.snRangeMap.GetValue(r.extHighestIncomingSN)
 	require.Error(t, err)
 
+	var snOffset uint64
 	// padding only packet with a gap should not report an error
 	params = &testutils.TestExtPacketParams{
 		SequenceNumber: 23335,
@@ -366,11 +368,11 @@ func TestGapInSequenceNumber(t *testing.T) {
 	}
 
 	tp, err = r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker)
-	require.ErrorIs(t, err, ErrPaddingOnlyPacket)
+	require.ErrorIs(t, err, errPaddingOnlyPacket)
 	require.Equal(t, tpExpected, tp)
 	require.Equal(t, uint64(65536+2), r.extHighestIncomingSN)
 	require.Equal(t, uint64(65536+1), r.extLastSN)
-	snOffset, err = r.snRangeMap.GetValue(r.extHighestIncomingSN)
+	_, err = r.snRangeMap.GetValue(r.extHighestIncomingSN)
 	require.Error(t, err)
 
 	// a packet with a gap should be adjusting for dropped padding packet
@@ -417,11 +419,11 @@ func TestGapInSequenceNumber(t *testing.T) {
 	}
 
 	tp, err = r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker)
-	require.ErrorIs(t, err, ErrPaddingOnlyPacket)
+	require.ErrorIs(t, err, errPaddingOnlyPacket)
 	require.Equal(t, tpExpected, tp)
 	require.Equal(t, uint64(65536+5), r.extHighestIncomingSN)
 	require.Equal(t, uint64(65536+3), r.extLastSN)
-	snOffset, err = r.snRangeMap.GetValue(r.extHighestIncomingSN)
+	_, err = r.snRangeMap.GetValue(r.extHighestIncomingSN)
 	require.Error(t, err)
 
 	// a packet with a gap should be adjusting for dropped packets
@@ -521,7 +523,7 @@ func TestUpdateAndGetPaddingSnTs(t *testing.T) {
 	// getting padding without forcing marker should fail
 	_, err := r.UpdateAndGetPaddingSnTs(10, 10, 5, false, 0)
 	require.Error(t, err)
-	require.ErrorIs(t, err, ErrPaddingNotOnFrameBoundary)
+	require.ErrorIs(t, err, errPaddingNotOnFrameBoundary)
 
 	// forcing a marker should not error out.
 	// And timestamp on first padding should be the same as the last one.
@@ -529,7 +531,7 @@ func TestUpdateAndGetPaddingSnTs(t *testing.T) {
 	clockRate := uint64(10)
 	frameRate := uint64(5)
 	var sntsExpected = make([]SnTs, numPadding)
-	for i := 0; i < numPadding; i++ {
+	for i := range numPadding {
 		sntsExpected[i] = SnTs{
 			extSequenceNumber: uint64(params.SequenceNumber) + uint64(i) + 1,
 			extTimestamp:      uint64(params.Timestamp) + ((uint64(i)*clockRate)+frameRate-1)/frameRate,
@@ -540,7 +542,7 @@ func TestUpdateAndGetPaddingSnTs(t *testing.T) {
 	require.Equal(t, sntsExpected, snts)
 
 	// now that there is a marker, timestamp should jump on first padding when asked again
-	for i := 0; i < numPadding; i++ {
+	for i := range numPadding {
 		sntsExpected[i] = SnTs{
 			extSequenceNumber: uint64(params.SequenceNumber) + uint64(len(snts)) + uint64(i) + 1,
 			extTimestamp:      snts[len(snts)-1].extTimestamp + ((uint64(i+1)*clockRate)+frameRate-1)/frameRate,
@@ -570,7 +572,7 @@ func TestIsOnFrameBoundary(t *testing.T) {
 
 	// packet with RTP marker
 	params = &testutils.TestExtPacketParams{
-		SetMarker:      true,
+		Marker:         true,
 		SequenceNumber: 23334,
 		Timestamp:      0xabcdef,
 		SSRC:           0x12345678,

livekit-server/pkg/sfu/rtpstats/rtpstats_base.go

@@ -25,13 +25,12 @@ import (
 	"github.com/livekit/protocol/livekit"
 	"github.com/livekit/protocol/utils"
 	"github.com/livekit/protocol/utils/mono"
+	"github.com/livekit/protocol/utils/rtputil"
 )
 
 const (
 	cFirstPacketTimeAdjustWindow    = 2 * time.Minute
 	cFirstPacketTimeAdjustThreshold = 15 * 1e9
-
-	cSequenceNumberLargeJumpThreshold = 100
 )
 
 // -------------------------------------------------------
@@ -192,6 +191,8 @@ func (w WrappedRTCPSenderReportStateLogger) MarshalLogObject(e zapcore.ObjectEnc
 	return nil
 }
 
+// ------------------------------------------------------------------
+
 func RTCPSenderReportPropagationDelay(rsrs *livekit.RTCPSenderReportState, passThrough bool) time.Duration {
 	if passThrough {
 		return 0
@@ -205,6 +206,8 @@ func RTCPSenderReportPropagationDelay(rsrs *livekit.RTCPSenderReportState, passT
 type rtpStatsBase struct {
 	*rtpStatsBaseLite
 
+	rtpConverter *rtputil.RTPConverter
+
 	firstTime           int64
 	firstTimeAdjustment time.Duration
 	highestTime         int64
@@ -251,6 +254,18 @@ func newRTPStatsBase(params RTPStatsParams) *rtpStatsBase {
 	}
 }
 
+func (r *rtpStatsBase) SetClockRate(clockRate uint32) {
+	r.lock.Lock()
+	defer r.lock.Unlock()
+
+	r.setClockRateLocked(clockRate)
+}
+
+func (r *rtpStatsBase) setClockRateLocked(clockRate uint32) {
+	r.rtpConverter = rtputil.NewRTPConverter(int64(clockRate))
+	r.rtpStatsBaseLite.setClockRateLocked(clockRate)
+}
+
 func (r *rtpStatsBase) seed(from *rtpStatsBase) bool {
 	if !r.rtpStatsBaseLite.seed(from.rtpStatsBaseLite) {
 		return false
@@ -346,6 +361,13 @@ func (r *rtpStatsBase) UpdateKeyFrame(kfCount uint32) {
 	r.lastKeyFrame = time.Now()
 }
 
+func (r *rtpStatsBase) KeyFrame() (uint32, time.Time) {
+	r.lock.RLock()
+	defer r.lock.RUnlock()
+
+	return r.keyFrames, r.lastKeyFrame
+}
+
 func (r *rtpStatsBase) UpdateRtt(rtt uint32) {
 	r.lock.Lock()
 	defer r.lock.Unlock()
@@ -374,7 +396,11 @@ func (r *rtpStatsBase) GetRtt() uint32 {
 	return r.rtt
 }
 
-func (r *rtpStatsBase) maybeAdjustFirstPacketTime(srData *livekit.RTCPSenderReportState, tsOffset uint64, extStartTS uint64) (err error, loggingFields []interface{}) {
+func (r *rtpStatsBase) maybeAdjustFirstPacketTime(
+	srData *livekit.RTCPSenderReportState,
+	tsOffset uint64,
+	extStartTS uint64,
+) (adjustment int64, loggingFields []any, err error) {
 	nowNano := mono.UnixNano()
 	if time.Duration(nowNano-r.startTime) > cFirstPacketTimeAdjustWindow {
 		return
@@ -387,49 +413,52 @@ func (r *rtpStatsBase) maybeAdjustFirstPacketTime(srData *livekit.RTCPSenderRepo
 	// in some network element along the way), push back first time
 	// to an earlier instance.
 	timeSinceReceive := time.Duration(nowNano - srData.AtAdjusted)
-	extNowTS := srData.RtpTimestampExt - tsOffset + uint64(timeSinceReceive.Nanoseconds()*int64(r.params.ClockRate)/1e9)
+	extNowTS := srData.RtpTimestampExt - tsOffset + r.rtpConverter.ToRTPExt(timeSinceReceive)
 	samplesDiff := int64(extNowTS - extStartTS)
 	if samplesDiff < 0 {
 		// out-of-order, skip
 		return
 	}
 
-	samplesDuration := time.Duration(float64(samplesDiff) / float64(r.params.ClockRate) * float64(time.Second))
+	samplesDuration := r.rtpConverter.ToDurationExt(uint64(samplesDiff))
 	timeSinceFirst := time.Duration(nowNano - r.firstTime)
 	now := r.firstTime + timeSinceFirst.Nanoseconds()
 	firstTime := now - samplesDuration.Nanoseconds()
-
-	getFields := func() []interface{} {
-		return []interface{}{
-			"startTime", time.Unix(0, r.startTime),
-			"nowTime", time.Unix(0, now),
-			"before", time.Unix(0, r.firstTime),
-			"after", time.Unix(0, firstTime),
-			"adjustment", time.Duration(r.firstTime - firstTime),
-			"extNowTS", extNowTS,
-			"extStartTS", extStartTS,
-			"srData", WrappedRTCPSenderReportStateLogger{srData},
-			"tsOffset", tsOffset,
-			"timeSinceReceive", timeSinceReceive,
-			"timeSinceFirst", timeSinceFirst,
-			"samplesDiff", samplesDiff,
-			"samplesDuration", samplesDuration,
-		}
-	}
+	adjustment = r.firstTime - firstTime
 
 	if firstTime < r.firstTime {
-		if r.firstTime-firstTime > cFirstPacketTimeAdjustThreshold {
+		if adjustment > cFirstPacketTimeAdjustThreshold {
 			err = errors.New("adjusting first packet time, too big, ignoring")
-			loggingFields = getFields()
+			loggingFields = r.firstPacketTimeAdjustLogFields(now, firstTime, adjustment, extNowTS, extStartTS, srData, tsOffset, timeSinceReceive, timeSinceFirst, samplesDuration, samplesDiff)
 		} else {
-			r.logger.Debugw("adjusting first packet time", getFields()...)
-			r.firstTimeAdjustment += time.Duration(r.firstTime - firstTime)
+			r.firstTimeAdjustment += time.Duration(adjustment)
+			r.logger.Debugw("adjusting first packet time", r.firstPacketTimeAdjustLogFields(now, firstTime, adjustment, extNowTS, extStartTS, srData, tsOffset, timeSinceReceive, timeSinceFirst, samplesDuration, samplesDiff)...)
 			r.firstTime = firstTime
 		}
 	}
+
 	return
 }
 
+func (r *rtpStatsBase) firstPacketTimeAdjustLogFields(now, firstTime int64, adjustment int64, extNowTS, extStartTS uint64, srData *livekit.RTCPSenderReportState, tsOffset uint64, timeSinceReceive, timeSinceFirst, samplesDuration time.Duration, samplesDiff int64) []any {
+	return []any{
+		"startTime", time.Unix(0, r.startTime),
+		"nowTime", time.Unix(0, now),
+		"before", time.Unix(0, r.firstTime),
+		"after", time.Unix(0, firstTime),
+		"adjustment", time.Duration(adjustment),
+		"firstTimeAdjustment", r.firstTimeAdjustment,
+		"extNowTS", extNowTS,
+		"extStartTS", extStartTS,
+		"srData", WrappedRTCPSenderReportStateLogger{srData},
+		"tsOffset", tsOffset,
+		"timeSinceReceive", timeSinceReceive,
+		"timeSinceFirst", timeSinceFirst,
+		"samplesDiff", samplesDiff,
+		"samplesDuration", samplesDuration,
+	}
+}
+
 func (r *rtpStatsBase) getPacketsSeenMinusPadding(extStartSN, extHighestSN uint64) uint64 {
 	packetsSeen := r.getPacketsSeen(extStartSN, extHighestSN)
 	if r.packetsPadding > packetsSeen {
@@ -447,9 +476,13 @@ func (r *rtpStatsBase) deltaInfo(
 	snapshotID uint32,
 	extStartSN uint64,
 	extHighestSN uint64,
-) (deltaInfo *RTPDeltaInfo, err error, loggingFields []interface{}) {
-	then, now := r.getAndResetSnapshot(snapshotID, extStartSN, extHighestSN)
-	if now == nil || then == nil {
+) (deltaInfo *RTPDeltaInfo, loggingFields []any, err error) {
+	if r.clockRate == 0 {
+		return
+	}
+
+	then, now, ok := r.getAndResetSnapshot(snapshotID, extStartSN, extHighestSN)
+	if !ok {
 		return
 	}
 
@@ -461,7 +494,7 @@ func (r *rtpStatsBase) deltaInfo(
 		packetsExpected = 0
 	}
 	if packetsExpected > cNumSequenceNumbers {
-		loggingFields = []interface{}{
+		loggingFields = []any{
 			"snapshotID", snapshotID,
 			"snapshotNow", now,
 			"snapshotThen", then,
@@ -487,7 +520,7 @@ func (r *rtpStatsBase) deltaInfo(
 	// padding packets delta could be higher than expected due to out-of-order padding packets
 	packetsPadding := now.packetsPadding - then.packetsPadding
 	if packetsExpected < packetsPadding {
-		loggingFields = []interface{}{
+		loggingFields = []any{
 			"snapshotID", snapshotID,
 			"snapshotNow", now,
 			"snapshotThen", then,
@@ -518,7 +551,7 @@ func (r *rtpStatsBase) deltaInfo(
 		PacketsOutOfOrder:    uint32(now.packetsOutOfOrder - then.packetsOutOfOrder),
 		Frames:               now.frames - then.frames,
 		RttMax:               then.maxRtt,
-		JitterMax:            then.maxJitter / float64(r.params.ClockRate) * 1e6,
+		JitterMax:            then.maxJitter / float64(r.clockRate) * 1e6,
 		Nacks:                now.nacks - then.nacks,
 		Plis:                 now.plis - then.plis,
 		Firs:                 now.firs - then.firs,
@@ -614,8 +647,8 @@ func (r *rtpStatsBase) toProto(
 	p.KeyFrames = r.keyFrames
 	p.LastKeyFrame = timestamppb.New(r.lastKeyFrame)
 
-	p.JitterCurrent = jitter / float64(r.params.ClockRate) * 1e6
-	p.JitterMax = maxJitter / float64(r.params.ClockRate) * 1e6
+	p.JitterCurrent = jitter / float64(r.clockRate) * 1e6
+	p.JitterMax = maxJitter / float64(r.clockRate) * 1e6
 
 	p.Firs = r.firs
 	p.LastFir = timestamppb.New(r.lastFir)
@@ -635,9 +668,9 @@ func (r *rtpStatsBase) updateJitter(ets uint64, packetTime int64) float64 {
 	//          p1f1 -> p1f2 -> p2f1
 	//       In this case, p2f1 (packet 2, frame 1) will still be used in jitter calculation
 	//       although it is the second packet of a frame because of out-of-order receival.
-	if r.lastJitterExtTimestamp != ets {
+	if r.lastJitterExtTimestamp != ets && r.rtpConverter != nil {
 		timeSinceFirst := packetTime - r.firstTime
-		packetTimeRTP := uint64(timeSinceFirst * int64(r.params.ClockRate) / 1e9)
+		packetTimeRTP := r.rtpConverter.ToRTPExt(time.Duration(timeSinceFirst))
 		transit := packetTimeRTP - ets
 
 		if r.lastTransit != 0 {
@@ -661,9 +694,9 @@ func (r *rtpStatsBase) updateJitter(ets uint64, packetTime int64) float64 {
 	return r.jitter
 }
 
-func (r *rtpStatsBase) getAndResetSnapshot(snapshotID uint32, extStartSN uint64, extHighestSN uint64) (*snapshot, *snapshot) {
-	if !r.initialized {
-		return nil, nil
+func (r *rtpStatsBase) getAndResetSnapshot(snapshotID uint32, extStartSN uint64, extHighestSN uint64) (snapshot, snapshot, bool) {
+	if !r.initialized || snapshotID < cFirstSnapshotID {
+		return snapshot{}, snapshot{}, false
 	}
 
 	idx := snapshotID - cFirstSnapshotID
@@ -676,7 +709,7 @@ func (r *rtpStatsBase) getAndResetSnapshot(snapshotID uint32, extStartSN uint64,
 	// snapshot now
 	now := r.getSnapshot(mono.UnixNano(), extHighestSN+1)
 	r.snapshots[idx] = now
-	return &then, &now
+	return then, now, true
 }
 
 func (r *rtpStatsBase) getDrift(extStartTS, extHighestTS uint64) (
@@ -686,21 +719,20 @@ func (r *rtpStatsBase) getDrift(extStartTS, extHighestTS uint64) (
 	rebasedReportDrift *livekit.RTPDrift,
 ) {
 	if r.firstTime != 0 {
-		elapsed := r.highestTime - r.firstTime
+		elapsed := time.Duration(r.highestTime - r.firstTime)
 		rtpClockTicks := extHighestTS - extStartTS
-		driftSamples := int64(rtpClockTicks - uint64(elapsed*int64(r.params.ClockRate)/1e9))
+		driftSamples := int64(rtpClockTicks - r.rtpConverter.ToRTPExt(elapsed))
 		if elapsed > 0 {
-			elapsedSeconds := time.Duration(elapsed).Seconds()
 			packetDrift = &livekit.RTPDrift{
 				StartTime:      timestamppb.New(time.Unix(0, r.firstTime)),
 				EndTime:        timestamppb.New(time.Unix(0, r.highestTime)),
-				Duration:       elapsedSeconds,
+				Duration:       elapsed.Seconds(),
 				StartTimestamp: extStartTS,
 				EndTimestamp:   extHighestTS,
 				RtpClockTicks:  rtpClockTicks,
 				DriftSamples:   driftSamples,
-				DriftMs:        (float64(driftSamples) * 1000) / float64(r.params.ClockRate),
-				ClockRate:      float64(rtpClockTicks) / elapsedSeconds,
+				DriftMs:        (float64(driftSamples) * 1000) / float64(r.clockRate),
+				ClockRate:      float64(rtpClockTicks) / elapsed.Seconds(),
 			}
 		}
 	}
@@ -708,25 +740,27 @@ func (r *rtpStatsBase) getDrift(extStartTS, extHighestTS uint64) (
 	if r.srFirst != nil && r.srNewest != nil && r.srFirst.RtpTimestamp != r.srNewest.RtpTimestamp {
 		rtpClockTicks := r.srNewest.RtpTimestampExt - r.srFirst.RtpTimestampExt
 
-		elapsed := mediatransportutil.NtpTime(r.srNewest.NtpTimestamp).Time().Sub(mediatransportutil.NtpTime(r.srFirst.NtpTimestamp).Time())
+		srFirstTime := mediatransportutil.NtpTime(r.srFirst.NtpTimestamp).Time()
+		srNewestTime := mediatransportutil.NtpTime(r.srNewest.NtpTimestamp).Time()
+		elapsed := srNewestTime.Sub(srFirstTime)
 		if elapsed.Seconds() > 0.0 {
-			driftSamples := int64(rtpClockTicks - uint64(elapsed.Nanoseconds()*int64(r.params.ClockRate)/1e9))
+			driftSamples := int64(rtpClockTicks - r.rtpConverter.ToRTPExt(elapsed))
 			ntpReportDrift = &livekit.RTPDrift{
-				StartTime:      timestamppb.New(mediatransportutil.NtpTime(r.srFirst.NtpTimestamp).Time()),
-				EndTime:        timestamppb.New(mediatransportutil.NtpTime(r.srNewest.NtpTimestamp).Time()),
+				StartTime:      timestamppb.New(srFirstTime),
+				EndTime:        timestamppb.New(srNewestTime),
 				Duration:       elapsed.Seconds(),
 				StartTimestamp: r.srFirst.RtpTimestampExt,
 				EndTimestamp:   r.srNewest.RtpTimestampExt,
 				RtpClockTicks:  rtpClockTicks,
 				DriftSamples:   driftSamples,
-				DriftMs:        (float64(driftSamples) * 1000) / float64(r.params.ClockRate),
+				DriftMs:        (float64(driftSamples) * 1000) / float64(r.clockRate),
 				ClockRate:      float64(rtpClockTicks) / elapsed.Seconds(),
 			}
 		}
 
 		elapsed = time.Duration(r.srNewest.At - r.srFirst.At)
 		if elapsed.Seconds() > 0.0 {
-			driftSamples := int64(rtpClockTicks - uint64(elapsed.Nanoseconds()*int64(r.params.ClockRate)/1e9))
+			driftSamples := int64(rtpClockTicks - r.rtpConverter.ToRTPExt(elapsed))
 			receivedReportDrift = &livekit.RTPDrift{
 				StartTime:      timestamppb.New(time.Unix(0, r.srFirst.At)),
 				EndTime:        timestamppb.New(time.Unix(0, r.srNewest.At)),
@@ -735,14 +769,14 @@ func (r *rtpStatsBase) getDrift(extStartTS, extHighestTS uint64) (
 				EndTimestamp:   r.srNewest.RtpTimestampExt,
 				RtpClockTicks:  rtpClockTicks,
 				DriftSamples:   driftSamples,
-				DriftMs:        (float64(driftSamples) * 1000) / float64(r.params.ClockRate),
+				DriftMs:        (float64(driftSamples) * 1000) / float64(r.clockRate),
 				ClockRate:      float64(rtpClockTicks) / elapsed.Seconds(),
 			}
 		}
 
 		elapsed = time.Duration(r.srNewest.AtAdjusted - r.srFirst.AtAdjusted)
 		if elapsed.Seconds() > 0.0 {
-			driftSamples := int64(rtpClockTicks - uint64(elapsed.Nanoseconds()*int64(r.params.ClockRate)/1e9))
+			driftSamples := int64(rtpClockTicks - r.rtpConverter.ToRTPExt(elapsed))
 			rebasedReportDrift = &livekit.RTPDrift{
 				StartTime:      timestamppb.New(time.Unix(0, r.srFirst.AtAdjusted)),
 				EndTime:        timestamppb.New(time.Unix(0, r.srNewest.AtAdjusted)),
@@ -751,7 +785,7 @@ func (r *rtpStatsBase) getDrift(extStartTS, extHighestTS uint64) (
 				EndTimestamp:   r.srNewest.RtpTimestampExt,
 				RtpClockTicks:  rtpClockTicks,
 				DriftSamples:   driftSamples,
-				DriftMs:        (float64(driftSamples) * 1000) / float64(r.params.ClockRate),
+				DriftMs:        (float64(driftSamples) * 1000) / float64(r.clockRate),
 				ClockRate:      float64(rtpClockTicks) / elapsed.Seconds(),
 			}
 		}

livekit-server/pkg/sfu/rtpstats/rtpstats_base_lite.go

@@ -17,6 +17,7 @@ package rtpstats
 import (
 	"errors"
 	"fmt"
+	"strings"
 	"sync"
 	"time"
 
@@ -95,14 +96,13 @@ func (s *snapshotLite) MarshalLogObject(e zapcore.ObjectEncoder) error {
 // ------------------------------------------------------------------
 
 type RTPStatsParams struct {
-	ClockRate uint32
-	IsRTX     bool
-	Logger    logger.Logger
+	IsRTX bool
 }
 
 type rtpStatsBaseLite struct {
-	params RTPStatsParams
-	logger logger.Logger
+	params    RTPStatsParams
+	clockRate uint32
+	logger    logger.Logger
 
 	lock sync.RWMutex
 
@@ -134,7 +134,7 @@ type rtpStatsBaseLite struct {
 func newRTPStatsBaseLite(params RTPStatsParams) *rtpStatsBaseLite {
 	return &rtpStatsBaseLite{
 		params:             params,
-		logger:             params.Logger,
+		logger:             logger.GetLogger(),
 		nextSnapshotLiteID: cFirstSnapshotID,
 		snapshotLites:      make([]snapshotLite, 2),
 	}
@@ -172,7 +172,21 @@ func (r *rtpStatsBaseLite) seed(from *rtpStatsBaseLite) bool {
 	return true
 }
 
+func (r *rtpStatsBaseLite) SetClockRate(clockRate uint32) {
+	r.lock.Lock()
+	defer r.lock.Unlock()
+
+	r.setClockRateLocked(clockRate)
+}
+
+func (r *rtpStatsBaseLite) setClockRateLocked(clockRate uint32) {
+	r.clockRate = clockRate
+}
+
 func (r *rtpStatsBaseLite) SetLogger(logger logger.Logger) {
+	r.lock.Lock()
+	defer r.lock.Unlock()
+
 	r.logger = logger
 }
 
@@ -305,9 +319,9 @@ func (r *rtpStatsBaseLite) deltaInfoLite(
 	snapshotLiteID uint32,
 	extStartSN uint64,
 	extHighestSN uint64,
-) (deltaInfoLite *RTPDeltaInfoLite, err error, loggingFields []interface{}) {
-	then, now := r.getAndResetSnapshotLite(snapshotLiteID, extStartSN, extHighestSN)
-	if now == nil || then == nil {
+) (deltaInfoLite *RTPDeltaInfoLite, loggingFields []any, err error) {
+	then, now, ok := r.getAndResetSnapshotLite(snapshotLiteID, extStartSN, extHighestSN)
+	if !ok {
 		return
 	}
 
@@ -319,7 +333,7 @@ func (r *rtpStatsBaseLite) deltaInfoLite(
 		packetsExpected = 0
 	}
 	if packetsExpected > cNumSequenceNumbers {
-		loggingFields = []interface{}{
+		loggingFields = []any{
 			"snapshotLiteID", snapshotLiteID,
 			"snapshotLiteNow", now,
 			"snapshotLiteThen", then,
@@ -342,7 +356,7 @@ func (r *rtpStatsBaseLite) deltaInfoLite(
 		packetsLost = 0
 	}
 	if packetsLost > packetsExpected {
-		loggingFields = []interface{}{
+		loggingFields = []any{
 			"snapshotLiteID", snapshotLiteID,
 			"snapshotLiteNow", now,
 			"snapshotLiteThen", then,
@@ -366,7 +380,7 @@ func (r *rtpStatsBaseLite) deltaInfoLite(
 }
 
 func (r *rtpStatsBaseLite) marshalLogObject(e zapcore.ObjectEncoder, packetsExpected, packetsSeenMinusPadding uint64) (float64, error) {
-	if r == nil || !r.initialized {
+	if r == nil || !r.initialized || r.clockRate == 0 {
 		return 0, errors.New("not initialized")
 	}
 
@@ -399,25 +413,18 @@ func (r *rtpStatsBaseLite) marshalLogObject(e zapcore.ObjectEncoder, packetsExpe
 		e.AddFloat32("packetLostPercentage", float32(r.packetsLost)/float32(packetsExpected)*100.0)
 	}
 
-	hasLoss := false
-	first := true
-	str := "["
+	var sb strings.Builder
 	for burst, count := range r.gapHistogram {
 		if count == 0 {
 			continue
 		}
-
-		hasLoss = true
-
-		if !first {
-			str += ", "
+		if sb.Len() > 0 {
+			sb.WriteString(", ")
 		}
-		first = false
-		str += fmt.Sprintf("%d:%d", burst+1, count)
+		fmt.Fprintf(&sb, "%d:%d", burst+1, count)
 	}
-	str += "]"
-	if hasLoss {
-		e.AddString("gapHistogram", str)
+	if sb.Len() > 0 {
+		e.AddString("gapHistogram", "["+sb.String()+"]")
 	}
 
 	e.AddUint32("nacks", r.nacks)
@@ -431,7 +438,7 @@ func (r *rtpStatsBaseLite) marshalLogObject(e zapcore.ObjectEncoder, packetsExpe
 }
 
 func (r *rtpStatsBaseLite) toProto(packetsExpected, packetsSeenMinusPadding, packetsLost uint64) *livekit.RTPStats {
-	if r.startTime == 0 {
+	if r.startTime == 0 || r.clockRate == 0 {
 		return nil
 	}
 
@@ -473,33 +480,22 @@ func (r *rtpStatsBaseLite) toProto(packetsExpected, packetsSeenMinusPadding, pac
 		LastPli:              timestamppb.New(time.Unix(0, r.lastPli)),
 	}
 
-	gapsPresent := false
-	for i := 0; i < len(r.gapHistogram); i++ {
+	for i := range len(r.gapHistogram) {
 		if r.gapHistogram[i] == 0 {
 			continue
 		}
-
-		gapsPresent = true
-		break
-	}
-
-	if gapsPresent {
-		p.GapHistogram = make(map[int32]uint32, len(r.gapHistogram))
-		for i := 0; i < len(r.gapHistogram); i++ {
-			if r.gapHistogram[i] == 0 {
-				continue
-			}
-
-			p.GapHistogram[int32(i+1)] = r.gapHistogram[i]
+		if p.GapHistogram == nil {
+			p.GapHistogram = make(map[int32]uint32, len(r.gapHistogram))
 		}
+		p.GapHistogram[int32(i+1)] = r.gapHistogram[i]
 	}
 
 	return p
 }
 
-func (r *rtpStatsBaseLite) getAndResetSnapshotLite(snapshotLiteID uint32, extStartSN uint64, extHighestSN uint64) (*snapshotLite, *snapshotLite) {
+func (r *rtpStatsBaseLite) getAndResetSnapshotLite(snapshotLiteID uint32, extStartSN uint64, extHighestSN uint64) (snapshotLite, snapshotLite, bool) {
 	if !r.initialized {
-		return nil, nil
+		return snapshotLite{}, snapshotLite{}, false
 	}
 
 	idx := snapshotLiteID - cFirstSnapshotID
@@ -512,7 +508,7 @@ func (r *rtpStatsBaseLite) getAndResetSnapshotLite(snapshotLiteID uint32, extSta
 	// snapshot now
 	now := r.getSnapshotLite(mono.UnixNano(), extHighestSN+1)
 	r.snapshotLites[idx] = now
-	return &then, &now
+	return then, now, true
 }
 
 func (r *rtpStatsBaseLite) updateGapHistogram(gap int) {
@@ -554,4 +550,8 @@ func getPacketsExpected(extStartSN, extHighestSN uint64) uint64 {
 	return extHighestSN - extStartSN + 1
 }
 
+func shouldLog(count int) bool {
+	return count < 20 || count%200 == 0
+}
+
 // ----------------------------------

livekit-server/pkg/sfu/rtpstats/rtpstats_receiver.go

@@ -20,10 +20,12 @@ import (
 	"time"
 
 	"github.com/pion/rtcp"
+	"go.uber.org/zap"
 	"go.uber.org/zap/zapcore"
 
-	"github.com/livekit/livekit-server/pkg/sfu/utils"
 	"github.com/livekit/mediatransportutil"
+	"github.com/livekit/mediatransportutil/pkg/latency"
+	"github.com/livekit/mediatransportutil/pkg/utils"
 	"github.com/livekit/protocol/livekit"
 	"github.com/livekit/protocol/logger"
 	protoutils "github.com/livekit/protocol/utils"
@@ -37,12 +39,53 @@ const (
 	cReportSlack = float64(60.0)
 
 	cTSJumpTooHighFactor = float64(1.5)
+
+	restartThreshold = 5
 )
 
 // ---------------------------------------------------------------------
 
+type RTPFlowUnhandledReason int
+
+const (
+	RTPFlowUnhandledReasonNone RTPFlowUnhandledReason = iota
+	RTPFlowUnhandledReasonEnded
+	RTPFlowUnhandledReasonUnconfigured
+	RTPFlowUnhandledReasonPaddingOnly
+	RTPFlowUnhandledReasonPreStartTimestamp
+	RTPFlowUnhandledReasonOldTimestamp
+	RTPFlowUnhandledReasonPreStartSequenceNumber
+	RTPFlowUnhandledReasonOldSequenceNumber
+	RTPFlowUnhandledReasonRestart
+)
+
+func (r RTPFlowUnhandledReason) String() string {
+	switch r {
+	case RTPFlowUnhandledReasonNone:
+		return "NONE"
+	case RTPFlowUnhandledReasonEnded:
+		return "ENDED"
+	case RTPFlowUnhandledReasonUnconfigured:
+		return "UNCONFIGURED"
+	case RTPFlowUnhandledReasonPaddingOnly:
+		return "PADDING_ONLY"
+	case RTPFlowUnhandledReasonPreStartTimestamp:
+		return "PRE_START_TIMESTAMP"
+	case RTPFlowUnhandledReasonOldTimestamp:
+		return "OLD_TIMESTAMP"
+	case RTPFlowUnhandledReasonPreStartSequenceNumber:
+		return "PRE_START_SEQUENCE_NUMBER"
+	case RTPFlowUnhandledReasonOldSequenceNumber:
+		return "OLD_SEQUENCE_NUMBER"
+	case RTPFlowUnhandledReasonRestart:
+		return "RESTART"
+	default:
+		return fmt.Sprintf("UNKNOWN: %d", int(r))
+	}
+}
+
 type RTPFlowState struct {
-	IsNotHandled bool
+	UnhandledReason RTPFlowUnhandledReason
 
 	LossStartInclusive uint64
 	LossEndExclusive   uint64
@@ -59,7 +102,7 @@ func (r *RTPFlowState) MarshalLogObject(e zapcore.ObjectEncoder) error {
 		return nil
 	}
 
-	e.AddBool("IsNotHandled", r.IsNotHandled)
+	e.AddString("UnhandledReason", r.UnhandledReason.String())
 	e.AddUint64("LossStartInclusive", r.LossStartInclusive)
 	e.AddUint64("LossEndExclusive", r.LossEndExclusive)
 	e.AddBool("IsDuplicate", r.IsDuplicate)
@@ -71,6 +114,62 @@ func (r *RTPFlowState) MarshalLogObject(e zapcore.ObjectEncoder) error {
 
 // ---------------------------------------------------------------------
 
+type packet struct {
+	sequenceNumber uint16
+	timestamp      uint32
+}
+
+func (p packet) MarshalLogObject(e zapcore.ObjectEncoder) error {
+	e.AddUint16("sequenceNumber", p.sequenceNumber)
+	e.AddUint32("timestamp", p.timestamp)
+	return nil
+}
+
+// ---------------------------------------------------------------------
+
+type receiverUpdateLoggingFields struct {
+	packetTime       int64
+	sequenceNumber   uint16
+	timestamp        uint32
+	marker           bool
+	hdrSize          int
+	payloadSize      int
+	paddingSize      int
+	resSN            utils.WrapAroundUpdateResult[uint64]
+	gapSN            int64
+	resTS            utils.WrapAroundUpdateResult[uint64]
+	gapTS            int64
+	snRolloverCount  int
+	expectedTSJump   int64
+	tsRolloverCount  int
+	timeSinceHighest int64
+	rtpStats         *RTPStatsReceiver
+}
+
+func (rulf *receiverUpdateLoggingFields) MarshalLogObject(e zapcore.ObjectEncoder) error {
+	if rulf != nil {
+		e.AddObject("resSN", &rulf.resSN)
+		e.AddInt64("gapSN", rulf.gapSN)
+		e.AddObject("resTS", &rulf.resTS)
+		e.AddInt64("gapTS", rulf.gapTS)
+		e.AddInt("snRolloverCount", rulf.snRolloverCount)
+		e.AddInt64("expectedTSJump", rulf.expectedTSJump)
+		e.AddInt("tsRolloverCount", rulf.tsRolloverCount)
+		e.AddTime("packetTime", time.Unix(0, rulf.packetTime))
+		e.AddDuration("timeSinceHighest", time.Duration(rulf.timeSinceHighest))
+		e.AddUint16("sequenceNumber", rulf.sequenceNumber)
+		e.AddUint32("timestamp", rulf.timestamp)
+		e.AddBool("marker", rulf.marker)
+		e.AddInt("hdrSize", rulf.hdrSize)
+		e.AddInt("payloadSize", rulf.payloadSize)
+		e.AddInt("paddingSize", rulf.paddingSize)
+		e.AddObject("rtpStats", lockedRTPStatsReceiverLogEncoder{rulf.rtpStats})
+	}
+	return nil
+}
+
+// ---------------------------------------------------------------------
+
 type RTPStatsReceiver struct {
 	*rtpStatsBase
 
@@ -81,27 +180,40 @@ type RTPStatsReceiver struct {
 
 	history *protoutils.Bitmap[uint64]
 
-	propagationDelayEstimator *utils.OWDEstimator
+	propagationDelayEstimator *latency.OWDEstimator
 
 	clockSkewCount              int
 	clockSkewMediaPathCount     int
 	outOfOrderSenderReportCount int
-	largeJumpCount              int
-	largeJumpNegativeCount      int
 	timeReversedCount           int
+
+	packetsDroppedPreStartTimestamp      int
+	packetsDroppedOldTimestamp           int
+	packetsDroppedPreStartSequenceNumber int
+	packetsDroppedOldSequenceNumber      int
+
+	restartPacketsBuf [restartThreshold]packet
+	restartPacketsN   int
 }
 
 func NewRTPStatsReceiver(params RTPStatsParams) *RTPStatsReceiver {
 	return &RTPStatsReceiver{
 		rtpStatsBase:              newRTPStatsBase(params),
 		sequenceNumber:            utils.NewWrapAround[uint16, uint64](utils.WrapAroundParams{IsRestartAllowed: false}),
-		tsRolloverThreshold:       (1 << 31) * 1e9 / int64(params.ClockRate),
 		timestamp:                 utils.NewWrapAround[uint32, uint64](utils.WrapAroundParams{IsRestartAllowed: false}),
 		history:                   protoutils.NewBitmap[uint64](cHistorySize),
-		propagationDelayEstimator: utils.NewOWDEstimator(utils.OWDEstimatorParamsDefault),
+		propagationDelayEstimator: latency.NewOWDEstimator(latency.OWDEstimatorParamsDefault),
 	}
 }
 
+func (r *RTPStatsReceiver) SetClockRate(clockRate uint32) {
+	r.lock.Lock()
+	defer r.lock.Unlock()
+
+	r.tsRolloverThreshold = (1 << 31) * 1e9 / int64(clockRate)
+	r.rtpStatsBase.setClockRateLocked(clockRate)
+}
+
 func (r *RTPStatsReceiver) NewSnapshotId() uint32 {
 	r.lock.Lock()
 	defer r.lock.Unlock()
@@ -115,17 +227,19 @@ func (r *RTPStatsReceiver) getTSRolloverCount(diffNano int64, ts uint32) int {
 		return -1
 	}
 
-	excess := (diffNano - r.tsRolloverThreshold*2) * int64(r.params.ClockRate) / 1e9
-	roc := excess / (1 << 32)
-	if roc < 0 {
-		roc = 0
-	}
+	excess := (diffNano - r.tsRolloverThreshold*2) * int64(r.clockRate) / 1e9
+	roc := max(excess/(1<<32), 0)
 	if r.timestamp.GetHighest() > ts {
 		roc++
 	}
 	return int(roc)
 }
 
+func (r *RTPStatsReceiver) undoUpdatesLocked(resSN utils.WrapAroundUpdateResult[uint64], resTS utils.WrapAroundUpdateResult[uint64]) {
+	r.sequenceNumber.UndoUpdate(resSN)
+	r.timestamp.UndoUpdate(resTS)
+}
+
 func (r *RTPStatsReceiver) Update(
 	packetTime int64,
 	sequenceNumber uint16,
@@ -139,7 +253,11 @@ func (r *RTPStatsReceiver) Update(
 	defer r.lock.Unlock()
 
 	if r.endTime != 0 {
-		flowState.IsNotHandled = true
+		flowState.UnhandledReason = RTPFlowUnhandledReasonEnded
+		return
+	}
+	if r.clockRate == 0 {
+		flowState.UnhandledReason = RTPFlowUnhandledReasonUnconfigured
 		return
 	}
 
@@ -152,31 +270,10 @@ func (r *RTPStatsReceiver) Update(
 	var tsRolloverCount int
 	var snRolloverCount int
 
-	logger := func() logger.UnlikelyLogger {
-		return r.logger.WithUnlikelyValues(
-			"resSN", resSN,
-			"gapSN", gapSN,
-			"resTS", resTS,
-			"gapTS", gapTS,
-			"snRolloverCount", snRolloverCount,
-			"expectedTSJump", expectedTSJump,
-			"tsRolloverCount", tsRolloverCount,
-			"packetTime", time.Unix(0, packetTime),
-			"timeSinceHighest", time.Duration(timeSinceHighest),
-			"sequenceNumber", sequenceNumber,
-			"timestamp", timestamp,
-			"marker", marker,
-			"hdrSize", hdrSize,
-			"payloadSize", payloadSize,
-			"paddingSize", paddingSize,
-			"rtpStats", lockedRTPStatsReceiverLogEncoder{r},
-		)
-	}
-
 	if !r.initialized {
 		if payloadSize == 0 {
 			// do not start on a padding only packet
-			flowState.IsNotHandled = true
+			flowState.UnhandledReason = RTPFlowUnhandledReasonPaddingOnly
 			return
 		}
 
@@ -201,57 +298,164 @@ func (r *RTPStatsReceiver) Update(
 		)
 	} else {
 		resSN = r.sequenceNumber.Update(sequenceNumber)
-		if resSN.IsUnhandled {
-			flowState.IsNotHandled = true
-			return
-		}
 		gapSN = int64(resSN.ExtendedVal - resSN.PreExtendedHighest)
 
 		timeSinceHighest = packetTime - r.highestTime
+		expectedTSJump = int64(r.rtpConverter.ToRTPExt(time.Duration(timeSinceHighest)))
 		tsRolloverCount = r.getTSRolloverCount(timeSinceHighest, timestamp)
-		if tsRolloverCount >= 0 {
-			logger().Warnw("potential time stamp roll over", nil)
-		}
 		resTS = r.timestamp.Rollover(timestamp, tsRolloverCount)
+		gapTS = int64(resTS.ExtendedVal - resTS.PreExtendedHighest)
 		if resTS.IsUnhandled {
-			flowState.IsNotHandled = true
+			r.undoUpdatesLocked(resSN, resTS)
+
+			r.packetsDroppedPreStartTimestamp++
+			rulf := &receiverUpdateLoggingFields{
+				packetTime:       packetTime,
+				sequenceNumber:   sequenceNumber,
+				timestamp:        timestamp,
+				marker:           marker,
+				hdrSize:          hdrSize,
+				payloadSize:      payloadSize,
+				paddingSize:      paddingSize,
+				resSN:            resSN,
+				gapSN:            gapSN,
+				resTS:            resTS,
+				gapTS:            gapTS,
+				snRolloverCount:  snRolloverCount,
+				expectedTSJump:   expectedTSJump,
+				tsRolloverCount:  tsRolloverCount,
+				timeSinceHighest: timeSinceHighest,
+				rtpStats:         r,
+			}
+			if shouldLog(r.packetsDroppedPreStartTimestamp) {
+				r.logger.Warnw("dropping packet, pre-start timestamp", nil, zap.Inline(rulf))
+			}
+
+			if r.maybeRestart(sequenceNumber, timestamp, payloadSize) {
+				r.logger.Infow("potential restart", zap.Inline(rulf))
+				r.resetRestart()
+				flowState.UnhandledReason = RTPFlowUnhandledReasonRestart
+			} else {
+				flowState.UnhandledReason = RTPFlowUnhandledReasonPreStartTimestamp
+			}
 			return
 		}
-		gapTS = int64(resTS.ExtendedVal - resTS.PreExtendedHighest)
 
-		// it is possible to receive old packets in two different scenarios
-		// as it is not possible to detect how far to roll back, ignore old packets
-		//
-		// Case 1:
-		//  Very old time stamp, happens under the following conditions
-		//  - resume after long mute, big time stamp jump
-		//  - an out of order packet from before the mute arrives (unsure what causes this
-		//    very old packet to be trasmitted from remote), causing time stamp to jump back
-		//    to before mute, but it appears like it has rolled over.
-		//  Use a threshold against expected to ignore these.
-		if gapSN < 0 && gapTS > 0 {
-			expectedTSJump = timeSinceHighest * int64(r.params.ClockRate) / 1e9
-			if gapTS > int64(float64(expectedTSJump)*cTSJumpTooHighFactor) {
-				r.sequenceNumber.UndoUpdate(resSN)
-				r.timestamp.UndoUpdate(resTS)
-				logger().Warnw("dropping old packet, timestamp", nil)
-				flowState.IsNotHandled = true
+		if tsRolloverCount >= 0 && payloadSize > 0 {
+			rulf := &receiverUpdateLoggingFields{
+				packetTime:       packetTime,
+				sequenceNumber:   sequenceNumber,
+				timestamp:        timestamp,
+				marker:           marker,
+				hdrSize:          hdrSize,
+				payloadSize:      payloadSize,
+				paddingSize:      paddingSize,
+				resSN:            resSN,
+				gapSN:            gapSN,
+				resTS:            resTS,
+				gapTS:            gapTS,
+				snRolloverCount:  snRolloverCount,
+				expectedTSJump:   expectedTSJump,
+				tsRolloverCount:  tsRolloverCount,
+				timeSinceHighest: timeSinceHighest,
+				rtpStats:         r,
+			}
+			r.logger.Warnw("potential time stamp roll over", nil, zap.Inline(rulf))
+		}
+
+		if !resSN.IsUnhandled {
+			// it is possible to receive old packets in two different scenarios
+			// as it is not possible to detect how far to roll back, ignore old packets
+			//
+			// Case 1:
+			//  Very old time stamp, happens under the following conditions
+			//  - resume after long mute, this casues big time stamp jump ahead for the packets
+			//    after unmute
+			//  - an out of order packet from before the mute arrives (unsure what causes this
+			//    very old packet to be transmitted from remote), causing time stamp to jump back
+			//    to before mute, but it appears like it has rolled over.
+			//  Use a threshold against expected to ignore these.
+			if gapSN < 0 && gapTS > 0 {
+				if gapTS > int64(float64(expectedTSJump)*cTSJumpTooHighFactor) {
+					r.undoUpdatesLocked(resSN, resTS)
+
+					r.packetsDroppedOldTimestamp++
+					rulf := &receiverUpdateLoggingFields{
+						packetTime:       packetTime,
+						sequenceNumber:   sequenceNumber,
+						timestamp:        timestamp,
+						marker:           marker,
+						hdrSize:          hdrSize,
+						payloadSize:      payloadSize,
+						paddingSize:      paddingSize,
+						resSN:            resSN,
+						gapSN:            gapSN,
+						resTS:            resTS,
+						gapTS:            gapTS,
+						snRolloverCount:  snRolloverCount,
+						expectedTSJump:   expectedTSJump,
+						tsRolloverCount:  tsRolloverCount,
+						timeSinceHighest: timeSinceHighest,
+						rtpStats:         r,
+					}
+					if shouldLog(r.packetsDroppedOldTimestamp) {
+						r.logger.Warnw("dropping packet, old timestamp", nil, zap.Inline(rulf))
+					}
+
+					if r.maybeRestart(sequenceNumber, timestamp, payloadSize) {
+						r.logger.Infow("potential restart", zap.Inline(rulf))
+						r.resetRestart()
+						flowState.UnhandledReason = RTPFlowUnhandledReasonRestart
+					} else {
+						flowState.UnhandledReason = RTPFlowUnhandledReasonOldTimestamp
+					}
+					return
+				}
+			}
+
+			// Case 2:
+			//  Sequence number looks like it is moving forward, but it is actually a very old packet.
+			if gapTS < 0 && gapSN > 0 {
+				r.undoUpdatesLocked(resSN, resTS)
+
+				expectedTSJump = int64(r.rtpConverter.ToRTPExt(time.Duration(timeSinceHighest)))
+
+				r.packetsDroppedOldSequenceNumber++
+				rulf := &receiverUpdateLoggingFields{
+					packetTime:       packetTime,
+					sequenceNumber:   sequenceNumber,
+					timestamp:        timestamp,
+					marker:           marker,
+					hdrSize:          hdrSize,
+					payloadSize:      payloadSize,
+					paddingSize:      paddingSize,
+					resSN:            resSN,
+					gapSN:            gapSN,
+					resTS:            resTS,
+					gapTS:            gapTS,
+					snRolloverCount:  snRolloverCount,
+					expectedTSJump:   expectedTSJump,
+					tsRolloverCount:  tsRolloverCount,
+					timeSinceHighest: timeSinceHighest,
+					rtpStats:         r,
+				}
+				if shouldLog(r.packetsDroppedOldSequenceNumber) {
+					r.logger.Warnw("dropping packet, old sequence number", nil, zap.Inline(rulf))
+				}
+
+				if r.maybeRestart(sequenceNumber, timestamp, payloadSize) {
+					r.logger.Infow("potential restart", zap.Inline(rulf))
+					r.resetRestart()
+					flowState.UnhandledReason = RTPFlowUnhandledReasonRestart
+				} else {
+					flowState.UnhandledReason = RTPFlowUnhandledReasonOldSequenceNumber
+				}
 				return
 			}
 		}
 
-		// Case 2:
-		//  Sequence number looks like it is moving forward, but it is actually a very old packet.
-		if gapTS < 0 && gapSN > 0 {
-			r.sequenceNumber.UndoUpdate(resSN)
-			r.timestamp.UndoUpdate(resTS)
-			logger().Warnw("dropping old packet, sequence number", nil)
-			flowState.IsNotHandled = true
-			return
-		}
-
 		// it is possible that sequence number has rolled over too
-		if gapSN < 0 && gapTS > 0 && payloadSize > 0 {
+		if (gapSN < 0 || gapSN > (1<<15)) && gapTS > 0 && payloadSize > 0 {
 			// not possible to know how many cycles of sequence number roll over could have happened,
 			// ensure that it at least does not go backwards
 			snRolloverCount = 0
@@ -259,12 +463,63 @@ func (r *RTPStatsReceiver) Update(
 				snRolloverCount = 1
 			}
 			resSN = r.sequenceNumber.Rollover(sequenceNumber, snRolloverCount)
-			if resSN.IsUnhandled {
-				flowState.IsNotHandled = true
-				return
+			if !resSN.IsUnhandled {
+				rulf := &receiverUpdateLoggingFields{
+					packetTime:       packetTime,
+					sequenceNumber:   sequenceNumber,
+					timestamp:        timestamp,
+					marker:           marker,
+					hdrSize:          hdrSize,
+					payloadSize:      payloadSize,
+					paddingSize:      paddingSize,
+					resSN:            resSN,
+					gapSN:            gapSN,
+					resTS:            resTS,
+					gapTS:            gapTS,
+					snRolloverCount:  snRolloverCount,
+					expectedTSJump:   expectedTSJump,
+					tsRolloverCount:  tsRolloverCount,
+					timeSinceHighest: timeSinceHighest,
+					rtpStats:         r,
+				}
+				r.logger.Warnw("forcing sequence number rollover", nil, zap.Inline(rulf))
+			}
+		}
+
+		if resSN.IsUnhandled {
+			r.undoUpdatesLocked(resSN, resTS)
+
+			r.packetsDroppedPreStartSequenceNumber++
+			rulf := &receiverUpdateLoggingFields{
+				packetTime:       packetTime,
+				sequenceNumber:   sequenceNumber,
+				timestamp:        timestamp,
+				marker:           marker,
+				hdrSize:          hdrSize,
+				payloadSize:      payloadSize,
+				paddingSize:      paddingSize,
+				resSN:            resSN,
+				gapSN:            gapSN,
+				resTS:            resTS,
+				gapTS:            gapTS,
+				snRolloverCount:  snRolloverCount,
+				expectedTSJump:   expectedTSJump,
+				tsRolloverCount:  tsRolloverCount,
+				timeSinceHighest: timeSinceHighest,
+				rtpStats:         r,
+			}
+			if shouldLog(r.packetsDroppedPreStartSequenceNumber) {
+				r.logger.Warnw("dropping packet, pre-start sequence number", nil, zap.Inline(rulf))
 			}
 
-			logger().Warnw("forcing sequence number rollover", nil)
+			if r.maybeRestart(sequenceNumber, timestamp, payloadSize) {
+				r.logger.Infow("potential restart", zap.Inline(rulf))
+				r.resetRestart()
+				flowState.UnhandledReason = RTPFlowUnhandledReasonRestart
+			} else {
+				flowState.UnhandledReason = RTPFlowUnhandledReasonPreStartSequenceNumber
+			}
+			return
 		}
 	}
 	gapSN = int64(resSN.ExtendedVal - resSN.PreExtendedHighest)
@@ -287,32 +542,31 @@ func (r *RTPStatsReceiver) Update(
 		}
 
 		flowState.IsOutOfOrder = true
-
-		if !flowState.IsDuplicate && -gapSN >= cSequenceNumberLargeJumpThreshold {
-			r.largeJumpNegativeCount++
-			if (r.largeJumpNegativeCount-1)%100 == 0 {
-				logger().Warnw(
-					"large sequence number gap negative", nil,
-					"count", r.largeJumpNegativeCount,
-				)
-			}
-		}
 	} else { // in-order
-		if gapSN >= cSequenceNumberLargeJumpThreshold {
-			r.largeJumpCount++
-			if (r.largeJumpCount-1)%100 == 0 {
-				logger().Warnw(
-					"large sequence number gap", nil,
-					"count", r.largeJumpCount,
-				)
-			}
-		}
-
-		if resTS.ExtendedVal < resTS.PreExtendedHighest {
+		if resTS.ExtendedVal < resTS.PreExtendedHighest && r.bytes > 0 {
 			r.timeReversedCount++
-			if (r.timeReversedCount-1)%100 == 0 {
-				logger().Warnw(
+			if shouldLog(r.timeReversedCount) {
+				rulf := &receiverUpdateLoggingFields{
+					packetTime:       packetTime,
+					sequenceNumber:   sequenceNumber,
+					timestamp:        timestamp,
+					marker:           marker,
+					hdrSize:          hdrSize,
+					payloadSize:      payloadSize,
+					paddingSize:      paddingSize,
+					resSN:            resSN,
+					gapSN:            gapSN,
+					resTS:            resTS,
+					gapTS:            gapTS,
+					snRolloverCount:  snRolloverCount,
+					expectedTSJump:   expectedTSJump,
+					tsRolloverCount:  tsRolloverCount,
+					timeSinceHighest: timeSinceHighest,
+					rtpStats:         r,
+				}
+				r.logger.Warnw(
 					"time reversed", nil,
+					zap.Inline(rulf),
 					"count", r.timeReversedCount,
 				)
 			}
@@ -355,6 +609,7 @@ func (r *RTPStatsReceiver) Update(
 			r.updateJitter(resTS.ExtendedVal, packetTime)
 		}
 	}
+	r.resetRestart()
 	return
 }
 
@@ -363,10 +618,12 @@ func (r *RTPStatsReceiver) getExtendedSenderReport(srData *livekit.RTCPSenderRep
 	if r.srNewest != nil {
 		// use time since last sender report to ensure long gaps where the time stamp might
 		// jump more than half the range
-		timeSinceLastReport := mediatransportutil.NtpTime(srData.NtpTimestamp).Time().Sub(mediatransportutil.NtpTime(r.srNewest.NtpTimestamp).Time())
-		expectedRTPTimestampExt := r.srNewest.RtpTimestampExt + uint64(timeSinceLastReport.Nanoseconds()*int64(r.params.ClockRate)/1e9)
-		lbound := expectedRTPTimestampExt - uint64(cReportSlack*float64(r.params.ClockRate))
-		ubound := expectedRTPTimestampExt + uint64(cReportSlack*float64(r.params.ClockRate))
+		srTime := mediatransportutil.NtpTime(srData.NtpTimestamp).Time()
+		srNewestTime := mediatransportutil.NtpTime(r.srNewest.NtpTimestamp).Time()
+		timeSinceLastReport := srTime.Sub(srNewestTime)
+		expectedRTPTimestampExt := r.srNewest.RtpTimestampExt + r.rtpConverter.ToRTPExt(timeSinceLastReport)
+		lbound := expectedRTPTimestampExt - uint64(cReportSlack*float64(r.clockRate))
+		ubound := expectedRTPTimestampExt + uint64(cReportSlack*float64(r.clockRate))
 		isInRange := (srData.RtpTimestamp-uint32(lbound) < (1 << 31)) && (uint32(ubound)-srData.RtpTimestamp < (1 << 31))
 		if isInRange {
 			lbTSCycles := lbound & 0xFFFF_FFFF_0000_0000
@@ -409,7 +666,7 @@ func (r *RTPStatsReceiver) checkOutOfOrderSenderReport(srData *livekit.RTCPSende
 		// Or it could be due bad report generation.
 		// In any case, ignore out-of-order reports.
 		r.outOfOrderSenderReportCount++
-		if (r.outOfOrderSenderReportCount-1)%10 == 0 {
+		if shouldLog(r.outOfOrderSenderReportCount) {
 			r.logger.Infow(
 				"received sender report, out-of-order, skipping",
 				"current", WrappedRTCPSenderReportStateLogger{srData},
@@ -428,18 +685,22 @@ func (r *RTPStatsReceiver) checkRTPClockSkewForSenderReport(srData *livekit.RTCP
 		return
 	}
 
-	timeSinceLast := mediatransportutil.NtpTime(srData.NtpTimestamp).Time().Sub(mediatransportutil.NtpTime(r.srNewest.NtpTimestamp).Time()).Seconds()
+	srTime := mediatransportutil.NtpTime(srData.NtpTimestamp).Time()
+	srNewestTime := mediatransportutil.NtpTime(r.srNewest.NtpTimestamp).Time()
+	srFirstTime := mediatransportutil.NtpTime(r.srFirst.NtpTimestamp).Time()
+
+	timeSinceLast := srTime.Sub(srNewestTime).Seconds()
 	rtpDiffSinceLast := srData.RtpTimestampExt - r.srNewest.RtpTimestampExt
 	calculatedClockRateFromLast := float64(rtpDiffSinceLast) / timeSinceLast
 
-	timeSinceFirst := mediatransportutil.NtpTime(srData.NtpTimestamp).Time().Sub(mediatransportutil.NtpTime(r.srFirst.NtpTimestamp).Time()).Seconds()
+	timeSinceFirst := srTime.Sub(srFirstTime).Seconds()
 	rtpDiffSinceFirst := srData.RtpTimestampExt - r.srFirst.RtpTimestampExt
 	calculatedClockRateFromFirst := float64(rtpDiffSinceFirst) / timeSinceFirst
 
-	if (timeSinceLast > 0.2 && math.Abs(float64(r.params.ClockRate)-calculatedClockRateFromLast) > 0.2*float64(r.params.ClockRate)) ||
-		(timeSinceFirst > 0.2 && math.Abs(float64(r.params.ClockRate)-calculatedClockRateFromFirst) > 0.2*float64(r.params.ClockRate)) {
+	if (timeSinceLast > 0.2 && math.Abs(float64(r.clockRate)-calculatedClockRateFromLast) > 0.2*float64(r.clockRate)) ||
+		(timeSinceFirst > 0.2 && math.Abs(float64(r.clockRate)-calculatedClockRateFromFirst) > 0.2*float64(r.clockRate)) {
 		r.clockSkewCount++
-		if (r.clockSkewCount-1)%100 == 0 {
+		if shouldLog(r.clockSkewCount) {
 			r.logger.Infow(
 				"received sender report, clock skew",
 				"current", WrappedRTCPSenderReportStateLogger{srData},
@@ -463,20 +724,20 @@ func (r *RTPStatsReceiver) checkRTPClockSkewAgainstMediaPathForSenderReport(srDa
 
 	nowNano := mono.UnixNano()
 	timeSinceSR := time.Duration(nowNano - srData.AtAdjusted)
-	extNowTSSR := srData.RtpTimestampExt + uint64(timeSinceSR.Nanoseconds()*int64(r.params.ClockRate)/1e9)
+	extNowTSSR := srData.RtpTimestampExt + r.rtpConverter.ToRTPExt(timeSinceSR)
 
 	timeSinceHighest := time.Duration(nowNano - r.highestTime)
-	extNowTSHighest := r.timestamp.GetExtendedHighest() + uint64(timeSinceHighest.Nanoseconds()*int64(r.params.ClockRate)/1e9)
+	extNowTSHighest := r.timestamp.GetExtendedHighest() + r.rtpConverter.ToRTPExt(timeSinceHighest)
 	diffHighest := extNowTSSR - extNowTSHighest
 
 	timeSinceFirst := time.Duration(nowNano - r.firstTime)
-	extNowTSFirst := r.timestamp.GetExtendedStart() + uint64(timeSinceFirst.Nanoseconds()*int64(r.params.ClockRate)/1e9)
+	extNowTSFirst := r.timestamp.GetExtendedStart() + r.rtpConverter.ToRTPExt(timeSinceFirst)
 	diffFirst := extNowTSSR - extNowTSFirst
 
 	// is it more than 5 seconds off?
-	if uint32(math.Abs(float64(int64(diffHighest)))) > 5*r.params.ClockRate || uint32(math.Abs(float64(int64(diffFirst)))) > 5*r.params.ClockRate {
+	if uint32(math.Abs(float64(int64(diffHighest)))) > 5*r.clockRate || uint32(math.Abs(float64(int64(diffFirst)))) > 5*r.clockRate {
 		r.clockSkewMediaPathCount++
-		if (r.clockSkewMediaPathCount-1)%100 == 0 {
+		if shouldLog(r.clockSkewMediaPathCount) {
 			r.logger.Infow(
 				"received sender report, clock skew against media path",
 				"current", WrappedRTCPSenderReportStateLogger{srData},
@@ -496,7 +757,8 @@ func (r *RTPStatsReceiver) checkRTPClockSkewAgainstMediaPathForSenderReport(srDa
 }
 
 func (r *RTPStatsReceiver) updatePropagationDelayAndRecordSenderReport(srData *livekit.RTCPSenderReportState) {
-	senderClockTime := mediatransportutil.NtpTime(srData.NtpTimestamp).Time().UnixNano()
+	srTime := mediatransportutil.NtpTime(srData.NtpTimestamp).Time()
+	senderClockTime := srTime.UnixNano()
 	estimatedPropagationDelay, stepChange := r.propagationDelayEstimator.Update(senderClockTime, srData.At)
 	if stepChange {
 		r.logger.Debugw(
@@ -518,7 +780,7 @@ func (r *RTPStatsReceiver) SetRtcpSenderReportData(srData *livekit.RTCPSenderRep
 	r.lock.Lock()
 	defer r.lock.Unlock()
 
-	if srData == nil || !r.initialized {
+	if srData == nil || !r.initialized || r.clockRate == 0 {
 		return false
 	}
 
@@ -542,9 +804,11 @@ func (r *RTPStatsReceiver) SetRtcpSenderReportData(srData *livekit.RTCPSenderRep
 	r.updatePropagationDelayAndRecordSenderReport(srDataExt)
 	r.checkRTPClockSkewAgainstMediaPathForSenderReport(srDataExt)
 
-	if err, loggingFields := r.maybeAdjustFirstPacketTime(r.srNewest, 0, r.timestamp.GetExtendedStart()); err != nil {
+	adjustment, loggingFields, err := r.maybeAdjustFirstPacketTime(r.srNewest, 0, r.timestamp.GetExtendedStart())
+	if err != nil {
 		r.logger.Infow(err.Error(), append(loggingFields, "rtpStats", lockedRTPStatsReceiverLogEncoder{r})...)
 	}
+	r.propagationDelayEstimator.InitialAdjustment(adjustment)
 	return true
 }
 
@@ -571,8 +835,8 @@ func (r *RTPStatsReceiver) GetRtcpReceptionReport(ssrc uint32, proxyFracLost uin
 	defer r.lock.Unlock()
 
 	extHighestSN := r.sequenceNumber.GetExtendedHighest()
-	then, now := r.getAndResetSnapshot(snapshotID, r.sequenceNumber.GetExtendedStart(), extHighestSN)
-	if now == nil || then == nil {
+	then, now, ok := r.getAndResetSnapshot(snapshotID, r.sequenceNumber.GetExtendedStart(), extHighestSN)
+	if !ok {
 		return nil
 	}
 
@@ -594,15 +858,9 @@ func (r *RTPStatsReceiver) GetRtcpReceptionReport(ssrc uint32, proxyFracLost uin
 		packetsLost = 0
 	}
 	lossRate := float32(packetsLost) / float32(packetsExpected)
-	fracLost := uint8(lossRate * 256.0)
-	if proxyFracLost > fracLost {
-		fracLost = proxyFracLost
-	}
+	fracLost := max(proxyFracLost, uint8(lossRate*256.0))
 
-	totalLost := r.packetsLost
-	if totalLost > 0xffffff { // 24-bits max
-		totalLost = 0xffffff
-	}
+	totalLost := min(r.packetsLost, 0xffffff) // 24-bits max
 
 	lastSR := uint32(0)
 	dlsr := uint32(0)
@@ -629,7 +887,7 @@ func (r *RTPStatsReceiver) DeltaInfo(snapshotID uint32) *RTPDeltaInfo {
 	r.lock.Lock()
 	defer r.lock.Unlock()
 
-	deltaInfo, err, loggingFields := r.deltaInfo(
+	deltaInfo, loggingFields, err := r.deltaInfo(
 		snapshotID,
 		r.sequenceNumber.GetExtendedStart(),
 		r.sequenceNumber.GetExtendedHighest(),
@@ -653,6 +911,10 @@ func (r *RTPStatsReceiver) MarshalLogObject(e zapcore.ObjectEncoder) error {
 }
 
 func (r *RTPStatsReceiver) ToProto() *livekit.RTPStats {
+	if r == nil {
+		return nil
+	}
+
 	r.lock.RLock()
 	defer r.lock.RUnlock()
 
@@ -696,6 +958,37 @@ func (r *RTPStatsReceiver) ExtendedHighestSequenceNumber() uint64 {
 	return r.sequenceNumber.GetExtendedHighest()
 }
 
+func (r *RTPStatsReceiver) maybeRestart(sn uint16, ts uint32, payloadSize int) bool {
+	if payloadSize > 0 {
+		if r.restartPacketsN < restartThreshold {
+			r.restartPacketsBuf[r.restartPacketsN] = packet{sn, ts}
+			r.restartPacketsN++
+		} else {
+			// keep last restartThreshold entries: shift left and append
+			copy(r.restartPacketsBuf[:], r.restartPacketsBuf[1:])
+			r.restartPacketsBuf[restartThreshold-1] = packet{sn, ts}
+		}
+	}
+	if r.restartPacketsN < restartThreshold {
+		return false
+	}
+
+	// check for contiguous sequence numbers and equal or increasing timestamps
+	for i := 1; i < r.restartPacketsN; i++ {
+		p := &r.restartPacketsBuf[i]
+		prev := &r.restartPacketsBuf[i-1]
+		if p.sequenceNumber != prev.sequenceNumber+1 || (p.timestamp-prev.timestamp) > (1<<31) {
+			return false
+		}
+	}
+
+	return true
+}
+
+func (r *RTPStatsReceiver) resetRestart() {
+	r.restartPacketsN = 0
+}
+
 // ----------------------------------
 
 type lockedRTPStatsReceiverLogEncoder struct {
@@ -725,6 +1018,18 @@ func (r lockedRTPStatsReceiverLogEncoder) MarshalLogObject(e zapcore.ObjectEncod
 	e.AddUint64("extHighestTS", extHighestTS)
 
 	e.AddObject("propagationDelayEstimator", r.propagationDelayEstimator)
+
+	e.AddInt("clockSkewCount", r.clockSkewCount)
+	e.AddInt("clockSkewMediaPathCount", r.clockSkewMediaPathCount)
+	e.AddInt("outOfOrderSenderReportCount", r.outOfOrderSenderReportCount)
+	e.AddInt("timeReversedCount", r.timeReversedCount)
+
+	e.AddInt("packetsDroppedPreStartTimestamp", r.packetsDroppedPreStartTimestamp)
+	e.AddInt("packetsDroppedOldTimestamp", r.packetsDroppedOldTimestamp)
+	e.AddInt("packetsDroppedPreStartSequenceNumber", r.packetsDroppedPreStartSequenceNumber)
+	e.AddInt("packetsDroppedOldSequenceNumber", r.packetsDroppedOldSequenceNumber)
+
+	e.AddArray("restartPackets", logger.ObjectSlice(r.restartPacketsBuf[:r.restartPacketsN]))
 	return nil
 }
 

livekit-server/pkg/sfu/rtpstats/rtpstats_receiver_lite.go

@@ -17,7 +17,7 @@ package rtpstats
 import (
 	"go.uber.org/zap/zapcore"
 
-	"github.com/livekit/livekit-server/pkg/sfu/utils"
+	"github.com/livekit/mediatransportutil/pkg/utils"
 	"github.com/livekit/protocol/livekit"
 	"github.com/livekit/protocol/utils/mono"
 )
@@ -102,7 +102,9 @@ func (r *RTPStatsReceiverLite) Update(packetTime int64, packetSize int, sequence
 	gapSN := int64(resSN.ExtendedVal - resSN.PreExtendedHighest)
 	if gapSN <= 0 { // duplicate OR out-of-order
 		r.packetsOutOfOrder++ // counting duplicate as out-of-order
-		r.packetsLost--
+		if gapSN != 0 && r.packetsLost > 0 {
+			r.packetsLost--
+		}
 	} else { // in-order
 		r.updateGapHistogram(int(gapSN))
 		r.packetsLost += uint64(gapSN - 1)
@@ -119,7 +121,7 @@ func (r *RTPStatsReceiverLite) DeltaInfoLite(snapshotLiteID uint32) *RTPDeltaInf
 	r.lock.Lock()
 	defer r.lock.Unlock()
 
-	deltaInfoLite, err, loggingFields := r.deltaInfoLite(
+	deltaInfoLite, loggingFields, err := r.deltaInfoLite(
 		snapshotLiteID,
 		r.sequenceNumber.GetExtendedStart(),
 		r.sequenceNumber.GetExtendedHighest(),

livekit-server/pkg/sfu/rtpstats/rtpstats_sender.go

@@ -21,12 +21,12 @@ import (
 	"time"
 
 	"github.com/pion/rtcp"
+	"go.uber.org/zap"
 	"go.uber.org/zap/zapcore"
 	"google.golang.org/protobuf/types/known/timestamppb"
 
 	"github.com/livekit/mediatransportutil"
 	"github.com/livekit/protocol/livekit"
-	"github.com/livekit/protocol/logger"
 	"github.com/livekit/protocol/utils/mono"
 )
 
@@ -40,6 +40,9 @@ const (
 	snInfoFlagOutOfOrder
 )
 
+// max number of receiver reports kept per sender snapshot for logging
+const cMaxProcessedReceptionReports = 64
+
 type snInfo struct {
 	pktSize uint16
 	hdrSize uint8
@@ -103,7 +106,8 @@ type wrappedReceptionReportsLogger struct {
 }
 
 func (w wrappedReceptionReportsLogger) MarshalLogObject(e zapcore.ObjectEncoder) error {
-	for i, rr := range w.senderSnapshotReceiverView.processedReceptionReports {
+	for i := 0; i < w.senderSnapshotReceiverView.processedReceptionReportsSize; i++ {
+		rr := w.senderSnapshotReceiverView.processedReceptionReports[(w.senderSnapshotReceiverView.processedReceptionReportsHead+i)%cMaxProcessedReceptionReports]
 		e.AddReflected(fmt.Sprintf("%d", i), rr)
 	}
 
@@ -192,9 +196,13 @@ type senderSnapshotReceiverView struct {
 	maxRtt    uint32
 	maxJitter float64
 
-	extLastRRSN                uint64
-	intervalStats              intervalStats
-	processedReceptionReports  []rtcp.ReceptionReport
+	extLastRRSN   uint64
+	intervalStats intervalStats
+
+	processedReceptionReports     [cMaxProcessedReceptionReports]rtcp.ReceptionReport
+	processedReceptionReportsHead int
+	processedReceptionReportsSize int
+
 	metadataCacheOverflowCount int
 }
 
@@ -272,6 +280,70 @@ func (s *senderSnapshot) maybeUpdateMaxJitter(jitter float64) {
 
 // -------------------------------------------------------------------
 
+type senderUpdateLoggingFields struct {
+	packetTime        int64
+	extSequenceNumber uint64
+	extTimestamp      uint64
+	marker            bool
+	hdrSize           int
+	payloadSize       int
+	paddingSize       int
+	gapSN             int64
+	gapTS             int64
+	timeSinceHighest  int64
+	rtpStats          *RTPStatsSender
+}
+
+func (sulf *senderUpdateLoggingFields) MarshalLogObject(e zapcore.ObjectEncoder) error {
+	if sulf != nil {
+		e.AddUint64("currSN", sulf.extSequenceNumber)
+		e.AddInt64("gapSN", sulf.gapSN)
+		e.AddUint64("currTS", sulf.extTimestamp)
+		e.AddInt64("gapTS", sulf.gapTS)
+		e.AddTime("packetTime", time.Unix(0, sulf.packetTime))
+		e.AddDuration("timeSinceHighest", time.Duration(sulf.timeSinceHighest))
+		e.AddBool("marker", sulf.marker)
+		e.AddInt("hdrSize", sulf.hdrSize)
+		e.AddInt("payloadSize", sulf.payloadSize)
+		e.AddInt("paddingSize", sulf.paddingSize)
+		e.AddObject("rtpStats", lockedRTPStatsSenderLogEncoder{sulf.rtpStats})
+	}
+	return nil
+}
+
+// -------------------------------------------------------------------
+
+type senderReportLoggingFields struct {
+	srData             *livekit.RTCPSenderReportState
+	publisherSRData    *livekit.RTCPSenderReportState
+	tsOffset           uint64
+	reportTime         int64
+	reportTimeAdjusted int64
+	nowNano            int64
+	nowRTPExt          uint64
+	rtpStats           *RTPStatsSender
+}
+
+func (srlf *senderReportLoggingFields) MarshalLogObject(e zapcore.ObjectEncoder) error {
+	if srlf != nil {
+		e.AddObject("curr", WrappedRTCPSenderReportStateLogger{srlf.srData})
+		e.AddObject("feed", WrappedRTCPSenderReportStateLogger{srlf.publisherSRData})
+		e.AddUint64("tsOffset", srlf.tsOffset)
+		e.AddTime("timeNow", time.Now())
+		e.AddInt64("reportTime", srlf.reportTime)
+		e.AddInt64("reportTimeAdjusted", srlf.reportTimeAdjusted)
+		e.AddDuration("timeSinceHighest", time.Duration(srlf.nowNano-srlf.rtpStats.highestTime))
+		e.AddDuration("timeSinceFirst", time.Duration(srlf.nowNano-srlf.rtpStats.firstTime))
+		e.AddDuration("timeSincePublisherSRAdjusted", time.Duration(srlf.nowNano-srlf.publisherSRData.AtAdjusted))
+		e.AddDuration("timeSincePublisherSR", time.Duration(srlf.nowNano-srlf.publisherSRData.At))
+		e.AddUint64("nowRTPExt", srlf.nowRTPExt)
+		e.AddObject("rtpStats", lockedRTPStatsSenderLogEncoder{srlf.rtpStats})
+	}
+	return nil
+}
+
+// -------------------------------------------------------------------
+
 type rttMarker struct {
 	ntpTime mediatransportutil.NtpTime
 	sentAt  time.Time
@@ -308,10 +380,8 @@ type RTPStatsSender struct {
 	nextSenderSnapshotID uint32
 	senderSnapshots      []senderSnapshot
 
-	clockSkewCount         int
-	largeJumpNegativeCount int
-	largeJumpCount         int
-	timeReversedCount      int
+	clockSkewCount    int
+	timeReversedCount int
 }
 
 func NewRTPStatsSender(params RTPStatsParams, cacheSize int) *RTPStatsSender {
@@ -399,7 +469,7 @@ func (r *RTPStatsSender) Update(
 	r.lock.Lock()
 	defer r.lock.Unlock()
 
-	if r.endTime != 0 {
+	if r.endTime != 0 || r.clockRate == 0 {
 		return
 	}
 
@@ -444,21 +514,6 @@ func (r *RTPStatsSender) Update(
 	pktSize := uint64(hdrSize + payloadSize + paddingSize)
 	isDuplicate := false
 	gapSN := int64(extSequenceNumber - r.extHighestSN)
-	ulgr := func() logger.UnlikelyLogger {
-		return r.logger.WithUnlikelyValues(
-			"currSN", extSequenceNumber,
-			"gapSN", gapSN,
-			"currTS", extTimestamp,
-			"gapTS", int64(extTimestamp-r.extHighestTS),
-			"packetTime", time.Unix(0, packetTime),
-			"timeSinceHighest", time.Duration(packetTime-r.highestTime),
-			"marker", marker,
-			"hdrSize", hdrSize,
-			"payloadSize", payloadSize,
-			"paddingSize", paddingSize,
-			"rtpStats", lockedRTPStatsSenderLogEncoder{r},
-		)
-	}
 	if gapSN <= 0 { // duplicate OR out-of-order
 		if payloadSize == 0 && extSequenceNumber < r.extStartSN && !r.params.IsRTX {
 			// do not start on a padding only packet
@@ -479,8 +534,22 @@ func (r *RTPStatsSender) Update(
 				r.senderSnapshots[i].maybeReinit(r.extStartSN, extSequenceNumber)
 			}
 
-			ulgr().Infow(
+			sulf := &senderUpdateLoggingFields{
+				packetTime:        packetTime,
+				extSequenceNumber: extSequenceNumber,
+				extTimestamp:      extTimestamp,
+				marker:            marker,
+				hdrSize:           hdrSize,
+				payloadSize:       payloadSize,
+				paddingSize:       paddingSize,
+				gapSN:             gapSN,
+				gapTS:             int64(extTimestamp - r.extHighestTS),
+				timeSinceHighest:  packetTime - r.highestTime,
+				rtpStats:          r,
+			}
+			r.logger.Infow(
 				"adjusting start sequence number",
+				zap.Inline(sulf),
 				"snAfter", extSequenceNumber,
 				"tsAfter", extTimestamp,
 			)
@@ -500,32 +569,26 @@ func (r *RTPStatsSender) Update(
 			r.packetsLost--
 			r.setSnInfo(extSequenceNumber, r.extHighestSN, uint16(pktSize), uint8(hdrSize), uint16(payloadSize), marker, true)
 		}
-
-		if !isDuplicate && -gapSN >= cSequenceNumberLargeJumpThreshold {
-			r.largeJumpNegativeCount++
-			if (r.largeJumpNegativeCount-1)%100 == 0 {
-				ulgr().Warnw(
-					"large sequence number gap negative", nil,
-					"count", r.largeJumpNegativeCount,
-				)
-			}
-		}
 	} else { // in-order
-		if gapSN >= cSequenceNumberLargeJumpThreshold {
-			r.largeJumpCount++
-			if (r.largeJumpCount-1)%100 == 0 {
-				ulgr().Warnw(
-					"large sequence number gap", nil,
-					"count", r.largeJumpCount,
-				)
-			}
-		}
-
 		if extTimestamp < r.extHighestTS {
 			r.timeReversedCount++
-			if (r.timeReversedCount-1)%100 == 0 {
-				ulgr().Warnw(
+			if shouldLog(r.timeReversedCount) {
+				sulf := &senderUpdateLoggingFields{
+					packetTime:        packetTime,
+					extSequenceNumber: extSequenceNumber,
+					extTimestamp:      extTimestamp,
+					marker:            marker,
+					hdrSize:           hdrSize,
+					payloadSize:       payloadSize,
+					paddingSize:       paddingSize,
+					gapSN:             gapSN,
+					gapTS:             int64(extTimestamp - r.extHighestTS),
+					timeSinceHighest:  packetTime - r.highestTime,
+					rtpStats:          r,
+				}
+				r.logger.Warnw(
 					"time reversed", nil,
+					zap.Inline(sulf),
 					"count", r.timeReversedCount,
 				)
 			}
@@ -544,8 +607,22 @@ func (r *RTPStatsSender) Update(
 	}
 
 	if extTimestamp < r.extStartTS {
-		ulgr().Infow(
+		sulf := &senderUpdateLoggingFields{
+			packetTime:        packetTime,
+			extSequenceNumber: extSequenceNumber,
+			extTimestamp:      extTimestamp,
+			marker:            marker,
+			hdrSize:           hdrSize,
+			payloadSize:       payloadSize,
+			paddingSize:       paddingSize,
+			gapSN:             gapSN,
+			gapTS:             int64(extTimestamp - r.extHighestTS),
+			timeSinceHighest:  packetTime - r.highestTime,
+			rtpStats:          r,
+		}
+		r.logger.Infow(
 			"adjusting start timestamp",
+			zap.Inline(sulf),
 			"snAfter", extSequenceNumber,
 			"tsAfter", extTimestamp,
 		)
@@ -628,14 +705,24 @@ func (r *RTPStatsSender) UpdateFromReceiverReport(rr rtcp.ReceptionReport) (rtt
 		r.lastRR = rr
 	}()
 
-	timeSinceLastRR := func() time.Duration {
-		if r.lastRRTime != 0 {
-			return time.Duration(nowNano - r.lastRRTime)
-		}
-		return time.Duration(nowNano - r.startTime)
+	timeSinceLastRR := time.Duration(nowNano - r.startTime)
+	if r.lastRRTime != 0 {
+		timeSinceLastRR = time.Duration(nowNano - r.lastRRTime)
 	}
 
 	extReceivedRRSN := extHighestSNFromRR + (r.extStartSN & 0xFFFF_FFFF_FFFF_0000)
+	if int64(r.extHighestSN-extReceivedRRSN) < 0 || int64(r.extHighestSN-extReceivedRRSN) > (1<<32) {
+		r.logger.Infow(
+			"receiver report runaway, dropping",
+			"timeSinceLastRR", timeSinceLastRR,
+			"receivedRR", rr,
+			"extHighestSNFromRR", extHighestSNFromRR,
+			"extReceivedRRSN", extReceivedRRSN,
+			"rtpStats", lockedRTPStatsSenderLogEncoder{r},
+		)
+		return
+	}
+
 	if r.extHighestSNFromRR != extHighestSNFromRR && int64(r.extHighestSN-extReceivedRRSN) >= (1<<16) {
 		// there are cases where remote does not send RTCP Receiver Report for extended periods of time,
 		// some times several minutes, in that interval the sequence number rolls over,
@@ -645,7 +732,7 @@ func (r *RTPStatsSender) UpdateFromReceiverReport(rr rtcp.ReceptionReport) (rtt
 		// should not roll over for long time, for e. g. it will approximately take 100 days at 500 pps).
 		// So, there seems to be a remote reporter issue where the sequence number rollover is missed.
 		//
-		// catch up till diffrence between highest sent and highest received via receiver report is
+		// catch up till difference between highest sent and highest received via receiver report is
 		// less than full 16-bit range.
 		//
 		// in a different flavor, there are clients that do not report properly,
@@ -654,7 +741,7 @@ func (r *RTPStatsSender) UpdateFromReceiverReport(rr rtcp.ReceptionReport) (rtt
 		// RTCP RR does not change.
 		r.logger.Infow(
 			"receiver report missed rollover, adjusting",
-			"timeSinceLastRR", timeSinceLastRR(),
+			"timeSinceLastRR", timeSinceLastRR,
 			"receivedRR", rr,
 			"extHighestSNFromRR", extHighestSNFromRR,
 			"extReceivedRRSN", extReceivedRRSN,
@@ -667,7 +754,7 @@ func (r *RTPStatsSender) UpdateFromReceiverReport(rr rtcp.ReceptionReport) (rtt
 		}
 		r.logger.Infow(
 			"receiver report missed rollover, adjusted",
-			"timeSinceLastRR", timeSinceLastRR(),
+			"timeSinceLastRR", timeSinceLastRR,
 			"receivedRR", rr,
 			"extHighestSNFromRR", extHighestSNFromRR,
 			"extReceivedRRSN", extReceivedRRSN,
@@ -679,7 +766,7 @@ func (r *RTPStatsSender) UpdateFromReceiverReport(rr rtcp.ReceptionReport) (rtt
 		// if remote adjusts somehow, roll back alignment
 		r.logger.Infow(
 			"receiver report caught up rollover, adjusting",
-			"timeSinceLastRR", timeSinceLastRR(),
+			"timeSinceLastRR", timeSinceLastRR,
 			"receivedRR", rr,
 			"extHighestSNFromRR", extHighestSNFromRR,
 			"extReceivedRRSN", extReceivedRRSN,
@@ -692,7 +779,7 @@ func (r *RTPStatsSender) UpdateFromReceiverReport(rr rtcp.ReceptionReport) (rtt
 		}
 		r.logger.Infow(
 			"receiver report caught up rollover, adjusted",
-			"timeSinceLastRR", timeSinceLastRR(),
+			"timeSinceLastRR", timeSinceLastRR,
 			"receivedRR", rr,
 			"extHighestSNFromRR", extHighestSNFromRR,
 			"extReceivedRRSN", extReceivedRRSN,
@@ -701,9 +788,9 @@ func (r *RTPStatsSender) UpdateFromReceiverReport(rr rtcp.ReceptionReport) (rtt
 	}
 
 	if r.extHighestSNFromRR > extHighestSNFromRR {
-		r.logger.Infow(
+		r.logger.Debugw(
 			"receiver report out-of-order, dropping",
-			"timeSinceLastRR", timeSinceLastRR(),
+			"timeSinceLastRR", timeSinceLastRR,
 			"receivedRR", rr,
 			"extHighestSNFromRR", extHighestSNFromRR,
 			"extReceivedRRSN", extReceivedRRSN,
@@ -768,7 +855,7 @@ func (r *RTPStatsSender) UpdateFromReceiverReport(rr rtcp.ReceptionReport) (rtt
 					"metadata cache overflow",
 					"senderSnapshotID", i+cFirstSnapshotID,
 					"senderSnapshot", s,
-					"timeSinceLastRR", timeSinceLastRR(),
+					"timeSinceLastRR", timeSinceLastRR,
 					"receivedRR", rr,
 					"extReceivedRRSN", extReceivedRRSN,
 					"packetsInInterval", extReceivedRRSN-s.receiverView.extLastRRSN,
@@ -779,7 +866,16 @@ func (r *RTPStatsSender) UpdateFromReceiverReport(rr rtcp.ReceptionReport) (rtt
 			}
 		}
 		s.receiverView.extLastRRSN = extReceivedRRSN
-		s.receiverView.processedReceptionReports = append(s.receiverView.processedReceptionReports, rr)
+
+		s.receiverView.processedReceptionReports[s.receiverView.processedReceptionReportsHead] = rr
+		s.receiverView.processedReceptionReportsHead++
+		if s.receiverView.processedReceptionReportsHead >= cMaxProcessedReceptionReports {
+			s.receiverView.processedReceptionReportsHead = 0
+		}
+		s.receiverView.processedReceptionReportsSize++
+		if s.receiverView.processedReceptionReportsSize > cMaxProcessedReceptionReports {
+			s.receiverView.processedReceptionReportsSize = cMaxProcessedReceptionReports
+		}
 	}
 
 	return
@@ -796,11 +892,11 @@ func (r *RTPStatsSender) MaybeAdjustFirstPacketTime(publisherSRData *livekit.RTC
 	r.lock.Lock()
 	defer r.lock.Unlock()
 
-	if !r.initialized || publisherSRData == nil {
+	if !r.initialized || publisherSRData == nil || r.clockRate == 0 {
 		return
 	}
 
-	if err, loggingFields := r.maybeAdjustFirstPacketTime(publisherSRData, tsOffset, r.extStartTS); err != nil {
+	if _, loggingFields, err := r.maybeAdjustFirstPacketTime(publisherSRData, tsOffset, r.extStartTS); err != nil {
 		r.logger.Infow(err.Error(), append(loggingFields, "rtpStats", lockedRTPStatsSenderLogEncoder{r})...)
 	}
 }
@@ -809,14 +905,13 @@ func (r *RTPStatsSender) GetExpectedRTPTimestamp(at time.Time) (expectedTSExt ui
 	r.lock.RLock()
 	defer r.lock.RUnlock()
 
-	if r.firstTime == 0 {
+	if r.firstTime == 0 || r.clockRate == 0 {
 		err = errors.New("uninitialized")
 		return
 	}
 
 	timeDiff := at.Sub(time.Unix(0, r.firstTime))
-	expectedRTPDiff := timeDiff.Nanoseconds() * int64(r.params.ClockRate) / 1e9
-	expectedTSExt = r.extStartTS + uint64(expectedRTPDiff)
+	expectedTSExt = r.extStartTS + r.rtpConverter.ToRTPExt(timeDiff)
 	return
 }
 
@@ -824,7 +919,7 @@ func (r *RTPStatsSender) GetRtcpSenderReport(ssrc uint32, publisherSRData *livek
 	r.lock.Lock()
 	defer r.lock.Unlock()
 
-	if !r.initialized || publisherSRData == nil {
+	if !r.initialized || publisherSRData == nil || r.clockRate == 0 {
 		return nil
 	}
 
@@ -834,19 +929,21 @@ func (r *RTPStatsSender) GetRtcpSenderReport(ssrc uint32, publisherSRData *livek
 		nowNTP             mediatransportutil.NtpTime
 		nowRTPExt          uint64
 	)
+	nowNano := mono.UnixNano()
 	if passThrough {
-		reportTime = publisherSRData.At
-		reportTimeAdjusted = publisherSRData.AtAdjusted
+		timeSincePublisherSR := time.Duration(nowNano - publisherSRData.At)
+		reportTime = publisherSRData.At + timeSincePublisherSR.Nanoseconds()
+		reportTimeAdjusted = publisherSRData.AtAdjusted + timeSincePublisherSR.Nanoseconds()
 
-		nowNTP = mediatransportutil.NtpTime(publisherSRData.NtpTimestamp)
-		nowRTPExt = publisherSRData.RtpTimestampExt - tsOffset
+		nowNTP = mediatransportutil.ToNtpTime(mediatransportutil.NtpTime(publisherSRData.NtpTimestamp).Time().Add(timeSincePublisherSR))
+		nowRTPExt = publisherSRData.RtpTimestampExt - tsOffset + r.rtpConverter.ToRTPExt(timeSincePublisherSR)
 	} else {
-		timeSincePublisherSRAdjusted := time.Duration(mono.UnixNano() - publisherSRData.AtAdjusted)
+		timeSincePublisherSRAdjusted := time.Duration(nowNano - publisherSRData.AtAdjusted)
 		reportTimeAdjusted = publisherSRData.AtAdjusted + timeSincePublisherSRAdjusted.Nanoseconds()
 		reportTime = reportTimeAdjusted
 
 		nowNTP = mediatransportutil.ToNtpTime(time.Unix(0, reportTime))
-		nowRTPExt = publisherSRData.RtpTimestampExt - tsOffset + uint64(timeSincePublisherSRAdjusted.Nanoseconds()*int64(r.params.ClockRate)/1e9)
+		nowRTPExt = publisherSRData.RtpTimestampExt - tsOffset + r.rtpConverter.ToRTPExt(timeSincePublisherSRAdjusted)
 	}
 
 	packetCount := uint32(r.getPacketsSeenPlusDuplicates(r.extStartSN, r.extHighestSN))
@@ -861,33 +958,28 @@ func (r *RTPStatsSender) GetRtcpSenderReport(ssrc uint32, publisherSRData *livek
 		Octets:          octetCount,
 	}
 
-	ulgr := func() logger.UnlikelyLogger {
-		nowNano := mono.UnixNano()
-		return r.logger.WithUnlikelyValues(
-			"curr", WrappedRTCPSenderReportStateLogger{srData},
-			"feed", WrappedRTCPSenderReportStateLogger{publisherSRData},
-			"tsOffset", tsOffset,
-			"timeNow", mono.Now(),
-			"reportTime", time.Unix(0, reportTime),
-			"reportTimeAdjusted", time.Unix(0, reportTimeAdjusted),
-			"timeSinceHighest", time.Duration(nowNano-r.highestTime),
-			"timeSinceFirst", time.Duration(nowNano-r.firstTime),
-			"timeSincePublisherSRAdjusted", time.Duration(nowNano-publisherSRData.AtAdjusted),
-			"timeSincePublisherSR", time.Duration(nowNano-publisherSRData.At),
-			"nowRTPExt", nowRTPExt,
-			"rtpStats", lockedRTPStatsSenderLogEncoder{r},
-		)
-	}
-
-	if r.srNewest != nil && nowRTPExt >= r.srNewest.RtpTimestampExt {
-		timeSinceLastReport := nowNTP.Time().Sub(mediatransportutil.NtpTime(r.srNewest.NtpTimestamp).Time())
+	if r.srNewest != nil && nowRTPExt >= r.srNewest.RtpTimestampExt && r.clockRate != 0 {
+		nowTime := nowNTP.Time()
+		srNewestTime := mediatransportutil.NtpTime(r.srNewest.NtpTimestamp).Time()
+		timeSinceLastReport := nowTime.Sub(srNewestTime)
 		rtpDiffSinceLastReport := nowRTPExt - r.srNewest.RtpTimestampExt
 		windowClockRate := float64(rtpDiffSinceLastReport) / timeSinceLastReport.Seconds()
-		if timeSinceLastReport.Seconds() > 0.2 && math.Abs(float64(r.params.ClockRate)-windowClockRate) > 0.2*float64(r.params.ClockRate) {
+		if timeSinceLastReport.Seconds() > 0.2 && math.Abs(float64(r.clockRate)-windowClockRate) > 0.2*float64(r.clockRate) {
 			r.clockSkewCount++
-			if (r.clockSkewCount-1)%100 == 0 {
-				ulgr().Infow(
+			if shouldLog(r.clockSkewCount) {
+				srlf := &senderReportLoggingFields{
+					srData:             srData,
+					publisherSRData:    publisherSRData,
+					tsOffset:           tsOffset,
+					reportTime:         reportTime,
+					reportTimeAdjusted: reportTimeAdjusted,
+					nowNano:            nowNano,
+					nowRTPExt:          nowRTPExt,
+					rtpStats:           r,
+				}
+				r.logger.Infow(
 					"sending sender report, clock skew",
+					zap.Inline(srlf),
 					"timeSinceLastReport", timeSinceLastReport,
 					"rtpDiffSinceLastReport", rtpDiffSinceLastReport,
 					"windowClockRate", windowClockRate,
@@ -900,7 +992,20 @@ func (r *RTPStatsSender) GetRtcpSenderReport(ssrc uint32, publisherSRData *livek
 	if r.srNewest != nil && nowRTPExt < r.srNewest.RtpTimestampExt {
 		// If report being generated is behind the last report, skip it.
 		// Should not happen.
-		ulgr().Infow("sending sender report, out-of-order, skipping")
+		srlf := &senderReportLoggingFields{
+			srData:             srData,
+			publisherSRData:    publisherSRData,
+			tsOffset:           tsOffset,
+			reportTime:         reportTime,
+			reportTimeAdjusted: reportTimeAdjusted,
+			nowNano:            nowNano,
+			nowRTPExt:          nowRTPExt,
+			rtpStats:           r,
+		}
+		r.logger.Infow(
+			"sending sender report, out-of-order, skipping",
+			zap.Inline(srlf),
+		)
 		return nil
 	}
 
@@ -911,7 +1016,7 @@ func (r *RTPStatsSender) GetRtcpSenderReport(ssrc uint32, publisherSRData *livek
 
 	r.rttMarker = rttMarker{
 		ntpTime: nowNTP,
-		sentAt:  mono.Now(),
+		sentAt:  time.Now(),
 	}
 
 	return &rtcp.SenderReport{
@@ -927,7 +1032,7 @@ func (r *RTPStatsSender) DeltaInfo(snapshotID uint32) *RTPDeltaInfo {
 	r.lock.Lock()
 	defer r.lock.Unlock()
 
-	deltaInfo, err, loggingFields := r.deltaInfo(
+	deltaInfo, loggingFields, err := r.deltaInfo(
 		snapshotID,
 		r.extStartSN,
 		r.extHighestSN,
@@ -943,9 +1048,13 @@ func (r *RTPStatsSender) DeltaInfoSender(senderSnapshotID uint32) (*RTPDeltaInfo
 	r.lock.Lock()
 	defer r.lock.Unlock()
 
+	if r.clockRate == 0 {
+		return nil, nil
+	}
+
 	var deltaStatsSenderView *RTPDeltaInfo
-	thenSenderView, nowSenderView := r.getAndResetSenderSnapshotWindow(senderSnapshotID)
-	if thenSenderView != nil && nowSenderView != nil {
+	thenSenderView, nowSenderView, ok := r.getAndResetSenderSnapshotWindow(senderSnapshotID)
+	if ok {
 		startTime := thenSenderView.startTime
 		endTime := nowSenderView.startTime
 
@@ -979,7 +1088,7 @@ func (r *RTPStatsSender) DeltaInfoSender(senderSnapshotID uint32) (*RTPDeltaInfo
 				packetsLostFeed = packetsExpected
 			}
 
-			maxJitterTime := thenSenderView.maxJitterFeed / float64(r.params.ClockRate) * 1e6
+			maxJitterTime := thenSenderView.maxJitterFeed / float64(r.clockRate) * 1e6
 
 			deltaStatsSenderView = &RTPDeltaInfo{
 				StartTime:            time.Unix(0, startTime),
@@ -1007,8 +1116,8 @@ func (r *RTPStatsSender) DeltaInfoSender(senderSnapshotID uint32) (*RTPDeltaInfo
 
 	var deltaStatsReceiverView *RTPDeltaInfo
 	if r.lastRRTime != 0 {
-		thenReceiverView, nowReceiverView := r.getAndResetSenderSnapshotReceiverView(senderSnapshotID)
-		if thenReceiverView != nil && nowReceiverView != nil {
+		thenReceiverView, nowReceiverView, ok := r.getAndResetSenderSnapshotReceiverView(senderSnapshotID)
+		if ok {
 			startTime := thenReceiverView.startTime
 			endTime := nowReceiverView.startTime
 
@@ -1048,12 +1157,7 @@ func (r *RTPStatsSender) DeltaInfoSender(senderSnapshotID uint32) (*RTPDeltaInfo
 					packetsLost = packetsExpected
 				}
 
-				// discount jitter from publisher side + internal processing
-				maxJitter := thenReceiverView.maxJitter - thenReceiverView.maxJitterFeed
-				if maxJitter < 0.0 {
-					maxJitter = 0.0
-				}
-				maxJitterTime := maxJitter / float64(r.params.ClockRate) * 1e6
+				maxJitterTime := thenReceiverView.maxJitter / float64(r.clockRate) * 1e6
 
 				deltaStatsReceiverView = &RTPDeltaInfo{
 					StartTime:            time.Unix(0, startTime),
@@ -1117,9 +1221,9 @@ func (r *RTPStatsSender) ToProto() *livekit.RTPStats {
 	return p
 }
 
-func (r *RTPStatsSender) getAndResetSenderSnapshotWindow(senderSnapshotID uint32) (*senderSnapshotWindow, *senderSnapshotWindow) {
-	if !r.initialized {
-		return nil, nil
+func (r *RTPStatsSender) getAndResetSenderSnapshotWindow(senderSnapshotID uint32) (senderSnapshotWindow, senderSnapshotWindow, bool) {
+	if !r.initialized || senderSnapshotID < cFirstSnapshotID {
+		return senderSnapshotWindow{}, senderSnapshotWindow{}, false
 	}
 
 	idx := senderSnapshotID - cFirstSnapshotID
@@ -1131,7 +1235,7 @@ func (r *RTPStatsSender) getAndResetSenderSnapshotWindow(senderSnapshotID uint32
 
 	// snapshot now
 	r.senderSnapshots[idx].senderView = r.getSenderSnapshotWindow(mono.UnixNano())
-	return &then.senderView, &r.senderSnapshots[idx].senderView
+	return then.senderView, r.senderSnapshots[idx].senderView, true
 }
 
 func (r *RTPStatsSender) getSenderSnapshotWindow(startTime int64) senderSnapshotWindow {
@@ -1158,9 +1262,9 @@ func (r *RTPStatsSender) getSenderSnapshotWindow(startTime int64) senderSnapshot
 	}
 }
 
-func (r *RTPStatsSender) getAndResetSenderSnapshotReceiverView(senderSnapshotID uint32) (*senderSnapshotReceiverView, *senderSnapshotReceiverView) {
-	if !r.initialized || r.lastRRTime == 0 {
-		return nil, nil
+func (r *RTPStatsSender) getAndResetSenderSnapshotReceiverView(senderSnapshotID uint32) (senderSnapshotReceiverView, senderSnapshotReceiverView, bool) {
+	if !r.initialized || r.lastRRTime == 0 || senderSnapshotID < cFirstSnapshotID {
+		return senderSnapshotReceiverView{}, senderSnapshotReceiverView{}, false
 	}
 
 	idx := senderSnapshotID - cFirstSnapshotID
@@ -1172,7 +1276,7 @@ func (r *RTPStatsSender) getAndResetSenderSnapshotReceiverView(senderSnapshotID
 
 	// snapshot now
 	r.senderSnapshots[idx].receiverView = r.getSenderSnapshotReceiverView(r.lastRRTime, &then.receiverView)
-	return &then.receiverView, &r.senderSnapshots[idx].receiverView
+	return then.receiverView, r.senderSnapshots[idx].receiverView, true
 }
 
 func (r *RTPStatsSender) getSenderSnapshotReceiverView(startTime int64, s *senderSnapshotReceiverView) senderSnapshotReceiverView {
@@ -1251,11 +1355,13 @@ func (r *RTPStatsSender) clearSnInfos(extStartInclusive uint64, extEndExclusive
 		return
 	}
 
+	if extEndExclusive-extStartInclusive > uint64(len(r.snInfos)) {
+		clear(r.snInfos)
+		return
+	}
+
 	for esn := extStartInclusive; esn != extEndExclusive; esn++ {
-		snInfo := &r.snInfos[int(esn)%len(r.snInfos)]
-		snInfo.pktSize = 0
-		snInfo.hdrSize = 0
-		snInfo.flags = 0
+		r.snInfos[int(esn)%len(r.snInfos)] = snInfo{}
 	}
 }
 
@@ -1273,14 +1379,18 @@ func (r *RTPStatsSender) getIntervalStats(
 	extEndExclusive uint64,
 	ehsn uint64,
 ) (intervalStats intervalStats) {
-	processESN := func(esn uint64, ehsn uint64) {
-		slot := r.getSnInfoOutOfOrderSlot(esn, ehsn)
-		if slot < 0 {
-			intervalStats.packetsNotFoundMetadata++
-			return
-		}
+	upperBound := ehsn + 1
+	lowerBound := uint64(0)
+	if n := uint64(len(r.snInfos)); n != 0 && ehsn >= n-1 {
+		lowerBound = ehsn - n + 1
+	}
+	extStartInclusiveClamped := max(min(extStartInclusive, upperBound), lowerBound)
+	extEndExclusiveClamped := max(min(extEndExclusive, upperBound), extStartInclusiveClamped)
 
-		snInfo := &r.snInfos[slot]
+	intervalStats.packetsNotFoundMetadata = (extEndExclusive - extStartInclusive) - (extEndExclusiveClamped - extStartInclusiveClamped)
+
+	for esn := extStartInclusiveClamped; esn != extEndExclusiveClamped; esn++ {
+		snInfo := &r.snInfos[int(esn)%len(r.snInfos)]
 		switch {
 		case snInfo.pktSize == 0:
 			intervalStats.packetsLostFeed++
@@ -1303,10 +1413,6 @@ func (r *RTPStatsSender) getIntervalStats(
 			intervalStats.frames++
 		}
 	}
-
-	for esn := extStartInclusive; esn != extEndExclusive; esn++ {
-		processESN(esn, ehsn)
-	}
 	return
 }
 

livekit-server/pkg/sfu/rtpstats/rtpstats_test.go

@@ -21,8 +21,6 @@ import (
 
 	"github.com/pion/rtp"
 	"github.com/stretchr/testify/require"
-
-	"github.com/livekit/protocol/logger"
 )
 
 func getPacket(sn uint16, ts uint32, payloadSize int) *rtp.Packet {
@@ -37,10 +35,8 @@ func getPacket(sn uint16, ts uint32, payloadSize int) *rtp.Packet {
 
 func Test_RTPStatsReceiver_Update(t *testing.T) {
 	clockRate := uint32(90000)
-	r := NewRTPStatsReceiver(RTPStatsParams{
-		ClockRate: clockRate,
-		Logger:    logger.GetLogger(),
-	})
+	r := NewRTPStatsReceiver(RTPStatsParams{})
+	r.SetClockRate(clockRate)
 
 	sequenceNumber := uint16(rand.Float64() * float64(1<<16))
 	timestamp := uint32(rand.Float64() * float64(1<<32))
@@ -89,7 +85,7 @@ func Test_RTPStatsReceiver_Update(t *testing.T) {
 		len(packet.Payload),
 		0,
 	)
-	require.True(t, flowState.IsNotHandled)
+	require.Equal(t, RTPFlowUnhandledReasonPreStartTimestamp, flowState.UnhandledReason)
 	require.Equal(t, sequenceNumber, r.sequenceNumber.GetHighest())
 	require.Equal(t, sequenceNumber, uint16(r.sequenceNumber.GetExtendedHighest()))
 	require.Equal(t, timestamp, r.timestamp.GetHighest())
@@ -108,7 +104,7 @@ func Test_RTPStatsReceiver_Update(t *testing.T) {
 		len(packet.Payload),
 		0,
 	)
-	require.True(t, flowState.IsNotHandled)
+	require.Equal(t, RTPFlowUnhandledReasonPreStartTimestamp, flowState.UnhandledReason)
 	require.Equal(t, sequenceNumber, r.sequenceNumber.GetHighest())
 	require.Equal(t, sequenceNumber, uint16(r.sequenceNumber.GetExtendedHighest()))
 	require.Equal(t, timestamp, r.timestamp.GetHighest())
@@ -218,7 +214,7 @@ func Test_RTPStatsReceiver_Update(t *testing.T) {
 		len(packet.Payload),
 		0,
 	)
-	require.True(t, flowState.IsNotHandled)
+	require.Equal(t, RTPFlowUnhandledReasonOldSequenceNumber, flowState.UnhandledReason)
 	require.Equal(t, sequenceNumber, r.sequenceNumber.GetHighest())
 	require.Equal(t, sequenceNumber, uint16(r.sequenceNumber.GetExtendedHighest()))
 	require.Equal(t, timestamp, r.timestamp.GetHighest())
@@ -226,3 +222,65 @@ func Test_RTPStatsReceiver_Update(t *testing.T) {
 
 	r.Stop()
 }
+
+func Test_RTPStatsReceiver_Restart(t *testing.T) {
+	clockRate := uint32(90000)
+	r := NewRTPStatsReceiver(RTPStatsParams{})
+	r.SetClockRate(clockRate)
+
+	// should not restart till there are at least threshold packets
+	require.False(t, r.maybeRestart(10, 20, 1000))
+	require.False(t, r.maybeRestart(11, 20, 1000))
+	require.False(t, r.maybeRestart(13, 20, 1000))
+	require.False(t, r.maybeRestart(14, 20, 1000))
+	// although adding 5th packet should have enough packets for a check,
+	// still should not restart as there is a sequence number gap between 11 and 13
+	require.False(t, r.maybeRestart(15, 20, 1000))
+	require.False(t, r.maybeRestart(16, 19, 1000))
+	// has enough packets, but still cannot restart because timestamps are not increasing
+	require.False(t, r.maybeRestart(17, 21, 1000))
+	require.False(t, r.maybeRestart(18, 21, 1000))
+	require.False(t, r.maybeRestart(19, 21, 1000))
+	// can restart as there are enough packets with proper sequencing
+	require.True(t, r.maybeRestart(20, 21, 1000))
+	require.Equal(t, restartThreshold, r.restartPacketsN)
+
+	r.resetRestart()
+	require.Zero(t, r.restartPacketsN)
+
+	r.Stop()
+}
+
+func Test_RTPStatsSender_getIntervalStats(t *testing.T) {
+	t.Run("packetsNotFoundMetadata should match lost packets", func(t *testing.T) {
+		r := NewRTPStatsSender(RTPStatsParams{}, 1024)
+		stats := r.getIntervalStats(0, 10000, 10000)
+		require.EqualValues(t, 8977, stats.packetsNotFoundMetadata)
+	})
+}
+
+func BenchmarkRTPStatsReceiver_Update(b *testing.B) {
+	const clockRate = 90000
+	const hdrSize = 12
+	const payloadSize = 1200
+	const paddingSize = 0
+	const tsIncrement = 3000 // ~33ms at 90kHz
+
+	r := NewRTPStatsReceiver(RTPStatsParams{})
+	r.SetClockRate(clockRate)
+
+	// initialize before running benchmark loop
+	packetTime := time.Now().UnixNano()
+	sn := uint16(0)
+	ts := uint32(0)
+	r.Update(packetTime, sn, ts, false, hdrSize, payloadSize, paddingSize)
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		sn++
+		ts += tsIncrement
+		packetTime += 33 * int64(time.Millisecond) // ~30fps
+		marker := (i+1)%100 == 0                   // marker every 100 packets
+		r.Update(packetTime, sn, ts, marker, hdrSize, payloadSize, paddingSize)
+	}
+}

livekit-server/pkg/sfu/sequencer.go

@@ -119,6 +119,10 @@ type sequencer struct {
 }
 
 func newSequencer(size int, maybeSparse bool, logger logger.Logger) *sequencer {
+	if size == 0 {
+		return nil
+	}
+
 	s := &sequencer{
 		size:      size,
 		startTime: time.Now().UnixNano(),

livekit-server/pkg/sfu/sfu.go

@@ -16,18 +16,21 @@ package sfu
 
 import (
 	"sync"
+
+	"github.com/pion/rtp"
 )
 
 var (
-	PacketFactory *sync.Pool
-)
-
-func init() {
-	// Init packet factory
 	PacketFactory = &sync.Pool{
-		New: func() interface{} {
+		New: func() any {
 			b := make([]byte, 1460)
 			return &b
 		},
 	}
-}
+
+	RTPHeaderFactory = &sync.Pool{
+		New: func() any {
+			return &rtp.Header{}
+		},
+	}
+)

livekit-server/pkg/sfu/streamallocator/streamallocator.go

@@ -16,6 +16,7 @@ package streamallocator
 
 import (
 	"fmt"
+	"slices"
 	"sort"
 	"sync"
 	"time"
@@ -37,21 +38,24 @@ import (
 )
 
 const (
-	ChannelCapacityInfinity = 100 * 1000 * 1000 // 100 Mbps
+	cChannelCapacityInfinity = 100 * 1000 * 1000 // 100 Mbps
 
-	PriorityMin                = uint8(1)
-	PriorityMax                = uint8(255)
-	PriorityDefaultScreenshare = PriorityMax
-	PriorityDefaultVideo       = PriorityMin
+	cPriorityMin                = uint8(1)
+	cPriorityMax                = uint8(255)
+	cPriorityDefaultScreenshare = cPriorityMax
+	cPriorityDefaultVideo       = cPriorityMin
 
-	FlagAllowOvershootWhileOptimal              = true
-	FlagAllowOvershootWhileDeficient            = false
-	FlagAllowOvershootExemptTrackWhileDeficient = true
-	FlagAllowOvershootInProbe                   = true
-	FlagAllowOvershootInCatchup                 = false
-	FlagAllowOvershootInBoost                   = true
+	cFlagAllowOvershootWhileOptimal              = true
+	cFlagAllowOvershootWhileDeficient            = false
+	cFlagAllowOvershootExemptTrackWhileDeficient = true
+	cFlagAllowOvershootInProbe                   = true
+	cFlagAllowOvershootInCatchup                 = false
+	cFlagAllowOvershootInBoost                   = true
 
 	cRTTPullInterval = 30 * time.Second
+
+	cPingLong  = cRTTPullInterval / 2
+	cPingShort = 100 * time.Millisecond
 )
 
 // ---------------------------------------------------------------------------
@@ -131,13 +135,45 @@ func (s streamAllocatorSignal) String() string {
 
 type Event struct {
 	*StreamAllocator
-	Signal  streamAllocatorSignal
-	TrackID livekit.TrackID
-	Data    interface{}
+	signal streamAllocatorSignal
+
+	// only one of these fields will be set based on event type
+	trackID                   livekit.TrackID
+	allowPause                bool
+	channelCapacity           int64
+	twccFeedback              *rtcp.TransportLayerCC
+	congestionStateChangeData congestionStateChangeData
+	probeBytesToSend          int
+	probeClusterId            ccutils.ProbeClusterId
+	probeClusterInfo          ccutils.ProbeClusterInfo
 }
 
 func (e Event) String() string {
-	return fmt.Sprintf("StreamAllocator:Event{signal: %s, trackID: %s, data: %+v}", e.Signal, e.TrackID, e.Data)
+	value := ""
+	switch e.signal {
+	case streamAllocatorSignalAllocateTrack, streamAllocatorSignalResume:
+		value = fmt.Sprintf("trackID: %s", e.trackID)
+	case streamAllocatorSignalEstimate, streamAllocatorSignalSetChannelCapacity:
+		value = fmt.Sprintf("channelCapacity: %d", e.channelCapacity)
+	case streamAllocatorSignalFeedback:
+		value = fmt.Sprintf("twccFeedback: %+v", e.twccFeedback)
+	case streamAllocatorSignalProbeClusterSwitch:
+		value = fmt.Sprintf("probeClusterInfo: %+v", e.probeClusterInfo)
+	case streamAllocatorSignalSendProbe:
+		value = fmt.Sprintf("probeBytesToSend: %d", e.probeBytesToSend)
+	case streamAllocatorSignalPacerProbeObserverClusterComplete:
+		value = fmt.Sprintf("probeClusterId: %v", e.probeClusterId)
+	case streamAllocatorSignalSetAllowPause:
+		value = fmt.Sprintf("allowPause: %v", e.allowPause)
+	case streamAllocatorSignalCongestionStateChange:
+		value = fmt.Sprintf("congestStatecChangeData: %+v", e.congestionStateChangeData)
+	}
+
+	if value == "" {
+		return fmt.Sprintf("StreamAllocator:Event{signal: %s}", e.signal)
+	}
+
+	return fmt.Sprintf("StreamAllocator:Event{signal: %s, %s}", e.signal, value)
 }
 
 // ---------------------------------------------------------------------------
@@ -202,6 +238,7 @@ type StreamAllocator struct {
 
 	videoTracksMu        sync.RWMutex
 	videoTracks          map[livekit.TrackID]*Track
+	videoTracksShadow    []*Track
 	isAllocateAllPending bool
 	rembTrackingSSRC     uint32
 
@@ -215,6 +252,8 @@ type StreamAllocator struct {
 
 	lastRTTTime time.Time
 
+	pingGeneration atomic.Uint32
+
 	isStopped atomic.Bool
 }
 
@@ -247,7 +286,7 @@ func NewStreamAllocator(params StreamAllocatorParams, enabled bool, allowPause b
 
 func (s *StreamAllocator) Start() {
 	s.eventsQueue.Start()
-	go s.ping()
+	go s.ping(s.pingGeneration.Inc(), cPingLong)
 }
 
 func (s *StreamAllocator) Stop() {
@@ -273,10 +312,10 @@ func (s *StreamAllocator) SetSendSideBWEInterceptor(sendSideBWEInterceptor cc.Ba
 }
 
 type AddTrackParams struct {
-	Source      livekit.TrackSource
-	Priority    uint8
-	IsSimulcast bool
-	PublisherID livekit.ParticipantID
+	Source         livekit.TrackSource
+	Priority       uint8
+	IsMultiLayered bool
+	PublisherID    livekit.ParticipantID
 }
 
 func (s *StreamAllocator) AddTrack(downTrack *sfu.DownTrack, params AddTrackParams) {
@@ -284,13 +323,14 @@ func (s *StreamAllocator) AddTrack(downTrack *sfu.DownTrack, params AddTrackPara
 		return
 	}
 
-	track := NewTrack(downTrack, params.Source, params.IsSimulcast, params.PublisherID, s.params.Logger)
+	track := NewTrack(downTrack, params.Source, params.IsMultiLayered, params.PublisherID, s.params.Logger)
 	track.SetPriority(params.Priority)
 
 	trackID := livekit.TrackID(downTrack.ID())
 	s.videoTracksMu.Lock()
 	oldTrack := s.videoTracks[trackID]
 	s.videoTracks[trackID] = track
+	s.shadowVideoTracksLocked()
 	s.videoTracksMu.Unlock()
 
 	if oldTrack != nil {
@@ -307,12 +347,13 @@ func (s *StreamAllocator) RemoveTrack(downTrack *sfu.DownTrack) {
 	s.videoTracksMu.Lock()
 	if existing := s.videoTracks[livekit.TrackID(downTrack.ID())]; existing != nil && existing.DownTrack() == downTrack {
 		delete(s.videoTracks, livekit.TrackID(downTrack.ID()))
+		s.shadowVideoTracksLocked()
 	}
 	s.videoTracksMu.Unlock()
 
 	// STREAM-ALLOCATOR-TODO: use any saved bandwidth to re-distribute
 	s.postEvent(Event{
-		Signal: streamAllocatorSignalAdjustState,
+		signal: streamAllocatorSignalAdjustState,
 	})
 }
 
@@ -324,7 +365,7 @@ func (s *StreamAllocator) SetTrackPriority(downTrack *sfu.DownTrack, priority ui
 			// do a full allocation on a track priority change to keep it simple
 			s.isAllocateAllPending = true
 			s.postEvent(Event{
-				Signal: streamAllocatorSignalAllocateAllTracks,
+				signal: streamAllocatorSignalAllocateAllTracks,
 			})
 		}
 	}
@@ -333,15 +374,15 @@ func (s *StreamAllocator) SetTrackPriority(downTrack *sfu.DownTrack, priority ui
 
 func (s *StreamAllocator) SetAllowPause(allowPause bool) {
 	s.postEvent(Event{
-		Signal: streamAllocatorSignalSetAllowPause,
-		Data:   allowPause,
+		signal:     streamAllocatorSignalSetAllowPause,
+		allowPause: allowPause,
 	})
 }
 
 func (s *StreamAllocator) SetChannelCapacity(channelCapacity int64) {
 	s.postEvent(Event{
-		Signal: streamAllocatorSignalSetChannelCapacity,
-		Data:   channelCapacity,
+		signal:          streamAllocatorSignalSetChannelCapacity,
+		channelCapacity: channelCapacity,
 	})
 }
 
@@ -385,13 +426,7 @@ func (s *StreamAllocator) OnREMB(downTrack *sfu.DownTrack, remb *rtcp.ReceiverEs
 		downTrackSSRCRTX = track.DownTrack().SSRCRTX()
 	}
 
-	found := false
-	for _, ssrc := range remb.SSRCs {
-		if ssrc == s.rembTrackingSSRC {
-			found = true
-			break
-		}
-	}
+	found := slices.Contains(remb.SSRCs, s.rembTrackingSSRC)
 	if !found {
 		if len(remb.SSRCs) == 0 {
 			s.params.Logger.Warnw("stream allocator: no SSRC to track REMB", nil)
@@ -429,24 +464,24 @@ func (s *StreamAllocator) OnREMB(downTrack *sfu.DownTrack, remb *rtcp.ReceiverEs
 	s.videoTracksMu.Unlock()
 
 	s.postEvent(Event{
-		Signal: streamAllocatorSignalEstimate,
-		Data:   int64(remb.Bitrate),
+		signal:          streamAllocatorSignalEstimate,
+		channelCapacity: int64(remb.Bitrate),
 	})
 }
 
 // called when a new transport-cc feedback is received
 func (s *StreamAllocator) OnTransportCCFeedback(downTrack *sfu.DownTrack, fb *rtcp.TransportLayerCC) {
 	s.postEvent(Event{
-		Signal: streamAllocatorSignalFeedback,
-		Data:   fb,
+		signal:       streamAllocatorSignalFeedback,
+		twccFeedback: fb,
 	})
 }
 
 // called when target bitrate changes (send side bandwidth estimation)
 func (s *StreamAllocator) onTargetBitrateChange(bitrate int) {
 	s.postEvent(Event{
-		Signal: streamAllocatorSignalEstimate,
-		Data:   int64(bitrate),
+		signal:          streamAllocatorSignalEstimate,
+		channelCapacity: int64(bitrate),
 	})
 }
 
@@ -460,8 +495,8 @@ type congestionStateChangeData struct {
 // BWEListener implementation
 func (s *StreamAllocator) OnCongestionStateChange(fromState bwe.CongestionState, toState bwe.CongestionState, estimatedAvailableChannelCapacity int64) {
 	s.postEvent(Event{
-		Signal: streamAllocatorSignalCongestionStateChange,
-		Data:   congestionStateChangeData{fromState, toState, estimatedAvailableChannelCapacity},
+		signal:                    streamAllocatorSignalCongestionStateChange,
+		congestionStateChangeData: congestionStateChangeData{fromState, toState, estimatedAvailableChannelCapacity},
 	})
 }
 
@@ -503,8 +538,8 @@ func (s *StreamAllocator) OnSubscribedLayerChanged(downTrack *sfu.DownTrack, lay
 
 	if shouldPost {
 		s.postEvent(Event{
-			Signal:  streamAllocatorSignalAllocateTrack,
-			TrackID: livekit.TrackID(downTrack.ID()),
+			signal:  streamAllocatorSignalAllocateTrack,
+			trackID: livekit.TrackID(downTrack.ID()),
 		})
 	}
 }
@@ -512,32 +547,32 @@ func (s *StreamAllocator) OnSubscribedLayerChanged(downTrack *sfu.DownTrack, lay
 // called when forwarder resumes a track
 func (s *StreamAllocator) OnResume(downTrack *sfu.DownTrack) {
 	s.postEvent(Event{
-		Signal:  streamAllocatorSignalResume,
-		TrackID: livekit.TrackID(downTrack.ID()),
+		signal:  streamAllocatorSignalResume,
+		trackID: livekit.TrackID(downTrack.ID()),
 	})
 }
 
 // called when probe cluster changes
 func (s *StreamAllocator) OnProbeClusterSwitch(pci ccutils.ProbeClusterInfo) {
 	s.postEvent(Event{
-		Signal: streamAllocatorSignalProbeClusterSwitch,
-		Data:   pci,
+		signal:           streamAllocatorSignalProbeClusterSwitch,
+		probeClusterInfo: pci,
 	})
 }
 
 // called when prober wants to send packet(s)
 func (s *StreamAllocator) OnSendProbe(bytesToSend int) {
 	s.postEvent(Event{
-		Signal: streamAllocatorSignalSendProbe,
-		Data:   bytesToSend,
+		signal:           streamAllocatorSignalSendProbe,
+		probeBytesToSend: bytesToSend,
 	})
 }
 
 // called when pacer probe observer observes a cluster completion
 func (s *StreamAllocator) OnPacerProbeObserverClusterComplete(probeClusterId ccutils.ProbeClusterId) {
 	s.postEvent(Event{
-		Signal: streamAllocatorSignalPacerProbeObserverClusterComplete,
-		Data:   probeClusterId,
+		signal:         streamAllocatorSignalPacerProbeObserverClusterComplete,
+		probeClusterId: probeClusterId,
 	})
 }
 
@@ -557,6 +592,10 @@ func (s *StreamAllocator) IsBWEEnabled(downTrack *sfu.DownTrack) bool {
 	return true
 }
 
+func (s *StreamAllocator) BWEType() bwe.BWEType {
+	return s.params.BWE.Type()
+}
+
 // called to check if track subscription mute can be applied
 func (s *StreamAllocator) IsSubscribeMutable(downTrack *sfu.DownTrack) bool {
 	s.videoTracksMu.Lock()
@@ -579,24 +618,24 @@ func (s *StreamAllocator) maybePostEventAllocateTrack(downTrack *sfu.DownTrack)
 
 	if shouldPost {
 		s.postEvent(Event{
-			Signal:  streamAllocatorSignalAllocateTrack,
-			TrackID: livekit.TrackID(downTrack.ID()),
+			signal:  streamAllocatorSignalAllocateTrack,
+			trackID: livekit.TrackID(downTrack.ID()),
 		})
 	}
 }
 
-func (s *StreamAllocator) ping() {
-	ticker := time.NewTicker(100 * time.Millisecond)
+func (s *StreamAllocator) ping(pingGeneration uint32, interval time.Duration) {
+	ticker := time.NewTicker(interval)
 	defer ticker.Stop()
 
 	for {
 		<-ticker.C
-		if s.isStopped.Load() {
+		if s.isStopped.Load() || (pingGeneration != s.pingGeneration.Load()) {
 			return
 		}
 
 		s.postEvent(Event{
-			Signal: streamAllocatorSignalPeriodicPing,
+			signal: streamAllocatorSignalPeriodicPing,
 		})
 	}
 }
@@ -604,7 +643,7 @@ func (s *StreamAllocator) ping() {
 func (s *StreamAllocator) postEvent(event Event) {
 	event.StreamAllocator = s
 	s.eventsQueue.Enqueue(func(event Event) {
-		switch event.Signal {
+		switch event.signal {
 		case streamAllocatorSignalAllocateTrack:
 			event.handleSignalAllocateTrack(event)
 		case streamAllocatorSignalAllocateAllTracks:
@@ -637,7 +676,7 @@ func (s *StreamAllocator) postEvent(event Event) {
 
 func (s *StreamAllocator) handleSignalAllocateTrack(event Event) {
 	s.videoTracksMu.Lock()
-	track := s.videoTracks[event.TrackID]
+	track := s.videoTracks[event.trackID]
 	if track != nil {
 		track.SetDirty(false)
 	}
@@ -663,13 +702,11 @@ func (s *StreamAllocator) handleSignalAdjustState(Event) {
 }
 
 func (s *StreamAllocator) handleSignalEstimate(event Event) {
-	receivedEstimate := event.Data.(int64)
-
 	// always update NACKs
 	packetDelta, repeatedNackDelta := s.getNackDelta()
 
 	s.params.BWE.HandleREMB(
-		receivedEstimate,
+		event.channelCapacity,
 		s.getExpectedBandwidthUsage(),
 		packetDelta,
 		repeatedNackDelta,
@@ -677,7 +714,7 @@ func (s *StreamAllocator) handleSignalEstimate(event Event) {
 }
 
 func (s *StreamAllocator) handleSignalFeedback(event Event) {
-	fb := event.Data.(*rtcp.TransportLayerCC)
+	fb := event.twccFeedback
 	if s.sendSideBWEInterceptor != nil {
 		s.sendSideBWEInterceptor.WriteRTCP([]rtcp.Packet{fb}, nil)
 	}
@@ -745,7 +782,7 @@ func (s *StreamAllocator) handleSignalPeriodicPing(Event) {
 }
 
 func (s *StreamAllocator) handleSignalProbeClusterSwitch(event Event) {
-	pci := event.Data.(ccutils.ProbeClusterInfo)
+	pci := event.probeClusterInfo
 	s.activeProbeClusterId = pci.Id
 	s.activeProbeGoalReached = false
 	s.activeProbeCongesting = false
@@ -754,19 +791,19 @@ func (s *StreamAllocator) handleSignalProbeClusterSwitch(event Event) {
 
 	s.params.Pacer.StartProbeCluster(pci)
 
-	for _, t := range s.getTracks() {
+	for _, t := range s.getVideoTracks() {
 		t.DownTrack().SetProbeClusterId(pci.Id)
 	}
 }
 
 func (s *StreamAllocator) handleSignalSendProbe(event Event) {
-	bytesToSend := event.Data.(int)
+	bytesToSend := event.probeBytesToSend
 	if bytesToSend <= 0 {
 		return
 	}
 
 	bytesSent := 0
-	for _, track := range s.getTracks() {
+	for _, track := range s.getVideoTracks() {
 		sent := track.WriteProbePackets(bytesToSend)
 		bytesSent += sent
 		bytesToSend -= sent
@@ -779,10 +816,9 @@ func (s *StreamAllocator) handleSignalSendProbe(event Event) {
 }
 
 func (s *StreamAllocator) handleSignalPacerProbeObserverClusterComplete(event Event) {
-	probeClusterId, _ := event.Data.(ccutils.ProbeClusterId)
-	pci := s.params.Pacer.EndProbeCluster(probeClusterId)
+	pci := s.params.Pacer.EndProbeCluster(event.probeClusterId)
 
-	for _, t := range s.getTracks() {
+	for _, t := range s.getVideoTracks() {
 		t.DownTrack().SwapProbeClusterId(pci.Id, ccutils.ProbeClusterIdInvalid)
 	}
 
@@ -792,7 +828,7 @@ func (s *StreamAllocator) handleSignalPacerProbeObserverClusterComplete(event Ev
 
 func (s *StreamAllocator) handleSignalResume(event Event) {
 	s.videoTracksMu.Lock()
-	track := s.videoTracks[event.TrackID]
+	track := s.videoTracks[event.trackID]
 	updated := track != nil && track.SetStreamState(StreamStateActive)
 	s.videoTracksMu.Unlock()
 
@@ -804,11 +840,11 @@ func (s *StreamAllocator) handleSignalResume(event Event) {
 }
 
 func (s *StreamAllocator) handleSignalSetAllowPause(event Event) {
-	s.allowPause = event.Data.(bool)
+	s.allowPause = event.allowPause
 }
 
 func (s *StreamAllocator) handleSignalSetChannelCapacity(event Event) {
-	s.overriddenChannelCapacity = event.Data.(int64)
+	s.overriddenChannelCapacity = event.channelCapacity
 	if s.overriddenChannelCapacity > 0 {
 		s.params.Logger.Infow("allocating on override channel capacity", "override", s.overriddenChannelCapacity)
 		s.allocateAllTracks()
@@ -818,7 +854,7 @@ func (s *StreamAllocator) handleSignalSetChannelCapacity(event Event) {
 }
 
 func (s *StreamAllocator) handleSignalCongestionStateChange(event Event) {
-	cscd := event.Data.(congestionStateChangeData)
+	cscd := event.congestionStateChangeData
 	if cscd.toState != bwe.CongestionStateNone {
 		// end/abort any running probe if channel is not clear
 		s.maybeStopProbe()
@@ -828,8 +864,8 @@ func (s *StreamAllocator) handleSignalCongestionStateChange(event Event) {
 	// during early warning hold (if there was one)
 	if isHoldableCongestionState(cscd.fromState) && cscd.toState == bwe.CongestionStateNone && s.state == streamAllocatorStateStable {
 		update := NewStreamStateUpdate()
-		for _, track := range s.getTracks() {
-			allocation := track.AllocateOptimal(FlagAllowOvershootWhileOptimal, false)
+		for _, track := range s.getVideoTracks() {
+			allocation := track.AllocateOptimal(cFlagAllowOvershootWhileOptimal, false)
 			updateStreamStateChange(track, allocation, update)
 		}
 		s.maybeSendUpdate(update)
@@ -875,11 +911,14 @@ func (s *StreamAllocator) setState(state streamAllocatorState) {
 		s.params.BWE.Reset()
 
 		s.activeProbeClusterId = ccutils.ProbeClusterIdInvalid
+		go s.ping(s.pingGeneration.Inc(), cPingLong)
+	} else {
+		go s.ping(s.pingGeneration.Inc(), cPingShort)
 	}
 }
 
 func (s *StreamAllocator) adjustState() {
-	for _, track := range s.getTracks() {
+	for _, track := range s.getVideoTracks() {
 		if track.IsDeficient() {
 			s.setState(streamAllocatorStateDeficient)
 			return
@@ -897,7 +936,7 @@ func (s *StreamAllocator) allocateTrack(track *Track) {
 	bweCongestionState := s.params.BWE.CongestionState()
 	if !s.enabled || (s.state == streamAllocatorStateStable && !isDeficientCongestionState(bweCongestionState)) || !track.IsManaged() {
 		update := NewStreamStateUpdate()
-		allocation := track.AllocateOptimal(FlagAllowOvershootWhileOptimal, isHoldableCongestionState(bweCongestionState))
+		allocation := track.AllocateOptimal(cFlagAllowOvershootWhileOptimal, isHoldableCongestionState(bweCongestionState))
 		updateStreamStateChange(track, allocation, update)
 		s.maybeSendUpdate(update)
 		return
@@ -923,7 +962,7 @@ func (s *StreamAllocator) allocateTrack(track *Track) {
 	//        track needing change is not currently streaming, i. e. it has to be resumed.
 	//
 	track.ProvisionalAllocatePrepare()
-	transition := track.ProvisionalAllocateGetCooperativeTransition(FlagAllowOvershootWhileDeficient)
+	transition := track.ProvisionalAllocateGetCooperativeTransition(cFlagAllowOvershootWhileDeficient)
 
 	// downgrade, giving back bits
 	if transition.From.GreaterThan(transition.To) {
@@ -969,7 +1008,7 @@ func (s *StreamAllocator) allocateTrack(track *Track) {
 					availableChannelCapacity,
 					layer,
 					s.allowPause,
-					FlagAllowOvershootWhileDeficient,
+					cFlagAllowOvershootWhileDeficient,
 				)
 				if availableChannelCapacity < usedChannelCapacity {
 					break alloc_loop
@@ -995,14 +1034,14 @@ func (s *StreamAllocator) allocateTrack(track *Track) {
 		}
 
 		track.ProvisionalAllocateReset()
-		transition = track.ProvisionalAllocateGetCooperativeTransition(FlagAllowOvershootWhileDeficient) // get transition again to reset above allocation attempt using available headroom
+		transition = track.ProvisionalAllocateGetCooperativeTransition(cFlagAllowOvershootWhileDeficient) // get transition again to reset above allocation attempt using available headroom
 	}
 
 	// if there is not enough headroom, try to redistribute starting with tracks that are closest to their desired.
 	bandwidthAcquired := int64(0)
 	var contributingTracks []*Track
 
-	minDistanceSorted := s.getMinDistanceSorted(track)
+	minDistanceSorted := s.getMinDistanceSortedVideoTracks(track)
 	for _, t := range minDistanceSorted {
 		t.ProvisionalAllocatePrepare()
 	}
@@ -1052,7 +1091,7 @@ func (s *StreamAllocator) maybeStopProbe() {
 
 	pci := s.params.Pacer.EndProbeCluster(s.activeProbeClusterId)
 
-	for _, t := range s.getTracks() {
+	for _, t := range s.getVideoTracks() {
 		t.DownTrack().SwapProbeClusterId(pci.Id, ccutils.ProbeClusterIdInvalid)
 	}
 
@@ -1074,11 +1113,11 @@ func (s *StreamAllocator) maybeBoostDeficientTracks() {
 
 	update := NewStreamStateUpdate()
 
-	sortedTracks := s.getMaxDistanceSortedDeficient()
+	sortedTracks := s.getMaxDistanceSortedDeficientVideoTracks()
 boost_loop:
 	for {
 		for idx, track := range sortedTracks {
-			allocation, boosted := track.AllocateNextHigher(availableChannelCapacity, FlagAllowOvershootInCatchup)
+			allocation, boosted := track.AllocateNextHigher(availableChannelCapacity, cFlagAllowOvershootInCatchup)
 			if !boosted {
 				if idx == len(sortedTracks)-1 {
 					// all tracks tried
@@ -1096,7 +1135,7 @@ boost_loop:
 
 			break // sort again below as the track that was just boosted could still be farthest from its desired
 		}
-		sortedTracks = s.getMaxDistanceSortedDeficient()
+		sortedTracks = s.getMaxDistanceSortedDeficientVideoTracks()
 		if len(sortedTracks) == 0 {
 			break // nothing available to boost
 		}
@@ -1133,13 +1172,13 @@ func (s *StreamAllocator) allocateAllTracks() {
 	// This pass is to find out if there is any leftover channel capacity after allocating exempt tracks.
 	// Exempt tracks are given optimal allocation (i. e. no bandwidth constraint) so that they do not fail allocation.
 	//
-	videoTracks := s.getTracks()
+	videoTracks := s.getVideoTracks()
 	for _, track := range videoTracks {
 		if track.IsManaged() {
 			continue
 		}
 
-		allocation := track.AllocateOptimal(FlagAllowOvershootExemptTrackWhileDeficient, false)
+		allocation := track.AllocateOptimal(cFlagAllowOvershootExemptTrackWhileDeficient, false)
 		updateStreamStateChange(track, allocation, update)
 
 		// STREAM-ALLOCATOR-TODO: optimistic allocation before bitrate is available will return 0. How to account for that?
@@ -1162,7 +1201,7 @@ func (s *StreamAllocator) allocateAllTracks() {
 			updateStreamStateChange(track, allocation, update)
 		}
 	} else {
-		sorted := s.getSorted()
+		sorted := s.getSortedVideoTracks()
 		for _, track := range sorted {
 			track.ProvisionalAllocatePrepare()
 		}
@@ -1175,7 +1214,7 @@ func (s *StreamAllocator) allocateAllTracks() {
 				}
 
 				for _, track := range sorted {
-					_, usedChannelCapacity := track.ProvisionalAllocate(availableChannelCapacity, layer, s.allowPause, FlagAllowOvershootWhileDeficient)
+					_, usedChannelCapacity := track.ProvisionalAllocate(availableChannelCapacity, layer, s.allowPause, cFlagAllowOvershootWhileDeficient)
 					availableChannelCapacity -= usedChannelCapacity
 					if availableChannelCapacity < 0 {
 						availableChannelCapacity = 0
@@ -1239,7 +1278,7 @@ func (s *StreamAllocator) getAvailableChannelCapacity(allowOverride bool) int64
 
 func (s *StreamAllocator) getExpectedBandwidthUsage() int64 {
 	expected := int64(0)
-	for _, track := range s.getTracks() {
+	for _, track := range s.getVideoTracks() {
 		expected += track.BandwidthRequested()
 	}
 
@@ -1248,14 +1287,8 @@ func (s *StreamAllocator) getExpectedBandwidthUsage() int64 {
 
 func (s *StreamAllocator) getExpectedBandwidthUsageWithoutTracks(filteredTracks []*Track) int64 {
 	expected := int64(0)
-	for _, track := range s.getTracks() {
-		filtered := false
-		for _, ft := range filteredTracks {
-			if ft == track {
-				filtered = true
-				break
-			}
-		}
+	for _, track := range s.getVideoTracks() {
+		filtered := slices.Contains(filteredTracks, track)
 		if !filtered {
 			expected += track.BandwidthRequested()
 		}
@@ -1275,7 +1308,7 @@ func (s *StreamAllocator) getAvailableHeadroomWithoutTracks(allowOverride bool,
 func (s *StreamAllocator) getNackDelta() (uint32, uint32) {
 	aggPacketDelta := uint32(0)
 	aggRepeatedNackDelta := uint32(0)
-	for _, track := range s.getTracks() {
+	for _, track := range s.getVideoTracks() {
 		packetDelta, nackDelta := track.GetNackDelta()
 		aggPacketDelta += packetDelta
 		aggRepeatedNackDelta += nackDelta
@@ -1305,8 +1338,8 @@ func (s *StreamAllocator) maybeProbe() {
 
 func (s *StreamAllocator) maybeProbeWithMedia() {
 	// boost deficient track farthest from desired layer
-	for _, track := range s.getMaxDistanceSortedDeficient() {
-		allocation, boosted := track.AllocateNextHigher(ChannelCapacityInfinity, FlagAllowOvershootInBoost)
+	for _, track := range s.getMaxDistanceSortedDeficientVideoTracks() {
+		allocation, boosted := track.AllocateNextHigher(cChannelCapacityInfinity, cFlagAllowOvershootInBoost)
 		if !boosted {
 			continue
 		}
@@ -1322,17 +1355,14 @@ func (s *StreamAllocator) maybeProbeWithMedia() {
 
 func (s *StreamAllocator) maybeProbeWithPadding() {
 	// use deficient track farthest from desired layer to find how much to probe
-	for _, track := range s.getMaxDistanceSortedDeficient() {
-		transition, available := track.GetNextHigherTransition(FlagAllowOvershootInProbe)
+	for _, track := range s.getMaxDistanceSortedDeficientVideoTracks() {
+		transition, available := track.GetNextHigherTransition(cFlagAllowOvershootInProbe)
 		if !available || transition.BandwidthDelta < 0 {
 			continue
 		}
 
 		// overshoot a bit to account for noise (in measurement/estimate etc)
-		desiredIncreaseBps := (transition.BandwidthDelta * s.params.Config.ProbeOveragePct) / 100
-		if desiredIncreaseBps < s.params.Config.ProbeMinBps {
-			desiredIncreaseBps = s.params.Config.ProbeMinBps
-		}
+		desiredIncreaseBps := max((transition.BandwidthDelta*s.params.Config.ProbeOveragePct)/100, s.params.Config.ProbeMinBps)
 		expectedBandwidthUsage := s.getExpectedBandwidthUsage()
 		pci := s.prober.AddCluster(
 			ccutils.ProbeClusterModeUniform,
@@ -1351,68 +1381,65 @@ func (s *StreamAllocator) maybeProbeWithPadding() {
 	}
 }
 
-func (s *StreamAllocator) getTracks() []*Track {
+func (s *StreamAllocator) getVideoTracks() []*Track {
 	s.videoTracksMu.RLock()
-	tracks := make([]*Track, 0, len(s.videoTracks))
-	for _, track := range s.videoTracks {
-		tracks = append(tracks, track)
-	}
-	s.videoTracksMu.RUnlock()
+	defer s.videoTracksMu.RUnlock()
 
-	return tracks
+	return s.videoTracksShadow
 }
 
-func (s *StreamAllocator) getSorted() TrackSorter {
-	s.videoTracksMu.RLock()
+func (s *StreamAllocator) getSortedVideoTracks() TrackSorter {
 	var trackSorter TrackSorter
-	for _, track := range s.videoTracks {
+	for _, track := range s.getVideoTracks() {
 		if !track.IsManaged() {
 			continue
 		}
 
 		trackSorter = append(trackSorter, track)
 	}
-	s.videoTracksMu.RUnlock()
 
 	sort.Sort(trackSorter)
 
 	return trackSorter
 }
 
-func (s *StreamAllocator) getMinDistanceSorted(exclude *Track) MinDistanceSorter {
-	s.videoTracksMu.RLock()
+func (s *StreamAllocator) getMinDistanceSortedVideoTracks(exclude *Track) MinDistanceSorter {
 	var minDistanceSorter MinDistanceSorter
-	for _, track := range s.videoTracks {
+	for _, track := range s.getVideoTracks() {
 		if !track.IsManaged() || track == exclude {
 			continue
 		}
 
 		minDistanceSorter = append(minDistanceSorter, track)
 	}
-	s.videoTracksMu.RUnlock()
 
 	sort.Sort(minDistanceSorter)
 
 	return minDistanceSorter
 }
 
-func (s *StreamAllocator) getMaxDistanceSortedDeficient() MaxDistanceSorter {
-	s.videoTracksMu.RLock()
+func (s *StreamAllocator) getMaxDistanceSortedDeficientVideoTracks() MaxDistanceSorter {
 	var maxDistanceSorter MaxDistanceSorter
-	for _, track := range s.videoTracks {
+	for _, track := range s.getVideoTracks() {
 		if !track.IsManaged() || !track.IsDeficient() {
 			continue
 		}
 
 		maxDistanceSorter = append(maxDistanceSorter, track)
 	}
-	s.videoTracksMu.RUnlock()
 
 	sort.Sort(maxDistanceSorter)
 
 	return maxDistanceSorter
 }
 
+func (s *StreamAllocator) shadowVideoTracksLocked() {
+	s.videoTracksShadow = make([]*Track, 0, len(s.videoTracks))
+	for _, t := range s.videoTracks {
+		s.videoTracksShadow = append(s.videoTracksShadow, t)
+	}
+}
+
 // ------------------------------------------------
 
 func updateStreamStateChange(track *Track, allocation sfu.VideoAllocation, update *StreamStateUpdate) {

livekit-server/pkg/sfu/streamallocator/track.go

@@ -23,12 +23,12 @@ import (
 )
 
 type Track struct {
-	downTrack   *sfu.DownTrack
-	source      livekit.TrackSource
-	isSimulcast bool
-	priority    uint8
-	publisherID livekit.ParticipantID
-	logger      logger.Logger
+	downTrack      *sfu.DownTrack
+	source         livekit.TrackSource
+	isMultiLayered bool
+	priority       uint8
+	publisherID    livekit.ParticipantID
+	logger         logger.Logger
 
 	maxLayer buffer.VideoLayer
 
@@ -43,17 +43,17 @@ type Track struct {
 func NewTrack(
 	downTrack *sfu.DownTrack,
 	source livekit.TrackSource,
-	isSimulcast bool,
+	isMultiLayered bool,
 	publisherID livekit.ParticipantID,
 	logger logger.Logger,
 ) *Track {
 	t := &Track{
-		downTrack:   downTrack,
-		source:      source,
-		isSimulcast: isSimulcast,
-		publisherID: publisherID,
-		logger:      logger,
-		streamState: StreamStateInactive,
+		downTrack:      downTrack,
+		source:         source,
+		isMultiLayered: isMultiLayered,
+		publisherID:    publisherID,
+		logger:         logger,
+		streamState:    StreamStateInactive,
 	}
 	t.SetPriority(0)
 	t.SetMaxLayer(downTrack.MaxLayer())
@@ -87,9 +87,9 @@ func (t *Track) SetPriority(priority uint8) bool {
 	if priority == 0 {
 		switch t.source {
 		case livekit.TrackSource_SCREEN_SHARE:
-			priority = PriorityDefaultScreenshare
+			priority = cPriorityDefaultScreenshare
 		default:
-			priority = PriorityDefaultVideo
+			priority = cPriorityDefaultVideo
 		}
 	}
 
@@ -110,7 +110,7 @@ func (t *Track) DownTrack() *sfu.DownTrack {
 }
 
 func (t *Track) IsManaged() bool {
-	return t.source != livekit.TrackSource_SCREEN_SHARE || t.isSimulcast
+	return t.source != livekit.TrackSource_SCREEN_SHARE || t.isMultiLayered
 }
 
 func (t *Track) ID() livekit.TrackID {

livekit-server/pkg/sfu/streamtracker/streamtracker.go

@@ -69,8 +69,8 @@ type StreamTracker struct {
 	lastNotifiedStatus StreamStatus
 
 	lastBitrateReport time.Time
-	bytesForBitrate   [4]int64
-	bitrate           [4]int64
+	bytesForBitrate   [buffer.DefaultMaxLayerTemporal + 1]int64
+	bitrate           [buffer.DefaultMaxLayerTemporal + 1]int64
 
 	isStopped bool
 }
@@ -158,10 +158,10 @@ func (s *StreamTracker) resetLocked() {
 
 	s.setStatusLocked(StreamStatusStopped)
 
-	for i := 0; i < len(s.bytesForBitrate); i++ {
+	for i := range len(s.bytesForBitrate) {
 		s.bytesForBitrate[i] = 0
 	}
-	for i := 0; i < len(s.bitrate); i++ {
+	for i := range len(s.bitrate) {
 		s.bitrate[i] = 0
 	}
 
@@ -207,8 +207,17 @@ func (s *StreamTracker) Observe(
 		go s.worker(s.generation.Load())
 	}
 
-	if temporalLayer >= 0 {
+	if temporalLayer >= 0 && int(temporalLayer) < len(s.bytesForBitrate) {
 		s.bytesForBitrate[temporalLayer] += int64(pktSize)
+	} else if int(temporalLayer) >= len(s.bytesForBitrate) {
+		s.params.Logger.Warnw(
+			"invalid temporal layer", nil,
+			"temporalLayer", temporalLayer,
+			"pktSize", pktSize,
+			"payloadSize", payloadSize,
+			"hasMarker", hasMarker,
+			"ts", ts,
+		)
 	}
 	s.lock.Unlock()
 
@@ -235,7 +244,7 @@ func (s *StreamTracker) BitrateTemporalCumulative() []int64 {
 	}
 
 	// clear higher layers
-	for i := 0; i < len(brs); i++ {
+	for i := range brs {
 		if brs[i] == 0 {
 			for j := i + 1; j < len(brs); j++ {
 				brs[j] = 0
@@ -291,7 +300,7 @@ func (s *StreamTracker) bitrateReport() {
 	s.lastBitrateReport = now
 
 	bitrateAvailabilityChanged := false
-	for i := 0; i < len(s.bytesForBitrate); i++ {
+	for i := range len(s.bytesForBitrate) {
 		bitrate := int64(float64(s.bytesForBitrate[i]*8) / diff.Seconds())
 		if (s.bitrate[i] == 0 && bitrate > 0) || (s.bitrate[i] > 0 && bitrate == 0) {
 			bitrateAvailabilityChanged = true

livekit-server/pkg/sfu/streamtracker/streamtracker_dd.go

@@ -107,13 +107,13 @@ func (s *StreamTrackerDependencyDescriptor) resetLocked() {
 	// bump generation to trigger exit of current worker
 	s.generation.Inc()
 
-	for i := 0; i < len(s.bytesForBitrate); i++ {
-		for j := 0; j < len(s.bytesForBitrate[i]); j++ {
+	for i := range len(s.bytesForBitrate) {
+		for j := range len(s.bytesForBitrate[i]) {
 			s.bytesForBitrate[i][j] = 0
 		}
 	}
-	for i := 0; i < len(s.bitrate); i++ {
-		for j := 0; j < len(s.bitrate[i]); j++ {
+	for i := range len(s.bitrate) {
+		for j := range len(s.bitrate[i]) {
 			s.bitrate[i][j] = 0
 		}
 	}
@@ -248,11 +248,11 @@ func (s *StreamTrackerDependencyDescriptor) bitrateReport() {
 	s.lastBitrateReport = now
 
 	var availableChangedFns []func()
-	for spatial := 0; spatial < len(s.bytesForBitrate); spatial++ {
+	for spatial := range len(s.bytesForBitrate) {
 		bytesForBitrate := s.bytesForBitrate[spatial][:]
 		bitrateAvailabilityChanged := false
 		bitrates := s.bitrate[spatial][:]
-		for i := 0; i < len(bytesForBitrate); i++ {
+		for i := range bytesForBitrate {
 			bitrate := int64(float64(bytesForBitrate[i]*8) / diff.Seconds())
 			if (bitrates[i] == 0 && bitrate > 0) || (bitrates[i] > 0 && bitrate == 0) {
 				bitrateAvailabilityChanged = true

livekit-server/pkg/sfu/streamtrackermanager.go

@@ -15,13 +15,13 @@
 package sfu
 
 import (
-	"sort"
+	"slices"
 	"sync"
 	"time"
 
 	"github.com/frostbyte73/core"
-	"go.uber.org/atomic"
 
+	"github.com/livekit/protocol/codecs/mime"
 	"github.com/livekit/protocol/livekit"
 	"github.com/livekit/protocol/logger"
 	"github.com/livekit/protocol/utils"
@@ -110,14 +110,15 @@ var (
 // ---------------------------------------------------
 
 type StreamTrackerManager struct {
-	logger    logger.Logger
-	trackInfo atomic.Pointer[livekit.TrackInfo]
-	isSVC     bool
-	clockRate uint32
+	logger         logger.Logger
+	mimeType       mime.MimeType
+	videoLayerMode livekit.VideoLayer_Mode
+	clockRate      uint32
 
 	trackerConfig StreamTrackerConfig
 
 	lock                 sync.RWMutex
+	trackInfo            *livekit.TrackInfo
 	maxPublishedLayer    int32
 	maxTemporalLayerSeen int32
 
@@ -126,7 +127,6 @@ type StreamTrackerManager struct {
 
 	availableLayers  []int32
 	maxExpectedLayer int32
-	paused           bool
 
 	closed core.Fuse
 
@@ -136,18 +136,19 @@ type StreamTrackerManager struct {
 func NewStreamTrackerManager(
 	logger logger.Logger,
 	trackInfo *livekit.TrackInfo,
-	isSVC bool,
+	mimeType mime.MimeType,
 	clockRate uint32,
 	config StreamTrackerManagerConfig,
 ) *StreamTrackerManager {
 	s := &StreamTrackerManager{
 		logger:               logger,
-		isSVC:                isSVC,
+		mimeType:             mimeType,
+		videoLayerMode:       buffer.GetVideoLayerModeForMimeType(mimeType, trackInfo),
+		trackInfo:            utils.CloneProto(trackInfo),
 		maxPublishedLayer:    buffer.InvalidLayerSpatial,
 		maxTemporalLayerSeen: buffer.InvalidLayerTemporal,
 		clockRate:            clockRate,
 	}
-	s.trackInfo.Store(utils.CloneProto(trackInfo))
 
 	switch trackInfo.Source {
 	case livekit.TrackSource_SCREEN_SHARE:
@@ -158,7 +159,8 @@ func NewStreamTrackerManager(
 		s.trackerConfig = config.Video
 	}
 
-	s.maxExpectedLayerFromTrackInfo()
+	s.maxExpectedLayer = buffer.InvalidLayerSpatial
+	s.maxExpectedLayerFromTrackInfo(true)
 
 	if trackInfo.Type == livekit.TrackType_VIDEO {
 		go s.bitrateReporter()
@@ -290,7 +292,7 @@ func (s *StreamTrackerManager) AddTracker(layer int32) streamtracker.StreamTrack
 	})
 
 	s.lock.Lock()
-	paused := s.paused
+	paused := s.trackInfo.GetMuted()
 	s.trackers[layer] = tracker
 
 	notify := false
@@ -329,8 +331,10 @@ func (s *StreamTrackerManager) RemoveAllTrackers() {
 		s.trackers[layer] = nil
 	}
 	s.availableLayers = make([]int32, 0)
-	s.maxExpectedLayerFromTrackInfoLocked()
-	s.paused = false
+
+	s.maxExpectedLayer = buffer.InvalidLayerSpatial
+	s.maxExpectedLayerFromTrackInfoLocked(true)
+
 	ddTracker := s.ddTracker
 	s.ddTracker = nil
 	s.lock.Unlock()
@@ -356,9 +360,8 @@ func (s *StreamTrackerManager) GetTracker(layer int32) streamtracker.StreamTrack
 	return s.trackers[layer]
 }
 
-func (s *StreamTrackerManager) SetPaused(paused bool) {
+func (s *StreamTrackerManager) setPaused(paused bool) {
 	s.lock.Lock()
-	s.paused = paused
 	trackers := s.trackers
 	s.lock.Unlock()
 
@@ -369,16 +372,15 @@ func (s *StreamTrackerManager) SetPaused(paused bool) {
 	}
 }
 
-func (s *StreamTrackerManager) IsPaused() bool {
-	s.lock.RLock()
-	defer s.lock.RUnlock()
-
-	return s.paused
-}
-
 func (s *StreamTrackerManager) UpdateTrackInfo(ti *livekit.TrackInfo) {
-	s.trackInfo.Store(utils.CloneProto(ti))
-	s.maxExpectedLayerFromTrackInfo()
+	s.lock.Lock()
+	s.trackInfo = utils.CloneProto(ti)
+	s.maxExpectedLayerFromTrackInfoLocked(false)
+
+	paused := s.trackInfo.GetMuted()
+	s.lock.Unlock()
+
+	s.setPaused(paused)
 }
 
 func (s *StreamTrackerManager) SetMaxExpectedSpatialLayer(layer int32) int32 {
@@ -417,7 +419,7 @@ func (s *StreamTrackerManager) DistanceToDesired() float64 {
 	s.lock.RLock()
 	defer s.lock.RUnlock()
 
-	if s.paused || s.maxExpectedLayer < 0 || s.maxTemporalLayerSeen < 0 {
+	if s.trackInfo.GetMuted() || s.maxExpectedLayer < 0 || s.maxTemporalLayerSeen < 0 {
 		return 0
 	}
 
@@ -480,18 +482,18 @@ func (s *StreamTrackerManager) getLayeredBitrateLocked() ([]int32, Bitrates) {
 	for i, tracker := range s.trackers {
 		if tracker != nil {
 			tls := make([]int64, buffer.DefaultMaxLayerTemporal+1)
-			if s.hasSpatialLayerLocked(int32(i)) {
+			if slices.Contains(s.availableLayers, int32(i)) {
 				tls = tracker.BitrateTemporalCumulative()
 			}
 
-			for j := 0; j < len(br[i]); j++ {
+			for j := range len(br[i]) {
 				br[i][j] = tls[j]
 			}
 		}
 	}
 
 	// accumulate bitrates for SVC streams without dependency descriptor
-	if s.isSVC && s.ddTracker == nil {
+	if s.videoLayerMode == livekit.VideoLayer_MULTIPLE_SPATIAL_LAYERS_PER_STREAM && s.ddTracker == nil {
 		for i := len(br) - 1; i >= 1; i-- {
 			for j := len(br[i]) - 1; j >= 0; j-- {
 				if br[i][j] != 0 {
@@ -509,32 +511,15 @@ func (s *StreamTrackerManager) getLayeredBitrateLocked() ([]int32, Bitrates) {
 	return availableLayers, br
 }
 
-func (s *StreamTrackerManager) hasSpatialLayerLocked(layer int32) bool {
-	for _, l := range s.availableLayers {
-		if l == layer {
-			return true
-		}
-	}
-
-	return false
-}
-
 func (s *StreamTrackerManager) addAvailableLayer(layer int32) {
 	s.lock.Lock()
-	hasLayer := false
-	for _, l := range s.availableLayers {
-		if l == layer {
-			hasLayer = true
-			break
-		}
-	}
-	if hasLayer {
+	if slices.Contains(s.availableLayers, layer) {
 		s.lock.Unlock()
 		return
 	}
 
 	s.availableLayers = append(s.availableLayers, layer)
-	sort.Slice(s.availableLayers, func(i, j int) bool { return s.availableLayers[i] < s.availableLayers[j] })
+	slices.Sort(s.availableLayers)
 
 	// check if new layer is the max layer
 	isMaxLayerChange := s.availableLayers[len(s.availableLayers)-1] == layer
@@ -568,7 +553,7 @@ func (s *StreamTrackerManager) removeAvailableLayer(layer int32) {
 			newLayers = append(newLayers, l)
 		}
 	}
-	sort.Slice(newLayers, func(i, j int) bool { return newLayers[i] < newLayers[j] })
+	slices.Sort(newLayers)
 	s.availableLayers = newLayers
 
 	s.logger.Debugw(
@@ -594,24 +579,29 @@ func (s *StreamTrackerManager) removeAvailableLayer(layer int32) {
 	}
 }
 
-func (s *StreamTrackerManager) maxExpectedLayerFromTrackInfo() {
+func (s *StreamTrackerManager) maxExpectedLayerFromTrackInfo(force bool) {
 	s.lock.Lock()
 	defer s.lock.Unlock()
 
-	s.maxExpectedLayerFromTrackInfoLocked()
+	s.maxExpectedLayerFromTrackInfoLocked(force)
 }
 
-func (s *StreamTrackerManager) maxExpectedLayerFromTrackInfoLocked() {
-	s.maxExpectedLayer = buffer.InvalidLayerSpatial
-	ti := s.trackInfo.Load()
-	if ti != nil {
-		for _, layer := range ti.Layers {
-			spatialLayer := buffer.VideoQualityToSpatialLayer(layer.Quality, ti)
-			if spatialLayer > s.maxExpectedLayer {
-				s.maxExpectedLayer = spatialLayer
+func (s *StreamTrackerManager) maxExpectedLayerFromTrackInfoLocked(force bool) {
+	maxExpectedLayer := buffer.InvalidLayerSpatial
+	if s.trackInfo != nil {
+		for _, layer := range buffer.GetVideoLayersForMimeType(s.mimeType, s.trackInfo) {
+			if layer.SpatialLayer > maxExpectedLayer {
+				maxExpectedLayer = layer.SpatialLayer
 			}
 		}
 	}
+
+	// when max expected is higher than current max, trackers are reset
+	// which allows a layer start to be detected on initial packets from that higher layer,
+	// so update max only on track info max being lower than current max
+	if force || maxExpectedLayer < s.maxExpectedLayer {
+		s.maxExpectedLayer = maxExpectedLayer
+	}
 }
 
 func (s *StreamTrackerManager) GetMaxTemporalLayerSeen() int32 {

livekit-server/pkg/sfu/testutils/data.go

@@ -21,12 +21,13 @@ import (
 	"github.com/pion/webrtc/v4"
 
 	"github.com/livekit/livekit-server/pkg/sfu/buffer"
+	"github.com/livekit/mediatransportutil/pkg/codec"
 )
 
 // -----------------------------------------------------------
 
 type TestExtPacketParams struct {
-	SetMarker      bool
+	Marker         bool
 	IsKeyFrame     bool
 	PayloadType    uint8
 	SequenceNumber uint16
@@ -38,6 +39,7 @@ type TestExtPacketParams struct {
 	PaddingSize    byte
 	ArrivalTime    time.Time
 	VideoLayer     buffer.VideoLayer
+	IsOutOfOrder   bool
 }
 
 // -----------------------------------------------------------
@@ -47,7 +49,7 @@ func GetTestExtPacket(params *TestExtPacketParams) (*buffer.ExtPacket, error) {
 		Header: rtp.Header{
 			Version:        2,
 			Padding:        params.PaddingSize != 0,
-			Marker:         params.SetMarker,
+			Marker:         params.Marker,
 			PayloadType:    params.PayloadType,
 			SequenceNumber: params.SequenceNumber,
 			Timestamp:      params.Timestamp,
@@ -68,8 +70,9 @@ func GetTestExtPacket(params *TestExtPacketParams) (*buffer.ExtPacket, error) {
 		ExtTimestamp:      uint64(params.TSCycles<<32) + uint64(params.Timestamp),
 		Arrival:           params.ArrivalTime.UnixNano(),
 		Packet:            &packet,
-		KeyFrame:          params.IsKeyFrame,
+		IsKeyFrame:        params.IsKeyFrame,
 		RawPacket:         raw,
+		IsOutOfOrder:      params.IsOutOfOrder,
 	}
 
 	return ep, nil
@@ -77,13 +80,13 @@ func GetTestExtPacket(params *TestExtPacketParams) (*buffer.ExtPacket, error) {
 
 // --------------------------------------
 
-func GetTestExtPacketVP8(params *TestExtPacketParams, vp8 *buffer.VP8) (*buffer.ExtPacket, error) {
+func GetTestExtPacketVP8(params *TestExtPacketParams, vp8 *codec.VP8) (*buffer.ExtPacket, error) {
 	ep, err := GetTestExtPacket(params)
 	if err != nil {
 		return nil, err
 	}
 
-	ep.KeyFrame = vp8.IsKeyFrame
+	ep.IsKeyFrame = vp8.IsKeyFrame
 	ep.Payload = *vp8
 	if ep.DependencyDescriptor == nil {
 		ep.Temporal = int32(vp8.TID)

livekit-server/pkg/sfu/track_remote.go

@@ -1,3 +1,17 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package sfu
 
 import "github.com/pion/webrtc/v4"
@@ -7,6 +21,7 @@ type TrackRemote interface {
 	RID() string
 	Msid() string
 	SSRC() webrtc.SSRC
+	RtxSSRC() webrtc.SSRC
 	StreamID() string
 	Kind() webrtc.RTPCodecType
 	Codec() webrtc.RTPCodecParameters

livekit-server/pkg/sfu/utils/downtrackspreader.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package sfu
+package utils
 
 import (
 	"sync"
@@ -22,55 +22,59 @@ import (
 	"github.com/livekit/protocol/utils"
 )
 
+type sender interface {
+	SubscriberID() livekit.ParticipantID
+}
+
 type DownTrackSpreaderParams struct {
 	Threshold int
 	Logger    logger.Logger
 }
 
-type DownTrackSpreader struct {
+type DownTrackSpreader[T sender] struct {
 	params DownTrackSpreaderParams
 
 	downTrackMu      sync.RWMutex
-	downTracks       map[livekit.ParticipantID]TrackSender
-	downTracksShadow []TrackSender
+	downTracks       map[livekit.ParticipantID]T
+	downTracksShadow []T
 }
 
-func NewDownTrackSpreader(params DownTrackSpreaderParams) *DownTrackSpreader {
-	d := &DownTrackSpreader{
+func NewDownTrackSpreader[T sender](params DownTrackSpreaderParams) *DownTrackSpreader[T] {
+	d := &DownTrackSpreader[T]{
 		params:     params,
-		downTracks: make(map[livekit.ParticipantID]TrackSender),
+		downTracks: make(map[livekit.ParticipantID]T),
 	}
 
 	return d
 }
 
-func (d *DownTrackSpreader) GetDownTracks() []TrackSender {
+func (d *DownTrackSpreader[T]) GetDownTracks() []T {
 	d.downTrackMu.RLock()
 	defer d.downTrackMu.RUnlock()
 	return d.downTracksShadow
 }
 
-func (d *DownTrackSpreader) ResetAndGetDownTracks() []TrackSender {
+func (d *DownTrackSpreader[T]) ResetAndGetDownTracks() []T {
 	d.downTrackMu.Lock()
 	defer d.downTrackMu.Unlock()
 
 	downTracks := d.downTracksShadow
 
-	d.downTracks = make(map[livekit.ParticipantID]TrackSender)
+	d.downTracks = make(map[livekit.ParticipantID]T)
 	d.downTracksShadow = nil
 
 	return downTracks
 }
 
-func (d *DownTrackSpreader) Store(ts TrackSender) {
+func (d *DownTrackSpreader[T]) Store(sender T) {
 	d.downTrackMu.Lock()
 	defer d.downTrackMu.Unlock()
 
-	d.downTracks[ts.SubscriberID()] = ts
+	d.downTracks[sender.SubscriberID()] = sender
 	d.shadowDownTracks()
 }
 
-func (d *DownTrackSpreader) Free(subscriberID livekit.ParticipantID) {
+func (d *DownTrackSpreader[T]) Free(subscriberID livekit.ParticipantID) {
 	d.downTrackMu.Lock()
 	defer d.downTrackMu.Unlock()
 
@@ -78,7 +82,7 @@ func (d *DownTrackSpreader) Free(subscriberID livekit.ParticipantID) {
 	d.shadowDownTracks()
 }
 
-func (d *DownTrackSpreader) HasDownTrack(subscriberID livekit.ParticipantID) bool {
+func (d *DownTrackSpreader[T]) HasDownTrack(subscriberID livekit.ParticipantID) bool {
 	d.downTrackMu.RLock()
 	defer d.downTrackMu.RUnlock()
 
@@ -86,10 +90,10 @@ func (d *DownTrackSpreader) HasDownTrack(subscriberID livekit.ParticipantID) boo
 	return ok
 }
 
-func (d *DownTrackSpreader) Broadcast(writer func(TrackSender)) int {
+func (d *DownTrackSpreader[T]) Broadcast(writer func(T)) {
 	downTracks := d.GetDownTracks()
 	if len(downTracks) == 0 {
-		return 0
+		return
 	}
 
 	threshold := uint64(d.params.Threshold)
@@ -101,17 +105,16 @@ func (d *DownTrackSpreader) Broadcast(writer func(TrackSender)) int {
 	// WriteRTP takes about 50µs on average, so we write to 2 down tracks per loop.
 	step := uint64(2)
 	utils.ParallelExec(downTracks, threshold, step, writer)
-	return len(downTracks)
 }
 
-func (d *DownTrackSpreader) DownTrackCount() int {
+func (d *DownTrackSpreader[T]) DownTrackCount() int {
 	d.downTrackMu.RLock()
 	defer d.downTrackMu.RUnlock()
 	return len(d.downTracksShadow)
 }
 
-func (d *DownTrackSpreader) shadowDownTracks() {
-	d.downTracksShadow = make([]TrackSender, 0, len(d.downTracks))
+func (d *DownTrackSpreader[T]) shadowDownTracks() {
+	d.downTracksShadow = make([]T, 0, len(d.downTracks))
 	for _, dt := range d.downTracks {
 		d.downTracksShadow = append(d.downTracksShadow, dt)
 	}

livekit-server/pkg/sfu/utils/helpers.go

@@ -18,10 +18,12 @@ import (
 	"errors"
 	"fmt"
 
-	"github.com/livekit/livekit-server/pkg/sfu/mime"
 	"github.com/pion/interceptor"
 	"github.com/pion/rtp"
 	"github.com/pion/webrtc/v4"
+
+	"github.com/livekit/protocol/codecs/mime"
+	"github.com/livekit/protocol/livekit"
 )
 
 // Do a fuzzy find for a codec in the list of codecs
@@ -89,3 +91,7 @@ func ValidateRTPPacket(pkt *rtp.Packet, expectedPayloadType uint8, expectedSSRC
 
 	return nil
 }
+
+func IsSimulcastMode(m livekit.VideoLayer_Mode) bool {
+	return m == livekit.VideoLayer_ONE_SPATIAL_LAYER_PER_STREAM || m == livekit.VideoLayer_ONE_SPATIAL_LAYER_PER_STREAM_INCOMPLETE_RTCP_SR
+}

livekit-server/pkg/sfu/utils/owd_estimator.go

@@ -1,179 +0,0 @@
-// Copyright 2023 LiveKit, Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package utils
-
-import (
-	"time"
-
-	"go.uber.org/zap/zapcore"
-)
-
-type OWDEstimatorParams struct {
-	PropagationDelayFallFactor float64
-	PropagationDelayRiseFactor float64
-
-	PropagationDelaySpikeAdaptationFactor float64
-
-	PropagationDelayDeltaThresholdMin                time.Duration
-	PropagationDelayDeltaThresholdMaxFactor          int64
-	PropagationDelayDeltaHighResetNumReports         int
-	PropagationDelayDeltaHighResetWait               time.Duration
-	PropagationDelayDeltaLongTermAdaptationThreshold time.Duration
-}
-
-var OWDEstimatorParamsDefault = OWDEstimatorParams{
-	// OWD (One-Way-Delay) Estimator is used to estimate propagation delay between sender and receicer.
-	// As they operate on different clock domains, it is not possible to get exact propagation delay easily.
-	// So, this module is an estimator using a simple approach explained below. It should not be used for
-	// things that require high accuracy.
-	//
-	// One example is RTCP Sender Reports getting re-based to SFU time base so that all subscriber side
-	// can have the same time base (i. e. SFU time base). To convert publisher side
-	// RTCP Sender Reports to SFU timebase, a propagation delay is maintained.
-	//    propagation_delay = time_of_report_reception - ntp_timestamp_in_report
-	//
-	// Propagation delay is adapted continuously. If it falls, adapt quickly to the
-	// lower value as that could be the real propagation delay. If it rises, adapt slowly
-	// as it might be a temporary change or slow drift. See below for handling of high deltas
-	// which could be a result of a path change.
-	PropagationDelayFallFactor: 0.9,
-	PropagationDelayRiseFactor: 0.1,
-
-	PropagationDelaySpikeAdaptationFactor: 0.5,
-
-	// To account for path changes mid-stream, if the delta of the propagation delay is consistently higher, reset.
-	// Reset at whichever of the below happens later.
-	//   1. 10 seconds of persistent high delta.
-	//   2. at least 2 consecutive reports with high delta.
-	//
-	// A long term estimate of delta of propagation delay is maintained and delta propagation delay exceeding
-	// a factor of the long term estimate is considered a sharp increase. That will trigger the start of the
-	// path change condition and if it persists, propagation delay will be reset.
-	PropagationDelayDeltaThresholdMin:                10 * time.Millisecond,
-	PropagationDelayDeltaThresholdMaxFactor:          2,
-	PropagationDelayDeltaHighResetNumReports:         2,
-	PropagationDelayDeltaHighResetWait:               10 * time.Second,
-	PropagationDelayDeltaLongTermAdaptationThreshold: 50 * time.Millisecond,
-}
-
-type OWDEstimator struct {
-	params OWDEstimatorParams
-
-	initialized                        bool
-	lastSenderClockTimeNs              int64
-	lastPropagationDelayNs             int64
-	lastDeltaPropagationDelayNs        int64
-	estimatedPropagationDelayNs        int64
-	longTermDeltaPropagationDelayNs    int64
-	propagationDelayDeltaHighCount     int
-	propagationDelayDeltaHighStartTime time.Time
-	propagationDelaySpikeNs            int64
-}
-
-func NewOWDEstimator(params OWDEstimatorParams) *OWDEstimator {
-	return &OWDEstimator{
-		params: params,
-	}
-}
-
-func (o *OWDEstimator) MarshalLogObject(e zapcore.ObjectEncoder) error {
-	if o != nil {
-		e.AddTime("lastSenderClockTimeNs", time.Unix(0, o.lastSenderClockTimeNs))
-		e.AddDuration("lastPropagationDelayNs", time.Duration(o.lastPropagationDelayNs))
-		e.AddDuration("lastDeltaPropagationDelayNs", time.Duration(o.lastDeltaPropagationDelayNs))
-		e.AddDuration("estimatedPropagationDelayNs", time.Duration(o.estimatedPropagationDelayNs))
-		e.AddDuration("longTermDeltaPropagationDelayNs", time.Duration(o.longTermDeltaPropagationDelayNs))
-		e.AddInt("propagationDelayDeltaHighCount", o.propagationDelayDeltaHighCount)
-		e.AddTime("propagationDelayDeltaHighStartTime", o.propagationDelayDeltaHighStartTime)
-		e.AddDuration("propagationDelaySpikeNs", time.Duration(o.propagationDelaySpikeNs))
-	}
-	return nil
-}
-
-func (o *OWDEstimator) Update(senderClockTimeNs int64, receiverClockTimeNs int64) (int64, bool) {
-	resetDelta := func() {
-		o.propagationDelayDeltaHighCount = 0
-		o.propagationDelayDeltaHighStartTime = time.Time{}
-		o.propagationDelaySpikeNs = 0
-	}
-
-	initPropagationDelay := func(pd int64) {
-		o.estimatedPropagationDelayNs = pd
-		o.longTermDeltaPropagationDelayNs = 0
-		resetDelta()
-	}
-
-	o.lastPropagationDelayNs = receiverClockTimeNs - senderClockTimeNs
-	if !o.initialized {
-		o.initialized = true
-		o.lastSenderClockTimeNs = senderClockTimeNs
-		initPropagationDelay(o.lastPropagationDelayNs)
-		return o.estimatedPropagationDelayNs, true
-	}
-
-	stepChange := false
-	o.lastDeltaPropagationDelayNs = o.lastPropagationDelayNs - o.estimatedPropagationDelayNs
-	// check for path changes, i. e. a step jump increase in propagation delay observed over time
-	if o.lastDeltaPropagationDelayNs > o.params.PropagationDelayDeltaThresholdMin.Nanoseconds() { // ignore small changes for path change consideration
-		if o.longTermDeltaPropagationDelayNs != 0 &&
-			o.lastDeltaPropagationDelayNs > o.longTermDeltaPropagationDelayNs*o.params.PropagationDelayDeltaThresholdMaxFactor {
-			o.propagationDelayDeltaHighCount++
-			if o.propagationDelayDeltaHighStartTime.IsZero() {
-				o.propagationDelayDeltaHighStartTime = time.Now()
-			}
-			if o.propagationDelaySpikeNs == 0 {
-				o.propagationDelaySpikeNs = o.lastPropagationDelayNs
-			} else {
-				o.propagationDelaySpikeNs += int64(o.params.PropagationDelaySpikeAdaptationFactor * float64(o.lastPropagationDelayNs-o.propagationDelaySpikeNs))
-			}
-
-			if o.propagationDelayDeltaHighCount >= o.params.PropagationDelayDeltaHighResetNumReports && time.Since(o.propagationDelayDeltaHighStartTime) >= o.params.PropagationDelayDeltaHighResetWait {
-				stepChange = true
-				initPropagationDelay(o.propagationDelaySpikeNs)
-			}
-		} else {
-			resetDelta()
-		}
-	} else {
-		resetDelta()
-
-		factor := o.params.PropagationDelayFallFactor
-		if o.lastPropagationDelayNs > o.estimatedPropagationDelayNs {
-			factor = o.params.PropagationDelayRiseFactor
-		}
-		o.estimatedPropagationDelayNs += int64(factor * float64(o.lastPropagationDelayNs-o.estimatedPropagationDelayNs))
-	}
-
-	if o.lastDeltaPropagationDelayNs < o.params.PropagationDelayDeltaLongTermAdaptationThreshold.Nanoseconds() {
-		if o.longTermDeltaPropagationDelayNs == 0 {
-			o.longTermDeltaPropagationDelayNs = o.lastDeltaPropagationDelayNs
-		} else {
-			// do not adapt to large +ve spikes, can happen when channel is congested and reports are delivered very late
-			// if the spike is in fact a path change, it will persist and handled by path change detection above
-			sinceLast := senderClockTimeNs - o.lastSenderClockTimeNs
-			adaptationFactor := min(1.0, float64(sinceLast)/float64(o.params.PropagationDelayDeltaHighResetWait))
-			o.longTermDeltaPropagationDelayNs += int64(adaptationFactor * float64(o.lastDeltaPropagationDelayNs-o.longTermDeltaPropagationDelayNs))
-		}
-	}
-	if o.longTermDeltaPropagationDelayNs < 0 {
-		o.longTermDeltaPropagationDelayNs = 0
-	}
-	o.lastSenderClockTimeNs = senderClockTimeNs
-	return o.estimatedPropagationDelayNs, stepChange
-}
-
-func (o *OWDEstimator) EstimatedPropagationDelay() int64 {
-	return o.estimatedPropagationDelayNs
-}

livekit-server/pkg/sfu/utils/rangemap.go

@@ -80,10 +80,7 @@ func (r *RangeMap[RT, VT]) MarshalLogObject(e zapcore.ObjectEncoder) error {
 	e.AddInt("numRanges", len(r.ranges))
 
 	// just the last 10 ranges max
-	startIdx := len(r.ranges) - 10
-	if startIdx < 0 {
-		startIdx = 0
-	}
+	startIdx := max(len(r.ranges)-10, 0)
 	for i := startIdx; i < len(r.ranges); i++ {
 		e.AddObject(fmt.Sprintf("range[%d]", i), r.ranges[i])
 	}

livekit-server/pkg/sfu/utils/rangemap_test.go

@@ -66,7 +66,7 @@ func TestRangeMapUint32(t *testing.T) {
 	require.Equal(t, uint32(1), value)
 
 	// excluded range should return error
-	value, err = r.GetValue(10)
+	_, err = r.GetValue(10)
 	require.ErrorIs(t, err, errKeyExcluded)
 
 	// out-of-order exclusion should return error
@@ -98,11 +98,11 @@ func TestRangeMapUint32(t *testing.T) {
 	require.Equal(t, expectedRanges, r.ranges)
 
 	// excluded range should return error, now is excluded because exclusion range could be extended
-	value, err = r.GetValue(11)
+	_, err = r.GetValue(11)
 	require.ErrorIs(t, err, errKeyExcluded)
 
 	// getting value in old range should return 0
-	value, err = r.GetValue(6)
+	_, err = r.GetValue(6)
 	require.NoError(t, err)
 
 	// newer should return 2
@@ -129,7 +129,7 @@ func TestRangeMapUint32(t *testing.T) {
 	require.Equal(t, expectedRanges, r.ranges)
 
 	// excluded range should return error, now is excluded because exclusion range could be extended
-	value, err = r.GetValue(15)
+	_, err = r.GetValue(15)
 	require.ErrorIs(t, err, errKeyExcluded)
 
 	// newer should return 12
@@ -190,15 +190,15 @@ func TestRangeMapUint32(t *testing.T) {
 	require.Equal(t, expectedRanges, r.ranges)
 
 	// excluded range should return error
-	value, err = r.GetValue(50)
+	_, err = r.GetValue(50)
 	require.ErrorIs(t, err, errKeyExcluded)
-	value, err = r.GetValue(28)
+	_, err = r.GetValue(28)
 	require.ErrorIs(t, err, errKeyExcluded)
-	value, err = r.GetValue(17)
+	_, err = r.GetValue(17)
 	require.ErrorIs(t, err, errKeyTooOld)
 
 	// previously valid, but aged out key should return error
-	value, err = r.GetValue(5)
+	_, err = r.GetValue(5)
 	require.ErrorIs(t, err, errKeyTooOld)
 
 	// valid range access should return values
@@ -279,7 +279,7 @@ func TestRangeMapUint32(t *testing.T) {
 	require.Equal(t, expectedRanges, r.ranges)
 
 	// before first range access
-	value, err = r.GetValue(5)
+	_, err = r.GetValue(5)
 	require.ErrorIs(t, err, errKeyTooOld)
 
 	// first range access
@@ -319,7 +319,7 @@ func TestRangeMapUint32(t *testing.T) {
 	require.Equal(t, expectedRanges, r.ranges)
 
 	// aged out range access
-	value, err = r.GetValue(25)
+	_, err = r.GetValue(25)
 	require.ErrorIs(t, err, errKeyTooOld)
 
 	// access closed range before decrementing value

livekit-server/pkg/sfu/utils/wraparound.go

@@ -1,224 +0,0 @@
-// Copyright 2023 LiveKit, Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package utils
-
-import (
-	"unsafe"
-
-	"go.uber.org/zap/zapcore"
-)
-
-type number interface {
-	uint16 | uint32
-}
-
-type extendedNumber interface {
-	uint32 | uint64
-}
-
-type WrapAroundParams struct {
-	IsRestartAllowed bool
-}
-
-type WrapAround[T number, ET extendedNumber] struct {
-	params    WrapAroundParams
-	fullRange ET
-
-	initialized     bool
-	start           T
-	highest         T
-	cycles          ET
-	extendedHighest ET
-}
-
-func NewWrapAround[T number, ET extendedNumber](params WrapAroundParams) *WrapAround[T, ET] {
-	var t T
-	return &WrapAround[T, ET]{
-		params:    params,
-		fullRange: 1 << (unsafe.Sizeof(t) * 8),
-	}
-}
-
-func (w *WrapAround[T, ET]) Seed(from *WrapAround[T, ET]) {
-	w.initialized = from.initialized
-	w.start = from.start
-	w.highest = from.highest
-	w.cycles = from.cycles
-	w.updateExtendedHighest()
-}
-
-type WrapAroundUpdateResult[ET extendedNumber] struct {
-	IsUnhandled        bool // when set, other fields are invalid
-	IsRestart          bool
-	PreExtendedStart   ET // valid only if IsRestart = true
-	PreExtendedHighest ET
-	ExtendedVal        ET
-}
-
-func (w *WrapAroundUpdateResult[ET]) MarshalLogObject(e zapcore.ObjectEncoder) error {
-	if w == nil {
-		return nil
-	}
-
-	e.AddBool("IsUnhandled", w.IsUnhandled)
-	e.AddBool("IsRestart", w.IsRestart)
-	e.AddUint64("PreExtendedStart", uint64(w.PreExtendedStart))
-	e.AddUint64("PreExtendedHighest", uint64(w.PreExtendedHighest))
-	e.AddUint64("ExtendedVal", uint64(w.ExtendedVal))
-	return nil
-}
-
-func (w *WrapAround[T, ET]) Update(val T) (result WrapAroundUpdateResult[ET]) {
-	if !w.initialized {
-		result.PreExtendedHighest = ET(val) - 1
-		result.ExtendedVal = ET(val)
-
-		w.start = val
-		w.highest = val
-		w.updateExtendedHighest()
-		w.initialized = true
-		return
-	}
-
-	gap := val - w.highest
-	if gap > T(w.fullRange>>1) {
-		// out-of-order
-		return w.maybeAdjustStart(val)
-	}
-
-	// in-order
-	result.PreExtendedHighest = w.extendedHighest
-
-	if val < w.highest {
-		w.cycles += w.fullRange
-	}
-	w.highest = val
-
-	w.updateExtendedHighest()
-	result.ExtendedVal = w.extendedHighest
-	return
-}
-
-func (w *WrapAround[T, ET]) UndoUpdate(result WrapAroundUpdateResult[ET]) {
-	if !w.initialized || result.PreExtendedHighest >= result.ExtendedVal {
-		return
-	}
-
-	w.ResetHighest(result.PreExtendedHighest)
-}
-
-func (w *WrapAround[T, ET]) Rollover(val T, numCycles int) (result WrapAroundUpdateResult[ET]) {
-	if numCycles < 0 || !w.initialized {
-		return w.Update(val)
-	}
-
-	result.PreExtendedHighest = w.extendedHighest
-
-	w.cycles += ET(numCycles) * w.fullRange
-	w.highest = val
-
-	w.updateExtendedHighest()
-	result.ExtendedVal = w.extendedHighest
-	return
-}
-
-func (w *WrapAround[T, ET]) RollbackRestart(ev ET) {
-	if w.isWrapBack(w.start, T(ev)) {
-		w.cycles -= w.fullRange
-		w.updateExtendedHighest()
-	}
-	w.start = T(ev)
-}
-
-func (w *WrapAround[T, ET]) ResetHighest(ev ET) {
-	w.highest = T(ev)
-	w.cycles = ev & ^(w.fullRange - 1)
-	w.updateExtendedHighest()
-}
-
-func (w *WrapAround[T, ET]) GetStart() T {
-	return w.start
-}
-
-func (w *WrapAround[T, ET]) GetExtendedStart() ET {
-	return ET(w.start)
-}
-
-func (w *WrapAround[T, ET]) GetHighest() T {
-	return w.highest
-}
-
-func (w *WrapAround[T, ET]) GetExtendedHighest() ET {
-	return w.extendedHighest
-}
-
-func (w *WrapAround[T, ET]) updateExtendedHighest() {
-	w.extendedHighest = getExtended(w.cycles, w.highest)
-}
-
-func (w *WrapAround[T, ET]) maybeAdjustStart(val T) (result WrapAroundUpdateResult[ET]) {
-	// re-adjust start if necessary. The conditions are
-	// 1. Not seen more than half the range yet
-	// 1. wrap back compared to start and not completed a half cycle, sequences like (10, 65530) in uint16 space
-	// 2. no wrap around, but out-of-order compared to start and not completed a half cycle , sequences like (10, 9), (65530, 65528) in uint16 space
-	cycles := w.cycles
-	totalNum := w.GetExtendedHighest() - w.GetExtendedStart() + 1
-	if totalNum > (w.fullRange >> 1) {
-		if w.isWrapBack(val, w.highest) {
-			cycles -= w.fullRange
-		}
-		result.PreExtendedHighest = w.extendedHighest
-		result.ExtendedVal = getExtended(cycles, val)
-		return
-	}
-
-	if val-w.start > T(w.fullRange>>1) {
-		if w.params.IsRestartAllowed {
-			// out-of-order with existing start => a new start
-			result.IsRestart = true
-			if val > w.start {
-				result.PreExtendedStart = w.fullRange + ET(w.start)
-			} else {
-				result.PreExtendedStart = ET(w.start)
-			}
-
-			if w.isWrapBack(val, w.highest) {
-				w.cycles = w.fullRange
-				w.updateExtendedHighest()
-				cycles = 0
-			}
-			w.start = val
-		} else {
-			result.IsUnhandled = true
-		}
-	} else {
-		if w.isWrapBack(val, w.highest) {
-			cycles -= w.fullRange
-		}
-	}
-	result.PreExtendedHighest = w.extendedHighest
-	result.ExtendedVal = getExtended(cycles, val)
-	return
-}
-
-func (w *WrapAround[T, ET]) isWrapBack(earlier T, later T) bool {
-	return ET(later) < (w.fullRange>>1) && ET(earlier) >= (w.fullRange>>1)
-}
-
-// ------------------------------------
-
-func getExtended[T number, ET extendedNumber](cycles ET, val T) ET {
-	return cycles + ET(val)
-}

livekit-server/pkg/sfu/utils/wraparound_test.go

@@ -1,679 +0,0 @@
-// Copyright 2023 LiveKit, Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package utils
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/require"
-)
-
-func TestWrapAroundUint16(t *testing.T) {
-	w := NewWrapAround[uint16, uint32](WrapAroundParams{IsRestartAllowed: true})
-	testCases := []struct {
-		name            string
-		input           uint16
-		updated         WrapAroundUpdateResult[uint32]
-		start           uint16
-		extendedStart   uint32
-		highest         uint16
-		extendedHighest uint32
-	}{
-		// initialize
-		{
-			name:  "initialize",
-			input: 10,
-			updated: WrapAroundUpdateResult[uint32]{
-				IsRestart:          false,
-				PreExtendedStart:   0,
-				PreExtendedHighest: 9,
-				ExtendedVal:        10,
-			},
-			start:           10,
-			extendedStart:   10,
-			highest:         10,
-			extendedHighest: 10,
-		},
-		// an older number without wrap around should reset start point
-		{
-			name:  "reset start no wrap around",
-			input: 8,
-			updated: WrapAroundUpdateResult[uint32]{
-				IsRestart:          true,
-				PreExtendedStart:   10,
-				PreExtendedHighest: 10,
-				ExtendedVal:        8,
-			},
-			start:           8,
-			extendedStart:   8,
-			highest:         10,
-			extendedHighest: 10,
-		},
-		// an older number with wrap around should reset start point
-		{
-			name:  "reset start wrap around",
-			input: (1 << 16) - 6,
-			updated: WrapAroundUpdateResult[uint32]{
-				IsRestart:          true,
-				PreExtendedStart:   (1 << 16) + 8,
-				PreExtendedHighest: (1 << 16) + 10,
-				ExtendedVal:        (1 << 16) - 6,
-			},
-			start:           (1 << 16) - 6,
-			extendedStart:   (1 << 16) - 6,
-			highest:         10,
-			extendedHighest: (1 << 16) + 10,
-		},
-		// an older number with wrap around should reset start point again
-		{
-			name:  "reset start again",
-			input: (1 << 16) - 12,
-			updated: WrapAroundUpdateResult[uint32]{
-				IsRestart:          true,
-				PreExtendedStart:   (1 << 16) - 6,
-				PreExtendedHighest: (1 << 16) + 10,
-				ExtendedVal:        (1 << 16) - 12,
-			},
-			start:           (1 << 16) - 12,
-			extendedStart:   (1 << 16) - 12,
-			highest:         10,
-			extendedHighest: (1 << 16) + 10,
-		},
-		// out of order with highest, wrap back, but no restart
-		{
-			name:  "out of order - no restart",
-			input: (1 << 16) - 3,
-			updated: WrapAroundUpdateResult[uint32]{
-				IsRestart:          false,
-				PreExtendedStart:   0,
-				PreExtendedHighest: (1 << 16) + 10,
-				ExtendedVal:        (1 << 16) - 3,
-			},
-			start:           (1 << 16) - 12,
-			extendedStart:   (1 << 16) - 12,
-			highest:         10,
-			extendedHighest: (1 << 16) + 10,
-		},
-		// duplicate should return same as highest
-		{
-			name:  "duplicate",
-			input: 10,
-			updated: WrapAroundUpdateResult[uint32]{
-				IsRestart:          false,
-				PreExtendedStart:   0,
-				PreExtendedHighest: (1 << 16) + 10,
-				ExtendedVal:        (1 << 16) + 10,
-			},
-			start:           (1 << 16) - 12,
-			extendedStart:   (1 << 16) - 12,
-			highest:         10,
-			extendedHighest: (1 << 16) + 10,
-		},
-		// a significant jump in order should not reset start
-		{
-			name:  "big in-order jump",
-			input: (1 << 15) - 10,
-			updated: WrapAroundUpdateResult[uint32]{
-				IsRestart:          false,
-				PreExtendedStart:   0,
-				PreExtendedHighest: (1 << 16) + 10,
-				ExtendedVal:        (1 << 16) + (1 << 15) - 10,
-			},
-			start:           (1 << 16) - 12,
-			extendedStart:   (1 << 16) - 12,
-			highest:         (1 << 15) - 10,
-			extendedHighest: (1 << 16) + (1 << 15) - 10,
-		},
-		// now out-of-order should not reset start as half the range has been seen
-		{
-			name:  "out-of-order after half range",
-			input: (1 << 15) - 11,
-			updated: WrapAroundUpdateResult[uint32]{
-				IsRestart:          false,
-				PreExtendedStart:   0,
-				PreExtendedHighest: (1 << 16) + (1 << 15) - 10,
-				ExtendedVal:        (1 << 16) + (1 << 15) - 11,
-			},
-			start:           (1 << 16) - 12,
-			extendedStart:   (1 << 16) - 12,
-			highest:         (1 << 15) - 10,
-			extendedHighest: (1 << 16) + (1 << 15) - 10,
-		},
-		// wrap back out-of-order
-		{
-			name:  "wrap back out-of-order after half range",
-			input: (1 << 16) - 1,
-			updated: WrapAroundUpdateResult[uint32]{
-				IsRestart:          false,
-				PreExtendedStart:   0,
-				PreExtendedHighest: (1 << 16) + (1 << 15) - 10,
-				ExtendedVal:        (1 << 16) - 1,
-			},
-			start:           (1 << 16) - 12,
-			extendedStart:   (1 << 16) - 12,
-			highest:         (1 << 15) - 10,
-			extendedHighest: (1 << 16) + (1 << 15) - 10,
-		},
-		// in-order, should update highest
-		{
-			name:  "in-order",
-			input: (1 << 15) + 3,
-			updated: WrapAroundUpdateResult[uint32]{
-				IsRestart:          false,
-				PreExtendedStart:   0,
-				PreExtendedHighest: (1 << 16) + (1 << 15) - 10,
-				ExtendedVal:        (1 << 16) + (1 << 15) + 3,
-			},
-			start:           (1 << 16) - 12,
-			extendedStart:   (1 << 16) - 12,
-			highest:         (1 << 15) + 3,
-			extendedHighest: (1 << 16) + (1 << 15) + 3,
-		},
-	}
-
-	for _, tc := range testCases {
-		t.Run(tc.name, func(t *testing.T) {
-			require.Equal(t, tc.updated, w.Update(tc.input))
-			require.Equal(t, tc.start, w.GetStart())
-			require.Equal(t, tc.extendedStart, w.GetExtendedStart())
-			require.Equal(t, tc.highest, w.GetHighest())
-			require.Equal(t, tc.extendedHighest, w.GetExtendedHighest())
-		})
-	}
-}
-
-func TestWrapAroundUint16NoRestart(t *testing.T) {
-	w := NewWrapAround[uint16, uint32](WrapAroundParams{IsRestartAllowed: false})
-	testCases := []struct {
-		name            string
-		input           uint16
-		updated         WrapAroundUpdateResult[uint32]
-		start           uint16
-		extendedStart   uint32
-		highest         uint16
-		extendedHighest uint32
-	}{
-		// initialize
-		{
-			name:  "initialize",
-			input: 10,
-			updated: WrapAroundUpdateResult[uint32]{
-				IsRestart:          false,
-				PreExtendedStart:   0,
-				PreExtendedHighest: 9,
-				ExtendedVal:        10,
-			},
-			start:           10,
-			extendedStart:   10,
-			highest:         10,
-			extendedHighest: 10,
-		},
-		// an older number without wrap around should not reset start point
-		{
-			name:  "no reset start no wrap around",
-			input: 8,
-			updated: WrapAroundUpdateResult[uint32]{
-				IsUnhandled: true,
-				// the following fields are not valid when `IsUnhandled = true`, but code fills it in
-				// and they are filled in here for testing purposes
-				PreExtendedHighest: 10,
-				ExtendedVal:        8,
-			},
-			start:           10,
-			extendedStart:   10,
-			highest:         10,
-			extendedHighest: 10,
-		},
-		// an older number with wrap around should not reset start point
-		{
-			name:  "no reset start wrap around",
-			input: (1 << 16) - 6,
-			updated: WrapAroundUpdateResult[uint32]{
-				IsUnhandled:        true,
-				PreExtendedHighest: 10,
-				ExtendedVal:        (1 << 16) - 6,
-			},
-			start:           10,
-			extendedStart:   10,
-			highest:         10,
-			extendedHighest: 10,
-		},
-		// yet another older number with wrap around should not reset start point
-		{
-			name:  "no reset start again",
-			input: (1 << 16) - 12,
-			updated: WrapAroundUpdateResult[uint32]{
-				IsUnhandled:        true,
-				PreExtendedHighest: 10,
-				ExtendedVal:        (1 << 16) - 12,
-			},
-			start:           10,
-			extendedStart:   10,
-			highest:         10,
-			extendedHighest: 10,
-		},
-		// duplicate should return same as highest
-		{
-			name:  "duplicate",
-			input: 10,
-			updated: WrapAroundUpdateResult[uint32]{
-				PreExtendedHighest: 10,
-				ExtendedVal:        10,
-			},
-			start:           10,
-			extendedStart:   10,
-			highest:         10,
-			extendedHighest: 10,
-		},
-		// a significant jump in order should move highest to that
-		{
-			name:  "big in-order jump",
-			input: (1 << 15) - 10,
-			updated: WrapAroundUpdateResult[uint32]{
-				PreExtendedHighest: 10,
-				ExtendedVal:        (1 << 15) - 10,
-			},
-			start:           10,
-			extendedStart:   10,
-			highest:         (1 << 15) - 10,
-			extendedHighest: (1 << 15) - 10,
-		},
-		// in-order, should update highest
-		{
-			name:  "in-order",
-			input: (1 << 15) + 13,
-			updated: WrapAroundUpdateResult[uint32]{
-				PreExtendedHighest: (1 << 15) - 10,
-				ExtendedVal:        (1 << 15) + 13,
-			},
-			start:           10,
-			extendedStart:   10,
-			highest:         (1 << 15) + 13,
-			extendedHighest: (1 << 15) + 13,
-		},
-		// now out-of-order should not reset start as half the range has been seen
-		{
-			name:  "out-of-order after half range",
-			input: (1 << 15) - 11,
-			updated: WrapAroundUpdateResult[uint32]{
-				PreExtendedHighest: (1 << 15) + 13,
-				ExtendedVal:        (1 << 15) - 11,
-			},
-			start:           10,
-			extendedStart:   10,
-			highest:         (1 << 15) + 13,
-			extendedHighest: (1 << 15) + 13,
-		},
-	}
-
-	for _, tc := range testCases {
-		t.Run(tc.name, func(t *testing.T) {
-			require.Equal(t, tc.updated, w.Update(tc.input))
-			require.Equal(t, tc.start, w.GetStart())
-			require.Equal(t, tc.extendedStart, w.GetExtendedStart())
-			require.Equal(t, tc.highest, w.GetHighest())
-			require.Equal(t, tc.extendedHighest, w.GetExtendedHighest())
-		})
-	}
-}
-
-func TestWrapAroundUint16RollbackRestartAndResetHighest(t *testing.T) {
-	w := NewWrapAround[uint16, uint64](WrapAroundParams{IsRestartAllowed: true})
-
-	// initialize
-	w.Update(23)
-	require.Equal(t, uint16(23), w.GetStart())
-	require.Equal(t, uint64(23), w.GetExtendedStart())
-	require.Equal(t, uint16(23), w.GetHighest())
-	require.Equal(t, uint64(23), w.GetExtendedHighest())
-
-	// an in-order update
-	w.Update(25)
-	require.Equal(t, uint16(23), w.GetStart())
-	require.Equal(t, uint64(23), w.GetExtendedStart())
-	require.Equal(t, uint16(25), w.GetHighest())
-	require.Equal(t, uint64(25), w.GetExtendedHighest())
-
-	// force restart without wrap
-	res := w.Update(12)
-	expectedResult := WrapAroundUpdateResult[uint64]{
-		IsRestart:          true,
-		PreExtendedStart:   23,
-		PreExtendedHighest: 25,
-		ExtendedVal:        12,
-	}
-	require.Equal(t, expectedResult, res)
-	require.Equal(t, uint16(12), w.GetStart())
-	require.Equal(t, uint64(12), w.GetExtendedStart())
-	require.Equal(t, uint16(25), w.GetHighest())
-	require.Equal(t, uint64(25), w.GetExtendedHighest())
-
-	// roll back restart
-	w.RollbackRestart(res.PreExtendedStart)
-	require.Equal(t, uint16(23), w.GetStart())
-	require.Equal(t, uint64(23), w.GetExtendedStart())
-	require.Equal(t, uint16(25), w.GetHighest())
-	require.Equal(t, uint64(25), w.GetExtendedHighest())
-
-	// force restart with wrap
-	res = w.Update(65533)
-	expectedResult = WrapAroundUpdateResult[uint64]{
-		IsRestart:          true,
-		PreExtendedStart:   (1 << 16) + 23,
-		PreExtendedHighest: (1 << 16) + 25,
-		ExtendedVal:        65533,
-	}
-	require.Equal(t, expectedResult, res)
-	require.Equal(t, uint16(65533), w.GetStart())
-	require.Equal(t, uint64(65533), w.GetExtendedStart())
-	require.Equal(t, uint16(25), w.GetHighest())
-	require.Equal(t, uint64(65536+25), w.GetExtendedHighest())
-
-	// roll back restart
-	w.RollbackRestart(res.PreExtendedStart)
-	require.Equal(t, uint16(23), w.GetStart())
-	require.Equal(t, uint64(23), w.GetExtendedStart())
-	require.Equal(t, uint16(25), w.GetHighest())
-	require.Equal(t, uint64(25), w.GetExtendedHighest())
-
-	// reset highest
-	w.ResetHighest(0x1234)
-	require.Equal(t, uint16(23), w.GetStart())
-	require.Equal(t, uint64(23), w.GetExtendedStart())
-	require.Equal(t, uint16(0x1234), w.GetHighest())
-	require.Equal(t, uint64(0x1234), w.GetExtendedHighest())
-
-	w.ResetHighest(0x7f1234)
-	require.Equal(t, uint16(23), w.GetStart())
-	require.Equal(t, uint64(23), w.GetExtendedStart())
-	require.Equal(t, uint16(0x1234), w.GetHighest())
-	require.Equal(t, uint64(0x7f1234), w.GetExtendedHighest())
-}
-
-func TestWrapAroundUint16WrapAroundRestartDuplicate(t *testing.T) {
-	w := NewWrapAround[uint16, uint64](WrapAroundParams{IsRestartAllowed: true})
-
-	// initialize
-	w.Update(65534)
-	require.Equal(t, uint16(65534), w.GetStart())
-	require.Equal(t, uint64(65534), w.GetExtendedStart())
-	require.Equal(t, uint16(65534), w.GetHighest())
-	require.Equal(t, uint64(65534), w.GetExtendedHighest())
-
-	// an in-order update with a roll over
-	w.Update(32)
-	require.Equal(t, uint16(65534), w.GetStart())
-	require.Equal(t, uint64(65534), w.GetExtendedStart())
-	require.Equal(t, uint16(32), w.GetHighest())
-	require.Equal(t, uint64(65568), w.GetExtendedHighest())
-
-	// duplicate of start
-	res := w.Update(65534)
-	expectedResult := WrapAroundUpdateResult[uint64]{
-		IsRestart:          false,
-		PreExtendedStart:   0,
-		PreExtendedHighest: 65568,
-		ExtendedVal:        65534,
-	}
-	require.Equal(t, expectedResult, res)
-	require.Equal(t, uint16(65534), w.GetStart())
-	require.Equal(t, uint64(65534), w.GetExtendedStart())
-	require.Equal(t, uint16(32), w.GetHighest())
-	require.Equal(t, uint64(65568), w.GetExtendedHighest())
-
-	// duplicate of start - again
-	res = w.Update(65534)
-	expectedResult = WrapAroundUpdateResult[uint64]{
-		IsRestart:          false,
-		PreExtendedStart:   0,
-		PreExtendedHighest: 65568,
-		ExtendedVal:        65534,
-	}
-	require.Equal(t, expectedResult, res)
-	require.Equal(t, uint16(65534), w.GetStart())
-	require.Equal(t, uint64(65534), w.GetExtendedStart())
-	require.Equal(t, uint16(32), w.GetHighest())
-	require.Equal(t, uint64(65568), w.GetExtendedHighest())
-}
-
-func TestWrapAroundUint16Rollover(t *testing.T) {
-	w := NewWrapAround[uint16, uint32](WrapAroundParams{IsRestartAllowed: false})
-	testCases := []struct {
-		name            string
-		input           uint16
-		numCycles       int
-		updated         WrapAroundUpdateResult[uint32]
-		start           uint16
-		extendedStart   uint32
-		highest         uint16
-		extendedHighest uint32
-	}{
-		// initialize - should initialize irrespective of numCycles
-		{
-			name:      "initialize",
-			input:     10,
-			numCycles: 10,
-			updated: WrapAroundUpdateResult[uint32]{
-				IsRestart:          false,
-				PreExtendedStart:   0,
-				PreExtendedHighest: 9,
-				ExtendedVal:        10,
-			},
-			start:           10,
-			extendedStart:   10,
-			highest:         10,
-			extendedHighest: 10,
-		},
-		// negative cycles - should just do an update
-		{
-			name:      "zero",
-			input:     8,
-			numCycles: -1,
-			updated: WrapAroundUpdateResult[uint32]{
-				IsUnhandled: true,
-				// the following fields are not valid when `IsUnhandled = true`, but code fills it in
-				// and they are filled in here for testing purposes
-				PreExtendedHighest: 10,
-				ExtendedVal:        8,
-			},
-			start:           10,
-			extendedStart:   10,
-			highest:         10,
-			extendedHighest: 10,
-		},
-		// one cycle
-		{
-			name:      "one cycle",
-			input:     (1 << 16) - 6,
-			numCycles: 1,
-			updated: WrapAroundUpdateResult[uint32]{
-				PreExtendedHighest: 10,
-				ExtendedVal:        (1 << 16) - 6 + (1 << 16),
-			},
-			start:           10,
-			extendedStart:   10,
-			highest:         (1 << 16) - 6,
-			extendedHighest: (1 << 16) - 6 + (1 << 16),
-		},
-		// two cycles
-		{
-			name:      "two cycles",
-			input:     (1 << 16) - 7,
-			numCycles: 2,
-			updated: WrapAroundUpdateResult[uint32]{
-				PreExtendedHighest: (1 << 16) - 6 + (1 << 16),
-				ExtendedVal:        (1 << 16) - 7 + 3*(1<<16),
-			},
-			start:           10,
-			extendedStart:   10,
-			highest:         (1 << 16) - 7,
-			extendedHighest: (1 << 16) - 7 + 3*(1<<16),
-		},
-	}
-
-	for _, tc := range testCases {
-		t.Run(tc.name, func(t *testing.T) {
-			require.Equal(t, tc.updated, w.Rollover(tc.input, tc.numCycles))
-			require.Equal(t, tc.start, w.GetStart())
-			require.Equal(t, tc.extendedStart, w.GetExtendedStart())
-			require.Equal(t, tc.highest, w.GetHighest())
-			require.Equal(t, tc.extendedHighest, w.GetExtendedHighest())
-		})
-	}
-}
-
-func TestWrapAroundUint32(t *testing.T) {
-	w := NewWrapAround[uint32, uint64](WrapAroundParams{IsRestartAllowed: true})
-	testCases := []struct {
-		name            string
-		input           uint32
-		updated         WrapAroundUpdateResult[uint64]
-		start           uint32
-		extendedStart   uint64
-		highest         uint32
-		extendedHighest uint64
-	}{
-		// initialize
-		{
-			name:  "initialize",
-			input: 10,
-			updated: WrapAroundUpdateResult[uint64]{
-				IsRestart:          false,
-				PreExtendedStart:   0,
-				PreExtendedHighest: 9,
-				ExtendedVal:        10,
-			},
-			start:           10,
-			extendedStart:   10,
-			highest:         10,
-			extendedHighest: 10,
-		},
-		// an older number without wrap around should reset start point
-		{
-			name:  "reset start no wrap around",
-			input: 8,
-			updated: WrapAroundUpdateResult[uint64]{
-				IsRestart:          true,
-				PreExtendedStart:   10,
-				PreExtendedHighest: 10,
-				ExtendedVal:        8,
-			},
-			start:           8,
-			extendedStart:   8,
-			highest:         10,
-			extendedHighest: 10,
-		},
-		// an older number with wrap around should reset start point
-		{
-			name:  "reset start wrap around",
-			input: (1 << 32) - 6,
-			updated: WrapAroundUpdateResult[uint64]{
-				IsRestart:          true,
-				PreExtendedStart:   (1 << 32) + 8,
-				PreExtendedHighest: (1 << 32) + 10,
-				ExtendedVal:        (1 << 32) - 6,
-			},
-			start:           (1 << 32) - 6,
-			extendedStart:   (1 << 32) - 6,
-			highest:         10,
-			extendedHighest: (1 << 32) + 10,
-		},
-		// an older number with wrap around should reset start point again
-		{
-			name:  "reset start again",
-			input: (1 << 32) - 12,
-			updated: WrapAroundUpdateResult[uint64]{
-				IsRestart:          true,
-				PreExtendedStart:   (1 << 32) - 6,
-				PreExtendedHighest: (1 << 32) + 10,
-				ExtendedVal:        (1 << 32) - 12,
-			},
-			start:           (1 << 32) - 12,
-			extendedStart:   (1 << 32) - 12,
-			highest:         10,
-			extendedHighest: (1 << 32) + 10,
-		},
-		// duplicate should return same as highest
-		{
-			name:  "duplicate",
-			input: 10,
-			updated: WrapAroundUpdateResult[uint64]{
-				IsRestart:          false,
-				PreExtendedStart:   0,
-				PreExtendedHighest: (1 << 32) + 10,
-				ExtendedVal:        (1 << 32) + 10,
-			},
-			start:           (1 << 32) - 12,
-			extendedStart:   (1 << 32) - 12,
-			highest:         10,
-			extendedHighest: (1 << 32) + 10,
-		},
-		// a significant jump in order should not reset start
-		{
-			name:  "big in-order jump",
-			input: 1 << 31,
-			updated: WrapAroundUpdateResult[uint64]{
-				IsRestart:          false,
-				PreExtendedStart:   0,
-				PreExtendedHighest: (1 << 32) + 10,
-				ExtendedVal:        (1 << 32) + (1 << 31),
-			},
-			start:           (1 << 32) - 12,
-			extendedStart:   (1 << 32) - 12,
-			highest:         1 << 31,
-			extendedHighest: (1 << 32) + (1 << 31),
-		},
-		// now out-of-order should not reset start as half the range has been seen
-		{
-			name:  "out-of-order after half range",
-			input: (1 << 31) - 1,
-			updated: WrapAroundUpdateResult[uint64]{
-				IsRestart:          false,
-				PreExtendedStart:   0,
-				PreExtendedHighest: (1 << 32) + (1 << 31),
-				ExtendedVal:        (1 << 32) + (1 << 31) - 1,
-			},
-			start:           (1 << 32) - 12,
-			extendedStart:   (1 << 32) - 12,
-			highest:         1 << 31,
-			extendedHighest: (1 << 32) + (1 << 31),
-		},
-		// in-order, should update highest
-		{
-			name:  "in-order",
-			input: (1 << 31) + 3,
-			updated: WrapAroundUpdateResult[uint64]{
-				IsRestart:          false,
-				PreExtendedStart:   0,
-				PreExtendedHighest: (1 << 32) + (1 << 31),
-				ExtendedVal:        (1 << 32) + (1 << 31) + 3,
-			},
-			start:           (1 << 32) - 12,
-			extendedStart:   (1 << 32) - 12,
-			highest:         (1 << 31) + 3,
-			extendedHighest: (1 << 32) + (1 << 31) + 3,
-		},
-	}
-
-	for _, tc := range testCases {
-		t.Run(tc.name, func(t *testing.T) {
-			require.Equal(t, tc.updated, w.Update(tc.input))
-			require.Equal(t, tc.start, w.GetStart())
-			require.Equal(t, tc.extendedStart, w.GetExtendedStart())
-			require.Equal(t, tc.highest, w.GetHighest())
-			require.Equal(t, tc.extendedHighest, w.GetExtendedHighest())
-		})
-	}
-}

livekit-server/pkg/sfu/videolayerselector/dependencydescriptor.go

@@ -24,6 +24,11 @@ import (
 	"github.com/livekit/protocol/logger"
 )
 
+const (
+	decisionCacheMaxElements = 256
+	decisionCacheNackEntries = 80
+)
+
 type DependencyDescriptor struct {
 	*Base
 
@@ -40,12 +45,14 @@ type DependencyDescriptor struct {
 	decodeTargetsLock sync.RWMutex
 	decodeTargets     []*DecodeTarget
 	fnWrapper         FrameNumberWrapper
+
+	restartGeneration int
 }
 
 func NewDependencyDescriptor(logger logger.Logger) *DependencyDescriptor {
 	return &DependencyDescriptor{
 		Base:      NewBase(logger),
-		decisions: NewSelectorDecisionCache(256, 80),
+		decisions: NewSelectorDecisionCache(decisionCacheMaxElements, decisionCacheNackEntries),
 		fnWrapper: FrameNumberWrapper{logger: logger},
 	}
 }
@@ -75,6 +82,21 @@ func (d *DependencyDescriptor) Select(extPkt *buffer.ExtPacket, _layer int32) (r
 		return
 	}
 
+	if ddwdt.RestartGeneration > d.restartGeneration {
+		d.logger.Debugw(
+			"stream restarted",
+			"packet", ddwdt.RestartGeneration,
+			"current", d.restartGeneration,
+			"structureKeyFrame", d.extKeyFrameNum,
+			"efn", ddwdt.ExtFrameNum,
+			"lastEfn", d.fnWrapper.LastOrigin(),
+		)
+		d.restart(ddwdt.RestartGeneration)
+	} else if ddwdt.RestartGeneration < d.restartGeneration {
+		// must not happen
+		d.logger.Warnw("packet from old generation", nil, "packet", ddwdt.RestartGeneration, "current", d.restartGeneration)
+	}
+
 	dd := ddwdt.Descriptor
 
 	extFrameNum := ddwdt.ExtFrameNum
@@ -269,7 +291,7 @@ func (d *DependencyDescriptor) Select(extPkt *buffer.ExtPacket, _layer int32) (r
 				"fn", dd.FrameNumber,
 				"efn", extFrameNum,
 				"sn", extPkt.Packet.SequenceNumber,
-				"isKeyFrame", extPkt.KeyFrame,
+				"isKeyFrame", extPkt.IsKeyFrame,
 			)
 		}
 
@@ -286,7 +308,7 @@ func (d *DependencyDescriptor) Select(extPkt *buffer.ExtPacket, _layer int32) (r
 			"fn", dd.FrameNumber,
 			"efn", extFrameNum,
 			"sn", extPkt.Packet.SequenceNumber,
-			"isKeyFrame", extPkt.KeyFrame,
+			"isKeyFrame", extPkt.IsKeyFrame,
 		)
 
 		result.IsRelevant = true
@@ -434,3 +456,9 @@ func (d *DependencyDescriptor) CheckSync() (locked bool, layer int32) {
 
 	return false, layer
 }
+
+func (d *DependencyDescriptor) restart(generation int) {
+	d.restartGeneration = generation
+	d.invalidateKeyFrame()
+	d.decisions = NewSelectorDecisionCache(decisionCacheMaxElements, decisionCacheNackEntries)
+}

livekit-server/pkg/sfu/videolayerselector/dependencydescriptor_test.go

@@ -15,7 +15,7 @@
 package videolayerselector
 
 import (
-	"sort"
+	"slices"
 	"testing"
 
 	"github.com/pion/rtp"
@@ -143,7 +143,7 @@ func TestDependencyDescriptor(t *testing.T) {
 
 	// non key frame, dropped
 	ret = ddSelector.Select(&buffer.ExtPacket{
-		KeyFrame: false,
+		IsKeyFrame: false,
 		DependencyDescriptor: &buffer.ExtDependencyDescriptor{
 			Descriptor: &dd.DependencyDescriptor{
 				FrameNumber: 1,
@@ -287,8 +287,14 @@ func createDDFrames(maxLayer buffer.VideoLayer, startFrameNumber uint16) []*buff
 			activeBitMask |= 1 << uint(i*int(maxLayer.Temporal+1)+j)
 		}
 	}
-	sort.Slice(decodeTargets, func(i, j int) bool {
-		return decodeTargets[i].Layer.GreaterThan(decodeTargets[j].Layer)
+	slices.SortFunc(decodeTargets, func(a, b buffer.DependencyDescriptorDecodeTarget) int {
+		if a.Layer.GreaterThan(b.Layer) {
+			return -1
+		}
+		if b.Layer.GreaterThan(a.Layer) {
+			return 1
+		}
+		return 0
 	})
 
 	chainDiffs := make([]int, int(maxLayer.Spatial)+1)
@@ -308,7 +314,7 @@ func createDDFrames(maxLayer buffer.VideoLayer, startFrameNumber uint16) []*buff
 		}
 	}
 	keyFrame := &buffer.ExtPacket{
-		KeyFrame: true,
+		IsKeyFrame: true,
 		DependencyDescriptor: &buffer.ExtDependencyDescriptor{
 			Descriptor: &dd.DependencyDescriptor{
 				FrameNumber: startFrameNumber,

livekit-server/pkg/sfu/videolayerselector/framechain.go

@@ -71,11 +71,11 @@ func (fc *FrameChain) OnFrame(extFrameNum uint64, fd *dd.FrameDependencyTemplate
 	}
 
 	var intact bool
-	switch {
-	case sd == selectorDecisionForwarded:
+	switch sd {
+	case selectorDecisionForwarded:
 		intact = true
 
-	case sd == selectorDecisionUnknown:
+	case selectorDecisionUnknown:
 		// If the previous frame is unknown, means it has not arrived but could be recovered by NACK / out-of-order arrival,
 		// set up a expected callback here to determine if the chain is broken or intact
 		if fc.decisions.ExpectDecision(prevFrameInChain, fc.OnExpectFrameChanged) {

livekit-server/pkg/sfu/videolayerselector/framenumberwrapper.go

@@ -52,3 +52,7 @@ func (f *FrameNumberWrapper) UpdateAndGet(new uint64, updateOffset bool) uint64
 	f.last = new
 	return new + f.offset
 }
+
+func (f *FrameNumberWrapper) LastOrigin() uint64 {
+	return f.last
+}

livekit-server/pkg/sfu/videolayerselector/framenumberwrapper_test.go

@@ -21,7 +21,7 @@ import (
 
 	"github.com/stretchr/testify/require"
 
-	"github.com/livekit/livekit-server/pkg/sfu/utils"
+	"github.com/livekit/mediatransportutil/pkg/utils"
 	"github.com/livekit/protocol/logger"
 )
 
@@ -65,7 +65,7 @@ func TestFrameNumberWrapper(t *testing.T) {
 		// frame out of order
 		firstF = secondF
 		secondF = getFrame(firstF, false)
-		// it is possile that an out of order non-keyframe has been converted to in order frame number if the diff is 32768
+		// it is possible that an out of order non-keyframe has been converted to in order frame number if the diff is 32768
 		// that is ok because the client can't decode in such case and always need to wait for the key frame.
 		// so it is just a failure of test case and increase the frame number here.
 		if secondF-firstF == 0x8000 {

livekit-server/pkg/sfu/videolayerselector/simulcast.go

@@ -93,7 +93,7 @@ func (s *Simulcast) Select(extPkt *buffer.ExtPacket, layer int32) (result VideoL
 		isActive := s.currentLayer.IsValid()
 		found := false
 		reason := ""
-		if extPkt.KeyFrame {
+		if extPkt.IsKeyFrame {
 			if layer > s.currentLayer.Spatial && layer <= s.targetLayer.Spatial {
 				reason = "upgrading layer"
 				found = true
@@ -124,7 +124,7 @@ func (s *Simulcast) Select(extPkt *buffer.ExtPacket, layer int32) (result VideoL
 	}
 
 	// if locked to higher than max layer due to overshoot, check if it can be dialed back
-	if s.currentLayer.Spatial > s.maxLayer.Spatial && layer <= s.maxLayer.Spatial && extPkt.KeyFrame {
+	if s.currentLayer.Spatial > s.maxLayer.Spatial && layer <= s.maxLayer.Spatial && extPkt.IsKeyFrame {
 		s.previousLayer = s.currentLayer
 		s.currentLayer.Spatial = layer
 

livekit-server/pkg/sfu/videolayerselector/temporallayerselector/vp8.go

@@ -16,6 +16,7 @@ package temporallayerselector
 
 import (
 	"github.com/livekit/livekit-server/pkg/sfu/buffer"
+	"github.com/livekit/mediatransportutil/pkg/codec"
 	"github.com/livekit/protocol/logger"
 )
 
@@ -36,7 +37,7 @@ func (v *VP8) Select(extPkt *buffer.ExtPacket, current int32, target int32) (thi
 		return
 	}
 
-	vp8, ok := extPkt.Payload.(buffer.VP8)
+	vp8, ok := extPkt.Payload.(codec.VP8)
 	if !ok {
 		return
 	}

livekit-server/pkg/sfu/videolayerselector/vp9.go

@@ -50,7 +50,7 @@ func (v *VP9) Select(extPkt *buffer.ExtPacket, _layer int32) (result VideoLayerS
 		updatedLayer := v.currentLayer
 
 		if !v.currentLayer.IsValid() {
-			if !extPkt.KeyFrame {
+			if !extPkt.IsKeyFrame {
 				return
 			}