ttipo/comrelay
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge commit '0da97ebd2149ec2a0412a273b17270f540431d81' as 'livekit-server'

Browse Source
This commit is contained in:
ttipo
2026-06-25 14:35:28 +09:00
parent 79194428fc 0da97ebd21
commit 8f49774aff
339 changed files with 114111 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
livekit-server/pkg/sfu/NOTICE
+16
View File
@@ -0,0 +1,16 @@
Portions of this package originated from ion-sfu: https://github.com/pion/ion-sfu.
MIT License
Copyright (c) 2019
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
----------------------------------------------------------------------------------------
livekit-server/pkg/sfu/audio/audiolevel.go
+156
View File
@@ -0,0 +1,156 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package audio
import (
"math"
"sync"
)
const (
silentAudioLevel = 127
negInv20 = -1.0 / 20
)
// --------------------------------------
type AudioLevelConfig struct {
// minimum level to be considered active, 0-127, where 0 is loudest
ActiveLevel uint8 `yaml:"active_level,omitempty"`
// percentile to measure, a participant is considered active if it has exceeded the ActiveLevel more than
// MinPercentile% of the time
MinPercentile uint8 `yaml:"min_percentile,omitempty"`
// interval to update clients, in ms
UpdateInterval uint32 `yaml:"update_interval,omitempty"`
// smoothing for audioLevel values sent to the client.
// audioLevel will be an average of `smooth_intervals`, 0 to disable
SmoothIntervals uint32 `yaml:"smooth_intervals,omitempty"`
}
var (
DefaultAudioLevelConfig = AudioLevelConfig{
ActiveLevel: 35, // -35dBov
MinPercentile: 40,
UpdateInterval: 400,
SmoothIntervals: 2,
}
)
// --------------------------------------
type AudioLevelParams struct {
Config AudioLevelConfig
}
// keeps track of audio level for a participant
type AudioLevel struct {
params AudioLevelParams
// min duration within an observe duration window to be considered active
minActiveDuration uint32
smoothFactor float64
activeThreshold float64
lock sync.Mutex
smoothedLevel float64
loudestObservedLevel uint8
activeDuration uint32 // ms
observedDuration uint32 // ms
lastObservedAt int64
}
func NewAudioLevel(params AudioLevelParams) *AudioLevel {
l := &AudioLevel{
params: params,
minActiveDuration: uint32(params.Config.MinPercentile) * params.Config.UpdateInterval / 100,
smoothFactor: 1,
activeThreshold: ConvertAudioLevel(float64(params.Config.ActiveLevel)),
loudestObservedLevel: silentAudioLevel,
}
if l.params.Config.SmoothIntervals > 0 {
// exponential moving average (EMA), same center of mass with simple moving average (SMA)
l.smoothFactor = float64(2) / (float64(l.params.Config.SmoothIntervals + 1))
}
return l
}
// Observes a new frame
func (l *AudioLevel) Observe(level uint8, durationMs uint32, arrivalTime int64) {
l.lock.Lock()
defer l.lock.Unlock()
l.lastObservedAt = arrivalTime
l.observedDuration += durationMs
if level <= l.params.Config.ActiveLevel {
l.activeDuration += durationMs
if l.loudestObservedLevel > level {
l.loudestObservedLevel = level
}
}
if l.observedDuration >= l.params.Config.UpdateInterval {
smoothedLevel := float64(0.0)
// compute and reset
if l.activeDuration >= l.minActiveDuration {
// adjust loudest observed level by how much of the window was active.
// Weight will be 0 if active the entire duration
// > 0 if active for longer than observe duration
// < 0 if active for less than observe duration
activityWeight := 20 * math.Log10(float64(l.activeDuration)/float64(l.params.Config.UpdateInterval))
adjustedLevel := float64(l.loudestObservedLevel) - activityWeight
linearLevel := ConvertAudioLevel(adjustedLevel)
// exponential smoothing to dampen transients
smoothedLevel = l.smoothedLevel + (linearLevel-l.smoothedLevel)*l.smoothFactor
}
l.resetLocked(smoothedLevel)
}
}
// returns current smoothed audio level
func (l *AudioLevel) GetLevel(now int64) (float64, bool) {
l.lock.Lock()
defer l.lock.Unlock()
l.resetIfStaleLocked(now)
return l.smoothedLevel, l.smoothedLevel >= l.activeThreshold
}
func (l *AudioLevel) resetIfStaleLocked(arrivalTime int64) {
if (arrivalTime-l.lastObservedAt)/1e6 < int64(2*l.params.Config.UpdateInterval) {
return
}
l.resetLocked(0.0)
}
func (l *AudioLevel) resetLocked(smoothedLevel float64) {
l.smoothedLevel = smoothedLevel
l.loudestObservedLevel = silentAudioLevel
l.activeDuration = 0
l.observedDuration = 0
}
// ---------------------------------------------------
// convert decibel back to linear
func ConvertAudioLevel(level float64) float64 {
return math.Pow(10, level*negInv20)
}
livekit-server/pkg/sfu/audio/audiolevel_test.go
+123
View File
@@ -0,0 +1,123 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package audio
import (
"testing"
"time"
"github.com/stretchr/testify/require"
)
const (
samplesPerBatch = 25
defaultActiveLevel = 30
// requires two noisy samples to count
defaultPercentile = 10
defaultObserveDuration = 500 // ms
)
func TestAudioLevel(t *testing.T) {
t.Run("initially to return not noisy, within a few samples", func(t *testing.T) {
clock := time.Now()
a := createAudioLevel(defaultActiveLevel, defaultPercentile, defaultObserveDuration)
_, noisy := a.GetLevel(clock.UnixNano())
require.False(t, noisy)
observeSamples(a, 28, 5, clock)
clock = clock.Add(5 * 20 * time.Millisecond)
_, noisy = a.GetLevel(clock.UnixNano())
require.False(t, noisy)
})
t.Run("not noisy when all samples are below threshold", func(t *testing.T) {
clock := time.Now()
a := createAudioLevel(defaultActiveLevel, defaultPercentile, defaultObserveDuration)
observeSamples(a, 35, 100, clock)
clock = clock.Add(100 * 20 * time.Millisecond)
_, noisy := a.GetLevel(clock.UnixNano())
require.False(t, noisy)
})
t.Run("not noisy when less than percentile samples are above threshold", func(t *testing.T) {
clock := time.Now()
a := createAudioLevel(defaultActiveLevel, defaultPercentile, defaultObserveDuration)
observeSamples(a, 35, samplesPerBatch-2, clock)
clock = clock.Add((samplesPerBatch - 2) * 20 * time.Millisecond)
observeSamples(a, 25, 1, clock)
clock = clock.Add(20 * time.Millisecond)
observeSamples(a, 35, 1, clock)
clock = clock.Add(20 * time.Millisecond)
_, noisy := a.GetLevel(clock.UnixNano())
require.False(t, noisy)
})
t.Run("noisy when higher than percentile samples are above threshold", func(t *testing.T) {
clock := time.Now()
a := createAudioLevel(defaultActiveLevel, defaultPercentile, defaultObserveDuration)
observeSamples(a, 35, samplesPerBatch-16, clock)
clock = clock.Add((samplesPerBatch - 16) * 20 * time.Millisecond)
observeSamples(a, 25, 8, clock)
clock = clock.Add(8 * 20 * time.Millisecond)
observeSamples(a, 29, 8, clock)
clock = clock.Add(8 * 20 * time.Millisecond)
level, noisy := a.GetLevel(clock.UnixNano())
require.True(t, noisy)
require.Greater(t, level, ConvertAudioLevel(float64(defaultActiveLevel)))
require.Less(t, level, ConvertAudioLevel(float64(25)))
})
t.Run("not noisy when samples are stale", func(t *testing.T) {
clock := time.Now()
a := createAudioLevel(defaultActiveLevel, defaultPercentile, defaultObserveDuration)
observeSamples(a, 25, 100, clock)
clock = clock.Add(100 * 20 * time.Millisecond)
level, noisy := a.GetLevel(clock.UnixNano())
require.True(t, noisy)
require.Greater(t, level, ConvertAudioLevel(float64(defaultActiveLevel)))
require.Less(t, level, ConvertAudioLevel(float64(20)))
// let enough time pass to make the samples stale
clock = clock.Add(1500 * time.Millisecond)
level, noisy = a.GetLevel(clock.UnixNano())
require.Equal(t, float64(0.0), level)
require.False(t, noisy)
})
}
func createAudioLevel(activeLevel uint8, minPercentile uint8, observeDuration uint32) *AudioLevel {
return NewAudioLevel(AudioLevelParams{
Config: AudioLevelConfig{
ActiveLevel: activeLevel,
MinPercentile: minPercentile,
UpdateInterval: observeDuration,
},
})
}
func observeSamples(a *AudioLevel, level uint8, count int, baseTime time.Time) {
for i := 0; i < count; i++ {
a.Observe(level, 20, baseTime.Add(+time.Duration(i*20)*time.Millisecond).UnixNano())
}
}
livekit-server/pkg/sfu/buffer/buffer.go
+1222
View File
File diff suppressed because it is too large Load Diff
livekit-server/pkg/sfu/buffer/buffer_test.go
+435
View File
@@ -0,0 +1,435 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package buffer
import (
"math"
"sync"
"testing"
"time"
"github.com/pion/rtcp"
"github.com/pion/rtp"
"github.com/pion/webrtc/v4"
"github.com/stretchr/testify/require"
"github.com/livekit/mediatransportutil/pkg/nack"
)
var h265Codec = webrtc.RTPCodecParameters{
RTPCodecCapability: webrtc.RTPCodecCapability{
MimeType: "video/h265",
ClockRate: 90000,
RTCPFeedback: []webrtc.RTCPFeedback{{
Type: "nack",
}},
},
PayloadType: 116,
}
var vp8Codec = webrtc.RTPCodecParameters{
RTPCodecCapability: webrtc.RTPCodecCapability{
MimeType: "video/vp8",
ClockRate: 90000,
RTCPFeedback: []webrtc.RTCPFeedback{{
Type: "nack",
}},
},
PayloadType: 96,
}
var opusCodec = webrtc.RTPCodecParameters{
RTPCodecCapability: webrtc.RTPCodecCapability{
MimeType: "audio/opus",
ClockRate: 48000,
},
PayloadType: 111,
}
func TestNack(t *testing.T) {
t.Run("nack normal", func(t *testing.T) {
buff := NewBuffer(123, 1, 1)
buff.codecType = webrtc.RTPCodecTypeVideo
require.NotNil(t, buff)
var wg sync.WaitGroup
// 5 tries
wg.Add(5)
buff.OnRtcpFeedback(func(fb []rtcp.Packet) {
for _, pkt := range fb {
switch p := pkt.(type) {
case *rtcp.TransportLayerNack:
if p.Nacks[0].PacketList()[0] == 1 && p.MediaSSRC == 123 {
wg.Done()
}
}
}
})
buff.Bind(webrtc.RTPParameters{
HeaderExtensions: nil,
Codecs: []webrtc.RTPCodecParameters{vp8Codec},
}, vp8Codec.RTPCodecCapability, 0)
rtt := uint32(20)
buff.nacker.SetRTT(rtt)
for i := 0; i < 15; i++ {
if i == 1 {
continue
}
if i < 14 {
time.Sleep(time.Duration(float64(rtt)*math.Pow(nack.NackQueueParamsDefault.BackoffFactor, float64(i))+10) * time.Millisecond)
} else {
time.Sleep(500 * time.Millisecond) // even a long wait should not exceed max retries
}
pkt := rtp.Packet{
Header: rtp.Header{
Version: 2,
PayloadType: 96,
SequenceNumber: uint16(i),
Timestamp: uint32(i),
SSRC: 123,
},
Payload: []byte{0xff, 0xff, 0xff, 0xfd, 0xb4, 0x9f, 0x94, 0x1},
}
b, err := pkt.Marshal()
require.NoError(t, err)
_, err = buff.Write(b)
require.NoError(t, err)
}
wg.Wait()
})
t.Run("nack with seq wrap", func(t *testing.T) {
buff := NewBuffer(123, 1, 1)
buff.codecType = webrtc.RTPCodecTypeVideo
require.NotNil(t, buff)
var wg sync.WaitGroup
expects := map[uint16]int{
65534: 0,
65535: 0,
0: 0,
1: 0,
}
wg.Add(5 * len(expects)) // retry 5 times
buff.OnRtcpFeedback(func(fb []rtcp.Packet) {
for _, pkt := range fb {
switch p := pkt.(type) {
case *rtcp.TransportLayerNack:
if p.MediaSSRC == 123 {
for _, v := range p.Nacks {
v.Range(func(seq uint16) bool {
if _, ok := expects[seq]; ok {
wg.Done()
} else {
require.Fail(t, "unexpected nack seq ", seq)
}
return true
})
}
}
}
}
})
buff.Bind(webrtc.RTPParameters{
HeaderExtensions: nil,
Codecs: []webrtc.RTPCodecParameters{vp8Codec},
}, vp8Codec.RTPCodecCapability, 0)
rtt := uint32(30)
buff.nacker.SetRTT(rtt)
for i := 0; i < 15; i++ {
if i > 0 && i < 5 {
continue
}
if i < 14 {
time.Sleep(time.Duration(float64(rtt)*math.Pow(nack.NackQueueParamsDefault.BackoffFactor, float64(i))+10) * time.Millisecond)
} else {
time.Sleep(500 * time.Millisecond) // even a long wait should not exceed max retries
}
pkt := rtp.Packet{
Header: rtp.Header{
Version: 2,
PayloadType: 96,
SequenceNumber: uint16(i + 65533),
Timestamp: uint32(i),
SSRC: 123,
},
Payload: []byte{0xff, 0xff, 0xff, 0xfd, 0xb4, 0x9f, 0x94, 0x1},
}
b, err := pkt.Marshal()
require.NoError(t, err)
_, err = buff.Write(b)
require.NoError(t, err)
}
wg.Wait()
})
}
func TestNewBuffer(t *testing.T) {
tests := []struct {
name string
}{
{
name: "Must not be nil and add packets in sequence",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
var TestPackets = []*rtp.Packet{
{
Header: rtp.Header{
Version: 2,
PayloadType: 96,
SequenceNumber: 65533,
SSRC: 123,
},
},
{
Header: rtp.Header{
Version: 2,
PayloadType: 96,
SequenceNumber: 65534,
SSRC: 123,
},
Payload: []byte{1},
},
{
Header: rtp.Header{
Version: 2,
PayloadType: 96,
SequenceNumber: 2,
SSRC: 123,
},
},
{
Header: rtp.Header{
Version: 2,
PayloadType: 96,
SequenceNumber: 65535,
SSRC: 123,
},
},
}
buff := NewBuffer(123, 1, 1)
buff.codecType = webrtc.RTPCodecTypeVideo
require.NotNil(t, buff)
buff.OnRtcpFeedback(func(_ []rtcp.Packet) {})
buff.Bind(webrtc.RTPParameters{
HeaderExtensions: nil,
Codecs: []webrtc.RTPCodecParameters{vp8Codec},
}, vp8Codec.RTPCodecCapability, 0)
for _, p := range TestPackets {
buf, _ := p.Marshal()
_, _ = buff.Write(buf)
}
require.Equal(t, uint16(2), buff.rtpStats.HighestSequenceNumber())
require.Equal(t, uint64(65536+2), buff.rtpStats.ExtendedHighestSequenceNumber())
})
}
}
func TestFractionLostReport(t *testing.T) {
buff := NewBuffer(123, 1, 1)
require.NotNil(t, buff)
var wg sync.WaitGroup
// with loss proxying
wg.Add(1)
buff.SetAudioLossProxying(true)
buff.SetLastFractionLostReport(55)
buff.OnRtcpFeedback(func(fb []rtcp.Packet) {
for _, pkt := range fb {
switch p := pkt.(type) {
case *rtcp.ReceiverReport:
for _, v := range p.Reports {
require.EqualValues(t, 55, v.FractionLost)
}
wg.Done()
}
}
})
buff.Bind(webrtc.RTPParameters{
HeaderExtensions: nil,
Codecs: []webrtc.RTPCodecParameters{opusCodec},
}, opusCodec.RTPCodecCapability, 0)
for i := 0; i < 15; i++ {
pkt := rtp.Packet{
Header: rtp.Header{
Version: 2,
PayloadType: 111,
SequenceNumber: uint16(i),
Timestamp: uint32(i),
SSRC: 123,
},
Payload: []byte{0xff, 0xff, 0xff, 0xfd, 0xb4, 0x9f, 0x94, 0x1},
}
b, err := pkt.Marshal()
require.NoError(t, err)
if i == 1 {
time.Sleep(1 * time.Second)
}
_, err = buff.Write(b)
require.NoError(t, err)
}
wg.Wait()
wg.Add(1)
buff.SetAudioLossProxying(false)
buff.OnRtcpFeedback(func(fb []rtcp.Packet) {
for _, pkt := range fb {
switch p := pkt.(type) {
case *rtcp.ReceiverReport:
for _, v := range p.Reports {
require.EqualValues(t, 0, v.FractionLost)
}
wg.Done()
}
}
})
buff.Bind(webrtc.RTPParameters{
HeaderExtensions: nil,
Codecs: []webrtc.RTPCodecParameters{opusCodec},
}, opusCodec.RTPCodecCapability, 0)
for i := 0; i < 15; i++ {
pkt := rtp.Packet{
Header: rtp.Header{
Version: 2,
PayloadType: 111,
SequenceNumber: uint16(i),
Timestamp: uint32(i),
SSRC: 123,
},
Payload: []byte{0xff, 0xff, 0xff, 0xfd, 0xb4, 0x9f, 0x94, 0x1},
}
b, err := pkt.Marshal()
require.NoError(t, err)
if i == 1 {
time.Sleep(1 * time.Second)
}
_, err = buff.Write(b)
require.NoError(t, err)
}
wg.Wait()
}
func TestCodecChange(t *testing.T) {
// codec change before bind
buff := NewBuffer(123, 1, 1)
require.NotNil(t, buff)
changedCodec := make(chan webrtc.RTPCodecParameters, 1)
buff.OnCodecChange(func(rp webrtc.RTPCodecParameters) {
select {
case changedCodec <- rp:
default:
t.Fatalf("codec change not consumed")
}
})
h265Pkt := rtp.Packet{
Header: rtp.Header{
Version: 2,
PayloadType: 116,
SequenceNumber: 1,
Timestamp: 1,
SSRC: 123,
},
Payload: []byte{0xff, 0xff, 0xff, 0xfd, 0xb4, 0x9f, 0x94, 0x1},
}
buf, err := h265Pkt.Marshal()
require.NoError(t, err)
_, err = buff.Write(buf)
require.NoError(t, err)
select {
case <-changedCodec:
t.Fatalf("unexpected codec change")
case <-time.After(100 * time.Millisecond):
}
buff.Bind(webrtc.RTPParameters{
HeaderExtensions: nil,
Codecs: []webrtc.RTPCodecParameters{vp8Codec, h265Codec},
}, vp8Codec.RTPCodecCapability, 0)
select {
case c := <-changedCodec:
require.Equal(t, h265Codec, c)
case <-time.After(1 * time.Second):
t.Fatalf("expected codec change")
}
// codec change after bind
vp8Pkt := rtp.Packet{
Header: rtp.Header{
Version: 2,
PayloadType: 96,
SequenceNumber: 3,
Timestamp: 3,
SSRC: 123,
},
Payload: []byte{0xff, 0xff, 0xff, 0xfd, 0xb4, 0x9f, 0x94, 0x1},
}
buf, err = vp8Pkt.Marshal()
require.NoError(t, err)
_, err = buff.Write(buf)
require.NoError(t, err)
select {
case c := <-changedCodec:
require.Equal(t, vp8Codec, c)
case <-time.After(1 * time.Second):
t.Fatalf("expected codec change")
}
// out of order pkts can't cause codec change
h265Pkt.SequenceNumber = 2
h265Pkt.Timestamp = 2
buf, err = h265Pkt.Marshal()
require.NoError(t, err)
_, err = buff.Write(buf)
require.NoError(t, err)
select {
case <-changedCodec:
t.Fatalf("unexpected codec change")
case <-time.After(100 * time.Millisecond):
}
// unknown codec should not cause change
h265Pkt.SequenceNumber = 4
h265Pkt.Timestamp = 4
h265Pkt.PayloadType = 117
buf, err = h265Pkt.Marshal()
require.NoError(t, err)
_, err = buff.Write(buf)
require.NoError(t, err)
select {
case <-changedCodec:
t.Fatalf("unexpected codec change")
case <-time.After(100 * time.Millisecond):
}
}
func BenchmarkMemcpu(b *testing.B) {
buf := make([]byte, 1500*1500*10)
buf2 := make([]byte, 1500*1500*20)
b.ResetTimer()
for i := 0; i < b.N; i++ {
copy(buf2, buf)
}
}
livekit-server/pkg/sfu/buffer/datastats.go
+102
View File
@@ -0,0 +1,102 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package buffer
import (
"sync"
"time"
"google.golang.org/protobuf/types/known/timestamppb"
"github.com/livekit/protocol/livekit"
)
type DataStatsParam struct {
WindowDuration time.Duration
}
type DataStats struct {
params DataStatsParam
lock sync.RWMutex
totalBytes int64
startTime time.Time
endTime time.Time
windowStart int64
windowBytes int64
}
func NewDataStats(params DataStatsParam) *DataStats {
return &DataStats{
params: params,
startTime: time.Now(),
windowStart: time.Now().UnixNano(),
}
}
func (s *DataStats) Update(bytes int, time int64) {
s.lock.Lock()
defer s.lock.Unlock()
s.totalBytes += int64(bytes)
if s.params.WindowDuration > 0 && time-s.windowStart > s.params.WindowDuration.Nanoseconds() {
s.windowBytes = 0
s.windowStart = time
}
s.windowBytes += int64(bytes)
}
func (s *DataStats) ToProtoActive() *livekit.RTPStats {
if s.params.WindowDuration == 0 {
return &livekit.RTPStats{}
}
s.lock.RLock()
defer s.lock.RUnlock()
now := time.Now().UnixNano()
duration := now - s.windowStart
if duration > s.params.WindowDuration.Nanoseconds() {
return &livekit.RTPStats{}
}
return &livekit.RTPStats{
StartTime: timestamppb.New(time.Unix(s.windowStart/1e9, s.windowStart%1e9)),
EndTime: timestamppb.New(time.Unix(0, now)),
Duration: float64(duration / 1e9),
Bytes: uint64(s.windowBytes),
Bitrate: float64(s.windowBytes) * 8 / float64(duration) / 1e9,
}
}
func (s *DataStats) Stop() {
s.lock.Lock()
s.endTime = time.Now()
s.lock.Unlock()
}
func (s *DataStats) ToProtoAggregateOnly() *livekit.RTPStats {
s.lock.RLock()
defer s.lock.RUnlock()
end := s.endTime
if end.IsZero() {
end = time.Now()
}
return &livekit.RTPStats{
StartTime: timestamppb.New(s.startTime),
EndTime: timestamppb.New(end),
Duration: end.Sub(s.startTime).Seconds(),
Bytes: uint64(s.totalBytes),
Bitrate: float64(s.totalBytes) * 8 / end.Sub(s.startTime).Seconds(),
}
}
livekit-server/pkg/sfu/buffer/datastats_test.go
+53
View File
@@ -0,0 +1,53 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package buffer
import (
"testing"
"time"
"github.com/stretchr/testify/require"
"google.golang.org/protobuf/proto"
"github.com/livekit/protocol/livekit"
)
func TestDataStats(t *testing.T) {
stats := NewDataStats(DataStatsParam{WindowDuration: time.Second})
time.Sleep(time.Millisecond)
r := stats.ToProtoAggregateOnly()
require.Equal(t, r.StartTime.AsTime().UnixNano(), stats.startTime.UnixNano())
require.NotZero(t, r.EndTime)
require.NotZero(t, r.Duration)
r.StartTime = nil
r.EndTime = nil
r.Duration = 0
require.True(t, proto.Equal(r, &livekit.RTPStats{}))
stats.Update(100, time.Now().UnixNano())
r = stats.ToProtoActive()
require.EqualValues(t, 100, r.Bytes)
require.NotZero(t, r.Bitrate)
// wait for window duration
time.Sleep(time.Second)
r = stats.ToProtoActive()
require.True(t, proto.Equal(r, &livekit.RTPStats{}))
stats.Stop()
r = stats.ToProtoAggregateOnly()
require.EqualValues(t, 100, r.Bytes)
require.NotZero(t, r.Bitrate)
}
livekit-server/pkg/sfu/buffer/dependencydescriptorparser.go
+218
View File
@@ -0,0 +1,218 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package buffer
import (
"fmt"
"sort"
"github.com/pion/rtp"
"go.uber.org/atomic"
dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor"
"github.com/livekit/livekit-server/pkg/sfu/utils"
"github.com/livekit/protocol/logger"
)
var (
ErrFrameEarlierThanKeyFrame = fmt.Errorf("frame is earlier than current keyframe")
ErrDDStructureAttachedToNonFirstPacket = fmt.Errorf("dependency descriptor structure is attached to non-first packet of a frame")
)
type DependencyDescriptorParser struct {
structure *dd.FrameDependencyStructure
ddExtID uint8
logger logger.Logger
onMaxLayerChanged func(int32, int32)
decodeTargets []DependencyDescriptorDecodeTarget
seqWrapAround *utils.WrapAround[uint16, uint64]
frameWrapAround *utils.WrapAround[uint16, uint64]
structureExtFrameNum uint64
activeDecodeTargetsExtSeq uint64
activeDecodeTargetsMask uint32
frameChecker *FrameIntegrityChecker
ddNotFoundCount atomic.Uint32
}
func NewDependencyDescriptorParser(ddExtID uint8, logger logger.Logger, onMaxLayerChanged func(int32, int32)) *DependencyDescriptorParser {
return &DependencyDescriptorParser{
ddExtID: ddExtID,
logger: logger,
onMaxLayerChanged: onMaxLayerChanged,
seqWrapAround: utils.NewWrapAround[uint16, uint64](utils.WrapAroundParams{IsRestartAllowed: false}),
frameWrapAround: utils.NewWrapAround[uint16, uint64](utils.WrapAroundParams{IsRestartAllowed: false}),
frameChecker: NewFrameIntegrityChecker(180, 1024), // 2seconds for L3T3 30fps video
}
}
type ExtDependencyDescriptor struct {
Descriptor *dd.DependencyDescriptor
DecodeTargets []DependencyDescriptorDecodeTarget
StructureUpdated bool
ActiveDecodeTargetsUpdated bool
Integrity bool
ExtFrameNum uint64
// the frame number of the keyframe which the current frame depends on
ExtKeyFrameNum uint64
}
func (r *DependencyDescriptorParser) Parse(pkt *rtp.Packet) (*ExtDependencyDescriptor, VideoLayer, error) {
var videoLayer VideoLayer
ddBuf := pkt.GetExtension(r.ddExtID)
if ddBuf == nil {
ddNotFoundCount := r.ddNotFoundCount.Inc()
if ddNotFoundCount%100 == 0 {
r.logger.Warnw("dependency descriptor extension is not present", nil, "seq", pkt.SequenceNumber, "count", ddNotFoundCount)
}
return nil, videoLayer, nil
}
var ddVal dd.DependencyDescriptor
ext := &dd.DependencyDescriptorExtension{
Descriptor: &ddVal,
Structure: r.structure,
}
_, err := ext.Unmarshal(ddBuf)
if err != nil {
if err != dd.ErrDDReaderNoStructure && err != dd.ErrDDReaderInvalidTemplateIndex {
r.logger.Infow("failed to parse generic dependency descriptor", err, "payload", pkt.PayloadType, "ddbufLen", len(ddBuf))
}
return nil, videoLayer, err
}
extSeq := r.seqWrapAround.Update(pkt.SequenceNumber).ExtendedVal
if ddVal.FrameDependencies != nil {
videoLayer.Spatial, videoLayer.Temporal = int32(ddVal.FrameDependencies.SpatialId), int32(ddVal.FrameDependencies.TemporalId)
}
unwrapped := r.frameWrapAround.Update(ddVal.FrameNumber)
extFN := unwrapped.ExtendedVal
if extFN < r.structureExtFrameNum {
r.logger.Debugw("drop frame which is earlier than current structure", "frameNum", extFN, "structureFrameNum", r.structureExtFrameNum)
return nil, videoLayer, ErrFrameEarlierThanKeyFrame
}
r.frameChecker.AddPacket(extSeq, extFN, &ddVal)
extDD := &ExtDependencyDescriptor{
Descriptor: &ddVal,
ExtFrameNum: extFN,
Integrity: r.frameChecker.FrameIntegrity(extFN),
}
if ddVal.AttachedStructure != nil {
if !ddVal.FirstPacketInFrame {
r.logger.Warnw("attached structure is not the first packet in frame", nil, "extSeq", extSeq, "extFN", extFN)
return nil, videoLayer, ErrDDStructureAttachedToNonFirstPacket
}
if r.structure == nil || ddVal.AttachedStructure.StructureId != r.structure.StructureId {
r.logger.Debugw("structure updated", "structureID", ddVal.AttachedStructure.StructureId, "extSeq", extSeq, "extFN", extFN, "descriptor", ddVal.String())
}
r.structure = ddVal.AttachedStructure
r.decodeTargets = ProcessFrameDependencyStructure(ddVal.AttachedStructure)
if extFN > unwrapped.PreExtendedHighest && extFN-unwrapped.PreExtendedHighest > 1000 {
r.logger.Debugw("large frame number jump on structure updating", "extFN", extFN, "preExtendedHighest", unwrapped.PreExtendedHighest, "structureExtFrameNum", r.structureExtFrameNum)
}
r.structureExtFrameNum = extFN
extDD.StructureUpdated = true
extDD.ActiveDecodeTargetsUpdated = true
// The dependency descriptor reader will always set ActiveDecodeTargetsBitmask for TemplateDependencyStructure is present,
// so don't need to notify max layer change here.
}
if mask := ddVal.ActiveDecodeTargetsBitmask; mask != nil && extSeq > r.activeDecodeTargetsExtSeq {
r.activeDecodeTargetsExtSeq = extSeq
if *mask != r.activeDecodeTargetsMask {
r.activeDecodeTargetsMask = *mask
extDD.ActiveDecodeTargetsUpdated = true
var maxSpatial, maxTemporal int32
for _, dt := range r.decodeTargets {
if *mask&(1<<dt.Target) != uint32(dd.DecodeTargetNotPresent) {
if maxSpatial < dt.Layer.Spatial {
maxSpatial = dt.Layer.Spatial
}
if maxTemporal < dt.Layer.Temporal {
maxTemporal = dt.Layer.Temporal
}
}
}
r.logger.Debugw("max layer changed", "maxSpatial", maxSpatial, "maxTemporal", maxTemporal)
r.onMaxLayerChanged(maxSpatial, maxTemporal)
}
}
extDD.DecodeTargets = r.decodeTargets
extDD.ExtKeyFrameNum = r.structureExtFrameNum
return extDD, videoLayer, nil
}
// ------------------------------------------------------------------------------
type DependencyDescriptorDecodeTarget struct {
Target int
Layer VideoLayer
}
func (dt *DependencyDescriptorDecodeTarget) String() string {
return fmt.Sprintf("DecodeTarget{t: %d, l: %+v}", dt.Target, dt.Layer)
}
// ------------------------------------------------------------------------------
func ProcessFrameDependencyStructure(structure *dd.FrameDependencyStructure) []DependencyDescriptorDecodeTarget {
decodeTargets := make([]DependencyDescriptorDecodeTarget, 0, structure.NumDecodeTargets)
for target := 0; target < structure.NumDecodeTargets; target++ {
layer := VideoLayer{Spatial: 0, Temporal: 0}
for _, t := range structure.Templates {
if t.DecodeTargetIndications[target] != dd.DecodeTargetNotPresent {
if layer.Spatial < int32(t.SpatialId) {
layer.Spatial = int32(t.SpatialId)
}
if layer.Temporal < int32(t.TemporalId) {
layer.Temporal = int32(t.TemporalId)
}
}
}
decodeTargets = append(decodeTargets, DependencyDescriptorDecodeTarget{target, layer})
}
// sort decode target layer by spatial and temporal from high to low
sort.Slice(decodeTargets, func(i, j int) bool {
return decodeTargets[i].Layer.GreaterThan(decodeTargets[j].Layer)
})
return decodeTargets
}
func GetActiveDecodeTargetBitmask(layer VideoLayer, decodeTargets []DependencyDescriptorDecodeTarget) *uint32 {
activeBitMask := uint32(0)
for _, dt := range decodeTargets {
if dt.Layer.Spatial <= layer.Spatial && dt.Layer.Temporal <= layer.Temporal {
activeBitMask |= 1 << dt.Target
}
}
return &activeBitMask
}
// ------------------------------------------------------------------------------
livekit-server/pkg/sfu/buffer/factory.go
+131
View File
@@ -0,0 +1,131 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package buffer
import (
"io"
"sync"
"github.com/pion/transport/v3/packetio"
)
type FactoryOfBufferFactory struct {
trackingPacketsVideo int
trackingPacketsAudio int
}
func NewFactoryOfBufferFactory(trackingPacketsVideo int, trackingPacketsAudio int) *FactoryOfBufferFactory {
return &FactoryOfBufferFactory{
trackingPacketsVideo: trackingPacketsVideo,
trackingPacketsAudio: trackingPacketsAudio,
}
}
func (f *FactoryOfBufferFactory) CreateBufferFactory() *Factory {
return &Factory{
trackingPacketsVideo: f.trackingPacketsVideo,
trackingPacketsAudio: f.trackingPacketsAudio,
rtpBuffers: make(map[uint32]*Buffer),
rtcpReaders: make(map[uint32]*RTCPReader),
rtxPair: make(map[uint32]uint32),
}
}
type Factory struct {
sync.RWMutex
trackingPacketsVideo int
trackingPacketsAudio int
rtpBuffers map[uint32]*Buffer
rtcpReaders map[uint32]*RTCPReader
rtxPair map[uint32]uint32 // repair -> base
}
func (f *Factory) GetOrNew(packetType packetio.BufferPacketType, ssrc uint32) io.ReadWriteCloser {
f.Lock()
defer f.Unlock()
switch packetType {
case packetio.RTCPBufferPacket:
if reader, ok := f.rtcpReaders[ssrc]; ok {
return reader
}
reader := NewRTCPReader(ssrc)
f.rtcpReaders[ssrc] = reader
reader.OnClose(func() {
f.Lock()
delete(f.rtcpReaders, ssrc)
f.Unlock()
})
return reader
case packetio.RTPBufferPacket:
if reader, ok := f.rtpBuffers[ssrc]; ok {
return reader
}
buffer := NewBuffer(ssrc, f.trackingPacketsVideo, f.trackingPacketsAudio)
f.rtpBuffers[ssrc] = buffer
for repair, base := range f.rtxPair {
if repair == ssrc {
baseBuffer, ok := f.rtpBuffers[base]
if ok {
buffer.SetPrimaryBufferForRTX(baseBuffer)
}
break
} else if base == ssrc {
repairBuffer, ok := f.rtpBuffers[repair]
if ok {
repairBuffer.SetPrimaryBufferForRTX(buffer)
}
break
}
}
buffer.OnClose(func() {
f.Lock()
delete(f.rtpBuffers, ssrc)
delete(f.rtxPair, ssrc)
f.Unlock()
})
return buffer
}
return nil
}
func (f *Factory) GetBufferPair(ssrc uint32) (*Buffer, *RTCPReader) {
f.RLock()
defer f.RUnlock()
return f.rtpBuffers[ssrc], f.rtcpReaders[ssrc]
}
func (f *Factory) GetBuffer(ssrc uint32) *Buffer {
f.RLock()
defer f.RUnlock()
return f.rtpBuffers[ssrc]
}
func (f *Factory) GetRTCPReader(ssrc uint32) *RTCPReader {
f.RLock()
defer f.RUnlock()
return f.rtcpReaders[ssrc]
}
func (f *Factory) SetRTXPair(repair, base uint32) {
f.Lock()
repairBuffer, baseBuffer := f.rtpBuffers[repair], f.rtpBuffers[base]
if repairBuffer == nil || baseBuffer == nil {
f.rtxPair[repair] = base
}
f.Unlock()
if repairBuffer != nil && baseBuffer != nil {
repairBuffer.SetPrimaryBufferForRTX(baseBuffer)
}
}
livekit-server/pkg/sfu/buffer/fps.go
+732
View File
@@ -0,0 +1,732 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package buffer
import (
"container/list"
"github.com/pion/rtp/codecs"
"github.com/livekit/protocol/logger"
)
var minFramesForCalculation = [...]int{8, 15, 40, 60}
type frameInfo struct {
startSeq uint16
endSeq uint16
ts uint32
fn uint16
spatial int32
temporal int32
frameDiff []int
}
type FrameRateCalculator interface {
RecvPacket(ep *ExtPacket) bool
GetFrameRate() []float32
Completed() bool
}
// -----------------------------
// FrameRateCalculator based on PictureID in VPx
type frameRateCalculatorVPx struct {
frameRates [DefaultMaxLayerTemporal + 1]float32
clockRate uint32
logger logger.Logger
firstFrames [DefaultMaxLayerTemporal + 1]*frameInfo
secondFrames [DefaultMaxLayerTemporal + 1]*frameInfo
fnReceived [64]*frameInfo
baseFrame *frameInfo
completed bool
}
func newFrameRateCalculatorVPx(clockRate uint32, logger logger.Logger) *frameRateCalculatorVPx {
return &frameRateCalculatorVPx{
clockRate: clockRate,
logger: logger,
}
}
func (f *frameRateCalculatorVPx) Completed() bool {
return f.completed
}
func (f *frameRateCalculatorVPx) RecvPacket(ep *ExtPacket, fn uint16) bool {
if f.completed {
return true
}
if ep.Temporal >= int32(len(f.frameRates)) {
f.logger.Warnw("invalid temporal layer", nil, "temporal", ep.Temporal)
return false
}
temporal := ep.Temporal
if temporal < 0 {
temporal = 0
}
if f.baseFrame == nil {
f.baseFrame = &frameInfo{ts: ep.Packet.Timestamp, fn: fn}
f.fnReceived[0] = f.baseFrame
f.firstFrames[temporal] = f.baseFrame
return false
}
baseDiff := fn - f.baseFrame.fn
if baseDiff == 0 || baseDiff > 0x4000 {
return false
}
if baseDiff >= uint16(len(f.fnReceived)) {
// frame number is not continuous, reset
f.reset()
return false
}
if f.fnReceived[baseDiff] != nil {
return false
}
fi := &frameInfo{
ts: ep.Packet.Timestamp,
fn: fn,
temporal: temporal,
}
f.fnReceived[baseDiff] = fi
firstFrame := f.firstFrames[temporal]
secondFrame := f.secondFrames[temporal]
if firstFrame == nil {
f.firstFrames[temporal] = fi
firstFrame = fi
} else {
if (secondFrame == nil || secondFrame.fn < fn) && fn != firstFrame.fn && (fn-firstFrame.fn) < 0x4000 {
f.secondFrames[temporal] = fi
}
}
return f.calc()
}
func (f *frameRateCalculatorVPx) calc() bool {
var rateCounter int
for currentTemporal := int32(0); currentTemporal <= DefaultMaxLayerTemporal; currentTemporal++ {
if f.frameRates[currentTemporal] > 0 {
rateCounter++
continue
}
ff := f.firstFrames[currentTemporal]
sf := f.secondFrames[currentTemporal]
// lower temporal layer has been calculated, but higher layer has not received any frames, it should not exist
if rateCounter > 0 && ff == nil {
rateCounter++
continue
}
if ff == nil || sf == nil {
continue
}
var frameCount int
lastTs := ff.ts
for j := ff.fn - f.baseFrame.fn + 1; j < sf.fn-f.baseFrame.fn+1; j++ {
if f := f.fnReceived[j]; f == nil {
break
} else if f.temporal <= currentTemporal {
frameCount++
lastTs = f.ts
}
}
if frameCount >= minFramesForCalculation[currentTemporal] {
f.frameRates[currentTemporal] = float32(f.clockRate) / float32(lastTs-ff.ts) * float32(frameCount)
rateCounter++
}
}
if rateCounter == len(f.frameRates) {
f.completed = true
// normalize frame rates, Microsoft Edge use 3 temporal layers for vp8 but the middle layer has chance to
// get a very low frame rate, so we need to normalize the frame rate(use fixed ration 1:2 of highest layer for that layer)
if f.frameRates[2] > 0 && f.frameRates[2] > f.frameRates[1]*3 {
f.frameRates[1] = f.frameRates[2] / 2
}
f.reset()
return true
}
return false
}
func (f *frameRateCalculatorVPx) reset() {
for i := range f.firstFrames {
f.firstFrames[i] = nil
f.secondFrames[i] = nil
}
for i := range f.fnReceived {
f.fnReceived[i] = nil
}
f.baseFrame = nil
}
func (f *frameRateCalculatorVPx) GetFrameRate() []float32 {
return f.frameRates[:]
}
// -----------------------------
// FrameRateCalculator based on PictureID in VP8
type FrameRateCalculatorVP8 struct {
*frameRateCalculatorVPx
logger logger.Logger
}
func NewFrameRateCalculatorVP8(clockRate uint32, logger logger.Logger) *FrameRateCalculatorVP8 {
return &FrameRateCalculatorVP8{
frameRateCalculatorVPx: newFrameRateCalculatorVPx(clockRate, logger),
logger: logger,
}
}
func (f *FrameRateCalculatorVP8) RecvPacket(ep *ExtPacket) bool {
if f.frameRateCalculatorVPx.Completed() {
return true
}
vp8, ok := ep.Payload.(VP8)
if !ok {
f.logger.Debugw("no vp8 payload", "sn", ep.Packet.SequenceNumber)
return false
}
success := f.frameRateCalculatorVPx.RecvPacket(ep, vp8.PictureID)
if f.frameRateCalculatorVPx.Completed() {
f.logger.Debugw("frame rate calculated", "rate", f.frameRateCalculatorVPx.GetFrameRate())
}
return success
}
// -----------------------------
// FrameRateCalculator based on PictureID in VP9
type FrameRateCalculatorVP9 struct {
logger logger.Logger
completed bool
// VP9-TODO - this is assuming three spatial layers. As `completed` marker relies on all layers being finished, have to assume this. FIX.
// Maybe look at number of layers in livekit.TrackInfo and declare completed once advertised layers are measured
frameRateCalculatorsVPx [DefaultMaxLayerSpatial + 1]*frameRateCalculatorVPx
}
func NewFrameRateCalculatorVP9(clockRate uint32, logger logger.Logger) *FrameRateCalculatorVP9 {
f := &FrameRateCalculatorVP9{
logger: logger,
}
for i := range f.frameRateCalculatorsVPx {
f.frameRateCalculatorsVPx[i] = newFrameRateCalculatorVPx(clockRate, logger)
}
return f
}
func (f *FrameRateCalculatorVP9) Completed() bool {
return f.completed
}
func (f *FrameRateCalculatorVP9) RecvPacket(ep *ExtPacket) bool {
if f.completed {
return true
}
vp9, ok := ep.Payload.(codecs.VP9Packet)
if !ok {
f.logger.Debugw("no vp9 payload", "sn", ep.Packet.SequenceNumber)
return false
}
if ep.Spatial < 0 || ep.Spatial >= int32(len(f.frameRateCalculatorsVPx)) || f.frameRateCalculatorsVPx[ep.Spatial] == nil {
f.logger.Debugw("invalid spatial layer", "sn", ep.Packet.SequenceNumber, "spatial", ep.Spatial)
return false
}
success := f.frameRateCalculatorsVPx[ep.Spatial].RecvPacket(ep, vp9.PictureID)
completed := true
for _, frc := range f.frameRateCalculatorsVPx {
if !frc.Completed() {
completed = false
break
}
}
if completed {
f.completed = true
var frameRates [DefaultMaxLayerSpatial + 1][]float32
for i := range f.frameRateCalculatorsVPx {
frameRates[i] = f.frameRateCalculatorsVPx[i].GetFrameRate()
}
f.logger.Debugw("frame rate calculated", "rate", frameRates)
}
return success
}
func (f *FrameRateCalculatorVP9) GetFrameRateForSpatial(spatial int32) []float32 {
if spatial < 0 || spatial >= int32(len(f.frameRateCalculatorsVPx)) || f.frameRateCalculatorsVPx[spatial] == nil {
return nil
}
return f.frameRateCalculatorsVPx[spatial].GetFrameRate()
}
func (f *FrameRateCalculatorVP9) GetFrameRateCalculatorForSpatial(spatial int32) *FrameRateCalculatorForVP9Layer {
return &FrameRateCalculatorForVP9Layer{
FrameRateCalculatorVP9: f,
spatial: spatial,
}
}
// -----------------------------
type FrameRateCalculatorForVP9Layer struct {
*FrameRateCalculatorVP9
spatial int32
}
func (f *FrameRateCalculatorForVP9Layer) GetFrameRate() []float32 {
return f.FrameRateCalculatorVP9.GetFrameRateForSpatial(f.spatial)
}
// -----------------------------------------------
// FrameRateCalculator based on Dependency descriptor
type FrameRateCalculatorDD struct {
frameRates [DefaultMaxLayerSpatial + 1][DefaultMaxLayerTemporal + 1]float32
clockRate uint32
logger logger.Logger
firstFrames [DefaultMaxLayerSpatial + 1][DefaultMaxLayerTemporal + 1]*frameInfo
secondFrames [DefaultMaxLayerSpatial + 1][DefaultMaxLayerTemporal + 1]*frameInfo
fnReceived [256]*frameInfo
baseFrame *frameInfo
completed bool
// frames for each decode target
targetFrames [DefaultMaxLayerSpatial + 1][DefaultMaxLayerTemporal + 1]list.List
maxSpatial, maxTemporal int32
}
func NewFrameRateCalculatorDD(clockRate uint32, logger logger.Logger) *FrameRateCalculatorDD {
return &FrameRateCalculatorDD{
clockRate: clockRate,
logger: logger,
maxSpatial: DefaultMaxLayerSpatial,
maxTemporal: DefaultMaxLayerTemporal,
}
}
func (f *FrameRateCalculatorDD) Completed() bool {
return f.completed
}
func (f *FrameRateCalculatorDD) SetMaxLayer(spatial, temporal int32) {
f.maxSpatial, f.maxTemporal = spatial, temporal
}
func (f *FrameRateCalculatorDD) RecvPacket(ep *ExtPacket) bool {
if f.completed {
return true
}
if ep.DependencyDescriptor == nil {
f.logger.Debugw("dependency descriptor is nil")
return false
}
spatial := ep.Spatial
// non-SVC codec will set spatial to -1
if spatial < 0 {
spatial = 0
}
temporal := ep.Temporal
if temporal < 0 || temporal > DefaultMaxLayerTemporal || spatial > DefaultMaxLayerSpatial {
f.logger.Warnw("invalid spatial or temporal", nil, "spatial", spatial, "temporal", temporal, "sn", ep.Packet.SequenceNumber)
return false
}
fn := ep.DependencyDescriptor.Descriptor.FrameNumber
if f.baseFrame == nil {
f.baseFrame = &frameInfo{ts: ep.Packet.Timestamp, fn: fn}
f.fnReceived[0] = f.baseFrame
f.firstFrames[spatial][temporal] = f.baseFrame
f.secondFrames[spatial][temporal] = f.baseFrame
return false
}
baseDiff := fn - f.baseFrame.fn
if baseDiff == 0 || baseDiff > 0x8000 {
return false
}
if baseDiff >= uint16(len(f.fnReceived)) {
// frame number is not continuous, reset
f.baseFrame = nil
for i := range f.firstFrames {
for j := range f.firstFrames[i] {
f.firstFrames[i][j] = nil
f.secondFrames[i][j] = nil
f.targetFrames[i][j].Init()
}
}
for i := range f.fnReceived {
f.fnReceived[i] = nil
}
return false
}
if f.fnReceived[baseDiff] != nil {
return false
}
fi := &frameInfo{
ts: ep.Packet.Timestamp,
fn: fn,
temporal: temporal,
spatial: spatial,
frameDiff: ep.DependencyDescriptor.Descriptor.FrameDependencies.FrameDiffs,
}
f.fnReceived[baseDiff] = fi
if f.firstFrames[spatial][temporal] == nil {
f.firstFrames[spatial][temporal] = fi
f.secondFrames[spatial][temporal] = fi
return false
}
chain := &f.targetFrames[spatial][temporal]
if chain.Len() == 0 {
chain.PushBack(fn)
}
for _, fdiff := range ep.DependencyDescriptor.Descriptor.FrameDependencies.FrameDiffs {
dependFrame := fn - uint16(fdiff)
// frame too old, ignore
if dependFrame-f.secondFrames[spatial][temporal].fn > 0x8000 {
continue
}
insertFrame:
for e := chain.Back(); e != nil; e = e.Prev() {
val := e.Value.(uint16)
switch {
case val == dependFrame:
break insertFrame
case sn16LT(val, dependFrame):
chain.InsertAfter(dependFrame, e)
break insertFrame
default:
if e == chain.Front() {
chain.PushFront(dependFrame)
break insertFrame
}
}
}
}
return f.calc()
}
func (f *FrameRateCalculatorDD) calc() bool {
var rateCounter int
for currentSpatial := int32(0); currentSpatial <= f.maxSpatial; currentSpatial++ {
var currentSpatialRateCounter int
for currentTemporal := int32(0); currentTemporal <= f.maxTemporal; currentTemporal++ {
if f.frameRates[currentSpatial][currentTemporal] > 0 {
rateCounter++
currentSpatialRateCounter++
continue
}
firstFrame := f.firstFrames[currentSpatial][currentTemporal]
// lower temporal layer has been calculated, but higher layer has not received any frames, it should not exist
if currentSpatialRateCounter > 0 && firstFrame == nil {
currentSpatialRateCounter++
rateCounter++
continue
}
chain := &f.targetFrames[currentSpatial][currentTemporal]
// find last decodable frame (no dependency frame is lost)
var lastFrame *frameInfo
for e := chain.Front(); e != nil; e = e.Next() {
diff := e.Value.(uint16) - f.baseFrame.fn
if diff >= uint16(len(f.fnReceived)) {
continue
}
fi := f.fnReceived[diff]
if fi == nil {
break
} else {
lastFrame = fi
if firstFrame == nil && fi.spatial == currentSpatial && fi.temporal == currentTemporal {
firstFrame = fi
}
}
}
if lastFrame != nil && lastFrame.fn > f.secondFrames[currentSpatial][currentTemporal].fn {
f.secondFrames[currentSpatial][currentTemporal] = lastFrame
} else {
continue
}
frameCount := 0
for i := firstFrame.fn - f.baseFrame.fn; i <= lastFrame.fn-f.baseFrame.fn; i++ {
fi := f.fnReceived[i]
if fi == nil {
continue
}
if fi.spatial == currentSpatial && fi.temporal <= currentTemporal {
frameCount++
}
}
if frameCount >= minFramesForCalculation[currentTemporal] && lastFrame.ts > firstFrame.ts {
f.frameRates[currentSpatial][currentTemporal] = float32(f.clockRate) / float32(lastFrame.ts-firstFrame.ts) * float32(frameCount)
rateCounter++
}
}
}
if rateCounter == int(f.maxSpatial+1)*int(f.maxTemporal+1) {
f.completed = true
f.close()
f.logger.Debugw("frame rate calculated", "rate", f.frameRates)
return true
}
return false
}
func (f *FrameRateCalculatorDD) GetFrameRateForSpatial(spatial int32) []float32 {
if spatial < 0 || spatial >= int32(len(f.frameRates)) {
return nil
}
return f.frameRates[spatial][:]
}
func (f *FrameRateCalculatorDD) close() {
f.baseFrame = nil
for i := range f.firstFrames {
for j := range f.firstFrames[i] {
f.firstFrames[i][j] = nil
f.secondFrames[i][j] = nil
}
}
for i := range f.fnReceived {
f.fnReceived[i] = nil
}
for i := range f.targetFrames {
for j := range f.targetFrames[i] {
f.targetFrames[i][j].Init()
}
}
}
func (f *FrameRateCalculatorDD) GetFrameRateCalculatorForSpatial(spatial int32) *FrameRateCalculatorForDDLayer {
return &FrameRateCalculatorForDDLayer{
FrameRateCalculatorDD: f,
spatial: spatial,
}
}
// -----------------------------------------------
type FrameRateCalculatorForDDLayer struct {
*FrameRateCalculatorDD
spatial int32
}
func (f *FrameRateCalculatorForDDLayer) GetFrameRate() []float32 {
return f.FrameRateCalculatorDD.GetFrameRateForSpatial(f.spatial)
}
// -----------------------------------------------
type FrameRateCalculatorH26x struct {
frameRates [DefaultMaxLayerTemporal + 1]float32
clockRate uint32
logger logger.Logger
fnReceived *list.List
baseFrame *frameInfo
completed bool
}
func NewFrameRateCalculatorH26x(clockRate uint32, logger logger.Logger) *FrameRateCalculatorH26x {
return &FrameRateCalculatorH26x{
clockRate: clockRate,
logger: logger,
}
}
func (f *FrameRateCalculatorH26x) Completed() bool {
return f.completed
}
func (f *FrameRateCalculatorH26x) RecvPacket(ep *ExtPacket) bool {
if f.completed {
return true
}
if ep.Temporal >= int32(len(f.frameRates)) {
f.logger.Warnw("invalid temporal layer", nil, "temporal", ep.Temporal)
return false
}
temporal := ep.Temporal
if temporal < 0 {
temporal = 0
}
if f.baseFrame == nil {
f.baseFrame = &frameInfo{
startSeq: ep.Packet.SequenceNumber,
endSeq: ep.Packet.SequenceNumber,
ts: ep.Packet.Timestamp,
temporal: temporal,
}
f.fnReceived = list.New()
f.fnReceived.PushBack(f.baseFrame)
return false
}
if sn16LTOrEqual(ep.Packet.SequenceNumber, f.baseFrame.startSeq) {
return false
}
insertFrame:
for e := f.fnReceived.Back(); e != nil; e = e.Prev() {
frame := e.Value.(*frameInfo)
switch {
case frame.ts == ep.Packet.Timestamp:
if sn16LT(frame.endSeq, ep.Packet.SequenceNumber) {
frame.endSeq = ep.Packet.SequenceNumber
}
if sn16LT(ep.Packet.SequenceNumber, frame.startSeq) {
frame.startSeq = ep.Packet.SequenceNumber
}
break insertFrame
case sn32LT(frame.ts, ep.Packet.Timestamp):
f.fnReceived.InsertAfter(&frameInfo{
startSeq: ep.Packet.SequenceNumber,
endSeq: ep.Packet.SequenceNumber,
ts: ep.Packet.Timestamp,
temporal: temporal,
}, e)
break insertFrame
default:
if e == f.fnReceived.Front() {
f.fnReceived.PushFront(&frameInfo{
startSeq: ep.Packet.SequenceNumber,
endSeq: ep.Packet.SequenceNumber,
ts: ep.Packet.Timestamp,
temporal: temporal,
})
break insertFrame
}
}
}
return f.calc()
}
func (f *FrameRateCalculatorH26x) calc() bool {
frameCounts := make([]int, DefaultMaxLayerTemporal+1)
var totalFrameCount int
var tsDuration int
cur := f.fnReceived.Front()
for {
next := cur.Next()
if next == nil {
break
}
ff := cur.Value.(*frameInfo)
nf := next.Value.(*frameInfo)
if nf.startSeq-ff.endSeq == 1 {
totalFrameCount++
tsDuration += int(nf.ts - ff.ts)
for i := int(nf.temporal); i < len(frameCounts); i++ {
frameCounts[i]++
}
} else {
// reset to find continuous frames
totalFrameCount = 0
for i := range frameCounts {
frameCounts[i] = 0
}
tsDuration = 0
}
// received enough continuous frames, calculate fps
if totalFrameCount >= minFramesForCalculation[DefaultMaxLayerTemporal] {
for currentTemporal := int32(0); currentTemporal <= DefaultMaxLayerTemporal; currentTemporal++ {
count := frameCounts[currentTemporal]
if currentTemporal > 0 && count == frameCounts[currentTemporal-1] {
// no frames for this temporal layer
f.frameRates[currentTemporal] = 0
} else {
f.frameRates[currentTemporal] = float32(f.clockRate) / float32(tsDuration) * float32(count)
}
}
f.logger.Debugw("fps changed", "fps", f.GetFrameRate())
f.completed = true
f.reset()
return true
}
cur = next
}
return false
}
func (f *FrameRateCalculatorH26x) reset() {
f.fnReceived.Init()
f.baseFrame = nil
}
func (f *FrameRateCalculatorH26x) GetFrameRate() []float32 {
return f.frameRates[:]
}
// -----------------------------------------------
func sn16LT(a, b uint16) bool {
return a-b > 0x8000
}
func sn16LTOrEqual(a, b uint16) bool {
return a == b || a-b > 0x8000
}
func sn32LT(a, b uint32) bool {
return a-b > 0x80000000
}
livekit-server/pkg/sfu/buffer/fps_test.go
+437
View File
@@ -0,0 +1,437 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package buffer
import (
"testing"
"github.com/pion/rtp"
"github.com/stretchr/testify/require"
dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor"
"github.com/livekit/protocol/logger"
)
type testFrameInfo struct {
header rtp.Header
framenumber uint16
spatial int
temporal int
frameDiff []int
}
func (f *testFrameInfo) toVP8() *ExtPacket {
return &ExtPacket{
Packet: &rtp.Packet{Header: f.header},
Payload: VP8{
PictureID: f.framenumber,
},
VideoLayer: VideoLayer{Spatial: InvalidLayerSpatial, Temporal: int32(f.temporal)},
}
}
func (f *testFrameInfo) toDD() *ExtPacket {
return &ExtPacket{
Packet: &rtp.Packet{Header: f.header},
DependencyDescriptor: &ExtDependencyDescriptor{
Descriptor: &dd.DependencyDescriptor{
FrameNumber: f.framenumber,
FrameDependencies: &dd.FrameDependencyTemplate{
FrameDiffs: f.frameDiff,
},
},
},
VideoLayer: VideoLayer{Spatial: int32(f.spatial), Temporal: int32(f.temporal)},
}
}
func (f *testFrameInfo) toH26x() *ExtPacket {
return &ExtPacket{
Packet: &rtp.Packet{Header: f.header},
VideoLayer: VideoLayer{Spatial: InvalidLayerSpatial, Temporal: int32(f.temporal)},
}
}
func createFrames(startFrameNumber uint16, startTs uint32, startSeq uint16, totalFramesPerSpatial int, fps [][]float32, spatialDependency bool) [][]*testFrameInfo {
spatials := len(fps)
temporals := len(fps[0])
frames := make([][]*testFrameInfo, spatials)
for s := 0; s < spatials; s++ {
frames[s] = make([]*testFrameInfo, 0, totalFramesPerSpatial)
}
fn := startFrameNumber
nextTs := make([][]uint32, spatials)
tsStep := make([][]uint32, spatials)
for i := 0; i < spatials; i++ {
nextTs[i] = make([]uint32, temporals)
tsStep[i] = make([]uint32, temporals)
for j := 0; j < temporals; j++ {
nextTs[i][j] = startTs
tsStep[i][j] = uint32(90000 / fps[i][j])
}
}
currentTs := make([]uint32, spatials)
for i := 0; i < spatials; i++ {
currentTs[i] = startTs
}
for i := 0; i < totalFramesPerSpatial; i++ {
for s := 0; s < spatials; s++ {
frame := &testFrameInfo{
header: rtp.Header{Timestamp: currentTs[s], SequenceNumber: startSeq},
framenumber: fn,
spatial: s,
}
for t := 0; t < temporals; t++ {
if currentTs[s] >= nextTs[s][t] {
frame.temporal = t
for nt := t; nt < temporals; nt++ {
nextTs[s][nt] += tsStep[s][nt]
}
break
}
}
currentTs[s] += tsStep[s][temporals-1]
frames[s] = append(frames[s], frame)
fn++
startSeq++
for fidx := len(frames[s]) - 1; fidx >= 0; fidx-- {
cf := frames[s][fidx]
if cf.header.Timestamp-frame.header.Timestamp > 0x80000000 {
frame.frameDiff = append(frame.frameDiff, int(frame.framenumber-cf.framenumber))
break
}
}
if spatialDependency && frame.spatial > 0 {
for fidx := len(frames[frame.spatial-1]) - 1; fidx >= 0; fidx-- {
cf := frames[frame.spatial-1][fidx]
if cf.header.Timestamp == frame.header.Timestamp {
frame.frameDiff = append(frame.frameDiff, int(frame.framenumber-cf.framenumber))
break
}
}
}
}
}
return frames
}
func verifyFps(t *testing.T, expect, got []float32) {
require.Equal(t, len(expect), len(got))
for i := 0; i < len(expect); i++ {
require.GreaterOrEqual(t, got[i], expect[i]*0.9, "expect %v, got %v", expect, got)
require.LessOrEqual(t, got[i], expect[i]*1.1, "expect %v, got %v", expect, got)
}
}
type testcase struct {
startTs uint32
startSeq uint16
startFrameNumber uint16
fps [][]float32
spatialDependency bool
}
func TestFpsVP8(t *testing.T) {
cases := map[string]testcase{
"normal": {
startTs: 12345678,
startFrameNumber: 100,
fps: [][]float32{{5, 10, 15}, {5, 10, 15}, {7.5, 15, 30}},
},
"frame number and timestamp wrap": {
startTs: (uint32(1) << 31) - 10,
startFrameNumber: (uint16(1) << 15) - 10,
fps: [][]float32{{5, 10, 15}, {5, 10, 15}, {7.5, 15, 30}},
},
"2 temporal layers": {
startTs: 12345678,
startFrameNumber: 100,
fps: [][]float32{{7.5, 15}, {7.5, 15}, {15, 30}},
},
}
for name, c := range cases {
testCase := c
t.Run(name, func(t *testing.T) {
fps := testCase.fps
frames := make([][]*testFrameInfo, 0)
vp8calcs := make([]*FrameRateCalculatorVP8, len(fps))
for i := range vp8calcs {
vp8calcs[i] = NewFrameRateCalculatorVP8(90000, logger.GetLogger())
frames = append(frames, createFrames(c.startFrameNumber, c.startTs, 10, 200, [][]float32{fps[i]}, false)[0])
}
var frameratesGot bool
for s, fs := range frames {
for _, f := range fs {
if vp8calcs[s].RecvPacket(f.toVP8()) {
frameratesGot = true
for _, calc := range vp8calcs {
if !calc.Completed() {
frameratesGot = false
break
}
}
}
}
}
require.True(t, frameratesGot)
for i, calc := range vp8calcs {
fpsExpected := fps[i]
fpsGot := calc.GetFrameRate()
verifyFps(t, fpsExpected, fpsGot[:len(fpsExpected)])
}
})
}
t.Run("packet lost and duplicate", func(t *testing.T) {
fps := [][]float32{{7.5, 15}, {7.5, 15}, {15, 30}}
frames := make([][]*testFrameInfo, 0)
vp8calcs := make([]*FrameRateCalculatorVP8, len(fps))
for i := range vp8calcs {
vp8calcs[i] = NewFrameRateCalculatorVP8(90000, logger.GetLogger())
frames = append(frames, createFrames(100, 12345678, 10, 300, [][]float32{fps[i]}, false)[0])
for j := 5; j < 130; j++ {
if j%2 == 0 {
frames[i][j] = frames[i][j-1]
}
}
}
var frameratesGot bool
for s, fs := range frames {
for _, f := range fs {
if vp8calcs[s].RecvPacket(f.toVP8()) {
frameratesGot = true
for _, calc := range vp8calcs {
if !calc.Completed() {
frameratesGot = false
break
}
}
}
}
}
require.True(t, frameratesGot)
for i, calc := range vp8calcs {
fpsExpected := fps[i]
fpsGot := calc.GetFrameRate()
verifyFps(t, fpsExpected, fpsGot[:len(fpsExpected)])
}
})
}
func TestFpsDD(t *testing.T) {
cases := map[string]testcase{
"normal": {
startTs: 12345678,
startFrameNumber: 100,
fps: [][]float32{{5.1, 10.1, 16}, {5.1, 10.1, 16}, {8, 15, 30.1}},
spatialDependency: true,
},
"frame number and timestamp wrap": {
startTs: (uint32(1) << 31) - 10,
startFrameNumber: (uint16(1) << 15) - 10,
fps: [][]float32{{7.5, 15, 30}, {7.5, 15, 30}, {7.5, 15, 30}},
spatialDependency: true,
},
"vp8": {
startTs: 12345678,
startFrameNumber: 100,
fps: [][]float32{{7.5, 15}, {7.5, 15}, {15, 30}},
spatialDependency: false,
},
}
for name, c := range cases {
testCase := c
t.Run(name, func(t *testing.T) {
fps := testCase.fps
frames := createFrames(c.startFrameNumber, c.startTs, 10, 500, fps, testCase.spatialDependency)
ddcalc := NewFrameRateCalculatorDD(90000, logger.GetLogger())
ddcalc.SetMaxLayer(int32(len(fps)-1), int32(len(fps[0])-1))
ddcalcs := make([]FrameRateCalculator, len(fps))
for i := range fps {
ddcalcs[i] = ddcalc.GetFrameRateCalculatorForSpatial(int32(i))
}
var frameratesGot bool
for s, fs := range frames {
for _, f := range fs {
if ddcalcs[s].RecvPacket(f.toDD()) {
frameratesGot = true
for _, calc := range ddcalcs {
if !calc.Completed() {
frameratesGot = false
break
}
}
}
}
}
require.True(t, frameratesGot)
for i, calc := range ddcalcs {
fpsExpected := fps[i]
fpsGot := calc.GetFrameRate()
verifyFps(t, fpsExpected, fpsGot[:len(fpsExpected)])
}
})
}
t.Run("packet lost and duplicate", func(t *testing.T) {
fps := [][]float32{{7.5, 15, 30}, {7.5, 15, 30}, {7.5, 15, 30}}
frames := createFrames(100, 12345678, 10, 500, fps, true)
ddcalc := NewFrameRateCalculatorDD(90000, logger.GetLogger())
ddcalc.SetMaxLayer(int32(len(fps)-1), int32(len(fps[0])-1))
ddcalcs := make([]FrameRateCalculator, len(fps))
for i := range fps {
ddcalcs[i] = ddcalc.GetFrameRateCalculatorForSpatial(int32(i))
for j := 5; j < 130; j++ {
if j%2 == 0 {
frames[i][j] = frames[i][j-1]
}
}
}
var frameratesGot bool
for s, fs := range frames {
for _, f := range fs {
if ddcalcs[s].RecvPacket(f.toDD()) {
frameratesGot = true
for _, calc := range ddcalcs {
if !calc.Completed() {
frameratesGot = false
break
}
}
}
}
}
require.True(t, frameratesGot)
for i, calc := range ddcalcs {
fpsExpected := fps[i]
fpsGot := calc.GetFrameRate()
verifyFps(t, fpsExpected, fpsGot[:len(fpsExpected)])
}
})
}
func TestFpsH26x(t *testing.T) {
cases := map[string]testcase{
"normal": {
startTs: 12345678,
startSeq: 100,
startFrameNumber: 100,
fps: [][]float32{{5, 10, 15}, {5, 10, 15}, {7.5, 15, 30}},
},
"frame number and timestamp wrap": {
startTs: (uint32(1) << 31) - 10,
startSeq: (uint16(1) << 15) - 10,
startFrameNumber: (uint16(1) << 15) - 10,
fps: [][]float32{{5, 10, 15}, {5, 10, 15}, {7.5, 15, 30}},
},
"2 temporal layers": {
startTs: 12345678,
startFrameNumber: 100,
fps: [][]float32{{7.5, 15}, {7.5, 15}, {15, 30}},
},
}
for name, c := range cases {
testCase := c
t.Run(name, func(t *testing.T) {
fps := testCase.fps
frames := make([][]*testFrameInfo, 0)
h26xcalcs := make([]*FrameRateCalculatorH26x, len(fps))
for i := range h26xcalcs {
h26xcalcs[i] = NewFrameRateCalculatorH26x(90000, logger.GetLogger())
frames = append(frames, createFrames(c.startFrameNumber, c.startTs, c.startSeq, 200, [][]float32{fps[i]}, false)[0])
}
var frameratesGot bool
for s, fs := range frames {
for _, f := range fs {
if h26xcalcs[s].RecvPacket(f.toH26x()) {
frameratesGot = true
for _, calc := range h26xcalcs {
if !calc.Completed() {
frameratesGot = false
break
}
}
}
}
}
require.True(t, frameratesGot)
for i, calc := range h26xcalcs {
fpsExpected := fps[i]
fpsGot := calc.GetFrameRate()
verifyFps(t, fpsExpected, fpsGot[:len(fpsExpected)])
}
})
}
t.Run("packet lost and duplicate", func(t *testing.T) {
fps := [][]float32{{7.5, 15, 30}, {7.5, 15, 30}, {7.5, 15, 30}}
frames := make([][]*testFrameInfo, 0, len(fps))
h26xcalcs := make([]FrameRateCalculator, len(fps))
for i := range fps {
frames = append(frames, createFrames(100, 12345678, 10, 500, [][]float32{fps[i]}, false)[0])
h26xcalcs[i] = NewFrameRateCalculatorH26x(90000, logger.GetLogger())
for j := 5; j < 130; j++ {
if j%2 == 0 {
frames[i][j] = frames[i][j-1]
}
}
for j := 130; j < 230; j++ {
if j%3 == 0 {
frames[i][j] = nil
}
}
for j := 230; j < 330; j++ {
if j%2 == 0 {
frames[i][j], frames[i][j-1] = frames[i][j-1], frames[i][j]
}
}
}
var frameratesGot bool
for s, fs := range frames {
for _, f := range fs {
if f == nil {
continue
}
if h26xcalcs[s].RecvPacket(f.toH26x()) {
frameratesGot = true
for _, calc := range h26xcalcs {
if !calc.Completed() {
frameratesGot = false
break
}
}
}
}
}
require.True(t, frameratesGot)
for i, calc := range h26xcalcs {
fpsExpected := fps[i]
fpsGot := calc.GetFrameRate()
verifyFps(t, fpsExpected, fpsGot[:len(fpsExpected)])
}
})
}
livekit-server/pkg/sfu/buffer/frameintegrity.go
+225
View File
@@ -0,0 +1,225 @@
// Copyright 2024 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package buffer
import (
dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor"
)
type FrameEntity struct {
startSeq *uint64
endSeq *uint64
integrity bool
pktHistory *PacketHistory
}
func (fe *FrameEntity) AddPacket(extSeq uint64, ddVal *dd.DependencyDescriptor) {
// duplicate packet
if fe.integrity {
return
}
if fe.startSeq == nil && ddVal.FirstPacketInFrame {
fe.startSeq = &extSeq
}
if fe.endSeq == nil && ddVal.LastPacketInFrame {
fe.endSeq = &extSeq
}
if fe.startSeq != nil && fe.endSeq != nil {
if fe.pktHistory.PacketsConsecutive(*fe.startSeq, *fe.endSeq) {
fe.integrity = true
}
}
}
func (fe *FrameEntity) Reset() {
fe.integrity = false
fe.startSeq, fe.endSeq = nil, nil
}
func (fe *FrameEntity) Integrity() bool {
return fe.integrity
}
// ------------------------------
type PacketHistory struct {
base uint64
last uint64
bits []uint64
packetCount int
inited bool
}
func NewPacketHistory(packetCount int) *PacketHistory {
packetCount = (packetCount + 63) / 64 * 64
return &PacketHistory{
bits: make([]uint64, packetCount/64),
packetCount: packetCount,
}
}
func (ph *PacketHistory) AddPacket(extSeq uint64) {
if !ph.inited {
ph.inited = true
ph.base = uint64(extSeq)
// set base to extSeq-100 to avoid out-of-order packets belongs to first frame to be dropped
if ph.base > 100 {
ph.base -= 100
} else {
ph.base = 0
}
ph.last = uint64(extSeq)
ph.set(extSeq, true)
return
}
if extSeq <= ph.base {
// too old
return
}
if extSeq <= ph.last {
if ph.last-extSeq < uint64(ph.packetCount) {
ph.set(extSeq, true)
}
return
}
for i := ph.last + 1; i < extSeq; i++ {
ph.set(i, false)
}
ph.set(extSeq, true)
ph.last = extSeq
}
func (ph *PacketHistory) getPos(seq uint64) (index, offset int) {
idx := (seq - ph.base) % uint64(ph.packetCount)
return int(idx >> 6), int(idx % 64)
}
func (ph *PacketHistory) set(seq uint64, received bool) {
idx, offset := ph.getPos(seq)
if !received {
ph.bits[idx] &= ^(1 << offset)
} else {
ph.bits[idx] |= 1 << (offset)
}
}
func (ph *PacketHistory) PacketsConsecutive(start, end uint64) bool {
if start > end {
return false
}
if end-start >= uint64(ph.packetCount) {
return false
}
startIndex, startOffset := ph.getPos(start)
endIndex, endOffset := ph.getPos(end)
if startIndex == endIndex && end-start <= 64 {
testBits := uint64((1<<(endOffset-startOffset+1))-1) << startOffset
return ph.bits[startIndex]&testBits == testBits
}
if (ph.bits[startIndex]>>(startOffset))+1 != 1<<(64-startOffset) {
return false
}
for i := startIndex + 1; i != endIndex; i++ {
if i == len(ph.bits) {
i = 0
if i == endIndex {
break
}
}
if ph.bits[i]+1 != 0 {
return false
}
}
testBits := uint64((1 << (endOffset + 1)) - 1)
return ph.bits[endIndex]&testBits == testBits
}
// ------------------------------
type FrameIntegrityChecker struct {
frameCount int
frames []FrameEntity
base uint64
last uint64
pktHistory *PacketHistory
inited bool
}
func NewFrameIntegrityChecker(frameCount, packetCount int) *FrameIntegrityChecker {
fc := &FrameIntegrityChecker{
frames: make([]FrameEntity, frameCount),
pktHistory: NewPacketHistory(packetCount),
frameCount: frameCount,
}
for i := range fc.frames {
fc.frames[i].pktHistory = fc.pktHistory
fc.frames[i].Reset()
}
return fc
}
func (fc *FrameIntegrityChecker) AddPacket(extSeq uint64, extFrameNum uint64, ddVal *dd.DependencyDescriptor) {
fc.pktHistory.AddPacket(extSeq)
if !fc.inited {
fc.inited = true
fc.base = extFrameNum
fc.last = extFrameNum
}
if extFrameNum < fc.base {
// frame too old
return
}
if extFrameNum <= fc.last {
if fc.last-extFrameNum >= uint64(fc.frameCount) {
// frame too old
return
}
fc.frames[int(extFrameNum-fc.base)%fc.frameCount].AddPacket(extSeq, ddVal)
return
}
// reset missing frames
for i := fc.last + 1; i <= extFrameNum; i++ {
fc.frames[int(i-fc.base)%fc.frameCount].Reset()
}
fc.frames[int(extFrameNum-fc.base)%fc.frameCount].AddPacket(extSeq, ddVal)
fc.last = extFrameNum
}
func (fc *FrameIntegrityChecker) FrameIntegrity(extFrameNum uint64) bool {
if extFrameNum < fc.base || extFrameNum > fc.last || fc.last-extFrameNum >= uint64(fc.frameCount) {
return false
}
return fc.frames[int(extFrameNum-fc.base)%fc.frameCount].Integrity()
}
livekit-server/pkg/sfu/buffer/frameintegrity_test.go
+86
View File
@@ -0,0 +1,86 @@
// Copyright 2024 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package buffer
import (
"math/rand"
"testing"
"github.com/stretchr/testify/require"
dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor"
)
func TestFrameIntegrityChecker(t *testing.T) {
fc := NewFrameIntegrityChecker(100, 1000)
// first frame out of order
fc.AddPacket(10, 10, &dd.DependencyDescriptor{})
require.False(t, fc.FrameIntegrity(10))
fc.AddPacket(9, 10, &dd.DependencyDescriptor{FirstPacketInFrame: true})
require.False(t, fc.FrameIntegrity(10))
fc.AddPacket(11, 10, &dd.DependencyDescriptor{LastPacketInFrame: true})
require.True(t, fc.FrameIntegrity(10))
// single packet frame
fc.AddPacket(100, 100, &dd.DependencyDescriptor{FirstPacketInFrame: true, LastPacketInFrame: true})
require.True(t, fc.FrameIntegrity(100))
require.False(t, fc.FrameIntegrity(101))
require.False(t, fc.FrameIntegrity(99))
// frame too old than first frame
fc.AddPacket(99, 99, &dd.DependencyDescriptor{FirstPacketInFrame: true, LastPacketInFrame: true})
// multiple packet frame, out of order
fc.AddPacket(2001, 2001, &dd.DependencyDescriptor{})
require.False(t, fc.FrameIntegrity(2001))
require.False(t, fc.FrameIntegrity(1999))
// out of frame count(100)
require.False(t, fc.FrameIntegrity(100))
require.False(t, fc.FrameIntegrity(1900))
fc.AddPacket(2000, 2001, &dd.DependencyDescriptor{FirstPacketInFrame: true})
require.False(t, fc.FrameIntegrity(2001))
fc.AddPacket(2002, 2001, &dd.DependencyDescriptor{LastPacketInFrame: true})
require.True(t, fc.FrameIntegrity(2001))
// duplicate packet
fc.AddPacket(2001, 2001, &dd.DependencyDescriptor{})
require.True(t, fc.FrameIntegrity(2001))
// frame too old
fc.AddPacket(900, 1900, &dd.DependencyDescriptor{FirstPacketInFrame: true, LastPacketInFrame: true})
require.False(t, fc.FrameIntegrity(1900))
for frame := uint64(2002); frame < 2102; frame++ {
// large frame (1000 packets) out of order / retransmitted
firstFrame := uint64(3000 + (frame-2002)*1000)
lastFrame := uint64(3999 + (frame-2002)*1000)
frames := make([]uint64, 0, lastFrame-firstFrame+1)
for i := firstFrame; i <= lastFrame; i++ {
frames = append(frames, i)
}
require.False(t, fc.FrameIntegrity(frame))
rand.Seed(int64(frame))
rand.Shuffle(len(frames), func(i, j int) { frames[i], frames[j] = frames[j], frames[i] })
for i, f := range frames {
fc.AddPacket(f, frame, &dd.DependencyDescriptor{
FirstPacketInFrame: f == firstFrame,
LastPacketInFrame: f == lastFrame,
})
require.Equal(t, i == len(frames)-1, fc.FrameIntegrity(frame), i)
}
require.True(t, fc.FrameIntegrity(frame))
}
}
livekit-server/pkg/sfu/buffer/helpers.go
+455
View File
@@ -0,0 +1,455 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package buffer
import (
"encoding/binary"
"errors"
"github.com/pion/rtp/codecs"
"github.com/livekit/protocol/logger"
)
var (
errShortPacket = errors.New("packet is not large enough")
errNilPacket = errors.New("invalid nil packet")
errInvalidPacket = errors.New("invalid packet")
)
// VP8 is a helper to get temporal data from VP8 packet header
/*
VP8 Payload Descriptor
0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7
+-+-+-+-+-+-+-+-+ +-+-+-+-+-+-+-+-+
|X|R|N|S|R| PID | (REQUIRED) |X|R|N|S|R| PID | (REQUIRED)
+-+-+-+-+-+-+-+-+ +-+-+-+-+-+-+-+-+
X: |I|L|T|K| RSV | (OPTIONAL) X: |I|L|T|K| RSV | (OPTIONAL)
+-+-+-+-+-+-+-+-+ +-+-+-+-+-+-+-+-+
I: |M| PictureID | (OPTIONAL) I: |M| PictureID | (OPTIONAL)
+-+-+-+-+-+-+-+-+ +-+-+-+-+-+-+-+-+
L: | TL0PICIDX | (OPTIONAL) | PictureID |
+-+-+-+-+-+-+-+-+ +-+-+-+-+-+-+-+-+
T/K:|TID|Y| KEYIDX | (OPTIONAL) L: | TL0PICIDX | (OPTIONAL)
+-+-+-+-+-+-+-+-+ +-+-+-+-+-+-+-+-+
T/K:|TID|Y| KEYIDX | (OPTIONAL)
+-+-+-+-+-+-+-+-+
*/
type VP8 struct {
FirstByte byte
S bool
I bool
M bool
PictureID uint16 /* 7 or 15 bits, picture ID */
L bool
TL0PICIDX uint8 /* 8 bits temporal level zero index */
// Optional Header If either of the T or K bits are set to 1,
// the TID/Y/KEYIDX extension field MUST be present.
T bool
TID uint8 /* 2 bits temporal layer idx */
Y bool
K bool
KEYIDX uint8 /* 5 bits of key frame idx */
HeaderSize int
// IsKeyFrame is a helper to detect if current packet is a keyframe
IsKeyFrame bool
}
// Unmarshal parses the passed byte slice and stores the result in the VP8 this method is called upon
func (v *VP8) Unmarshal(payload []byte) error {
if payload == nil {
return errNilPacket
}
payloadLen := len(payload)
if payloadLen < 1 {
return errShortPacket
}
idx := 0
v.FirstByte = payload[idx]
v.S = payload[idx]&0x10 > 0
// Check for extended bit control
if payload[idx]&0x80 > 0 {
idx++
if payloadLen < idx+1 {
return errShortPacket
}
v.I = payload[idx]&0x80 > 0
v.L = payload[idx]&0x40 > 0
v.T = payload[idx]&0x20 > 0
v.K = payload[idx]&0x10 > 0
if v.L && !v.T {
return errInvalidPacket
}
if v.I {
idx++
if payloadLen < idx+1 {
return errShortPacket
}
pid := payload[idx] & 0x7f
// if m is 1, then Picture ID is 15 bits
v.M = payload[idx]&0x80 > 0
if v.M {
idx++
if payloadLen < idx+1 {
return errShortPacket
}
v.PictureID = binary.BigEndian.Uint16([]byte{pid, payload[idx]})
} else {
v.PictureID = uint16(pid)
}
}
if v.L {
idx++
if payloadLen < idx+1 {
return errShortPacket
}
v.TL0PICIDX = payload[idx]
}
if v.T || v.K {
idx++
if payloadLen < idx+1 {
return errShortPacket
}
if v.T {
v.TID = (payload[idx] & 0xc0) >> 6
v.Y = (payload[idx] & 0x20) > 0
}
if v.K {
v.KEYIDX = payload[idx] & 0x1f
}
}
idx++
if payloadLen < idx+1 {
return errShortPacket
}
// Check is packet is a keyframe by looking at P bit in vp8 payload
v.IsKeyFrame = payload[idx]&0x01 == 0 && v.S
} else {
idx++
if payloadLen < idx+1 {
return errShortPacket
}
// Check is packet is a keyframe by looking at P bit in vp8 payload
v.IsKeyFrame = payload[idx]&0x01 == 0 && v.S
}
v.HeaderSize = idx
return nil
}
func (v *VP8) Marshal() ([]byte, error) {
buf := make([]byte, v.HeaderSize)
n, err := v.MarshalTo(buf)
if err != nil {
return nil, err
}
return buf[:n], err
}
func (v *VP8) MarshalTo(buf []byte) (int, error) {
if len(buf) < v.HeaderSize {
return 0, errShortPacket
}
idx := 0
buf[idx] = v.FirstByte
if v.I || v.L || v.T || v.K {
buf[idx] |= 0x80 // X bit
idx++
xpos := idx
xval := byte(0)
idx++
if v.I {
xval |= (1 << 7)
if v.M {
buf[idx] = 0x80 | byte((v.PictureID>>8)&0x7f)
buf[idx+1] = byte(v.PictureID & 0xff)
idx += 2
} else {
buf[idx] = byte(v.PictureID)
idx++
}
}
if v.L {
xval |= (1 << 6)
buf[idx] = v.TL0PICIDX
idx++
}
if v.T || v.K {
buf[idx] = 0
if v.T {
xval |= (1 << 5)
buf[idx] = v.TID << 6
if v.Y {
buf[idx] |= (1 << 5)
}
}
if v.K {
xval |= (1 << 4)
buf[idx] |= v.KEYIDX & 0x1f
}
idx++
}
buf[xpos] = xval
} else {
buf[idx] &^= 0x80 // X bit
idx++
}
return idx, nil
}
// -------------------------------------
func VPxPictureIdSizeDiff(mBit1 bool, mBit2 bool) int {
if mBit1 == mBit2 {
return 0
}
if mBit1 {
return 1
}
return -1
}
// -------------------------------------
// IsH264KeyFrame detects if h264 payload is a keyframe
// this code was taken from https://github.com/jech/galene/blob/codecs/rtpconn/rtpreader.go#L45
// all credits belongs to Juliusz Chroboczek @jech and the awesome Galene SFU
func IsH264KeyFrame(payload []byte) bool {
if len(payload) < 1 {
return false
}
nalu := payload[0] & 0x1F
if nalu == 0 {
// reserved
return false
} else if nalu <= 23 {
// simple NALU
return nalu == 7
} else if nalu == 24 || nalu == 25 || nalu == 26 || nalu == 27 {
// STAP-A, STAP-B, MTAP16 or MTAP24
i := 1
if nalu == 25 || nalu == 26 || nalu == 27 {
// skip DON
i += 2
}
for i < len(payload) {
if i+2 > len(payload) {
return false
}
length := uint16(payload[i])<<8 |
uint16(payload[i+1])
i += 2
if i+int(length) > len(payload) {
return false
}
offset := 0
if nalu == 26 {
offset = 3
} else if nalu == 27 {
offset = 4
}
if offset >= int(length) {
return false
}
n := payload[i+offset] & 0x1F
if n == 7 {
return true
} else if n >= 24 {
// is this legal?
logger.Debugw("Non-simple NALU within a STAP")
}
i += int(length)
}
if i == len(payload) {
return false
}
return false
} else if nalu == 28 || nalu == 29 {
// FU-A or FU-B
if len(payload) < 2 {
return false
}
if (payload[1] & 0x80) == 0 {
// not a starting fragment
return false
}
return payload[1]&0x1F == 7
}
return false
}
// -------------------------------------
// IsVP9KeyFrame detects if vp9 payload is a keyframe
// taken from https://github.com/jech/galene/blob/master/codecs/codecs.go
// all credits belongs to Juliusz Chroboczek @jech and the awesome Galene SFU
func IsVP9KeyFrame(payload []byte) bool {
var vp9 codecs.VP9Packet
_, err := vp9.Unmarshal(payload)
if err != nil || len(vp9.Payload) < 1 {
return false
}
if !vp9.B {
return false
}
if (vp9.Payload[0] & 0xc0) != 0x80 {
return false
}
profile := (vp9.Payload[0] >> 4) & 0x3
if profile != 3 {
return (vp9.Payload[0] & 0xC) == 0
}
return (vp9.Payload[0] & 0x6) == 0
}
// -------------------------------------
// IsAV1KeyFrame detects if av1 payload is a keyframe
// taken from https://github.com/jech/galene/blob/master/codecs/codecs.go
// all credits belongs to Juliusz Chroboczek @jech and the awesome Galene SFU
func IsAV1KeyFrame(payload []byte) bool {
if len(payload) < 2 {
return false
}
// Z=0, N=1
if (payload[0] & 0x88) != 0x08 {
return false
}
w := (payload[0] & 0x30) >> 4
getObu := func(data []byte, last bool) ([]byte, int, bool) {
if last {
return data, len(data), false
}
offset := 0
length := 0
for {
if len(data) <= offset {
return nil, offset, offset > 0
}
l := data[offset]
length |= int(l&0x7f) << (offset * 7)
offset++
if (l & 0x80) == 0 {
break
}
}
if len(data) < offset+length {
return data[offset:], len(data), true
}
return data[offset : offset+length],
offset + length, false
}
offset := 1
i := 0
for {
obu, length, truncated :=
getObu(payload[offset:], int(w) == i+1)
if len(obu) < 1 {
return false
}
tpe := (obu[0] & 0x38) >> 3
switch i {
case 0:
// OBU_SEQUENCE_HEADER
if tpe != 1 {
return false
}
default:
// OBU_FRAME_HEADER or OBU_FRAME
if tpe == 3 || tpe == 6 {
if len(obu) < 2 {
return false
}
// show_existing_frame == 0
if (obu[1] & 0x80) != 0 {
return false
}
// frame_type == KEY_FRAME
return (obu[1] & 0x60) == 0
}
}
if truncated || i >= int(w) {
// the first frame header is in a second
// packet, give up.
return false
}
offset += length
i++
}
}
func IsH265KeyFrame(payload []byte) (kf bool) {
if len(payload) < 2 {
return false
}
naluType := (payload[0] & 0x7E) >> 1
switch {
case naluType == 33 || naluType == 34:
return true
case naluType == 48: // AP
idx := 2
for idx < len(payload)-2 {
// TODO: check the DONL field (controled by sprop-max-don-diff)
size := binary.BigEndian.Uint16(payload[idx:])
idx += 2
if idx >= len(payload) {
return false
}
naluType = (payload[idx] & 0x7E) >> 1
if naluType == 33 || naluType == 34 {
return true
}
idx += int(size)
}
return false
case naluType == 49: // FU
if len(payload) < 3 {
return false
}
naluType = (payload[2] & 0x7E) >> 1
return naluType == 33 || naluType == 34
default:
return false
}
}
// -------------------------------------
livekit-server/pkg/sfu/buffer/helpers_test.go
+110
View File
@@ -0,0 +1,110 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package buffer
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestVP8Helper_Unmarshal(t *testing.T) {
type args struct {
payload []byte
}
tests := []struct {
name string
args args
wantErr bool
checkTemporal bool
temporalSupport bool
checkKeyFrame bool
keyFrame bool
checkPictureID bool
pictureID uint16
checkTlzIdx bool
tlzIdx uint8
checkTempID bool
temporalID uint8
}{
{
name: "Empty or nil payload must return error",
args: args{payload: []byte{}},
wantErr: true,
},
{
name: "Temporal must be supported by setting T bit to 1",
args: args{payload: []byte{0xff, 0x20, 0x1, 0x2, 0x3, 0x4}},
checkTemporal: true,
temporalSupport: true,
},
{
name: "Picture must be ID 7 bits by setting M bit to 0 and present by I bit set to 1",
args: args{payload: []byte{0xff, 0xff, 0x11, 0x2, 0x3, 0x4}},
checkPictureID: true,
pictureID: 17,
},
{
name: "Picture ID must be 15 bits by setting M bit to 1 and present by I bit set to 1",
args: args{payload: []byte{0xff, 0xff, 0x92, 0x67, 0x3, 0x4, 0x5}},
checkPictureID: true,
pictureID: 4711,
},
{
name: "Temporal level zero index must be present if L set to 1",
args: args{payload: []byte{0xff, 0xff, 0xff, 0xfd, 0xb4, 0x4, 0x5}},
checkTlzIdx: true,
tlzIdx: 180,
},
{
name: "Temporal index must be present and used if T bit set to 1",
args: args{payload: []byte{0xff, 0xff, 0xff, 0xfd, 0xb4, 0x9f, 0x5, 0x6}},
checkTempID: true,
temporalID: 2,
},
{
name: "Check if packet is a keyframe by looking at P bit set to 0",
args: args{payload: []byte{0xff, 0xff, 0xff, 0xfd, 0xb4, 0x9f, 0x94, 0x1}},
checkKeyFrame: true,
keyFrame: true,
},
}
for _, tt := range tests {
tt := tt
t.Run(tt.name, func(t *testing.T) {
p := &VP8{}
if err := p.Unmarshal(tt.args.payload); (err != nil) != tt.wantErr {
t.Errorf("Unmarshal() error = %v, wantErr %v", err, tt.wantErr)
}
if tt.checkTemporal {
require.Equal(t, tt.temporalSupport, p.T)
}
if tt.checkKeyFrame {
require.Equal(t, tt.keyFrame, p.IsKeyFrame)
}
if tt.checkPictureID {
require.Equal(t, tt.pictureID, p.PictureID)
}
if tt.checkTlzIdx {
require.Equal(t, tt.tlzIdx, p.TL0PICIDX)
}
if tt.checkTempID {
require.Equal(t, tt.temporalID, p.TID)
}
})
}
}
// ------------------------------------------
livekit-server/pkg/sfu/buffer/rtcpreader.go
+62
View File
@@ -0,0 +1,62 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package buffer
import (
"io"
"go.uber.org/atomic"
)
type RTCPReader struct {
ssrc uint32
closed atomic.Bool
onPacket atomic.Value // func([]byte)
onClose func()
}
func NewRTCPReader(ssrc uint32) *RTCPReader {
return &RTCPReader{ssrc: ssrc}
}
func (r *RTCPReader) Write(p []byte) (n int, err error) {
if r.closed.Load() {
err = io.EOF
return
}
if f, ok := r.onPacket.Load().(func([]byte)); ok && f != nil {
f(p)
}
return
}
func (r *RTCPReader) OnClose(fn func()) {
r.onClose = fn
}
func (r *RTCPReader) Close() error {
if r.closed.Swap(true) {
return nil
}
r.onClose()
return nil
}
func (r *RTCPReader) OnPacket(f func([]byte)) {
r.onPacket.Store(f)
}
func (r *RTCPReader) Read(_ []byte) (n int, err error) { return }
livekit-server/pkg/sfu/buffer/streamstats.go
+24
View File
@@ -0,0 +1,24 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package buffer
import "github.com/livekit/livekit-server/pkg/sfu/rtpstats"
type StreamStatsWithLayers struct {
RTPStats *rtpstats.RTPDeltaInfo
Layers map[int32]*rtpstats.RTPDeltaInfo
RTPStatsRemoteView *rtpstats.RTPDeltaInfo
}
livekit-server/pkg/sfu/buffer/videolayer.go
+58
View File
@@ -0,0 +1,58 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package buffer
import "fmt"
const (
InvalidLayerSpatial = int32(-1)
InvalidLayerTemporal = int32(-1)
DefaultMaxLayerSpatial = int32(2)
DefaultMaxLayerTemporal = int32(3)
)
var (
InvalidLayer = VideoLayer{
Spatial: InvalidLayerSpatial,
Temporal: InvalidLayerTemporal,
}
DefaultMaxLayer = VideoLayer{
Spatial: DefaultMaxLayerSpatial,
Temporal: DefaultMaxLayerTemporal,
}
)
type VideoLayer struct {
Spatial int32
Temporal int32
}
func (v VideoLayer) String() string {
return fmt.Sprintf("VideoLayer{s: %d, t: %d}", v.Spatial, v.Temporal)
}
func (v VideoLayer) GreaterThan(v2 VideoLayer) bool {
return v.Spatial > v2.Spatial || (v.Spatial == v2.Spatial && v.Temporal > v2.Temporal)
}
func (v VideoLayer) SpatialGreaterThanOrEqual(v2 VideoLayer) bool {
return v.Spatial >= v2.Spatial
}
func (v VideoLayer) IsValid() bool {
return v.Spatial != InvalidLayerSpatial && v.Temporal != InvalidLayerTemporal
}
livekit-server/pkg/sfu/buffer/videolayerutils.go
+326
View File
@@ -0,0 +1,326 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package buffer
import (
"github.com/livekit/protocol/livekit"
"github.com/livekit/protocol/logger"
)
const (
QuarterResolution = "q"
HalfResolution = "h"
FullResolution = "f"
)
// SIMULCAST-CODEC-TODO: these need to be codec mime aware if and when each codec suppports different layers
func LayerPresenceFromTrackInfo(trackInfo *livekit.TrackInfo) *[livekit.VideoQuality_HIGH + 1]bool {
if trackInfo == nil || len(trackInfo.Layers) == 0 {
return nil
}
var layerPresence [livekit.VideoQuality_HIGH + 1]bool
for _, layer := range trackInfo.Layers {
// WARNING: comparing protobuf enum
if layer.Quality <= livekit.VideoQuality_HIGH {
layerPresence[layer.Quality] = true
} else {
logger.Warnw("unexpected quality in track info", nil, "trackID", trackInfo.Sid, "trackInfo", logger.Proto(trackInfo))
}
}
return &layerPresence
}
func RidToSpatialLayer(rid string, trackInfo *livekit.TrackInfo) int32 {
lp := LayerPresenceFromTrackInfo(trackInfo)
if lp == nil {
switch rid {
case QuarterResolution:
return 0
case HalfResolution:
return 1
case FullResolution:
return 2
default:
return 0
}
}
switch rid {
case QuarterResolution:
switch {
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
fallthrough
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM]:
fallthrough
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_HIGH]:
fallthrough
case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
return 0
default:
// only one quality published, could be any
return 0
}
case HalfResolution:
switch {
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
fallthrough
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM]:
fallthrough
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_HIGH]:
fallthrough
case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
return 1
default:
// only one quality published, could be any
return 0
}
case FullResolution:
switch {
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
return 2
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM]:
logger.Warnw("unexpected rid f with only two qualities, low and medium", nil, "trackID", trackInfo.Sid, "trackInfo", logger.Proto(trackInfo))
return 1
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_HIGH]:
logger.Warnw("unexpected rid f with only two qualities, low and high", nil, "trackID", trackInfo.Sid, "trackInfo", logger.Proto(trackInfo))
return 1
case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
logger.Warnw("unexpected rid f with only two qualities, medium and high", nil, "trackID", trackInfo.Sid, "trackInfo", logger.Proto(trackInfo))
return 1
default:
// only one quality published, could be any
return 0
}
default:
// no rid, should be single layer
return 0
}
}
func SpatialLayerToRid(layer int32, trackInfo *livekit.TrackInfo) string {
lp := LayerPresenceFromTrackInfo(trackInfo)
if lp == nil {
switch layer {
case 0:
return QuarterResolution
case 1:
return HalfResolution
case 2:
return FullResolution
default:
return QuarterResolution
}
}
switch layer {
case 0:
switch {
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
fallthrough
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM]:
fallthrough
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_HIGH]:
fallthrough
case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
return QuarterResolution
default:
return QuarterResolution
}
case 1:
switch {
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
fallthrough
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM]:
fallthrough
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_HIGH]:
fallthrough
case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
return HalfResolution
default:
return QuarterResolution
}
case 2:
switch {
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
return FullResolution
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM]:
logger.Warnw("unexpected layer 2 with only two qualities, low and medium", nil, "trackID", trackInfo.Sid, "trackInfo", logger.Proto(trackInfo))
return HalfResolution
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_HIGH]:
logger.Warnw("unexpected layer 2 with only two qualities, low and high", nil, "trackID", trackInfo.Sid, "trackInfo", logger.Proto(trackInfo))
return HalfResolution
case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
logger.Warnw("unexpected layer 2 with only two qualities, medium and high", nil, "trackID", trackInfo.Sid, "trackInfo", logger.Proto(trackInfo))
return HalfResolution
default:
return QuarterResolution
}
default:
return QuarterResolution
}
}
func VideoQualityToRid(quality livekit.VideoQuality, trackInfo *livekit.TrackInfo) string {
return SpatialLayerToRid(VideoQualityToSpatialLayer(quality, trackInfo), trackInfo)
}
func SpatialLayerToVideoQuality(layer int32, trackInfo *livekit.TrackInfo) livekit.VideoQuality {
lp := LayerPresenceFromTrackInfo(trackInfo)
if lp == nil {
switch layer {
case 0:
return livekit.VideoQuality_LOW
case 1:
return livekit.VideoQuality_MEDIUM
case 2:
return livekit.VideoQuality_HIGH
default:
return livekit.VideoQuality_OFF
}
}
switch layer {
case 0:
switch {
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
fallthrough
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM]:
fallthrough
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_HIGH]:
fallthrough
case lp[livekit.VideoQuality_LOW]:
return livekit.VideoQuality_LOW
case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
fallthrough
case lp[livekit.VideoQuality_MEDIUM]:
return livekit.VideoQuality_MEDIUM
default:
return livekit.VideoQuality_HIGH
}
case 1:
switch {
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
fallthrough
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM]:
return livekit.VideoQuality_MEDIUM
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_HIGH]:
fallthrough
case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
return livekit.VideoQuality_HIGH
default:
logger.Errorw("invalid layer", nil, "trackID", trackInfo.Sid, "layer", layer, "trackInfo", logger.Proto(trackInfo))
return livekit.VideoQuality_HIGH
}
case 2:
switch {
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
return livekit.VideoQuality_HIGH
default:
logger.Errorw("invalid layer", nil, "trackID", trackInfo.Sid, "layer", layer, "trackInfo", logger.Proto(trackInfo))
return livekit.VideoQuality_HIGH
}
}
return livekit.VideoQuality_OFF
}
func VideoQualityToSpatialLayer(quality livekit.VideoQuality, trackInfo *livekit.TrackInfo) int32 {
lp := LayerPresenceFromTrackInfo(trackInfo)
if lp == nil {
switch quality {
case livekit.VideoQuality_LOW:
return 0
case livekit.VideoQuality_MEDIUM:
return 1
case livekit.VideoQuality_HIGH:
return 2
default:
return InvalidLayerSpatial
}
}
switch quality {
case livekit.VideoQuality_LOW:
switch {
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
fallthrough
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM]:
fallthrough
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_HIGH]:
fallthrough
case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
fallthrough
default: // only one quality published, could be any
return 0
}
case livekit.VideoQuality_MEDIUM:
switch {
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
fallthrough
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM]:
fallthrough
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_HIGH]:
return 1
case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
return 0
default: // only one quality published, could be any
return 0
}
case livekit.VideoQuality_HIGH:
switch {
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
return 2
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM]:
fallthrough
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_HIGH]:
fallthrough
case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
return 1
default: // only one quality published, could be any
return 0
}
}
return InvalidLayerSpatial
}
livekit-server/pkg/sfu/buffer/videolayerutils_test.go
+442
View File
@@ -0,0 +1,442 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package buffer
import (
"testing"
"github.com/stretchr/testify/require"
"github.com/livekit/protocol/livekit"
)
func TestRidConversion(t *testing.T) {
type RidAndLayer struct {
rid string
layer int32
}
tests := []struct {
name string
trackInfo *livekit.TrackInfo
ridToLayer map[string]RidAndLayer
}{
{
"no track info",
nil,
map[string]RidAndLayer{
"": {rid: QuarterResolution, layer: 0},
QuarterResolution: {rid: QuarterResolution, layer: 0},
HalfResolution: {rid: HalfResolution, layer: 1},
FullResolution: {rid: FullResolution, layer: 2},
},
},
{
"no layers",
&livekit.TrackInfo{},
map[string]RidAndLayer{
"": {rid: QuarterResolution, layer: 0},
QuarterResolution: {rid: QuarterResolution, layer: 0},
HalfResolution: {rid: HalfResolution, layer: 1},
FullResolution: {rid: FullResolution, layer: 2},
},
},
{
"single layer, low",
&livekit.TrackInfo{
Layers: []*livekit.VideoLayer{
{Quality: livekit.VideoQuality_LOW},
},
},
map[string]RidAndLayer{
"": {rid: QuarterResolution, layer: 0},
QuarterResolution: {rid: QuarterResolution, layer: 0},
HalfResolution: {rid: QuarterResolution, layer: 0},
FullResolution: {rid: QuarterResolution, layer: 0},
},
},
{
"single layer, medium",
&livekit.TrackInfo{
Layers: []*livekit.VideoLayer{
{Quality: livekit.VideoQuality_MEDIUM},
},
},
map[string]RidAndLayer{
"": {rid: QuarterResolution, layer: 0},
QuarterResolution: {rid: QuarterResolution, layer: 0},
HalfResolution: {rid: QuarterResolution, layer: 0},
FullResolution: {rid: QuarterResolution, layer: 0},
},
},
{
"single layer, high",
&livekit.TrackInfo{
Layers: []*livekit.VideoLayer{
{Quality: livekit.VideoQuality_HIGH},
},
},
map[string]RidAndLayer{
"": {rid: QuarterResolution, layer: 0},
QuarterResolution: {rid: QuarterResolution, layer: 0},
HalfResolution: {rid: QuarterResolution, layer: 0},
FullResolution: {rid: QuarterResolution, layer: 0},
},
},
{
"two layers, low and medium",
&livekit.TrackInfo{
Layers: []*livekit.VideoLayer{
{Quality: livekit.VideoQuality_LOW},
{Quality: livekit.VideoQuality_MEDIUM},
},
},
map[string]RidAndLayer{
"": {rid: QuarterResolution, layer: 0},
QuarterResolution: {rid: QuarterResolution, layer: 0},
HalfResolution: {rid: HalfResolution, layer: 1},
FullResolution: {rid: HalfResolution, layer: 1},
},
},
{
"two layers, low and high",
&livekit.TrackInfo{
Layers: []*livekit.VideoLayer{
{Quality: livekit.VideoQuality_LOW},
{Quality: livekit.VideoQuality_HIGH},
},
},
map[string]RidAndLayer{
"": {rid: QuarterResolution, layer: 0},
QuarterResolution: {rid: QuarterResolution, layer: 0},
HalfResolution: {rid: HalfResolution, layer: 1},
FullResolution: {rid: HalfResolution, layer: 1},
},
},
{
"two layers, medium and high",
&livekit.TrackInfo{
Layers: []*livekit.VideoLayer{
{Quality: livekit.VideoQuality_MEDIUM},
{Quality: livekit.VideoQuality_HIGH},
},
},
map[string]RidAndLayer{
"": {rid: QuarterResolution, layer: 0},
QuarterResolution: {rid: QuarterResolution, layer: 0},
HalfResolution: {rid: HalfResolution, layer: 1},
FullResolution: {rid: HalfResolution, layer: 1},
},
},
{
"three layers",
&livekit.TrackInfo{
Layers: []*livekit.VideoLayer{
{Quality: livekit.VideoQuality_LOW},
{Quality: livekit.VideoQuality_MEDIUM},
{Quality: livekit.VideoQuality_HIGH},
},
},
map[string]RidAndLayer{
"": {rid: QuarterResolution, layer: 0},
QuarterResolution: {rid: QuarterResolution, layer: 0},
HalfResolution: {rid: HalfResolution, layer: 1},
FullResolution: {rid: FullResolution, layer: 2},
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
for testRid, expectedResult := range test.ridToLayer {
actualLayer := RidToSpatialLayer(testRid, test.trackInfo)
require.Equal(t, expectedResult.layer, actualLayer)
actualRid := SpatialLayerToRid(actualLayer, test.trackInfo)
require.Equal(t, expectedResult.rid, actualRid)
}
})
}
}
func TestQualityConversion(t *testing.T) {
type QualityAndLayer struct {
quality livekit.VideoQuality
layer int32
}
tests := []struct {
name string
trackInfo *livekit.TrackInfo
qualityToLayer map[livekit.VideoQuality]QualityAndLayer
}{
{
"no track info",
nil,
map[livekit.VideoQuality]QualityAndLayer{
livekit.VideoQuality_LOW: {quality: livekit.VideoQuality_LOW, layer: 0},
livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_MEDIUM, layer: 1},
livekit.VideoQuality_HIGH: {quality: livekit.VideoQuality_HIGH, layer: 2},
},
},
{
"no layers",
&livekit.TrackInfo{},
map[livekit.VideoQuality]QualityAndLayer{
livekit.VideoQuality_LOW: {quality: livekit.VideoQuality_LOW, layer: 0},
livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_MEDIUM, layer: 1},
livekit.VideoQuality_HIGH: {quality: livekit.VideoQuality_HIGH, layer: 2},
},
},
{
"single layer, low",
&livekit.TrackInfo{
Layers: []*livekit.VideoLayer{
{Quality: livekit.VideoQuality_LOW},
},
},
map[livekit.VideoQuality]QualityAndLayer{
livekit.VideoQuality_LOW: {quality: livekit.VideoQuality_LOW, layer: 0},
livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_LOW, layer: 0},
livekit.VideoQuality_HIGH: {quality: livekit.VideoQuality_LOW, layer: 0},
},
},
{
"single layer, medium",
&livekit.TrackInfo{
Layers: []*livekit.VideoLayer{
{Quality: livekit.VideoQuality_MEDIUM},
},
},
map[livekit.VideoQuality]QualityAndLayer{
livekit.VideoQuality_LOW: {quality: livekit.VideoQuality_MEDIUM, layer: 0},
livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_MEDIUM, layer: 0},
livekit.VideoQuality_HIGH: {quality: livekit.VideoQuality_MEDIUM, layer: 0},
},
},
{
"single layer, high",
&livekit.TrackInfo{
Layers: []*livekit.VideoLayer{
{Quality: livekit.VideoQuality_HIGH},
},
},
map[livekit.VideoQuality]QualityAndLayer{
livekit.VideoQuality_LOW: {quality: livekit.VideoQuality_HIGH, layer: 0},
livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_HIGH, layer: 0},
livekit.VideoQuality_HIGH: {quality: livekit.VideoQuality_HIGH, layer: 0},
},
},
{
"two layers, low and medium",
&livekit.TrackInfo{
Layers: []*livekit.VideoLayer{
{Quality: livekit.VideoQuality_LOW},
{Quality: livekit.VideoQuality_MEDIUM},
},
},
map[livekit.VideoQuality]QualityAndLayer{
livekit.VideoQuality_LOW: {quality: livekit.VideoQuality_LOW, layer: 0},
livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_MEDIUM, layer: 1},
livekit.VideoQuality_HIGH: {quality: livekit.VideoQuality_MEDIUM, layer: 1},
},
},
{
"two layers, low and high",
&livekit.TrackInfo{
Layers: []*livekit.VideoLayer{
{Quality: livekit.VideoQuality_LOW},
{Quality: livekit.VideoQuality_HIGH},
},
},
map[livekit.VideoQuality]QualityAndLayer{
livekit.VideoQuality_LOW: {quality: livekit.VideoQuality_LOW, layer: 0},
livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_HIGH, layer: 1},
livekit.VideoQuality_HIGH: {quality: livekit.VideoQuality_HIGH, layer: 1},
},
},
{
"two layers, medium and high",
&livekit.TrackInfo{
Layers: []*livekit.VideoLayer{
{Quality: livekit.VideoQuality_MEDIUM},
{Quality: livekit.VideoQuality_HIGH},
},
},
map[livekit.VideoQuality]QualityAndLayer{
livekit.VideoQuality_LOW: {quality: livekit.VideoQuality_MEDIUM, layer: 0},
livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_MEDIUM, layer: 0},
livekit.VideoQuality_HIGH: {quality: livekit.VideoQuality_HIGH, layer: 1},
},
},
{
"three layers",
&livekit.TrackInfo{
Layers: []*livekit.VideoLayer{
{Quality: livekit.VideoQuality_LOW},
{Quality: livekit.VideoQuality_MEDIUM},
{Quality: livekit.VideoQuality_HIGH},
},
},
map[livekit.VideoQuality]QualityAndLayer{
livekit.VideoQuality_LOW: {quality: livekit.VideoQuality_LOW, layer: 0},
livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_MEDIUM, layer: 1},
livekit.VideoQuality_HIGH: {quality: livekit.VideoQuality_HIGH, layer: 2},
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
for testQuality, expectedResult := range test.qualityToLayer {
actualLayer := VideoQualityToSpatialLayer(testQuality, test.trackInfo)
require.Equal(t, expectedResult.layer, actualLayer)
actualQuality := SpatialLayerToVideoQuality(actualLayer, test.trackInfo)
require.Equal(t, expectedResult.quality, actualQuality)
}
})
}
}
func TestVideoQualityToRidConversion(t *testing.T) {
tests := []struct {
name string
trackInfo *livekit.TrackInfo
qualityToRid map[livekit.VideoQuality]string
}{
{
"no track info",
nil,
map[livekit.VideoQuality]string{
livekit.VideoQuality_LOW: QuarterResolution,
livekit.VideoQuality_MEDIUM: HalfResolution,
livekit.VideoQuality_HIGH: FullResolution,
},
},
{
"no layers",
&livekit.TrackInfo{},
map[livekit.VideoQuality]string{
livekit.VideoQuality_LOW: QuarterResolution,
livekit.VideoQuality_MEDIUM: HalfResolution,
livekit.VideoQuality_HIGH: FullResolution,
},
},
{
"single layer, low",
&livekit.TrackInfo{
Layers: []*livekit.VideoLayer{
{Quality: livekit.VideoQuality_LOW},
},
},
map[livekit.VideoQuality]string{
livekit.VideoQuality_LOW: QuarterResolution,
livekit.VideoQuality_MEDIUM: QuarterResolution,
livekit.VideoQuality_HIGH: QuarterResolution,
},
},
{
"single layer, medium",
&livekit.TrackInfo{
Layers: []*livekit.VideoLayer{
{Quality: livekit.VideoQuality_MEDIUM},
},
},
map[livekit.VideoQuality]string{
livekit.VideoQuality_LOW: QuarterResolution,
livekit.VideoQuality_MEDIUM: QuarterResolution,
livekit.VideoQuality_HIGH: QuarterResolution,
},
},
{
"single layer, high",
&livekit.TrackInfo{
Layers: []*livekit.VideoLayer{
{Quality: livekit.VideoQuality_HIGH},
},
},
map[livekit.VideoQuality]string{
livekit.VideoQuality_LOW: QuarterResolution,
livekit.VideoQuality_MEDIUM: QuarterResolution,
livekit.VideoQuality_HIGH: QuarterResolution,
},
},
{
"two layers, low and medium",
&livekit.TrackInfo{
Layers: []*livekit.VideoLayer{
{Quality: livekit.VideoQuality_LOW},
{Quality: livekit.VideoQuality_MEDIUM},
},
},
map[livekit.VideoQuality]string{
livekit.VideoQuality_LOW: QuarterResolution,
livekit.VideoQuality_MEDIUM: HalfResolution,
livekit.VideoQuality_HIGH: HalfResolution,
},
},
{
"two layers, low and high",
&livekit.TrackInfo{
Layers: []*livekit.VideoLayer{
{Quality: livekit.VideoQuality_LOW},
{Quality: livekit.VideoQuality_HIGH},
},
},
map[livekit.VideoQuality]string{
livekit.VideoQuality_LOW: QuarterResolution,
livekit.VideoQuality_MEDIUM: HalfResolution,
livekit.VideoQuality_HIGH: HalfResolution,
},
},
{
"two layers, medium and high",
&livekit.TrackInfo{
Layers: []*livekit.VideoLayer{
{Quality: livekit.VideoQuality_MEDIUM},
{Quality: livekit.VideoQuality_HIGH},
},
},
map[livekit.VideoQuality]string{
livekit.VideoQuality_LOW: QuarterResolution,
livekit.VideoQuality_MEDIUM: QuarterResolution,
livekit.VideoQuality_HIGH: HalfResolution,
},
},
{
"three layers",
&livekit.TrackInfo{
Layers: []*livekit.VideoLayer{
{Quality: livekit.VideoQuality_LOW},
{Quality: livekit.VideoQuality_MEDIUM},
{Quality: livekit.VideoQuality_HIGH},
},
},
map[livekit.VideoQuality]string{
livekit.VideoQuality_LOW: QuarterResolution,
livekit.VideoQuality_MEDIUM: HalfResolution,
livekit.VideoQuality_HIGH: FullResolution,
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
for testQuality, expectedRid := range test.qualityToRid {
actualRid := VideoQualityToRid(testQuality, test.trackInfo)
require.Equal(t, expectedRid, actualRid)
}
})
}
}
livekit-server/pkg/sfu/bwe/bwe.go
+98
View File
@@ -0,0 +1,98 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package bwe
import (
"fmt"
"time"
"github.com/livekit/livekit-server/pkg/sfu/ccutils"
"github.com/pion/rtcp"
)
// ------------------------------------------------
const (
DefaultRTT = float64(0.070) // 70 ms
RTTSmoothingFactor = float64(0.5)
)
// ------------------------------------------------
type CongestionState int
const (
CongestionStateNone CongestionState = iota
CongestionStateEarlyWarning
CongestionStateCongested
)
func (c CongestionState) String() string {
switch c {
case CongestionStateNone:
return "NONE"
case CongestionStateEarlyWarning:
return "EARLY_WARNING"
case CongestionStateCongested:
return "CONGESTED"
default:
return fmt.Sprintf("%d", int(c))
}
}
// ------------------------------------------------
type BWE interface {
SetBWEListener(bweListner BWEListener)
Reset()
HandleREMB(
receivedEstimate int64,
expectedBandwidthUsage int64,
sentPackets uint32,
repeatedNacks uint32,
)
// TWCC sequence number
RecordPacketSendAndGetSequenceNumber(
atMicro int64,
size int,
isRTX bool,
probeClusterId ccutils.ProbeClusterId,
isProbe bool,
) uint16
HandleTWCCFeedback(report *rtcp.TransportLayerCC)
UpdateRTT(rtt float64)
CongestionState() CongestionState
CanProbe() bool
ProbeDuration() time.Duration
ProbeClusterStarting(pci ccutils.ProbeClusterInfo)
ProbeClusterDone(pci ccutils.ProbeClusterInfo)
ProbeClusterIsGoalReached() bool
ProbeClusterFinalize() (ccutils.ProbeSignal, int64, bool)
}
// ------------------------------------------------
type BWEListener interface {
OnCongestionStateChange(fromState CongestionState, toState CongestionState, estimatedAvailableChannelCapacity int64)
}
// ------------------------------------------------
livekit-server/pkg/sfu/bwe/null_bwe.go
+77
View File
@@ -0,0 +1,77 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package bwe
import (
"time"
"github.com/livekit/livekit-server/pkg/sfu/ccutils"
"github.com/pion/rtcp"
)
type NullBWE struct {
}
func (n *NullBWE) SetBWEListener(_bweListener BWEListener) {}
func (n *NullBWE) Reset() {}
func (n *NullBWE) RecordPacketSendAndGetSequenceNumber(
_atMicro int64,
_size int,
_isRTX bool,
_probeClusterId ccutils.ProbeClusterId,
_isProbe bool,
) uint16 {
return 0
}
func (n *NullBWE) HandleREMB(
_receivedEstimate int64,
_expectedBandwidthUsage int64,
_sentPackets uint32,
_repeatedNacks uint32,
) {
}
func (n *NullBWE) HandleTWCCFeedback(_report *rtcp.TransportLayerCC) {}
func (n *NullBWE) UpdateRTT(rtt float64) {}
func (n *NullBWE) CongestionState() CongestionState {
return CongestionStateNone
}
func (n *NullBWE) CanProbe() bool {
return false
}
func (n *NullBWE) ProbeDuration() time.Duration {
return 0
}
func (n *NullBWE) ProbeClusterStarting(_pci ccutils.ProbeClusterInfo) {}
func (n *NullBWE) ProbeClusterDone(_pci ccutils.ProbeClusterInfo) {}
func (n *NullBWE) ProbeClusterIsGoalReached() bool {
return false
}
func (n *NullBWE) ProbeClusterFinalize() (ccutils.ProbeSignal, int64, bool) {
return ccutils.ProbeSignalInconclusive, 0, false
}
// ------------------------------------------------
livekit-server/pkg/sfu/bwe/remotebwe/channel_observer.go
+202
View File
@@ -0,0 +1,202 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package remotebwe
import (
"fmt"
"time"
"go.uber.org/zap/zapcore"
"github.com/livekit/livekit-server/pkg/sfu/ccutils"
"github.com/livekit/protocol/logger"
)
// ------------------------------------------------
type channelTrend int
const (
channelTrendInconclusive channelTrend = iota
channelTrendClearing
channelTrendCongesting
)
func (c channelTrend) String() string {
switch c {
case channelTrendInconclusive:
return "INCONCLUSIVE"
case channelTrendClearing:
return "CLEARING"
case channelTrendCongesting:
return "CONGESTING"
default:
return fmt.Sprintf("%d", int(c))
}
}
// ------------------------------------------------
type channelCongestionReason int
const (
channelCongestionReasonNone channelCongestionReason = iota
channelCongestionReasonEstimate
channelCongestionReasonLoss
)
func (c channelCongestionReason) String() string {
switch c {
case channelCongestionReasonNone:
return "NONE"
case channelCongestionReasonEstimate:
return "ESTIMATE"
case channelCongestionReasonLoss:
return "LOSS"
default:
return fmt.Sprintf("%d", int(c))
}
}
// ------------------------------------------------
type ChannelObserverConfig struct {
Estimate ccutils.TrendDetectorConfig `yaml:"estimate,omitempty"`
Nack NackTrackerConfig `yaml:"nack,omitempty"`
}
var (
defaultTrendDetectorConfigProbe = ccutils.TrendDetectorConfig{
RequiredSamples: 3,
RequiredSamplesMin: 3,
DownwardTrendThreshold: 0.0,
DownwardTrendMaxWait: 5 * time.Second,
CollapseThreshold: 0,
ValidityWindow: 10 * time.Second,
}
defaultChannelObserverConfigProbe = ChannelObserverConfig{
Estimate: defaultTrendDetectorConfigProbe,
Nack: defaultNackTrackerConfigProbe,
}
defaultTrendDetectorConfigNonProbe = ccutils.TrendDetectorConfig{
RequiredSamples: 12,
RequiredSamplesMin: 8,
DownwardTrendThreshold: -0.6,
DownwardTrendMaxWait: 5 * time.Second,
CollapseThreshold: 500 * time.Millisecond,
ValidityWindow: 10 * time.Second,
}
defaultChannelObserverConfigNonProbe = ChannelObserverConfig{
Estimate: defaultTrendDetectorConfigNonProbe,
Nack: defaultNackTrackerConfigNonProbe,
}
)
// ------------------------------------------------
type channelObserverParams struct {
Name string
Config ChannelObserverConfig
}
type channelObserver struct {
params channelObserverParams
logger logger.Logger
estimateTrend *ccutils.TrendDetector[int64]
nackTracker *nackTracker
}
func newChannelObserver(params channelObserverParams, logger logger.Logger) *channelObserver {
return &channelObserver{
params: params,
logger: logger,
estimateTrend: ccutils.NewTrendDetector[int64](ccutils.TrendDetectorParams{
Name: params.Name + "-estimate",
Logger: logger,
Config: params.Config.Estimate,
}),
nackTracker: newNackTracker(nackTrackerParams{
Name: params.Name + "-nack",
Logger: logger,
Config: params.Config.Nack,
}),
}
}
func (c *channelObserver) SeedEstimate(estimate int64) {
c.estimateTrend.Seed(estimate)
}
func (c *channelObserver) AddEstimate(estimate int64) {
c.estimateTrend.AddValue(estimate)
}
func (c *channelObserver) AddNack(packets uint32, repeatedNacks uint32) {
c.nackTracker.Add(packets, repeatedNacks)
}
func (c *channelObserver) GetLowestEstimate() int64 {
return c.estimateTrend.GetLowest()
}
func (c *channelObserver) GetHighestEstimate() int64 {
return c.estimateTrend.GetHighest()
}
func (c *channelObserver) HasEnoughEstimateSamples() bool {
return c.estimateTrend.HasEnoughSamples()
}
func (c *channelObserver) GetNackRatio() float64 {
return c.nackTracker.GetRatio()
}
func (c *channelObserver) GetTrend() (channelTrend, channelCongestionReason) {
estimateDirection := c.estimateTrend.GetDirection()
switch {
case estimateDirection == ccutils.TrendDirectionDownward:
return channelTrendCongesting, channelCongestionReasonEstimate
case c.nackTracker.IsTriggered():
return channelTrendCongesting, channelCongestionReasonLoss
case estimateDirection == ccutils.TrendDirectionUpward:
return channelTrendClearing, channelCongestionReasonNone
}
return channelTrendInconclusive, channelCongestionReasonNone
}
func (c *channelObserver) MarshalLogObject(e zapcore.ObjectEncoder) error {
if c == nil {
return nil
}
e.AddString("name", c.params.Name)
e.AddObject("estimate", c.estimateTrend)
e.AddObject("nack", c.nackTracker)
channelTrend, channelCongestionReason := c.GetTrend()
e.AddString("channelTrend", channelTrend.String())
e.AddString("channelCongestionReason", channelCongestionReason.String())
return nil
}
// ------------------------------------------------
livekit-server/pkg/sfu/bwe/remotebwe/nack_tracker.go
+130
View File
@@ -0,0 +1,130 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package remotebwe
import (
"time"
"go.uber.org/zap/zapcore"
"github.com/livekit/protocol/logger"
)
// ------------------------------------------------
type NackTrackerConfig struct {
WindowMinDuration time.Duration `yaml:"window_min_duration,omitempty"`
WindowMaxDuration time.Duration `yaml:"window_max_duration,omitempty"`
RatioThreshold float64 `yaml:"ratio_threshold,omitempty"`
}
var (
defaultNackTrackerConfigProbe = NackTrackerConfig{
WindowMinDuration: 500 * time.Millisecond,
WindowMaxDuration: 1 * time.Second,
RatioThreshold: 0.04,
}
defaultNackTrackerConfigNonProbe = NackTrackerConfig{
WindowMinDuration: 2 * time.Second,
WindowMaxDuration: 3 * time.Second,
RatioThreshold: 0.08,
}
)
// ------------------------------------------------
type nackTrackerParams struct {
Name string
Logger logger.Logger
Config NackTrackerConfig
}
type nackTracker struct {
params nackTrackerParams
windowStartTime time.Time
packets uint32
repeatedNacks uint32
}
func newNackTracker(params nackTrackerParams) *nackTracker {
return &nackTracker{
params: params,
}
}
func (n *nackTracker) Add(packets uint32, repeatedNacks uint32) {
if n.params.Config.WindowMaxDuration != 0 && !n.windowStartTime.IsZero() && time.Since(n.windowStartTime) > n.params.Config.WindowMaxDuration {
n.windowStartTime = time.Time{}
n.packets = 0
n.repeatedNacks = 0
}
//
// Start NACK monitoring window only when a repeated NACK happens.
// This allows locking tightly to when NACKs start happening and
// check if the NACKs keep adding up (potentially a sign of congestion)
// or isolated losses
//
if n.repeatedNacks == 0 && repeatedNacks != 0 {
n.windowStartTime = time.Now()
}
if !n.windowStartTime.IsZero() {
n.packets += packets
n.repeatedNacks += repeatedNacks
}
}
func (n *nackTracker) GetRatio() float64 {
ratio := 0.0
if n.packets != 0 {
ratio = float64(n.repeatedNacks) / float64(n.packets)
if ratio > 1.0 {
ratio = 1.0
}
}
return ratio
}
func (n *nackTracker) IsTriggered() bool {
if n.params.Config.WindowMinDuration != 0 && !n.windowStartTime.IsZero() && time.Since(n.windowStartTime) > n.params.Config.WindowMinDuration {
return n.GetRatio() > n.params.Config.RatioThreshold
}
return false
}
func (n *nackTracker) MarshalLogObject(e zapcore.ObjectEncoder) error {
if n == nil {
return nil
}
e.AddString("name", n.params.Name)
if n.windowStartTime.IsZero() {
e.AddString("window", "inactive")
} else {
e.AddTime("windowStartTime", n.windowStartTime)
e.AddDuration("windowDuration", time.Since(n.windowStartTime))
e.AddUint32("packets", n.packets)
e.AddUint32("repeatedNacks", n.repeatedNacks)
e.AddFloat64("nackRatio", n.GetRatio())
}
return nil
}
// ------------------------------------------------
livekit-server/pkg/sfu/bwe/remotebwe/probe_controller.go
+184
View File
@@ -0,0 +1,184 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package remotebwe
import (
"fmt"
"time"
"github.com/livekit/livekit-server/pkg/sfu/bwe"
"github.com/livekit/livekit-server/pkg/sfu/ccutils"
"github.com/livekit/protocol/logger"
"github.com/livekit/protocol/utils/mono"
)
// ---------------------------------------------------------------------------
type probeControllerState int
const (
probeControllerStateNone probeControllerState = iota
probeControllerStateProbing
probeControllerStateHangover
)
func (p probeControllerState) String() string {
switch p {
case probeControllerStateNone:
return "NONE"
case probeControllerStateProbing:
return "PROBING"
case probeControllerStateHangover:
return "HANGOVER"
default:
return fmt.Sprintf("%d", int(p))
}
}
// ------------------------------------------------
type ProbeControllerConfig struct {
ProbeRegulator ccutils.ProbeRegulatorConfig `yaml:"probe_regulator,omitempty"`
SettleWaitNumRTT uint32 `yaml:"settle_wait_num_rtt,omitempty"`
SettleWaitMin time.Duration `yaml:"settle_wait_min,omitempty"`
SettleWaitMax time.Duration `yaml:"settle_wait_max,omitempty"`
}
var (
DefaultProbeControllerConfig = ProbeControllerConfig{
ProbeRegulator: ccutils.DefaultProbeRegulatorConfig,
SettleWaitNumRTT: 5,
SettleWaitMin: 250 * time.Millisecond,
SettleWaitMax: 5 * time.Second,
}
)
// ---------------------------------------------------------------------------
type probeControllerParams struct {
Config ProbeControllerConfig
Logger logger.Logger
}
type probeController struct {
params probeControllerParams
state probeControllerState
stateSwitchedAt time.Time
pci ccutils.ProbeClusterInfo
rtt float64
*ccutils.ProbeRegulator
}
func newProbeController(params probeControllerParams) *probeController {
return &probeController{
params: params,
state: probeControllerStateNone,
stateSwitchedAt: mono.Now(),
pci: ccutils.ProbeClusterInfoInvalid,
rtt: bwe.DefaultRTT,
ProbeRegulator: ccutils.NewProbeRegulator(
ccutils.ProbeRegulatorParams{
Config: params.Config.ProbeRegulator,
Logger: params.Logger,
},
),
}
}
func (p *probeController) UpdateRTT(rtt float64) {
if rtt == 0 {
p.rtt = bwe.DefaultRTT
} else {
if p.rtt == 0 {
p.rtt = rtt
} else {
p.rtt = bwe.RTTSmoothingFactor*rtt + (1.0-bwe.RTTSmoothingFactor)*p.rtt
}
}
}
func (p *probeController) GetRTT() float64 {
return p.rtt
}
func (p *probeController) CanProbe() bool {
return p.state == probeControllerStateNone && p.ProbeRegulator.CanProbe()
}
func (p *probeController) IsInProbe() bool {
return p.state != probeControllerStateNone
}
func (p *probeController) ProbeClusterStarting(pci ccutils.ProbeClusterInfo) {
if p.state != probeControllerStateNone {
p.params.Logger.Warnw("unexpected probe controller state", nil, "state", p.state)
}
p.setState(probeControllerStateProbing)
p.pci = pci
}
func (p *probeController) ProbeClusterDone(pci ccutils.ProbeClusterInfo) {
if p.pci.Id != pci.Id {
return
}
p.pci.Result = pci.Result
p.setState(probeControllerStateHangover)
}
func (p *probeController) ProbeClusterIsGoalReached(estimate int64) bool {
if p.pci.Id == ccutils.ProbeClusterIdInvalid {
return false
}
return estimate > int64(p.pci.Goal.DesiredBps)
}
func (p *probeController) MaybeFinalizeProbe() (ccutils.ProbeClusterInfo, bool) {
if p.state != probeControllerStateHangover {
return ccutils.ProbeClusterInfoInvalid, false
}
settleWait := time.Duration(float64(p.params.Config.SettleWaitNumRTT) * p.rtt * float64(time.Second))
if settleWait < p.params.Config.SettleWaitMin {
settleWait = p.params.Config.SettleWaitMin
}
if settleWait > p.params.Config.SettleWaitMax {
settleWait = p.params.Config.SettleWaitMax
}
if time.Since(p.stateSwitchedAt) < settleWait {
return ccutils.ProbeClusterInfoInvalid, false
}
p.setState(probeControllerStateNone)
return p.pci, true
}
func (p *probeController) setState(state probeControllerState) {
if state == p.state {
return
}
p.state = state
p.stateSwitchedAt = mono.Now()
}
// ------------------------------------------------
livekit-server/pkg/sfu/bwe/remotebwe/remote_bwe.go
+353
View File
@@ -0,0 +1,353 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package remotebwe
import (
"sync"
"time"
"github.com/livekit/livekit-server/pkg/sfu/bwe"
"github.com/livekit/livekit-server/pkg/sfu/ccutils"
"github.com/livekit/protocol/logger"
"github.com/livekit/protocol/utils/mono"
)
// ---------------------------------------------------------------------------
type RemoteBWEConfig struct {
NackRatioAttenuator float64 `yaml:"nack_ratio_attenuator,omitempty"`
ExpectedUsageThreshold float64 `yaml:"expected_usage_threshold,omitempty"`
ChannelObserverProbe ChannelObserverConfig `yaml:"channel_observer_probe,omitempty"`
ChannelObserverNonProbe ChannelObserverConfig `yaml:"channel_observer_non_probe,omitempty"`
ProbeController ProbeControllerConfig `yaml:"probe_controller,omitempty"`
}
var (
DefaultRemoteBWEConfig = RemoteBWEConfig{
NackRatioAttenuator: 0.4,
ExpectedUsageThreshold: 0.95,
ChannelObserverProbe: defaultChannelObserverConfigProbe,
ChannelObserverNonProbe: defaultChannelObserverConfigNonProbe,
ProbeController: DefaultProbeControllerConfig,
}
)
// ---------------------------------------------------------------------------
type RemoteBWEParams struct {
Config RemoteBWEConfig
Logger logger.Logger
}
type RemoteBWE struct {
bwe.NullBWE
params RemoteBWEParams
lock sync.RWMutex
lastReceivedEstimate int64
lastExpectedBandwidthUsage int64
committedChannelCapacity int64
probeController *probeController
channelObserver *channelObserver
congestionState bwe.CongestionState
congestionStateSwitchedAt time.Time
bweListener bwe.BWEListener
}
func NewRemoteBWE(params RemoteBWEParams) *RemoteBWE {
r := &RemoteBWE{
params: params,
}
r.Reset()
return r
}
func (r *RemoteBWE) SetBWEListener(bweListener bwe.BWEListener) {
r.lock.Lock()
defer r.lock.Unlock()
r.bweListener = bweListener
}
func (r *RemoteBWE) getBWEListener() bwe.BWEListener {
r.lock.RLock()
defer r.lock.RUnlock()
return r.bweListener
}
func (r *RemoteBWE) Reset() {
r.lock.Lock()
defer r.lock.Unlock()
r.lastReceivedEstimate = 0
r.lastExpectedBandwidthUsage = 0
r.committedChannelCapacity = 100_000_000
r.congestionState = bwe.CongestionStateNone
r.congestionStateSwitchedAt = mono.Now()
r.probeController = newProbeController(probeControllerParams{
Config: r.params.Config.ProbeController,
Logger: r.params.Logger,
})
r.newChannelObserver()
}
func (r *RemoteBWE) HandleREMB(
receivedEstimate int64,
expectedBandwidthUsage int64,
sentPackets uint32,
repeatedNacks uint32,
) {
r.lock.Lock()
r.lastReceivedEstimate = receivedEstimate
r.lastExpectedBandwidthUsage = expectedBandwidthUsage
// in probe, freeze channel observer state if probe causes congestion till the probe is done,
// this is to ensure that probe result is not marked a success,
// an unsuccessful probe will not up allocate any tracks
if r.congestionState != bwe.CongestionStateNone && r.probeController.IsInProbe() {
r.lock.Unlock()
return
}
r.channelObserver.AddEstimate(r.lastReceivedEstimate)
r.channelObserver.AddNack(sentPackets, repeatedNacks)
shouldNotify, fromState, toState, committedChannelCapacity := r.congestionDetectionStateMachine()
r.lock.Unlock()
if shouldNotify {
if bweListener := r.getBWEListener(); bweListener != nil {
bweListener.OnCongestionStateChange(fromState, toState, committedChannelCapacity)
}
}
}
func (r *RemoteBWE) UpdateRTT(rtt float64) {
r.lock.Lock()
defer r.lock.Unlock()
r.probeController.UpdateRTT(rtt)
}
func (r *RemoteBWE) congestionDetectionStateMachine() (bool, bwe.CongestionState, bwe.CongestionState, int64) {
fromState := r.congestionState
toState := r.congestionState
update := false
trend, reason := r.channelObserver.GetTrend()
switch fromState {
case bwe.CongestionStateNone:
if trend == channelTrendCongesting {
if r.probeController.IsInProbe() || r.estimateAvailableChannelCapacity(reason) {
// when in probe, if congested, stays there till probe is done,
// the estimate stays at pre-probe level
toState = bwe.CongestionStateCongested
}
}
case bwe.CongestionStateCongested:
if trend == channelTrendCongesting {
if r.estimateAvailableChannelCapacity(reason) {
// update state as this needs to reset switch time to wait for congestion min duration again
update = true
}
} else {
toState = bwe.CongestionStateNone
}
}
shouldNotify := false
if toState != fromState || update {
fromState, toState = r.updateCongestionState(toState, reason)
shouldNotify = true
}
return shouldNotify, fromState, toState, r.committedChannelCapacity
}
func (r *RemoteBWE) estimateAvailableChannelCapacity(reason channelCongestionReason) bool {
var estimateToCommit int64
switch reason {
case channelCongestionReasonLoss:
estimateToCommit = int64(float64(r.lastExpectedBandwidthUsage) * (1.0 - r.params.Config.NackRatioAttenuator*r.channelObserver.GetNackRatio()))
default:
estimateToCommit = r.lastReceivedEstimate
}
if estimateToCommit > r.lastReceivedEstimate {
estimateToCommit = r.lastReceivedEstimate
}
commitThreshold := int64(r.params.Config.ExpectedUsageThreshold * float64(r.lastExpectedBandwidthUsage))
if estimateToCommit > commitThreshold || r.committedChannelCapacity == estimateToCommit {
return false
}
r.params.Logger.Infow(
"remote bwe: channel congestion detected, applying channel capacity update",
"reason", reason,
"old(bps)", r.committedChannelCapacity,
"new(bps)", estimateToCommit,
"lastReceived(bps)", r.lastReceivedEstimate,
"expectedUsage(bps)", r.lastExpectedBandwidthUsage,
"commitThreshold(bps)", commitThreshold,
"channel", r.channelObserver,
)
r.committedChannelCapacity = estimateToCommit
return true
}
func (r *RemoteBWE) updateCongestionState(state bwe.CongestionState, reason channelCongestionReason) (bwe.CongestionState, bwe.CongestionState) {
r.params.Logger.Debugw(
"remote bwe: congestion state change",
"from", r.congestionState,
"to", state,
"reason", reason,
"committedChannelCapacity", r.committedChannelCapacity,
)
fromState := r.congestionState
r.congestionState = state
r.congestionStateSwitchedAt = mono.Now()
return fromState, r.congestionState
}
func (r *RemoteBWE) CongestionState() bwe.CongestionState {
r.lock.Lock()
defer r.lock.Unlock()
return r.congestionState
}
func (r *RemoteBWE) CanProbe() bool {
r.lock.Lock()
defer r.lock.Unlock()
return r.congestionState == bwe.CongestionStateNone && r.probeController.CanProbe()
}
func (r *RemoteBWE) ProbeDuration() time.Duration {
r.lock.Lock()
defer r.lock.Unlock()
return r.probeController.ProbeDuration()
}
func (r *RemoteBWE) ProbeClusterStarting(pci ccutils.ProbeClusterInfo) {
r.lock.Lock()
defer r.lock.Unlock()
r.lastExpectedBandwidthUsage = int64(pci.Goal.ExpectedUsageBps)
r.params.Logger.Debugw(
"remote bwe: starting probe",
"lastReceived", r.lastReceivedEstimate,
"expectedBandwidthUsage", r.lastExpectedBandwidthUsage,
"channel", r.channelObserver,
)
r.probeController.ProbeClusterStarting(pci)
r.newChannelObserver()
}
func (r *RemoteBWE) ProbeClusterDone(pci ccutils.ProbeClusterInfo) {
r.lock.Lock()
defer r.lock.Unlock()
r.probeController.ProbeClusterDone(pci)
}
func (r *RemoteBWE) ProbeClusterIsGoalReached() bool {
r.lock.Lock()
defer r.lock.Unlock()
if !r.probeController.IsInProbe() ||
r.congestionState != bwe.CongestionStateNone ||
!r.channelObserver.HasEnoughEstimateSamples() {
return false
}
return r.probeController.ProbeClusterIsGoalReached(r.channelObserver.GetHighestEstimate())
}
func (r *RemoteBWE) ProbeClusterFinalize() (ccutils.ProbeSignal, int64, bool) {
r.lock.Lock()
defer r.lock.Unlock()
pci, isFinalized := r.probeController.MaybeFinalizeProbe()
if !isFinalized {
return ccutils.ProbeSignalInconclusive, 0, isFinalized
}
// switch to a non-probe channel observer on probe end,
// reset congestion state to get a fresh trend
pco := r.channelObserver
probeCongestionState := r.congestionState
r.congestionState = bwe.CongestionStateNone
r.newChannelObserver()
r.params.Logger.Infow(
"remote bwe: probe finalized",
"lastReceived", r.lastReceivedEstimate,
"expectedBandwidthUsage", r.lastExpectedBandwidthUsage,
"channel", pco,
"isSignalValid", pco.HasEnoughEstimateSamples(),
"probeClusterInfo", pci,
"rtt", r.probeController.GetRTT(),
)
probeSignal := ccutils.ProbeSignalNotCongesting
if probeCongestionState != bwe.CongestionStateNone {
probeSignal = ccutils.ProbeSignalCongesting
} else if !pco.HasEnoughEstimateSamples() {
probeSignal = ccutils.ProbeSignalInconclusive
} else {
highestEstimate := pco.GetHighestEstimate()
if highestEstimate > r.committedChannelCapacity {
r.committedChannelCapacity = highestEstimate
}
}
r.probeController.ProbeSignal(probeSignal, pci.CreatedAt)
return probeSignal, r.committedChannelCapacity, true
}
func (r *RemoteBWE) newChannelObserver() {
var params channelObserverParams
if r.probeController.IsInProbe() {
params = channelObserverParams{
Name: "probe",
Config: r.params.Config.ChannelObserverProbe,
}
} else {
params = channelObserverParams{
Name: "non-probe",
Config: r.params.Config.ChannelObserverNonProbe,
}
}
r.channelObserver = newChannelObserver(params, r.params.Logger)
}
livekit-server/pkg/sfu/bwe/sendsidebwe/congestion_detector.go
+1122
View File
File diff suppressed because it is too large Load Diff
livekit-server/pkg/sfu/bwe/sendsidebwe/packet_group.go
+310
View File
@@ -0,0 +1,310 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package sendsidebwe
import (
"errors"
"time"
"github.com/livekit/protocol/logger"
"github.com/livekit/protocol/utils"
"go.uber.org/zap/zapcore"
)
// -------------------------------------------------------------
var (
errGroupFinalized = errors.New("packet group is finalized")
errOldPacket = errors.New("packet is older than packet group start")
)
// -------------------------------------------------------------
type PacketGroupConfig struct {
MinPackets int `yaml:"min_packets,omitempty"`
MaxWindowDuration time.Duration `yaml:"max_window_duration,omitempty"`
}
var (
defaultPacketGroupConfig = PacketGroupConfig{
MinPackets: 30,
MaxWindowDuration: 500 * time.Millisecond,
}
)
// -------------------------------------------------------------
type stat struct {
numPackets int
numBytes int
}
func (s *stat) add(size int) {
s.numPackets++
s.numBytes += size
}
func (s *stat) remove(size int) {
s.numPackets--
s.numBytes -= size
}
func (s *stat) getNumPackets() int {
return s.numPackets
}
func (s *stat) getNumBytes() int {
return s.numBytes
}
func (s stat) MarshalLogObject(e zapcore.ObjectEncoder) error {
e.AddInt("numPackets", s.numPackets)
e.AddInt("numBytes", s.numBytes)
return nil
}
// -------------------------------------------------------------
type classStat struct {
primary stat
rtx stat
probe stat
}
func (c *classStat) add(size int, isRTX bool, isProbe bool) {
if isRTX {
c.rtx.add(size)
} else if isProbe {
c.probe.add(size)
} else {
c.primary.add(size)
}
}
func (c *classStat) remove(size int, isRTX bool, isProbe bool) {
if isRTX {
c.rtx.remove(size)
} else if isProbe {
c.probe.remove(size)
} else {
c.primary.remove(size)
}
}
func (c *classStat) numPackets() int {
return c.primary.getNumPackets() + c.rtx.getNumPackets() + c.probe.getNumPackets()
}
func (c *classStat) numBytes() int {
return c.primary.getNumBytes() + c.rtx.getNumBytes() + c.probe.getNumBytes()
}
func (c classStat) MarshalLogObject(e zapcore.ObjectEncoder) error {
e.AddObject("primary", c.primary)
e.AddObject("rtx", c.rtx)
e.AddObject("probe", c.probe)
return nil
}
// -------------------------------------------------------------
type packetGroupParams struct {
Config PacketGroupConfig
WeightedLoss WeightedLossConfig
Logger logger.Logger
}
type packetGroup struct {
params packetGroupParams
minSequenceNumber uint64
maxSequenceNumber uint64
minSendTime int64
maxSendTime int64
minRecvTime int64 // for information only
maxRecvTime int64 // for information only
acked classStat
lost classStat
snBitmap *utils.Bitmap[uint64]
aggregateSendDelta int64
aggregateRecvDelta int64
inheritedQueuingDelay int64
isFinalized bool
}
func newPacketGroup(params packetGroupParams, inheritedQueuingDelay int64) *packetGroup {
return &packetGroup{
params: params,
inheritedQueuingDelay: inheritedQueuingDelay,
snBitmap: utils.NewBitmap[uint64](params.Config.MinPackets),
}
}
func (p *packetGroup) Add(pi *packetInfo, sendDelta, recvDelta int64, isLost bool) error {
if isLost {
return p.lostPacket(pi)
}
if err := p.inGroup(pi.sequenceNumber); err != nil {
return err
}
if p.minSequenceNumber == 0 || pi.sequenceNumber < p.minSequenceNumber {
p.minSequenceNumber = pi.sequenceNumber
}
p.maxSequenceNumber = max(p.maxSequenceNumber, pi.sequenceNumber)
if p.minSendTime == 0 || (pi.sendTime-sendDelta) < p.minSendTime {
p.minSendTime = pi.sendTime - sendDelta
}
p.maxSendTime = max(p.maxSendTime, pi.sendTime)
if p.minRecvTime == 0 || (pi.recvTime-recvDelta) < p.minRecvTime {
p.minRecvTime = pi.recvTime - recvDelta
}
p.maxRecvTime = max(p.maxRecvTime, pi.recvTime)
p.acked.add(int(pi.size), pi.isRTX, pi.isProbe)
if int(pi.sequenceNumber-p.minSequenceNumber) < p.snBitmap.Len() && p.snBitmap.IsSet(pi.sequenceNumber-p.minSequenceNumber) {
// an earlier packet reported as lost has been received
p.snBitmap.Clear(pi.sequenceNumber - p.minSequenceNumber)
p.lost.remove(int(pi.size), pi.isRTX, pi.isProbe)
}
// note that out-of-order deliveries will amplify the queueing delay.
// for e.g. a, b, c getting delivered as a, c, b.
// let us say packets are delivered with interval of `x`
// send delta aggregate will go up by x((a, c) = 2x + (c, b) -1x)
// recv delta aggregate will go up by 3x((a, c) = 2x + (c, b) 1x)
p.aggregateSendDelta += sendDelta
p.aggregateRecvDelta += recvDelta
if p.acked.numPackets() == p.params.Config.MinPackets || (pi.sendTime-p.minSendTime) > p.params.Config.MaxWindowDuration.Microseconds() {
p.isFinalized = true
}
return nil
}
func (p *packetGroup) lostPacket(pi *packetInfo) error {
if pi.recvTime != 0 {
// previously received packet, so not lost
return nil
}
if err := p.inGroup(pi.sequenceNumber); err != nil {
return err
}
if p.minSequenceNumber == 0 || pi.sequenceNumber < p.minSequenceNumber {
p.minSequenceNumber = pi.sequenceNumber
}
p.maxSequenceNumber = max(p.maxSequenceNumber, pi.sequenceNumber)
p.snBitmap.Set(pi.sequenceNumber - p.minSequenceNumber)
p.lost.add(int(pi.size), pi.isRTX, pi.isProbe)
return nil
}
func (p *packetGroup) MinSendTime() int64 {
return p.minSendTime
}
func (p *packetGroup) SendWindow() (int64, int64) {
return p.minSendTime, p.maxSendTime
}
func (p *packetGroup) PropagatedQueuingDelay() int64 {
if p.inheritedQueuingDelay+p.aggregateRecvDelta-p.aggregateSendDelta > 0 {
return p.inheritedQueuingDelay + p.aggregateRecvDelta - p.aggregateSendDelta
}
return max(0, p.aggregateRecvDelta-p.aggregateSendDelta)
}
func (p *packetGroup) FinalizedPropagatedQueuingDelay() (int64, bool) {
if !p.isFinalized {
return 0, false
}
return p.PropagatedQueuingDelay(), true
}
func (p *packetGroup) IsFinalized() bool {
return p.isFinalized
}
func (p *packetGroup) Traffic() *trafficStats {
return &trafficStats{
minSendTime: p.minSendTime,
maxSendTime: p.maxSendTime,
sendDelta: p.aggregateSendDelta,
recvDelta: p.aggregateRecvDelta,
ackedPackets: p.acked.numPackets(),
ackedBytes: p.acked.numBytes(),
lostPackets: p.lost.numPackets(),
lostBytes: p.lost.numBytes(),
}
}
func (p *packetGroup) MarshalLogObject(e zapcore.ObjectEncoder) error {
if p == nil {
return nil
}
e.AddUint64("minSequenceNumber", p.minSequenceNumber)
e.AddUint64("maxSequenceNumber", p.maxSequenceNumber)
e.AddObject("acked", p.acked)
e.AddObject("lost", p.lost)
e.AddInt64("minRecvTime", p.minRecvTime)
e.AddInt64("maxRecvTime", p.maxRecvTime)
recvDuration := time.Duration((p.maxRecvTime - p.minRecvTime) * 1000)
e.AddDuration("recvDuration", recvDuration)
recvBitrate := float64(0)
if recvDuration != 0 {
recvBitrate = float64(p.acked.numBytes()*8) / recvDuration.Seconds()
e.AddFloat64("recvBitrate", recvBitrate)
}
ts := newTrafficStats(trafficStatsParams{
Config: p.params.WeightedLoss,
Logger: p.params.Logger,
})
ts.Merge(p.Traffic())
e.AddObject("trafficStats", ts)
e.AddInt64("inheritedQueuingDelay", p.inheritedQueuingDelay)
e.AddInt64("propagatedQueuingDelay", p.PropagatedQueuingDelay())
e.AddBool("isFinalized", p.isFinalized)
return nil
}
func (p *packetGroup) inGroup(sequenceNumber uint64) error {
if p.isFinalized && sequenceNumber > p.maxSequenceNumber {
return errGroupFinalized
}
if sequenceNumber < p.minSequenceNumber {
return errOldPacket
}
return nil
}
livekit-server/pkg/sfu/bwe/sendsidebwe/packet_info.go
+45
View File
@@ -0,0 +1,45 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package sendsidebwe
import (
"github.com/livekit/livekit-server/pkg/sfu/ccutils"
"go.uber.org/zap/zapcore"
)
type packetInfo struct {
sequenceNumber uint64
sendTime int64
recvTime int64
probeClusterId ccutils.ProbeClusterId
size uint16
isRTX bool
isProbe bool
}
func (pi *packetInfo) MarshalLogObject(e zapcore.ObjectEncoder) error {
if pi == nil {
return nil
}
e.AddUint64("sequenceNumber", pi.sequenceNumber)
e.AddInt64("sendTime", pi.sendTime)
e.AddInt64("recvTime", pi.recvTime)
e.AddUint32("probeClusterId", uint32(pi.probeClusterId))
e.AddUint16("size", pi.size)
e.AddBool("isRTX", pi.isRTX)
e.AddBool("isProbe", pi.isProbe)
return nil
}
livekit-server/pkg/sfu/bwe/sendsidebwe/packet_tracker.go
+168
View File
@@ -0,0 +1,168 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package sendsidebwe
import (
"math/rand"
"sync"
"github.com/livekit/livekit-server/pkg/sfu/ccutils"
"github.com/livekit/protocol/logger"
"github.com/livekit/protocol/utils/mono"
)
// -------------------------------------------------------------------------------
type packetTrackerParams struct {
Logger logger.Logger
}
type packetTracker struct {
params packetTrackerParams
lock sync.Mutex
sequenceNumber uint64
baseSendTime int64
packetInfos [2048]packetInfo
baseRecvTime int64
piLastRecv *packetInfo
probeClusterId ccutils.ProbeClusterId
probeMaxSequenceNumber uint64
}
func newPacketTracker(params packetTrackerParams) *packetTracker {
return &packetTracker{
params: params,
sequenceNumber: uint64(rand.Intn(1<<14)) + uint64(1<<15), // a random number in third quartile of sequence number space
}
}
func (p *packetTracker) RecordPacketSendAndGetSequenceNumber(
atMicro int64,
size int,
isRTX bool,
probeClusterId ccutils.ProbeClusterId,
isProbe bool,
) uint16 {
p.lock.Lock()
defer p.lock.Unlock()
if p.baseSendTime == 0 {
p.baseSendTime = atMicro
}
pi := p.getPacketInfo(uint16(p.sequenceNumber))
*pi = packetInfo{
sequenceNumber: p.sequenceNumber,
sendTime: atMicro - p.baseSendTime,
size: uint16(size),
isRTX: isRTX,
probeClusterId: probeClusterId,
isProbe: isProbe,
}
p.sequenceNumber++
// extreme case of wrap around before receiving any feedback
if pi == p.piLastRecv {
p.piLastRecv = nil
}
if p.probeClusterId != ccutils.ProbeClusterIdInvalid && p.probeClusterId == pi.probeClusterId && pi.sequenceNumber > p.probeMaxSequenceNumber {
p.probeMaxSequenceNumber = pi.sequenceNumber
}
return uint16(pi.sequenceNumber)
}
func (p *packetTracker) BaseSendTimeThreshold(threshold int64) (int64, bool) {
p.lock.Lock()
defer p.lock.Unlock()
if p.baseSendTime == 0 {
return 0, false
}
return mono.UnixMicro() - p.baseSendTime - threshold, true
}
func (p *packetTracker) RecordPacketIndicationFromRemote(sn uint16, recvTime int64) (piRecv packetInfo, sendDelta, recvDelta int64) {
p.lock.Lock()
defer p.lock.Unlock()
pi := p.getPacketInfoExisting(sn)
if pi == nil {
return
}
if recvTime == 0 {
// maybe lost OR already received but reported lost in a later report
piRecv = *pi
return
}
if p.baseRecvTime == 0 {
p.baseRecvTime = recvTime
p.piLastRecv = pi
}
pi.recvTime = recvTime - p.baseRecvTime
piRecv = *pi
if p.piLastRecv != nil {
sendDelta, recvDelta = pi.sendTime-p.piLastRecv.sendTime, pi.recvTime-p.piLastRecv.recvTime
}
p.piLastRecv = pi
return
}
func (p *packetTracker) getPacketInfo(sn uint16) *packetInfo {
return &p.packetInfos[int(sn)%len(p.packetInfos)]
}
func (p *packetTracker) getPacketInfoExisting(sn uint16) *packetInfo {
pi := &p.packetInfos[int(sn)%len(p.packetInfos)]
if uint16(pi.sequenceNumber) == sn {
return pi
}
return nil
}
func (p *packetTracker) ProbeClusterStarting(probeClusterId ccutils.ProbeClusterId) {
p.lock.Lock()
defer p.lock.Unlock()
p.probeClusterId = probeClusterId
}
func (p *packetTracker) ProbeClusterDone(probeClusterId ccutils.ProbeClusterId) {
p.lock.Lock()
defer p.lock.Unlock()
if p.probeClusterId == probeClusterId {
p.probeClusterId = ccutils.ProbeClusterIdInvalid
}
}
func (p *packetTracker) ProbeMaxSequenceNumber() uint64 {
p.lock.Lock()
defer p.lock.Unlock()
return p.probeMaxSequenceNumber
}
livekit-server/pkg/sfu/bwe/sendsidebwe/probe_packet_group.go
+137
View File
@@ -0,0 +1,137 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package sendsidebwe
import (
"time"
"github.com/livekit/livekit-server/pkg/sfu/ccutils"
"github.com/livekit/protocol/logger"
"github.com/livekit/protocol/utils/mono"
"go.uber.org/zap/zapcore"
)
// -------------------------------------------------------------
type ProbePacketGroupConfig struct {
PacketGroup PacketGroupConfig `yaml:"packet_group,omitempty"`
SettleWaitNumRTT uint32 `yaml:"settle_wait_num_rtt,omitempty"`
SettleWaitMin time.Duration `yaml:"settle_wait_min,omitempty"`
SettleWaitMax time.Duration `yaml:"settle_wait_max,omitempty"`
}
var (
// large numbers to treat a probe packet group as one
defaultProbePacketGroupConfig = ProbePacketGroupConfig{
PacketGroup: PacketGroupConfig{
MinPackets: 16384,
MaxWindowDuration: time.Minute,
},
SettleWaitNumRTT: 5,
SettleWaitMin: 250 * time.Millisecond,
SettleWaitMax: 5 * time.Second,
}
)
// -------------------------------------------------------------
type probePacketGroupParams struct {
Config ProbePacketGroupConfig
WeightedLoss WeightedLossConfig
Logger logger.Logger
}
type probePacketGroup struct {
params probePacketGroupParams
pci ccutils.ProbeClusterInfo
*packetGroup
maxSequenceNumber uint64
doneAt time.Time
}
func newProbePacketGroup(params probePacketGroupParams, pci ccutils.ProbeClusterInfo) *probePacketGroup {
return &probePacketGroup{
params: params,
pci: pci,
packetGroup: newPacketGroup(
packetGroupParams{
Config: params.Config.PacketGroup,
WeightedLoss: params.WeightedLoss,
Logger: params.Logger,
},
0,
),
}
}
func (p *probePacketGroup) ProbeClusterDone(pci ccutils.ProbeClusterInfo) {
if p.pci.Id != pci.Id {
return
}
p.pci.Result = pci.Result
p.doneAt = mono.Now()
}
func (p *probePacketGroup) ProbeClusterInfo() ccutils.ProbeClusterInfo {
return p.pci
}
func (p *probePacketGroup) MaybeFinalizeProbe(maxSequenceNumber uint64, rtt float64) (ccutils.ProbeClusterInfo, bool) {
if p.doneAt.IsZero() {
return ccutils.ProbeClusterInfoInvalid, false
}
if maxSequenceNumber != 0 && p.maxSequenceNumber >= maxSequenceNumber {
return p.pci, true
}
settleWait := time.Duration(float64(p.params.Config.SettleWaitNumRTT) * rtt * float64(time.Second))
if settleWait < p.params.Config.SettleWaitMin {
settleWait = p.params.Config.SettleWaitMin
}
if settleWait > p.params.Config.SettleWaitMax {
settleWait = p.params.Config.SettleWaitMax
}
if time.Since(p.doneAt) < settleWait {
return ccutils.ProbeClusterInfoInvalid, false
}
return p.pci, true
}
func (p *probePacketGroup) Add(pi *packetInfo, sendDelta, recvDelta int64, isLost bool) error {
if pi.probeClusterId != p.pci.Id {
return nil
}
p.maxSequenceNumber = max(p.maxSequenceNumber, pi.sequenceNumber)
return p.packetGroup.Add(pi, sendDelta, recvDelta, isLost)
}
func (p *probePacketGroup) MarshalLogObject(e zapcore.ObjectEncoder) error {
if p == nil {
return nil
}
e.AddObject("pci", p.pci)
e.AddObject("packetGroup", p.packetGroup)
e.AddUint64("maxSequenceNumber", p.maxSequenceNumber)
e.AddTime("doneAt", p.doneAt)
return nil
}
livekit-server/pkg/sfu/bwe/sendsidebwe/send_side_bwe.go
+145
View File
@@ -0,0 +1,145 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package sendsidebwe
import (
"time"
"github.com/livekit/livekit-server/pkg/sfu/bwe"
"github.com/livekit/livekit-server/pkg/sfu/ccutils"
"github.com/livekit/protocol/logger"
"github.com/pion/rtcp"
)
//
// Based on a simplified/modified version of JitterPath paper
// (https://homepage.iis.sinica.edu.tw/papers/lcs/2114-F.pdf)
//
// TWCC feedback is uesed to calcualte delta one-way-delay.
// It is accumulated/propagated to determine in which region
// groups of packets are operating in.
//
// In simplified terms,
// o JQR (Join Queuing Region) is when channel is congested.
// o DQR (Disjoint Queuing Region) is when channel is not.
//
// Packets are grouped and thresholds applied to smooth over
// small variations. For example, in the paper,
// if propagated_queuing_delay + delta_one_way_delay > 0 {
// possibly_operating_in_jqr
// }
// But, in this implementation it is checked at packet group level,
// i. e. using queuing delay and aggreated delta one-way-delay of
// the group and a minimum value threshold is applied before declaring
// that a group is in JQR.
//
// There is also hysteresis to make transisitons smoother, i.e. if the
// metric is above a certain threshold, it is JQR and it is DQR only if it
// is below a certain value and the gap in between those two thresholds
// are treated as interdeterminate groups.
//
// ---------------------------------------------------------------------------
type SendSideBWEConfig struct {
CongestionDetector CongestionDetectorConfig `yaml:"congestion_detector,omitempty"`
}
var (
DefaultSendSideBWEConfig = SendSideBWEConfig{
CongestionDetector: defaultCongestionDetectorConfig,
}
)
// ---------------------------------------------------------------------------
type SendSideBWEParams struct {
Config SendSideBWEConfig
Logger logger.Logger
}
type SendSideBWE struct {
bwe.NullBWE
params SendSideBWEParams
*congestionDetector
}
func NewSendSideBWE(params SendSideBWEParams) *SendSideBWE {
return &SendSideBWE{
params: params,
congestionDetector: newCongestionDetector(congestionDetectorParams{
Config: params.Config.CongestionDetector,
Logger: params.Logger,
}),
}
}
func (s *SendSideBWE) SetBWEListener(bweListener bwe.BWEListener) {
s.congestionDetector.SetBWEListener(bweListener)
}
func (s *SendSideBWE) Reset() {
s.congestionDetector.Reset()
}
func (s *SendSideBWE) RecordPacketSendAndGetSequenceNumber(
atMicro int64,
size int,
isRTX bool,
probeClusterId ccutils.ProbeClusterId,
isProbe bool,
) uint16 {
return s.congestionDetector.RecordPacketSendAndGetSequenceNumber(atMicro, size, isRTX, probeClusterId, isProbe)
}
func (s *SendSideBWE) HandleTWCCFeedback(report *rtcp.TransportLayerCC) {
s.congestionDetector.HandleTWCCFeedback(report)
}
func (s *SendSideBWE) UpdateRTT(rtt float64) {
s.congestionDetector.UpdateRTT(rtt)
}
func (s *SendSideBWE) CongestionState() bwe.CongestionState {
return s.congestionDetector.CongestionState()
}
func (s *SendSideBWE) CanProbe() bool {
return s.congestionDetector.CanProbe()
}
func (s *SendSideBWE) ProbeDuration() time.Duration {
return s.congestionDetector.ProbeDuration()
}
func (s *SendSideBWE) ProbeClusterStarting(pci ccutils.ProbeClusterInfo) {
s.congestionDetector.ProbeClusterStarting(pci)
}
func (s *SendSideBWE) ProbeClusterDone(pci ccutils.ProbeClusterInfo) {
s.congestionDetector.ProbeClusterDone(pci)
}
func (s *SendSideBWE) ProbeClusterIsGoalReached() bool {
return s.congestionDetector.ProbeClusterIsGoalReached()
}
func (s *SendSideBWE) ProbeClusterFinalize() (ccutils.ProbeSignal, int64, bool) {
return s.congestionDetector.ProbeClusterFinalize()
}
// ------------------------------------------------
livekit-server/pkg/sfu/bwe/sendsidebwe/traffic_stats.go
+191
View File
@@ -0,0 +1,191 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package sendsidebwe
import (
"math"
"time"
"github.com/livekit/protocol/logger"
"go.uber.org/zap/zapcore"
)
// -----------------------------------------------------------
type WeightedLossConfig struct {
MinDurationForLossValidity time.Duration `yaml:"min_duration_for_loss_validity,omitempty"`
BaseDuration time.Duration `yaml:"base_duration,omitempty"`
BasePPS int `yaml:"base_pps,omitempty"`
LossPenaltyFactor float64 `yaml:"loss_penalty_factor,omitempty"`
}
var (
defaultWeightedLossConfig = WeightedLossConfig{
MinDurationForLossValidity: 100 * time.Millisecond,
BaseDuration: 500 * time.Millisecond,
BasePPS: 30,
LossPenaltyFactor: 0.25,
}
)
// -----------------------------------------------------------
type trafficStatsParams struct {
Config WeightedLossConfig
Logger logger.Logger
}
type trafficStats struct {
params trafficStatsParams
minSendTime int64
maxSendTime int64
sendDelta int64
recvDelta int64
ackedPackets int
ackedBytes int
lostPackets int
lostBytes int
}
func newTrafficStats(params trafficStatsParams) *trafficStats {
return &trafficStats{
params: params,
}
}
func (ts *trafficStats) Merge(rhs *trafficStats) {
if ts.minSendTime == 0 || rhs.minSendTime < ts.minSendTime {
ts.minSendTime = rhs.minSendTime
}
if rhs.maxSendTime > ts.maxSendTime {
ts.maxSendTime = rhs.maxSendTime
}
ts.sendDelta += rhs.sendDelta
ts.recvDelta += rhs.recvDelta
ts.ackedPackets += rhs.ackedPackets
ts.ackedBytes += rhs.ackedBytes
ts.lostPackets += rhs.lostPackets
ts.lostBytes += rhs.lostBytes
}
func (ts *trafficStats) NumBytes() int {
return ts.ackedBytes + ts.lostBytes
}
func (ts *trafficStats) Duration() int64 {
return ts.maxSendTime - ts.minSendTime
}
func (ts *trafficStats) AcknowledgedBitrate() int64 {
duration := ts.Duration()
if duration == 0 {
return 0
}
ackedBitrate := float64(ts.ackedBytes) * 8 * 1e6 / float64(ts.Duration())
return int64(ackedBitrate * ts.CapturedTrafficRatio())
}
func (ts *trafficStats) CapturedTrafficRatio() float64 {
if ts.recvDelta == 0 {
return 0.0
}
// apply a penalty for lost packets,
// the rationale being packet dropping is a strategy to relieve congestion
// and if they were not dropped, they would have increased queuing delay,
// as it is not possible to know the reason for the losses,
// apply a small penalty to receive delta aggregate to simulate those packets
// building up queuing delay.
return min(1.0, float64(ts.sendDelta)/float64(ts.recvDelta+ts.lossPenalty()))
}
func (ts *trafficStats) WeightedLoss() float64 {
durationMicro := ts.Duration()
if time.Duration(durationMicro*1000) < ts.params.Config.MinDurationForLossValidity {
return 0.0
}
totalPackets := float64(ts.lostPackets + ts.ackedPackets)
pps := totalPackets * 1e6 / float64(durationMicro)
// longer duration, i. e. more time resolution, lower pps is acceptable as the measurement is more stable
deltaDuration := time.Duration(durationMicro*1000) - ts.params.Config.BaseDuration
if deltaDuration < 0 {
deltaDuration = 0
}
threshold := math.Exp(-deltaDuration.Seconds()) * float64(ts.params.Config.BasePPS)
if pps < threshold {
return 0.0
}
lossRatio := float64(0.0)
if totalPackets != 0 {
lossRatio = float64(ts.lostPackets) / totalPackets
}
// Log10 is used to give higher weight for the same loss ratio at higher packet rates,
// for e.g.
// - 10% loss at 20 pps = 0.1 * log10(20) = 0.130
// - 10% loss at 100 pps = 0.1 * log10(100) = 0.2
// - 10% loss at 1000 pps = 0.1 * log10(1000) = 0.3
return lossRatio * math.Log10(pps)
}
func (ts *trafficStats) lossPenalty() int64 {
return int64(float64(ts.recvDelta) * ts.WeightedLoss() * ts.params.Config.LossPenaltyFactor)
}
func (ts *trafficStats) MarshalLogObject(e zapcore.ObjectEncoder) error {
if ts == nil {
return nil
}
e.AddInt64("minSendTime", ts.minSendTime)
e.AddInt64("maxSendTime", ts.maxSendTime)
duration := time.Duration(ts.Duration() * 1000)
e.AddDuration("duration", duration)
e.AddInt("ackedPackets", ts.ackedPackets)
e.AddInt("ackedBytes", ts.ackedBytes)
e.AddInt("lostPackets", ts.lostPackets)
e.AddInt("lostBytes", ts.lostBytes)
bitrate := float64(0)
if duration != 0 {
bitrate = float64(ts.ackedBytes*8) / duration.Seconds()
e.AddFloat64("bitrate", bitrate)
}
e.AddInt64("sendDelta", ts.sendDelta)
e.AddInt64("recvDelta", ts.recvDelta)
e.AddInt64("groupDelay", ts.recvDelta-ts.sendDelta)
totalPackets := ts.lostPackets + ts.ackedPackets
if duration != 0 {
e.AddFloat64("pps", float64(totalPackets)/duration.Seconds())
}
if (totalPackets) != 0 {
e.AddFloat64("rawLoss", float64(ts.lostPackets)/float64(totalPackets))
}
e.AddFloat64("weightedLoss", ts.WeightedLoss())
e.AddInt64("lossPenalty", ts.lossPenalty())
capturedTrafficRatio := ts.CapturedTrafficRatio()
e.AddFloat64("capturedTrafficRatio", capturedTrafficRatio)
e.AddFloat64("estimatedAvailableChannelCapacity", bitrate*capturedTrafficRatio)
return nil
}
livekit-server/pkg/sfu/bwe/sendsidebwe/twcc_feedback.go
+128
View File
@@ -0,0 +1,128 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package sendsidebwe
import (
"errors"
"time"
"github.com/livekit/protocol/logger"
"github.com/pion/rtcp"
"go.uber.org/zap/zapcore"
)
// ------------------------------------------------------
const (
cOutlierReportFactor = 3
cEstimatedFeedbackIntervalAlpha = float64(0.9)
cReferenceTimeMask = (1 << 24) - 1
cReferenceTimeResolution = 64 // 64 ms
)
// ------------------------------------------------------
var (
errFeedbackReportOutOfOrder = errors.New("feedback report out-of-order")
)
// ------------------------------------------------------
type twccFeedbackParams struct {
Logger logger.Logger
}
type twccFeedback struct {
params twccFeedbackParams
lastFeedbackTime time.Time
estimatedFeedbackInterval time.Duration
numReports int
numReportsOutOfOrder int
highestFeedbackCount uint8
cycles int64
highestReferenceTime uint32
}
func newTWCCFeedback(params twccFeedbackParams) *twccFeedback {
return &twccFeedback{
params: params,
}
}
func (t *twccFeedback) ProcessReport(report *rtcp.TransportLayerCC, at time.Time) (int64, bool) {
t.numReports++
if t.lastFeedbackTime.IsZero() {
t.lastFeedbackTime = at
t.highestReferenceTime = report.ReferenceTime
t.highestFeedbackCount = report.FbPktCount
return (t.cycles + int64(report.ReferenceTime)) * cReferenceTimeResolution * 1000, false
}
isOutOfOrder := false
if (report.FbPktCount - t.highestFeedbackCount) > (1 << 7) {
t.numReportsOutOfOrder++
isOutOfOrder = true
}
// reference time wrap around handling
var referenceTime int64
if (report.ReferenceTime-t.highestReferenceTime)&cReferenceTimeMask < (1 << 23) {
if report.ReferenceTime < t.highestReferenceTime {
t.cycles += (1 << 24)
}
t.highestReferenceTime = report.ReferenceTime
referenceTime = t.cycles + int64(report.ReferenceTime)
} else {
cycles := t.cycles
if report.ReferenceTime > t.highestReferenceTime && cycles >= (1<<24) {
cycles -= (1 << 24)
}
referenceTime = cycles + int64(report.ReferenceTime)
}
if !isOutOfOrder {
sinceLast := at.Sub(t.lastFeedbackTime)
if t.estimatedFeedbackInterval == 0 {
t.estimatedFeedbackInterval = sinceLast
} else {
// filter out outliers from estimate
if sinceLast > t.estimatedFeedbackInterval/cOutlierReportFactor && sinceLast < cOutlierReportFactor*t.estimatedFeedbackInterval {
// smoothed version of inter feedback interval
t.estimatedFeedbackInterval = time.Duration(cEstimatedFeedbackIntervalAlpha*float64(t.estimatedFeedbackInterval) + (1.0-cEstimatedFeedbackIntervalAlpha)*float64(sinceLast))
}
}
t.lastFeedbackTime = at
t.highestFeedbackCount = report.FbPktCount
}
return referenceTime * cReferenceTimeResolution * 1000, isOutOfOrder
}
func (t *twccFeedback) MarshalLogObject(e zapcore.ObjectEncoder) error {
if t == nil {
return nil
}
e.AddTime("lastFeedbackTime", t.lastFeedbackTime)
e.AddDuration("estimatedFeedbackInterval", t.estimatedFeedbackInterval)
e.AddInt("numReports", t.numReports)
e.AddInt("numReportsOutOfOrder", t.numReportsOutOfOrder)
e.AddInt64("cycles", t.cycles/(1<<24))
return nil
}
livekit-server/pkg/sfu/ccutils/probe_regulator.go
+106
View File
@@ -0,0 +1,106 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package ccutils
import (
"time"
"github.com/livekit/protocol/logger"
"github.com/livekit/protocol/utils/mono"
)
// ------------------------------------------------
type ProbeRegulatorConfig struct {
BaseInterval time.Duration `yaml:"base_interval,omitempty"`
BackoffFactor float64 `yaml:"backoff_factor,omitempty"`
MaxInterval time.Duration `yaml:"max_interval,omitempty"`
MinDuration time.Duration `yaml:"min_duration,omitempty"`
MaxDuration time.Duration `yaml:"max_duration,omitempty"`
DurationIncreaseFactor float64 `yaml:"duration_increase_factor,omitempty"`
}
var (
DefaultProbeRegulatorConfig = ProbeRegulatorConfig{
BaseInterval: 3 * time.Second,
BackoffFactor: 1.5,
MaxInterval: 2 * time.Minute,
MinDuration: 200 * time.Millisecond,
MaxDuration: 20 * time.Second,
DurationIncreaseFactor: 1.5,
}
)
// ---------------------------------------------------------------------------
type ProbeRegulatorParams struct {
Config ProbeRegulatorConfig
Logger logger.Logger
}
type ProbeRegulator struct {
params ProbeRegulatorParams
probeInterval time.Duration
probeDuration time.Duration
nextProbeEarliestAt time.Time
}
func NewProbeRegulator(params ProbeRegulatorParams) *ProbeRegulator {
return &ProbeRegulator{
params: params,
probeInterval: params.Config.BaseInterval,
probeDuration: params.Config.MinDuration,
nextProbeEarliestAt: mono.Now(),
}
}
func (p *ProbeRegulator) CanProbe() bool {
return mono.Now().After(p.nextProbeEarliestAt)
}
func (p *ProbeRegulator) ProbeDuration() time.Duration {
return p.probeDuration
}
func (p *ProbeRegulator) ProbeSignal(probeSignal ProbeSignal, baseTime time.Time) {
if probeSignal == ProbeSignalCongesting {
// wait longer till next probe
p.probeInterval = time.Duration(p.probeInterval.Seconds()*p.params.Config.BackoffFactor) * time.Second
if p.probeInterval > p.params.Config.MaxInterval {
p.probeInterval = p.params.Config.MaxInterval
}
// revert back to starting with shortest probe
p.probeDuration = p.params.Config.MinDuration
} else {
// probe can be started again after minimal interval as previous congestion signal indicated congestion clearing
p.probeInterval = p.params.Config.BaseInterval
// can do longer probe after a good probe
p.probeDuration = time.Duration(float64(p.probeDuration.Milliseconds())*p.params.Config.DurationIncreaseFactor) * time.Millisecond
if p.probeDuration > p.params.Config.MaxDuration {
p.probeDuration = p.params.Config.MaxDuration
}
}
if baseTime.IsZero() {
p.nextProbeEarliestAt = mono.Now().Add(p.probeInterval)
} else {
p.nextProbeEarliestAt = baseTime.Add(p.probeInterval)
}
}
livekit-server/pkg/sfu/ccutils/probe_signal.go
+40
View File
@@ -0,0 +1,40 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package ccutils
import "fmt"
// ------------------------------------------------
type ProbeSignal int
const (
ProbeSignalInconclusive ProbeSignal = iota
ProbeSignalCongesting
ProbeSignalNotCongesting
)
func (p ProbeSignal) String() string {
switch p {
case ProbeSignalInconclusive:
return "INCONCLUSIVE"
case ProbeSignalCongesting:
return "CONGESTING"
case ProbeSignalNotCongesting:
return "NOT_CONGESTING"
default:
return fmt.Sprintf("%d", int(p))
}
}
livekit-server/pkg/sfu/ccutils/prober.go
+596
View File
@@ -0,0 +1,596 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Design of Prober
//
// Probing is used to check for existence of excess channel capacity.
// This is especially useful in the downstream direction of SFU.
// SFU forwards audio/video streams from one or more publishers to
// all the subscribers. But, the downstream channel of a subscriber
// may not be big enough to carry all the streams. It is also a time
// varying quantity.
//
// When there is not enough capacity, some streams will be paused.
// To resume a stream, SFU would need to know that the channel has
// enough capacity. That's where probing comes in. When conditions
// are favorable, SFU can send probe packets so that the bandwidth
// estimator has more data to estimate available channel capacity
// better.
// NOTE: What defines `favorable conditions` is implementation dependent.
//
// There are two options for probing
// - Use padding only RTP packets: This one is preferable as
// probe rate can be controlled more tightly.
// - Resume a paused stream or forward a higher spatial layer:
// Have to find a stream at probing rate. Also, a stream could
// get a key frame unexpectedly boosting rate in the probing
// window.
//
// The strategy used depends on stream allocator implementation.
// This module can be used if the stream allocator decides to use
// padding only RTP packets for probing purposes.
//
// Implementation:
// There are a couple of options
// - Check prober in the forwarding path (pull from prober).
// This is preferred for scalability reasons. But, this
// suffers from not being able to probe when all streams
// are paused (could be due to downstream bandwidth
// constraints or the corresponding upstream tracks may
// have paused due to upstream bandwidth constraints).
// Another issue is not being able to have tight control on
// probing window boundary as the packet forwarding path
// may not have a packet to forward. But, it should not
// be a major concern as long as some stream(s) is/are
// forwarded as there should be a packet at least every
// 60 ms or so (forwarding only one stream at 15 fps).
// Usually, it will be serviced much more frequently when
// there are multiple streams getting forwarded.
// - Run it a go routine. But, that would have to wake up
// very often to prevent bunching up of probe
// packets. So, a scalability concern as there is one prober
// per subscriber peer connection. But, probe windows
// should be very short (of the order of 100s of ms).
// So, this approach might be fine.
//
// The implementation here follows the second approach of using a
// go routine.
//
// Pacing:
// ------
// Ideally, the subscriber peer connection should have a pacer which
// trickles data out at the estimated channel capacity rate (and
// estimated channel capacity + probing rate when actively probing).
//
// But, there a few significant challenges
// 1. Pacer will require buffering of forwarded packets. That means
// more memory, more CPU (have to make copy of packets) and
// more latency in the media stream.
// 2. Scalability concern as SFU may be handling hundreds of
// subscriber peer connections and each one processing the pacing
// loop at 5ms interval will add up.
//
// So, this module assumes that pacing is inherently provided by the
// publishers for media streams. That is a reasonable assumption given
// that publishing clients will run their own pacer and pacing data out
// at a steady rate.
//
// A further assumption is that if there are multiple publishers for
// a subscriber peer connection, all the publishers are not pacing
// in sync, i.e. each publisher's pacer is completely independent
// and SFU will be receiving the media packets with a good spread and
// not clumped together.
//
// Given those assumptions, this module monitors media send rate and
// adjusts probing packet sends accordingly. Although the probing may
// have a high enough wake up frequency, it is for short windows.
// For example, probing at 5 Mbps for 1/2 second and sending 1000 byte
// probe per iteration will wake up every 1.6 ms. That is very high,
// but should last for 1/2 second or so.
//
// 5 Mbps over 1/2 second = 2.5 Mbps
// 2.5 Mbps = 312500 bytes = 313 probes at 1000 byte probes
// 313 probes over 1/2 second = 1.6 ms between probes
//
// A few things to note
// 1. When a probe cluster is added, the expected media rate is provided.
// So, the wake-up interval takes that into account. For example,
// if probing at 5 Mbps for 1/2 second and if 4 Mbps of it is expected
// to be provided by media traffic, the wake-up interval becomes 8 ms.
// 2. The amount of probing should actually be capped at some value to
// avoid too much self-induced congestion. It maybe something like 500 kbps.
// That will increase the wake-up interval to 16 ms in the above example.
// 3. In practice, the probing interval may also be shorter. Typically,
// it can be run for 2 - 3 RTTs to get a good measurement. For
// the longest hauls, RTT could be 250 ms or so leading to the probing
// window being long(ish). But, RTT should be much shorter especially if
// the subscriber peer connection of the client is able to connect to
// the nearest data center.
package ccutils
import (
"fmt"
"math"
"sync"
"time"
"github.com/gammazero/deque"
"go.uber.org/atomic"
"go.uber.org/zap/zapcore"
"github.com/livekit/protocol/logger"
"github.com/livekit/protocol/utils/mono"
)
type ProberListener interface {
OnProbeClusterSwitch(info ProbeClusterInfo)
OnSendProbe(bytesToSend int)
}
type ProberParams struct {
Listener ProberListener
Logger logger.Logger
}
type Prober struct {
params ProberParams
clusterId atomic.Uint32
clustersMu sync.RWMutex
clusters deque.Deque[*Cluster]
activeCluster *Cluster
}
func NewProber(params ProberParams) *Prober {
p := &Prober{
params: params,
}
p.clusters.SetBaseCap(2)
return p
}
func (p *Prober) IsRunning() bool {
p.clustersMu.RLock()
defer p.clustersMu.RUnlock()
return p.clusters.Len() > 0
}
func (p *Prober) Reset(info ProbeClusterInfo) {
p.clustersMu.Lock()
defer p.clustersMu.Unlock()
if p.activeCluster != nil && p.activeCluster.Id() == info.Id {
p.activeCluster.MarkCompleted(info.Result)
p.params.Logger.Debugw("prober: resetting active cluster", "cluster", p.activeCluster)
}
p.clusters.Clear()
p.activeCluster = nil
}
func (p *Prober) AddCluster(mode ProbeClusterMode, pcg ProbeClusterGoal) ProbeClusterInfo {
if pcg.DesiredBps <= 0 {
return ProbeClusterInfoInvalid
}
clusterId := ProbeClusterId(p.clusterId.Inc())
cluster := newCluster(clusterId, mode, pcg, p.params.Listener)
p.params.Logger.Debugw("cluster added", "cluster", cluster)
p.pushBackClusterAndMaybeStart(cluster)
return cluster.Info()
}
func (p *Prober) ProbesSent(bytesSent int) {
cluster := p.getFrontCluster()
if cluster == nil {
return
}
cluster.ProbesSent(bytesSent)
}
func (p *Prober) ClusterDone(info ProbeClusterInfo) {
cluster := p.getFrontCluster()
if cluster == nil {
return
}
if cluster.Id() == info.Id {
cluster.MarkCompleted(info.Result)
p.params.Logger.Debugw("cluster done", "cluster", cluster)
p.popFrontCluster(cluster)
}
}
func (p *Prober) GetActiveClusterId() ProbeClusterId {
p.clustersMu.RLock()
defer p.clustersMu.RUnlock()
if p.activeCluster != nil {
return p.activeCluster.Id()
}
return ProbeClusterIdInvalid
}
func (p *Prober) getFrontCluster() *Cluster {
p.clustersMu.Lock()
defer p.clustersMu.Unlock()
if p.activeCluster != nil {
return p.activeCluster
}
if p.clusters.Len() == 0 {
p.activeCluster = nil
} else {
p.activeCluster = p.clusters.Front()
p.activeCluster.Start()
}
return p.activeCluster
}
func (p *Prober) popFrontCluster(cluster *Cluster) {
p.clustersMu.Lock()
if p.clusters.Len() == 0 {
p.activeCluster = nil
p.clustersMu.Unlock()
return
}
if p.clusters.Front() == cluster {
p.clusters.PopFront()
}
if cluster == p.activeCluster {
p.activeCluster = nil
}
p.clustersMu.Unlock()
}
func (p *Prober) pushBackClusterAndMaybeStart(cluster *Cluster) {
p.clustersMu.Lock()
p.clusters.PushBack(cluster)
if p.clusters.Len() == 1 {
go p.run()
}
p.clustersMu.Unlock()
}
func (p *Prober) run() {
ticker := time.NewTicker(100 * time.Millisecond)
defer ticker.Stop()
for {
cluster := p.getFrontCluster()
if cluster == nil {
return
}
sleepDuration := cluster.Process()
if sleepDuration == 0 {
p.popFrontCluster(cluster)
continue
}
ticker.Reset(sleepDuration)
<-ticker.C
}
}
// ---------------------------------
type ProbeClusterId uint32
const (
ProbeClusterIdInvalid ProbeClusterId = 0
// padding only packets are 255 bytes max + 20 byte header = 4 packets per probe,
// when not using padding only packets, this is a min and actual sent could be higher
cBytesPerProbe = 1100
cSleepDuration = 20 * time.Millisecond
cSleepDurationMin = 10 * time.Millisecond
)
// -----------------------------------
type ProbeClusterMode int
const (
ProbeClusterModeUniform ProbeClusterMode = iota
ProbeClusterModeLinearChirp
)
func (p ProbeClusterMode) String() string {
switch p {
case ProbeClusterModeUniform:
return "UNIFORM"
case ProbeClusterModeLinearChirp:
return "LINEAR_CHIRP"
default:
return fmt.Sprintf("%d", int(p))
}
}
// ---------------------------------------------------------------------------
type ProbeClusterGoal struct {
AvailableBandwidthBps int
ExpectedUsageBps int
DesiredBps int
Duration time.Duration
DesiredBytes int
}
func (p ProbeClusterGoal) MarshalLogObject(e zapcore.ObjectEncoder) error {
e.AddInt("AvailableBandwidthBps", p.AvailableBandwidthBps)
e.AddInt("ExpectedUsageBps", p.ExpectedUsageBps)
e.AddInt("DesiredBps", p.DesiredBps)
e.AddDuration("Duration", p.Duration)
e.AddInt("DesiredBytes", p.DesiredBytes)
return nil
}
type ProbeClusterResult struct {
StartTime int64
EndTime int64
PacketsProbe int
BytesProbe int
PacketsNonProbePrimary int
BytesNonProbePrimary int
PacketsNonProbeRTX int
BytesNonProbeRTX int
IsCompleted bool
}
func (p ProbeClusterResult) Bytes() int {
return p.BytesProbe + p.BytesNonProbePrimary + p.BytesNonProbeRTX
}
func (p ProbeClusterResult) Duration() time.Duration {
return time.Duration(p.EndTime - p.StartTime)
}
func (p ProbeClusterResult) Bitrate() float64 {
duration := p.Duration().Seconds()
if duration != 0 {
return float64(p.Bytes()*8) / duration
}
return 0
}
func (p ProbeClusterResult) MarshalLogObject(e zapcore.ObjectEncoder) error {
e.AddTime("StartTime", time.Unix(0, p.StartTime))
e.AddTime("EndTime", time.Unix(0, p.EndTime))
e.AddDuration("Duration", p.Duration())
e.AddInt("PacketsProbe", p.PacketsProbe)
e.AddInt("BytesProbe", p.BytesProbe)
e.AddInt("PacketsNonProbePrimary", p.PacketsNonProbePrimary)
e.AddInt("BytesNonProbePrimary", p.BytesNonProbePrimary)
e.AddInt("PacketsNonProbeRTX", p.PacketsNonProbeRTX)
e.AddInt("BytesNonProbeRTX", p.BytesNonProbeRTX)
e.AddInt("Bytes", p.Bytes())
e.AddFloat64("Bitrate", p.Bitrate())
e.AddBool("IsCompleted", p.IsCompleted)
return nil
}
type ProbeClusterInfo struct {
Id ProbeClusterId
CreatedAt time.Time
Goal ProbeClusterGoal
Result ProbeClusterResult
}
var (
ProbeClusterInfoInvalid = ProbeClusterInfo{Id: ProbeClusterIdInvalid}
)
func (p ProbeClusterInfo) MarshalLogObject(e zapcore.ObjectEncoder) error {
e.AddUint32("Id", uint32(p.Id))
e.AddTime("CreatedAt", p.CreatedAt)
e.AddObject("Goal", p.Goal)
e.AddObject("Result", p.Result)
return nil
}
// ---------------------------------------------------------------------------
type bucket struct {
expectedElapsedDuration time.Duration
expectedProbeBytesSent int
}
func (b bucket) MarshalLogObject(e zapcore.ObjectEncoder) error {
e.AddDuration("expectedElapsedDuration", b.expectedElapsedDuration)
e.AddInt("expectedProbesBytesSent", b.expectedProbeBytesSent)
return nil
}
// ---------------------------------------------------------------------------
type Cluster struct {
lock sync.RWMutex
info ProbeClusterInfo
mode ProbeClusterMode
listener ProberListener
baseSleepDuration time.Duration
buckets []bucket
bucketIdx int
probeBytesSent int
startTime time.Time
isComplete bool
}
func newCluster(id ProbeClusterId, mode ProbeClusterMode, pcg ProbeClusterGoal, listener ProberListener) *Cluster {
c := &Cluster{
mode: mode,
info: ProbeClusterInfo{
Id: id,
CreatedAt: mono.Now(),
Goal: pcg,
},
listener: listener,
}
c.initProbes()
return c
}
func (c *Cluster) initProbes() {
c.info.Goal.DesiredBytes = int(math.Round(float64(c.info.Goal.DesiredBps)*c.info.Goal.Duration.Seconds()/8 + 0.5))
numBuckets := int(math.Round(c.info.Goal.Duration.Seconds()/cSleepDuration.Seconds() + 0.5))
if numBuckets < 1 {
numBuckets = 1
}
numIntervals := numBuckets
// for linear chirp, group intervals with decreasing duration, i.e. incraasing bitrate,
// by aiming to send same number of bytes in each interval, as intervals get shorter, the bitrate is higher
if c.mode == ProbeClusterModeLinearChirp {
sum := 0
i := 1
for {
sum += i
if sum >= numBuckets {
break
}
i++
}
numBuckets = i
numIntervals = sum
}
c.baseSleepDuration = c.info.Goal.Duration / time.Duration(numIntervals)
if c.baseSleepDuration < cSleepDurationMin {
c.baseSleepDuration = cSleepDurationMin
}
numIntervals = int(math.Round(c.info.Goal.Duration.Seconds()/c.baseSleepDuration.Seconds() + 0.5))
desiredProbeBytesPerInterval := int(math.Round(((c.info.Goal.Duration.Seconds()*float64(c.info.Goal.DesiredBps-c.info.Goal.ExpectedUsageBps)/8)+float64(numIntervals)-1)/float64(numIntervals) + 0.5))
c.buckets = make([]bucket, numBuckets)
for i := 0; i < numBuckets; i++ {
switch c.mode {
case ProbeClusterModeUniform:
c.buckets[i] = bucket{
expectedElapsedDuration: c.baseSleepDuration,
}
case ProbeClusterModeLinearChirp:
c.buckets[i] = bucket{
expectedElapsedDuration: time.Duration(numBuckets-i) * c.baseSleepDuration,
}
}
if i > 0 {
c.buckets[i].expectedElapsedDuration += c.buckets[i-1].expectedElapsedDuration
}
c.buckets[i].expectedProbeBytesSent = (i + 1) * desiredProbeBytesPerInterval
}
}
func (c *Cluster) Start() {
if c.listener != nil {
c.listener.OnProbeClusterSwitch(c.info)
}
}
func (c *Cluster) Id() ProbeClusterId {
return c.info.Id
}
func (c *Cluster) Info() ProbeClusterInfo {
c.lock.RLock()
defer c.lock.RUnlock()
return c.info
}
func (c *Cluster) ProbesSent(bytesSent int) {
c.lock.Lock()
defer c.lock.Unlock()
c.probeBytesSent += bytesSent
}
func (c *Cluster) MarkCompleted(result ProbeClusterResult) {
c.lock.Lock()
defer c.lock.Unlock()
c.isComplete = true
c.info.Result = result
}
func (c *Cluster) Process() time.Duration {
c.lock.Lock()
if c.isComplete {
c.lock.Unlock()
return 0
}
bytesToSend := 0
if c.startTime.IsZero() {
c.startTime = mono.Now()
bytesToSend = cBytesPerProbe
} else {
sinceStart := time.Since(c.startTime)
if sinceStart > c.buckets[c.bucketIdx].expectedElapsedDuration {
c.bucketIdx++
overflow := false
if c.bucketIdx >= len(c.buckets) {
// when overflowing, repeat the last bucket
c.bucketIdx = len(c.buckets) - 1
overflow = true
}
if c.buckets[c.bucketIdx].expectedProbeBytesSent > c.probeBytesSent || overflow {
bytesToSend = max(cBytesPerProbe, c.buckets[c.bucketIdx].expectedProbeBytesSent-c.probeBytesSent)
}
}
}
c.lock.Unlock()
if bytesToSend != 0 && c.listener != nil {
c.listener.OnSendProbe(bytesToSend)
}
return cSleepDurationMin
}
func (c *Cluster) MarshalLogObject(e zapcore.ObjectEncoder) error {
if c != nil {
e.AddString("mode", c.mode.String())
e.AddObject("info", c.info)
e.AddDuration("baseSleepDuration", c.baseSleepDuration)
e.AddInt("numBuckets", len(c.buckets))
e.AddInt("bucketIdx", c.bucketIdx)
e.AddInt("probeBytesSent", c.probeBytesSent)
e.AddTime("startTime", c.startTime)
e.AddDuration("elapsed", time.Since(c.startTime))
e.AddBool("isComplete", c.isComplete)
}
return nil
}
// ----------------------------------------------------------------------
livekit-server/pkg/sfu/ccutils/trenddetector.go
+274
View File
@@ -0,0 +1,274 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package ccutils
import (
"fmt"
"time"
"github.com/livekit/protocol/logger"
"go.uber.org/zap/zapcore"
)
// ------------------------------------------------
type TrendDirection int
const (
TrendDirectionInconclusive TrendDirection = iota
TrendDirectionUpward
TrendDirectionDownward
)
func (t TrendDirection) String() string {
switch t {
case TrendDirectionInconclusive:
return "INCONCLUSIVE"
case TrendDirectionUpward:
return "UPWARD"
case TrendDirectionDownward:
return "DOWNWARD"
default:
return fmt.Sprintf("%d", int(t))
}
}
// ------------------------------------------------
type trendDetectorNumber interface {
int64 | float64
}
// ------------------------------------------------
type trendDetectorSample[T trendDetectorNumber] struct {
value T
at time.Time
}
type trendDetectorSampleElapsed[T trendDetectorNumber] struct {
value T
sinceFirst time.Duration
}
func (t trendDetectorSampleElapsed[T]) MarshalLogObject(e zapcore.ObjectEncoder) error {
e.AddFloat64("value", float64(t.value))
e.AddDuration("sinceFirst", t.sinceFirst)
return nil
}
// ------------------------------------------------
type TrendDetectorConfig struct {
RequiredSamples int `yaml:"required_samples,omitempty"`
RequiredSamplesMin int `yaml:"required_samples_min,omitempty"`
DownwardTrendThreshold float64 `yaml:"downward_trend_threshold,omitempty"`
DownwardTrendMaxWait time.Duration `yaml:"downward_trend_max_wait,omitempty"`
CollapseThreshold time.Duration `yaml:"collapse_threshold,omitempty"`
ValidityWindow time.Duration `yaml:"validity_window,omitempty"`
}
// ------------------------------------------------
type TrendDetectorParams struct {
Name string
Logger logger.Logger
Config TrendDetectorConfig
}
type TrendDetector[T trendDetectorNumber] struct {
params TrendDetectorParams
startTime time.Time
numSamples int
samples []trendDetectorSample[T]
lowestValue T
highestValue T
direction TrendDirection
}
func NewTrendDetector[T trendDetectorNumber](params TrendDetectorParams) *TrendDetector[T] {
return &TrendDetector[T]{
params: params,
startTime: time.Now(),
direction: TrendDirectionInconclusive,
}
}
func (t *TrendDetector[T]) Seed(value T) {
if len(t.samples) != 0 {
return
}
t.samples = append(t.samples, trendDetectorSample[T]{value: value, at: time.Now()})
}
func (t *TrendDetector[T]) AddValue(value T) {
t.numSamples++
if t.lowestValue == 0 || value < t.lowestValue {
t.lowestValue = value
}
if value > t.highestValue {
t.highestValue = value
}
// Ignore duplicate values in collapse window.
//
// Bandwidth estimate is received periodically. If the estimate does not change, it will be repeated.
// When there is congestion, there are several estimates received with decreasing values.
//
// Using a sliding window, collapsing repeated values and waiting for falling trend to ensure that
// the reaction is not too fast, i. e. reacting to falling values too quick could mean a lot of re-allocation
// resulting in layer switches, key frames and more congestion.
//
// But, on the flip side, estimate could fall once or twice within a sliding window and stay there.
// In those cases, using a collapse window to record a value even if it is duplicate. By doing that,
// a trend could be detected eventually. It will be delayed, but that is fine with slow changing estimates.
var lastSample *trendDetectorSample[T]
if len(t.samples) != 0 {
lastSample = &t.samples[len(t.samples)-1]
}
if lastSample != nil && lastSample.value == value && t.params.Config.CollapseThreshold > 0 && time.Since(lastSample.at) < t.params.Config.CollapseThreshold {
return
}
t.samples = append(t.samples, trendDetectorSample[T]{value: value, at: time.Now()})
t.prune()
t.updateDirection()
}
func (t *TrendDetector[T]) GetLowest() T {
return t.lowestValue
}
func (t *TrendDetector[T]) GetHighest() T {
return t.highestValue
}
func (t *TrendDetector[T]) GetDirection() TrendDirection {
return t.direction
}
func (t *TrendDetector[T]) HasEnoughSamples() bool {
return t.numSamples >= t.params.Config.RequiredSamples
}
func (t *TrendDetector[T]) MarshalLogObject(e zapcore.ObjectEncoder) error {
if t == nil {
return nil
}
var samples []trendDetectorSampleElapsed[T]
if len(t.samples) > 0 {
firstTime := t.samples[0].at
for _, sample := range t.samples {
samples = append(samples, trendDetectorSampleElapsed[T]{sample.value, sample.at.Sub(firstTime)})
}
}
e.AddString("name", t.params.Name)
e.AddTime("startTime", t.startTime)
e.AddDuration("elapsed", time.Since(t.startTime))
e.AddInt("numSamples", t.numSamples)
e.AddArray("samples", logger.ObjectSlice(samples))
e.AddFloat64("lowestValue", float64(t.lowestValue))
e.AddFloat64("highestValue", float64(t.highestValue))
e.AddFloat64("kendallsTau", t.kendallsTau())
e.AddString("direction", t.direction.String())
return nil
}
func (t *TrendDetector[T]) prune() {
// prune based on a few rules
// 1. If there are more than required samples
if len(t.samples) > t.params.Config.RequiredSamples {
t.samples = t.samples[len(t.samples)-t.params.Config.RequiredSamples:]
}
// 2. drop samples that are too old
if len(t.samples) != 0 && t.params.Config.ValidityWindow > 0 {
cutoffTime := time.Now().Add(-t.params.Config.ValidityWindow)
cutoffIndex := -1
for i := 0; i < len(t.samples); i++ {
if t.samples[i].at.After(cutoffTime) {
cutoffIndex = i
break
}
}
if cutoffIndex >= 0 {
t.samples = t.samples[cutoffIndex:]
}
}
// 3. collapse same values at the front to just the last of those samples
if len(t.samples) != 0 {
cutoffIndex := -1
firstValue := t.samples[0].value
for i := 1; i < len(t.samples); i++ {
if t.samples[i].value != firstValue {
cutoffIndex = i - 1
break
}
}
if cutoffIndex >= 0 {
t.samples = t.samples[cutoffIndex:]
} else {
// all values are the same, just keep the last one
t.samples = t.samples[len(t.samples)-1:]
}
}
}
func (t *TrendDetector[T]) updateDirection() {
if len(t.samples) < t.params.Config.RequiredSamplesMin {
t.direction = TrendDirectionInconclusive
return
}
// using Kendall's Tau to find trend
kt := t.kendallsTau()
t.direction = TrendDirectionInconclusive
switch {
case kt > 0 && len(t.samples) >= t.params.Config.RequiredSamples:
t.direction = TrendDirectionUpward
case kt < t.params.Config.DownwardTrendThreshold && (len(t.samples) >= t.params.Config.RequiredSamples || t.samples[len(t.samples)-1].at.Sub(t.samples[0].at) > t.params.Config.DownwardTrendMaxWait):
t.direction = TrendDirectionDownward
}
}
func (t *TrendDetector[T]) kendallsTau() float64 {
concordantPairs := 0
discordantPairs := 0
for i := 0; i < len(t.samples)-1; i++ {
for j := i + 1; j < len(t.samples); j++ {
if t.samples[i].value < t.samples[j].value {
concordantPairs++
} else if t.samples[i].value > t.samples[j].value {
discordantPairs++
}
}
}
if (concordantPairs + discordantPairs) == 0 {
return 0.0
}
return (float64(concordantPairs) - float64(discordantPairs)) / (float64(concordantPairs) + float64(discordantPairs))
}
livekit-server/pkg/sfu/codecmunger/codecmunger.go
+39
View File
@@ -0,0 +1,39 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package codecmunger
import (
"errors"
"github.com/livekit/livekit-server/pkg/sfu/buffer"
)
var (
ErrNotVP8 = errors.New("not VP8")
ErrOutOfOrderVP8PictureIdCacheMiss = errors.New("out-of-order VP8 picture id not found in cache")
ErrFilteredVP8TemporalLayer = errors.New("filtered VP8 temporal layer")
)
type CodecMunger interface {
GetState() interface{}
SeedState(state interface{})
SetLast(extPkt *buffer.ExtPacket)
UpdateOffsets(extPkt *buffer.ExtPacket)
UpdateAndGet(extPkt *buffer.ExtPacket, snOutOfOrder bool, snHasGap bool, maxTemporal int32) (int, []byte, error)
UpdateAndGetPadding(newPicture bool) ([]byte, error)
}
livekit-server/pkg/sfu/codecmunger/null.go
+54
View File
@@ -0,0 +1,54 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package codecmunger
import (
"github.com/livekit/livekit-server/pkg/sfu/buffer"
"github.com/livekit/protocol/logger"
)
type Null struct {
seededState interface{}
}
func NewNull(_logger logger.Logger) *Null {
return &Null{}
}
func (n *Null) GetState() interface{} {
return nil
}
func (n *Null) SeedState(state interface{}) {
n.seededState = state
}
func (n *Null) GetSeededState() interface{} {
return n.seededState
}
func (n *Null) SetLast(_extPkt *buffer.ExtPacket) {
}
func (n *Null) UpdateOffsets(_extPkt *buffer.ExtPacket) {
}
func (n *Null) UpdateAndGet(_extPkt *buffer.ExtPacket, snOutOfOrder bool, snHasGap bool, maxTemporal int32) (int, []byte, error) {
return 0, nil, nil
}
func (n *Null) UpdateAndGetPadding(newPicture bool) ([]byte, error) {
return nil, nil
}
livekit-server/pkg/sfu/codecmunger/vp8.go
+480
View File
@@ -0,0 +1,480 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package codecmunger
import (
"github.com/elliotchance/orderedmap/v2"
"github.com/livekit/protocol/livekit"
"github.com/livekit/protocol/logger"
"github.com/livekit/livekit-server/pkg/sfu/buffer"
)
const (
missingPictureIdsThreshold = 50
droppedPictureIdsThreshold = 20
exemptedPictureIdsThreshold = 20
)
// -----------------------------------------------------------
type VP8 struct {
logger logger.Logger
pictureIdWrapHandler VP8PictureIdWrapHandler
extLastPictureId int32
pictureIdOffset int32
pictureIdUsed bool
lastTl0PicIdx uint8
tl0PicIdxOffset uint8
tl0PicIdxUsed bool
tidUsed bool
lastKeyIdx uint8
keyIdxOffset uint8
keyIdxUsed bool
missingPictureIds *orderedmap.OrderedMap[int32, int32]
droppedPictureIds *orderedmap.OrderedMap[int32, bool]
exemptedPictureIds *orderedmap.OrderedMap[int32, bool]
}
func NewVP8(logger logger.Logger) *VP8 {
return &VP8{
logger: logger,
missingPictureIds: orderedmap.NewOrderedMap[int32, int32](),
droppedPictureIds: orderedmap.NewOrderedMap[int32, bool](),
exemptedPictureIds: orderedmap.NewOrderedMap[int32, bool](),
}
}
func NewVP8FromNull(cm CodecMunger, logger logger.Logger) *VP8 {
v := NewVP8(logger)
v.SeedState(cm.(*Null).GetSeededState())
return v
}
func (v *VP8) GetState() interface{} {
return &livekit.VP8MungerState{
ExtLastPictureId: v.extLastPictureId,
PictureIdUsed: v.pictureIdUsed,
LastTl0PicIdx: uint32(v.lastTl0PicIdx),
Tl0PicIdxUsed: v.tl0PicIdxUsed,
TidUsed: v.tidUsed,
LastKeyIdx: uint32(v.lastKeyIdx),
KeyIdxUsed: v.keyIdxUsed,
}
}
func (v *VP8) SeedState(seed interface{}) {
switch cm := seed.(type) {
case *livekit.RTPForwarderState_Vp8Munger:
state := cm.Vp8Munger
v.extLastPictureId = state.ExtLastPictureId
v.pictureIdUsed = state.PictureIdUsed
v.lastTl0PicIdx = uint8(state.LastTl0PicIdx)
v.tl0PicIdxUsed = state.Tl0PicIdxUsed
v.tidUsed = state.TidUsed
v.lastKeyIdx = uint8(state.LastKeyIdx)
v.keyIdxUsed = state.KeyIdxUsed
}
}
func (v *VP8) SetLast(extPkt *buffer.ExtPacket) {
vp8, ok := extPkt.Payload.(buffer.VP8)
if !ok {
return
}
v.pictureIdUsed = vp8.I
if v.pictureIdUsed {
v.pictureIdWrapHandler.Init(int32(vp8.PictureID)-1, vp8.M)
v.extLastPictureId = int32(vp8.PictureID)
}
v.tl0PicIdxUsed = vp8.L
if v.tl0PicIdxUsed {
v.lastTl0PicIdx = vp8.TL0PICIDX
}
v.tidUsed = vp8.T
v.keyIdxUsed = vp8.K
if v.keyIdxUsed {
v.lastKeyIdx = vp8.KEYIDX
}
}
func (v *VP8) UpdateOffsets(extPkt *buffer.ExtPacket) {
vp8, ok := extPkt.Payload.(buffer.VP8)
if !ok {
return
}
if v.pictureIdUsed {
v.pictureIdWrapHandler.Init(int32(vp8.PictureID)-1, vp8.M)
v.pictureIdOffset = int32(vp8.PictureID) - v.extLastPictureId - 1
}
if v.tl0PicIdxUsed {
v.tl0PicIdxOffset = vp8.TL0PICIDX - v.lastTl0PicIdx - 1
}
if v.keyIdxUsed {
v.keyIdxOffset = (vp8.KEYIDX - v.lastKeyIdx - 1) & 0x1f
}
// clear picture id caches on layer switch
v.missingPictureIds = orderedmap.NewOrderedMap[int32, int32]()
v.droppedPictureIds = orderedmap.NewOrderedMap[int32, bool]()
v.exemptedPictureIds = orderedmap.NewOrderedMap[int32, bool]()
}
func (v *VP8) UpdateAndGet(extPkt *buffer.ExtPacket, snOutOfOrder bool, snHasGap bool, maxTemporalLayer int32) (int, []byte, error) {
vp8, ok := extPkt.Payload.(buffer.VP8)
if !ok {
return 0, nil, ErrNotVP8
}
extPictureId := v.pictureIdWrapHandler.Unwrap(vp8.PictureID, vp8.M)
// if out-of-order, look up missing picture id cache
if snOutOfOrder {
pictureIdOffset, ok := v.missingPictureIds.Get(extPictureId)
if !ok {
return 0, nil, ErrOutOfOrderVP8PictureIdCacheMiss
}
// the out-of-order picture id cannot be deleted from the cache
// as there could more than one packet in a picture and more
// than one packet of a picture could come out-of-order.
// To prevent picture id cache from growing, it is truncated
// when it reaches a certain size.
mungedPictureId := uint16((extPictureId - pictureIdOffset) & 0x7fff)
vp8Packet := &buffer.VP8{
FirstByte: vp8.FirstByte,
I: vp8.I,
M: mungedPictureId > 127,
PictureID: mungedPictureId,
L: vp8.L,
TL0PICIDX: vp8.TL0PICIDX - v.tl0PicIdxOffset,
T: vp8.T,
TID: vp8.TID,
Y: vp8.Y,
K: vp8.K,
KEYIDX: vp8.KEYIDX - v.keyIdxOffset,
IsKeyFrame: vp8.IsKeyFrame,
HeaderSize: vp8.HeaderSize + buffer.VPxPictureIdSizeDiff(mungedPictureId > 127, vp8.M),
}
vp8HeaderBytes, err := vp8Packet.Marshal()
if err != nil {
return 0, nil, err
}
return vp8.HeaderSize, vp8HeaderBytes, nil
}
prevMaxPictureId := v.pictureIdWrapHandler.MaxPictureId()
v.pictureIdWrapHandler.UpdateMaxPictureId(extPictureId, vp8.M)
// if there is a gap in sequence number, record possible pictures that
// the missing packets can belong to in missing picture id cache.
// The missing picture cache should contain the previous picture id
// and the current picture id and all the intervening pictures.
// This is to handle a scenario as follows
// o Packet 10 -> Picture ID 10
// o Packet 11 -> missing
// o Packet 12 -> Picture ID 11
// In this case, Packet 11 could belong to either Picture ID 10 (last packet of that picture)
// or Picture ID 11 (first packet of the current picture). Although in this simple case,
// it is possible to deduce that (for example by looking at previous packet's RTP marker
// and check if that was the last packet of Picture 10), it could get complicated when
// the gap is larger.
if snHasGap {
for lostPictureId := prevMaxPictureId; lostPictureId <= extPictureId; lostPictureId++ {
// Record missing only if picture id was not dropped. This is to avoid a subsequent packet of dropped frame going through.
// A sequence like this
// o Packet 10 - Picture 11 - TID that should be dropped
// o Packet 11 - missing - belongs to Picture 11 still
// o Packet 12 - Picture 12 - will be reported as GAP, so missing picture id mapping will be set up for Picture 11 also.
// o Next packet - Packet 11 - this will use the wrong offset from missing pictures cache
_, ok := v.droppedPictureIds.Get(lostPictureId)
if !ok {
v.missingPictureIds.Set(lostPictureId, v.pictureIdOffset)
}
}
// trim cache if necessary
for v.missingPictureIds.Len() > missingPictureIdsThreshold {
el := v.missingPictureIds.Front()
v.missingPictureIds.Delete(el.Key)
}
// if there is a gap, packet is forwarded irrespective of temporal layer as it cannot be determined
// which layer the missing packets belong to. A layer could have multiple packets. So, keep track
// of pictures that are forwarded even though they will be filtered out based on temporal layer
// requirements. That allows forwarding of the complete picture.
if extPkt.Temporal > maxTemporalLayer {
v.exemptedPictureIds.Set(extPictureId, true)
// trim cache if necessary
for v.exemptedPictureIds.Len() > exemptedPictureIdsThreshold {
el := v.exemptedPictureIds.Front()
v.exemptedPictureIds.Delete(el.Key)
}
}
} else {
if extPkt.Temporal > maxTemporalLayer {
// drop only if not exempted
_, ok := v.exemptedPictureIds.Get(extPictureId)
if !ok {
// adjust only once per picture as a picture could have multiple packets
if vp8.I && prevMaxPictureId != extPictureId {
// keep track of dropped picture ids so that they do not get into the missing picture cache
v.droppedPictureIds.Set(extPictureId, true)
// trim cache if necessary
for v.droppedPictureIds.Len() > droppedPictureIdsThreshold {
el := v.droppedPictureIds.Front()
v.droppedPictureIds.Delete(el.Key)
}
v.pictureIdOffset += 1
}
return 0, nil, ErrFilteredVP8TemporalLayer
}
}
}
// in-order incoming sequence number, may or may not be contiguous.
// In the case of loss (i.e. incoming sequence number is not contiguous),
// forward even if it is a filtered layer. With temporal scalability,
// it is unclear if the current packet should be dropped if it is not
// contiguous. Hence, forward anything that is not contiguous.
// Reference: http://www.rtcbits.com/2017/04/howto-implement-temporal-scalability.html
extMungedPictureId := extPictureId - v.pictureIdOffset
mungedPictureId := uint16(extMungedPictureId & 0x7fff)
mungedTl0PicIdx := vp8.TL0PICIDX - v.tl0PicIdxOffset
mungedKeyIdx := (vp8.KEYIDX - v.keyIdxOffset) & 0x1f
v.extLastPictureId = extMungedPictureId
v.lastTl0PicIdx = mungedTl0PicIdx
v.lastKeyIdx = mungedKeyIdx
vp8Packet := &buffer.VP8{
FirstByte: vp8.FirstByte,
I: vp8.I,
M: mungedPictureId > 127,
PictureID: mungedPictureId,
L: vp8.L,
TL0PICIDX: mungedTl0PicIdx,
T: vp8.T,
TID: vp8.TID,
Y: vp8.Y,
K: vp8.K,
KEYIDX: mungedKeyIdx,
IsKeyFrame: vp8.IsKeyFrame,
HeaderSize: vp8.HeaderSize + buffer.VPxPictureIdSizeDiff(mungedPictureId > 127, vp8.M),
}
vp8HeaderBytes, err := vp8Packet.Marshal()
if err != nil {
return 0, nil, err
}
return vp8.HeaderSize, vp8HeaderBytes, nil
}
func (v *VP8) UpdateAndGetPadding(newPicture bool) ([]byte, error) {
offset := 0
if newPicture {
offset = 1
}
headerSize := 1
if v.pictureIdUsed || v.tl0PicIdxUsed || v.tidUsed || v.keyIdxUsed {
headerSize += 1
}
extPictureId := v.extLastPictureId
if v.pictureIdUsed {
extPictureId = v.extLastPictureId + int32(offset)
v.extLastPictureId = extPictureId
v.pictureIdOffset -= int32(offset)
if (extPictureId & 0x7fff) > 127 {
headerSize += 2
} else {
headerSize += 1
}
}
pictureId := uint16(extPictureId & 0x7fff)
tl0PicIdx := uint8(0)
if v.tl0PicIdxUsed {
tl0PicIdx = v.lastTl0PicIdx + uint8(offset)
v.lastTl0PicIdx = tl0PicIdx
v.tl0PicIdxOffset -= uint8(offset)
headerSize += 1
}
if v.tidUsed || v.keyIdxUsed {
headerSize += 1
}
keyIdx := uint8(0)
if v.keyIdxUsed {
keyIdx = (v.lastKeyIdx + uint8(offset)) & 0x1f
v.lastKeyIdx = keyIdx
v.keyIdxOffset -= uint8(offset)
}
vp8Packet := &buffer.VP8{
FirstByte: 0x10, // partition 0, start of VP8 Partition, reference frame
I: v.pictureIdUsed,
M: pictureId > 127,
PictureID: pictureId,
L: v.tl0PicIdxUsed,
TL0PICIDX: tl0PicIdx,
T: v.tidUsed,
TID: 0,
Y: true,
K: v.keyIdxUsed,
KEYIDX: keyIdx,
IsKeyFrame: true,
HeaderSize: headerSize,
}
return vp8Packet.Marshal()
}
// for testing only
func (v *VP8) PictureIdOffset(extPictureId int32) (int32, bool) {
return v.missingPictureIds.Get(extPictureId)
}
// -----------------------------
// VP8PictureIdWrapHandler
func isWrapping7Bit(val1 int32, val2 int32) bool {
return val2 < val1 && (val1-val2) > (1<<6)
}
func isWrapping15Bit(val1 int32, val2 int32) bool {
return val2 < val1 && (val1-val2) > (1<<14)
}
type VP8PictureIdWrapHandler struct {
maxPictureId int32
maxMBit bool
totalWrap int32
lastWrap int32
}
func (v *VP8PictureIdWrapHandler) Init(extPictureId int32, mBit bool) {
v.maxPictureId = extPictureId
v.maxMBit = mBit
v.totalWrap = 0
v.lastWrap = 0
}
func (v *VP8PictureIdWrapHandler) MaxPictureId() int32 {
return v.maxPictureId
}
// unwrap picture id and update the maxPictureId. return unwrapped value
func (v *VP8PictureIdWrapHandler) Unwrap(pictureId uint16, mBit bool) int32 {
//
// VP8 Picture ID is specified very flexibly.
//
// Reference: https://datatracker.ietf.org/doc/html/draft-ietf-payload-vp8
//
// Quoting from the RFC
// ----------------------------
// PictureID: 7 or 15 bits (shown left and right, respectively, in
// Figure 2) not including the M bit. This is a running index of
// the frames, which MAY start at a random value, MUST increase by
// 1 for each subsequent frame, and MUST wrap to 0 after reaching
// the maximum ID (all bits set). The 7 or 15 bits of the
// PictureID go from most significant to least significant,
// beginning with the first bit after the M bit. The sender
// chooses a 7 or 15 bit index and sets the M bit accordingly.
// The receiver MUST NOT assume that the number of bits in
// PictureID stay the same through the session. Having sent a
// 7-bit PictureID with all bits set to 1, the sender may either
// wrap the PictureID to 0, or extend to 15 bits and continue
// incrementing
// ----------------------------
//
// While in practice, senders may not switch between modes indiscriminately,
// it is possible that small picture ids are sent in 7 bits and then switch
// to 15 bits. But, to ensure correctness, this code keeps track of how much
// quantity has wrapped and uses that to figure out if the incoming picture id
// is newer OR out-of-order.
//
maxPictureId := v.maxPictureId
// maxPictureId can be -1 at the start
if maxPictureId > 0 {
if v.maxMBit {
maxPictureId = v.maxPictureId & 0x7fff
} else {
maxPictureId = v.maxPictureId & 0x7f
}
}
var newPictureId int32
if mBit {
newPictureId = int32(pictureId & 0x7fff)
} else {
newPictureId = int32(pictureId & 0x7f)
}
//
// if the new picture id is too far ahead of max, i.e. more than half of last wrap,
// it is out-of-order, unwrap backwards
//
if v.totalWrap > 0 {
if (v.maxPictureId + (v.lastWrap >> 1)) < (newPictureId + v.totalWrap) {
return newPictureId + v.totalWrap - v.lastWrap
}
}
//
// check for wrap around based on mode of previous picture id.
// There are three cases here
// 1. Wrapping from 15-bit -> 8-bit (32767 -> 0)
// 2. Wrapping from 15-bit -> 15-bit (32767 -> 0)
// 3. Wrapping from 8-bit -> 8-bit (127 -> 0)
// In all cases, looking at the mode of previous picture id will
// ensure that we are calculating the wrap properly.
//
wrap := int32(0)
if v.maxMBit {
if isWrapping15Bit(maxPictureId, newPictureId) {
wrap = 1 << 15
}
} else {
if isWrapping7Bit(maxPictureId, newPictureId) {
wrap = 1 << 7
}
}
v.totalWrap += wrap
if wrap != 0 {
v.lastWrap = wrap
}
newPictureId += v.totalWrap
return newPictureId
}
func (v *VP8PictureIdWrapHandler) UpdateMaxPictureId(extPictureId int32, mBit bool) {
v.maxPictureId = extPictureId
v.maxMBit = mBit
}
livekit-server/pkg/sfu/codecmunger/vp8_test.go
+532
View File
@@ -0,0 +1,532 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package codecmunger
import (
"reflect"
"testing"
"github.com/stretchr/testify/require"
"github.com/livekit/protocol/logger"
"github.com/livekit/livekit-server/pkg/sfu/buffer"
"github.com/livekit/livekit-server/pkg/sfu/testutils"
)
func compare(expected *VP8, actual *VP8) bool {
return reflect.DeepEqual(expected.pictureIdWrapHandler, actual.pictureIdWrapHandler) &&
expected.extLastPictureId == actual.extLastPictureId &&
expected.pictureIdOffset == actual.pictureIdOffset &&
expected.pictureIdUsed == actual.pictureIdUsed &&
expected.lastTl0PicIdx == actual.lastTl0PicIdx &&
expected.tl0PicIdxOffset == actual.tl0PicIdxOffset &&
expected.tl0PicIdxUsed == actual.tl0PicIdxUsed &&
expected.tidUsed == actual.tidUsed &&
expected.lastKeyIdx == actual.lastKeyIdx &&
expected.keyIdxOffset == actual.keyIdxOffset &&
expected.keyIdxUsed == actual.keyIdxUsed
}
func newVP8() *VP8 {
return NewVP8(logger.GetLogger())
}
func TestSetLast(t *testing.T) {
v := newVP8()
params := &testutils.TestExtPacketParams{
SequenceNumber: 23333,
Timestamp: 0xabcdef,
SSRC: 0x12345678,
}
vp8 := &buffer.VP8{
FirstByte: 25,
I: true,
M: true,
PictureID: 13467,
L: true,
TL0PICIDX: 233,
T: true,
TID: 13,
Y: true,
K: true,
KEYIDX: 23,
HeaderSize: 6,
IsKeyFrame: true,
}
extPkt, err := testutils.GetTestExtPacketVP8(params, vp8)
require.NoError(t, err)
require.NotNil(t, extPkt)
expectedVP8 := VP8{
pictureIdWrapHandler: VP8PictureIdWrapHandler{
maxPictureId: 13466,
maxMBit: true,
totalWrap: 0,
lastWrap: 0,
},
extLastPictureId: 13467,
pictureIdOffset: 0,
pictureIdUsed: true,
lastTl0PicIdx: 233,
tl0PicIdxOffset: 0,
tl0PicIdxUsed: true,
tidUsed: true,
lastKeyIdx: 23,
keyIdxOffset: 0,
keyIdxUsed: true,
}
v.SetLast(extPkt)
require.True(t, compare(&expectedVP8, v))
}
func TestUpdateOffsets(t *testing.T) {
v := newVP8()
params := &testutils.TestExtPacketParams{
SequenceNumber: 23333,
Timestamp: 0xabcdef,
SSRC: 0x12345678,
}
vp8 := &buffer.VP8{
FirstByte: 25,
I: true,
M: true,
PictureID: 13467,
L: true,
TL0PICIDX: 233,
T: true,
TID: 13,
Y: true,
K: true,
KEYIDX: 23,
HeaderSize: 6,
IsKeyFrame: true,
}
extPkt, _ := testutils.GetTestExtPacketVP8(params, vp8)
v.SetLast(extPkt)
params = &testutils.TestExtPacketParams{
SequenceNumber: 56789,
Timestamp: 0xabcdef,
SSRC: 0x87654321,
}
vp8 = &buffer.VP8{
FirstByte: 25,
I: true,
M: true,
PictureID: 345,
L: true,
TL0PICIDX: 12,
T: true,
TID: 13,
Y: true,
K: true,
KEYIDX: 4,
HeaderSize: 6,
IsKeyFrame: true,
}
extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8)
v.UpdateOffsets(extPkt)
expectedVP8 := VP8{
pictureIdWrapHandler: VP8PictureIdWrapHandler{
maxPictureId: 344,
maxMBit: true,
totalWrap: 0,
lastWrap: 0,
},
extLastPictureId: 13467,
pictureIdOffset: 345 - 13467 - 1,
pictureIdUsed: true,
lastTl0PicIdx: 233,
tl0PicIdxOffset: (12 - 233 - 1) & 0xff,
tl0PicIdxUsed: true,
tidUsed: true,
lastKeyIdx: 23,
keyIdxOffset: (4 - 23 - 1) & 0x1f,
keyIdxUsed: true,
}
require.True(t, compare(&expectedVP8, v))
}
func TestOutOfOrderPictureId(t *testing.T) {
v := newVP8()
params := &testutils.TestExtPacketParams{
SequenceNumber: 23333,
Timestamp: 0xabcdef,
SSRC: 0x12345678,
}
vp8 := &buffer.VP8{
FirstByte: 25,
I: true,
M: true,
PictureID: 13467,
L: true,
TL0PICIDX: 233,
T: true,
TID: 1,
Y: true,
K: true,
KEYIDX: 23,
HeaderSize: 6,
IsKeyFrame: true,
}
extPkt, _ := testutils.GetTestExtPacketVP8(params, vp8)
v.SetLast(extPkt)
v.UpdateAndGet(extPkt, false, false, 2)
// out-of-order sequence number not in the missing picture id cache
vp8.PictureID = 13466
extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8)
nIn, buf, err := v.UpdateAndGet(extPkt, true, false, 2)
require.Error(t, err)
require.ErrorIs(t, err, ErrOutOfOrderVP8PictureIdCacheMiss)
require.Equal(t, 0, nIn)
require.Nil(t, buf)
// create a hole in picture id
vp8.PictureID = 13469
extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8)
expectedVP8 := &buffer.VP8{
FirstByte: 25,
I: true,
M: true,
PictureID: 13469,
L: true,
TL0PICIDX: 233,
T: true,
TID: 1,
Y: true,
K: true,
KEYIDX: 23,
HeaderSize: 6,
IsKeyFrame: true,
}
marshalledVP8, err := expectedVP8.Marshal()
require.NoError(t, err)
nIn, buf, err = v.UpdateAndGet(extPkt, false, true, 2)
require.NoError(t, err)
require.Equal(t, 6, nIn)
require.Equal(t, marshalledVP8, buf)
// all three, the last, the current and the in-between should have been added to missing picture id cache
value, ok := v.PictureIdOffset(13467)
require.True(t, ok)
require.EqualValues(t, 0, value)
value, ok = v.PictureIdOffset(13468)
require.True(t, ok)
require.EqualValues(t, 0, value)
value, ok = v.PictureIdOffset(13469)
require.True(t, ok)
require.EqualValues(t, 0, value)
// out-of-order sequence number should be in the missing picture id cache
vp8.PictureID = 13468
extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8)
expectedVP8 = &buffer.VP8{
FirstByte: 25,
I: true,
M: true,
PictureID: 13468,
L: true,
TL0PICIDX: 233,
T: true,
TID: 1,
Y: true,
K: true,
KEYIDX: 23,
HeaderSize: 6,
IsKeyFrame: true,
}
marshalledVP8, err = expectedVP8.Marshal()
require.NoError(t, err)
nIn, buf, err = v.UpdateAndGet(extPkt, true, false, 2)
require.NoError(t, err)
require.Equal(t, 6, nIn)
require.Equal(t, marshalledVP8, buf)
}
func TestTemporalLayerFiltering(t *testing.T) {
v := newVP8()
params := &testutils.TestExtPacketParams{
SequenceNumber: 23333,
Timestamp: 0xabcdef,
SSRC: 0x12345678,
}
vp8 := &buffer.VP8{
FirstByte: 25,
I: true,
M: true,
PictureID: 13467,
L: true,
TL0PICIDX: 233,
T: true,
TID: 1,
Y: true,
K: true,
KEYIDX: 23,
HeaderSize: 6,
IsKeyFrame: true,
}
extPkt, _ := testutils.GetTestExtPacketVP8(params, vp8)
v.SetLast(extPkt)
// translate
nIn, buf, err := v.UpdateAndGet(extPkt, false, false, 0)
require.Error(t, err)
require.ErrorIs(t, err, ErrFilteredVP8TemporalLayer)
require.Equal(t, 0, nIn)
require.Nil(t, buf)
dropped, _ := v.droppedPictureIds.Get(13467)
require.True(t, dropped)
require.EqualValues(t, 1, v.pictureIdOffset)
// another packet with the same picture id.
// It should be dropped, but offset should not be updated.
params.SequenceNumber = 23334
extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8)
nIn, buf, err = v.UpdateAndGet(extPkt, false, false, 0)
require.Error(t, err)
require.ErrorIs(t, err, ErrFilteredVP8TemporalLayer)
require.Equal(t, 0, nIn)
require.Nil(t, buf)
dropped, _ = v.droppedPictureIds.Get(13467)
require.True(t, dropped)
require.EqualValues(t, 1, v.pictureIdOffset)
// another packet with the same picture id, but a gap in sequence number.
// It should be dropped, but offset should not be updated.
params.SequenceNumber = 23337
extPkt, _ = testutils.GetTestExtPacketVP8(params, vp8)
nIn, buf, err = v.UpdateAndGet(extPkt, false, false, 0)
require.Error(t, err)
require.ErrorIs(t, err, ErrFilteredVP8TemporalLayer)
require.Equal(t, 0, nIn)
require.Nil(t, buf)
dropped, _ = v.droppedPictureIds.Get(13467)
require.True(t, dropped)
require.EqualValues(t, 1, v.pictureIdOffset)
}
func TestGapInSequenceNumberSamePicture(t *testing.T) {
v := newVP8()
params := &testutils.TestExtPacketParams{
SequenceNumber: 65533,
Timestamp: 0xabcdef,
SSRC: 0x12345678,
PayloadSize: 33,
}
vp8 := &buffer.VP8{
FirstByte: 25,
I: true,
M: true,
PictureID: 13467,
L: true,
TL0PICIDX: 233,
T: true,
TID: 1,
Y: true,
K: true,
KEYIDX: 23,
HeaderSize: 6,
IsKeyFrame: true,
}
extPkt, _ := testutils.GetTestExtPacketVP8(params, vp8)
v.SetLast(extPkt)
expectedVP8 := &buffer.VP8{
FirstByte: 25,
I: true,
M: true,
PictureID: 13467,
L: true,
TL0PICIDX: 233,
T: true,
TID: 1,
Y: true,
K: true,
KEYIDX: 23,
HeaderSize: 6,
IsKeyFrame: true,
}
marshalledVP8, err := expectedVP8.Marshal()
require.NoError(t, err)
nIn, buf, err := v.UpdateAndGet(extPkt, false, false, 2)
require.NoError(t, err)
require.Equal(t, 6, nIn)
require.Equal(t, marshalledVP8, buf)
// telling there is a gap in sequence number will add pictures to missing picture cache
expectedVP8 = &buffer.VP8{
FirstByte: 25,
I: true,
M: true,
PictureID: 13467,
L: true,
TL0PICIDX: 233,
T: true,
TID: 1,
Y: true,
K: true,
KEYIDX: 23,
HeaderSize: 6,
IsKeyFrame: true,
}
marshalledVP8, err = expectedVP8.Marshal()
require.NoError(t, err)
nIn, buf, err = v.UpdateAndGet(extPkt, false, true, 2)
require.NoError(t, err)
require.Equal(t, 6, nIn)
require.Equal(t, marshalledVP8, buf)
value, ok := v.PictureIdOffset(13467)
require.True(t, ok)
require.EqualValues(t, 0, value)
}
func TestUpdateAndGetPadding(t *testing.T) {
v := newVP8()
params := &testutils.TestExtPacketParams{
SequenceNumber: 23333,
Timestamp: 0xabcdef,
SSRC: 0x12345678,
PayloadSize: 20,
}
vp8 := &buffer.VP8{
FirstByte: 25,
I: true,
M: true,
PictureID: 13467,
L: true,
TL0PICIDX: 233,
T: true,
TID: 13,
Y: true,
K: true,
KEYIDX: 23,
HeaderSize: 6,
IsKeyFrame: true,
}
extPkt, _ := testutils.GetTestExtPacketVP8(params, vp8)
v.SetLast(extPkt)
// getting padding with repeat of last picture
buf, err := v.UpdateAndGetPadding(false)
require.NoError(t, err)
expectedVP8 := buffer.VP8{
FirstByte: 16,
I: true,
M: true,
PictureID: 13467,
L: true,
TL0PICIDX: 233,
T: true,
TID: 0,
Y: true,
K: true,
KEYIDX: 23,
HeaderSize: 6,
IsKeyFrame: true,
}
marshalledVP8, err := expectedVP8.Marshal()
require.NoError(t, err)
require.Equal(t, marshalledVP8, buf)
// getting padding with new picture
buf, err = v.UpdateAndGetPadding(true)
require.NoError(t, err)
expectedVP8 = buffer.VP8{
FirstByte: 16,
I: true,
M: true,
PictureID: 13468,
L: true,
TL0PICIDX: 234,
T: true,
TID: 0,
Y: true,
K: true,
KEYIDX: 24,
HeaderSize: 6,
IsKeyFrame: true,
}
marshalledVP8, err = expectedVP8.Marshal()
require.NoError(t, err)
require.Equal(t, marshalledVP8, buf)
}
func TestVP8PictureIdWrapHandler(t *testing.T) {
v := &VP8PictureIdWrapHandler{}
v.Init(109, false)
require.Equal(t, int32(109), v.MaxPictureId())
require.False(t, v.maxMBit)
v.UpdateMaxPictureId(109350, true)
require.Equal(t, int32(109350), v.MaxPictureId())
require.True(t, v.maxMBit)
// start with something close to the 15-bit wrap around point
v.Init(32766, true)
// out-of-order, do not wrap
extPictureId := v.Unwrap(32750, true)
require.Equal(t, int32(32750), extPictureId)
require.Equal(t, int32(0), v.totalWrap)
require.Equal(t, int32(0), v.lastWrap)
// wrap at 15-bits
extPictureId = v.Unwrap(5, false)
require.Equal(t, int32(32773), extPictureId) // 15-bit wrap at 32768 + 5 = 32773
require.Equal(t, int32(32768), v.totalWrap)
require.Equal(t, int32(32768), v.lastWrap)
// set things near 7-bit wrap point
v.UpdateMaxPictureId(32893, false) // 32768 + 125
// wrap at 7-bits
extPictureId = v.Unwrap(5, true)
require.Equal(t, int32(32901), extPictureId) // 15-bit wrap at 32768 + 7-bit wrap at 128 + 5 = 32901
require.Equal(t, int32(32896), v.totalWrap) // one 15-bit wrap + one 7-bit wrap
require.Equal(t, int32(128), v.lastWrap)
// a new picture in 7-bit mode much with a gap in between.
// A big enough gap which would have been treated as out-of-order in 7-bit mode.
v.UpdateMaxPictureId(32901, false)
extPictureId = v.Unwrap(73, false)
require.Equal(t, int32(32841), extPictureId) // 15-bit wrap at 32768 + 73 = 32841
// a new picture in 15-bit mode much with a gap in between.
// A big enough gap which would have been treated as out-of-order in 7-bit mode.
v.UpdateMaxPictureId(32901, true)
v.lastWrap = int32(32768)
extPictureId = v.Unwrap(73, false)
require.Equal(t, int32(32969), extPictureId) // 15-bit wrap at 32768 + 7-bit wrap at 128 + 73 = 32969
}
livekit-server/pkg/sfu/connectionquality/connectionstats.go
+499
View File
@@ -0,0 +1,499 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package connectionquality
import (
"sync"
"time"
"github.com/frostbyte73/core"
"go.uber.org/atomic"
"google.golang.org/protobuf/types/known/timestamppb"
"github.com/livekit/protocol/livekit"
"github.com/livekit/protocol/logger"
"github.com/livekit/livekit-server/pkg/sfu/buffer"
"github.com/livekit/livekit-server/pkg/sfu/mime"
"github.com/livekit/livekit-server/pkg/sfu/rtpstats"
)
const (
UpdateInterval = 5 * time.Second
noReceiverReportTooLongThreshold = 30 * time.Second
)
type ConnectionStatsReceiverProvider interface {
GetDeltaStats() map[uint32]*buffer.StreamStatsWithLayers
GetLastSenderReportTime() time.Time
}
type ConnectionStatsSenderProvider interface {
GetDeltaStatsSender() map[uint32]*buffer.StreamStatsWithLayers
GetPrimaryStreamLastReceiverReportTime() time.Time
GetPrimaryStreamPacketsSent() uint64
}
type ConnectionStatsParams struct {
UpdateInterval time.Duration
IncludeRTT bool
IncludeJitter bool
EnableBitrateScore bool
ReceiverProvider ConnectionStatsReceiverProvider
SenderProvider ConnectionStatsSenderProvider
Logger logger.Logger
}
type ConnectionStats struct {
params ConnectionStatsParams
codecMimeType atomic.Value // mime.MimeType
isStarted atomic.Bool
isVideo atomic.Bool
onStatsUpdate func(cs *ConnectionStats, stat *livekit.AnalyticsStat)
lock sync.RWMutex
packetsSent uint64
streamingStartedAt time.Time
scorer *qualityScorer
done core.Fuse
}
func NewConnectionStats(params ConnectionStatsParams) *ConnectionStats {
return &ConnectionStats{
params: params,
scorer: newQualityScorer(qualityScorerParams{
IncludeRTT: params.IncludeRTT,
IncludeJitter: params.IncludeJitter,
EnableBitrateScore: params.EnableBitrateScore,
Logger: params.Logger,
}),
}
}
func (cs *ConnectionStats) StartAt(codecMimeType mime.MimeType, isFECEnabled bool, at time.Time) {
if cs.isStarted.Swap(true) {
return
}
cs.isVideo.Store(mime.IsMimeTypeVideo(codecMimeType))
cs.codecMimeType.Store(codecMimeType)
cs.scorer.StartAt(getPacketLossWeight(codecMimeType, isFECEnabled), at)
go cs.updateStatsWorker()
}
func (cs *ConnectionStats) Start(codecMimeType mime.MimeType, isFECEnabled bool) {
cs.StartAt(codecMimeType, isFECEnabled, time.Now())
}
func (cs *ConnectionStats) Close() {
cs.done.Break()
}
func (cs *ConnectionStats) UpdateCodec(codecMimeType mime.MimeType, isFECEnabled bool) {
cs.isVideo.Store(mime.IsMimeTypeVideo(codecMimeType))
cs.codecMimeType.Store(codecMimeType)
cs.scorer.UpdatePacketLossWeight(getPacketLossWeight(codecMimeType, isFECEnabled))
}
func (cs *ConnectionStats) OnStatsUpdate(fn func(cs *ConnectionStats, stat *livekit.AnalyticsStat)) {
cs.onStatsUpdate = fn
}
func (cs *ConnectionStats) UpdateMuteAt(isMuted bool, at time.Time) {
if cs.done.IsBroken() {
return
}
cs.scorer.UpdateMuteAt(isMuted, at)
}
func (cs *ConnectionStats) UpdateMute(isMuted bool) {
if cs.done.IsBroken() {
return
}
cs.scorer.UpdateMute(isMuted)
}
func (cs *ConnectionStats) AddBitrateTransitionAt(bitrate int64, at time.Time) {
if cs.done.IsBroken() {
return
}
cs.scorer.AddBitrateTransitionAt(bitrate, at)
}
func (cs *ConnectionStats) AddBitrateTransition(bitrate int64) {
if cs.done.IsBroken() {
return
}
cs.scorer.AddBitrateTransition(bitrate)
}
func (cs *ConnectionStats) UpdateLayerMuteAt(isMuted bool, at time.Time) {
if cs.done.IsBroken() {
return
}
cs.scorer.UpdateLayerMuteAt(isMuted, at)
}
func (cs *ConnectionStats) UpdateLayerMute(isMuted bool) {
if cs.done.IsBroken() {
return
}
cs.scorer.UpdateLayerMute(isMuted)
}
func (cs *ConnectionStats) UpdatePauseAt(isPaused bool, at time.Time) {
if cs.done.IsBroken() {
return
}
cs.scorer.UpdatePauseAt(isPaused, at)
}
func (cs *ConnectionStats) UpdatePause(isPaused bool) {
if cs.done.IsBroken() {
return
}
cs.scorer.UpdatePause(isPaused)
}
func (cs *ConnectionStats) AddLayerTransitionAt(distance float64, at time.Time) {
if cs.done.IsBroken() {
return
}
cs.scorer.AddLayerTransitionAt(distance, at)
}
func (cs *ConnectionStats) AddLayerTransition(distance float64) {
if cs.done.IsBroken() {
return
}
cs.scorer.AddLayerTransition(distance)
}
func (cs *ConnectionStats) GetScoreAndQuality() (float32, livekit.ConnectionQuality) {
return cs.scorer.GetMOSAndQuality()
}
func (cs *ConnectionStats) updateScoreWithAggregate(agg *rtpstats.RTPDeltaInfo, lastRTCPAt time.Time, at time.Time) float32 {
var stat windowStat
if agg != nil {
stat.startedAt = agg.StartTime
stat.duration = agg.EndTime.Sub(agg.StartTime)
stat.packets = agg.Packets
stat.packetsPadding = agg.PacketsPadding
stat.packetsLost = agg.PacketsLost
stat.packetsMissing = agg.PacketsMissing
stat.packetsOutOfOrder = agg.PacketsOutOfOrder
stat.bytes = agg.Bytes - agg.HeaderBytes // only use media payload size
stat.rttMax = agg.RttMax
stat.jitterMax = agg.JitterMax
stat.lastRTCPAt = lastRTCPAt
}
if at.IsZero() {
cs.scorer.Update(&stat)
} else {
cs.scorer.UpdateAt(&stat, at)
}
mos, _ := cs.scorer.GetMOSAndQuality()
return mos
}
func (cs *ConnectionStats) updateScoreFromReceiverReport(at time.Time) (float32, map[uint32]*buffer.StreamStatsWithLayers) {
if cs.params.SenderProvider == nil {
return MinMOS, nil
}
streamingStartedAt := cs.updateStreamingStart(at)
if streamingStartedAt.IsZero() {
// not streaming, just return current score
mos, _ := cs.scorer.GetMOSAndQuality()
return mos, nil
}
streams := cs.params.SenderProvider.GetDeltaStatsSender()
if len(streams) == 0 {
// check for receiver report not received for a while
marker := cs.params.SenderProvider.GetPrimaryStreamLastReceiverReportTime()
if marker.IsZero() || streamingStartedAt.After(marker) {
marker = streamingStartedAt
}
if time.Since(marker) > noReceiverReportTooLongThreshold {
// have not received receiver report for a long time when streaming, run with nil stat
return cs.updateScoreWithAggregate(nil, time.Time{}, at), nil
}
// wait for receiver report, return current score
mos, _ := cs.scorer.GetMOSAndQuality()
return mos, nil
}
// delta stat duration could be large due to not receiving receiver report for a long time (for example, due to mute),
// adjust to streaming start if necessary
if streamingStartedAt.After(cs.params.SenderProvider.GetPrimaryStreamLastReceiverReportTime()) {
// last receiver report was before streaming started, wait for next one
mos, _ := cs.scorer.GetMOSAndQuality()
return mos, streams
}
agg := toAggregateDeltaInfo(streams, true)
if agg == nil {
// no receiver report in the window
mos, _ := cs.scorer.GetMOSAndQuality()
return mos, streams
}
if streamingStartedAt.After(agg.StartTime) {
agg.StartTime = streamingStartedAt
}
return cs.updateScoreWithAggregate(agg, time.Time{}, at), streams
}
func (cs *ConnectionStats) updateScoreAt(at time.Time) (float32, map[uint32]*buffer.StreamStatsWithLayers) {
if cs.params.SenderProvider != nil {
// receiver report based quality scoring, use stats from receiver report for scoring
return cs.updateScoreFromReceiverReport(at)
}
if cs.params.ReceiverProvider == nil {
return MinMOS, nil
}
streams := cs.params.ReceiverProvider.GetDeltaStats()
if len(streams) == 0 {
mos, _ := cs.scorer.GetMOSAndQuality()
return mos, nil
}
agg := toAggregateDeltaInfo(streams, false)
if agg == nil {
// no receiver report in the window
mos, _ := cs.scorer.GetMOSAndQuality()
return mos, streams
}
return cs.updateScoreWithAggregate(agg, cs.params.ReceiverProvider.GetLastSenderReportTime(), at), streams
}
func (cs *ConnectionStats) updateStreamingStart(at time.Time) time.Time {
cs.lock.Lock()
defer cs.lock.Unlock()
packetsSent := cs.params.SenderProvider.GetPrimaryStreamPacketsSent()
if packetsSent > cs.packetsSent {
if cs.streamingStartedAt.IsZero() {
// the start could be anywhere after last update, but using `at` as this is not required to be accurate
if at.IsZero() {
cs.streamingStartedAt = time.Now()
} else {
cs.streamingStartedAt = at
}
}
} else {
cs.streamingStartedAt = time.Time{}
}
cs.packetsSent = packetsSent
return cs.streamingStartedAt
}
func (cs *ConnectionStats) getStat() {
score, streams := cs.updateScoreAt(time.Time{})
if cs.onStatsUpdate != nil && len(streams) != 0 {
analyticsStreams := make([]*livekit.AnalyticsStream, 0, len(streams))
for ssrc, stream := range streams {
as := toAnalyticsStream(ssrc, stream.RTPStats, stream.RTPStatsRemoteView)
if as == nil {
continue
}
//
// add video layer if either
// 1. Simulcast - even if there is only one layer per stream as it provides layer id
// 2. A stream has multiple layers
//
if (len(streams) > 1 || len(stream.Layers) > 1) && cs.isVideo.Load() {
for layer, layerStats := range stream.Layers {
avl := toAnalyticsVideoLayer(layer, layerStats)
if avl != nil {
as.VideoLayers = append(as.VideoLayers, avl)
}
}
}
analyticsStreams = append(analyticsStreams, as)
}
if len(analyticsStreams) != 0 {
cs.onStatsUpdate(cs, &livekit.AnalyticsStat{
Score: score,
Streams: analyticsStreams,
Mime: cs.codecMimeType.Load().(mime.MimeType).String(),
})
}
}
}
func (cs *ConnectionStats) updateStatsWorker() {
interval := cs.params.UpdateInterval
if interval == 0 {
interval = UpdateInterval
}
tk := time.NewTicker(interval)
defer tk.Stop()
for {
select {
case <-cs.done.Watch():
return
case <-tk.C:
if cs.done.IsBroken() {
return
}
cs.getStat()
}
}
}
// -----------------------------------------------------------------------
// how much weight to give to packet loss rate when calculating score.
// It is codec dependent.
// For audio:
//
// o Opus without FEC or RED suffers the most through packet loss, hence has the highest weight
// o RED with two packet redundancy can absorb one out of every two packets lost, so packet loss is not as detrimental and therefore lower weight
//
// For video:
//
// o No in-built codec repair available, hence same for all codecs
func getPacketLossWeight(mimeType mime.MimeType, isFecEnabled bool) float64 {
var plw float64
switch {
case mimeType == mime.MimeTypeOpus:
// 2.5%: fall to GOOD, 7.5%: fall to POOR
plw = 8.0
if isFecEnabled {
// 3.75%: fall to GOOD, 11.25%: fall to POOR
plw /= 1.5
}
case mimeType == mime.MimeTypeRED:
// 5%: fall to GOOD, 15.0%: fall to POOR
plw = 4.0
if isFecEnabled {
// 7.5%: fall to GOOD, 22.5%: fall to POOR
plw /= 1.5
}
case mime.IsMimeTypeVideo(mimeType):
// 2%: fall to GOOD, 6%: fall to POOR
plw = 10.0
}
return plw
}
func toAggregateDeltaInfo(streams map[uint32]*buffer.StreamStatsWithLayers, useRemoteView bool) *rtpstats.RTPDeltaInfo {
deltaInfoList := make([]*rtpstats.RTPDeltaInfo, 0, len(streams))
for _, s := range streams {
if useRemoteView {
if s.RTPStatsRemoteView != nil {
deltaInfoList = append(deltaInfoList, s.RTPStatsRemoteView)
}
} else {
if s.RTPStats != nil {
deltaInfoList = append(deltaInfoList, s.RTPStats)
}
}
}
return rtpstats.AggregateRTPDeltaInfo(deltaInfoList)
}
func toAnalyticsStream(
ssrc uint32,
deltaStats *rtpstats.RTPDeltaInfo,
deltaStatsRemoteView *rtpstats.RTPDeltaInfo,
) *livekit.AnalyticsStream {
if deltaStats == nil {
return nil
}
// discount the feed side loss when reporting forwarded track stats,
packetsLost := deltaStats.PacketsLost
if deltaStatsRemoteView != nil {
packetsLost = deltaStatsRemoteView.PacketsLost
if deltaStatsRemoteView.PacketsMissing > packetsLost {
packetsLost = 0
} else {
packetsLost -= deltaStatsRemoteView.PacketsMissing
}
}
return &livekit.AnalyticsStream{
StartTime: timestamppb.New(deltaStats.StartTime),
EndTime: timestamppb.New(deltaStats.EndTime),
Ssrc: ssrc,
PrimaryPackets: deltaStats.Packets,
PrimaryBytes: deltaStats.Bytes,
RetransmitPackets: deltaStats.PacketsDuplicate,
RetransmitBytes: deltaStats.BytesDuplicate,
PaddingPackets: deltaStats.PacketsPadding,
PaddingBytes: deltaStats.BytesPadding,
PacketsLost: packetsLost,
PacketsOutOfOrder: deltaStats.PacketsOutOfOrder,
Frames: deltaStats.Frames,
Rtt: deltaStats.RttMax,
Jitter: uint32(deltaStats.JitterMax),
Nacks: deltaStats.Nacks,
Plis: deltaStats.Plis,
Firs: deltaStats.Firs,
}
}
func toAnalyticsVideoLayer(layer int32, layerStats *rtpstats.RTPDeltaInfo) *livekit.AnalyticsVideoLayer {
if layerStats == nil {
return nil
}
avl := &livekit.AnalyticsVideoLayer{
Layer: layer,
Packets: layerStats.Packets + layerStats.PacketsDuplicate + layerStats.PacketsPadding,
Bytes: layerStats.Bytes + layerStats.BytesDuplicate + layerStats.BytesPadding,
Frames: layerStats.Frames,
}
if avl.Packets == 0 || avl.Bytes == 0 || avl.Frames == 0 {
return nil
}
return avl
}
livekit-server/pkg/sfu/connectionquality/connectionstats_test.go
+903
View File
@@ -0,0 +1,903 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package connectionquality
import (
"math"
"testing"
"time"
"github.com/stretchr/testify/require"
"github.com/livekit/livekit-server/pkg/sfu/buffer"
"github.com/livekit/livekit-server/pkg/sfu/mime"
"github.com/livekit/livekit-server/pkg/sfu/rtpstats"
"github.com/livekit/protocol/livekit"
"github.com/livekit/protocol/logger"
)
// -----------------------------------------------
type testReceiverProvider struct {
streams map[uint32]*buffer.StreamStatsWithLayers
lastSenderReportTime time.Time
}
func newTestReceiverProvider() *testReceiverProvider {
return &testReceiverProvider{}
}
func (trp *testReceiverProvider) setStreams(streams map[uint32]*buffer.StreamStatsWithLayers) {
trp.streams = streams
}
func (trp *testReceiverProvider) GetDeltaStats() map[uint32]*buffer.StreamStatsWithLayers {
return trp.streams
}
func (trp *testReceiverProvider) setLastSenderReportTime(at time.Time) {
trp.lastSenderReportTime = at
}
func (trp *testReceiverProvider) GetLastSenderReportTime() time.Time {
return trp.lastSenderReportTime
}
// -----------------------------------------------
func TestConnectionQuality(t *testing.T) {
trp := newTestReceiverProvider()
t.Run("quality scorer operation", func(t *testing.T) {
cs := NewConnectionStats(ConnectionStatsParams{
IncludeRTT: true,
IncludeJitter: true,
EnableBitrateScore: true,
ReceiverProvider: trp,
Logger: logger.GetLogger(),
})
duration := 5 * time.Second
now := time.Now()
cs.StartAt(mime.MimeTypeOpus, false, now.Add(-duration))
cs.UpdateMuteAt(false, now.Add(-1*time.Second))
// no data and not enough unmute time should return default state which is EXCELLENT quality
cs.updateScoreAt(now)
mos, quality := cs.GetScoreAndQuality()
require.Greater(t, float32(4.6), mos)
require.Equal(t, livekit.ConnectionQuality_EXCELLENT, quality)
// best conditions (no loss, jitter/rtt = 0) - quality should stay EXCELLENT
trp.setStreams(map[uint32]*buffer.StreamStatsWithLayers{
1: {
RTPStats: &rtpstats.RTPDeltaInfo{
StartTime: now,
EndTime: now.Add(duration),
Packets: 250,
},
},
})
cs.updateScoreAt(now.Add(duration))
mos, quality = cs.GetScoreAndQuality()
require.Greater(t, float32(4.6), mos)
require.Equal(t, livekit.ConnectionQuality_EXCELLENT, quality)
// introduce loss and the score should drop - 12% loss for Opus -> POOR
now = now.Add(duration)
trp.setStreams(map[uint32]*buffer.StreamStatsWithLayers{
1: {
RTPStats: &rtpstats.RTPDeltaInfo{
StartTime: now,
EndTime: now.Add(duration),
Packets: 120,
PacketsLost: 30,
},
},
2: {
RTPStats: &rtpstats.RTPDeltaInfo{
StartTime: now,
EndTime: now.Add(duration),
Packets: 130,
PacketsLost: 0,
},
},
})
cs.updateScoreAt(now.Add(duration))
mos, quality = cs.GetScoreAndQuality()
require.Greater(t, float32(2.1), mos)
require.Equal(t, livekit.ConnectionQuality_POOR, quality)
// should climb to GOOD quality in one iteration if the conditions improve.
// although significant loss (12%) in the previous window, lowest score is
// bound so that climbing back does not take too long even under excellent conditions.
now = now.Add(duration)
trp.setStreams(map[uint32]*buffer.StreamStatsWithLayers{
1: {
RTPStats: &rtpstats.RTPDeltaInfo{
StartTime: now,
EndTime: now.Add(duration),
Packets: 250,
},
},
})
cs.updateScoreAt(now.Add(duration))
mos, quality = cs.GetScoreAndQuality()
require.Greater(t, float32(4.1), mos)
require.Equal(t, livekit.ConnectionQuality_GOOD, quality)
// should stay at GOOD if conditions continue to be good
now = now.Add(duration)
trp.setStreams(map[uint32]*buffer.StreamStatsWithLayers{
1: {
RTPStats: &rtpstats.RTPDeltaInfo{
StartTime: now,
EndTime: now.Add(duration),
Packets: 250,
},
},
})
cs.updateScoreAt(now.Add(duration))
mos, quality = cs.GetScoreAndQuality()
require.Greater(t, float32(4.1), mos)
require.Equal(t, livekit.ConnectionQuality_GOOD, quality)
// should climb up to EXCELLENT if conditions continue to be good
now = now.Add(duration)
trp.setStreams(map[uint32]*buffer.StreamStatsWithLayers{
1: {
RTPStats: &rtpstats.RTPDeltaInfo{
StartTime: now,
EndTime: now.Add(duration),
Packets: 250,
},
},
})
cs.updateScoreAt(now.Add(duration))
mos, quality = cs.GetScoreAndQuality()
require.Greater(t, float32(4.6), mos)
require.Equal(t, livekit.ConnectionQuality_EXCELLENT, quality)
// introduce loss and the score should drop - 5% loss for Opus -> GOOD
now = now.Add(duration)
trp.setStreams(map[uint32]*buffer.StreamStatsWithLayers{
1: {
RTPStats: &rtpstats.RTPDeltaInfo{
StartTime: now,
EndTime: now.Add(duration),
Packets: 250,
PacketsLost: 13,
},
},
})
cs.updateScoreAt(now.Add(duration))
mos, quality = cs.GetScoreAndQuality()
require.Greater(t, float32(4.1), mos)
require.Equal(t, livekit.ConnectionQuality_GOOD, quality)
// should stay at GOOD quality for another iteration even if the conditions improve
now = now.Add(duration)
trp.setStreams(map[uint32]*buffer.StreamStatsWithLayers{
1: {
RTPStats: &rtpstats.RTPDeltaInfo{
StartTime: now,
EndTime: now.Add(duration),
Packets: 250,
},
},
})
cs.updateScoreAt(now.Add(duration))
mos, quality = cs.GetScoreAndQuality()
require.Greater(t, float32(4.1), mos)
require.Equal(t, livekit.ConnectionQuality_GOOD, quality)
// should climb up to EXCELLENT if conditions continue to be good
now = now.Add(duration)
trp.setStreams(map[uint32]*buffer.StreamStatsWithLayers{
1: {
RTPStats: &rtpstats.RTPDeltaInfo{
StartTime: now,
EndTime: now.Add(duration),
Packets: 250,
},
},
})
cs.updateScoreAt(now.Add(duration))
mos, quality = cs.GetScoreAndQuality()
require.Greater(t, float32(4.6), mos)
require.Equal(t, livekit.ConnectionQuality_EXCELLENT, quality)
// mute when quality is POOR should return quality to EXCELLENT
now = now.Add(duration)
trp.setStreams(map[uint32]*buffer.StreamStatsWithLayers{
1: {
RTPStats: &rtpstats.RTPDeltaInfo{
StartTime: now,
EndTime: now.Add(duration),
Packets: 250,
PacketsLost: 30,
},
},
})
cs.updateScoreAt(now.Add(duration))
mos, quality = cs.GetScoreAndQuality()
require.Greater(t, float32(2.1), mos)
require.Equal(t, livekit.ConnectionQuality_POOR, quality)
now = now.Add(duration)
cs.UpdateMuteAt(true, now.Add(1*time.Second))
mos, quality = cs.GetScoreAndQuality()
require.Greater(t, float32(4.6), mos)
require.Equal(t, livekit.ConnectionQuality_EXCELLENT, quality)
// unmute at specific time to ensure next window does not satisfy the unmute time threshold.
// that means even if the next update has 0 packets, it should hold state and stay at EXCELLENT quality
cs.UpdateMuteAt(false, now.Add(3*time.Second))
trp.setStreams(map[uint32]*buffer.StreamStatsWithLayers{
1: {
RTPStats: &rtpstats.RTPDeltaInfo{
StartTime: now,
EndTime: now.Add(duration),
Packets: 0,
},
},
})
cs.updateScoreAt(now.Add(duration))
mos, quality = cs.GetScoreAndQuality()
require.Greater(t, float32(4.6), mos)
require.Equal(t, livekit.ConnectionQuality_EXCELLENT, quality)
// next update with no packets,
// but last RTCP is not set, should knock quality down to POOR
now = now.Add(duration)
trp.setStreams(map[uint32]*buffer.StreamStatsWithLayers{
1: {
RTPStats: &rtpstats.RTPDeltaInfo{
StartTime: now,
EndTime: now.Add(duration),
Packets: 0,
},
},
})
cs.updateScoreAt(now.Add(duration))
mos, quality = cs.GetScoreAndQuality()
require.Greater(t, float32(2.1), mos)
require.Equal(t, livekit.ConnectionQuality_POOR, quality)
// another dry spell, but last RTCP is not stale, should keep quality at POOR
now = now.Add(duration)
trp.setLastSenderReportTime(now.Add(time.Second))
trp.setStreams(map[uint32]*buffer.StreamStatsWithLayers{
1: {
RTPStats: &rtpstats.RTPDeltaInfo{
StartTime: now,
EndTime: now.Add(duration),
Packets: 0,
},
},
})
cs.updateScoreAt(now.Add(duration))
mos, quality = cs.GetScoreAndQuality()
require.Greater(t, float32(2.1), mos)
require.Equal(t, livekit.ConnectionQuality_POOR, quality)
// yet another dry spell, but last RTCP is stale, should knock down quality at LOST
now = now.Add(duration)
trp.setStreams(map[uint32]*buffer.StreamStatsWithLayers{
1: {
RTPStats: &rtpstats.RTPDeltaInfo{
StartTime: now,
EndTime: now.Add(duration),
Packets: 0,
},
},
})
cs.updateScoreAt(now.Add(duration))
mos, quality = cs.GetScoreAndQuality()
require.Greater(t, float32(1.3), mos)
require.Equal(t, livekit.ConnectionQuality_LOST, quality)
// mute when LOST should not bump up score/quality
now = now.Add(duration)
cs.UpdateMuteAt(true, now.Add(1*time.Second))
mos, quality = cs.GetScoreAndQuality()
require.Greater(t, float32(1.3), mos)
require.Equal(t, livekit.ConnectionQuality_LOST, quality)
// unmute and send packets to bring quality back up
now = now.Add(duration)
cs.UpdateMuteAt(false, now.Add(2*time.Second))
for i := 0; i < 3; i++ {
trp.setStreams(map[uint32]*buffer.StreamStatsWithLayers{
1: {
RTPStats: &rtpstats.RTPDeltaInfo{
StartTime: now,
EndTime: now.Add(duration),
Packets: 250,
PacketsLost: 0,
},
},
})
cs.updateScoreAt(now.Add(duration))
now = now.Add(duration)
}
cs.updateScoreAt(now.Add(duration))
mos, quality = cs.GetScoreAndQuality()
require.Greater(t, float32(4.6), mos)
require.Equal(t, livekit.ConnectionQuality_EXCELLENT, quality)
// with lesser number of packet (simulating DTX).
// even higher loss (like 10%) should not knock down quality due to quadratic weighting of packet loss ratio
trp.setStreams(map[uint32]*buffer.StreamStatsWithLayers{
1: {
RTPStats: &rtpstats.RTPDeltaInfo{
StartTime: now,
EndTime: now.Add(duration),
Packets: 50,
PacketsLost: 5,
},
},
})
cs.updateScoreAt(now.Add(duration))
mos, quality = cs.GetScoreAndQuality()
require.Greater(t, float32(4.6), mos)
require.Equal(t, livekit.ConnectionQuality_EXCELLENT, quality)
// mute/unmute to bring quality back up
now = now.Add(duration)
cs.UpdateMuteAt(true, now.Add(1*time.Second))
cs.UpdateMuteAt(false, now.Add(2*time.Second))
// RTT and jitter can knock quality down.
// at 2% loss, quality should stay at EXCELLENT purely based on loss, but with added RTT/jitter, should drop to GOOD
trp.setStreams(map[uint32]*buffer.StreamStatsWithLayers{
1: {
RTPStats: &rtpstats.RTPDeltaInfo{
StartTime: now,
EndTime: now.Add(duration),
Packets: 250,
PacketsLost: 5,
RttMax: 400,
JitterMax: 30000,
},
},
})
cs.updateScoreAt(now.Add(duration))
mos, quality = cs.GetScoreAndQuality()
require.Greater(t, float32(4.1), mos)
require.Equal(t, livekit.ConnectionQuality_GOOD, quality)
// mute/unmute to bring quality back up
now = now.Add(duration)
cs.UpdateMuteAt(true, now.Add(1*time.Second))
cs.UpdateMuteAt(false, now.Add(2*time.Second))
// bitrate based calculation can drop quality even if there is no loss
cs.AddBitrateTransitionAt(1_000_000, now)
cs.AddBitrateTransitionAt(2_000_000, now.Add(2*time.Second))
trp.setStreams(map[uint32]*buffer.StreamStatsWithLayers{
1: {
RTPStats: &rtpstats.RTPDeltaInfo{
StartTime: now,
EndTime: now.Add(duration),
Packets: 250,
Bytes: 8_000_000 / 8 / 5,
},
},
})
cs.updateScoreAt(now.Add(duration))
mos, quality = cs.GetScoreAndQuality()
require.Greater(t, float32(4.1), mos)
require.Equal(t, livekit.ConnectionQuality_GOOD, quality)
// test layer mute via UpdateLayerMute API
cs.AddBitrateTransitionAt(1_000_000, now)
cs.AddBitrateTransitionAt(2_000_000, now.Add(2*time.Second))
trp.setStreams(map[uint32]*buffer.StreamStatsWithLayers{
1: {
RTPStats: &rtpstats.RTPDeltaInfo{
StartTime: now,
EndTime: now.Add(duration),
Packets: 250,
Bytes: 8_000_000 / 8 / 5,
},
},
})
cs.updateScoreAt(now.Add(duration))
mos, quality = cs.GetScoreAndQuality()
require.Greater(t, float32(4.1), mos)
require.Equal(t, livekit.ConnectionQuality_GOOD, quality)
now = now.Add(duration)
cs.UpdateLayerMuteAt(true, now)
mos, quality = cs.GetScoreAndQuality()
require.Greater(t, float32(4.6), mos)
require.Equal(t, livekit.ConnectionQuality_EXCELLENT, quality)
// unmute layer
cs.UpdateLayerMuteAt(false, now.Add(2*time.Second))
trp.setStreams(map[uint32]*buffer.StreamStatsWithLayers{
1: {
RTPStats: &rtpstats.RTPDeltaInfo{
StartTime: now,
EndTime: now.Add(duration),
Packets: 250,
Bytes: 8_000_000 / 8 / 5,
},
},
})
cs.updateScoreAt(now.Add(duration))
mos, quality = cs.GetScoreAndQuality()
require.Greater(t, float32(4.6), mos)
require.Equal(t, livekit.ConnectionQuality_EXCELLENT, quality)
// pause
now = now.Add(duration)
cs.UpdatePauseAt(true, now)
mos, quality = cs.GetScoreAndQuality()
require.Greater(t, float32(2.1), mos)
require.Equal(t, livekit.ConnectionQuality_POOR, quality)
// resume
cs.UpdatePauseAt(false, now.Add(2*time.Second))
// although conditions are perfect, climbing back from POOR (because of pause above)
// will only climb to GOOD.
trp.setStreams(map[uint32]*buffer.StreamStatsWithLayers{
1: {
RTPStats: &rtpstats.RTPDeltaInfo{
StartTime: now,
EndTime: now.Add(duration),
Packets: 250,
Bytes: 8_000_000 / 8 / 5,
},
},
})
cs.updateScoreAt(now.Add(duration))
mos, quality = cs.GetScoreAndQuality()
require.Greater(t, float32(4.1), mos)
require.Equal(t, livekit.ConnectionQuality_GOOD, quality)
})
t.Run("quality scorer dependent rtt", func(t *testing.T) {
cs := NewConnectionStats(ConnectionStatsParams{
IncludeRTT: false,
IncludeJitter: true,
ReceiverProvider: trp,
Logger: logger.GetLogger(),
})
duration := 5 * time.Second
now := time.Now()
cs.StartAt(mime.MimeTypeOpus, false, now.Add(-duration))
cs.UpdateMuteAt(false, now.Add(-1*time.Second))
// RTT does not knock quality down because it is dependent and hence not taken into account
// at 2% loss, quality should stay at EXCELLENT purely based on loss. With high RTT (700 ms)
// quality should drop to GOOD if RTT were taken into consideration
trp.setStreams(map[uint32]*buffer.StreamStatsWithLayers{
1: {
RTPStats: &rtpstats.RTPDeltaInfo{
StartTime: now,
EndTime: now.Add(duration),
Packets: 250,
PacketsLost: 5,
RttMax: 700,
},
},
})
cs.updateScoreAt(now.Add(duration))
mos, quality := cs.GetScoreAndQuality()
require.Greater(t, float32(4.6), mos)
require.Equal(t, livekit.ConnectionQuality_EXCELLENT, quality)
})
t.Run("quality scorer dependent jitter", func(t *testing.T) {
cs := NewConnectionStats(ConnectionStatsParams{
IncludeRTT: true,
IncludeJitter: false,
ReceiverProvider: trp,
Logger: logger.GetLogger(),
})
duration := 5 * time.Second
now := time.Now()
cs.StartAt(mime.MimeTypeOpus, false, now.Add(-duration))
cs.UpdateMuteAt(false, now.Add(-1*time.Second))
// Jitter does not knock quality down because it is dependent and hence not taken into account
// at 2% loss, quality should stay at EXCELLENT purely based on loss. With high jitter (200 ms)
// quality should drop to GOOD if jitter were taken into consideration
trp.setStreams(map[uint32]*buffer.StreamStatsWithLayers{
1: {
RTPStats: &rtpstats.RTPDeltaInfo{
StartTime: now,
EndTime: now.Add(duration),
Packets: 250,
PacketsLost: 5,
JitterMax: 200,
},
},
})
cs.updateScoreAt(now.Add(duration))
mos, quality := cs.GetScoreAndQuality()
require.Greater(t, float32(4.6), mos)
require.Equal(t, livekit.ConnectionQuality_EXCELLENT, quality)
})
t.Run("codecs - packet", func(t *testing.T) {
type expectedQuality struct {
packetLossPercentage float64
expectedMOS float32
expectedQuality livekit.ConnectionQuality
}
testCases := []struct {
name string
mimeType mime.MimeType
isFECEnabled bool
packetsExpected uint32
expectedQualities []expectedQuality
}{
// NOTE: Because of EWMA (Exponentially Weighted Moving Average), these cut off points are not exact
// "audio/opus" - no fec - 0 <= loss < 2.5%: EXCELLENT, 2.5% <= loss < 7.5%: GOOD, >= 7.5%: POOR
{
name: "audio/opus - no fec",
mimeType: mime.MimeTypeOpus,
isFECEnabled: false,
packetsExpected: 200,
expectedQualities: []expectedQuality{
{
packetLossPercentage: 1.0,
expectedMOS: 4.6,
expectedQuality: livekit.ConnectionQuality_EXCELLENT,
},
{
packetLossPercentage: 4.0,
expectedMOS: 4.1,
expectedQuality: livekit.ConnectionQuality_GOOD,
},
{
packetLossPercentage: 9.2,
expectedMOS: 2.1,
expectedQuality: livekit.ConnectionQuality_POOR,
},
},
},
// "audio/opus" - fec - 0 <= loss < 3.75%: EXCELLENT, 3.75% <= loss < 11.25%: GOOD, >= 11.25%: POOR
{
name: "audio/opus - fec",
mimeType: mime.MimeTypeOpus,
isFECEnabled: true,
packetsExpected: 200,
expectedQualities: []expectedQuality{
{
packetLossPercentage: 3.0,
expectedMOS: 4.6,
expectedQuality: livekit.ConnectionQuality_EXCELLENT,
},
{
packetLossPercentage: 4.4,
expectedMOS: 4.1,
expectedQuality: livekit.ConnectionQuality_GOOD,
},
{
packetLossPercentage: 15.0,
expectedMOS: 2.1,
expectedQuality: livekit.ConnectionQuality_POOR,
},
},
},
// "audio/red" - no fec - 0 <= loss < 5%: EXCELLENT, 5% <= loss < 15%: GOOD, >= 15%: POOR
{
name: "audio/red - no fec",
mimeType: mime.MimeTypeRED,
isFECEnabled: false,
packetsExpected: 200,
expectedQualities: []expectedQuality{
{
packetLossPercentage: 4.0,
expectedMOS: 4.6,
expectedQuality: livekit.ConnectionQuality_EXCELLENT,
},
{
packetLossPercentage: 6.0,
expectedMOS: 4.1,
expectedQuality: livekit.ConnectionQuality_GOOD,
},
{
packetLossPercentage: 19.5,
expectedMOS: 2.1,
expectedQuality: livekit.ConnectionQuality_POOR,
},
},
},
// "audio/red" - fec - 0 <= loss < 7.5%: EXCELLENT, 7.5% <= loss < 22.5%: GOOD, >= 22.5%: POOR
{
name: "audio/red - fec",
mimeType: mime.MimeTypeRED,
isFECEnabled: true,
packetsExpected: 200,
expectedQualities: []expectedQuality{
{
packetLossPercentage: 6.0,
expectedMOS: 4.6,
expectedQuality: livekit.ConnectionQuality_EXCELLENT,
},
{
packetLossPercentage: 10.0,
expectedMOS: 4.1,
expectedQuality: livekit.ConnectionQuality_GOOD,
},
{
packetLossPercentage: 30.0,
expectedMOS: 2.1,
expectedQuality: livekit.ConnectionQuality_POOR,
},
},
},
// "video/*" - 0 <= loss < 2%: EXCELLENT, 2% <= loss < 6%: GOOD, >= 6%: POOR
{
name: "video/*",
mimeType: mime.MimeTypeVP8,
isFECEnabled: false,
packetsExpected: 200,
expectedQualities: []expectedQuality{
{
packetLossPercentage: 1.0,
expectedMOS: 4.6,
expectedQuality: livekit.ConnectionQuality_EXCELLENT,
},
{
packetLossPercentage: 3.5,
expectedMOS: 4.1,
expectedQuality: livekit.ConnectionQuality_GOOD,
},
{
packetLossPercentage: 8.0,
expectedMOS: 2.1,
expectedQuality: livekit.ConnectionQuality_POOR,
},
},
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
cs := NewConnectionStats(ConnectionStatsParams{
IncludeRTT: true,
IncludeJitter: true,
ReceiverProvider: trp,
Logger: logger.GetLogger(),
})
duration := 5 * time.Second
now := time.Now()
cs.StartAt(tc.mimeType, tc.isFECEnabled, now.Add(-duration))
for _, eq := range tc.expectedQualities {
trp.setStreams(map[uint32]*buffer.StreamStatsWithLayers{
123: {
RTPStats: &rtpstats.RTPDeltaInfo{
StartTime: now,
EndTime: now.Add(duration),
Packets: tc.packetsExpected,
PacketsLost: uint32(math.Ceil(eq.packetLossPercentage * float64(tc.packetsExpected) / 100.0)),
},
},
})
cs.updateScoreAt(now.Add(duration))
mos, quality := cs.GetScoreAndQuality()
require.Greater(t, eq.expectedMOS, mos)
require.Equal(t, eq.expectedQuality, quality)
now = now.Add(duration)
}
})
}
})
t.Run("bitrate", func(t *testing.T) {
type transition struct {
bitrate int64
offset time.Duration
}
testCases := []struct {
name string
transitions []transition
bytes uint64
expectedMOS float32
expectedQuality livekit.ConnectionQuality
}{
// NOTE: Because of EWMA (Exponentially Weighted Moving Average), these cut off points are not exact
// 1.0 <= expectedBits / actualBits < ~2.7 = EXCELLENT
// ~2.7 <= expectedBits / actualBits < ~20.1 = GOOD
// expectedBits / actualBits >= ~20.1 = POOR
{
name: "excellent",
transitions: []transition{
{
bitrate: 1_000_000,
},
{
bitrate: 2_000_000,
offset: 3 * time.Second,
},
},
bytes: 6_000_000 / 8,
expectedMOS: 4.6,
expectedQuality: livekit.ConnectionQuality_EXCELLENT,
},
{
name: "good",
transitions: []transition{
{
bitrate: 1_000_000,
},
{
bitrate: 2_000_000,
offset: 3 * time.Second,
},
},
bytes: uint64(math.Ceil(7_000_000.0 / 8.0 / 4.2)),
expectedMOS: 4.1,
expectedQuality: livekit.ConnectionQuality_GOOD,
},
{
name: "poor",
transitions: []transition{
{
bitrate: 2_000_000,
},
{
bitrate: 1_000_000,
offset: 3 * time.Second,
},
},
bytes: uint64(math.Ceil(8_000_000.0 / 8.0 / 75.0)),
expectedMOS: 2.1,
expectedQuality: livekit.ConnectionQuality_POOR,
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
cs := NewConnectionStats(ConnectionStatsParams{
IncludeRTT: true,
IncludeJitter: true,
EnableBitrateScore: true,
ReceiverProvider: trp,
Logger: logger.GetLogger(),
})
duration := 5 * time.Second
now := time.Now()
cs.StartAt(mime.MimeTypeVP8, false, now)
for _, tr := range tc.transitions {
cs.AddBitrateTransitionAt(tr.bitrate, now.Add(tr.offset))
}
trp.setStreams(map[uint32]*buffer.StreamStatsWithLayers{
123: {
RTPStats: &rtpstats.RTPDeltaInfo{
StartTime: now,
EndTime: now.Add(duration),
Packets: 100,
Bytes: tc.bytes,
},
},
})
cs.updateScoreAt(now.Add(duration))
mos, quality := cs.GetScoreAndQuality()
require.Greater(t, tc.expectedMOS, mos)
require.Equal(t, tc.expectedQuality, quality)
})
}
})
t.Run("layer", func(t *testing.T) {
type transition struct {
distance float64
offset time.Duration
}
testCases := []struct {
name string
transitions []transition
expectedMOS float32
expectedQuality livekit.ConnectionQuality
}{
// NOTE: Because of EWMA (Exponentially Weighted Moving Average), these cut off points are not exact
// each spatial layer missed drops o quality level
{
name: "excellent",
transitions: []transition{
{
distance: 0.5,
},
{
distance: 0.0,
offset: 3 * time.Second,
},
},
expectedMOS: 4.6,
expectedQuality: livekit.ConnectionQuality_EXCELLENT,
},
{
name: "good",
transitions: []transition{
{
distance: 1.0,
},
{
distance: 1.5,
offset: 2 * time.Second,
},
},
expectedMOS: 4.1,
expectedQuality: livekit.ConnectionQuality_GOOD,
},
{
name: "poor",
transitions: []transition{
{
distance: 2.0,
},
{
distance: 2.6,
offset: 1 * time.Second,
},
},
expectedMOS: 2.1,
expectedQuality: livekit.ConnectionQuality_POOR,
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
cs := NewConnectionStats(ConnectionStatsParams{
IncludeRTT: true,
IncludeJitter: true,
ReceiverProvider: trp,
Logger: logger.GetLogger(),
})
duration := 5 * time.Second
now := time.Now()
cs.StartAt(mime.MimeTypeVP8, false, now)
for _, tr := range tc.transitions {
cs.AddLayerTransitionAt(tr.distance, now.Add(tr.offset))
}
trp.setStreams(map[uint32]*buffer.StreamStatsWithLayers{
123: {
RTPStats: &rtpstats.RTPDeltaInfo{
StartTime: now,
EndTime: now.Add(duration),
Packets: 200,
},
},
})
cs.updateScoreAt(now.Add(duration))
mos, quality := cs.GetScoreAndQuality()
require.Greater(t, tc.expectedMOS, mos)
require.Equal(t, tc.expectedQuality, quality)
})
}
})
}
livekit-server/pkg/sfu/connectionquality/scorer.go
+644
View File
@@ -0,0 +1,644 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package connectionquality
import (
"fmt"
"math"
"sync"
"time"
"github.com/livekit/protocol/livekit"
"github.com/livekit/protocol/logger"
"github.com/livekit/protocol/utils"
"go.uber.org/zap/zapcore"
)
const (
MaxMOS = float32(4.5)
MinMOS = float32(1.0)
cMaxScore = float64(100.0)
cMinScore = float64(30.0)
cIncreaseFactor = float64(0.4) // slower increase, i. e. when score is recovering move up slower -> conservative
cDecreaseFactor = float64(0.8) // faster decrease, i. e. when score is dropping move down faster -> aggressive to be responsive to quality drops
cDistanceWeight = float64(35.0) // each spatial layer missed drops a quality level
cUnmuteTimeThreshold = float64(0.5)
cPPSQuantization = float64(2)
cPPSMinReadings = 10
cModeCalculationInterval = 2 * time.Minute
)
var (
qualityTransitionScore = map[livekit.ConnectionQuality]float64{
livekit.ConnectionQuality_GOOD: 80,
livekit.ConnectionQuality_POOR: 40,
livekit.ConnectionQuality_LOST: 20,
}
)
// ------------------------------------------
type windowStat struct {
startedAt time.Time
duration time.Duration
packets uint32
packetsPadding uint32
packetsLost uint32
packetsMissing uint32
packetsOutOfOrder uint32
bytes uint64
rttMax uint32
jitterMax float64
lastRTCPAt time.Time
}
func (w *windowStat) calculatePacketScore(aplw float64, includeRTT bool, includeJitter bool) float64 {
// this is based on simplified E-model based on packet loss, rtt, jitter as
// outlined at https://www.pingman.com/kb/article/how-is-mos-calculated-in-pingplotter-pro-50.html.
effectiveDelay := 0.0
// discount the dependent factors if dependency indicated.
// for example,
// 1. in the up stream, RTT cannot be measured without RTCP-XR, it is using down stream RTT.
// 2. in the down stream, up stream jitter affects it. although jitter can be adjusted to account for up stream
// jitter, this lever can be used to discount jitter in scoring.
if includeRTT {
effectiveDelay += float64(w.rttMax) / 2.0
}
if includeJitter {
effectiveDelay += (w.jitterMax * 2.0) / 1000.0
}
delayEffect := effectiveDelay / 40.0
if effectiveDelay > 160.0 {
delayEffect = (effectiveDelay - 120.0) / 10.0
}
// discount out-of-order packets from loss to deal with a scenario like
// 1. up stream has loss
// 2. down stream forwards with loss/hole in sequence number
// 3. down stream client reports a certain number of loss via RTCP RR
// 4. while processing that RTCP RR, up stream could have retransmitted missing packets
// 5. those retransmitted packets are forwarded,
// - server's view: it has forwarded those packets
// - client's view: it had not seen those packets when sending RTCP RR
// so those retransmitted packets appear like down stream loss to server.
//
// retransmitted packets would have arrived out-of-order. So, discounting them
// will account for it.
//
// Note that packets can arrive out-of-order in the upstream during regular
// streaming as well, i. e. without loss + NACK + retransmit. Those will be
// discounted too. And that will skew the real loss. For example, let
// us say that 40 out of 100 packets were reported lost by down stream.
// These could be real losses. In the same window, 40 packets could have been
// delivered out-of-order by the up stream, thus cancelling out the real loss.
// But, those situations should be rare and is a compromise for not letting
// up stream loss penalise down stream.
actualLost := w.packetsLost - w.packetsMissing - w.packetsOutOfOrder
if int32(actualLost) < 0 {
actualLost = 0
}
var lossEffect float64
if w.packets+w.packetsPadding > 0 {
lossEffect = float64(actualLost) * 100.0 / float64(w.packets+w.packetsPadding)
}
lossEffect *= aplw
score := cMaxScore - delayEffect - lossEffect
if score < 0.0 {
score = 0.0
}
return score
}
func (w *windowStat) calculateBitrateScore(expectedBits int64, isEnabled bool) float64 {
if expectedBits == 0 || !isEnabled {
// unsupported mode OR all layers stopped
return cMaxScore
}
var score float64
if w.bytes != 0 {
// using the ratio of expectedBits / actualBits
// the quality inflection points are approximately
// GOOD at ~2.7x, POOR at ~20.1x
score = cMaxScore - 20*math.Log(float64(expectedBits)/float64(w.bytes*8))
if score > cMaxScore {
score = cMaxScore
}
if score < 0.0 {
score = 0.0
}
}
return score
}
func (w *windowStat) String() string {
return fmt.Sprintf("start: %+v, dur: %+v, p: %d, pp: %d, pl: %d, pm: %d, pooo: %d, b: %d, rtt: %d, jitter: %0.2f, lastRTCP: %+v",
w.startedAt,
w.duration,
w.packets,
w.packetsPadding,
w.packetsLost,
w.packetsMissing,
w.packetsOutOfOrder,
w.bytes,
w.rttMax,
w.jitterMax,
w.lastRTCPAt,
)
}
func (w *windowStat) MarshalLogObject(e zapcore.ObjectEncoder) error {
if w == nil {
return nil
}
e.AddTime("startedAt", w.startedAt)
e.AddString("duration", w.duration.String())
e.AddUint32("packets", w.packets)
e.AddUint32("packetsPadding", w.packetsPadding)
e.AddUint32("packetsLost", w.packetsLost)
e.AddUint32("packetsMissing", w.packetsMissing)
e.AddUint32("packetsOutOfOrder", w.packetsOutOfOrder)
e.AddUint64("bytes", w.bytes)
e.AddUint32("rttMax", w.rttMax)
e.AddFloat64("jitterMax", w.jitterMax)
e.AddTime("lastRTCPAt", w.lastRTCPAt)
return nil
}
// ------------------------------------------
type qualityScorerParams struct {
IncludeRTT bool
IncludeJitter bool
EnableBitrateScore bool
Logger logger.Logger
}
type qualityScorer struct {
params qualityScorerParams
lock sync.RWMutex
lastUpdateAt time.Time
packetLossWeight float64
score float64
stat windowStat
mutedAt time.Time
unmutedAt time.Time
layerMutedAt time.Time
layerUnmutedAt time.Time
pausedAt time.Time
resumedAt time.Time
ppsHistogram [250]int
numPPSReadings int
ppsMode int
modeCalculatedAt time.Time
aggregateBitrate *utils.TimedAggregator[int64]
layerDistance *utils.TimedAggregator[float64]
}
func newQualityScorer(params qualityScorerParams) *qualityScorer {
return &qualityScorer{
params: params,
score: cMaxScore,
aggregateBitrate: utils.NewTimedAggregator[int64](utils.TimedAggregatorParams{
CapNegativeValues: true,
}),
layerDistance: utils.NewTimedAggregator[float64](utils.TimedAggregatorParams{
CapNegativeValues: true,
}),
modeCalculatedAt: time.Now().Add(-cModeCalculationInterval),
}
}
func (q *qualityScorer) startAtLocked(packetLossWeight float64, at time.Time) {
q.packetLossWeight = packetLossWeight
q.lastUpdateAt = at
}
func (q *qualityScorer) StartAt(packetLossWeight float64, at time.Time) {
q.lock.Lock()
defer q.lock.Unlock()
q.startAtLocked(packetLossWeight, at)
}
func (q *qualityScorer) Start(packetLossWeight float64) {
q.lock.Lock()
defer q.lock.Unlock()
q.startAtLocked(packetLossWeight, time.Now())
}
func (q *qualityScorer) UpdatePacketLossWeight(packetLossWeight float64) {
q.lock.Lock()
defer q.lock.Unlock()
q.packetLossWeight = packetLossWeight
}
func (q *qualityScorer) updateMuteAtLocked(isMuted bool, at time.Time) {
if isMuted {
q.mutedAt = at
// muting when LOST should not push quality to EXCELLENT
if q.score != qualityTransitionScore[livekit.ConnectionQuality_LOST] {
q.score = cMaxScore
}
} else {
q.unmutedAt = at
}
}
func (q *qualityScorer) UpdateMuteAt(isMuted bool, at time.Time) {
q.lock.Lock()
defer q.lock.Unlock()
q.updateMuteAtLocked(isMuted, at)
}
func (q *qualityScorer) UpdateMute(isMuted bool) {
q.lock.Lock()
defer q.lock.Unlock()
q.updateMuteAtLocked(isMuted, time.Now())
}
func (q *qualityScorer) addBitrateTransitionAtLocked(bitrate int64, at time.Time) {
q.aggregateBitrate.AddSampleAt(bitrate, at)
}
func (q *qualityScorer) AddBitrateTransitionAt(bitrate int64, at time.Time) {
q.lock.Lock()
defer q.lock.Unlock()
q.addBitrateTransitionAtLocked(bitrate, at)
}
func (q *qualityScorer) AddBitrateTransition(bitrate int64) {
q.lock.Lock()
defer q.lock.Unlock()
q.addBitrateTransitionAtLocked(bitrate, time.Now())
}
func (q *qualityScorer) updateLayerMuteAtLocked(isMuted bool, at time.Time) {
if isMuted {
if !q.isLayerMuted() {
q.aggregateBitrate.Reset()
q.layerDistance.Reset()
q.layerMutedAt = at
q.score = cMaxScore
}
} else {
if q.isLayerMuted() {
q.layerUnmutedAt = at
}
}
}
func (q *qualityScorer) UpdateLayerMuteAt(isMuted bool, at time.Time) {
q.lock.Lock()
defer q.lock.Unlock()
q.updateLayerMuteAtLocked(isMuted, at)
}
func (q *qualityScorer) UpdateLayerMute(isMuted bool) {
q.lock.Lock()
defer q.lock.Unlock()
q.updateLayerMuteAtLocked(isMuted, time.Now())
}
func (q *qualityScorer) updatePauseAtLocked(isPaused bool, at time.Time) {
if isPaused {
if !q.isPaused() {
q.aggregateBitrate.Reset()
q.layerDistance.Reset()
q.pausedAt = at
q.score = cMinScore
}
} else {
if q.isPaused() {
q.resumedAt = at
}
}
}
func (q *qualityScorer) UpdatePauseAt(isPaused bool, at time.Time) {
q.lock.Lock()
defer q.lock.Unlock()
q.updatePauseAtLocked(isPaused, at)
}
func (q *qualityScorer) UpdatePause(isPaused bool) {
q.lock.Lock()
defer q.lock.Unlock()
q.updatePauseAtLocked(isPaused, time.Now())
}
func (q *qualityScorer) addLayerTransitionAtLocked(distance float64, at time.Time) {
q.layerDistance.AddSampleAt(distance, at)
}
func (q *qualityScorer) AddLayerTransitionAt(distance float64, at time.Time) {
q.lock.Lock()
defer q.lock.Unlock()
q.addLayerTransitionAtLocked(distance, at)
}
func (q *qualityScorer) AddLayerTransition(distance float64) {
q.lock.Lock()
defer q.lock.Unlock()
q.addLayerTransitionAtLocked(distance, time.Now())
}
func (q *qualityScorer) updateAtLocked(stat *windowStat, at time.Time) {
// always update transitions
expectedBits, _, err := q.aggregateBitrate.GetAggregateAndRestartAt(at)
if err != nil {
q.params.Logger.Warnw("error getting expected bitrate", err)
}
expectedDistance, err := q.layerDistance.GetAverageAndRestartAt(at)
if err != nil {
q.params.Logger.Warnw("error getting expected distance", err)
}
// nothing to do when muted or not unmuted for long enough
// NOTE: it is possible that unmute -> mute -> unmute transition happens in the
// same analysis window. On a transition to mute, quality is immediately moved
// EXCELLENT for responsiveness. On an unmute, the entire window data is
// considered (as long as enough time has passed since unmute).
//
// Similarly, when paused (possibly due to congestion), score is immediately
// set to cMinScore for responsiveness. The layer transition is reset.
// On a resume, quality climbs back up using normal operation.
if q.isMuted() || !q.isUnmutedEnough(at) || q.isLayerMuted() || q.isPaused() {
q.lastUpdateAt = at
return
}
aplw := q.getAdjustedPacketLossWeight(stat)
reason := "none"
var score, packetScore, bitrateScore, layerScore float64
if stat.packets+stat.packetsPadding == 0 {
if !stat.lastRTCPAt.IsZero() && at.Sub(stat.lastRTCPAt) > stat.duration {
reason = "rtcp"
score = qualityTransitionScore[livekit.ConnectionQuality_LOST]
} else {
reason = "dry"
score = qualityTransitionScore[livekit.ConnectionQuality_POOR]
}
} else {
packetScore = stat.calculatePacketScore(aplw, q.params.IncludeRTT, q.params.IncludeJitter)
bitrateScore = stat.calculateBitrateScore(expectedBits, q.params.EnableBitrateScore)
layerScore = math.Max(math.Min(cMaxScore, cMaxScore-(expectedDistance*cDistanceWeight)), 0.0)
minScore := math.Min(packetScore, bitrateScore)
minScore = math.Min(minScore, layerScore)
switch {
case packetScore == minScore:
reason = "packet"
score = packetScore
case bitrateScore == minScore:
reason = "bitrate"
score = bitrateScore
case layerScore == minScore:
reason = "layer"
score = layerScore
}
factor := cIncreaseFactor
if score < q.score {
factor = cDecreaseFactor
}
score = factor*score + (1.0-factor)*q.score
if score < cMinScore {
// lower bound to prevent score from becoming very small values due to extreme conditions.
// Without a lower bound, it can get so low that it takes a long time to climb back to
// better quality even under excellent conditions.
score = cMinScore
}
}
prevCQ := scoreToConnectionQuality(q.score)
currCQ := scoreToConnectionQuality(score)
ulgr := q.params.Logger.WithUnlikelyValues(
"reason", reason,
"prevScore", q.score,
"prevQuality", prevCQ,
"prevStat", &q.stat,
"score", score,
"packetScore", packetScore,
"layerScore", layerScore,
"bitrateScore", bitrateScore,
"quality", currCQ,
"stat", stat,
"packetLossWeight", q.packetLossWeight,
"adjustedPacketLossWeight", aplw,
"modePPS", q.ppsMode*int(cPPSQuantization),
"expectedBits", expectedBits,
"expectedDistance", expectedDistance,
)
switch {
case utils.IsConnectionQualityLower(prevCQ, currCQ):
ulgr.Debugw("quality drop")
case utils.IsConnectionQualityHigher(prevCQ, currCQ):
ulgr.Debugw("quality rise")
default:
packets := stat.packets + stat.packetsPadding
if packets != 0 && (stat.packetsLost*100/packets) > 10 {
ulgr.Debugw("quality hold - high loss")
}
}
q.score = score
q.stat = *stat
q.lastUpdateAt = at
}
func (q *qualityScorer) UpdateAt(stat *windowStat, at time.Time) {
q.lock.Lock()
defer q.lock.Unlock()
q.updateAtLocked(stat, at)
}
func (q *qualityScorer) Update(stat *windowStat) {
q.lock.Lock()
defer q.lock.Unlock()
q.updateAtLocked(stat, time.Now())
}
func (q *qualityScorer) isMuted() bool {
return !q.mutedAt.IsZero() && (q.unmutedAt.IsZero() || q.mutedAt.After(q.unmutedAt))
}
func (q *qualityScorer) isUnmutedEnough(at time.Time) bool {
var sinceUnmute time.Duration
if q.unmutedAt.IsZero() {
sinceUnmute = at.Sub(q.lastUpdateAt)
} else {
sinceUnmute = at.Sub(q.unmutedAt)
}
var sinceLayerUnmute time.Duration
if q.layerUnmutedAt.IsZero() {
sinceLayerUnmute = at.Sub(q.lastUpdateAt)
} else {
sinceLayerUnmute = at.Sub(q.layerUnmutedAt)
}
validDuration := sinceUnmute
if sinceLayerUnmute < validDuration {
validDuration = sinceLayerUnmute
}
sinceLastUpdate := at.Sub(q.lastUpdateAt)
return validDuration.Seconds()/sinceLastUpdate.Seconds() > cUnmuteTimeThreshold
}
func (q *qualityScorer) isLayerMuted() bool {
return !q.layerMutedAt.IsZero() && (q.layerUnmutedAt.IsZero() || q.layerMutedAt.After(q.layerUnmutedAt))
}
func (q *qualityScorer) isPaused() bool {
return !q.pausedAt.IsZero() && (q.resumedAt.IsZero() || q.pausedAt.After(q.resumedAt))
}
func (q *qualityScorer) getAdjustedPacketLossWeight(stat *windowStat) float64 {
if stat == nil || stat.duration <= 0 {
return q.packetLossWeight
}
// packet loss is weighted by comparing against mode of packet rate seen.
// this is to handle situations like DTX in audio and variable bit rate tracks like screen share.
// and the effect of loss is not pronounced in those scenarios (audio silence, static screen share).
// for example, DTX typically uses only 5% of packets of full packet rate. at that rate,
// packet loss weight is reduced to ~22% of configured weight (i. e. sqrt(0.05) * configured weight)
pps := float64(stat.packets) / stat.duration.Seconds()
ppsQuantized := int(pps/cPPSQuantization + 0.5)
if ppsQuantized < len(q.ppsHistogram)-1 {
q.ppsHistogram[ppsQuantized]++
} else {
q.ppsHistogram[len(q.ppsHistogram)-1]++
}
q.numPPSReadings++
// calculate mode sparingly, do it under the following conditions
// 1. minimum number of readings available (AND)
// 2. enough time has elapsed since last calculation
if q.numPPSReadings > cPPSMinReadings && time.Since(q.modeCalculatedAt) > cModeCalculationInterval {
q.ppsMode = 0
for i := 0; i < len(q.ppsHistogram); i++ {
if q.ppsHistogram[i] > q.ppsMode {
q.ppsMode = i
}
}
q.modeCalculatedAt = time.Now()
q.params.Logger.Debugw("updating pps mode", "expected", stat.packets, "duration", stat.duration.Seconds(), "pps", pps, "ppsMode", q.ppsMode)
}
if q.ppsMode == 0 || q.ppsMode == len(q.ppsHistogram)-1 {
return q.packetLossWeight
}
packetRatio := pps / (float64(q.ppsMode) * cPPSQuantization)
if packetRatio > 1.0 {
packetRatio = 1.0
}
return math.Sqrt(packetRatio) * q.packetLossWeight
}
func (q *qualityScorer) GetScoreAndQuality() (float32, livekit.ConnectionQuality) {
q.lock.RLock()
defer q.lock.RUnlock()
return float32(q.score), scoreToConnectionQuality(q.score)
}
func (q *qualityScorer) GetMOSAndQuality() (float32, livekit.ConnectionQuality) {
q.lock.RLock()
defer q.lock.RUnlock()
return scoreToMOS(q.score), scoreToConnectionQuality(q.score)
}
// ------------------------------------------
func scoreToConnectionQuality(score float64) livekit.ConnectionQuality {
// R-factor -> livekit.ConnectionQuality scale mapping roughly based on
// https://www.itu.int/ITU-T/2005-2008/com12/emodelv1/tut.htm
//
// As there are only three levels in livekit.ConnectionQuality scale,
// using a larger range for middling quality. Empirical evidence suggests
// that a score of 60 does not correspond to `POOR` quality. Repair
// mechanisms and use of algorithms like de-jittering makes the experience
// better even under harsh conditions.
if score > qualityTransitionScore[livekit.ConnectionQuality_GOOD] {
return livekit.ConnectionQuality_EXCELLENT
}
if score > qualityTransitionScore[livekit.ConnectionQuality_POOR] {
return livekit.ConnectionQuality_GOOD
}
if score > qualityTransitionScore[livekit.ConnectionQuality_LOST] {
return livekit.ConnectionQuality_POOR
}
return livekit.ConnectionQuality_LOST
}
// ------------------------------------------
func scoreToMOS(score float64) float32 {
if score <= 0.0 {
return 1.0
}
if score >= 100.0 {
return 4.5
}
return float32(1.0 + 0.035*score + (0.000007 * score * (score - 60.0) * (100.0 - score)))
}
// ------------------------------------------
livekit-server/pkg/sfu/datachannel/bitrate.go
+109
View File
@@ -0,0 +1,109 @@
package datachannel
import (
"sync"
"time"
"github.com/gammazero/deque"
"github.com/livekit/protocol/utils/mono"
)
const (
BitrateDuration = 2 * time.Second
BitrateWindow = 100 * time.Millisecond
)
// BitrateCalculator calculates bitrate over sliding window
type BitrateCalculator struct {
lock sync.Mutex
windowDuration time.Duration
duration time.Duration
windows deque.Deque[bitrateWindow]
active bitrateWindow
bytes int
lastBufferedAmount int
start time.Time
}
func NewBitrateCalculator(duration time.Duration, window time.Duration) *BitrateCalculator {
windowCnt := int((duration + (window - 1)) / window)
if windowCnt == 0 {
windowCnt = 1
}
now := mono.Now()
c := &BitrateCalculator{
duration: duration,
windowDuration: window,
start: now,
active: bitrateWindow{start: now},
}
c.windows.SetBaseCap(windowCnt + 1)
return c
}
func (c *BitrateCalculator) AddBytes(bytes int, bufferedAmout int, ts time.Time) {
c.lock.Lock()
defer c.lock.Unlock()
bytes -= bufferedAmout - c.lastBufferedAmount
if bytes < 0 {
// it is possible that internal buffering (non-data like DCEP packet from webrtc) caused bytes to be negative
bytes = 0
}
c.lastBufferedAmount = bufferedAmout
if ts.Sub(c.active.start) >= c.windowDuration {
c.windows.PushBack(c.active)
c.active.start = ts
c.active.bytes = 0
for c.windows.Len() > 0 {
// pop expired windows
if w := c.windows.Front(); ts.Sub(w.start) > (c.duration + c.windowDuration) {
c.bytes -= w.bytes
c.windows.PopFront()
} else {
c.start = w.start
break
}
}
if c.windows.Len() == 0 {
c.start = ts
c.bytes = 0
}
}
c.bytes += bytes
c.active.bytes += bytes
}
func (c *BitrateCalculator) Bitrate(ts time.Time) (int, bool) {
return c.bitrate(ts, false)
}
func (c *BitrateCalculator) ForceBitrate(ts time.Time) (int, bool) {
return c.bitrate(ts, true)
}
func (c *BitrateCalculator) bitrate(ts time.Time, force bool) (int, bool) {
c.lock.Lock()
defer c.lock.Unlock()
duration := ts.Sub(c.start)
if duration < c.windowDuration {
if force {
duration = c.windowDuration
} else {
return 0, false
}
}
return c.bytes * 8 * 1000 / int(duration.Milliseconds()), true
}
type bitrateWindow struct {
start time.Time
bytes int
}
livekit-server/pkg/sfu/datachannel/bitrate_test.go
+36
View File
@@ -0,0 +1,36 @@
package datachannel
import (
"testing"
"time"
"github.com/stretchr/testify/require"
)
func TestBitrateCalculator(t *testing.T) {
c := NewBitrateCalculator(BitrateDuration, BitrateWindow)
require.NotNil(t, c)
t0 := time.Now()
c.AddBytes(100, 0, t0)
// bytes buffered
c.AddBytes(100, 100, t0.Add(50*time.Millisecond))
bitrate, ok := c.Bitrate(t0.Add(50 * time.Millisecond))
require.Equal(t, 0, bitrate)
require.False(t, ok)
// 50 bytes sent (50 bytes buffer flushed)
c.AddBytes(100, 50, t0.Add(time.Second))
// 250 bytes sent in 1 second
bitrate, ok = c.Bitrate(t0.Add(time.Second))
require.Equal(t, 2000, bitrate)
require.True(t, ok)
// silence for long time
t1 := t0.Add(2 * BitrateDuration)
// 150 bytes sent (50 bytes buffer flushed)
c.AddBytes(100, 0, t1)
bitrate, ok = c.Bitrate(t1.Add(time.Second))
require.Equal(t, 1200, bitrate)
require.True(t, ok)
}
livekit-server/pkg/sfu/datachannel/datachannel_writer.go
+80
View File
@@ -0,0 +1,80 @@
package datachannel
import (
"context"
"errors"
"fmt"
"time"
"github.com/pion/datachannel"
"github.com/livekit/protocol/utils/mono"
)
const (
singleWriteTimeout = 50 * time.Millisecond
)
var ErrDataDroppedBySlowReader = errors.New("data dropped by slow reader")
type BufferedAmountGetter interface {
BufferedAmount() uint64
}
type DataChannelWriter[T BufferedAmountGetter] struct {
bufferGetter T
rawDC datachannel.ReadWriteCloserDeadliner
slowThreshold int
rate *BitrateCalculator
}
// NewDataChannelWriter creates a new DataChannelWriter by detaching the data channel, when
// writing to the datachanel times out, it will block and retry if the receiver's bitrate is
// above the slowThreshold or drop the data if it's below the threshold. If the slowThreshold
// is 0, it will never retry on write timeout.
func NewDataChannelWriter[T BufferedAmountGetter](bufferGetter T, rawDC datachannel.ReadWriteCloserDeadliner, slowThreshold int) *DataChannelWriter[T] {
var rate *BitrateCalculator
if slowThreshold > 0 {
rate = NewBitrateCalculator(BitrateDuration, BitrateWindow)
}
return &DataChannelWriter[T]{
bufferGetter: bufferGetter,
rawDC: rawDC,
slowThreshold: slowThreshold,
rate: rate,
}
}
func (w *DataChannelWriter[T]) BufferedAmountGetter() T {
return w.bufferGetter
}
func (w *DataChannelWriter[T]) Write(p []byte) (n int, err error) {
for {
err = w.rawDC.SetWriteDeadline(time.Now().Add(singleWriteTimeout))
if err != nil {
return 0, err
}
n, err = w.rawDC.Write(p)
if w.slowThreshold == 0 {
return
}
now := mono.Now()
w.rate.AddBytes(n, int(w.bufferGetter.BufferedAmount()), now)
// retry if the write timed out on a non-slow receiver
if errors.Is(err, context.DeadlineExceeded) {
if bitrate, ok := w.rate.Bitrate(now); !ok || bitrate >= w.slowThreshold {
continue
} else {
err = fmt.Errorf("%w: bitrate %d, threshold %d", ErrDataDroppedBySlowReader, bitrate, w.slowThreshold)
}
}
return
}
}
func (w *DataChannelWriter[T]) Close() error {
return w.rawDC.Close()
}
livekit-server/pkg/sfu/datachannel/datachannel_writer_test.go
+94
View File
@@ -0,0 +1,94 @@
package datachannel
import (
"context"
"testing"
"time"
"github.com/pion/datachannel"
"github.com/pion/transport/v3/deadline"
"github.com/stretchr/testify/require"
)
func TestDataChannelWriter(t *testing.T) {
mockDC := newMockDataChannelWriter()
// slow threshold is 1000B/s
w := NewDataChannelWriter(mockDC, mockDC, 8000)
require.Equal(t, mockDC, w.BufferedAmountGetter())
buf := make([]byte, 2000)
// write 2000 bytes so it should not drop in 2 seconds
t0 := time.Now()
n, err := w.Write(buf)
require.NoError(t, err)
require.Equal(t, 2000, n)
t1 := time.Now()
mockDC.SetNextWriteCompleteAt(t0.Add(time.Second))
n, err = w.Write(buf[:10])
require.NoError(t, err)
require.Equal(t, 10, n)
require.GreaterOrEqual(t, time.Since(t1), time.Second)
// bitrate below slow threshold(2000bytes/3sec), should drop by timeout
mockDC.SetNextWriteCompleteAt(t0.Add(3 * time.Second))
n, err = w.Write(buf[:1000])
require.ErrorIs(t, err, ErrDataDroppedBySlowReader, err)
require.Equal(t, 0, n)
}
func TestDataChannelWriter_NoSlowThreshold(t *testing.T) {
mockDC := newMockDataChannelWriter()
w := NewDataChannelWriter(mockDC, mockDC, 0)
buf := make([]byte, 2000)
n, err := w.Write(buf)
require.NoError(t, err)
require.Equal(t, 2000, n)
mockDC.SetNextWriteCompleteAt(time.Now().Add(singleWriteTimeout / 2))
n, err = w.Write(buf[:10])
require.NoError(t, err)
require.Equal(t, 10, n)
// slow threshold is 0, should not block & retry
mockDC.SetNextWriteCompleteAt(time.Now().Add(singleWriteTimeout * 2))
n, err = w.Write(buf[:1000])
require.ErrorIs(t, err, context.DeadlineExceeded, err)
require.Equal(t, 0, n)
}
type mockDataChannelWriter struct {
datachannel.ReadWriteCloserDeadliner
nextWriteCompleteAt time.Time
deadline *deadline.Deadline
}
func newMockDataChannelWriter() *mockDataChannelWriter {
return &mockDataChannelWriter{
deadline: deadline.New(),
}
}
func (m *mockDataChannelWriter) BufferedAmount() uint64 {
return 0
}
func (m *mockDataChannelWriter) Write(b []byte) (int, error) {
wait := time.Until(m.nextWriteCompleteAt)
if wait <= 0 {
return len(b), nil
}
select {
case <-m.deadline.Done():
return 0, m.deadline.Err()
case <-time.After(wait):
return len(b), nil
}
}
func (m *mockDataChannelWriter) SetWriteDeadline(t time.Time) error {
m.deadline.Set(t)
return nil
}
func (m *mockDataChannelWriter) SetNextWriteCompleteAt(t time.Time) {
m.nextWriteCompleteAt = t
}
livekit-server/pkg/sfu/downtrack.go
+2536
View File
File diff suppressed because it is too large Load Diff
livekit-server/pkg/sfu/downtrackspreader.go
+118
View File
@@ -0,0 +1,118 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package sfu
import (
"sync"
"github.com/livekit/protocol/livekit"
"github.com/livekit/protocol/logger"
"github.com/livekit/protocol/utils"
)
type DownTrackSpreaderParams struct {
Threshold int
Logger logger.Logger
}
type DownTrackSpreader struct {
params DownTrackSpreaderParams
downTrackMu sync.RWMutex
downTracks map[livekit.ParticipantID]TrackSender
downTracksShadow []TrackSender
}
func NewDownTrackSpreader(params DownTrackSpreaderParams) *DownTrackSpreader {
d := &DownTrackSpreader{
params: params,
downTracks: make(map[livekit.ParticipantID]TrackSender),
}
return d
}
func (d *DownTrackSpreader) GetDownTracks() []TrackSender {
d.downTrackMu.RLock()
defer d.downTrackMu.RUnlock()
return d.downTracksShadow
}
func (d *DownTrackSpreader) ResetAndGetDownTracks() []TrackSender {
d.downTrackMu.Lock()
defer d.downTrackMu.Unlock()
downTracks := d.downTracksShadow
d.downTracks = make(map[livekit.ParticipantID]TrackSender)
d.downTracksShadow = nil
return downTracks
}
func (d *DownTrackSpreader) Store(ts TrackSender) {
d.downTrackMu.Lock()
defer d.downTrackMu.Unlock()
d.downTracks[ts.SubscriberID()] = ts
d.shadowDownTracks()
}
func (d *DownTrackSpreader) Free(subscriberID livekit.ParticipantID) {
d.downTrackMu.Lock()
defer d.downTrackMu.Unlock()
delete(d.downTracks, subscriberID)
d.shadowDownTracks()
}
func (d *DownTrackSpreader) HasDownTrack(subscriberID livekit.ParticipantID) bool {
d.downTrackMu.RLock()
defer d.downTrackMu.RUnlock()
_, ok := d.downTracks[subscriberID]
return ok
}
func (d *DownTrackSpreader) Broadcast(writer func(TrackSender)) int {
downTracks := d.GetDownTracks()
if len(downTracks) == 0 {
return 0
}
threshold := uint64(d.params.Threshold)
if threshold == 0 {
threshold = 1000000
}
// 100µs is enough to amortize the overhead and provide sufficient load balancing.
// WriteRTP takes about 50µs on average, so we write to 2 down tracks per loop.
step := uint64(2)
utils.ParallelExec(downTracks, threshold, step, writer)
return len(downTracks)
}
func (d *DownTrackSpreader) DownTrackCount() int {
d.downTrackMu.RLock()
defer d.downTrackMu.RUnlock()
return len(d.downTracksShadow)
}
func (d *DownTrackSpreader) shadowDownTracks() {
d.downTracksShadow = make([]TrackSender, 0, len(d.downTracks))
for _, dt := range d.downTracks {
d.downTracksShadow = append(d.downTracksShadow, dt)
}
}
livekit-server/pkg/sfu/errors.go
+15
View File
@@ -0,0 +1,15 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package sfu
livekit-server/pkg/sfu/forwarder.go
+2296
View File
File diff suppressed because it is too large Load Diff
livekit-server/pkg/sfu/forwarder_test.go
+2074
View File
File diff suppressed because it is too large Load Diff
livekit-server/pkg/sfu/forwardstats.go
+89
View File
@@ -0,0 +1,89 @@
package sfu
import (
"fmt"
"sync"
"time"
"go.uber.org/atomic"
"github.com/livekit/livekit-server/pkg/telemetry/prometheus"
"github.com/livekit/protocol/logger"
"github.com/livekit/protocol/utils"
)
type ForwardStats struct {
lock sync.Mutex
lastLeftNano atomic.Int64
latency *utils.LatencyAggregate
closeCh chan struct{}
}
func NewForwardStats(latencyUpdateInterval, reportInterval, latencyWindowLength time.Duration) *ForwardStats {
s := &ForwardStats{
latency: utils.NewLatencyAggregate(latencyUpdateInterval, latencyWindowLength),
closeCh: make(chan struct{}),
}
go s.report(reportInterval)
return s
}
func (s *ForwardStats) Update(arrival, left int64) {
transit := left - arrival
// ignore if transit is too large or negative, this could happen if system time is adjusted
if transit < 0 || time.Duration(transit) > 5*time.Second {
return
}
lastLeftNano := s.lastLeftNano.Load()
if left < lastLeftNano || !s.lastLeftNano.CompareAndSwap(lastLeftNano, left) {
return
}
s.lock.Lock()
defer s.lock.Unlock()
s.latency.Update(time.Duration(arrival), float64(transit))
}
func (s *ForwardStats) GetStats() (latency, jitter time.Duration) {
s.lock.Lock()
w := s.latency.Summarize()
s.lock.Unlock()
latency, jitter = time.Duration(w.Mean()), time.Duration(w.StdDev())
// TODO: remove this check after debugging unexpected jitter issue
if jitter > 10*time.Second {
logger.Infow("unexpected forward jitter",
"jitter", jitter,
"stats", fmt.Sprintf("count %.2f, mean %.2f, stdDev %.2f", w.Count(), w.Mean(), w.StdDev()),
)
}
return
}
func (s *ForwardStats) GetLastStats(duration time.Duration) (latency, jitter time.Duration) {
s.lock.Lock()
defer s.lock.Unlock()
w := s.latency.SummarizeLast(duration)
return time.Duration(w.Mean()), time.Duration(w.StdDev())
}
func (s *ForwardStats) Stop() {
close(s.closeCh)
}
func (s *ForwardStats) report(reportInterval time.Duration) {
ticker := time.NewTicker(reportInterval)
defer ticker.Stop()
for {
select {
case <-s.closeCh:
return
case <-ticker.C:
latency, jitter := s.GetLastStats(reportInterval)
latencySlow, jitterSlow := s.GetStats()
prometheus.RecordForwardJitter(uint32(jitter/time.Millisecond), uint32(jitterSlow/time.Millisecond))
prometheus.RecordForwardLatency(uint32(latency/time.Millisecond), uint32(latencySlow/time.Millisecond))
}
}
}
livekit-server/pkg/sfu/interceptor/rtx.go
+182
View File
@@ -0,0 +1,182 @@
// Copyright 2024 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package interceptor
import (
"sync"
"github.com/pion/interceptor"
"github.com/pion/sdp/v3"
"github.com/pion/webrtc/v4"
"github.com/livekit/livekit-server/pkg/sfu/utils"
"github.com/livekit/protocol/logger"
)
const (
SDESRepairRTPStreamIDURI = "urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id"
rtxProbeCount = 10
)
type streamInfo struct {
mid string
rid string
rsid string
}
type RTXInfoExtractorFactory struct {
onStreamFound func(*interceptor.StreamInfo)
onRTXPairFound func(repair, base uint32)
lock sync.Mutex
streams map[uint32]streamInfo
logger logger.Logger
}
func NewRTXInfoExtractorFactory(onStreamFound func(*interceptor.StreamInfo), onRTXPairFound func(repair, base uint32), logger logger.Logger) *RTXInfoExtractorFactory {
return &RTXInfoExtractorFactory{
onStreamFound: onStreamFound,
onRTXPairFound: onRTXPairFound,
streams: make(map[uint32]streamInfo),
logger: logger,
}
}
func (f *RTXInfoExtractorFactory) NewInterceptor(id string) (interceptor.Interceptor, error) {
return &RTXInfoExtractor{
factory: f,
logger: f.logger,
}, nil
}
func (f *RTXInfoExtractorFactory) setStreamInfo(ssrc uint32, mid, rid, rsid string) {
var repairSsrc, baseSsrc uint32
f.lock.Lock()
if rsid != "" {
// repair stream found, find base stream
for base, info := range f.streams {
if info.mid == mid && info.rid == rsid {
repairSsrc = ssrc
baseSsrc = base
delete(f.streams, base)
break
}
}
} else {
// base stream found, find repair stream
for repair, info := range f.streams {
if info.mid == mid && info.rsid == rid {
repairSsrc = repair
baseSsrc = ssrc
delete(f.streams, repair)
break
}
}
}
// no rtx pair found, save it for later
if repairSsrc == 0 || baseSsrc == 0 {
f.streams[ssrc] = streamInfo{
mid: mid,
rid: rid,
rsid: rsid,
}
}
f.lock.Unlock()
if repairSsrc != 0 && baseSsrc != 0 {
f.onRTXPairFound(repairSsrc, baseSsrc)
}
}
type RTXInfoExtractor struct {
interceptor.NoOp
factory *RTXInfoExtractorFactory
logger logger.Logger
}
func (u *RTXInfoExtractor) BindRemoteStream(info *interceptor.StreamInfo, reader interceptor.RTPReader) interceptor.RTPReader {
u.factory.onStreamFound(info)
midExtensionID := utils.GetHeaderExtensionID(info.RTPHeaderExtensions, webrtc.RTPHeaderExtensionCapability{URI: sdp.SDESMidURI})
streamIDExtensionID := utils.GetHeaderExtensionID(info.RTPHeaderExtensions, webrtc.RTPHeaderExtensionCapability{URI: sdp.SDESRTPStreamIDURI})
repairStreamIDExtensionID := utils.GetHeaderExtensionID(info.RTPHeaderExtensions, webrtc.RTPHeaderExtensionCapability{URI: SDESRepairRTPStreamIDURI})
if midExtensionID == 0 || streamIDExtensionID == 0 || repairStreamIDExtensionID == 0 {
return reader
}
return &rtxInfoReader{
tryTimes: rtxProbeCount,
reader: reader,
midExtID: uint8(midExtensionID),
ridExtID: uint8(streamIDExtensionID),
rsidExtID: uint8(repairStreamIDExtensionID),
factory: u.factory,
logger: u.logger,
}
}
type rtxInfoReader struct {
tryTimes int
reader interceptor.RTPReader
midExtID uint8
ridExtID uint8
rsidExtID uint8
factory *RTXInfoExtractorFactory
logger logger.Logger
}
func (r *rtxInfoReader) Read(b []byte, a interceptor.Attributes) (int, interceptor.Attributes, error) {
n, a, err := r.reader.Read(b, a)
if r.tryTimes < 0 || err != nil {
return n, a, err
}
if a == nil {
a = make(interceptor.Attributes)
}
header, err := a.GetRTPHeader(b[:n])
if err != nil {
return n, a, nil
}
var mid, rid, rsid string
if payload := header.GetExtension(r.midExtID); payload != nil {
mid = string(payload)
}
if payload := header.GetExtension(r.ridExtID); payload != nil {
rid = string(payload)
}
if payload := header.GetExtension(r.rsidExtID); payload != nil {
rsid = string(payload)
}
if mid != "" && (rid != "" || rsid != "") {
r.logger.Debugw("stream found", "mid", mid, "rid", rid, "rsid", rsid, "ssrc", header.SSRC)
r.tryTimes = -1
go r.factory.setStreamInfo(header.SSRC, mid, rid, rsid)
} else {
// ignore padding only packet for probe count
if !(header.Padding && n-header.MarshalSize()-int(b[n-1]) == 0) {
r.tryTimes--
}
}
return n, a, nil
}
livekit-server/pkg/sfu/mime/mimetype.go
+281
View File
@@ -0,0 +1,281 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package mime
import (
"strings"
"github.com/pion/webrtc/v4"
)
const (
MimeTypePrefixAudio = "audio/"
MimeTypePrefixVideo = "video/"
)
type MimeTypeCodec int
const (
MimeTypeCodecUnknown MimeTypeCodec = iota
MimeTypeCodecH264
MimeTypeCodecH265
MimeTypeCodecOpus
MimeTypeCodecRED
MimeTypeCodecVP8
MimeTypeCodecVP9
MimeTypeCodecAV1
MimeTypeCodecG722
MimeTypeCodecPCMU
MimeTypeCodecPCMA
MimeTypeCodecRTX
MimeTypeCodecFlexFEC
MimeTypeCodecULPFEC
)
func (m MimeTypeCodec) String() string {
switch m {
case MimeTypeCodecUnknown:
return "MimeTypeCodecUnknown"
case MimeTypeCodecH264:
return "H264"
case MimeTypeCodecH265:
return "H265"
case MimeTypeCodecOpus:
return "opus"
case MimeTypeCodecRED:
return "red"
case MimeTypeCodecVP8:
return "VP8"
case MimeTypeCodecVP9:
return "VP9"
case MimeTypeCodecAV1:
return "AV1"
case MimeTypeCodecG722:
return "G722"
case MimeTypeCodecPCMU:
return "PCMU"
case MimeTypeCodecPCMA:
return "PCMA"
case MimeTypeCodecRTX:
return "rtx"
case MimeTypeCodecFlexFEC:
return "flexfec"
case MimeTypeCodecULPFEC:
return "ulpfec"
}
return "MimeTypeCodecUnknown"
}
func NormalizeMimeTypeCodec(codec string) MimeTypeCodec {
switch {
case strings.EqualFold(codec, "h264"):
return MimeTypeCodecH264
case strings.EqualFold(codec, "h265"):
return MimeTypeCodecH265
case strings.EqualFold(codec, "opus"):
return MimeTypeCodecOpus
case strings.EqualFold(codec, "red"):
return MimeTypeCodecRED
case strings.EqualFold(codec, "vp8"):
return MimeTypeCodecVP8
case strings.EqualFold(codec, "vp9"):
return MimeTypeCodecVP9
case strings.EqualFold(codec, "av1"):
return MimeTypeCodecAV1
case strings.EqualFold(codec, "g722"):
return MimeTypeCodecG722
case strings.EqualFold(codec, "pcmu"):
return MimeTypeCodecPCMU
case strings.EqualFold(codec, "pcma"):
return MimeTypeCodecPCMA
case strings.EqualFold(codec, "rtx"):
return MimeTypeCodecRTX
case strings.EqualFold(codec, "flexfec"):
return MimeTypeCodecFlexFEC
case strings.EqualFold(codec, "ulpfec"):
return MimeTypeCodecULPFEC
}
return MimeTypeCodecUnknown
}
func GetMimeTypeCodec(mime string) MimeTypeCodec {
i := strings.IndexByte(mime, '/')
if i == -1 {
return MimeTypeCodecUnknown
}
return NormalizeMimeTypeCodec(mime[i+1:])
}
func IsMimeTypeCodecStringOpus(codec string) bool {
return NormalizeMimeTypeCodec(codec) == MimeTypeCodecOpus
}
func IsMimeTypeCodecStringRED(codec string) bool {
return NormalizeMimeTypeCodec(codec) == MimeTypeCodecRED
}
func IsMimeTypeCodecStringH264(codec string) bool {
return NormalizeMimeTypeCodec(codec) == MimeTypeCodecH264
}
type MimeType int
const (
MimeTypeUnknown MimeType = iota
MimeTypeH264
MimeTypeH265
MimeTypeOpus
MimeTypeRED
MimeTypeVP8
MimeTypeVP9
MimeTypeAV1
MimeTypeG722
MimeTypePCMU
MimeTypePCMA
MimeTypeRTX
MimeTypeFlexFEC
MimeTypeULPFEC
)
func (m MimeType) String() string {
switch m {
case MimeTypeUnknown:
return "MimeTypeUnknown"
case MimeTypeH264:
return webrtc.MimeTypeH264
case MimeTypeH265:
return webrtc.MimeTypeH265
case MimeTypeOpus:
return webrtc.MimeTypeOpus
case MimeTypeRED:
return "audio/red"
case MimeTypeVP8:
return webrtc.MimeTypeVP8
case MimeTypeVP9:
return webrtc.MimeTypeVP9
case MimeTypeAV1:
return webrtc.MimeTypeAV1
case MimeTypeG722:
return webrtc.MimeTypeG722
case MimeTypePCMU:
return webrtc.MimeTypePCMU
case MimeTypePCMA:
return webrtc.MimeTypePCMA
case MimeTypeRTX:
return webrtc.MimeTypeRTX
case MimeTypeFlexFEC:
return webrtc.MimeTypeFlexFEC
case MimeTypeULPFEC:
return "video/ulpfec"
}
return "MimeTypeUnknown"
}
func NormalizeMimeType(mime string) MimeType {
switch {
case strings.EqualFold(mime, webrtc.MimeTypeH264):
return MimeTypeH264
case strings.EqualFold(mime, webrtc.MimeTypeH265):
return MimeTypeH265
case strings.EqualFold(mime, webrtc.MimeTypeOpus):
return MimeTypeOpus
case strings.EqualFold(mime, "audio/red"):
return MimeTypeRED
case strings.EqualFold(mime, webrtc.MimeTypeVP8):
return MimeTypeVP8
case strings.EqualFold(mime, webrtc.MimeTypeVP9):
return MimeTypeVP9
case strings.EqualFold(mime, webrtc.MimeTypeAV1):
return MimeTypeAV1
case strings.EqualFold(mime, webrtc.MimeTypeG722):
return MimeTypeG722
case strings.EqualFold(mime, webrtc.MimeTypePCMU):
return MimeTypePCMU
case strings.EqualFold(mime, webrtc.MimeTypePCMA):
return MimeTypePCMA
case strings.EqualFold(mime, webrtc.MimeTypeRTX):
return MimeTypeRTX
case strings.EqualFold(mime, webrtc.MimeTypeFlexFEC):
return MimeTypeFlexFEC
case strings.EqualFold(mime, "video/ulpfec"):
return MimeTypeULPFEC
}
return MimeTypeUnknown
}
func IsMimeTypeStringEqual(mime1 string, mime2 string) bool {
return NormalizeMimeType(mime1) == NormalizeMimeType(mime2)
}
func IsMimeTypeStringAudio(mime string) bool {
return strings.HasPrefix(mime, MimeTypePrefixAudio)
}
func IsMimeTypeAudio(mimeType MimeType) bool {
return strings.HasPrefix(mimeType.String(), MimeTypePrefixAudio)
}
func IsMimeTypeStringVideo(mime string) bool {
return strings.HasPrefix(mime, MimeTypePrefixVideo)
}
func IsMimeTypeVideo(mimeType MimeType) bool {
return strings.HasPrefix(mimeType.String(), MimeTypePrefixVideo)
}
// SVC-TODO: Have to use more conditions to differentiate between
// SVC-TODO: SVC and non-SVC (could be single layer or simulcast).
// SVC-TODO: May only need to differentiate between simulcast and non-simulcast
// SVC-TODO: i. e. may be possible to treat single layer as SVC to get proper/intended functionality.
func IsMimeTypeSVC(mimeType MimeType) bool {
switch mimeType {
case MimeTypeAV1, MimeTypeVP9:
return true
}
return false
}
func IsMimeTypeStringSVC(mime string) bool {
return IsMimeTypeSVC(NormalizeMimeType(mime))
}
func IsMimeTypeStringRED(mime string) bool {
return NormalizeMimeType(mime) == MimeTypeRED
}
func IsMimeTypeStringOpus(mime string) bool {
return NormalizeMimeType(mime) == MimeTypeOpus
}
func IsMimeTypeStringRTX(mime string) bool {
return NormalizeMimeType(mime) == MimeTypeRTX
}
func IsMimeTypeStringVP8(mime string) bool {
return NormalizeMimeType(mime) == MimeTypeVP8
}
func IsMimeTypeStringVP9(mime string) bool {
return NormalizeMimeType(mime) == MimeTypeVP9
}
func IsMimeTypeStringH264(mime string) bool {
return NormalizeMimeType(mime) == MimeTypeH264
}
livekit-server/pkg/sfu/pacer/base.go
+127
View File
@@ -0,0 +1,127 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pacer
import (
"errors"
"io"
"time"
"github.com/livekit/livekit-server/pkg/sfu/bwe"
"github.com/livekit/protocol/logger"
"github.com/livekit/protocol/utils/mono"
"github.com/pion/rtp"
"go.uber.org/atomic"
)
type Base struct {
logger logger.Logger
bwe bwe.BWE
lastPacketSentAt atomic.Int64
*ProbeObserver
}
func NewBase(logger logger.Logger, bwe bwe.BWE) *Base {
return &Base{
logger: logger,
bwe: bwe,
ProbeObserver: NewProbeObserver(logger),
}
}
func (b *Base) SetInterval(_interval time.Duration) {
}
func (b *Base) SetBitrate(_bitrate int) {
}
func (b *Base) TimeSinceLastSentPacket() time.Duration {
return time.Duration(mono.UnixNano() - b.lastPacketSentAt.Load())
}
func (b *Base) SendPacket(p *Packet) (int, error) {
defer func() {
if p.Pool != nil && p.PoolEntity != nil {
p.Pool.Put(p.PoolEntity)
}
}()
err := b.patchRTPHeaderExtensions(p)
if err != nil {
b.logger.Errorw("patching rtp header extensions err", err)
return 0, err
}
var written int
written, err = p.WriteStream.WriteRTP(p.Header, p.Payload)
if err != nil {
if !errors.Is(err, io.ErrClosedPipe) {
b.logger.Errorw("write rtp packet failed", err)
}
return 0, err
}
return written, nil
}
// patch just abs-send-time and transport-cc extensions if applicable
func (b *Base) patchRTPHeaderExtensions(p *Packet) error {
sendingAt := mono.Now()
if p.AbsSendTimeExtID != 0 {
absSendTime := rtp.NewAbsSendTimeExtension(sendingAt)
absSendTimeBytes, err := absSendTime.Marshal()
if err != nil {
return err
}
if err = p.Header.SetExtension(p.AbsSendTimeExtID, absSendTimeBytes); err != nil {
return err
}
b.lastPacketSentAt.Store(sendingAt.UnixNano())
}
packetSize := p.HeaderSize + len(p.Payload)
if p.TransportWideExtID != 0 && b.bwe != nil {
twccSN := b.bwe.RecordPacketSendAndGetSequenceNumber(
sendingAt.UnixMicro(),
packetSize,
p.IsRTX,
p.ProbeClusterId,
p.IsProbe,
)
twccExt := rtp.TransportCCExtension{
TransportSequence: twccSN,
}
twccExtBytes, err := twccExt.Marshal()
if err != nil {
return err
}
if err = p.Header.SetExtension(p.TransportWideExtID, twccExtBytes); err != nil {
return err
}
b.lastPacketSentAt.Store(sendingAt.UnixNano())
}
b.ProbeObserver.RecordPacket(packetSize, p.IsRTX, p.ProbeClusterId, p.IsProbe)
return nil
}
// ------------------------------------------------
livekit-server/pkg/sfu/pacer/leaky_bucket.go
+142
View File
@@ -0,0 +1,142 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pacer
import (
"sync"
"time"
"github.com/frostbyte73/core"
"github.com/gammazero/deque"
"github.com/livekit/livekit-server/pkg/sfu/bwe"
"github.com/livekit/protocol/logger"
)
const (
maxOvershootFactor = 2.0
)
type LeakyBucket struct {
*Base
logger logger.Logger
lock sync.RWMutex
packets deque.Deque[*Packet]
interval time.Duration
bitrate int
stop core.Fuse
}
func NewLeakyBucket(logger logger.Logger, bwe bwe.BWE, interval time.Duration, bitrate int) *LeakyBucket {
l := &LeakyBucket{
Base: NewBase(logger, bwe),
logger: logger,
interval: interval,
bitrate: bitrate,
}
l.packets.SetBaseCap(512)
go l.sendWorker()
return l
}
func (l *LeakyBucket) SetInterval(interval time.Duration) {
l.lock.Lock()
defer l.lock.Unlock()
l.interval = interval
}
func (l *LeakyBucket) SetBitrate(bitrate int) {
l.lock.Lock()
defer l.lock.Unlock()
l.bitrate = bitrate
}
func (l *LeakyBucket) Stop() {
l.stop.Break()
}
func (l *LeakyBucket) Enqueue(p *Packet) {
l.lock.Lock()
l.packets.PushBack(p)
l.lock.Unlock()
}
func (l *LeakyBucket) sendWorker() {
l.lock.RLock()
interval := l.interval
bitrate := l.bitrate
l.lock.RUnlock()
timer := time.NewTimer(interval)
overage := 0
for {
<-timer.C
l.lock.RLock()
interval = l.interval
bitrate = l.bitrate
l.lock.RUnlock()
// calculate number of bytes that can be sent in this interval
// adjusting for overage.
intervalBytes := int(interval.Seconds() * float64(bitrate) / 8.0)
maxOvershootBytes := int(float64(intervalBytes) * maxOvershootFactor)
toSendBytes := intervalBytes - overage
if toSendBytes < 0 {
// too much overage, wait for next interval
overage = -toSendBytes
timer.Reset(interval)
continue
}
// do not allow too much overshoot in an interval
if toSendBytes > maxOvershootBytes {
toSendBytes = maxOvershootBytes
}
for {
if l.stop.IsBroken() {
return
}
l.lock.Lock()
if l.packets.Len() == 0 {
l.lock.Unlock()
// allow overshoot in next interval with shortage in this interval
overage = -toSendBytes
timer.Reset(interval)
break
}
p := l.packets.PopFront()
l.lock.Unlock()
written, _ := l.Base.SendPacket(p)
toSendBytes -= written
if toSendBytes < 0 {
// overage, wait for next interval
overage = -toSendBytes
timer.Reset(interval)
break
}
}
}
}
// ------------------------------------------------
livekit-server/pkg/sfu/pacer/no_queue.go
+90
View File
@@ -0,0 +1,90 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pacer
import (
"sync"
"github.com/frostbyte73/core"
"github.com/gammazero/deque"
"github.com/livekit/livekit-server/pkg/sfu/bwe"
"github.com/livekit/protocol/logger"
)
type NoQueue struct {
*Base
logger logger.Logger
lock sync.RWMutex
packets deque.Deque[*Packet]
wake chan struct{}
stop core.Fuse
}
func NewNoQueue(logger logger.Logger, bwe bwe.BWE) *NoQueue {
n := &NoQueue{
Base: NewBase(logger, bwe),
logger: logger,
wake: make(chan struct{}, 1),
}
n.packets.SetBaseCap(512)
go n.sendWorker()
return n
}
func (n *NoQueue) Stop() {
n.stop.Break()
select {
case n.wake <- struct{}{}:
default:
}
}
func (n *NoQueue) Enqueue(p *Packet) {
n.lock.Lock()
n.packets.PushBack(p)
n.lock.Unlock()
select {
case n.wake <- struct{}{}:
default:
}
}
func (n *NoQueue) sendWorker() {
for {
<-n.wake
for {
if n.stop.IsBroken() {
return
}
n.lock.Lock()
if n.packets.Len() == 0 {
n.lock.Unlock()
break
}
p := n.packets.PopFront()
n.lock.Unlock()
n.Base.SendPacket(p)
}
}
}
// ------------------------------------------------
livekit-server/pkg/sfu/pacer/pacer.go
+58
View File
@@ -0,0 +1,58 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pacer
import (
"sync"
"time"
"github.com/livekit/livekit-server/pkg/sfu/ccutils"
"github.com/pion/rtp"
"github.com/pion/webrtc/v4"
)
type Packet struct {
Header *rtp.Header
HeaderSize int
Payload []byte
IsRTX bool
ProbeClusterId ccutils.ProbeClusterId
IsProbe bool
AbsSendTimeExtID uint8
TransportWideExtID uint8
WriteStream webrtc.TrackLocalWriter
Pool *sync.Pool
PoolEntity *[]byte
}
type Pacer interface {
Enqueue(p *Packet)
Stop()
SetInterval(interval time.Duration)
SetBitrate(bitrate int)
TimeSinceLastSentPacket() time.Duration
SetPacerProbeObserverListener(listener PacerProbeObserverListener)
StartProbeCluster(pci ccutils.ProbeClusterInfo)
EndProbeCluster(probeClusterId ccutils.ProbeClusterId) ccutils.ProbeClusterInfo
}
type PacerProbeObserverListener interface {
OnPacerProbeObserverClusterComplete(probeClusterId ccutils.ProbeClusterId)
}
// ------------------------------------------------
livekit-server/pkg/sfu/pacer/pass_through.go
+39
View File
@@ -0,0 +1,39 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pacer
import (
"github.com/livekit/livekit-server/pkg/sfu/bwe"
"github.com/livekit/protocol/logger"
)
type PassThrough struct {
*Base
}
func NewPassThrough(logger logger.Logger, bwe bwe.BWE) *PassThrough {
return &PassThrough{
Base: NewBase(logger, bwe),
}
}
func (p *PassThrough) Stop() {
}
func (p *PassThrough) Enqueue(pkt *Packet) {
p.Base.SendPacket(pkt)
}
// ------------------------------------------------
livekit-server/pkg/sfu/pacer/probe_observer.go
+143
View File
@@ -0,0 +1,143 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pacer
import (
"sync"
"time"
"go.uber.org/atomic"
"github.com/livekit/livekit-server/pkg/sfu/ccutils"
"github.com/livekit/protocol/logger"
"github.com/livekit/protocol/utils/mono"
)
type ProbeObserver struct {
logger logger.Logger
listener PacerProbeObserverListener
isInProbe atomic.Bool
lock sync.Mutex
pci ccutils.ProbeClusterInfo
}
func NewProbeObserver(logger logger.Logger) *ProbeObserver {
return &ProbeObserver{
logger: logger,
}
}
func (po *ProbeObserver) SetPacerProbeObserverListener(listener PacerProbeObserverListener) {
po.listener = listener
}
func (po *ProbeObserver) StartProbeCluster(pci ccutils.ProbeClusterInfo) {
if po.isInProbe.Load() {
po.logger.Warnw(
"ignoring start of a new probe cluster when already active", nil,
"probeClusterInfo", pci,
)
return
}
po.lock.Lock()
defer po.lock.Unlock()
po.pci = pci
po.pci.Result = ccutils.ProbeClusterResult{
StartTime: mono.UnixNano(),
}
po.isInProbe.Store(true)
}
func (po *ProbeObserver) EndProbeCluster(probeClusterId ccutils.ProbeClusterId) ccutils.ProbeClusterInfo {
if !po.isInProbe.Load() {
// probe not active
if probeClusterId != ccutils.ProbeClusterIdInvalid {
po.logger.Debugw(
"ignoring end of a probe cluster when not active",
"probeClusterId", probeClusterId,
)
}
return ccutils.ProbeClusterInfoInvalid
}
po.lock.Lock()
defer po.lock.Unlock()
if po.pci.Id != probeClusterId {
// probe cluster id not active
po.logger.Warnw(
"ignoring end of a probe cluster of a non-active one", nil,
"probeClusterId", probeClusterId,
"active", po.pci.Id,
)
return ccutils.ProbeClusterInfoInvalid
}
if po.pci.Result.EndTime == 0 {
po.pci.Result.EndTime = mono.UnixNano()
}
po.isInProbe.Store(false)
return po.pci
}
func (po *ProbeObserver) RecordPacket(size int, isRTX bool, probeClusterId ccutils.ProbeClusterId, isProbe bool) {
if !po.isInProbe.Load() {
return
}
po.lock.Lock()
if probeClusterId != po.pci.Id || po.pci.Result.EndTime != 0 {
po.lock.Unlock()
return
}
if isProbe {
po.pci.Result.PacketsProbe++
po.pci.Result.BytesProbe += size
} else {
if isRTX {
po.pci.Result.PacketsNonProbeRTX++
po.pci.Result.BytesNonProbeRTX += size
} else {
po.pci.Result.PacketsNonProbePrimary++
po.pci.Result.BytesNonProbePrimary += size
}
}
notify := false
var clusterId ccutils.ProbeClusterId
if po.pci.Result.EndTime == 0 && ((po.pci.Result.Bytes() >= po.pci.Goal.DesiredBytes) && time.Duration(mono.UnixNano()-po.pci.Result.StartTime) >= po.pci.Goal.Duration) {
po.pci.Result.EndTime = mono.UnixNano()
po.pci.Result.IsCompleted = true
notify = true
clusterId = po.pci.Id
}
po.lock.Unlock()
if notify && po.listener != nil {
po.listener.OnPacerProbeObserverClusterComplete(clusterId)
}
}
// ------------------------------------------------
livekit-server/pkg/sfu/playoutdelay.go
+202
View File
@@ -0,0 +1,202 @@
// Copyright 2024 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package sfu
import (
"sync"
"time"
pd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/playoutdelay"
"github.com/livekit/livekit-server/pkg/sfu/rtpstats"
"github.com/livekit/protocol/logger"
"go.uber.org/atomic"
"go.uber.org/zap/zapcore"
)
const (
jitterMultiToDelay = 10
targetDelayLogThreshold = 500
// limit max delay change to make it smoother for a/v sync
maxDelayChangePerSec = 80
)
// ----------------------------------------------------
type PlayoutDelayState int32
const (
PlayoutDelayStateChanged PlayoutDelayState = iota
PlayoutDelaySending
PlayoutDelayAcked
)
func (s PlayoutDelayState) String() string {
switch s {
case PlayoutDelayStateChanged:
return "StateChanged"
case PlayoutDelaySending:
return "Sending"
case PlayoutDelayAcked:
return "Acked"
}
return "Unknown"
}
// ----------------------------------------------------
type PlayoutDelayControllerState struct {
SenderSnapshotID uint32
}
func (p PlayoutDelayControllerState) MarshalLogObject(e zapcore.ObjectEncoder) error {
e.AddUint32("SenderSnapshotID", p.SenderSnapshotID)
return nil
}
// ----------------------------------------------------
type PlayoutDelayController struct {
lock sync.Mutex
state atomic.Int32
minDelay, maxDelay uint32
currentDelay uint32
extBytes atomic.Value //[]byte
sendingAtSeq uint16
sendingAtTime time.Time
logger logger.Logger
rtpStats *rtpstats.RTPStatsSender
senderSnapshotID uint32
highDelayCount atomic.Uint32
}
func NewPlayoutDelayController(minDelay, maxDelay uint32, logger logger.Logger, rtpStats *rtpstats.RTPStatsSender) (*PlayoutDelayController, error) {
if maxDelay == 0 && minDelay > 0 {
maxDelay = pd.MaxPlayoutDelayDefault
}
if maxDelay > pd.PlayoutDelayMaxValue {
maxDelay = pd.PlayoutDelayMaxValue
}
c := &PlayoutDelayController{
currentDelay: minDelay,
minDelay: minDelay,
maxDelay: maxDelay,
logger: logger,
rtpStats: rtpStats,
senderSnapshotID: rtpStats.NewSenderSnapshotId(),
}
return c, c.createExtData()
}
func (c *PlayoutDelayController) GetState() PlayoutDelayControllerState {
c.lock.Lock()
defer c.lock.Unlock()
return PlayoutDelayControllerState{
SenderSnapshotID: c.senderSnapshotID,
}
}
func (c *PlayoutDelayController) SeedState(pdcs PlayoutDelayControllerState) {
c.lock.Lock()
defer c.lock.Unlock()
c.senderSnapshotID = pdcs.SenderSnapshotID
}
func (c *PlayoutDelayController) SetJitter(jitter uint32) {
c.lock.Lock()
deltaInfoSender, _ := c.rtpStats.DeltaInfoSender(c.senderSnapshotID)
var nackPercent uint32
if deltaInfoSender != nil && deltaInfoSender.Packets > 0 {
nackPercent = deltaInfoSender.Nacks * 100 / deltaInfoSender.Packets
}
targetDelay := jitter * jitterMultiToDelay
if nackPercent > 60 {
targetDelay += (nackPercent - 60) * 2
}
elapsed := time.Since(c.sendingAtTime)
delayChangeLimit := uint32(maxDelayChangePerSec * elapsed.Seconds())
if delayChangeLimit > maxDelayChangePerSec {
delayChangeLimit = maxDelayChangePerSec
}
if targetDelay > c.currentDelay+delayChangeLimit {
targetDelay = c.currentDelay + delayChangeLimit
} else if c.currentDelay > targetDelay+delayChangeLimit {
targetDelay = c.currentDelay - delayChangeLimit
}
if targetDelay < c.minDelay {
targetDelay = c.minDelay
}
if targetDelay > c.maxDelay {
targetDelay = c.maxDelay
}
if c.currentDelay == targetDelay {
c.lock.Unlock()
return
}
if targetDelay > targetDelayLogThreshold {
if c.highDelayCount.Add(1)%100 == 1 {
c.logger.Infow("high playout delay", "target", targetDelay, "jitter", jitter, "nackPercent", nackPercent, "current", c.currentDelay)
}
}
c.currentDelay = targetDelay
c.lock.Unlock()
c.createExtData()
}
func (c *PlayoutDelayController) OnSeqAcked(seq uint16) {
c.lock.Lock()
defer c.lock.Unlock()
if PlayoutDelayState(c.state.Load()) == PlayoutDelaySending && (seq-c.sendingAtSeq) < 0x8000 {
c.state.Store(int32(PlayoutDelayAcked))
}
}
func (c *PlayoutDelayController) GetDelayExtension(seq uint16) []byte {
switch PlayoutDelayState(c.state.Load()) {
case PlayoutDelayStateChanged:
c.lock.Lock()
c.state.Store(int32(PlayoutDelaySending))
c.sendingAtSeq = seq
c.sendingAtTime = time.Now()
c.lock.Unlock()
return c.extBytes.Load().([]byte)
case PlayoutDelaySending:
return c.extBytes.Load().([]byte)
case PlayoutDelayAcked:
return nil
}
return nil
}
func (c *PlayoutDelayController) createExtData() error {
delay := pd.PlayoutDelayFromValue(
uint16(c.currentDelay),
uint16(c.maxDelay),
)
b, err := delay.Marshal()
if err == nil {
c.extBytes.Store(b)
c.state.Store(int32(PlayoutDelayStateChanged))
} else {
c.logger.Errorw("failed to marshal playout delay", err, "playoutDelay", delay)
}
return err
}
livekit-server/pkg/sfu/playoutdelay_test.go
+79
View File
@@ -0,0 +1,79 @@
// Copyright 2024 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package sfu
import (
"testing"
"time"
"github.com/stretchr/testify/require"
pd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/playoutdelay"
"github.com/livekit/livekit-server/pkg/sfu/rtpstats"
"github.com/livekit/protocol/logger"
)
func TestPlayoutDelay(t *testing.T) {
stats := rtpstats.NewRTPStatsSender(rtpstats.RTPStatsParams{ClockRate: 900000, Logger: logger.GetLogger()}, 128)
c, err := NewPlayoutDelayController(100, 120, logger.GetLogger(), stats)
require.NoError(t, err)
ext := c.GetDelayExtension(100)
playoutDelayEqual(t, ext, 100, 120)
ext = c.GetDelayExtension(105)
playoutDelayEqual(t, ext, 100, 120)
// seq acked before delay changed
c.OnSeqAcked(65534)
ext = c.GetDelayExtension(105)
playoutDelayEqual(t, ext, 100, 120)
c.OnSeqAcked(90)
ext = c.GetDelayExtension(105)
playoutDelayEqual(t, ext, 100, 120)
// seq acked, no extension sent for new packet
c.OnSeqAcked(103)
ext = c.GetDelayExtension(106)
require.Nil(t, ext)
// delay on change(can't go below min), no extension sent
c.SetJitter(0)
ext = c.GetDelayExtension(107)
require.Nil(t, ext)
// delay changed, generate new extension to send
time.Sleep(200 * time.Millisecond)
c.SetJitter(50)
t.Log(c.currentDelay, c.state.Load())
ext = c.GetDelayExtension(108)
var delay pd.PlayOutDelay
require.NoError(t, delay.Unmarshal(ext))
require.Greater(t, delay.Min, uint16(100))
// can't go above max
time.Sleep(200 * time.Millisecond)
c.SetJitter(10000)
ext = c.GetDelayExtension(109)
playoutDelayEqual(t, ext, 120, 120)
}
func playoutDelayEqual(t *testing.T, data []byte, min, max uint16) {
var delay pd.PlayOutDelay
require.NoError(t, delay.Unmarshal(data))
require.Equal(t, min, delay.Min)
require.Equal(t, max, delay.Max)
}
livekit-server/pkg/sfu/receiver.go
+957
View File
@@ -0,0 +1,957 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package sfu
import (
"errors"
"io"
"strings"
"sync"
"time"
"github.com/pion/rtcp"
"github.com/pion/webrtc/v4"
"go.uber.org/atomic"
"github.com/livekit/mediatransportutil/pkg/bucket"
"github.com/livekit/protocol/livekit"
"github.com/livekit/protocol/logger"
"github.com/livekit/protocol/utils"
"github.com/livekit/livekit-server/pkg/sfu/audio"
"github.com/livekit/livekit-server/pkg/sfu/buffer"
"github.com/livekit/livekit-server/pkg/sfu/connectionquality"
"github.com/livekit/livekit-server/pkg/sfu/mime"
dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor"
"github.com/livekit/livekit-server/pkg/sfu/rtpstats"
"github.com/livekit/livekit-server/pkg/sfu/streamtracker"
)
var (
ErrReceiverClosed = errors.New("receiver closed")
ErrDownTrackAlreadyExist = errors.New("DownTrack already exist")
ErrBufferNotFound = errors.New("buffer not found")
ErrDuplicateLayer = errors.New("duplicate layer")
)
// --------------------------------------
type PLIThrottleConfig struct {
LowQuality time.Duration `yaml:"low_quality,omitempty"`
MidQuality time.Duration `yaml:"mid_quality,omitempty"`
HighQuality time.Duration `yaml:"high_quality,omitempty"`
}
var (
DefaultPLIThrottleConfig = PLIThrottleConfig{
LowQuality: 500 * time.Millisecond,
MidQuality: time.Second,
HighQuality: time.Second,
}
)
// --------------------------------------
type AudioConfig struct {
audio.AudioLevelConfig `yaml:",inline"`
// enable red encoding downtrack for opus only audio up track
ActiveREDEncoding bool `yaml:"active_red_encoding,omitempty"`
// enable proxying weakest subscriber loss to publisher in RTCP Receiver Report
EnableLossProxying bool `yaml:"enable_loss_proxying,omitempty"`
}
var (
DefaultAudioConfig = AudioConfig{
AudioLevelConfig: audio.DefaultAudioLevelConfig,
}
)
// --------------------------------------
type AudioLevelHandle func(level uint8, duration uint32)
type Bitrates [buffer.DefaultMaxLayerSpatial + 1][buffer.DefaultMaxLayerTemporal + 1]int64
type ReceiverCodecState int
const (
ReceiverCodecStateNormal ReceiverCodecState = iota
ReceiverCodecStateSuspended
ReceiverCodecStateInvalid
)
// TrackReceiver defines an interface receive media from remote peer
type TrackReceiver interface {
TrackID() livekit.TrackID
StreamID() string
// returns the initial codec of the receiver, it is determined by the track's codec
// and will not change if the codec changes during the session (publisher changes codec)
Codec() webrtc.RTPCodecParameters
Mime() mime.MimeType
HeaderExtensions() []webrtc.RTPHeaderExtensionParameter
IsClosed() bool
ReadRTP(buf []byte, layer uint8, esn uint64) (int, error)
GetLayeredBitrate() ([]int32, Bitrates)
GetAudioLevel() (float64, bool)
SendPLI(layer int32, force bool)
SetUpTrackPaused(paused bool)
SetMaxExpectedSpatialLayer(layer int32)
AddDownTrack(track TrackSender) error
DeleteDownTrack(participantID livekit.ParticipantID)
GetDownTracks() []TrackSender
DebugInfo() map[string]interface{}
TrackInfo() *livekit.TrackInfo
UpdateTrackInfo(ti *livekit.TrackInfo)
// Get primary receiver if this receiver represents a RED codec; otherwise it will return itself
GetPrimaryReceiverForRed() TrackReceiver
// Get red receiver for primary codec, used by forward red encodings for opus only codec
GetRedReceiver() TrackReceiver
GetTemporalLayerFpsForSpatial(layer int32) []float32
GetTrackStats() *livekit.RTPStats
// AddOnReady adds a function to be called when the receiver is ready, the callback
// could be called immediately if the receiver is ready when the callback is added
AddOnReady(func())
AddOnCodecStateChange(func(webrtc.RTPCodecParameters, ReceiverCodecState))
CodecState() ReceiverCodecState
}
type redPktWriteFunc func(pkt *buffer.ExtPacket, spatialLayer int32) int
// WebRTCReceiver receives a media track
type WebRTCReceiver struct {
logger logger.Logger
pliThrottleConfig PLIThrottleConfig
audioConfig AudioConfig
trackID livekit.TrackID
streamID string
kind webrtc.RTPCodecType
receiver *webrtc.RTPReceiver
codec webrtc.RTPCodecParameters
codecState ReceiverCodecState
codecStateLock sync.Mutex
onCodecStateChange []func(webrtc.RTPCodecParameters, ReceiverCodecState)
isSVC bool
isRED bool
onCloseHandler func()
closeOnce sync.Once
closed atomic.Bool
useTrackers bool
trackInfo atomic.Pointer[livekit.TrackInfo]
onRTCP func([]rtcp.Packet)
bufferMu sync.RWMutex
buffers [buffer.DefaultMaxLayerSpatial + 1]*buffer.Buffer
upTracks [buffer.DefaultMaxLayerSpatial + 1]TrackRemote
rtt uint32
lbThreshold int
streamTrackerManager *StreamTrackerManager
downTrackSpreader *DownTrackSpreader
connectionStats *connectionquality.ConnectionStats
onStatsUpdate func(w *WebRTCReceiver, stat *livekit.AnalyticsStat)
onMaxLayerChange func(maxLayer int32)
primaryReceiver atomic.Pointer[RedPrimaryReceiver]
redReceiver atomic.Pointer[RedReceiver]
redPktWriter atomic.Value // redPktWriteFunc
forwardStats *ForwardStats
}
type ReceiverOpts func(w *WebRTCReceiver) *WebRTCReceiver
// WithPliThrottleConfig indicates minimum time(ms) between sending PLIs
func WithPliThrottleConfig(pliThrottleConfig PLIThrottleConfig) ReceiverOpts {
return func(w *WebRTCReceiver) *WebRTCReceiver {
w.pliThrottleConfig = pliThrottleConfig
return w
}
}
// WithAudioConfig sets up parameters for active speaker detection
func WithAudioConfig(audioConfig AudioConfig) ReceiverOpts {
return func(w *WebRTCReceiver) *WebRTCReceiver {
w.audioConfig = audioConfig
return w
}
}
// WithStreamTrackers enables StreamTracker use for simulcast
func WithStreamTrackers() ReceiverOpts {
return func(w *WebRTCReceiver) *WebRTCReceiver {
w.useTrackers = true
return w
}
}
// WithLoadBalanceThreshold enables parallelization of packet writes when downTracks exceeds threshold
// Value should be between 3 and 150.
// For a server handling a few large rooms, use a smaller value (required to handle very large (250+ participant) rooms).
// For a server handling many small rooms, use a larger value or disable.
// Set to 0 (disabled) by default.
func WithLoadBalanceThreshold(downTracks int) ReceiverOpts {
return func(w *WebRTCReceiver) *WebRTCReceiver {
w.lbThreshold = downTracks
return w
}
}
func WithForwardStats(forwardStats *ForwardStats) ReceiverOpts {
return func(w *WebRTCReceiver) *WebRTCReceiver {
w.forwardStats = forwardStats
return w
}
}
// NewWebRTCReceiver creates a new webrtc track receiver
func NewWebRTCReceiver(
receiver *webrtc.RTPReceiver,
track TrackRemote,
trackInfo *livekit.TrackInfo,
logger logger.Logger,
onRTCP func([]rtcp.Packet),
streamTrackerManagerConfig StreamTrackerManagerConfig,
opts ...ReceiverOpts,
) *WebRTCReceiver {
w := &WebRTCReceiver{
logger: logger,
receiver: receiver,
trackID: livekit.TrackID(track.ID()),
streamID: track.StreamID(),
codec: track.Codec(),
codecState: ReceiverCodecStateNormal,
kind: track.Kind(),
onRTCP: onRTCP,
isSVC: mime.IsMimeTypeStringSVC(track.Codec().MimeType),
isRED: mime.IsMimeTypeStringRED(track.Codec().MimeType),
}
for _, opt := range opts {
w = opt(w)
}
w.trackInfo.Store(utils.CloneProto(trackInfo))
w.downTrackSpreader = NewDownTrackSpreader(DownTrackSpreaderParams{
Threshold: w.lbThreshold,
Logger: logger,
})
w.connectionStats = connectionquality.NewConnectionStats(connectionquality.ConnectionStatsParams{
ReceiverProvider: w,
Logger: w.logger.WithValues("direction", "up"),
})
w.connectionStats.OnStatsUpdate(func(_cs *connectionquality.ConnectionStats, stat *livekit.AnalyticsStat) {
if w.onStatsUpdate != nil {
w.onStatsUpdate(w, stat)
}
})
w.connectionStats.Start(
mime.NormalizeMimeType(w.codec.MimeType),
// TODO: technically not correct to declare FEC on when RED. Need the primary codec's fmtp line to check.
mime.IsMimeTypeStringRED(w.codec.MimeType) || strings.Contains(strings.ToLower(w.codec.SDPFmtpLine), "useinbandfec=1"),
)
w.streamTrackerManager = NewStreamTrackerManager(logger, trackInfo, w.isSVC, w.codec.ClockRate, streamTrackerManagerConfig)
w.streamTrackerManager.SetListener(w)
// SVC-TODO: Handle DD for non-SVC cases???
if w.isSVC {
for _, ext := range receiver.GetParameters().HeaderExtensions {
if ext.URI == dd.ExtensionURI {
w.streamTrackerManager.AddDependencyDescriptorTrackers()
break
}
}
}
return w
}
func (w *WebRTCReceiver) TrackInfo() *livekit.TrackInfo {
return w.trackInfo.Load()
}
func (w *WebRTCReceiver) UpdateTrackInfo(ti *livekit.TrackInfo) {
w.trackInfo.Store(utils.CloneProto(ti))
w.streamTrackerManager.UpdateTrackInfo(ti)
}
func (w *WebRTCReceiver) OnStatsUpdate(fn func(w *WebRTCReceiver, stat *livekit.AnalyticsStat)) {
w.onStatsUpdate = fn
}
func (w *WebRTCReceiver) OnMaxLayerChange(fn func(maxLayer int32)) {
w.bufferMu.Lock()
w.onMaxLayerChange = fn
w.bufferMu.Unlock()
}
func (w *WebRTCReceiver) getOnMaxLayerChange() func(maxLayer int32) {
w.bufferMu.RLock()
defer w.bufferMu.RUnlock()
return w.onMaxLayerChange
}
func (w *WebRTCReceiver) GetConnectionScoreAndQuality() (float32, livekit.ConnectionQuality) {
return w.connectionStats.GetScoreAndQuality()
}
func (w *WebRTCReceiver) IsClosed() bool {
return w.closed.Load()
}
func (w *WebRTCReceiver) SetRTT(rtt uint32) {
w.bufferMu.Lock()
if w.rtt == rtt {
w.bufferMu.Unlock()
return
}
w.rtt = rtt
buffers := w.buffers
w.bufferMu.Unlock()
for _, buff := range buffers {
if buff == nil {
continue
}
buff.SetRTT(rtt)
}
}
func (w *WebRTCReceiver) StreamID() string {
return w.streamID
}
func (w *WebRTCReceiver) TrackID() livekit.TrackID {
return w.trackID
}
func (w *WebRTCReceiver) ssrc(layer int) uint32 {
if track := w.upTracks[layer]; track != nil {
return uint32(track.SSRC())
}
return 0
}
func (w *WebRTCReceiver) Codec() webrtc.RTPCodecParameters {
return w.codec
}
func (w *WebRTCReceiver) Mime() mime.MimeType {
return mime.NormalizeMimeType(w.codec.MimeType)
}
func (w *WebRTCReceiver) HeaderExtensions() []webrtc.RTPHeaderExtensionParameter {
return w.receiver.GetParameters().HeaderExtensions
}
func (w *WebRTCReceiver) Kind() webrtc.RTPCodecType {
return w.kind
}
func (w *WebRTCReceiver) AddUpTrack(track TrackRemote, buff *buffer.Buffer) error {
if w.closed.Load() {
return ErrReceiverClosed
}
layer := int32(0)
if w.Kind() == webrtc.RTPCodecTypeVideo && !w.isSVC {
layer = buffer.RidToSpatialLayer(track.RID(), w.trackInfo.Load())
}
buff.SetLogger(w.logger.WithValues("layer", layer))
buff.SetAudioLevelParams(audio.AudioLevelParams{
Config: w.audioConfig.AudioLevelConfig,
})
buff.SetAudioLossProxying(w.audioConfig.EnableLossProxying)
buff.OnRtcpFeedback(w.sendRTCP)
buff.OnRtcpSenderReport(func() {
srData := buff.GetSenderReportData()
w.downTrackSpreader.Broadcast(func(dt TrackSender) {
_ = dt.HandleRTCPSenderReportData(w.codec.PayloadType, w.isSVC, layer, srData)
})
})
if w.Kind() == webrtc.RTPCodecTypeVideo && layer == 0 {
buff.OnCodecChange(w.handleCodecChange)
}
var duration time.Duration
switch layer {
case 2:
duration = w.pliThrottleConfig.HighQuality
case 1:
duration = w.pliThrottleConfig.MidQuality
case 0:
duration = w.pliThrottleConfig.LowQuality
default:
duration = w.pliThrottleConfig.MidQuality
}
if duration != 0 {
buff.SetPLIThrottle(duration.Nanoseconds())
}
w.bufferMu.Lock()
if w.upTracks[layer] != nil {
w.bufferMu.Unlock()
return ErrDuplicateLayer
}
w.upTracks[layer] = track
w.buffers[layer] = buff
rtt := w.rtt
w.bufferMu.Unlock()
buff.SetRTT(rtt)
buff.SetPaused(w.streamTrackerManager.IsPaused())
if w.Kind() == webrtc.RTPCodecTypeVideo && w.useTrackers {
w.streamTrackerManager.AddTracker(layer)
}
go w.forwardRTP(layer, buff)
return nil
}
// SetUpTrackPaused indicates upstream will not be sending any data.
// this will reflect the "muted" status and will pause streamtracker to ensure we don't turn off
// the layer
func (w *WebRTCReceiver) SetUpTrackPaused(paused bool) {
w.streamTrackerManager.SetPaused(paused)
w.bufferMu.RLock()
for _, buff := range w.buffers {
if buff == nil {
continue
}
buff.SetPaused(paused)
}
w.bufferMu.RUnlock()
w.connectionStats.UpdateMute(paused)
}
func (w *WebRTCReceiver) AddDownTrack(track TrackSender) error {
if w.closed.Load() {
return ErrReceiverClosed
}
if w.downTrackSpreader.HasDownTrack(track.SubscriberID()) {
w.logger.Infow("subscriberID already exists, replacing downtrack", "subscriberID", track.SubscriberID())
}
track.UpTrackMaxPublishedLayerChange(w.streamTrackerManager.GetMaxPublishedLayer())
track.UpTrackMaxTemporalLayerSeenChange(w.streamTrackerManager.GetMaxTemporalLayerSeen())
w.downTrackSpreader.Store(track)
w.logger.Debugw("downtrack added", "subscriberID", track.SubscriberID())
return nil
}
func (w *WebRTCReceiver) GetDownTracks() []TrackSender {
return w.downTrackSpreader.GetDownTracks()
}
func (w *WebRTCReceiver) notifyMaxExpectedLayer(layer int32) {
ti := w.TrackInfo()
if ti == nil {
return
}
if w.Kind() == webrtc.RTPCodecTypeAudio || ti.Source == livekit.TrackSource_SCREEN_SHARE {
// screen share tracks have highly variable bitrate, do not use bit rate based quality for those
return
}
expectedBitrate := int64(0)
for _, vl := range ti.Layers {
l := buffer.VideoQualityToSpatialLayer(vl.Quality, ti)
if l <= layer {
expectedBitrate += int64(vl.Bitrate)
}
}
w.connectionStats.AddBitrateTransition(expectedBitrate)
}
func (w *WebRTCReceiver) SetMaxExpectedSpatialLayer(layer int32) {
w.streamTrackerManager.SetMaxExpectedSpatialLayer(layer)
w.notifyMaxExpectedLayer(layer)
if layer == buffer.InvalidLayerSpatial {
w.connectionStats.UpdateLayerMute(true)
} else {
w.connectionStats.UpdateLayerMute(false)
w.connectionStats.AddLayerTransition(w.streamTrackerManager.DistanceToDesired())
}
}
// StreamTrackerManagerListener.OnAvailableLayersChanged
func (w *WebRTCReceiver) OnAvailableLayersChanged() {
w.downTrackSpreader.Broadcast(func(dt TrackSender) {
dt.UpTrackLayersChange()
})
w.connectionStats.AddLayerTransition(w.streamTrackerManager.DistanceToDesired())
}
// StreamTrackerManagerListener.OnBitrateAvailabilityChanged
func (w *WebRTCReceiver) OnBitrateAvailabilityChanged() {
w.downTrackSpreader.Broadcast(func(dt TrackSender) {
dt.UpTrackBitrateAvailabilityChange()
})
}
// StreamTrackerManagerListener.OnMaxPublishedLayerChanged
func (w *WebRTCReceiver) OnMaxPublishedLayerChanged(maxPublishedLayer int32) {
w.downTrackSpreader.Broadcast(func(dt TrackSender) {
dt.UpTrackMaxPublishedLayerChange(maxPublishedLayer)
})
w.notifyMaxExpectedLayer(maxPublishedLayer)
w.connectionStats.AddLayerTransition(w.streamTrackerManager.DistanceToDesired())
}
// StreamTrackerManagerListener.OnMaxTemporalLayerSeenChanged
func (w *WebRTCReceiver) OnMaxTemporalLayerSeenChanged(maxTemporalLayerSeen int32) {
w.downTrackSpreader.Broadcast(func(dt TrackSender) {
dt.UpTrackMaxTemporalLayerSeenChange(maxTemporalLayerSeen)
})
w.connectionStats.AddLayerTransition(w.streamTrackerManager.DistanceToDesired())
}
// StreamTrackerManagerListener.OnMaxAvailableLayerChanged
func (w *WebRTCReceiver) OnMaxAvailableLayerChanged(maxAvailableLayer int32) {
if onMaxLayerChange := w.getOnMaxLayerChange(); onMaxLayerChange != nil {
onMaxLayerChange(maxAvailableLayer)
}
}
// StreamTrackerManagerListener.OnBitrateReport
func (w *WebRTCReceiver) OnBitrateReport(availableLayers []int32, bitrates Bitrates) {
w.downTrackSpreader.Broadcast(func(dt TrackSender) {
dt.UpTrackBitrateReport(availableLayers, bitrates)
})
w.connectionStats.AddLayerTransition(w.streamTrackerManager.DistanceToDesired())
}
func (w *WebRTCReceiver) GetLayeredBitrate() ([]int32, Bitrates) {
return w.streamTrackerManager.GetLayeredBitrate()
}
// OnCloseHandler method to be called on remote tracked removed
func (w *WebRTCReceiver) OnCloseHandler(fn func()) {
w.onCloseHandler = fn
}
// DeleteDownTrack removes a DownTrack from a Receiver
func (w *WebRTCReceiver) DeleteDownTrack(subscriberID livekit.ParticipantID) {
if w.closed.Load() {
return
}
w.downTrackSpreader.Free(subscriberID)
w.logger.Debugw("downtrack deleted", "subscriberID", subscriberID)
}
func (w *WebRTCReceiver) sendRTCP(packets []rtcp.Packet) {
if packets == nil || w.closed.Load() {
return
}
if w.onRTCP != nil {
w.onRTCP(packets)
}
}
func (w *WebRTCReceiver) SendPLI(layer int32, force bool) {
// SVC-TODO : should send LRR (Layer Refresh Request) instead of PLI
buff := w.getBuffer(layer)
if buff == nil {
return
}
buff.SendPLI(force)
}
func (w *WebRTCReceiver) getBuffer(layer int32) *buffer.Buffer {
w.bufferMu.RLock()
defer w.bufferMu.RUnlock()
return w.getBufferLocked(layer)
}
func (w *WebRTCReceiver) getBufferLocked(layer int32) *buffer.Buffer {
// for svc codecs, use layer = 0 always.
// spatial layers are in-built and handled by single buffer
if w.isSVC {
layer = 0
}
if layer < 0 || int(layer) >= len(w.buffers) {
return nil
}
return w.buffers[layer]
}
func (w *WebRTCReceiver) ReadRTP(buf []byte, layer uint8, esn uint64) (int, error) {
b := w.getBuffer(int32(layer))
if b == nil {
return 0, ErrBufferNotFound
}
return b.GetPacket(buf, esn)
}
func (w *WebRTCReceiver) GetTrackStats() *livekit.RTPStats {
w.bufferMu.RLock()
defer w.bufferMu.RUnlock()
stats := make([]*livekit.RTPStats, 0, len(w.buffers))
for _, buff := range w.buffers {
if buff == nil {
continue
}
sswl := buff.GetStats()
if sswl == nil {
continue
}
stats = append(stats, sswl)
}
return rtpstats.AggregateRTPStats(stats)
}
func (w *WebRTCReceiver) GetAudioLevel() (float64, bool) {
if w.Kind() == webrtc.RTPCodecTypeVideo {
return 0, false
}
w.bufferMu.RLock()
defer w.bufferMu.RUnlock()
for _, buff := range w.buffers {
if buff == nil {
continue
}
return buff.GetAudioLevel()
}
return 0, false
}
func (w *WebRTCReceiver) GetDeltaStats() map[uint32]*buffer.StreamStatsWithLayers {
w.bufferMu.RLock()
defer w.bufferMu.RUnlock()
deltaStats := make(map[uint32]*buffer.StreamStatsWithLayers, len(w.buffers))
for layer, buff := range w.buffers {
if buff == nil {
continue
}
sswl := buff.GetDeltaStats()
if sswl == nil {
continue
}
// patch buffer stats with correct layer
patched := make(map[int32]*rtpstats.RTPDeltaInfo, 1)
patched[int32(layer)] = sswl.Layers[0]
sswl.Layers = patched
deltaStats[w.ssrc(layer)] = sswl
}
return deltaStats
}
func (w *WebRTCReceiver) GetLastSenderReportTime() time.Time {
w.bufferMu.RLock()
defer w.bufferMu.RUnlock()
latestSRTime := time.Time{}
for _, buff := range w.buffers {
if buff == nil {
continue
}
srAt := buff.GetLastSenderReportTime()
if srAt.After(latestSRTime) {
latestSRTime = srAt
}
}
return latestSRTime
}
func (w *WebRTCReceiver) forwardRTP(layer int32, buff *buffer.Buffer) {
defer func() {
w.closeOnce.Do(func() {
w.closed.Store(true)
w.closeTracks()
if pr := w.primaryReceiver.Load(); pr != nil {
pr.Close()
}
if pr := w.redReceiver.Load(); pr != nil {
pr.Close()
}
})
w.streamTrackerManager.RemoveTracker(layer)
if w.isSVC {
w.streamTrackerManager.RemoveAllTrackers()
}
}()
var spatialTrackers [buffer.DefaultMaxLayerSpatial + 1]streamtracker.StreamTrackerWorker
if layer < 0 || int(layer) >= len(spatialTrackers) {
w.logger.Errorw("invalid layer", nil, "layer", layer)
return
}
spatialTrackers[layer] = w.streamTrackerManager.GetTracker(layer)
pktBuf := make([]byte, bucket.MaxPktSize)
for {
pkt, err := buff.ReadExtended(pktBuf)
if err == io.EOF {
return
}
if pkt.Packet.PayloadType != uint8(w.codec.PayloadType) {
// drop packets as we don't support codec fallback directly
continue
}
spatialLayer := layer
if pkt.Spatial >= 0 {
// svc packet, take spatial layer info from packet
spatialLayer = pkt.Spatial
}
if int(spatialLayer) >= len(spatialTrackers) {
w.logger.Errorw(
"unexpected spatial layer", nil,
"spatialLayer", spatialLayer,
"pktSpatialLayer", pkt.Spatial,
)
continue
}
writeCount := w.downTrackSpreader.Broadcast(func(dt TrackSender) {
_ = dt.WriteRTP(pkt, spatialLayer)
})
if f := w.redPktWriter.Load(); f != nil {
writeCount += f.(redPktWriteFunc)(pkt, spatialLayer)
}
// track delay/jitter
if writeCount > 0 && w.forwardStats != nil {
w.forwardStats.Update(pkt.Arrival, time.Now().UnixNano())
}
// track video layers
if w.Kind() == webrtc.RTPCodecTypeVideo {
if spatialTrackers[spatialLayer] == nil {
spatialTrackers[spatialLayer] = w.streamTrackerManager.GetTracker(spatialLayer)
if spatialTrackers[spatialLayer] == nil {
spatialTrackers[spatialLayer] = w.streamTrackerManager.AddTracker(spatialLayer)
}
}
if spatialTrackers[spatialLayer] != nil {
spatialTrackers[spatialLayer].Observe(
pkt.Temporal,
len(pkt.RawPacket),
len(pkt.Packet.Payload),
pkt.Packet.Marker,
pkt.Packet.Timestamp,
pkt.DependencyDescriptor,
)
}
}
}
}
// closeTracks close all tracks from Receiver
func (w *WebRTCReceiver) closeTracks() {
w.connectionStats.Close()
w.streamTrackerManager.Close()
closeTrackSenders(w.downTrackSpreader.ResetAndGetDownTracks())
if w.onCloseHandler != nil {
w.onCloseHandler()
}
}
func (w *WebRTCReceiver) DebugInfo() map[string]interface{} {
isSimulcast := !w.isSVC
if ti := w.trackInfo.Load(); ti != nil {
isSimulcast = isSimulcast && len(ti.Layers) > 1
}
info := map[string]interface{}{
"SVC": w.isSVC,
"Simulcast": isSimulcast,
}
w.bufferMu.RLock()
upTrackInfo := make([]map[string]interface{}, 0, len(w.upTracks))
for layer, ut := range w.upTracks {
if ut != nil {
upTrackInfo = append(upTrackInfo, map[string]interface{}{
"Layer": layer,
"SSRC": ut.SSRC(),
"Msid": ut.Msid(),
"RID": ut.RID(),
})
}
}
w.bufferMu.RUnlock()
info["UpTracks"] = upTrackInfo
return info
}
func (w *WebRTCReceiver) GetPrimaryReceiverForRed() TrackReceiver {
if !w.isRED || w.closed.Load() {
return w
}
if w.primaryReceiver.Load() == nil {
pr := NewRedPrimaryReceiver(w, DownTrackSpreaderParams{
Threshold: w.lbThreshold,
Logger: w.logger,
})
if w.primaryReceiver.CompareAndSwap(nil, pr) {
w.redPktWriter.Store(redPktWriteFunc(pr.ForwardRTP))
}
}
return w.primaryReceiver.Load()
}
func (w *WebRTCReceiver) GetRedReceiver() TrackReceiver {
if w.isRED || w.closed.Load() {
return w
}
if w.redReceiver.Load() == nil {
pr := NewRedReceiver(w, DownTrackSpreaderParams{
Threshold: w.lbThreshold,
Logger: w.logger,
})
if w.redReceiver.CompareAndSwap(nil, pr) {
w.redPktWriter.Store(redPktWriteFunc(pr.ForwardRTP))
}
}
return w.redReceiver.Load()
}
func (w *WebRTCReceiver) GetTemporalLayerFpsForSpatial(layer int32) []float32 {
b := w.getBuffer(layer)
if b == nil {
return nil
}
if !w.isSVC {
return b.GetTemporalLayerFpsForSpatial(0)
}
return b.GetTemporalLayerFpsForSpatial(layer)
}
func (w *WebRTCReceiver) AddOnReady(fn func()) {
// webRTCReceiver is always ready after created
fn()
}
func (w *WebRTCReceiver) handleCodecChange(newCodec webrtc.RTPCodecParameters) {
// we don't support the codec fallback directly, set the codec state to invalid once it happens
w.SetCodecState(ReceiverCodecStateInvalid)
}
func (w *WebRTCReceiver) AddOnCodecStateChange(f func(webrtc.RTPCodecParameters, ReceiverCodecState)) {
w.codecStateLock.Lock()
w.onCodecStateChange = append(w.onCodecStateChange, f)
w.codecStateLock.Unlock()
}
func (w *WebRTCReceiver) CodecState() ReceiverCodecState {
w.codecStateLock.Lock()
defer w.codecStateLock.Unlock()
return w.codecState
}
func (w *WebRTCReceiver) SetCodecState(state ReceiverCodecState) {
w.codecStateLock.Lock()
if w.codecState == state || w.codecState == ReceiverCodecStateInvalid {
w.codecStateLock.Unlock()
return
}
w.codecState = state
fns := w.onCodecStateChange
w.codecStateLock.Unlock()
for _, f := range fns {
f(w.codec, state)
}
}
// -----------------------------------------------------------
// closes all track senders in parallel, returns when all are closed
func closeTrackSenders(senders []TrackSender) {
wg := sync.WaitGroup{}
for _, dt := range senders {
dt := dt
wg.Add(1)
go func() {
defer wg.Done()
dt.Close()
}()
}
wg.Wait()
}
livekit-server/pkg/sfu/receiver_test.go
+213
View File
@@ -0,0 +1,213 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package sfu
import (
"fmt"
"hash/fnv"
"math/rand"
"runtime"
"sync"
"testing"
"github.com/gammazero/workerpool"
"github.com/stretchr/testify/assert"
"go.uber.org/atomic"
)
func TestWebRTCReceiver_OnCloseHandler(t *testing.T) {
type args struct {
fn func()
}
tests := []struct {
name string
args args
}{
{
name: "Must set on close handler function",
args: args{
fn: func() {},
},
},
}
for _, tt := range tests {
tt := tt
t.Run(tt.name, func(t *testing.T) {
w := &WebRTCReceiver{}
w.OnCloseHandler(tt.args.fn)
assert.NotNil(t, w.onCloseHandler)
})
}
}
func BenchmarkWriteRTP(b *testing.B) {
cases := []int{1, 2, 5, 10, 100, 250, 500}
workers := runtime.NumCPU()
wp := workerpool.New(workers)
for _, c := range cases {
// fills each bucket with a max of 50, i.e. []int{50, 50} for c=100
fill := make([]int, 0)
for i := 50; ; i += 50 {
if i > c {
fill = append(fill, c%50)
break
}
fill = append(fill, 50)
if i == c {
break
}
}
// splits c into numCPU buckets, i.e. []int{9, 9, 9, 9, 8, 8, 8, 8, 8, 8, 8, 8} for 12 cpus and c=100
split := make([]int, workers)
for i := range split {
split[i] = c / workers
}
for i := 0; i < c%workers; i++ {
split[i]++
}
b.Run(fmt.Sprintf("%d-Downtracks/Control", c), func(b *testing.B) {
benchmarkNoPool(b, c)
})
b.Run(fmt.Sprintf("%d-Downtracks/Pool(Fill)", c), func(b *testing.B) {
benchmarkPool(b, wp, fill)
})
b.Run(fmt.Sprintf("%d-Downtracks/Pool(Hash)", c), func(b *testing.B) {
benchmarkPool(b, wp, split)
})
b.Run(fmt.Sprintf("%d-Downtracks/Goroutines", c), func(b *testing.B) {
benchmarkGoroutine(b, split)
})
b.Run(fmt.Sprintf("%d-Downtracks/LoadBalanced", c), func(b *testing.B) {
benchmarkLoadBalanced(b, workers, 2, c)
})
b.Run(fmt.Sprintf("%d-Downtracks/LBPool", c), func(b *testing.B) {
benchmarkLoadBalancedPool(b, wp, workers, 2, c)
})
}
}
func benchmarkNoPool(b *testing.B, downTracks int) {
for i := 0; i < b.N; i++ {
for dt := 0; dt < downTracks; dt++ {
writeRTP()
}
}
}
func benchmarkPool(b *testing.B, wp *workerpool.WorkerPool, buckets []int) {
for i := 0; i < b.N; i++ {
var wg sync.WaitGroup
for j := range buckets {
downTracks := buckets[j]
if downTracks == 0 {
continue
}
wg.Add(1)
wp.Submit(func() {
defer wg.Done()
for dt := 0; dt < downTracks; dt++ {
writeRTP()
}
})
}
wg.Wait()
}
}
func benchmarkGoroutine(b *testing.B, buckets []int) {
for i := 0; i < b.N; i++ {
var wg sync.WaitGroup
for j := range buckets {
downTracks := buckets[j]
if downTracks == 0 {
continue
}
wg.Add(1)
go func() {
defer wg.Done()
for dt := 0; dt < downTracks; dt++ {
writeRTP()
}
}()
}
wg.Wait()
}
}
func benchmarkLoadBalanced(b *testing.B, numProcs, step, downTracks int) {
for i := 0; i < b.N; i++ {
start := atomic.NewUint64(0)
step := uint64(step)
end := uint64(downTracks)
var wg sync.WaitGroup
wg.Add(numProcs)
for p := 0; p < numProcs; p++ {
go func() {
defer wg.Done()
for {
n := start.Add(step)
if n >= end+step {
return
}
for i := n - step; i < n && i < end; i++ {
writeRTP()
}
}
}()
}
wg.Wait()
}
}
func benchmarkLoadBalancedPool(b *testing.B, wp *workerpool.WorkerPool, numProcs, step, downTracks int) {
for i := 0; i < b.N; i++ {
start := atomic.NewUint64(0)
step := uint64(step)
end := uint64(downTracks)
var wg sync.WaitGroup
wg.Add(numProcs)
for p := 0; p < numProcs; p++ {
wp.Submit(func() {
defer wg.Done()
for {
n := start.Add(step)
if n >= end+step {
return
}
for i := n - step; i < n && i < end; i++ {
writeRTP()
}
}
})
}
wg.Wait()
}
}
func writeRTP() {
s := []byte("simulate some work")
stop := 1900 + rand.Intn(200)
for j := 0; j < stop; j++ {
h := fnv.New128()
s = h.Sum(s)
}
}
livekit-server/pkg/sfu/redprimaryreceiver.go
+326
View File
@@ -0,0 +1,326 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package sfu
import (
"encoding/binary"
"errors"
"go.uber.org/atomic"
"github.com/pion/rtp"
"github.com/livekit/livekit-server/pkg/sfu/buffer"
"github.com/livekit/protocol/livekit"
"github.com/livekit/protocol/logger"
)
var (
ErrIncompleteRedHeader = errors.New("incomplete red block header")
ErrIncompleteRedBlock = errors.New("incomplete red block payload")
)
type RedPrimaryReceiver struct {
TrackReceiver
downTrackSpreader *DownTrackSpreader
logger logger.Logger
closed atomic.Bool
redPT uint8
firstPktReceived bool
lastSeq uint16
// bitset for upstream packet receive history [lastSeq-8, lastSeq-1], bit 1 represents packet received
pktHistory byte
}
func NewRedPrimaryReceiver(receiver TrackReceiver, dsp DownTrackSpreaderParams) *RedPrimaryReceiver {
return &RedPrimaryReceiver{
TrackReceiver: receiver,
downTrackSpreader: NewDownTrackSpreader(dsp),
logger: dsp.Logger,
redPT: uint8(receiver.Codec().PayloadType),
}
}
func (r *RedPrimaryReceiver) ForwardRTP(pkt *buffer.ExtPacket, spatialLayer int32) int {
// extract primary payload from RED and forward to downtracks
if r.downTrackSpreader.DownTrackCount() == 0 {
return 0
}
if pkt.Packet.PayloadType != r.redPT {
// forward non-red packet directly
return r.downTrackSpreader.Broadcast(func(dt TrackSender) {
_ = dt.WriteRTP(pkt, spatialLayer)
})
}
pkts, err := r.getSendPktsFromRed(pkt.Packet)
if err != nil {
r.logger.Errorw("get encoding for red failed", err, "payloadtype", pkt.Packet.PayloadType)
return 0
}
var count int
for i, sendPkt := range pkts {
pPkt := *pkt
if i != len(pkts)-1 {
// patch extended sequence number and time stamp for all but the last packet,
// last packet is the primary payload
pPkt.ExtSequenceNumber -= uint64(pkts[len(pkts)-1].SequenceNumber - pkts[i].SequenceNumber)
pPkt.ExtTimestamp -= uint64(pkts[len(pkts)-1].Timestamp - pkts[i].Timestamp)
}
pPkt.Packet = sendPkt
// not modify the ExtPacket.RawPacket here for performance since it is not used by the DownTrack,
// otherwise it should be set to the correct value (marshal the primary rtp packet)
count += r.downTrackSpreader.Broadcast(func(dt TrackSender) {
_ = dt.WriteRTP(&pPkt, spatialLayer)
})
}
return count
}
func (r *RedPrimaryReceiver) AddDownTrack(track TrackSender) error {
if r.closed.Load() {
return ErrReceiverClosed
}
if r.downTrackSpreader.HasDownTrack(track.SubscriberID()) {
r.logger.Infow("subscriberID already exists, replacing downtrack", "subscriberID", track.SubscriberID())
}
r.downTrackSpreader.Store(track)
r.logger.Debugw("red primary receiver downtrack added", "subscriberID", track.SubscriberID())
return nil
}
func (r *RedPrimaryReceiver) DeleteDownTrack(subscriberID livekit.ParticipantID) {
if r.closed.Load() {
return
}
r.downTrackSpreader.Free(subscriberID)
r.logger.Debugw("red primary receiver downtrack deleted", "subscriberID", subscriberID)
}
func (r *RedPrimaryReceiver) GetDownTracks() []TrackSender {
return r.downTrackSpreader.GetDownTracks()
}
func (r *RedPrimaryReceiver) ResyncDownTracks() {
r.downTrackSpreader.Broadcast(func(dt TrackSender) {
dt.Resync()
})
}
func (r *RedPrimaryReceiver) IsClosed() bool {
return r.closed.Load()
}
func (r *RedPrimaryReceiver) CanClose() bool {
return r.closed.Load() || r.downTrackSpreader.DownTrackCount() == 0
}
func (r *RedPrimaryReceiver) Close() {
r.closed.Store(true)
closeTrackSenders(r.downTrackSpreader.ResetAndGetDownTracks())
}
func (r *RedPrimaryReceiver) ReadRTP(buf []byte, layer uint8, esn uint64) (int, error) {
n, err := r.TrackReceiver.ReadRTP(buf, layer, esn)
if err != nil {
return n, err
}
var pkt rtp.Packet
pkt.Unmarshal(buf[:n])
payload, err := extractPrimaryEncodingForRED(pkt.Payload)
if err != nil {
return 0, err
}
pkt.Payload = payload
return pkt.MarshalTo(buf)
}
func (r *RedPrimaryReceiver) getSendPktsFromRed(rtp *rtp.Packet) ([]*rtp.Packet, error) {
var needRecover bool
if !r.firstPktReceived {
r.lastSeq = rtp.SequenceNumber
r.pktHistory = 0
r.firstPktReceived = true
} else {
diff := rtp.SequenceNumber - r.lastSeq
switch {
case diff == 0: // duplicate
break
case diff > 0x8000: // unorder
// in history
if 65535-diff < 8 {
r.pktHistory |= 1 << (65535 - diff)
needRecover = true
}
case diff > 8: // long jump
r.lastSeq = rtp.SequenceNumber
r.pktHistory = 0
needRecover = true
default:
r.lastSeq = rtp.SequenceNumber
r.pktHistory = (r.pktHistory << byte(diff)) | 1<<(diff-1)
needRecover = true
}
}
var recoverBits byte
if needRecover {
bitIndex := r.lastSeq - rtp.SequenceNumber
for i := 0; i < maxRedCount; i++ {
if bitIndex > 7 {
break
}
if r.pktHistory&byte(1<<bitIndex) == 0 {
recoverBits |= byte(1 << i)
}
bitIndex++
}
}
return extractPktsFromRed(rtp, recoverBits)
}
type block struct {
tsOffset uint32
length int
pt uint8
primary bool
}
func extractPktsFromRed(redPkt *rtp.Packet, recoverBits byte) ([]*rtp.Packet, error) {
payload := redPkt.Payload
var blocks []block
var blockLength int
for {
if len(payload) < 1 {
// illegal data, need at least one byte for primary encoding
return nil, ErrIncompleteRedHeader
}
if payload[0]&0x80 == 0 {
// last block is primary encoding data
pt := uint8(payload[0] & 0x7F)
blocks = append(blocks, block{pt: pt, primary: true})
payload = payload[1:]
break
} else {
if len(payload) < 4 {
// illegal data
return nil, ErrIncompleteRedHeader
}
blockHead := binary.BigEndian.Uint32(payload[0:])
length := int(blockHead & 0x03FF)
blockHead >>= 10
tsOffset := blockHead & 0x3FFF
blockHead >>= 14
pt := uint8(blockHead & 0x7F)
blocks = append(blocks, block{pt: pt, length: length, tsOffset: tsOffset})
blockLength += length
payload = payload[4:]
}
}
if len(payload) < blockLength {
return nil, ErrIncompleteRedBlock
}
pkts := make([]*rtp.Packet, 0, len(blocks))
for i, b := range blocks {
if b.primary {
header := redPkt.Header
header.PayloadType = b.pt
pkts = append(pkts, &rtp.Packet{Header: header, Payload: payload})
break
}
recoverIndex := len(blocks) - i - 1
if recoverIndex < 1 || recoverBits&(1<<(recoverIndex-1)) == 0 {
// skip past packet/block that does not need recovery
payload = payload[b.length:]
continue
}
// recover missing packet
header := redPkt.Header
header.SequenceNumber -= uint16(recoverIndex)
header.Timestamp -= b.tsOffset
header.PayloadType = b.pt
pkts = append(pkts, &rtp.Packet{Header: header, Payload: payload[:b.length]})
payload = payload[b.length:]
}
return pkts, nil
}
func extractPrimaryEncodingForRED(payload []byte) ([]byte, error) {
/* RED payload https://datatracker.ietf.org/doc/html/rfc2198#section-3
0 1 2 3
0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
|F| block PT | timestamp offset | block length |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
F: 1 bit First bit in header indicates whether another header block
follows. If 1 further header blocks follow, if 0 this is the
last header block.
*/
var blockLength int
for {
if len(payload) < 1 {
// illegal data, need at least one byte for primary encoding
return nil, ErrIncompleteRedHeader
}
if payload[0]&0x80 == 0 {
// last block is primary encoding data
payload = payload[1:]
break
} else {
if len(payload) < 4 {
// illegal data
return nil, ErrIncompleteRedHeader
}
blockLength += int(binary.BigEndian.Uint16(payload[2:]) & 0x03FF)
payload = payload[4:]
}
}
if len(payload) < blockLength {
return nil, ErrIncompleteRedBlock
}
return payload[blockLength:], nil
}
livekit-server/pkg/sfu/redreceiver.go
+226
View File
@@ -0,0 +1,226 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package sfu
import (
"encoding/binary"
"fmt"
"go.uber.org/atomic"
"github.com/pion/rtp"
"github.com/livekit/livekit-server/pkg/sfu/buffer"
"github.com/livekit/mediatransportutil/pkg/bucket"
"github.com/livekit/protocol/livekit"
"github.com/livekit/protocol/logger"
)
const (
maxRedCount = 2
mtuSize = 1500
maxRedPayload = 1 << 10 // fit into 10 bits length field
// the RedReceiver is only for chrome / native webrtc now, we always negotiate opus payload to 111 with those clients,
// so it is safe to use a fixed payload 111 here for performance(avoid encoding red blocks for each downtrack that
// have a different opus payload type).
opusPT = 111
opusRedPT = 63
)
type RedReceiver struct {
TrackReceiver
downTrackSpreader *DownTrackSpreader
logger logger.Logger
closed atomic.Bool
pktBuff [maxRedCount]*rtp.Packet
redPayloadBuf [mtuSize]byte
}
func NewRedReceiver(receiver TrackReceiver, dsp DownTrackSpreaderParams) *RedReceiver {
return &RedReceiver{
TrackReceiver: receiver,
downTrackSpreader: NewDownTrackSpreader(dsp),
logger: dsp.Logger,
}
}
func (r *RedReceiver) ForwardRTP(pkt *buffer.ExtPacket, spatialLayer int32) int {
// encode RED payload from primary payload and forward to downtracks
if r.downTrackSpreader.DownTrackCount() == 0 {
return 0
}
// fallback to primary codec if payload size exceeds redundant block length
if len(pkt.Packet.Payload) >= maxRedPayload {
return r.downTrackSpreader.Broadcast(func(dt TrackSender) {
_ = dt.WriteRTP(pkt, spatialLayer)
})
}
redLen, err := r.encodeRedForPrimary(pkt.Packet, r.redPayloadBuf[:])
if err != nil {
r.logger.Errorw("red encoding failed", err)
return 0
}
pPkt := *pkt
redRtpPacket := *pkt.Packet
redRtpPacket.PayloadType = 63
redRtpPacket.Payload = r.redPayloadBuf[:redLen]
pPkt.Packet = &redRtpPacket
// not modify the ExtPacket.RawPacket here for performance since it is not used by the DownTrack,
// otherwise it should be set to the correct value (marshal the primary rtp packet)
return r.downTrackSpreader.Broadcast(func(dt TrackSender) {
_ = dt.WriteRTP(&pPkt, spatialLayer)
})
}
func (r *RedReceiver) AddDownTrack(track TrackSender) error {
if r.closed.Load() {
return ErrReceiverClosed
}
if r.downTrackSpreader.HasDownTrack(track.SubscriberID()) {
r.logger.Infow("subscriberID already exists, replacing downtrack", "subscriberID", track.SubscriberID())
}
r.downTrackSpreader.Store(track)
r.logger.Debugw("red receiver downtrack added", "subscriberID", track.SubscriberID())
return nil
}
func (r *RedReceiver) DeleteDownTrack(subscriberID livekit.ParticipantID) {
if r.closed.Load() {
return
}
r.downTrackSpreader.Free(subscriberID)
r.logger.Debugw("red receiver downtrack deleted", "subscriberID", subscriberID)
}
func (r *RedReceiver) GetDownTracks() []TrackSender {
return r.downTrackSpreader.GetDownTracks()
}
func (r *RedReceiver) ResyncDownTracks() {
r.downTrackSpreader.Broadcast(func(dt TrackSender) {
dt.Resync()
})
}
func (r *RedReceiver) CanClose() bool {
return r.closed.Load() || r.downTrackSpreader.DownTrackCount() == 0
}
func (r *RedReceiver) IsClosed() bool {
return r.closed.Load()
}
func (r *RedReceiver) Close() {
r.closed.Store(true)
closeTrackSenders(r.downTrackSpreader.ResetAndGetDownTracks())
}
func (r *RedReceiver) ReadRTP(buf []byte, layer uint8, esn uint64) (int, error) {
// red encoding doesn't support nack
return 0, bucket.ErrPacketMismatch
}
func (r *RedReceiver) encodeRedForPrimary(pkt *rtp.Packet, redPayload []byte) (int, error) {
redLength := len(r.pktBuff)
redPkts := make([]*rtp.Packet, 0, redLength+1)
lastNilPkt := -1
for i := redLength - 1; i >= 0; i-- {
if r.pktBuff[i] == nil {
lastNilPkt = i
break
}
}
for _, prev := range r.pktBuff[lastNilPkt+1:] {
if pkt.SequenceNumber == prev.SequenceNumber ||
(pkt.SequenceNumber-prev.SequenceNumber) > uint16(redLength) ||
(pkt.Timestamp-prev.Timestamp) >= (1<<14) {
continue
}
redPkts = append(redPkts, prev)
}
// insert primary packet in history buffer
// NOTE: packet is copied from retransmission buffer and used in forwarding path. So, not making another
// copy here and just maintaining pointer to the packet as the forwarding path should not alter the packet.
for i := redLength - 1; i >= 0; i-- {
if r.pktBuff[i] == nil || // history is empty
pkt.SequenceNumber-r.pktBuff[i].SequenceNumber < (1<<15) { // received packet has more recent sequence number
// age out older ones
for j := 0; j < i; j++ {
r.pktBuff[j] = r.pktBuff[j+1]
}
r.pktBuff[i] = pkt
break
}
}
return encodeRedForPrimary(redPkts, pkt, redPayload)
}
func encodeRedForPrimary(redPkts []*rtp.Packet, primary *rtp.Packet, redPayload []byte) (int, error) {
payloadSize := len(primary.Payload) + 1
for _, p := range redPkts {
payloadSize += len(p.Payload) + 4
}
// if required payload size is larger than the redPayload buffer, encode the primary packet only
if payloadSize > len(redPayload) {
redPkts = redPkts[:0]
}
var index int
for _, p := range redPkts {
/* RED payload https://datatracker.ietf.org/doc/html/rfc2198#section-3
0 1 2 3
0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
|F| block PT | timestamp offset | block length |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
F: 1 bit First bit in header indicates whether another header block
follows. If 1 further header blocks follow, if 0 this is the
last header block.
*/
header := uint32(0x80 | uint8(opusPT))
header <<= 14
header |= (primary.Timestamp - p.Timestamp) & 0x3FFF
header <<= 10
header |= uint32(len(p.Payload)) & 0x3FF
binary.BigEndian.PutUint32(redPayload[index:], header)
index += 4
}
// last block header
redPayload[index] = uint8(opusPT)
index++
// append data blocks
redPkts = append(redPkts, primary)
for _, p := range redPkts {
if copy(redPayload[index:], p.Payload) < len(p.Payload) {
return 0, fmt.Errorf("red payload don't have enough space, needsize %d", len(p.Payload))
}
index += len(p.Payload)
}
return index, nil
}
livekit-server/pkg/sfu/redreceiver_test.go
+427
View File
@@ -0,0 +1,427 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package sfu
import (
"testing"
"github.com/pion/rtp"
"github.com/pion/webrtc/v4"
"github.com/stretchr/testify/require"
"github.com/livekit/livekit-server/pkg/sfu/buffer"
"github.com/livekit/protocol/logger"
)
const (
tsStep = uint32(48000 / 1000 * 10)
opusREDPT = 63
)
type dummyDowntrack struct {
TrackSender
lastReceivedPkt *rtp.Packet
receivedPkts []*rtp.Packet
}
func (dt *dummyDowntrack) WriteRTP(p *buffer.ExtPacket, _ int32) error {
dt.lastReceivedPkt = p.Packet
dt.receivedPkts = append(dt.receivedPkts, p.Packet)
return nil
}
func (dt *dummyDowntrack) TrackInfoAvailable() {}
func TestRedReceiver(t *testing.T) {
dt := &dummyDowntrack{TrackSender: &DownTrack{}}
t.Run("normal", func(t *testing.T) {
w := &WebRTCReceiver{
isRED: true,
kind: webrtc.RTPCodecTypeAudio,
logger: logger.GetLogger(),
}
require.Equal(t, w.GetRedReceiver(), w)
w.isRED = false
red := w.GetRedReceiver().(*RedReceiver)
require.NotNil(t, red)
require.NoError(t, red.AddDownTrack(dt))
header := rtp.Header{SequenceNumber: 65534, Timestamp: (uint32(1) << 31) - 2*tsStep, PayloadType: 111}
expectPkt := make([]*rtp.Packet, 0, maxRedCount+1)
for _, pkt := range generatePkts(header, 10, tsStep) {
expectPkt = append(expectPkt, pkt)
if len(expectPkt) > maxRedCount+1 {
expectPkt = expectPkt[1:]
}
red.ForwardRTP(&buffer.ExtPacket{
Packet: pkt,
}, 0)
verifyRedEncodings(t, dt.lastReceivedPkt, expectPkt)
}
})
t.Run("packet lost and jump", func(t *testing.T) {
w := &WebRTCReceiver{
kind: webrtc.RTPCodecTypeAudio,
logger: logger.GetLogger(),
}
red := w.GetRedReceiver().(*RedReceiver)
require.NoError(t, red.AddDownTrack(dt))
header := rtp.Header{SequenceNumber: 65534, Timestamp: (uint32(1) << 31) - 2*tsStep, PayloadType: 111}
expectPkt := make([]*rtp.Packet, 0, maxRedCount+1)
for i := 0; i < 10; i++ {
if i%2 == 0 {
header.SequenceNumber++
header.Timestamp += tsStep
expectPkt = append(expectPkt, nil)
continue
}
hbuf, _ := header.Marshal()
pkt1 := &rtp.Packet{
Header: header,
Payload: hbuf,
}
expectPkt = append(expectPkt, pkt1)
if len(expectPkt) > maxRedCount+1 {
expectPkt = expectPkt[len(expectPkt)-maxRedCount-1:]
}
red.ForwardRTP(&buffer.ExtPacket{
Packet: pkt1,
}, 0)
verifyRedEncodings(t, dt.lastReceivedPkt, expectPkt)
header.SequenceNumber++
header.Timestamp += tsStep
}
// jump
header.SequenceNumber += 10
header.Timestamp += 10 * tsStep
expectPkt = expectPkt[:0]
for _, pkt := range generatePkts(header, 3, tsStep) {
expectPkt = append(expectPkt, pkt)
if len(expectPkt) > maxRedCount+1 {
expectPkt = expectPkt[len(expectPkt)-maxRedCount-1:]
}
red.ForwardRTP(&buffer.ExtPacket{
Packet: pkt,
}, 0)
verifyRedEncodings(t, dt.lastReceivedPkt, expectPkt)
}
})
t.Run("unorder and repeat", func(t *testing.T) {
w := &WebRTCReceiver{
kind: webrtc.RTPCodecTypeAudio,
logger: logger.GetLogger(),
}
red := w.GetRedReceiver().(*RedReceiver)
require.NoError(t, red.AddDownTrack(dt))
header := rtp.Header{SequenceNumber: 65534, Timestamp: (uint32(1) << 31) - 2*tsStep, PayloadType: 111}
var prevPkts []*rtp.Packet
for _, pkt := range generatePkts(header, 10, tsStep) {
red.ForwardRTP(&buffer.ExtPacket{
Packet: pkt,
}, 0)
prevPkts = append(prevPkts, pkt)
}
// old unorder data don't have red records
expectPkt := prevPkts[len(prevPkts)-3 : len(prevPkts)-2]
red.ForwardRTP(&buffer.ExtPacket{
Packet: expectPkt[0],
}, 0)
verifyRedEncodings(t, dt.lastReceivedPkt, expectPkt)
// repeat packet only have 1 red records
expectPkt = prevPkts[len(prevPkts)-2:]
red.ForwardRTP(&buffer.ExtPacket{
Packet: expectPkt[1],
}, 0)
verifyRedEncodings(t, dt.lastReceivedPkt, expectPkt)
})
t.Run("encoding exceed space", func(t *testing.T) {
w := &WebRTCReceiver{
isRED: true,
kind: webrtc.RTPCodecTypeAudio,
logger: logger.GetLogger(),
}
require.Equal(t, w.GetRedReceiver(), w)
w.isRED = false
red := w.GetRedReceiver().(*RedReceiver)
require.NotNil(t, red)
require.NoError(t, red.AddDownTrack(dt))
header := rtp.Header{SequenceNumber: 65534, Timestamp: (uint32(1) << 31) - 2*tsStep, PayloadType: 111}
expectPkt := make([]*rtp.Packet, 0, maxRedCount+1)
for _, pkt := range generatePkts(header, 10, tsStep) {
// make sure red encodings don't have enough space to encoding redundant packet
pkt.Payload = make([]byte, 1000)
expectPkt = append(expectPkt[:0], pkt)
red.ForwardRTP(&buffer.ExtPacket{
Packet: pkt,
}, 0)
verifyRedEncodings(t, dt.lastReceivedPkt, expectPkt)
}
})
t.Run("large timestamp gap", func(t *testing.T) {
w := &WebRTCReceiver{
isRED: true,
kind: webrtc.RTPCodecTypeAudio,
logger: logger.GetLogger(),
}
require.Equal(t, w.GetRedReceiver(), w)
w.isRED = false
red := w.GetRedReceiver().(*RedReceiver)
require.NotNil(t, red)
require.NoError(t, red.AddDownTrack(dt))
header := rtp.Header{SequenceNumber: 65534, Timestamp: (uint32(1) << 31) - 2*tsStep, PayloadType: 111}
// first few packets normal
expectPkt := make([]*rtp.Packet, 0, maxRedCount+1)
for _, pkt := range generatePkts(header, 4, tsStep) {
expectPkt = append(expectPkt, pkt)
if len(expectPkt) > maxRedCount+1 {
expectPkt = expectPkt[1:]
}
red.ForwardRTP(&buffer.ExtPacket{
Packet: pkt,
}, 0)
verifyRedEncodings(t, dt.lastReceivedPkt, expectPkt)
}
// and then a few packets with a large timestamp jump, should contain only primary
for _, pkt := range generatePkts(header, 4, 40*tsStep) {
red.ForwardRTP(&buffer.ExtPacket{
Packet: pkt,
}, 0)
verifyRedEncodings(t, dt.lastReceivedPkt, []*rtp.Packet{pkt})
}
})
}
func verifyRedEncodings(t *testing.T, red *rtp.Packet, redPkts []*rtp.Packet) {
solidPkts := make([]*rtp.Packet, 0, len(redPkts))
for _, pkt := range redPkts {
if pkt != nil {
solidPkts = append(solidPkts, pkt)
}
}
pktsFromRed, err := extractPktsFromRed(red, 0xFF)
require.NoError(t, err)
require.Len(t, pktsFromRed, len(solidPkts))
for i, pkt := range pktsFromRed {
verifyEncodingEqual(t, pkt, solidPkts[i])
}
}
func verifyPktsEqual(t *testing.T, p1s, p2s []*rtp.Packet) {
require.Len(t, p1s, len(p2s))
for i, pkt := range p1s {
verifyEncodingEqual(t, pkt, p2s[i])
}
}
func verifyEncodingEqual(t *testing.T, p1, p2 *rtp.Packet) {
require.Equal(t, p1.Header.Timestamp, p2.Header.Timestamp)
require.Equal(t, p1.PayloadType, p2.PayloadType)
require.EqualValues(t, p1.Payload, p2.Payload, "seq1 %s", p1.SequenceNumber)
}
func generatePkts(header rtp.Header, count int, tsStep uint32) []*rtp.Packet {
pkts := make([]*rtp.Packet, 0, count)
for i := 0; i < count; i++ {
hbuf, _ := header.Marshal()
pkts = append(pkts, &rtp.Packet{
Header: header,
Payload: hbuf,
})
header.SequenceNumber++
header.Timestamp += tsStep
}
return pkts
}
func generateRedPkts(t *testing.T, pkts []*rtp.Packet, redCount int) []*rtp.Packet {
redPkts := make([]*rtp.Packet, 0, len(pkts))
for i, pkt := range pkts {
encodingPkts := make([]*rtp.Packet, 0, redCount)
for j := i - redCount; j < i; j++ {
if j < 0 {
continue
}
encodingPkts = append(encodingPkts, pkts[j])
}
buf := make([]byte, mtuSize)
redPkt := *pkt
redPkt.PayloadType = opusREDPT
encoded, err := encodeRedForPrimary(encodingPkts, pkt, buf)
require.NoError(t, err)
redPkt.Payload = buf[:encoded]
redPkts = append(redPkts, &redPkt)
}
return redPkts
}
func testRedRedPrimaryReceiver(t *testing.T, maxPktCount, redCount int, sendPktIdx, expectPktIdx []int) {
dt := &dummyDowntrack{TrackSender: &DownTrack{}}
w := &WebRTCReceiver{
kind: webrtc.RTPCodecTypeAudio,
logger: logger.GetLogger(),
codec: webrtc.RTPCodecParameters{PayloadType: opusREDPT, RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: "audio/red"}},
}
require.Equal(t, w.GetPrimaryReceiverForRed(), w)
w.isRED = true
red := w.GetPrimaryReceiverForRed().(*RedPrimaryReceiver)
require.NotNil(t, red)
require.NoError(t, red.AddDownTrack(dt))
header := rtp.Header{SequenceNumber: 65530, Timestamp: (uint32(1) << 31) - 2*tsStep, PayloadType: 111}
primaryPkts := generatePkts(header, maxPktCount, tsStep)
redPkts := generateRedPkts(t, primaryPkts, redCount)
for _, i := range sendPktIdx {
red.ForwardRTP(&buffer.ExtPacket{
Packet: redPkts[i],
}, 0)
}
expectPkts := make([]*rtp.Packet, 0, len(expectPktIdx))
for _, i := range expectPktIdx {
expectPkts = append(expectPkts, primaryPkts[i])
}
verifyPktsEqual(t, expectPkts, dt.receivedPkts)
}
func TestRedPrimaryReceiver(t *testing.T) {
w := &WebRTCReceiver{
kind: webrtc.RTPCodecTypeAudio,
logger: logger.GetLogger(),
}
require.Equal(t, w.GetPrimaryReceiverForRed(), w)
w.isRED = true
red := w.GetPrimaryReceiverForRed().(*RedPrimaryReceiver)
require.NotNil(t, red)
t.Run("packet should send only once", func(t *testing.T) {
maxPktCount := 19
var sendPktIndex []int
for i := 0; i < maxPktCount; i++ {
sendPktIndex = append(sendPktIndex, i)
}
testRedRedPrimaryReceiver(t, maxPktCount, maxRedCount, sendPktIndex, sendPktIndex)
})
t.Run("packet duplicate and unorder", func(t *testing.T) {
maxPktCount := 19
var sendPktIndex []int
for i := 0; i < maxPktCount; i++ {
sendPktIndex = append(sendPktIndex, i)
if i > 0 {
sendPktIndex = append(sendPktIndex, i-1)
}
sendPktIndex = append(sendPktIndex, i)
}
testRedRedPrimaryReceiver(t, maxPktCount, maxRedCount, sendPktIndex, sendPktIndex)
})
t.Run("full recover", func(t *testing.T) {
maxPktCount := 19
var sendPktIndex, recvPktIndex []int
for i := 0; i < maxPktCount; i++ {
recvPktIndex = append(recvPktIndex, i)
// drop packets covered by red encoding
if i%(maxRedCount+1) != 0 {
continue
}
sendPktIndex = append(sendPktIndex, i)
}
testRedRedPrimaryReceiver(t, maxPktCount, maxRedCount, sendPktIndex, recvPktIndex)
})
t.Run("lost 2 but red recover 1", func(t *testing.T) {
maxPktCount := 19
sendPktIndex := []int{0, 3, 6, 9, 12}
recvPktIndex := []int{0, 2, 3, 5, 6, 8, 9, 11, 12}
testRedRedPrimaryReceiver(t, maxPktCount, 1, sendPktIndex, recvPktIndex)
})
t.Run("part recover and long jump", func(t *testing.T) {
maxPktCount := 50
sendPktIndex := []int{0, 5, 12, 21 /*long jump*/, 24, 27}
recvPktIndex := []int{0, 3, 4, 5, 10, 11, 12, 19, 20, 21, 22, 23, 24, 25, 26, 27}
testRedRedPrimaryReceiver(t, maxPktCount, maxRedCount, sendPktIndex, recvPktIndex)
})
t.Run("unorder", func(t *testing.T) {
maxPktCount := 50
sendPktIndex := []int{20, 10 /*unorder can't recover*/, 25, 23, 34}
recvPktIndex := []int{20, 10, 23, 24, 25, 21, 22, 23, 32, 33, 34}
testRedRedPrimaryReceiver(t, maxPktCount, maxRedCount, sendPktIndex, recvPktIndex)
})
t.Run("mixed primary codec", func(t *testing.T) {
dt := &dummyDowntrack{TrackSender: &DownTrack{}}
w := &WebRTCReceiver{
kind: webrtc.RTPCodecTypeAudio,
logger: logger.GetLogger(),
codec: webrtc.RTPCodecParameters{PayloadType: opusREDPT, RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: "audio/red"}},
}
require.Equal(t, w.GetPrimaryReceiverForRed(), w)
w.isRED = true
red := w.GetPrimaryReceiverForRed().(*RedPrimaryReceiver)
require.NotNil(t, red)
require.NoError(t, red.AddDownTrack(dt))
primaryPkt := &rtp.Packet{
Header: rtp.Header{SequenceNumber: 65530, Timestamp: (uint32(1) << 31) - 2*tsStep, PayloadType: 111},
Payload: []byte{1, 3, 5, 7, 9},
}
red.ForwardRTP(&buffer.ExtPacket{
Packet: primaryPkt,
}, 0)
verifyPktsEqual(t, []*rtp.Packet{primaryPkt}, dt.receivedPkts)
})
}
func TestExtractPrimaryEncodingForRED(t *testing.T) {
header := rtp.Header{SequenceNumber: 65530, Timestamp: (uint32(1) << 31) - 2*tsStep, PayloadType: 111}
pkts := generatePkts(header, 10, tsStep)
redPkts := generateRedPkts(t, pkts, 2)
primaryPkts := make([]*rtp.Packet, 0, len(redPkts))
for _, redPkt := range redPkts {
payload, err := extractPrimaryEncodingForRED(redPkt.Payload)
require.NoError(t, err)
primaryHeader := redPkt.Header
primaryHeader.PayloadType = 111
primaryPkts = append(primaryPkts, &rtp.Packet{
Header: primaryHeader,
Payload: payload,
})
}
verifyPktsEqual(t, pkts, primaryPkts)
}
livekit-server/pkg/sfu/rtpextension/abscapturetime/abscapturetime.go
+114
View File
@@ -0,0 +1,114 @@
// Copyright 2024 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package abscapturetime
import (
"encoding/binary"
"errors"
"time"
"github.com/livekit/mediatransportutil"
)
const (
AbsCaptureTimeURI = "http://www.webrtc.org/experiments/rtp-hdrext/abs-capture-time"
)
var (
errInvalidData = errors.New("invalid data")
errTooSmall = errors.New("buffer too small")
)
// Reference: https://webrtc.googlesource.com/src/+/refs/heads/main/docs/native-code/rtp-hdrext/abs-capture-time/
//
// Data layout of the shortened version of abs-capture-time with a 1-byte header + 8 bytes of data:
//
// 0 1 2 3
// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// | ID | len=7 | absolute capture timestamp (bit 0-23) |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// | absolute capture timestamp (bit 24-55) |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// | ... (56-63) |
// +-+-+-+-+-+-+-+-+
//
//Data layout of the extended version of abs-capture-time with a 1-byte header + 16 bytes of data:
//
// 0 1 2 3
// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// | ID | len=15| absolute capture timestamp (bit 0-23) |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// | absolute capture timestamp (bit 24-55) |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// | ... (56-63) | estimated capture clock offset (bit 0-23) |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// | estimated capture clock offset (bit 24-55) |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// | ... (56-63) |
// +-+-+-+-+-+-+-+-+
type AbsCaptureTime struct {
absoluteCaptureTimestamp mediatransportutil.NtpTime
estimatedCaptureClockOffset int64
}
func AbsCaptureTimeFromValue(absoluteCaptureTimestamp uint64, estimatedCaptureClockOffset int64) *AbsCaptureTime {
return &AbsCaptureTime{
absoluteCaptureTimestamp: mediatransportutil.NtpTime(absoluteCaptureTimestamp),
estimatedCaptureClockOffset: estimatedCaptureClockOffset,
}
}
func (a *AbsCaptureTime) Rewrite(offset time.Duration) error {
if a.absoluteCaptureTimestamp == 0 {
return errInvalidData
}
capturedAt := a.absoluteCaptureTimestamp.Time().Add(offset)
a.absoluteCaptureTimestamp = mediatransportutil.ToNtpTime(capturedAt)
a.estimatedCaptureClockOffset = 0
return nil
}
func (a *AbsCaptureTime) Marshal() ([]byte, error) {
if a.absoluteCaptureTimestamp == 0 {
return nil, errInvalidData
}
size := 8
if a.estimatedCaptureClockOffset != 0 {
size += 8
}
marshalled := make([]byte, size)
binary.BigEndian.PutUint64(marshalled, uint64(a.absoluteCaptureTimestamp))
if a.estimatedCaptureClockOffset != 0 {
binary.BigEndian.PutUint64(marshalled[8:], uint64(a.estimatedCaptureClockOffset))
}
return marshalled, nil
}
func (a *AbsCaptureTime) Unmarshal(marshalled []byte) error {
if len(marshalled) < 8 {
return errTooSmall
}
a.absoluteCaptureTimestamp = mediatransportutil.NtpTime(binary.BigEndian.Uint64(marshalled))
if len(marshalled) >= 16 {
a.estimatedCaptureClockOffset = int64(binary.BigEndian.Uint64(marshalled[8:]))
}
return nil
}
livekit-server/pkg/sfu/rtpextension/dependencydescriptor/bitstreamreader.go
+138
View File
@@ -0,0 +1,138 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dependencydescriptor
import (
"errors"
"io"
)
type BitStreamReader struct {
buf []byte
pos int
remainingBits int
}
func NewBitStreamReader(buf []byte) *BitStreamReader {
return &BitStreamReader{buf: buf, remainingBits: len(buf) * 8}
}
func (b *BitStreamReader) RemainingBits() int {
return b.remainingBits
}
// Reads `bits` from the bitstream. `bits` must be in range [0, 64].
// Returns an unsigned integer in range [0, 2^bits - 1].
// On failure sets `BitstreamReader` into the failure state and returns 0.
func (b *BitStreamReader) ReadBits(bits int) (uint64, error) {
if bits < 0 || bits > 64 {
return 0, errors.New("invalid number of bits, expected 0-64")
}
if b.remainingBits < bits {
b.remainingBits -= bits
return 0, io.EOF
}
remainingBitsInFirstByte := b.remainingBits % 8
b.remainingBits -= bits
if bits < remainingBitsInFirstByte {
// Reading fewer bits than what's left in the current byte, just
// return the portion of this byte that is needed.
offset := remainingBitsInFirstByte - bits
return uint64((b.buf[b.pos] >> offset) & ((1 << bits) - 1)), nil
}
var result uint64
if remainingBitsInFirstByte > 0 {
// Read all bits that were left in the current byte and consume that byte.
bits -= remainingBitsInFirstByte
mask := byte((1 << remainingBitsInFirstByte) - 1)
result = uint64(b.buf[b.pos]&mask) << bits
b.pos++
}
// Read as many full bytes as we can.
for bits >= 8 {
bits -= 8
result |= uint64(b.buf[b.pos]) << bits
b.pos++
}
// Whatever is left to read is smaller than a byte, so grab just the needed
// bits and shift them into the lowest bits.
if bits > 0 {
result |= uint64(b.buf[b.pos] >> (8 - bits))
}
return result, nil
}
func (b *BitStreamReader) ReadBool() (bool, error) {
val, err := b.ReadBits(1)
return val != 0, err
}
func (b *BitStreamReader) Ok() bool {
return b.remainingBits >= 0
}
func (b *BitStreamReader) Invalidate() {
b.remainingBits = -1
}
// Reads value in range [0, `num_values` - 1].
// This encoding is similar to ReadBits(val, Ceil(Log2(num_values)),
// but reduces wastage incurred when encoding non-power of two value ranges
// Non symmetric values are encoded as:
// 1) n = bit_width(num_values)
// 2) k = (1 << n) - num_values
// Value v in range [0, k - 1] is encoded in (n-1) bits.
// Value v in range [k, num_values - 1] is encoded as (v+k) in n bits.
// https://aomediacodec.github.io/av1-spec/#nsn
func (b *BitStreamReader) ReadNonSymmetric(numValues uint32) (uint32, error) {
if numValues >= (uint32(1) << 31) {
return 0, errors.New("invalid number of values, expected 0-2^31")
}
width := bitwidth(numValues)
numMinBitsValues := (uint32(1) << width) - numValues
val, err := b.ReadBits(width - 1)
if err != nil {
return 0, err
}
if val < uint64(numMinBitsValues) {
return uint32(val), nil
}
bit, err := b.ReadBits(1)
if err != nil {
return 0, err
}
return uint32((val << 1) + bit - uint64(numMinBitsValues)), nil
}
func (b *BitStreamReader) BytesRead() int {
if b.remainingBits%8 > 0 {
return b.pos + 1
}
return b.pos
}
func bitwidth(n uint32) int {
var w int
for n != 0 {
n >>= 1
w++
}
return w
}
livekit-server/pkg/sfu/rtpextension/dependencydescriptor/bitstreamwriter.go
+132
View File
@@ -0,0 +1,132 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dependencydescriptor
import (
"errors"
"fmt"
)
type BitStreamWriter struct {
buf []byte
pos int
bitOffset int // bit offset in the current byte
}
func NewBitStreamWriter(buf []byte) *BitStreamWriter {
return &BitStreamWriter{buf: buf}
}
func (w *BitStreamWriter) RemainingBits() int {
return (len(w.buf)-w.pos)*8 - w.bitOffset
}
func (w *BitStreamWriter) WriteBits(val uint64, bitCount int) error {
if bitCount > w.RemainingBits() {
return errors.New("insufficient space")
}
totalBits := bitCount
// push bits to the highest bits of uint64
val <<= 64 - bitCount
buf := w.buf[w.pos:]
// The first byte is relatively special; the bit offset to write to may put us
// in the middle of the byte, and the total bit count to write may require we
// save the bits at the end of the byte.
remainingBitsInCurrentByte := 8 - w.bitOffset
bitsInFirstByte := bitCount
if bitsInFirstByte > remainingBitsInCurrentByte {
bitsInFirstByte = remainingBitsInCurrentByte
}
buf[0] = w.writePartialByte(uint8(val>>56), bitsInFirstByte, buf[0], w.bitOffset)
if bitCount <= remainingBitsInCurrentByte {
// no bit left to write
return w.consumeBits(totalBits)
}
// write the rest of the bits
val <<= bitsInFirstByte
buf = buf[1:]
bitCount -= bitsInFirstByte
for bitCount >= 8 {
buf[0] = uint8(val >> 56)
buf = buf[1:]
val <<= 8
bitCount -= 8
}
// write the last bits
if bitCount > 0 {
buf[0] = w.writePartialByte(uint8(val>>56), bitCount, buf[0], 0)
}
return w.consumeBits(totalBits)
}
func (w *BitStreamWriter) consumeBits(bitCount int) error {
if bitCount > w.RemainingBits() {
return errors.New("insufficient space")
}
w.pos += (w.bitOffset + bitCount) / 8
w.bitOffset = (w.bitOffset + bitCount) % 8
return nil
}
func (w *BitStreamWriter) writePartialByte(source uint8, sourceBitCount int, target uint8, targetBitOffset int) uint8 {
// if !(targetBitOffset < 8 && sourceBitCount <= (8-targetBitOffset)) {
// return fmt.Errorf("invalid argument, source %d, sourceBitCount %d, target %d, targetBitOffset %d", source, sourceBitCount, target, targetBitOffset)
// }
// generate mask for bits to overwrite, shift source bits to highest bits, then position to target bit offset
mask := uint8(0xff<<(8-sourceBitCount)) >> uint8(targetBitOffset)
// clear target bits and write source bits
return (target & ^mask) | (source >> targetBitOffset)
}
func (w *BitStreamWriter) WriteNonSymmetric(val, numValues uint32) error {
if !(val < numValues && numValues <= 1<<31) {
return fmt.Errorf("invalid argument, val %d, numValues %d", val, numValues)
}
if numValues == 1 {
// When there is only one possible value, it requires zero bits to store it.
// But WriteBits doesn't support writing zero bits.
return nil
}
countBits := bitwidth(numValues)
numMinBitsValues := (uint32(1) << countBits) - numValues
if val < numMinBitsValues {
return w.WriteBits(uint64(val), countBits-1)
} else {
return w.WriteBits(uint64(val+numMinBitsValues), countBits)
}
}
func SizeNonSymmetricBits(val, numValues uint32) int {
countBits := bitwidth(numValues)
numMinBitsValues := (uint32(1) << countBits) - numValues
if val < numMinBitsValues {
return countBits - 1
} else {
return countBits
}
}
livekit-server/pkg/sfu/rtpextension/dependencydescriptor/dependencydescriptorextension.go
+199
View File
@@ -0,0 +1,199 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dependencydescriptor
import (
"fmt"
"math"
"strconv"
)
// DependencyDescriptorExtension is a extension payload format in
// https://aomediacodec.github.io/av1-rtp-spec/#dependency-descriptor-rtp-header-extension
func formatBitmask(b *uint32) string {
if b == nil {
return "-"
}
return strconv.FormatInt(int64(*b), 2)
}
// ------------------------------------------------------------------------------
type DependencyDescriptorExtension struct {
Descriptor *DependencyDescriptor
Structure *FrameDependencyStructure
}
func (d *DependencyDescriptorExtension) Marshal() ([]byte, error) {
return d.MarshalWithActiveChains(^uint32(0))
}
func (d *DependencyDescriptorExtension) MarshalWithActiveChains(activeChains uint32) ([]byte, error) {
writer, err := NewDependencyDescriptorWriter(nil, d.Structure, activeChains, d.Descriptor)
if err != nil {
return nil, err
}
buf := make([]byte, int(math.Ceil(float64(writer.ValueSizeBits())/8)))
writer.ResetBuf(buf)
if err = writer.Write(); err != nil {
return nil, err
}
return buf, nil
}
func (d *DependencyDescriptorExtension) Unmarshal(buf []byte) (int, error) {
reader := NewDependencyDescriptorReader(buf, d.Structure, d.Descriptor)
return reader.Parse()
}
// ------------------------------------------------------------------------------
const (
MaxSpatialIds = 4
MaxTemporalIds = 8
MaxDecodeTargets = 32
MaxTemplates = 64
AllChainsAreActive = uint32(0)
ExtensionURI = "https://aomediacodec.github.io/av1-rtp-spec/#dependency-descriptor-rtp-header-extension"
)
// ------------------------------------------------------------------------------
type DependencyDescriptor struct {
FirstPacketInFrame bool
LastPacketInFrame bool
FrameNumber uint16
FrameDependencies *FrameDependencyTemplate
Resolution *RenderResolution
ActiveDecodeTargetsBitmask *uint32
AttachedStructure *FrameDependencyStructure
}
func (d *DependencyDescriptor) MarshalSize() (int, error) {
return d.MarshalSizeWithActiveChains(^uint32(0))
}
func (d *DependencyDescriptor) MarshalSizeWithActiveChains(activeChains uint32) (int, error) {
writer, err := NewDependencyDescriptorWriter(nil, d.AttachedStructure, activeChains, d)
if err != nil {
return 0, err
}
return int(math.Ceil(float64(writer.ValueSizeBits()) / 8)), nil
}
func (d *DependencyDescriptor) String() string {
resolution, dependencies := "-", "-"
if d.Resolution != nil {
resolution = fmt.Sprintf("%+v", *d.Resolution)
}
if d.FrameDependencies != nil {
dependencies = fmt.Sprintf("%+v", *d.FrameDependencies)
}
return fmt.Sprintf("DependencyDescriptor{FirstPacketInFrame: %v, LastPacketInFrame: %v, FrameNumber: %v, FrameDependencies: %s, Resolution: %s, ActiveDecodeTargetsBitmask: %v, AttachedStructure: %v}",
d.FirstPacketInFrame, d.LastPacketInFrame, d.FrameNumber, dependencies, resolution, formatBitmask(d.ActiveDecodeTargetsBitmask), d.AttachedStructure)
}
// ------------------------------------------------------------------------------
// Relationship of a frame to a Decode target.
type DecodeTargetIndication int
const (
DecodeTargetNotPresent DecodeTargetIndication = iota // DecodeTargetInfo symbol '-'
DecodeTargetDiscardable // DecodeTargetInfo symbol 'D'
DecodeTargetSwitch // DecodeTargetInfo symbol 'S'
DecodeTargetRequired // DecodeTargetInfo symbol 'R'
)
func (i DecodeTargetIndication) String() string {
switch i {
case DecodeTargetNotPresent:
return "-"
case DecodeTargetDiscardable:
return "D"
case DecodeTargetSwitch:
return "S"
case DecodeTargetRequired:
return "R"
default:
return "Unknown"
}
}
// ------------------------------------------------------------------------------
type FrameDependencyTemplate struct {
SpatialId int
TemporalId int
DecodeTargetIndications []DecodeTargetIndication
FrameDiffs []int
ChainDiffs []int
}
func (t *FrameDependencyTemplate) Clone() *FrameDependencyTemplate {
t2 := &FrameDependencyTemplate{
SpatialId: t.SpatialId,
TemporalId: t.TemporalId,
}
t2.DecodeTargetIndications = make([]DecodeTargetIndication, len(t.DecodeTargetIndications))
copy(t2.DecodeTargetIndications, t.DecodeTargetIndications)
t2.FrameDiffs = make([]int, len(t.FrameDiffs))
copy(t2.FrameDiffs, t.FrameDiffs)
t2.ChainDiffs = make([]int, len(t.ChainDiffs))
copy(t2.ChainDiffs, t.ChainDiffs)
return t2
}
// ------------------------------------------------------------------------------
type FrameDependencyStructure struct {
StructureId int
NumDecodeTargets int
NumChains int
// If chains are used (num_chains > 0), maps decode target index into index of
// the chain protecting that target.
DecodeTargetProtectedByChain []int
Resolutions []RenderResolution
Templates []*FrameDependencyTemplate
}
func (f *FrameDependencyStructure) String() string {
str := fmt.Sprintf("FrameDependencyStructure{StructureId: %v, NumDecodeTargets: %v, NumChains: %v, DecodeTargetProtectedByChain: %v, Resolutions: %+v, Templates: [",
f.StructureId, f.NumDecodeTargets, f.NumChains, f.DecodeTargetProtectedByChain, f.Resolutions)
// templates
for _, t := range f.Templates {
str += fmt.Sprintf("%+v, ", t)
}
str += "]}"
return str
}
// ------------------------------------------------------------------------------
type RenderResolution struct {
Width int
Height int
}
// ------------------------------------------------------------------------------
livekit-server/pkg/sfu/rtpextension/dependencydescriptor/dependencydescriptorextension_test.go
+65
View File
@@ -0,0 +1,65 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dependencydescriptor
import (
"encoding/hex"
"testing"
)
func TestDependencyDescriptorUnmarshal(t *testing.T) {
// hex bytes from traffic capture
hexes := []string{
"c1017280081485214eafffaaaa863cf0430c10c302afc0aaa0063c00430010c002a000a80006000040001d954926e082b04a0941b820ac1282503157f974000ca864330e222222eca8655304224230eca877530077004200ef008601df010d",
"86017340fc",
"46017340fc",
"c3017540fc",
"88017640fc",
"48017640fc",
"c2017840fc",
//
"c1017280081485214eafffaaaa863cf0430c10c302afc0aaa0063c00430010c002a000a80006000040001d954926e082b04a0941b820ac1282503157f974000ca864330e222222eca8655304224230eca877530077004200ef008601df010d",
"860173",
"460173",
"8b0174",
"0b0174",
"0b0174",
"c30175",
}
var structure *FrameDependencyStructure
for _, h := range hexes {
buf, err := hex.DecodeString(h)
if err != nil {
t.Fatal(err)
}
var ddVal DependencyDescriptor
var d = DependencyDescriptorExtension{
Structure: structure,
Descriptor: &ddVal,
}
if _, err := d.Unmarshal(buf); err != nil {
t.Fatal(err)
}
if ddVal.AttachedStructure != nil {
structure = ddVal.AttachedStructure
}
t.Log(ddVal.String())
}
}
livekit-server/pkg/sfu/rtpextension/dependencydescriptor/dependencydescriptorreader.go
+446
View File
@@ -0,0 +1,446 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dependencydescriptor
import (
"errors"
)
var (
ErrDDReaderNoStructure = errors.New("DependencyDescriptorReader: Structure is nil")
ErrDDReaderTemplateWithoutStructure = errors.New("DependencyDescriptorReader: has templateDependencyStructurePresentFlag but AttachedStructure is nil")
ErrDDReaderTooManyTemplates = errors.New("DependencyDescriptorReader: too many templates")
ErrDDReaderTooManyTemporalLayers = errors.New("DependencyDescriptorReader: too many temporal layers")
ErrDDReaderTooManySpatialLayers = errors.New("DependencyDescriptorReader: too many spatial layers")
ErrDDReaderInvalidTemplateIndex = errors.New("DependencyDescriptorReader: invalid template index")
ErrDDReaderInvalidSpatialLayer = errors.New("DependencyDescriptorReader: invalid spatial layer, should be less than the number of resolutions")
ErrDDReaderNumDTIMismatch = errors.New("DependencyDescriptorReader: decode target indications length mismatch with structure num decode targets")
ErrDDReaderNumChainDiffsMismatch = errors.New("DependencyDescriptorReader: chain diffs length mismatch with structure num chains")
)
type DependencyDescriptorReader struct {
// Output.
descriptor *DependencyDescriptor
// Values that are needed while reading the descriptor, but can be discarded
// when reading is complete.
buffer *BitStreamReader
frameDependencyTemplateId int
activeDecodeTargetsPresentFlag bool
customDtisFlag bool
customFdiffsFlag bool
customChainsFlag bool
structure *FrameDependencyStructure
}
func NewDependencyDescriptorReader(buf []byte, structure *FrameDependencyStructure, descriptor *DependencyDescriptor) *DependencyDescriptorReader {
buffer := NewBitStreamReader(buf)
return &DependencyDescriptorReader{
buffer: buffer,
descriptor: descriptor,
structure: structure,
}
}
func (r *DependencyDescriptorReader) Parse() (int, error) {
if err := r.readMandatoryFields(); err != nil {
return 0, err
}
if len(r.buffer.buf) > 3 {
err := r.readExtendedFields()
if err != nil {
return 0, err
}
}
if r.descriptor.AttachedStructure != nil {
r.structure = r.descriptor.AttachedStructure
}
if r.structure == nil {
r.buffer.Invalidate()
return 0, ErrDDReaderNoStructure
}
if r.activeDecodeTargetsPresentFlag {
bitmask, err := r.buffer.ReadBits(r.structure.NumDecodeTargets)
if err != nil {
return 0, err
}
mask := uint32(bitmask)
r.descriptor.ActiveDecodeTargetsBitmask = &mask
}
err := r.readFrameDependencyDefinition()
if err != nil {
return 0, err
}
return r.buffer.BytesRead(), nil
}
func (r *DependencyDescriptorReader) readMandatoryFields() error {
var err error
r.descriptor.FirstPacketInFrame, err = r.buffer.ReadBool()
if err != nil {
return err
}
r.descriptor.LastPacketInFrame, err = r.buffer.ReadBool()
if err != nil {
return err
}
templateID, err := r.buffer.ReadBits(6)
if err != nil {
return err
}
r.frameDependencyTemplateId = int(templateID)
frameNumber, err := r.buffer.ReadBits(16)
if err != nil {
return err
}
r.descriptor.FrameNumber = uint16(frameNumber)
return nil
}
func (r *DependencyDescriptorReader) readExtendedFields() error {
templateDependencyStructurePresentFlag, err := r.buffer.ReadBool()
if err != nil {
return err
}
flag, err := r.buffer.ReadBool()
if err != nil {
return err
}
r.activeDecodeTargetsPresentFlag = flag
flag, err = r.buffer.ReadBool()
if err != nil {
return err
}
r.customDtisFlag = flag
flag, err = r.buffer.ReadBool()
if err != nil {
return err
}
r.customFdiffsFlag = flag
flag, err = r.buffer.ReadBool()
if err != nil {
return err
}
r.customChainsFlag = flag
if templateDependencyStructurePresentFlag {
err = r.readTemplateDependencyStructure()
if err != nil {
return err
}
if r.descriptor.AttachedStructure == nil {
return ErrDDReaderTemplateWithoutStructure
}
bitmask := uint32((uint64(1) << r.descriptor.AttachedStructure.NumDecodeTargets) - 1)
r.descriptor.ActiveDecodeTargetsBitmask = &bitmask
}
return nil
}
func (r *DependencyDescriptorReader) readTemplateDependencyStructure() error {
r.descriptor.AttachedStructure = &FrameDependencyStructure{}
structureId, err := r.buffer.ReadBits(6)
if err != nil {
return err
}
r.descriptor.AttachedStructure.StructureId = int(structureId)
numDecodeTargets, err := r.buffer.ReadBits(5)
if err != nil {
return err
}
r.descriptor.AttachedStructure.NumDecodeTargets = int(numDecodeTargets) + 1
if err = r.readTemplateLayers(); err != nil {
return err
}
if err = r.readTemplateDtis(); err != nil {
return err
}
if err = r.readTemplateFdiffs(); err != nil {
return err
}
if err = r.readTemplateChains(); err != nil {
return err
}
flag, err := r.buffer.ReadBool()
if err != nil {
return err
}
if flag {
return r.readResolutions()
}
return nil
}
type nextLayerIdcType int
const (
sameLayer nextLayerIdcType = iota
nextTemporalLayer
nextSpatialLayer
noMoreLayer
invalidLayer
)
func (r *DependencyDescriptorReader) readTemplateLayers() error {
var (
templates []*FrameDependencyTemplate
temporalId, spatialId int
nextLayerIdc nextLayerIdcType
)
for {
if len(templates) == MaxTemplates {
return ErrDDReaderTooManyTemplates
}
var lastTemplate FrameDependencyTemplate
templates = append(templates, &lastTemplate)
lastTemplate.TemporalId = temporalId
lastTemplate.SpatialId = spatialId
idc, err := r.buffer.ReadBits(2)
if err != nil {
return err
}
nextLayerIdc = nextLayerIdcType(idc)
if nextLayerIdc == nextTemporalLayer {
temporalId++
if temporalId >= MaxTemporalIds {
return ErrDDReaderTooManyTemporalLayers
}
} else if nextLayerIdc == nextSpatialLayer {
spatialId++
temporalId = 0
if spatialId >= MaxSpatialIds {
return ErrDDReaderTooManySpatialLayers
}
}
if !(nextLayerIdc != noMoreLayer && r.buffer.Ok()) {
break
}
}
r.descriptor.AttachedStructure.Templates = templates
return nil
}
func (r *DependencyDescriptorReader) readTemplateDtis() error {
structure := r.descriptor.AttachedStructure
for _, template := range structure.Templates {
if len(template.DecodeTargetIndications) < structure.NumDecodeTargets {
template.DecodeTargetIndications = append(template.DecodeTargetIndications, make([]DecodeTargetIndication, structure.NumDecodeTargets-len(template.DecodeTargetIndications))...)
} else {
template.DecodeTargetIndications = template.DecodeTargetIndications[:structure.NumDecodeTargets]
}
for i := range template.DecodeTargetIndications {
indication, err := r.buffer.ReadBits(2)
if err != nil {
return err
}
template.DecodeTargetIndications[i] = DecodeTargetIndication(indication)
}
}
return nil
}
func (r *DependencyDescriptorReader) readTemplateFdiffs() error {
for _, template := range r.descriptor.AttachedStructure.Templates {
for {
fdiffFollow, err := r.buffer.ReadBool()
if err != nil {
return err
}
if !fdiffFollow {
break
}
fDiffMinusOne, err := r.buffer.ReadBits(4)
if err != nil {
return err
}
template.FrameDiffs = append(template.FrameDiffs, int(fDiffMinusOne+1))
}
}
return nil
}
func (r *DependencyDescriptorReader) readTemplateChains() error {
structure := r.descriptor.AttachedStructure
numChains, err := r.buffer.ReadNonSymmetric(uint32(structure.NumDecodeTargets) + 1)
if err != nil {
return err
}
structure.NumChains = int(numChains)
if structure.NumChains == 0 {
return nil
}
for i := 0; i < structure.NumDecodeTargets; i++ {
protectedByChain, err := r.buffer.ReadNonSymmetric(uint32(structure.NumChains))
if err != nil {
return err
}
structure.DecodeTargetProtectedByChain = append(structure.DecodeTargetProtectedByChain, int(protectedByChain))
}
for _, frameTemplate := range structure.Templates {
for chainId := 0; chainId < structure.NumChains; chainId++ {
chainDiff, err := r.buffer.ReadBits(4)
if err != nil {
return err
}
frameTemplate.ChainDiffs = append(frameTemplate.ChainDiffs, int(chainDiff))
}
}
return nil
}
func (r *DependencyDescriptorReader) readResolutions() error {
structure := r.descriptor.AttachedStructure
// The way templates are bitpacked, they are always ordered by spatial_id.
spatialLayers := structure.Templates[len(structure.Templates)-1].SpatialId + 1
for sid := 0; sid < spatialLayers; sid++ {
widthMinus1, err := r.buffer.ReadBits(16)
if err != nil {
return err
}
heightMinus1, err := r.buffer.ReadBits(16)
if err != nil {
return err
}
structure.Resolutions = append(structure.Resolutions, RenderResolution{
Width: int(widthMinus1 + 1),
Height: int(heightMinus1 + 1),
})
}
return nil
}
func (r *DependencyDescriptorReader) readFrameDependencyDefinition() error {
templateIndex := (r.frameDependencyTemplateId + MaxTemplates - r.structure.StructureId) % MaxTemplates
if templateIndex >= len(r.structure.Templates) {
r.buffer.Invalidate()
return ErrDDReaderInvalidTemplateIndex
}
// Copy all the fields from the matching template
r.descriptor.FrameDependencies = r.structure.Templates[templateIndex].Clone()
if r.customDtisFlag {
err := r.readFrameDtis()
if err != nil {
return err
}
}
if r.customFdiffsFlag {
err := r.readFrameFdiffs()
if err != nil {
return err
}
}
if r.customChainsFlag {
err := r.readFrameChains()
if err != nil {
return err
}
}
if len(r.structure.Resolutions) == 0 {
r.descriptor.Resolution = nil
} else {
// Format guarantees that if there were resolutions in the last structure,
// then each spatial layer got one.
if r.descriptor.FrameDependencies.SpatialId >= len(r.structure.Resolutions) {
r.buffer.Invalidate()
return ErrDDReaderInvalidSpatialLayer
}
res := r.structure.Resolutions[r.descriptor.FrameDependencies.SpatialId]
r.descriptor.Resolution = &res
}
return nil
}
func (r *DependencyDescriptorReader) readFrameDtis() error {
if len(r.descriptor.FrameDependencies.DecodeTargetIndications) != r.structure.NumDecodeTargets {
return ErrDDReaderNumDTIMismatch
}
for i := range r.descriptor.FrameDependencies.DecodeTargetIndications {
indication, err := r.buffer.ReadBits(2)
if err != nil {
return err
}
r.descriptor.FrameDependencies.DecodeTargetIndications[i] = DecodeTargetIndication(indication)
}
return nil
}
func (r *DependencyDescriptorReader) readFrameFdiffs() error {
r.descriptor.FrameDependencies.FrameDiffs = r.descriptor.FrameDependencies.FrameDiffs[:0]
for {
nexFdiffSize, err := r.buffer.ReadBits(2)
if err != nil {
return err
}
if nexFdiffSize == 0 {
break
}
fDiffMinusOne, err := r.buffer.ReadBits(int(nexFdiffSize * 4))
if err != nil {
return err
}
r.descriptor.FrameDependencies.FrameDiffs = append(r.descriptor.FrameDependencies.FrameDiffs, int(fDiffMinusOne+1))
}
return nil
}
func (r *DependencyDescriptorReader) readFrameChains() error {
if len(r.descriptor.FrameDependencies.ChainDiffs) != r.structure.NumChains {
return ErrDDReaderNumChainDiffsMismatch
}
for i := range r.descriptor.FrameDependencies.ChainDiffs {
chainDiff, err := r.buffer.ReadBits(8)
if err != nil {
return err
}
r.descriptor.FrameDependencies.ChainDiffs[i] = int(chainDiff)
}
return nil
}
livekit-server/pkg/sfu/rtpextension/dependencydescriptor/dependencydescriptorwriter.go
+501
View File
@@ -0,0 +1,501 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dependencydescriptor
import (
"fmt"
"golang.org/x/exp/slices"
)
type TemplateMatch struct {
TemplateIdx int
NeedCustomDtis bool
NeedCustomFdiffs bool
NeedCustomChains bool
// Size in bits to store frame-specific details, i.e.
// excluding mandatory fields and template dependency structure.
ExtraSizeBits int
}
type DependencyDescriptorWriter struct {
descriptor *DependencyDescriptor
structure *FrameDependencyStructure
activeChains uint32
writer *BitStreamWriter
bestTemplate TemplateMatch
}
func NewDependencyDescriptorWriter(buf []byte, structure *FrameDependencyStructure, activeChains uint32, descriptor *DependencyDescriptor) (*DependencyDescriptorWriter, error) {
writer := NewBitStreamWriter(buf)
w := &DependencyDescriptorWriter{
descriptor: descriptor,
structure: structure,
activeChains: activeChains,
writer: writer,
}
return w, w.findBestTemplate()
}
func (w *DependencyDescriptorWriter) ResetBuf(buf []byte) {
w.writer = NewBitStreamWriter(buf)
}
func (w *DependencyDescriptorWriter) Write() error {
if err := w.findBestTemplate(); err != nil {
return err
}
if err := w.writeMandatoryFields(); err != nil {
return err
}
if w.hasExtendedFields() {
if err := w.writeExtendedFields(); err != nil {
return err
}
if err := w.writeFrameDependencyDefinition(); err != nil {
return err
}
}
remainingBits := w.writer.RemainingBits()
// Zero remaining memory to avoid leaving it uninitialized.
if remainingBits%64 != 0 {
if err := w.writeBits(0, remainingBits%64); err != nil {
return err
}
}
for i := 0; i < remainingBits/64; i++ {
if err := w.writeBits(0, 64); err != nil {
return err
}
}
return nil
}
func (w *DependencyDescriptorWriter) findBestTemplate() error {
// Find templates with same spatial and temporal layer of frame dependency.
var (
firstSameLayer *FrameDependencyTemplate
firstSameLayerIdx, lastSameLayerIdx int
)
for i, t := range w.structure.Templates {
if w.descriptor.FrameDependencies.SpatialId == t.SpatialId &&
w.descriptor.FrameDependencies.TemporalId == t.TemporalId {
firstSameLayer = t
firstSameLayerIdx = i
break
}
}
if firstSameLayer == nil {
return fmt.Errorf("no template found for spatial layer %d and temporal layer %d", w.descriptor.FrameDependencies.SpatialId, w.descriptor.FrameDependencies.TemporalId)
}
for i, t := range w.structure.Templates[firstSameLayerIdx:] {
if w.descriptor.FrameDependencies.SpatialId != t.SpatialId ||
w.descriptor.FrameDependencies.TemporalId != t.TemporalId {
lastSameLayerIdx = i + firstSameLayerIdx
}
}
// Search if there any better template that have small extra size.
w.bestTemplate = w.calculateMatch(firstSameLayerIdx, firstSameLayer)
for i := firstSameLayerIdx + 1; i <= lastSameLayerIdx; i++ {
t := w.structure.Templates[i]
match := w.calculateMatch(i, t)
if match.ExtraSizeBits < w.bestTemplate.ExtraSizeBits {
w.bestTemplate = match
}
}
return nil
}
func (w *DependencyDescriptorWriter) calculateMatch(idx int, template *FrameDependencyTemplate) TemplateMatch {
var result TemplateMatch
result.TemplateIdx = idx
result.NeedCustomFdiffs = w.descriptor.FrameDependencies.FrameDiffs != nil && !slices.Equal(w.descriptor.FrameDependencies.FrameDiffs, template.FrameDiffs)
result.NeedCustomDtis = w.descriptor.FrameDependencies.DecodeTargetIndications != nil && !slices.Equal(w.descriptor.FrameDependencies.DecodeTargetIndications, template.DecodeTargetIndications)
for i := 0; i < w.structure.NumChains; i++ {
if w.activeChains&(1<<i) != 0 && (len(w.descriptor.FrameDependencies.ChainDiffs) <= i || len(template.ChainDiffs) <= i || w.descriptor.FrameDependencies.ChainDiffs[i] != template.ChainDiffs[i]) {
result.NeedCustomChains = true
break
}
}
if result.NeedCustomFdiffs {
result.ExtraSizeBits = 2 * (1 + len(w.descriptor.FrameDependencies.FrameDiffs))
for _, fdiff := range w.descriptor.FrameDependencies.FrameDiffs {
if fdiff <= (1 << 4) {
result.ExtraSizeBits += 4
} else if fdiff <= (1 << 8) {
result.ExtraSizeBits += 8
} else {
result.ExtraSizeBits += 12
}
}
}
if result.NeedCustomDtis {
result.ExtraSizeBits += 2 * len(w.descriptor.FrameDependencies.DecodeTargetIndications)
}
if result.NeedCustomChains {
result.ExtraSizeBits += 8 * w.structure.NumChains
}
return result
}
func (w *DependencyDescriptorWriter) writeMandatoryFields() error {
if err := w.writeBool(w.descriptor.FirstPacketInFrame); err != nil {
return err
}
if err := w.writeBool(w.descriptor.LastPacketInFrame); err != nil {
return err
}
templateId := (w.bestTemplate.TemplateIdx + w.structure.StructureId) % MaxTemplates
if err := w.writeBits(uint64(templateId), 6); err != nil {
return err
}
return w.writeBits(uint64(w.descriptor.FrameNumber), 16)
}
func (w *DependencyDescriptorWriter) writeBool(val bool) error {
v := uint64(0)
if val {
v = 1
}
return w.writer.WriteBits(v, 1)
}
func (w *DependencyDescriptorWriter) writeBits(val uint64, bitCount int) error {
if err := w.writer.WriteBits(val, bitCount); err != nil {
return err
}
return nil
}
func (w *DependencyDescriptorWriter) hasExtendedFields() bool {
return w.bestTemplate.ExtraSizeBits > 0 || w.descriptor.AttachedStructure != nil || w.descriptor.ActiveDecodeTargetsBitmask != nil
}
func (w *DependencyDescriptorWriter) writeExtendedFields() error {
// template_dependency_structure_present_flag
if err := w.writeBool(w.descriptor.AttachedStructure != nil); err != nil {
return err
}
// active_decode_targets_present_flag
activeDecodeTargetsPresentFlag := w.shouldWriteActiveDecodeTargetsBitmask()
if err := w.writeBool(activeDecodeTargetsPresentFlag); err != nil {
return err
}
// need_custom_dtis
if err := w.writeBool(w.bestTemplate.NeedCustomDtis); err != nil {
return err
}
// need_custom_fdiffs
if err := w.writeBool(w.bestTemplate.NeedCustomFdiffs); err != nil {
return err
}
// need_custom_chains
if err := w.writeBool(w.bestTemplate.NeedCustomChains); err != nil {
return err
}
// template_dependency_structure
if w.descriptor.AttachedStructure != nil {
if err := w.writeTemplateDependencyStructure(); err != nil {
return err
}
}
// active_decode_targets_bitmask
if activeDecodeTargetsPresentFlag {
if err := w.writeBits(uint64(*w.descriptor.ActiveDecodeTargetsBitmask), w.structure.NumDecodeTargets); err != nil {
return err
}
}
return nil
}
func (w *DependencyDescriptorWriter) writeTemplateDependencyStructure() error {
if !(w.structure.StructureId >= 0 && w.structure.StructureId < MaxTemplates &&
w.structure.NumDecodeTargets > 0 && w.structure.NumDecodeTargets <= MaxDecodeTargets) {
return fmt.Errorf("invalid arguments, structureId: %d, numDecodeTargets: %d", w.structure.StructureId, w.structure.NumDecodeTargets)
}
if err := w.writeBits(uint64(w.structure.StructureId), 6); err != nil {
return err
}
if err := w.writeBits(uint64(w.structure.NumDecodeTargets-1), 5); err != nil {
return err
}
if err := w.writeTemplateLayers(); err != nil {
return err
}
if err := w.writeTemplateDtis(); err != nil {
return err
}
if err := w.writeTemplateFdiffs(); err != nil {
return err
}
if err := w.writeTemplateChains(); err != nil {
return err
}
hasResolutions := len(w.structure.Resolutions) > 0
if err := w.writeBool(hasResolutions); err != nil {
return err
}
return w.writeResolutions()
}
func (w *DependencyDescriptorWriter) writeTemplateLayers() error {
if !(len(w.structure.Templates) > 0 && len(w.structure.Templates) <= MaxTemplates &&
w.structure.Templates[0].SpatialId == 0 && w.structure.Templates[0].TemporalId == 0) {
return fmt.Errorf("invalid templates, len %d, templates[0]: spatialId %d, temporalId %d", len(w.structure.Templates), w.structure.Templates[0].SpatialId, w.structure.Templates[0].TemporalId)
}
for i := 1; i < len(w.structure.Templates); i++ {
nextLayerIdc := getNextLayerIdc(w.structure.Templates[i-1], w.structure.Templates[i])
if nextLayerIdc >= 3 {
return fmt.Errorf("invalid next_layer_idc %d", nextLayerIdc)
}
if err := w.writeBits(uint64(nextLayerIdc), 2); err != nil {
return err
}
}
return w.writeBits(uint64(noMoreLayer), 2)
}
func getNextLayerIdc(prevTemplate, nextTemplate *FrameDependencyTemplate) nextLayerIdcType {
if nextTemplate.SpatialId == prevTemplate.SpatialId && nextTemplate.TemporalId == prevTemplate.TemporalId {
return sameLayer
} else if nextTemplate.SpatialId == prevTemplate.SpatialId && nextTemplate.TemporalId == prevTemplate.TemporalId+1 {
return nextTemporalLayer
} else if nextTemplate.SpatialId == prevTemplate.SpatialId+1 && nextTemplate.TemporalId == 0 {
return nextSpatialLayer
}
return invalidLayer
}
func (w *DependencyDescriptorWriter) writeTemplateDtis() error {
for _, t := range w.structure.Templates {
for _, dti := range t.DecodeTargetIndications {
if err := w.writeBits(uint64(dti), 2); err != nil {
return err
}
}
}
return nil
}
func (w *DependencyDescriptorWriter) writeTemplateFdiffs() error {
for _, t := range w.structure.Templates {
for _, fdiff := range t.FrameDiffs {
if err := w.writeBits(uint64(1<<4)|uint64(fdiff-1), 1+4); err != nil {
return err
}
}
// no more fdiffs for this template
if err := w.writeBits(uint64(0), 1); err != nil {
return err
}
}
return nil
}
func (w *DependencyDescriptorWriter) writeTemplateChains() error {
if err := w.writeNonSymmetric(uint32(w.structure.NumChains), uint32(w.structure.NumDecodeTargets+1)); err != nil {
return err
}
if w.structure.NumChains == 0 {
return nil
}
for _, protectedBy := range w.structure.DecodeTargetProtectedByChain {
if err := w.writeNonSymmetric(uint32(protectedBy), uint32(w.structure.NumChains)); err != nil {
return err
}
}
for _, t := range w.structure.Templates {
for _, chainDiff := range t.ChainDiffs {
if err := w.writeBits(uint64(chainDiff), 4); err != nil {
return err
}
}
}
return nil
}
func (w *DependencyDescriptorWriter) writeNonSymmetric(value, numValues uint32) error {
return w.writer.WriteNonSymmetric(value, numValues)
}
func (w *DependencyDescriptorWriter) writeResolutions() error {
for _, res := range w.structure.Resolutions {
if err := w.writeBits(uint64(res.Width)-1, 16); err != nil {
return err
}
if err := w.writeBits(uint64(res.Height)-1, 16); err != nil {
return err
}
}
return nil
}
func (w *DependencyDescriptorWriter) writeFrameDependencyDefinition() error {
if w.bestTemplate.NeedCustomDtis {
if err := w.writeFrameDtis(); err != nil {
return err
}
}
if w.bestTemplate.NeedCustomFdiffs {
if err := w.writeFrameFdiffs(); err != nil {
return err
}
}
if w.bestTemplate.NeedCustomChains {
if err := w.writeFrameChains(); err != nil {
return err
}
}
return nil
}
func (w *DependencyDescriptorWriter) writeFrameDtis() error {
for _, dti := range w.descriptor.FrameDependencies.DecodeTargetIndications {
if err := w.writeBits(uint64(dti), 2); err != nil {
return err
}
}
return nil
}
func (w *DependencyDescriptorWriter) writeFrameFdiffs() error {
for _, fdiff := range w.descriptor.FrameDependencies.FrameDiffs {
if fdiff <= (1 << 4) {
if err := w.writeBits(uint64(1<<4)|uint64(fdiff-1), 2+4); err != nil {
return err
}
} else if fdiff <= (1 << 8) {
if err := w.writeBits(uint64(2<<8)|uint64(fdiff-1), 2+8); err != nil {
return err
}
} else { // fdiff <= (1<<12)
if err := w.writeBits(uint64(3<<12)|uint64(fdiff-1), 2+12); err != nil {
return err
}
}
}
// no more fdiffs
return w.writeBits(uint64(0), 2)
}
func (w *DependencyDescriptorWriter) writeFrameChains() error {
for i := 0; i < w.structure.NumChains; i++ {
chainDiff := 0
if w.activeChains&(1<<i) != 0 {
chainDiff = w.descriptor.FrameDependencies.ChainDiffs[i]
}
if err := w.writeBits(uint64(chainDiff), 8); err != nil {
return err
}
}
return nil
}
const mandatoryFieldSize = 1 + 1 + 6 + 16
func (w *DependencyDescriptorWriter) ValueSizeBits() int {
valueSizeBits := mandatoryFieldSize + w.bestTemplate.ExtraSizeBits
if w.hasExtendedFields() {
valueSizeBits += 5
if w.descriptor.AttachedStructure != nil {
valueSizeBits += w.structureSizeBits()
}
if w.shouldWriteActiveDecodeTargetsBitmask() {
valueSizeBits += w.structure.NumDecodeTargets
}
}
return valueSizeBits
}
func (w *DependencyDescriptorWriter) shouldWriteActiveDecodeTargetsBitmask() bool {
if w.descriptor.ActiveDecodeTargetsBitmask == nil {
return false
}
allDecodeTargetsBitmask := (uint64(1) << w.structure.NumDecodeTargets) - 1
if w.descriptor.AttachedStructure != nil && uint64(*w.descriptor.ActiveDecodeTargetsBitmask) == allDecodeTargetsBitmask {
return false
}
return true
}
func (w *DependencyDescriptorWriter) structureSizeBits() int {
// template_id offset (6 bits) and number of decode targets (5 bits)
bits := 11
// template layers
bits += 2 * len(w.structure.Templates)
// dtis
bits += 2 * len(w.structure.Templates) * w.structure.NumDecodeTargets
// fdiffs. each templates uses 1 + 5 * sizeof(fdiff) bits.
bits += len(w.structure.Templates)
for _, t := range w.structure.Templates {
bits += 5 * len(t.FrameDiffs)
}
bits += SizeNonSymmetricBits(uint32(w.structure.NumChains), uint32(w.structure.NumDecodeTargets+1))
if w.structure.NumChains > 0 {
for _, protectedBy := range w.structure.DecodeTargetProtectedByChain {
bits += SizeNonSymmetricBits(uint32(protectedBy), uint32(w.structure.NumChains))
}
bits += 4 * len(w.structure.Templates) * w.structure.NumChains
}
// resolutions
bits += 1 + 32*len(w.structure.Resolutions)
return bits
}
livekit-server/pkg/sfu/rtpextension/playoutdelay/playoutdelay.go
+72
View File
@@ -0,0 +1,72 @@
// Copyright 2024 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package playoutdelay
import (
"encoding/binary"
"errors"
)
const (
PlayoutDelayURI = "http://www.webrtc.org/experiments/rtp-hdrext/playout-delay"
MaxPlayoutDelayDefault = 10000 // 10s, equal to chrome's default max playout delay
PlayoutDelayMaxValue = 10 * (1<<12 - 1) // max value for playout delay can be represented
playoutDelayExtensionSize = 3
)
var (
errPlayoutDelayOverflow = errors.New("playout delay overflow")
errTooSmall = errors.New("buffer too small")
)
// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// | ID | len=2 | MIN delay | MAX delay |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// The wired MIN/MAX delay is in 10ms unit
type PlayOutDelay struct {
Min, Max uint16 // delay in ms
}
func PlayoutDelayFromValue(min, max uint16) PlayOutDelay {
if min > PlayoutDelayMaxValue {
min = PlayoutDelayMaxValue
}
if max > PlayoutDelayMaxValue {
max = PlayoutDelayMaxValue
}
return PlayOutDelay{Min: min, Max: max}
}
func (p PlayOutDelay) Marshal() ([]byte, error) {
min, max := p.Min/10, p.Max/10
if min >= 1<<12 || max >= 1<<12 {
return nil, errPlayoutDelayOverflow
}
return []byte{byte(min >> 4), byte(min<<4) | byte(max>>8), byte(max)}, nil
}
func (p *PlayOutDelay) Unmarshal(rawData []byte) error {
if len(rawData) < playoutDelayExtensionSize {
return errTooSmall
}
p.Min = (binary.BigEndian.Uint16(rawData) >> 4) * 10
p.Max = (binary.BigEndian.Uint16(rawData[1:]) & 0x0FFF) * 10
return nil
}
livekit-server/pkg/sfu/rtpextension/playoutdelay/playoutdelay_test.go
+57
View File
@@ -0,0 +1,57 @@
// Copyright 2024 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package playoutdelay
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestPlayoutDelay(t *testing.T) {
p1 := PlayOutDelay{Min: 100, Max: 200}
b, err := p1.Marshal()
require.NoError(t, err)
require.Len(t, b, playoutDelayExtensionSize)
var p2 PlayOutDelay
err = p2.Unmarshal(b)
require.NoError(t, err)
require.Equal(t, p1, p2)
// overflow
p3 := PlayOutDelay{Min: 100, Max: (1 << 12) * 10}
_, err = p3.Marshal()
require.ErrorIs(t, err, errPlayoutDelayOverflow)
// too small
p4 := PlayOutDelay{}
err = p4.Unmarshal([]byte{0x00, 0x00})
require.ErrorIs(t, err, errTooSmall)
// from value
p5 := PlayoutDelayFromValue(1<<12*10, 1<<12*10+10)
_, err = p5.Marshal()
require.NoError(t, err)
require.Equal(t, uint16((1<<12)-1)*10, p5.Min)
require.Equal(t, uint16((1<<12)-1)*10, p5.Max)
p6 := PlayOutDelay{Min: 100, Max: PlayoutDelayMaxValue}
bytes, err := p6.Marshal()
require.NoError(t, err)
p6Unmarshal := PlayOutDelay{}
err = p6Unmarshal.Unmarshal(bytes)
require.NoError(t, err)
require.Equal(t, p6, p6Unmarshal)
}
livekit-server/pkg/sfu/rtpmunger.go
+342
View File
@@ -0,0 +1,342 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package sfu
import (
"github.com/livekit/protocol/livekit"
"github.com/livekit/protocol/logger"
"github.com/livekit/livekit-server/pkg/sfu/buffer"
"github.com/livekit/livekit-server/pkg/sfu/utils"
)
// RTPMunger
type SequenceNumberOrdering int
const (
SequenceNumberOrderingContiguous SequenceNumberOrdering = iota
SequenceNumberOrderingOutOfOrder
SequenceNumberOrderingGap
SequenceNumberOrderingDuplicate
)
const (
RtxGateWindow = 2000
)
type TranslationParamsRTP struct {
snOrdering SequenceNumberOrdering
extSequenceNumber uint64
extTimestamp uint64
}
type SnTs struct {
extSequenceNumber uint64
extTimestamp uint64
}
// ----------------------------------------------------------------------
type RTPMunger struct {
logger logger.Logger
extHighestIncomingSN uint64
snRangeMap *utils.RangeMap[uint64, uint64]
extLastSN uint64
extSecondLastSN uint64
snOffset uint64
extLastTS uint64
extSecondLastTS uint64
tsOffset uint64
lastMarker bool
secondLastMarker bool
extRtxGateSn uint64
isInRtxGateRegion bool
}
func NewRTPMunger(logger logger.Logger) *RTPMunger {
return &RTPMunger{
logger: logger,
snRangeMap: utils.NewRangeMap[uint64, uint64](100),
}
}
func (r *RTPMunger) DebugInfo() map[string]interface{} {
return map[string]interface{}{
"ExtHighestIncomingSN": r.extHighestIncomingSN,
"ExtLastSN": r.extLastSN,
"ExtSecondLastSN": r.extSecondLastSN,
"SNOffset": r.snOffset,
"ExtLastTS": r.extLastTS,
"ExtSecondLastTS": r.extSecondLastTS,
"TSOffset": r.tsOffset,
"LastMarker": r.lastMarker,
"SecondLastMarker": r.secondLastMarker,
}
}
func (r *RTPMunger) GetState() *livekit.RTPMungerState {
return &livekit.RTPMungerState{
ExtLastSequenceNumber: r.extLastSN,
ExtSecondLastSequenceNumber: r.extSecondLastSN,
ExtLastTimestamp: r.extLastTS,
ExtSecondLastTimestamp: r.extSecondLastTS,
LastMarker: r.lastMarker,
SecondLastMarker: r.secondLastMarker,
}
}
func (r *RTPMunger) GetTSOffset() uint64 {
return r.tsOffset
}
func (r *RTPMunger) SeedState(state *livekit.RTPMungerState) {
r.extLastSN = state.ExtLastSequenceNumber
r.extSecondLastSN = state.ExtSecondLastSequenceNumber
r.extLastTS = state.ExtLastTimestamp
r.extSecondLastTS = state.ExtSecondLastTimestamp
r.lastMarker = state.LastMarker
r.secondLastMarker = state.SecondLastMarker
}
func (r *RTPMunger) SetLastSnTs(extPkt *buffer.ExtPacket) {
r.extHighestIncomingSN = extPkt.ExtSequenceNumber - 1
r.extLastSN = extPkt.ExtSequenceNumber
r.extSecondLastSN = r.extLastSN - 1
r.snRangeMap.ClearAndResetValue(extPkt.ExtSequenceNumber, 0)
r.updateSnOffset()
r.extLastTS = extPkt.ExtTimestamp
r.extSecondLastTS = extPkt.ExtTimestamp
r.tsOffset = 0
}
func (r *RTPMunger) UpdateSnTsOffsets(extPkt *buffer.ExtPacket, snAdjust uint64, tsAdjust uint64) {
r.extHighestIncomingSN = extPkt.ExtSequenceNumber - 1
r.snRangeMap.ClearAndResetValue(extPkt.ExtSequenceNumber, extPkt.ExtSequenceNumber-r.extLastSN-snAdjust)
r.updateSnOffset()
r.tsOffset = extPkt.ExtTimestamp - r.extLastTS - tsAdjust
}
func (r *RTPMunger) PacketDropped(extPkt *buffer.ExtPacket) {
if r.extHighestIncomingSN != extPkt.ExtSequenceNumber {
return
}
snOffset, err := r.snRangeMap.GetValue(extPkt.ExtSequenceNumber)
if err == nil {
outSN := extPkt.ExtSequenceNumber - snOffset
if outSN != r.extLastSN {
r.logger.Warnw("last outgoing sequence number mismatch", nil, "expected", r.extLastSN, "got", outSN)
}
}
if r.extLastSN == r.extSecondLastSN {
r.logger.Warnw("cannot roll back on drop", nil, "extLastSN", r.extLastSN, "secondLastSN", r.extSecondLastSN)
}
if err := r.snRangeMap.ExcludeRange(r.extHighestIncomingSN, r.extHighestIncomingSN+1); err != nil {
r.logger.Errorw("could not exclude range", err, "sn", r.extHighestIncomingSN)
}
r.extLastSN = r.extSecondLastSN
r.updateSnOffset()
r.extLastTS = r.extSecondLastTS
r.lastMarker = r.secondLastMarker
}
func (r *RTPMunger) UpdateAndGetSnTs(extPkt *buffer.ExtPacket, marker bool) (TranslationParamsRTP, error) {
diff := int64(extPkt.ExtSequenceNumber - r.extHighestIncomingSN)
if (diff == 1 && len(extPkt.Packet.Payload) != 0) || diff > 1 {
// in-order - either contiguous packet with payload OR packet following a gap, may or may not have payload
r.extHighestIncomingSN = extPkt.ExtSequenceNumber
ordering := SequenceNumberOrderingContiguous
if diff > 1 {
ordering = SequenceNumberOrderingGap
}
extMungedSN := extPkt.ExtSequenceNumber - r.snOffset
extMungedTS := extPkt.ExtTimestamp - r.tsOffset
r.extSecondLastSN = r.extLastSN
r.extLastSN = extMungedSN
r.extSecondLastTS = r.extLastTS
r.extLastTS = extMungedTS
r.secondLastMarker = r.lastMarker
r.lastMarker = marker
if extPkt.KeyFrame {
r.extRtxGateSn = extMungedSN
r.isInRtxGateRegion = true
}
if r.isInRtxGateRegion && (extMungedSN-r.extRtxGateSn) > RtxGateWindow {
r.isInRtxGateRegion = false
}
return TranslationParamsRTP{
snOrdering: ordering,
extSequenceNumber: extMungedSN,
extTimestamp: extMungedTS,
}, nil
}
if diff < 0 {
// out-of-order, look up sequence number offset cache
snOffset, err := r.snRangeMap.GetValue(extPkt.ExtSequenceNumber)
if err != nil {
return TranslationParamsRTP{
snOrdering: SequenceNumberOrderingOutOfOrder,
}, ErrOutOfOrderSequenceNumberCacheMiss
}
extSequenceNumber := extPkt.ExtSequenceNumber - snOffset
if extSequenceNumber >= r.extLastSN {
// should not happen, just being paranoid
r.logger.Errorw(
"unexpected packet ordering", nil,
"extIncomingSN", extPkt.ExtSequenceNumber,
"extHighestIncomingSN", r.extHighestIncomingSN,
"extLastSN", r.extLastSN,
"snOffsetIncoming", snOffset,
"snOffsetHighest", r.snOffset,
)
return TranslationParamsRTP{
snOrdering: SequenceNumberOrderingOutOfOrder,
}, ErrOutOfOrderSequenceNumberCacheMiss
}
return TranslationParamsRTP{
snOrdering: SequenceNumberOrderingOutOfOrder,
extSequenceNumber: extSequenceNumber,
extTimestamp: extPkt.ExtTimestamp - r.tsOffset,
}, nil
}
// if padding only packet, can be dropped and sequence number adjusted, if contiguous
if diff == 1 {
r.extHighestIncomingSN = extPkt.ExtSequenceNumber
if err := r.snRangeMap.ExcludeRange(r.extHighestIncomingSN, r.extHighestIncomingSN+1); err != nil {
r.logger.Errorw("could not exclude range", err, "sn", r.extHighestIncomingSN)
}
r.updateSnOffset()
return TranslationParamsRTP{
snOrdering: SequenceNumberOrderingContiguous,
}, ErrPaddingOnlyPacket
}
// can get duplicate packet due to FEC
return TranslationParamsRTP{
snOrdering: SequenceNumberOrderingDuplicate,
}, ErrDuplicatePacket
}
func (r *RTPMunger) FilterRTX(nacks []uint16) []uint16 {
if !r.isInRtxGateRegion {
return nacks
}
filtered := make([]uint16, 0, len(nacks))
for _, sn := range nacks {
if (sn - uint16(r.extRtxGateSn)) < (1 << 15) {
filtered = append(filtered, sn)
}
}
return filtered
}
func (r *RTPMunger) UpdateAndGetPaddingSnTs(num int, clockRate uint32, frameRate uint32, forceMarker bool, extRtpTimestamp uint64) ([]SnTs, error) {
if num == 0 {
return nil, nil
}
useLastTSForFirst := false
tsOffset := 0
if !r.lastMarker {
if !forceMarker {
return nil, ErrPaddingNotOnFrameBoundary
}
// if forcing frame end, use timestamp of latest received frame for the first one
useLastTSForFirst = true
tsOffset = 1
}
extLastSN := r.extLastSN
extLastTS := r.extLastTS
vals := make([]SnTs, num)
for i := 0; i < num; i++ {
extLastSN++
vals[i].extSequenceNumber = extLastSN
if frameRate != 0 {
if useLastTSForFirst && i == 0 {
vals[i].extTimestamp = r.extLastTS
} else {
ets := extRtpTimestamp + uint64(((uint32(i+1-tsOffset)*clockRate)+frameRate-1)/frameRate)
if int64(ets-extLastTS) <= 0 {
ets = extLastTS + 1
}
extLastTS = ets
vals[i].extTimestamp = ets
}
} else {
vals[i].extTimestamp = r.extLastTS
}
}
r.extSecondLastSN = extLastSN - 1
r.extLastSN = extLastSN
r.snRangeMap.DecValue(r.extHighestIncomingSN, uint64(num))
r.updateSnOffset()
if len(vals) == 1 {
r.extSecondLastTS = r.extLastTS
} else {
r.extSecondLastTS = vals[len(vals)-2].extTimestamp
}
r.tsOffset -= extLastTS - r.extLastTS
r.extLastTS = extLastTS
if forceMarker {
r.lastMarker = true
}
return vals, nil
}
func (r *RTPMunger) IsOnFrameBoundary() bool {
return r.lastMarker
}
func (r *RTPMunger) updateSnOffset() {
snOffset, err := r.snRangeMap.GetValue(r.extHighestIncomingSN + 1)
if err != nil {
r.logger.Errorw("could not get sequence number offset", err)
}
r.snOffset = snOffset
}
livekit-server/pkg/sfu/rtpmunger_test.go
+585
View File
@@ -0,0 +1,585 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package sfu
import (
"testing"
"github.com/stretchr/testify/require"
"github.com/livekit/protocol/logger"
"github.com/livekit/livekit-server/pkg/sfu/testutils"
)
func newRTPMunger() *RTPMunger {
return NewRTPMunger(logger.GetLogger())
}
func TestSetLastSnTs(t *testing.T) {
r := newRTPMunger()
params := &testutils.TestExtPacketParams{
SequenceNumber: 23333,
Timestamp: 0xabcdef,
SSRC: 0x12345678,
}
extPkt, err := testutils.GetTestExtPacket(params)
require.NoError(t, err)
require.NotNil(t, extPkt)
r.SetLastSnTs(extPkt)
require.Equal(t, uint64(23332), r.extHighestIncomingSN)
require.Equal(t, uint64(23333), r.extLastSN)
require.Equal(t, uint64(0xabcdef), r.extLastTS)
snOffset, err := r.snRangeMap.GetValue(r.extHighestIncomingSN)
require.Error(t, err)
snOffset, err = r.snRangeMap.GetValue(r.extLastSN)
require.NoError(t, err)
require.Equal(t, uint64(0), snOffset)
require.Equal(t, uint64(0), r.snOffset)
require.Equal(t, uint64(0), r.tsOffset)
}
func TestUpdateSnTsOffsets(t *testing.T) {
r := newRTPMunger()
params := &testutils.TestExtPacketParams{
SequenceNumber: 23333,
Timestamp: 0xabcdef,
SSRC: 0x12345678,
}
extPkt, _ := testutils.GetTestExtPacket(params)
r.SetLastSnTs(extPkt)
params = &testutils.TestExtPacketParams{
SequenceNumber: 33333,
Timestamp: 0xabcdef,
SSRC: 0x87654321,
}
extPkt, _ = testutils.GetTestExtPacket(params)
r.UpdateSnTsOffsets(extPkt, 1, 1)
require.Equal(t, uint64(33332), r.extHighestIncomingSN)
require.Equal(t, uint64(23333), r.extLastSN)
require.Equal(t, uint64(0xabcdef), r.extLastTS)
_, err := r.snRangeMap.GetValue(r.extHighestIncomingSN)
require.Error(t, err)
_, err = r.snRangeMap.GetValue(r.extLastSN)
require.Error(t, err)
require.Equal(t, uint64(9999), r.snOffset)
require.Equal(t, uint64(0xffff_ffff_ffff_ffff), r.tsOffset)
}
func TestPacketDropped(t *testing.T) {
r := newRTPMunger()
params := &testutils.TestExtPacketParams{
SequenceNumber: 23333,
Timestamp: 0xabcdef,
SSRC: 0x12345678,
PayloadSize: 10,
}
extPkt, _ := testutils.GetTestExtPacket(params)
r.SetLastSnTs(extPkt)
require.Equal(t, uint64(23332), r.extHighestIncomingSN)
require.Equal(t, uint64(23333), r.extLastSN)
require.Equal(t, uint64(0xabcdef), r.extLastTS)
snOffset, err := r.snRangeMap.GetValue(r.extHighestIncomingSN)
require.Error(t, err)
snOffset, err = r.snRangeMap.GetValue(r.extLastSN)
require.NoError(t, err)
require.Equal(t, uint64(0), snOffset)
require.Equal(t, uint64(0), r.tsOffset)
r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker) // update sequence number offset
// drop a non-head packet, should cause no change in internals
params = &testutils.TestExtPacketParams{
SequenceNumber: 33333,
Timestamp: 0xabcdef,
SSRC: 0x12345678,
}
extPkt, _ = testutils.GetTestExtPacket(params)
r.PacketDropped(extPkt)
require.Equal(t, uint64(23333), r.extHighestIncomingSN)
require.Equal(t, uint64(23333), r.extLastSN)
snOffset, err = r.snRangeMap.GetValue(r.extHighestIncomingSN)
require.NoError(t, err)
require.Equal(t, uint64(0), snOffset)
// drop a head packet and check offset increases
params = &testutils.TestExtPacketParams{
SequenceNumber: 44444,
Timestamp: 0xabcdef,
SSRC: 0x12345678,
PayloadSize: 20,
}
extPkt, _ = testutils.GetTestExtPacket(params)
r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker) // update sequence number offset
require.Equal(t, uint64(44444), r.extLastSN)
r.PacketDropped(extPkt)
require.Equal(t, uint64(23333), r.extLastSN)
snOffset, err = r.snRangeMap.GetValue(r.extHighestIncomingSN)
require.Error(t, err)
snOffset, err = r.snRangeMap.GetValue(r.extHighestIncomingSN + 1)
require.NoError(t, err)
require.Equal(t, uint64(1), snOffset)
params = &testutils.TestExtPacketParams{
SequenceNumber: 44445,
Timestamp: 0xabcdef,
SSRC: 0x12345678,
PayloadSize: 20,
}
extPkt, _ = testutils.GetTestExtPacket(params)
r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker) // update sequence number offset
require.Equal(t, r.extLastSN, uint64(44444))
snOffset, err = r.snRangeMap.GetValue(r.extHighestIncomingSN)
require.NoError(t, err)
require.Equal(t, uint64(1), snOffset)
}
func TestOutOfOrderSequenceNumber(t *testing.T) {
r := newRTPMunger()
params := &testutils.TestExtPacketParams{
SequenceNumber: 23333,
Timestamp: 0xabcdef,
SSRC: 0x12345678,
PayloadSize: 10,
}
extPkt, _ := testutils.GetTestExtPacket(params)
r.SetLastSnTs(extPkt)
r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker)
// should not be able to add a missing sequence number to the cache that is before start
err := r.snRangeMap.ExcludeRange(23332, 23333)
require.Error(t, err)
// out-of-order sequence number before start should miss
params = &testutils.TestExtPacketParams{
SequenceNumber: 23331,
Timestamp: 0xabcdef,
SSRC: 0x12345678,
PayloadSize: 10,
}
extPkt, _ = testutils.GetTestExtPacket(params)
tp, err := r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker)
require.Error(t, err)
// add a missing sequence number to the cache
err = r.snRangeMap.ExcludeRange(23334, 23335)
require.NoError(t, err)
params = &testutils.TestExtPacketParams{
SequenceNumber: 23336,
Timestamp: 0xabcdef,
SSRC: 0x12345678,
PayloadSize: 10,
}
extPkt, _ = testutils.GetTestExtPacket(params)
r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker)
// out-of-order sequence number should be munged from cache
params = &testutils.TestExtPacketParams{
SequenceNumber: 23335,
Timestamp: 0xabcdef,
SSRC: 0x12345678,
PayloadSize: 10,
}
extPkt, _ = testutils.GetTestExtPacket(params)
tpExpected := TranslationParamsRTP{
snOrdering: SequenceNumberOrderingOutOfOrder,
extSequenceNumber: 23334,
extTimestamp: 0xabcdef,
}
tp, err = r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker)
require.NoError(t, err)
require.Equal(t, tpExpected, tp)
params = &testutils.TestExtPacketParams{
SequenceNumber: 23332,
Timestamp: 0xabcdef,
SSRC: 0x12345678,
PayloadSize: 10,
}
extPkt, _ = testutils.GetTestExtPacket(params)
tpExpected = TranslationParamsRTP{
snOrdering: SequenceNumberOrderingOutOfOrder,
}
tp, err = r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker)
require.Error(t, err, ErrOutOfOrderSequenceNumberCacheMiss)
require.Equal(t, tpExpected, tp)
}
func TestDuplicateSequenceNumber(t *testing.T) {
r := newRTPMunger()
params := &testutils.TestExtPacketParams{
SequenceNumber: 23333,
Timestamp: 0xabcdef,
SSRC: 0x12345678,
}
extPkt, _ := testutils.GetTestExtPacket(params)
r.SetLastSnTs(extPkt)
// send first packet through
r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker)
// send it again - duplicate packet
tpExpected := TranslationParamsRTP{
snOrdering: SequenceNumberOrderingDuplicate,
}
tp, err := r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker)
require.ErrorIs(t, err, ErrDuplicatePacket)
require.Equal(t, tpExpected, tp)
}
func TestPaddingOnlyPacket(t *testing.T) {
r := newRTPMunger()
params := &testutils.TestExtPacketParams{
SequenceNumber: 23333,
Timestamp: 0xabcdef,
SSRC: 0x12345678,
}
extPkt, _ := testutils.GetTestExtPacket(params)
r.SetLastSnTs(extPkt)
// contiguous padding only packet should report an error
tpExpected := TranslationParamsRTP{
snOrdering: SequenceNumberOrderingContiguous,
}
tp, err := r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker)
require.Error(t, err)
require.ErrorIs(t, err, ErrPaddingOnlyPacket)
require.Equal(t, tpExpected, tp)
require.Equal(t, uint64(23333), r.extHighestIncomingSN)
require.Equal(t, uint64(23333), r.extLastSN)
snOffset, err := r.snRangeMap.GetValue(r.extHighestIncomingSN)
require.Error(t, err)
// padding only packet with a gap should not report an error
params = &testutils.TestExtPacketParams{
SequenceNumber: 23335,
Timestamp: 0xabcdef,
SSRC: 0x12345678,
}
extPkt, _ = testutils.GetTestExtPacket(params)
tpExpected = TranslationParamsRTP{
snOrdering: SequenceNumberOrderingGap,
extSequenceNumber: 23334,
extTimestamp: 0xabcdef,
}
tp, err = r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker)
require.NoError(t, err)
require.Equal(t, tpExpected, tp)
require.Equal(t, uint64(23335), r.extHighestIncomingSN)
require.Equal(t, uint64(23334), r.extLastSN)
snOffset, err = r.snRangeMap.GetValue(r.extHighestIncomingSN)
require.NoError(t, err)
require.Equal(t, uint64(1), snOffset)
}
func TestGapInSequenceNumber(t *testing.T) {
r := newRTPMunger()
params := &testutils.TestExtPacketParams{
SequenceNumber: 65533,
Timestamp: 0xabcdef,
SSRC: 0x12345678,
PayloadSize: 33,
}
extPkt, _ := testutils.GetTestExtPacket(params)
r.SetLastSnTs(extPkt)
_, err := r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker)
require.NoError(t, err)
// three lost packets
params = &testutils.TestExtPacketParams{
SequenceNumber: 1,
SNCycles: 1,
Timestamp: 0xabcdef,
SSRC: 0x12345678,
PayloadSize: 33,
}
extPkt, _ = testutils.GetTestExtPacket(params)
tpExpected := TranslationParamsRTP{
snOrdering: SequenceNumberOrderingGap,
extSequenceNumber: 65536 + 1,
extTimestamp: 0xabcdef,
}
tp, err := r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker)
require.NoError(t, err)
require.Equal(t, tpExpected, tp)
require.Equal(t, uint64(65536+1), r.extHighestIncomingSN)
require.Equal(t, uint64(65536+1), r.extLastSN)
snOffset, err := r.snRangeMap.GetValue(r.extHighestIncomingSN)
require.NoError(t, err)
require.Equal(t, uint64(0), snOffset)
// ensure missing sequence numbers have correct cached offset
for i := uint64(65534); i != 65536+1; i++ {
offset, err := r.snRangeMap.GetValue(i)
require.NoError(t, err)
require.Equal(t, uint64(0), offset)
}
// a padding only packet should be dropped
params = &testutils.TestExtPacketParams{
SequenceNumber: 2,
SNCycles: 1,
Timestamp: 0xabcdef,
SSRC: 0x12345678,
}
extPkt, _ = testutils.GetTestExtPacket(params)
tpExpected = TranslationParamsRTP{
snOrdering: SequenceNumberOrderingContiguous,
}
tp, err = r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker)
require.ErrorIs(t, err, ErrPaddingOnlyPacket)
require.Equal(t, tpExpected, tp)
require.Equal(t, uint64(65536+2), r.extHighestIncomingSN)
require.Equal(t, uint64(65536+1), r.extLastSN)
snOffset, err = r.snRangeMap.GetValue(r.extHighestIncomingSN)
require.Error(t, err)
// a packet with a gap should be adjusting for dropped padding packet
params = &testutils.TestExtPacketParams{
SequenceNumber: 4,
SNCycles: 1,
Timestamp: 0xabcdef,
SSRC: 0x12345678,
PayloadSize: 22,
}
extPkt, _ = testutils.GetTestExtPacket(params)
tpExpected = TranslationParamsRTP{
snOrdering: SequenceNumberOrderingGap,
extSequenceNumber: 65536 + 3,
extTimestamp: 0xabcdef,
}
tp, err = r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker)
require.NoError(t, err)
require.Equal(t, tpExpected, tp)
require.Equal(t, uint64(65536+4), r.extHighestIncomingSN)
require.Equal(t, uint64(65536+3), r.extLastSN)
snOffset, err = r.snRangeMap.GetValue(r.extHighestIncomingSN)
require.NoError(t, err)
require.Equal(t, uint64(1), snOffset)
// ensure missing sequence number has correct cached offset
offset, err := r.snRangeMap.GetValue(65536 + 3)
require.NoError(t, err)
require.Equal(t, uint64(1), offset)
// another contiguous padding only packet should be dropped
params = &testutils.TestExtPacketParams{
SequenceNumber: 5,
SNCycles: 1,
Timestamp: 0xabcdef,
SSRC: 0x12345678,
}
extPkt, _ = testutils.GetTestExtPacket(params)
tpExpected = TranslationParamsRTP{
snOrdering: SequenceNumberOrderingContiguous,
}
tp, err = r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker)
require.ErrorIs(t, err, ErrPaddingOnlyPacket)
require.Equal(t, tpExpected, tp)
require.Equal(t, uint64(65536+5), r.extHighestIncomingSN)
require.Equal(t, uint64(65536+3), r.extLastSN)
snOffset, err = r.snRangeMap.GetValue(r.extHighestIncomingSN)
require.Error(t, err)
// a packet with a gap should be adjusting for dropped packets
params = &testutils.TestExtPacketParams{
SequenceNumber: 7,
SNCycles: 1,
Timestamp: 0xabcdef,
SSRC: 0x12345678,
PayloadSize: 22,
}
extPkt, _ = testutils.GetTestExtPacket(params)
tpExpected = TranslationParamsRTP{
snOrdering: SequenceNumberOrderingGap,
extSequenceNumber: 65536 + 5,
extTimestamp: 0xabcdef,
}
tp, err = r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker)
require.NoError(t, err)
require.Equal(t, tpExpected, tp)
require.Equal(t, uint64(65536+7), r.extHighestIncomingSN)
require.Equal(t, uint64(65536+5), r.extLastSN)
snOffset, err = r.snRangeMap.GetValue(r.extHighestIncomingSN)
require.NoError(t, err)
require.Equal(t, uint64(2), snOffset)
// ensure missing sequence number has correct cached offset
offset, err = r.snRangeMap.GetValue(65536 + 3)
require.NoError(t, err)
require.Equal(t, uint64(1), offset)
offset, err = r.snRangeMap.GetValue(65536 + 6)
require.NoError(t, err)
require.Equal(t, uint64(2), offset)
// check the missing packets
params = &testutils.TestExtPacketParams{
SequenceNumber: 6,
SNCycles: 1,
Timestamp: 0xabcdef,
SSRC: 0x12345678,
}
extPkt, _ = testutils.GetTestExtPacket(params)
tpExpected = TranslationParamsRTP{
snOrdering: SequenceNumberOrderingOutOfOrder,
extSequenceNumber: 65536 + 4,
extTimestamp: 0xabcdef,
}
tp, err = r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker)
require.NoError(t, err)
require.Equal(t, tpExpected, tp)
require.Equal(t, uint64(65536+7), r.extHighestIncomingSN)
require.Equal(t, uint64(65536+5), r.extLastSN)
snOffset, err = r.snRangeMap.GetValue(r.extHighestIncomingSN)
require.NoError(t, err)
require.Equal(t, uint64(2), snOffset)
params = &testutils.TestExtPacketParams{
SequenceNumber: 3,
SNCycles: 1,
Timestamp: 0xabcdef,
SSRC: 0x12345678,
}
extPkt, _ = testutils.GetTestExtPacket(params)
tpExpected = TranslationParamsRTP{
snOrdering: SequenceNumberOrderingOutOfOrder,
extSequenceNumber: 65536 + 2,
extTimestamp: 0xabcdef,
}
tp, err = r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker)
require.NoError(t, err)
require.Equal(t, tpExpected, tp)
require.Equal(t, uint64(65536+7), r.extHighestIncomingSN)
require.Equal(t, uint64(65536+5), r.extLastSN)
snOffset, err = r.snRangeMap.GetValue(r.extHighestIncomingSN)
require.NoError(t, err)
require.Equal(t, uint64(2), snOffset)
}
func TestUpdateAndGetPaddingSnTs(t *testing.T) {
r := newRTPMunger()
params := &testutils.TestExtPacketParams{
SequenceNumber: 23333,
Timestamp: 0xabcdef,
SSRC: 0x12345678,
PayloadSize: 20,
}
extPkt, _ := testutils.GetTestExtPacket(params)
r.SetLastSnTs(extPkt)
// getting padding without forcing marker should fail
_, err := r.UpdateAndGetPaddingSnTs(10, 10, 5, false, 0)
require.Error(t, err)
require.ErrorIs(t, err, ErrPaddingNotOnFrameBoundary)
// forcing a marker should not error out.
// And timestamp on first padding should be the same as the last one.
numPadding := 10
clockRate := uint64(10)
frameRate := uint64(5)
var sntsExpected = make([]SnTs, numPadding)
for i := 0; i < numPadding; i++ {
sntsExpected[i] = SnTs{
extSequenceNumber: uint64(params.SequenceNumber) + uint64(i) + 1,
extTimestamp: uint64(params.Timestamp) + ((uint64(i)*clockRate)+frameRate-1)/frameRate,
}
}
snts, err := r.UpdateAndGetPaddingSnTs(numPadding, uint32(clockRate), uint32(frameRate), true, extPkt.ExtTimestamp)
require.NoError(t, err)
require.Equal(t, sntsExpected, snts)
// now that there is a marker, timestamp should jump on first padding when asked again
for i := 0; i < numPadding; i++ {
sntsExpected[i] = SnTs{
extSequenceNumber: uint64(params.SequenceNumber) + uint64(len(snts)) + uint64(i) + 1,
extTimestamp: snts[len(snts)-1].extTimestamp + ((uint64(i+1)*clockRate)+frameRate-1)/frameRate,
}
}
snts, err = r.UpdateAndGetPaddingSnTs(numPadding, uint32(clockRate), uint32(frameRate), false, snts[len(snts)-1].extTimestamp)
require.NoError(t, err)
require.Equal(t, sntsExpected, snts)
}
func TestIsOnFrameBoundary(t *testing.T) {
r := newRTPMunger()
params := &testutils.TestExtPacketParams{
SequenceNumber: 23333,
Timestamp: 0xabcdef,
SSRC: 0x12345678,
PayloadSize: 20,
}
extPkt, _ := testutils.GetTestExtPacket(params)
r.SetLastSnTs(extPkt)
// send it through
_, err := r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker)
require.NoError(t, err)
require.False(t, r.IsOnFrameBoundary())
// packet with RTP marker
params = &testutils.TestExtPacketParams{
SetMarker: true,
SequenceNumber: 23334,
Timestamp: 0xabcdef,
SSRC: 0x12345678,
PayloadSize: 20,
}
extPkt, _ = testutils.GetTestExtPacket(params)
// send it through
_, err = r.UpdateAndGetSnTs(extPkt, extPkt.Packet.Marker)
require.NoError(t, err)
require.True(t, r.IsOnFrameBoundary())
}
livekit-server/pkg/sfu/rtpstats/rtpstats_base.go
+971
View File
@@ -0,0 +1,971 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package rtpstats
import (
"errors"
"time"
"go.uber.org/zap/zapcore"
"google.golang.org/protobuf/types/known/timestamppb"
"github.com/livekit/mediatransportutil"
"github.com/livekit/protocol/livekit"
"github.com/livekit/protocol/utils"
"github.com/livekit/protocol/utils/mono"
)
const (
cFirstPacketTimeAdjustWindow = 2 * time.Minute
cFirstPacketTimeAdjustThreshold = 15 * 1e9
cSequenceNumberLargeJumpThreshold = 100
)
// -------------------------------------------------------
type RTPDeltaInfo struct {
StartTime time.Time
EndTime time.Time
Packets uint32
Bytes uint64
HeaderBytes uint64
PacketsDuplicate uint32
BytesDuplicate uint64
HeaderBytesDuplicate uint64
PacketsPadding uint32
BytesPadding uint64
HeaderBytesPadding uint64
PacketsLost uint32
PacketsMissing uint32
PacketsOutOfOrder uint32
Frames uint32
RttMax uint32
JitterMax float64
Nacks uint32
NackRepeated uint32
Plis uint32
Firs uint32
}
func (r *RTPDeltaInfo) MarshalLogObject(e zapcore.ObjectEncoder) error {
if r == nil {
return nil
}
e.AddTime("StartTime", r.StartTime)
e.AddTime("EndTime", r.EndTime)
e.AddUint32("Packets", r.Packets)
e.AddUint64("Bytes", r.Bytes)
e.AddUint64("HeaderBytes", r.HeaderBytes)
e.AddUint32("PacketsDuplicate", r.PacketsDuplicate)
e.AddUint64("BytesDuplicate", r.BytesDuplicate)
e.AddUint64("HeaderBytesDuplicate", r.HeaderBytesDuplicate)
e.AddUint32("PacketsPadding", r.PacketsPadding)
e.AddUint64("BytesPadding", r.BytesPadding)
e.AddUint64("HeaderBytesPadding", r.HeaderBytesPadding)
e.AddUint32("PacketsLost", r.PacketsLost)
e.AddUint32("PacketsMissing", r.PacketsMissing)
e.AddUint32("PacketsOutOfOrder", r.PacketsOutOfOrder)
e.AddUint32("Frames", r.Frames)
e.AddUint32("RttMax", r.RttMax)
e.AddFloat64("JitterMax", r.JitterMax)
e.AddUint32("Nacks", r.Nacks)
e.AddUint32("NackRepeated", r.NackRepeated)
e.AddUint32("Plis", r.Plis)
e.AddUint32("Firs", r.Firs)
return nil
}
// -------------------------------------------------------
type snapshot struct {
snapshotLite
headerBytes uint64
packetsDuplicate uint64
bytesDuplicate uint64
headerBytesDuplicate uint64
packetsPadding uint64
bytesPadding uint64
headerBytesPadding uint64
frames uint32
plis uint32
firs uint32
maxRtt uint32
maxJitter float64
}
func (s *snapshot) MarshalLogObject(e zapcore.ObjectEncoder) error {
if s == nil {
return nil
}
e.AddObject("snapshotLite", &s.snapshotLite)
e.AddUint64("headerBytes", s.headerBytes)
e.AddUint64("packetsDuplicate", s.packetsDuplicate)
e.AddUint64("bytesDuplicate", s.bytesDuplicate)
e.AddUint64("headerBytesDuplicate", s.headerBytesDuplicate)
e.AddUint64("packetsPadding", s.packetsPadding)
e.AddUint64("bytesPadding", s.bytesPadding)
e.AddUint64("headerBytesPadding", s.headerBytesPadding)
e.AddUint32("frames", s.frames)
e.AddUint32("plis", s.plis)
e.AddUint32("firs", s.firs)
e.AddUint32("maxRtt", s.maxRtt)
e.AddFloat64("maxJitter", s.maxJitter)
return nil
}
func (s *snapshot) maybeUpdateMaxRTT(rtt uint32) {
if rtt > s.maxRtt {
s.maxRtt = rtt
}
}
func (s *snapshot) maybeUpdateMaxJitter(jitter float64) {
if jitter > s.maxJitter {
s.maxJitter = jitter
}
}
// ------------------------------------------------------------------
type wrappedRTPDriftLogger struct {
*livekit.RTPDrift
}
func (w wrappedRTPDriftLogger) MarshalLogObject(e zapcore.ObjectEncoder) error {
rd := w.RTPDrift
if rd == nil {
return nil
}
e.AddTime("StartTime", rd.StartTime.AsTime())
e.AddTime("EndTime", rd.EndTime.AsTime())
e.AddFloat64("Duration", rd.Duration)
e.AddUint64("StartTimestamp", rd.StartTimestamp)
e.AddUint64("EndTimestamp", rd.EndTimestamp)
e.AddUint64("RtpClockTicks", rd.RtpClockTicks)
e.AddInt64("DriftSamples", rd.DriftSamples)
e.AddFloat64("DriftMs", rd.DriftMs)
e.AddFloat64("ClockRate", rd.ClockRate)
return nil
}
// ------------------------------------------------------------------
type WrappedRTCPSenderReportStateLogger struct {
*livekit.RTCPSenderReportState
}
func (w WrappedRTCPSenderReportStateLogger) MarshalLogObject(e zapcore.ObjectEncoder) error {
rsrs := w.RTCPSenderReportState
if rsrs == nil {
return nil
}
e.AddUint32("RtpTimestamp", rsrs.RtpTimestamp)
e.AddUint64("RtpTimestampExt", rsrs.RtpTimestampExt)
e.AddTime("NtpTimestamp", mediatransportutil.NtpTime(rsrs.NtpTimestamp).Time())
e.AddTime("At", time.Unix(0, rsrs.At))
e.AddTime("AtAdjusted", time.Unix(0, rsrs.AtAdjusted))
e.AddUint32("Packets", rsrs.Packets)
e.AddUint64("Octets", rsrs.Octets)
return nil
}
func RTCPSenderReportPropagationDelay(rsrs *livekit.RTCPSenderReportState, passThrough bool) time.Duration {
if passThrough {
return 0
}
return time.Unix(0, rsrs.AtAdjusted).Sub(mediatransportutil.NtpTime(rsrs.NtpTimestamp).Time())
}
// ------------------------------------------------------------------
type rtpStatsBase struct {
*rtpStatsBaseLite
firstTime int64
firstTimeAdjustment time.Duration
highestTime int64
lastTransit uint64
lastJitterExtTimestamp uint64
headerBytes uint64
packetsDuplicate uint64
bytesDuplicate uint64
headerBytesDuplicate uint64
packetsPadding uint64
bytesPadding uint64
headerBytesPadding uint64
frames uint32
jitter float64
maxJitter float64
firs uint32
lastFir time.Time
keyFrames uint32
lastKeyFrame time.Time
rtt uint32
maxRtt uint32
srFirst *livekit.RTCPSenderReportState
srNewest *livekit.RTCPSenderReportState
nextSnapshotID uint32
snapshots []snapshot
}
func newRTPStatsBase(params RTPStatsParams) *rtpStatsBase {
return &rtpStatsBase{
rtpStatsBaseLite: newRTPStatsBaseLite(params),
nextSnapshotID: cFirstSnapshotID,
snapshots: make([]snapshot, 2),
}
}
func (r *rtpStatsBase) seed(from *rtpStatsBase) bool {
if !r.rtpStatsBaseLite.seed(from.rtpStatsBaseLite) {
return false
}
r.firstTime = from.firstTime
r.firstTimeAdjustment = from.firstTimeAdjustment
r.highestTime = from.highestTime
r.lastTransit = from.lastTransit
r.lastJitterExtTimestamp = from.lastJitterExtTimestamp
r.headerBytes = from.headerBytes
r.packetsDuplicate = from.packetsDuplicate
r.bytesDuplicate = from.bytesDuplicate
r.headerBytesDuplicate = from.headerBytesDuplicate
r.packetsPadding = from.packetsPadding
r.bytesPadding = from.bytesPadding
r.headerBytesPadding = from.headerBytesPadding
r.frames = from.frames
r.jitter = from.jitter
r.maxJitter = from.maxJitter
r.firs = from.firs
r.lastFir = from.lastFir
r.keyFrames = from.keyFrames
r.lastKeyFrame = from.lastKeyFrame
r.rtt = from.rtt
r.maxRtt = from.maxRtt
r.srFirst = utils.CloneProto(from.srFirst)
r.srNewest = utils.CloneProto(from.srNewest)
r.nextSnapshotID = from.nextSnapshotID
r.snapshots = make([]snapshot, cap(from.snapshots))
copy(r.snapshots, from.snapshots)
return true
}
func (r *rtpStatsBase) newSnapshotID(extStartSN uint64) uint32 {
id := r.nextSnapshotID
r.nextSnapshotID++
if cap(r.snapshots) < int(r.nextSnapshotID-cFirstSnapshotID) {
snapshots := make([]snapshot, r.nextSnapshotID-cFirstSnapshotID)
copy(snapshots, r.snapshots)
r.snapshots = snapshots
}
if r.initialized {
r.snapshots[id-cFirstSnapshotID] = initSnapshot(mono.UnixNano(), extStartSN)
}
return id
}
func (r *rtpStatsBase) UpdateFir(firCount uint32) {
r.lock.Lock()
defer r.lock.Unlock()
if r.endTime != 0 {
return
}
r.firs += firCount
}
func (r *rtpStatsBase) UpdateFirTime() {
r.lock.Lock()
defer r.lock.Unlock()
if r.endTime != 0 {
return
}
r.lastFir = time.Now()
}
func (r *rtpStatsBase) UpdateKeyFrame(kfCount uint32) {
r.lock.Lock()
defer r.lock.Unlock()
if r.endTime != 0 {
return
}
r.keyFrames += kfCount
r.lastKeyFrame = time.Now()
}
func (r *rtpStatsBase) UpdateRtt(rtt uint32) {
r.lock.Lock()
defer r.lock.Unlock()
if r.endTime != 0 {
return
}
r.rtt = rtt
if rtt > r.maxRtt {
r.maxRtt = rtt
}
for i := uint32(0); i < r.nextSnapshotID-cFirstSnapshotID; i++ {
s := &r.snapshots[i]
if rtt > s.maxRtt {
s.maxRtt = rtt
}
}
}
func (r *rtpStatsBase) GetRtt() uint32 {
r.lock.RLock()
defer r.lock.RUnlock()
return r.rtt
}
func (r *rtpStatsBase) maybeAdjustFirstPacketTime(srData *livekit.RTCPSenderReportState, tsOffset uint64, extStartTS uint64) (err error, loggingFields []interface{}) {
nowNano := mono.UnixNano()
if time.Duration(nowNano-r.startTime) > cFirstPacketTimeAdjustWindow {
return
}
// for some time after the start, adjust time of first packet.
// Helps improve accuracy of expected timestamp calculation.
// Adjusting only one way, i. e. if the first sample experienced
// abnormal delay (maybe due to pacing or maybe due to queuing
// in some network element along the way), push back first time
// to an earlier instance.
timeSinceReceive := time.Duration(nowNano - srData.AtAdjusted)
extNowTS := srData.RtpTimestampExt - tsOffset + uint64(timeSinceReceive.Nanoseconds()*int64(r.params.ClockRate)/1e9)
samplesDiff := int64(extNowTS - extStartTS)
if samplesDiff < 0 {
// out-of-order, skip
return
}
samplesDuration := time.Duration(float64(samplesDiff) / float64(r.params.ClockRate) * float64(time.Second))
timeSinceFirst := time.Duration(nowNano - r.firstTime)
now := r.firstTime + timeSinceFirst.Nanoseconds()
firstTime := now - samplesDuration.Nanoseconds()
getFields := func() []interface{} {
return []interface{}{
"startTime", time.Unix(0, r.startTime),
"nowTime", time.Unix(0, now),
"before", time.Unix(0, r.firstTime),
"after", time.Unix(0, firstTime),
"adjustment", time.Duration(r.firstTime - firstTime),
"extNowTS", extNowTS,
"extStartTS", extStartTS,
"srData", WrappedRTCPSenderReportStateLogger{srData},
"tsOffset", tsOffset,
"timeSinceReceive", timeSinceReceive,
"timeSinceFirst", timeSinceFirst,
"samplesDiff", samplesDiff,
"samplesDuration", samplesDuration,
}
}
if firstTime < r.firstTime {
if r.firstTime-firstTime > cFirstPacketTimeAdjustThreshold {
err = errors.New("adjusting first packet time, too big, ignoring")
loggingFields = getFields()
} else {
r.logger.Debugw("adjusting first packet time", getFields()...)
r.firstTimeAdjustment += time.Duration(r.firstTime - firstTime)
r.firstTime = firstTime
}
}
return
}
func (r *rtpStatsBase) getPacketsSeenMinusPadding(extStartSN, extHighestSN uint64) uint64 {
packetsSeen := r.getPacketsSeen(extStartSN, extHighestSN)
if r.packetsPadding > packetsSeen {
return 0
}
return packetsSeen - r.packetsPadding
}
func (r *rtpStatsBase) getPacketsSeenPlusDuplicates(extStartSN, extHighestSN uint64) uint64 {
return r.getPacketsSeen(extStartSN, extHighestSN) + r.packetsDuplicate
}
func (r *rtpStatsBase) deltaInfo(
snapshotID uint32,
extStartSN uint64,
extHighestSN uint64,
) (deltaInfo *RTPDeltaInfo, err error, loggingFields []interface{}) {
then, now := r.getAndResetSnapshot(snapshotID, extStartSN, extHighestSN)
if now == nil || then == nil {
return
}
startTime := then.startTime
endTime := now.startTime
packetsExpected := now.extStartSN - then.extStartSN
if then.extStartSN > extHighestSN {
packetsExpected = 0
}
if packetsExpected > cNumSequenceNumbers {
loggingFields = []interface{}{
"snapshotID", snapshotID,
"snapshotNow", now,
"snapshotThen", then,
"duration", time.Duration(endTime - startTime),
"packetsExpected", packetsExpected,
}
err = errors.New("too many packets expected in delta")
return
}
if packetsExpected == 0 {
deltaInfo = &RTPDeltaInfo{
StartTime: time.Unix(0, startTime),
EndTime: time.Unix(0, endTime),
}
return
}
packetsLost := uint32(now.packetsLost - then.packetsLost)
if int32(packetsLost) < 0 {
packetsLost = 0
}
// padding packets delta could be higher than expected due to out-of-order padding packets
packetsPadding := now.packetsPadding - then.packetsPadding
if packetsExpected < packetsPadding {
loggingFields = []interface{}{
"snapshotID", snapshotID,
"snapshotNow", now,
"snapshotThen", then,
"duration", time.Duration(endTime - startTime),
"packetsExpected", packetsExpected,
"packetsPadding", packetsPadding,
"packetsLost", packetsLost,
}
err = errors.New("padding packets more than expected")
packetsExpected = 0
} else {
packetsExpected -= packetsPadding
}
deltaInfo = &RTPDeltaInfo{
StartTime: time.Unix(0, startTime),
EndTime: time.Unix(0, endTime),
Packets: uint32(packetsExpected),
Bytes: now.bytes - then.bytes,
HeaderBytes: now.headerBytes - then.headerBytes,
PacketsDuplicate: uint32(now.packetsDuplicate - then.packetsDuplicate),
BytesDuplicate: now.bytesDuplicate - then.bytesDuplicate,
HeaderBytesDuplicate: now.headerBytesDuplicate - then.headerBytesDuplicate,
PacketsPadding: uint32(packetsPadding),
BytesPadding: now.bytesPadding - then.bytesPadding,
HeaderBytesPadding: now.headerBytesPadding - then.headerBytesPadding,
PacketsLost: packetsLost,
PacketsOutOfOrder: uint32(now.packetsOutOfOrder - then.packetsOutOfOrder),
Frames: now.frames - then.frames,
RttMax: then.maxRtt,
JitterMax: then.maxJitter / float64(r.params.ClockRate) * 1e6,
Nacks: now.nacks - then.nacks,
Plis: now.plis - then.plis,
Firs: now.firs - then.firs,
}
return
}
func (r *rtpStatsBase) marshalLogObject(
e zapcore.ObjectEncoder,
packetsExpected, packetsSeenMinusPadding uint64,
extStartTS, extHighestTS uint64,
) (float64, error) {
if r == nil {
return 0, nil
}
elapsedSeconds, err := r.rtpStatsBaseLite.marshalLogObject(e, packetsExpected, packetsSeenMinusPadding)
if err != nil {
return 0, err
}
e.AddTime("firstTime", time.Unix(0, r.firstTime))
e.AddDuration("firstTimeAdjustment", r.firstTimeAdjustment)
e.AddTime("highestTime", time.Unix(0, r.highestTime))
e.AddUint64("headerBytes", r.headerBytes)
e.AddUint64("packetsDuplicate", r.packetsDuplicate)
e.AddFloat64("packetsDuplicateRate", float64(r.packetsDuplicate)/elapsedSeconds)
e.AddUint64("bytesDuplicate", r.bytesDuplicate)
e.AddFloat64("bitrateDuplicate", float64(r.bytesDuplicate)*8.0/elapsedSeconds)
e.AddUint64("headerBytesDuplicate", r.headerBytesDuplicate)
e.AddUint64("packetsPadding", r.packetsPadding)
e.AddFloat64("packetsPaddingRate", float64(r.packetsPadding)/elapsedSeconds)
e.AddUint64("bytesPadding", r.bytesPadding)
e.AddFloat64("bitratePadding", float64(r.bytesPadding)*8.0/elapsedSeconds)
e.AddUint64("headerBytesPadding", r.headerBytesPadding)
e.AddUint32("frames", r.frames)
e.AddFloat64("frameRate", float64(r.frames)/elapsedSeconds)
e.AddFloat64("jitter", r.jitter)
e.AddFloat64("maxJitter", r.maxJitter)
e.AddUint32("firs", r.firs)
e.AddTime("lastFir", r.lastFir)
e.AddUint32("keyFrames", r.keyFrames)
e.AddTime("lastKeyFrame", r.lastKeyFrame)
e.AddUint32("rtt", r.rtt)
e.AddUint32("maxRtt", r.maxRtt)
e.AddObject("srFirst", WrappedRTCPSenderReportStateLogger{r.srFirst})
e.AddObject("srNewest", WrappedRTCPSenderReportStateLogger{r.srNewest})
packetDrift, ntpReportDrift, receivedReportDrift, rebasedReportDrift := r.getDrift(extStartTS, extHighestTS)
e.AddObject("packetDrift", wrappedRTPDriftLogger{packetDrift})
e.AddObject("ntpReportDrift", wrappedRTPDriftLogger{ntpReportDrift})
e.AddObject("receivedReportDrift", wrappedRTPDriftLogger{receivedReportDrift})
e.AddObject("rebasedReportDrift", wrappedRTPDriftLogger{rebasedReportDrift})
return elapsedSeconds, nil
}
func (r *rtpStatsBase) toProto(
packetsExpected, packetsSeenMinusPadding, packetsLost uint64,
extStartTS, extHighestTS uint64,
jitter, maxJitter float64,
) *livekit.RTPStats {
p := r.rtpStatsBaseLite.toProto(packetsExpected, packetsSeenMinusPadding, packetsLost)
if p == nil {
return nil
}
p.HeaderBytes = r.headerBytes
p.PacketsDuplicate = uint32(r.packetsDuplicate)
p.PacketDuplicateRate = float64(r.packetsDuplicate) / p.Duration
p.BytesDuplicate = r.bytesDuplicate
p.BitrateDuplicate = float64(r.bytesDuplicate) * 8.0 / p.Duration
p.HeaderBytesDuplicate = r.headerBytesDuplicate
p.PacketsPadding = uint32(r.packetsPadding)
p.PacketPaddingRate = float64(r.packetsPadding) / p.Duration
p.BytesPadding = r.bytesPadding
p.BitratePadding = float64(r.bytesPadding) * 8.0 / p.Duration
p.HeaderBytesPadding = r.headerBytesPadding
p.Frames = r.frames
p.FrameRate = float64(r.frames) / p.Duration
p.KeyFrames = r.keyFrames
p.LastKeyFrame = timestamppb.New(r.lastKeyFrame)
p.JitterCurrent = jitter / float64(r.params.ClockRate) * 1e6
p.JitterMax = maxJitter / float64(r.params.ClockRate) * 1e6
p.Firs = r.firs
p.LastFir = timestamppb.New(r.lastFir)
p.RttCurrent = r.rtt
p.RttMax = r.maxRtt
p.PacketDrift, p.NtpReportDrift, p.ReceivedReportDrift, p.RebasedReportDrift = r.getDrift(extStartTS, extHighestTS)
return p
}
func (r *rtpStatsBase) updateJitter(ets uint64, packetTime int64) float64 {
// Do not update jitter on multiple packets of same frame.
// All packets of a frame have the same time stamp.
// NOTE: This does not protect against using more than one packet of the same frame
// if packets arrive out-of-order. For example,
// p1f1 -> p1f2 -> p2f1
// In this case, p2f1 (packet 2, frame 1) will still be used in jitter calculation
// although it is the second packet of a frame because of out-of-order receival.
if r.lastJitterExtTimestamp != ets {
timeSinceFirst := packetTime - r.firstTime
packetTimeRTP := uint64(timeSinceFirst * int64(r.params.ClockRate) / 1e9)
transit := packetTimeRTP - ets
if r.lastTransit != 0 {
d := int64(transit - r.lastTransit)
if d < 0 {
d = -d
}
r.jitter += (float64(d) - r.jitter) / 16
if r.jitter > r.maxJitter {
r.maxJitter = r.jitter
}
for i := uint32(0); i < r.nextSnapshotID-cFirstSnapshotID; i++ {
r.snapshots[i].maybeUpdateMaxJitter(r.jitter)
}
}
r.lastTransit = transit
r.lastJitterExtTimestamp = ets
}
return r.jitter
}
func (r *rtpStatsBase) getAndResetSnapshot(snapshotID uint32, extStartSN uint64, extHighestSN uint64) (*snapshot, *snapshot) {
if !r.initialized {
return nil, nil
}
idx := snapshotID - cFirstSnapshotID
then := r.snapshots[idx]
if !then.isValid {
then = initSnapshot(r.startTime, extStartSN)
r.snapshots[idx] = then
}
// snapshot now
now := r.getSnapshot(mono.UnixNano(), extHighestSN+1)
r.snapshots[idx] = now
return &then, &now
}
func (r *rtpStatsBase) getDrift(extStartTS, extHighestTS uint64) (
packetDrift *livekit.RTPDrift,
ntpReportDrift *livekit.RTPDrift,
receivedReportDrift *livekit.RTPDrift,
rebasedReportDrift *livekit.RTPDrift,
) {
if r.firstTime != 0 {
elapsed := r.highestTime - r.firstTime
rtpClockTicks := extHighestTS - extStartTS
driftSamples := int64(rtpClockTicks - uint64(elapsed*int64(r.params.ClockRate)/1e9))
if elapsed > 0 {
elapsedSeconds := time.Duration(elapsed).Seconds()
packetDrift = &livekit.RTPDrift{
StartTime: timestamppb.New(time.Unix(0, r.firstTime)),
EndTime: timestamppb.New(time.Unix(0, r.highestTime)),
Duration: elapsedSeconds,
StartTimestamp: extStartTS,
EndTimestamp: extHighestTS,
RtpClockTicks: rtpClockTicks,
DriftSamples: driftSamples,
DriftMs: (float64(driftSamples) * 1000) / float64(r.params.ClockRate),
ClockRate: float64(rtpClockTicks) / elapsedSeconds,
}
}
}
if r.srFirst != nil && r.srNewest != nil && r.srFirst.RtpTimestamp != r.srNewest.RtpTimestamp {
rtpClockTicks := r.srNewest.RtpTimestampExt - r.srFirst.RtpTimestampExt
elapsed := mediatransportutil.NtpTime(r.srNewest.NtpTimestamp).Time().Sub(mediatransportutil.NtpTime(r.srFirst.NtpTimestamp).Time())
if elapsed.Seconds() > 0.0 {
driftSamples := int64(rtpClockTicks - uint64(elapsed.Nanoseconds()*int64(r.params.ClockRate)/1e9))
ntpReportDrift = &livekit.RTPDrift{
StartTime: timestamppb.New(mediatransportutil.NtpTime(r.srFirst.NtpTimestamp).Time()),
EndTime: timestamppb.New(mediatransportutil.NtpTime(r.srNewest.NtpTimestamp).Time()),
Duration: elapsed.Seconds(),
StartTimestamp: r.srFirst.RtpTimestampExt,
EndTimestamp: r.srNewest.RtpTimestampExt,
RtpClockTicks: rtpClockTicks,
DriftSamples: driftSamples,
DriftMs: (float64(driftSamples) * 1000) / float64(r.params.ClockRate),
ClockRate: float64(rtpClockTicks) / elapsed.Seconds(),
}
}
elapsed = time.Duration(r.srNewest.At - r.srFirst.At)
if elapsed.Seconds() > 0.0 {
driftSamples := int64(rtpClockTicks - uint64(elapsed.Nanoseconds()*int64(r.params.ClockRate)/1e9))
receivedReportDrift = &livekit.RTPDrift{
StartTime: timestamppb.New(time.Unix(0, r.srFirst.At)),
EndTime: timestamppb.New(time.Unix(0, r.srNewest.At)),
Duration: elapsed.Seconds(),
StartTimestamp: r.srFirst.RtpTimestampExt,
EndTimestamp: r.srNewest.RtpTimestampExt,
RtpClockTicks: rtpClockTicks,
DriftSamples: driftSamples,
DriftMs: (float64(driftSamples) * 1000) / float64(r.params.ClockRate),
ClockRate: float64(rtpClockTicks) / elapsed.Seconds(),
}
}
elapsed = time.Duration(r.srNewest.AtAdjusted - r.srFirst.AtAdjusted)
if elapsed.Seconds() > 0.0 {
driftSamples := int64(rtpClockTicks - uint64(elapsed.Nanoseconds()*int64(r.params.ClockRate)/1e9))
rebasedReportDrift = &livekit.RTPDrift{
StartTime: timestamppb.New(time.Unix(0, r.srFirst.AtAdjusted)),
EndTime: timestamppb.New(time.Unix(0, r.srNewest.AtAdjusted)),
Duration: elapsed.Seconds(),
StartTimestamp: r.srFirst.RtpTimestampExt,
EndTimestamp: r.srNewest.RtpTimestampExt,
RtpClockTicks: rtpClockTicks,
DriftSamples: driftSamples,
DriftMs: (float64(driftSamples) * 1000) / float64(r.params.ClockRate),
ClockRate: float64(rtpClockTicks) / elapsed.Seconds(),
}
}
}
return
}
func (r *rtpStatsBase) updateGapHistogram(gap int) {
if gap < 2 {
return
}
missing := gap - 1
if missing > len(r.gapHistogram) {
r.gapHistogram[len(r.gapHistogram)-1]++
} else {
r.gapHistogram[missing-1]++
}
}
func (r *rtpStatsBase) getSnapshot(startTime int64, extStartSN uint64) snapshot {
return snapshot{
snapshotLite: r.getSnapshotLite(startTime, extStartSN),
headerBytes: r.headerBytes,
packetsDuplicate: r.packetsDuplicate,
bytesDuplicate: r.bytesDuplicate,
headerBytesDuplicate: r.headerBytesDuplicate,
packetsPadding: r.packetsPadding,
bytesPadding: r.bytesPadding,
headerBytesPadding: r.headerBytesPadding,
frames: r.frames,
plis: r.plis,
firs: r.firs,
maxRtt: r.rtt,
maxJitter: r.jitter,
}
}
// ----------------------------------
func initSnapshot(startTime int64, extStartSN uint64) snapshot {
return snapshot{
snapshotLite: initSnapshotLite(startTime, extStartSN),
}
}
func AggregateRTPStats(statsList []*livekit.RTPStats) *livekit.RTPStats {
return utils.AggregateRTPStats(statsList, cGapHistogramNumBins)
}
func AggregateRTPDeltaInfo(deltaInfoList []*RTPDeltaInfo) *RTPDeltaInfo {
if len(deltaInfoList) == 0 {
return nil
}
startTime := int64(0)
endTime := int64(0)
packets := uint32(0)
bytes := uint64(0)
headerBytes := uint64(0)
packetsDuplicate := uint32(0)
bytesDuplicate := uint64(0)
headerBytesDuplicate := uint64(0)
packetsPadding := uint32(0)
bytesPadding := uint64(0)
headerBytesPadding := uint64(0)
packetsLost := uint32(0)
packetsMissing := uint32(0)
packetsOutOfOrder := uint32(0)
frames := uint32(0)
maxRtt := uint32(0)
maxJitter := float64(0)
nacks := uint32(0)
plis := uint32(0)
firs := uint32(0)
for _, deltaInfo := range deltaInfoList {
if deltaInfo == nil {
continue
}
if startTime == 0 || startTime > deltaInfo.StartTime.UnixNano() {
startTime = deltaInfo.StartTime.UnixNano()
}
if endTime == 0 || endTime < deltaInfo.EndTime.UnixNano() {
endTime = deltaInfo.EndTime.UnixNano()
}
packets += deltaInfo.Packets
bytes += deltaInfo.Bytes
headerBytes += deltaInfo.HeaderBytes
packetsDuplicate += deltaInfo.PacketsDuplicate
bytesDuplicate += deltaInfo.BytesDuplicate
headerBytesDuplicate += deltaInfo.HeaderBytesDuplicate
packetsPadding += deltaInfo.PacketsPadding
bytesPadding += deltaInfo.BytesPadding
headerBytesPadding += deltaInfo.HeaderBytesPadding
packetsLost += deltaInfo.PacketsLost
packetsMissing += deltaInfo.PacketsMissing
packetsOutOfOrder += deltaInfo.PacketsOutOfOrder
frames += deltaInfo.Frames
if deltaInfo.RttMax > maxRtt {
maxRtt = deltaInfo.RttMax
}
if deltaInfo.JitterMax > maxJitter {
maxJitter = deltaInfo.JitterMax
}
nacks += deltaInfo.Nacks
plis += deltaInfo.Plis
firs += deltaInfo.Firs
}
if startTime == 0 || endTime == 0 {
return nil
}
return &RTPDeltaInfo{
StartTime: time.Unix(0, startTime),
EndTime: time.Unix(0, endTime),
Packets: packets,
Bytes: bytes,
HeaderBytes: headerBytes,
PacketsDuplicate: packetsDuplicate,
BytesDuplicate: bytesDuplicate,
HeaderBytesDuplicate: headerBytesDuplicate,
PacketsPadding: packetsPadding,
BytesPadding: bytesPadding,
HeaderBytesPadding: headerBytesPadding,
PacketsLost: packetsLost,
PacketsMissing: packetsMissing,
PacketsOutOfOrder: packetsOutOfOrder,
Frames: frames,
RttMax: maxRtt,
JitterMax: maxJitter,
Nacks: nacks,
Plis: plis,
Firs: firs,
}
}
func ReconcileRTPStatsWithRTX(primaryStats *livekit.RTPStats, rtxStats *livekit.RTPStats) *livekit.RTPStats {
if primaryStats == nil || rtxStats == nil {
return primaryStats
}
primaryStats.PacketsDuplicate += rtxStats.Packets
primaryStats.PacketDuplicateRate = float64(primaryStats.PacketsDuplicate) / primaryStats.Duration
primaryStats.BytesDuplicate += rtxStats.Bytes
primaryStats.HeaderBytesDuplicate += rtxStats.HeaderBytes
primaryStats.BitrateDuplicate = float64(primaryStats.BytesDuplicate) * 8.0 / primaryStats.Duration
primaryStats.PacketsPadding += rtxStats.PacketsPadding
primaryStats.PacketPaddingRate = float64(primaryStats.PacketsPadding) / primaryStats.Duration
primaryStats.BytesPadding += rtxStats.BytesPadding
primaryStats.HeaderBytesPadding += rtxStats.HeaderBytesPadding
primaryStats.BitratePadding = float64(primaryStats.BytesPadding) * 8.0 / primaryStats.Duration
// RTX non-padding packets are responses to NACKs, that should discount packets lost,
lossAdjustment := rtxStats.Packets - rtxStats.PacketsLost - primaryStats.NackRepeated
if int32(lossAdjustment) < 0 {
lossAdjustment = 0
}
if lossAdjustment >= primaryStats.PacketsLost {
primaryStats.PacketsLost = 0
} else {
primaryStats.PacketsLost -= lossAdjustment
}
primaryStats.PacketLossRate = float64(primaryStats.PacketsLost) / primaryStats.Duration
primaryStats.PacketLossPercentage = float32(primaryStats.PacketsLost) / float32(primaryStats.Packets+primaryStats.PacketsPadding+primaryStats.PacketsLost) * 100.0
return primaryStats
}
func ReconcileRTPDeltaInfoWithRTX(primaryDeltaInfo *RTPDeltaInfo, rtxDeltaInfo *RTPDeltaInfo) *RTPDeltaInfo {
if primaryDeltaInfo == nil || rtxDeltaInfo == nil {
return primaryDeltaInfo
}
primaryDeltaInfo.PacketsDuplicate += rtxDeltaInfo.Packets
primaryDeltaInfo.BytesDuplicate += rtxDeltaInfo.Bytes
primaryDeltaInfo.HeaderBytesDuplicate += rtxDeltaInfo.HeaderBytes
primaryDeltaInfo.PacketsPadding += rtxDeltaInfo.PacketsPadding
primaryDeltaInfo.BytesPadding += rtxDeltaInfo.BytesPadding
primaryDeltaInfo.HeaderBytesPadding += rtxDeltaInfo.HeaderBytesPadding
// RTX non-padding packets are responses to NACKs, that should discount packets lost
lossAdjustment := rtxDeltaInfo.Packets - rtxDeltaInfo.PacketsLost - primaryDeltaInfo.NackRepeated
if int32(lossAdjustment) < 0 {
lossAdjustment = 0
}
if lossAdjustment >= primaryDeltaInfo.PacketsLost {
primaryDeltaInfo.PacketsLost = 0
} else {
primaryDeltaInfo.PacketsLost -= lossAdjustment
}
return primaryDeltaInfo
}
// -------------------------------------------------------------------
livekit-server/pkg/sfu/rtpstats/rtpstats_base_lite.go
+557
View File
@@ -0,0 +1,557 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package rtpstats
import (
"errors"
"fmt"
"sync"
"time"
"github.com/livekit/protocol/livekit"
"github.com/livekit/protocol/logger"
"github.com/livekit/protocol/utils/mono"
"go.uber.org/zap/zapcore"
"google.golang.org/protobuf/types/known/timestamppb"
)
const (
cGapHistogramNumBins = 101
cNumSequenceNumbers = 65536
cFirstSnapshotID = 1
)
// -------------------------------------------------------
type RTPDeltaInfoLite struct {
StartTime time.Time
EndTime time.Time
Packets uint32
Bytes uint64
PacketsLost uint32
PacketsOutOfOrder uint32
Nacks uint32
}
func (r *RTPDeltaInfoLite) MarshalLogObject(e zapcore.ObjectEncoder) error {
if r == nil {
return nil
}
e.AddTime("StartTime", r.StartTime)
e.AddTime("EndTime", r.EndTime)
e.AddUint32("Packets", r.Packets)
e.AddUint64("Bytes", r.Bytes)
e.AddUint32("PacketsLost", r.PacketsLost)
e.AddUint32("PacketsOutOfOrder", r.PacketsOutOfOrder)
e.AddUint32("Nacks", r.Nacks)
return nil
}
// -------------------------------------------------------
type snapshotLite struct {
isValid bool
startTime int64
extStartSN uint64
bytes uint64
packetsOutOfOrder uint64
packetsLost uint64
nacks uint32
}
func (s *snapshotLite) MarshalLogObject(e zapcore.ObjectEncoder) error {
if s == nil {
return nil
}
e.AddBool("isValid", s.isValid)
e.AddTime("startTime", time.Unix(0, s.startTime))
e.AddUint64("extStartSN", s.extStartSN)
e.AddUint64("bytes", s.bytes)
e.AddUint64("packetsOutOfOrder", s.packetsOutOfOrder)
e.AddUint64("packetsLost", s.packetsLost)
e.AddUint32("nacks", s.nacks)
return nil
}
// ------------------------------------------------------------------
type RTPStatsParams struct {
ClockRate uint32
IsRTX bool
Logger logger.Logger
}
type rtpStatsBaseLite struct {
params RTPStatsParams
logger logger.Logger
lock sync.RWMutex
initialized bool
startTime int64
endTime int64
bytes uint64
packetsOutOfOrder uint64
packetsLost uint64
gapHistogram [cGapHistogramNumBins]uint32
nacks uint32
nackAcks uint32
nackMisses uint32
nackRepeated uint32
plis uint32
lastPli int64
nextSnapshotLiteID uint32
snapshotLites []snapshotLite
}
func newRTPStatsBaseLite(params RTPStatsParams) *rtpStatsBaseLite {
return &rtpStatsBaseLite{
params: params,
logger: params.Logger,
nextSnapshotLiteID: cFirstSnapshotID,
snapshotLites: make([]snapshotLite, 2),
}
}
func (r *rtpStatsBaseLite) seed(from *rtpStatsBaseLite) bool {
if from == nil || !from.initialized || r.initialized {
return false
}
r.initialized = from.initialized
r.startTime = from.startTime
// do not clone endTime as a non-zero endTime indicates an ended object
r.bytes = from.bytes
r.packetsOutOfOrder = from.packetsOutOfOrder
r.packetsLost = from.packetsLost
r.gapHistogram = from.gapHistogram
r.nacks = from.nacks
r.nackAcks = from.nackAcks
r.nackMisses = from.nackMisses
r.nackRepeated = from.nackRepeated
r.plis = from.plis
r.lastPli = from.lastPli
r.nextSnapshotLiteID = from.nextSnapshotLiteID
r.snapshotLites = make([]snapshotLite, cap(from.snapshotLites))
copy(r.snapshotLites, from.snapshotLites)
return true
}
func (r *rtpStatsBaseLite) SetLogger(logger logger.Logger) {
r.logger = logger
}
func (r *rtpStatsBaseLite) Stop() {
r.lock.Lock()
defer r.lock.Unlock()
r.endTime = mono.UnixNano()
}
func (r *rtpStatsBaseLite) newSnapshotLiteID(extStartSN uint64) uint32 {
id := r.nextSnapshotLiteID
r.nextSnapshotLiteID++
if cap(r.snapshotLites) < int(r.nextSnapshotLiteID-cFirstSnapshotID) {
snapshotLites := make([]snapshotLite, r.nextSnapshotLiteID-cFirstSnapshotID)
copy(snapshotLites, r.snapshotLites)
r.snapshotLites = snapshotLites
}
if r.initialized {
r.snapshotLites[id-cFirstSnapshotID] = initSnapshotLite(mono.UnixNano(), extStartSN)
}
return id
}
func (r *rtpStatsBaseLite) IsActive() bool {
r.lock.RLock()
defer r.lock.RUnlock()
return r.initialized && r.endTime == 0
}
func (r *rtpStatsBaseLite) UpdateNack(nackCount uint32) {
r.lock.Lock()
defer r.lock.Unlock()
if r.endTime != 0 {
return
}
r.nacks += nackCount
}
func (r *rtpStatsBaseLite) UpdateNackProcessed(nackAckCount uint32, nackMissCount uint32, nackRepeatedCount uint32) {
r.lock.Lock()
defer r.lock.Unlock()
if r.endTime != 0 {
return
}
r.nackAcks += nackAckCount
r.nackMisses += nackMissCount
r.nackRepeated += nackRepeatedCount
}
func (r *rtpStatsBaseLite) CheckAndUpdatePli(throttle int64, force bool) bool {
r.lock.Lock()
defer r.lock.Unlock()
if r.endTime != 0 || (!force && mono.UnixNano()-r.lastPli < throttle) {
return false
}
r.updatePliLocked(1)
r.updatePliTimeLocked()
return true
}
func (r *rtpStatsBaseLite) UpdatePliAndTime(pliCount uint32) {
r.lock.Lock()
defer r.lock.Unlock()
if r.endTime != 0 {
return
}
r.updatePliLocked(pliCount)
r.updatePliTimeLocked()
}
func (r *rtpStatsBaseLite) UpdatePli(pliCount uint32) {
r.lock.Lock()
defer r.lock.Unlock()
if r.endTime != 0 {
return
}
r.updatePliLocked(pliCount)
}
func (r *rtpStatsBaseLite) updatePliLocked(pliCount uint32) {
r.plis += pliCount
}
func (r *rtpStatsBaseLite) UpdatePliTime() {
r.lock.Lock()
defer r.lock.Unlock()
if r.endTime != 0 {
return
}
r.updatePliTimeLocked()
}
func (r *rtpStatsBaseLite) updatePliTimeLocked() {
r.lastPli = mono.UnixNano()
}
func (r *rtpStatsBaseLite) LastPli() int64 {
r.lock.RLock()
defer r.lock.RUnlock()
return r.lastPli
}
func (r *rtpStatsBaseLite) getPacketsSeen(extStartSN, extHighestSN uint64) uint64 {
packetsExpected := getPacketsExpected(extStartSN, extHighestSN)
if r.packetsLost > packetsExpected {
// should not happen
return 0
}
return packetsExpected - r.packetsLost
}
func (r *rtpStatsBaseLite) deltaInfoLite(
snapshotLiteID uint32,
extStartSN uint64,
extHighestSN uint64,
) (deltaInfoLite *RTPDeltaInfoLite, err error, loggingFields []interface{}) {
then, now := r.getAndResetSnapshotLite(snapshotLiteID, extStartSN, extHighestSN)
if now == nil || then == nil {
return
}
startTime := then.startTime
endTime := now.startTime
packetsExpected := uint32(now.extStartSN - then.extStartSN)
if then.extStartSN > extHighestSN {
packetsExpected = 0
}
if packetsExpected > cNumSequenceNumbers {
loggingFields = []interface{}{
"snapshotLiteID", snapshotLiteID,
"snapshotLiteNow", now,
"snapshotLiteThen", then,
"packetsExpected", packetsExpected,
"duration", time.Duration(endTime - startTime),
}
err = errors.New("too many packets expected in delta lite")
return
}
if packetsExpected == 0 {
deltaInfoLite = &RTPDeltaInfoLite{
StartTime: time.Unix(0, startTime),
EndTime: time.Unix(0, endTime),
}
return
}
packetsLost := uint32(now.packetsLost - then.packetsLost)
if int32(packetsLost) < 0 {
packetsLost = 0
}
if packetsLost > packetsExpected {
loggingFields = []interface{}{
"snapshotLiteID", snapshotLiteID,
"snapshotLiteNow", now,
"snapshotLiteThen", then,
"packetsExpected", packetsExpected,
"packetsLost", packetsLost,
"duration", time.Duration(endTime - startTime),
}
err = errors.New("unexpected number of packets lost in delta lite")
}
deltaInfoLite = &RTPDeltaInfoLite{
StartTime: time.Unix(0, startTime),
EndTime: time.Unix(0, endTime),
Packets: packetsExpected,
Bytes: now.bytes - then.bytes,
PacketsLost: packetsLost,
PacketsOutOfOrder: uint32(now.packetsOutOfOrder - then.packetsOutOfOrder),
Nacks: now.nacks - then.nacks,
}
return
}
func (r *rtpStatsBaseLite) marshalLogObject(e zapcore.ObjectEncoder, packetsExpected, packetsSeenMinusPadding uint64) (float64, error) {
if r == nil || !r.initialized {
return 0, errors.New("not initialized")
}
endTime := r.endTime
if endTime == 0 {
endTime = mono.UnixNano()
}
elapsed := time.Duration(endTime - r.startTime)
if elapsed == 0 {
return 0, errors.New("no time elapsed")
}
elapsedSeconds := elapsed.Seconds()
e.AddTime("startTime", time.Unix(0, r.startTime))
e.AddTime("endTime", time.Unix(0, r.endTime))
e.AddDuration("elapsed", elapsed)
e.AddUint64("packetsExpected", packetsExpected)
e.AddFloat64("packetsExpectedRate", float64(packetsExpected)/elapsedSeconds)
e.AddUint64("packetsSeenPrimary", packetsSeenMinusPadding)
e.AddFloat64("packetsSeenPrimaryRate", float64(packetsSeenMinusPadding)/elapsedSeconds)
e.AddUint64("bytes", r.bytes)
e.AddFloat64("bitrate", float64(r.bytes)*8.0/elapsedSeconds)
e.AddUint64("packetsOutOfOrder", r.packetsOutOfOrder)
e.AddUint64("packetsLost", r.packetsLost)
e.AddFloat64("packetsLostRate", float64(r.packetsLost)/elapsedSeconds)
if packetsExpected != 0 {
e.AddFloat32("packetLostPercentage", float32(r.packetsLost)/float32(packetsExpected)*100.0)
}
hasLoss := false
first := true
str := "["
for burst, count := range r.gapHistogram {
if count == 0 {
continue
}
hasLoss = true
if !first {
str += ", "
}
first = false
str += fmt.Sprintf("%d:%d", burst+1, count)
}
str += "]"
if hasLoss {
e.AddString("gapHistogram", str)
}
e.AddUint32("nacks", r.nacks)
e.AddUint32("nackAcks", r.nackAcks)
e.AddUint32("nackMisses", r.nackMisses)
e.AddUint32("nackRepeated", r.nackRepeated)
e.AddUint32("plis", r.plis)
e.AddTime("lastPli", time.Unix(0, r.lastPli))
return elapsedSeconds, nil
}
func (r *rtpStatsBaseLite) toProto(packetsExpected, packetsSeenMinusPadding, packetsLost uint64) *livekit.RTPStats {
if r.startTime == 0 {
return nil
}
endTime := r.endTime
if endTime == 0 {
endTime = mono.UnixNano()
}
elapsed := time.Duration(endTime - r.startTime).Seconds()
if elapsed == 0.0 {
return nil
}
packetRate := float64(packetsSeenMinusPadding) / elapsed
bitrate := float64(r.bytes) * 8.0 / elapsed
packetLostRate := float64(packetsLost) / elapsed
packetLostPercentage := float32(0)
if packetsExpected != 0 {
packetLostPercentage = float32(packetsLost) / float32(packetsExpected) * 100.0
}
p := &livekit.RTPStats{
StartTime: timestamppb.New(time.Unix(0, r.startTime)),
EndTime: timestamppb.New(time.Unix(0, endTime)),
Duration: elapsed,
Packets: uint32(packetsSeenMinusPadding),
PacketRate: packetRate,
Bytes: r.bytes,
Bitrate: bitrate,
PacketsLost: uint32(packetsLost),
PacketLossRate: packetLostRate,
PacketLossPercentage: packetLostPercentage,
PacketsOutOfOrder: uint32(r.packetsOutOfOrder),
Nacks: r.nacks,
NackAcks: r.nackAcks,
NackMisses: r.nackMisses,
NackRepeated: r.nackRepeated,
Plis: r.plis,
LastPli: timestamppb.New(time.Unix(0, r.lastPli)),
}
gapsPresent := false
for i := 0; i < len(r.gapHistogram); i++ {
if r.gapHistogram[i] == 0 {
continue
}
gapsPresent = true
break
}
if gapsPresent {
p.GapHistogram = make(map[int32]uint32, len(r.gapHistogram))
for i := 0; i < len(r.gapHistogram); i++ {
if r.gapHistogram[i] == 0 {
continue
}
p.GapHistogram[int32(i+1)] = r.gapHistogram[i]
}
}
return p
}
func (r *rtpStatsBaseLite) getAndResetSnapshotLite(snapshotLiteID uint32, extStartSN uint64, extHighestSN uint64) (*snapshotLite, *snapshotLite) {
if !r.initialized {
return nil, nil
}
idx := snapshotLiteID - cFirstSnapshotID
then := r.snapshotLites[idx]
if !then.isValid {
then = initSnapshotLite(r.startTime, extStartSN)
r.snapshotLites[idx] = then
}
// snapshot now
now := r.getSnapshotLite(mono.UnixNano(), extHighestSN+1)
r.snapshotLites[idx] = now
return &then, &now
}
func (r *rtpStatsBaseLite) updateGapHistogram(gap int) {
if gap < 2 {
return
}
missing := gap - 1
if missing > len(r.gapHistogram) {
r.gapHistogram[len(r.gapHistogram)-1]++
} else {
r.gapHistogram[missing-1]++
}
}
func (r *rtpStatsBaseLite) getSnapshotLite(startTime int64, extStartSN uint64) snapshotLite {
return snapshotLite{
isValid: true,
startTime: startTime,
extStartSN: extStartSN,
bytes: r.bytes,
packetsOutOfOrder: r.packetsOutOfOrder,
packetsLost: r.packetsLost,
nacks: r.nacks,
}
}
// ----------------------------------
func initSnapshotLite(startTime int64, extStartSN uint64) snapshotLite {
return snapshotLite{
isValid: true,
startTime: startTime,
extStartSN: extStartSN,
}
}
func getPacketsExpected(extStartSN, extHighestSN uint64) uint64 {
return extHighestSN - extStartSN + 1
}
// ----------------------------------
livekit-server/pkg/sfu/rtpstats/rtpstats_receiver.go
+731
View File
@@ -0,0 +1,731 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package rtpstats
import (
"fmt"
"math"
"time"
"github.com/pion/rtcp"
"go.uber.org/zap/zapcore"
"github.com/livekit/livekit-server/pkg/sfu/utils"
"github.com/livekit/mediatransportutil"
"github.com/livekit/protocol/livekit"
"github.com/livekit/protocol/logger"
protoutils "github.com/livekit/protocol/utils"
"github.com/livekit/protocol/utils/mono"
)
const (
cHistorySize = 8192
// number of seconds the current report RTP timestamp can be off from expected RTP timestamp
cReportSlack = float64(60.0)
cTSJumpTooHighFactor = float64(1.5)
)
// ---------------------------------------------------------------------
type RTPFlowState struct {
IsNotHandled bool
LossStartInclusive uint64
LossEndExclusive uint64
IsDuplicate bool
IsOutOfOrder bool
ExtSequenceNumber uint64
ExtTimestamp uint64
}
func (r *RTPFlowState) MarshalLogObject(e zapcore.ObjectEncoder) error {
if r == nil {
return nil
}
e.AddBool("IsNotHandled", r.IsNotHandled)
e.AddUint64("LossStartInclusive", r.LossStartInclusive)
e.AddUint64("LossEndExclusive", r.LossEndExclusive)
e.AddBool("IsDuplicate", r.IsDuplicate)
e.AddBool("IsOutOfOrder", r.IsOutOfOrder)
e.AddUint64("ExtSequenceNumber", r.ExtSequenceNumber)
e.AddUint64("ExtTimestamp", r.ExtTimestamp)
return nil
}
// ---------------------------------------------------------------------
type RTPStatsReceiver struct {
*rtpStatsBase
sequenceNumber *utils.WrapAround[uint16, uint64]
tsRolloverThreshold int64
timestamp *utils.WrapAround[uint32, uint64]
history *protoutils.Bitmap[uint64]
propagationDelayEstimator *utils.OWDEstimator
clockSkewCount int
clockSkewMediaPathCount int
outOfOrderSenderReportCount int
largeJumpCount int
largeJumpNegativeCount int
timeReversedCount int
}
func NewRTPStatsReceiver(params RTPStatsParams) *RTPStatsReceiver {
return &RTPStatsReceiver{
rtpStatsBase: newRTPStatsBase(params),
sequenceNumber: utils.NewWrapAround[uint16, uint64](utils.WrapAroundParams{IsRestartAllowed: false}),
tsRolloverThreshold: (1 << 31) * 1e9 / int64(params.ClockRate),
timestamp: utils.NewWrapAround[uint32, uint64](utils.WrapAroundParams{IsRestartAllowed: false}),
history: protoutils.NewBitmap[uint64](cHistorySize),
propagationDelayEstimator: utils.NewOWDEstimator(utils.OWDEstimatorParamsDefault),
}
}
func (r *RTPStatsReceiver) NewSnapshotId() uint32 {
r.lock.Lock()
defer r.lock.Unlock()
return r.newSnapshotID(r.sequenceNumber.GetExtendedHighest())
}
func (r *RTPStatsReceiver) getTSRolloverCount(diffNano int64, ts uint32) int {
if diffNano < r.tsRolloverThreshold {
// time not more than rollover threshold
return -1
}
excess := (diffNano - r.tsRolloverThreshold*2) * int64(r.params.ClockRate) / 1e9
roc := excess / (1 << 32)
if roc < 0 {
roc = 0
}
if r.timestamp.GetHighest() > ts {
roc++
}
return int(roc)
}
func (r *RTPStatsReceiver) Update(
packetTime int64,
sequenceNumber uint16,
timestamp uint32,
marker bool,
hdrSize int,
payloadSize int,
paddingSize int,
) (flowState RTPFlowState) {
r.lock.Lock()
defer r.lock.Unlock()
if r.endTime != 0 {
flowState.IsNotHandled = true
return
}
var resSN utils.WrapAroundUpdateResult[uint64]
var gapSN int64
var resTS utils.WrapAroundUpdateResult[uint64]
var gapTS int64
var expectedTSJump int64
var timeSinceHighest int64
var tsRolloverCount int
var snRolloverCount int
logger := func() logger.UnlikelyLogger {
return r.logger.WithUnlikelyValues(
"resSN", resSN,
"gapSN", gapSN,
"resTS", resTS,
"gapTS", gapTS,
"snRolloverCount", snRolloverCount,
"expectedTSJump", expectedTSJump,
"tsRolloverCount", tsRolloverCount,
"packetTime", time.Unix(0, packetTime),
"timeSinceHighest", time.Duration(timeSinceHighest),
"sequenceNumber", sequenceNumber,
"timestamp", timestamp,
"marker", marker,
"hdrSize", hdrSize,
"payloadSize", payloadSize,
"paddingSize", paddingSize,
"rtpStats", lockedRTPStatsReceiverLogEncoder{r},
)
}
if !r.initialized {
if payloadSize == 0 {
// do not start on a padding only packet
flowState.IsNotHandled = true
return
}
r.initialized = true
r.startTime = mono.UnixNano()
r.firstTime = packetTime
r.highestTime = packetTime
resSN = r.sequenceNumber.Update(sequenceNumber)
resTS = r.timestamp.Update(timestamp)
// initialize snapshots if any
for i := uint32(0); i < r.nextSnapshotID-cFirstSnapshotID; i++ {
r.snapshots[i] = initSnapshot(r.startTime, r.sequenceNumber.GetExtendedStart())
}
r.logger.Debugw(
"rtp receiver stream start",
"rtpStats", lockedRTPStatsReceiverLogEncoder{r},
)
} else {
resSN = r.sequenceNumber.Update(sequenceNumber)
if resSN.IsUnhandled {
flowState.IsNotHandled = true
return
}
gapSN = int64(resSN.ExtendedVal - resSN.PreExtendedHighest)
timeSinceHighest = packetTime - r.highestTime
tsRolloverCount = r.getTSRolloverCount(timeSinceHighest, timestamp)
if tsRolloverCount >= 0 {
logger().Warnw("potential time stamp roll over", nil)
}
resTS = r.timestamp.Rollover(timestamp, tsRolloverCount)
if resTS.IsUnhandled {
flowState.IsNotHandled = true
return
}
gapTS = int64(resTS.ExtendedVal - resTS.PreExtendedHighest)
// it is possible to receive old packets in two different scenarios
// as it is not possible to detect how far to roll back, ignore old packets
//
// Case 1:
// Very old time stamp, happens under the following conditions
// - resume after long mute, big time stamp jump
// - an out of order packet from before the mute arrives (unsure what causes this
// very old packet to be trasmitted from remote), causing time stamp to jump back
// to before mute, but it appears like it has rolled over.
// Use a threshold against expected to ignore these.
if gapSN < 0 && gapTS > 0 {
expectedTSJump = timeSinceHighest * int64(r.params.ClockRate) / 1e9
if gapTS > int64(float64(expectedTSJump)*cTSJumpTooHighFactor) {
r.sequenceNumber.UndoUpdate(resSN)
r.timestamp.UndoUpdate(resTS)
logger().Warnw("dropping old packet, timestamp", nil)
flowState.IsNotHandled = true
return
}
}
// Case 2:
// Sequence number looks like it is moving forward, but it is actually a very old packet.
if gapTS < 0 && gapSN > 0 {
r.sequenceNumber.UndoUpdate(resSN)
r.timestamp.UndoUpdate(resTS)
logger().Warnw("dropping old packet, sequence number", nil)
flowState.IsNotHandled = true
return
}
// it is possible that sequence number has rolled over too
if gapSN < 0 && gapTS > 0 && payloadSize > 0 {
// not possible to know how many cycles of sequence number roll over could have happened,
// ensure that it at least does not go backwards
snRolloverCount = 0
if sequenceNumber < r.sequenceNumber.GetHighest() {
snRolloverCount = 1
}
resSN = r.sequenceNumber.Rollover(sequenceNumber, snRolloverCount)
if resSN.IsUnhandled {
flowState.IsNotHandled = true
return
}
logger().Warnw("forcing sequence number rollover", nil)
}
}
gapSN = int64(resSN.ExtendedVal - resSN.PreExtendedHighest)
pktSize := uint64(hdrSize + payloadSize + paddingSize)
if gapSN <= 0 { // duplicate OR out-of-order
if gapSN != 0 {
r.packetsOutOfOrder++
}
if r.isInRange(resSN.ExtendedVal, resSN.PreExtendedHighest) {
if r.history.GetAndSet(resSN.ExtendedVal) {
r.bytesDuplicate += pktSize
r.headerBytesDuplicate += uint64(hdrSize)
r.packetsDuplicate++
flowState.IsDuplicate = true
} else {
r.packetsLost--
}
}
flowState.IsOutOfOrder = true
if !flowState.IsDuplicate && -gapSN >= cSequenceNumberLargeJumpThreshold {
r.largeJumpNegativeCount++
if (r.largeJumpNegativeCount-1)%100 == 0 {
logger().Warnw(
"large sequence number gap negative", nil,
"count", r.largeJumpNegativeCount,
)
}
}
} else { // in-order
if gapSN >= cSequenceNumberLargeJumpThreshold {
r.largeJumpCount++
if (r.largeJumpCount-1)%100 == 0 {
logger().Warnw(
"large sequence number gap", nil,
"count", r.largeJumpCount,
)
}
}
if resTS.ExtendedVal < resTS.PreExtendedHighest {
r.timeReversedCount++
if (r.timeReversedCount-1)%100 == 0 {
logger().Warnw(
"time reversed", nil,
"count", r.timeReversedCount,
)
}
}
// update gap histogram
r.updateGapHistogram(int(gapSN))
// update missing sequence numbers
r.history.ClearRange(resSN.PreExtendedHighest+1, resSN.ExtendedVal-1)
r.packetsLost += uint64(gapSN - 1)
r.history.Set(resSN.ExtendedVal)
if timestamp != uint32(resTS.PreExtendedHighest) {
// update only on first packet as same timestamp could be in multiple packets.
// NOTE: this may not be the first packet with this time stamp if there is packet loss.
r.highestTime = packetTime
}
flowState.LossStartInclusive = resSN.PreExtendedHighest + 1
flowState.LossEndExclusive = resSN.ExtendedVal
}
flowState.ExtSequenceNumber = resSN.ExtendedVal
flowState.ExtTimestamp = resTS.ExtendedVal
if !flowState.IsDuplicate {
if payloadSize == 0 {
r.packetsPadding++
r.bytesPadding += pktSize
r.headerBytesPadding += uint64(hdrSize)
} else {
r.bytes += pktSize
r.headerBytes += uint64(hdrSize)
if marker {
r.frames++
}
r.updateJitter(resTS.ExtendedVal, packetTime)
}
}
return
}
func (r *RTPStatsReceiver) getExtendedSenderReport(srData *livekit.RTCPSenderReportState) *livekit.RTCPSenderReportState {
tsCycles := uint64(0)
if r.srNewest != nil {
// use time since last sender report to ensure long gaps where the time stamp might
// jump more than half the range
timeSinceLastReport := mediatransportutil.NtpTime(srData.NtpTimestamp).Time().Sub(mediatransportutil.NtpTime(r.srNewest.NtpTimestamp).Time())
expectedRTPTimestampExt := r.srNewest.RtpTimestampExt + uint64(timeSinceLastReport.Nanoseconds()*int64(r.params.ClockRate)/1e9)
lbound := expectedRTPTimestampExt - uint64(cReportSlack*float64(r.params.ClockRate))
ubound := expectedRTPTimestampExt + uint64(cReportSlack*float64(r.params.ClockRate))
isInRange := (srData.RtpTimestamp-uint32(lbound) < (1 << 31)) && (uint32(ubound)-srData.RtpTimestamp < (1 << 31))
if isInRange {
lbTSCycles := lbound & 0xFFFF_FFFF_0000_0000
ubTSCycles := ubound & 0xFFFF_FFFF_0000_0000
if lbTSCycles == ubTSCycles {
tsCycles = lbTSCycles
} else {
if srData.RtpTimestamp < (1 << 31) {
// rolled over
tsCycles = ubTSCycles
} else {
tsCycles = lbTSCycles
}
}
} else {
// ideally this method should not be required, but there are clients
// negotiating one clock rate, but actually send media at a different rate.
tsCycles = r.srNewest.RtpTimestampExt & 0xFFFF_FFFF_0000_0000
if (srData.RtpTimestamp-r.srNewest.RtpTimestamp) < (1<<31) && srData.RtpTimestamp < r.srNewest.RtpTimestamp {
tsCycles += (1 << 32)
}
if tsCycles >= (1 << 32) {
if (srData.RtpTimestamp-r.srNewest.RtpTimestamp) >= (1<<31) && srData.RtpTimestamp > r.srNewest.RtpTimestamp {
tsCycles -= (1 << 32)
}
}
}
}
srDataExt := protoutils.CloneProto(srData)
srDataExt.RtpTimestampExt = uint64(srDataExt.RtpTimestamp) + tsCycles
return srDataExt
}
func (r *RTPStatsReceiver) checkOutOfOrderSenderReport(srData *livekit.RTCPSenderReportState) bool {
if r.srNewest != nil && srData.RtpTimestampExt < r.srNewest.RtpTimestampExt {
// This can happen when a track is replaced with a null and then restored -
// i. e. muting replacing with null and unmute restoring the original track.
// Or it could be due bad report generation.
// In any case, ignore out-of-order reports.
r.outOfOrderSenderReportCount++
if (r.outOfOrderSenderReportCount-1)%10 == 0 {
r.logger.Infow(
"received sender report, out-of-order, skipping",
"current", WrappedRTCPSenderReportStateLogger{srData},
"count", r.outOfOrderSenderReportCount,
"rtpStats", lockedRTPStatsReceiverLogEncoder{r},
)
}
return true
}
return false
}
func (r *RTPStatsReceiver) checkRTPClockSkewForSenderReport(srData *livekit.RTCPSenderReportState) {
if r.srNewest == nil {
return
}
timeSinceLast := mediatransportutil.NtpTime(srData.NtpTimestamp).Time().Sub(mediatransportutil.NtpTime(r.srNewest.NtpTimestamp).Time()).Seconds()
rtpDiffSinceLast := srData.RtpTimestampExt - r.srNewest.RtpTimestampExt
calculatedClockRateFromLast := float64(rtpDiffSinceLast) / timeSinceLast
timeSinceFirst := mediatransportutil.NtpTime(srData.NtpTimestamp).Time().Sub(mediatransportutil.NtpTime(r.srFirst.NtpTimestamp).Time()).Seconds()
rtpDiffSinceFirst := srData.RtpTimestampExt - r.srFirst.RtpTimestampExt
calculatedClockRateFromFirst := float64(rtpDiffSinceFirst) / timeSinceFirst
if (timeSinceLast > 0.2 && math.Abs(float64(r.params.ClockRate)-calculatedClockRateFromLast) > 0.2*float64(r.params.ClockRate)) ||
(timeSinceFirst > 0.2 && math.Abs(float64(r.params.ClockRate)-calculatedClockRateFromFirst) > 0.2*float64(r.params.ClockRate)) {
r.clockSkewCount++
if (r.clockSkewCount-1)%100 == 0 {
r.logger.Infow(
"received sender report, clock skew",
"current", WrappedRTCPSenderReportStateLogger{srData},
"timeSinceFirst", timeSinceFirst,
"rtpDiffSinceFirst", rtpDiffSinceFirst,
"calculatedFirst", calculatedClockRateFromFirst,
"timeSinceLast", timeSinceLast,
"rtpDiffSinceLast", rtpDiffSinceLast,
"calculatedLast", calculatedClockRateFromLast,
"count", r.clockSkewCount,
"rtpStats", lockedRTPStatsReceiverLogEncoder{r},
)
}
}
}
func (r *RTPStatsReceiver) checkRTPClockSkewAgainstMediaPathForSenderReport(srData *livekit.RTCPSenderReportState) {
if r.highestTime == 0 {
return
}
nowNano := mono.UnixNano()
timeSinceSR := time.Duration(nowNano - srData.AtAdjusted)
extNowTSSR := srData.RtpTimestampExt + uint64(timeSinceSR.Nanoseconds()*int64(r.params.ClockRate)/1e9)
timeSinceHighest := time.Duration(nowNano - r.highestTime)
extNowTSHighest := r.timestamp.GetExtendedHighest() + uint64(timeSinceHighest.Nanoseconds()*int64(r.params.ClockRate)/1e9)
diffHighest := extNowTSSR - extNowTSHighest
timeSinceFirst := time.Duration(nowNano - r.firstTime)
extNowTSFirst := r.timestamp.GetExtendedStart() + uint64(timeSinceFirst.Nanoseconds()*int64(r.params.ClockRate)/1e9)
diffFirst := extNowTSSR - extNowTSFirst
// is it more than 5 seconds off?
if uint32(math.Abs(float64(int64(diffHighest)))) > 5*r.params.ClockRate || uint32(math.Abs(float64(int64(diffFirst)))) > 5*r.params.ClockRate {
r.clockSkewMediaPathCount++
if (r.clockSkewMediaPathCount-1)%100 == 0 {
r.logger.Infow(
"received sender report, clock skew against media path",
"current", WrappedRTCPSenderReportStateLogger{srData},
"timeSinceSR", timeSinceSR,
"extNowTSSR", extNowTSSR,
"timeSinceHighest", timeSinceHighest,
"extNowTSHighest", extNowTSHighest,
"diffHighest", int64(diffHighest),
"timeSinceFirst", timeSinceFirst,
"extNowTSFirst", extNowTSFirst,
"diffFirst", int64(diffFirst),
"count", r.clockSkewMediaPathCount,
"rtpStats", lockedRTPStatsReceiverLogEncoder{r},
)
}
}
}
func (r *RTPStatsReceiver) updatePropagationDelayAndRecordSenderReport(srData *livekit.RTCPSenderReportState) {
senderClockTime := mediatransportutil.NtpTime(srData.NtpTimestamp).Time().UnixNano()
estimatedPropagationDelay, stepChange := r.propagationDelayEstimator.Update(senderClockTime, srData.At)
if stepChange {
r.logger.Debugw(
"propagation delay step change",
"currentSenderReport", WrappedRTCPSenderReportStateLogger{srData},
"rtpStats", lockedRTPStatsReceiverLogEncoder{r},
)
}
if r.srFirst == nil {
r.srFirst = srData
}
// adjust receive time to estimated propagation delay
srData.AtAdjusted = senderClockTime + estimatedPropagationDelay
r.srNewest = srData
}
func (r *RTPStatsReceiver) SetRtcpSenderReportData(srData *livekit.RTCPSenderReportState) bool {
r.lock.Lock()
defer r.lock.Unlock()
if srData == nil || !r.initialized {
return false
}
// prevent against extreme case of anachronous sender reports
if r.srNewest != nil && r.srNewest.NtpTimestamp > srData.NtpTimestamp {
r.logger.Infow(
"received sender report, anachronous, dropping",
"current", WrappedRTCPSenderReportStateLogger{srData},
"rtpStats", lockedRTPStatsReceiverLogEncoder{r},
)
return false
}
srDataExt := r.getExtendedSenderReport(srData)
if r.checkOutOfOrderSenderReport(srDataExt) {
return false
}
r.checkRTPClockSkewForSenderReport(srDataExt)
r.updatePropagationDelayAndRecordSenderReport(srDataExt)
r.checkRTPClockSkewAgainstMediaPathForSenderReport(srDataExt)
if err, loggingFields := r.maybeAdjustFirstPacketTime(r.srNewest, 0, r.timestamp.GetExtendedStart()); err != nil {
r.logger.Infow(err.Error(), append(loggingFields, "rtpStats", lockedRTPStatsReceiverLogEncoder{r})...)
}
return true
}
func (r *RTPStatsReceiver) GetRtcpSenderReportData() *livekit.RTCPSenderReportState {
r.lock.RLock()
defer r.lock.RUnlock()
return protoutils.CloneProto(r.srNewest)
}
func (r *RTPStatsReceiver) LastSenderReportTime() time.Time {
r.lock.RLock()
defer r.lock.RUnlock()
if r.srNewest != nil {
return time.Unix(0, r.srNewest.At)
}
return time.Time{}
}
func (r *RTPStatsReceiver) GetRtcpReceptionReport(ssrc uint32, proxyFracLost uint8, snapshotID uint32) *rtcp.ReceptionReport {
r.lock.Lock()
defer r.lock.Unlock()
extHighestSN := r.sequenceNumber.GetExtendedHighest()
then, now := r.getAndResetSnapshot(snapshotID, r.sequenceNumber.GetExtendedStart(), extHighestSN)
if now == nil || then == nil {
return nil
}
packetsExpected := now.extStartSN - then.extStartSN
if packetsExpected > cNumSequenceNumbers {
r.logger.Warnw(
"too many packets expected in receiver report",
fmt.Errorf("start: %d, end: %d, expected: %d", then.extStartSN, now.extStartSN, packetsExpected),
"rtpStats", lockedRTPStatsReceiverLogEncoder{r},
)
return nil
}
if packetsExpected == 0 {
return nil
}
packetsLost := uint32(now.packetsLost - then.packetsLost)
if int32(packetsLost) < 0 {
packetsLost = 0
}
lossRate := float32(packetsLost) / float32(packetsExpected)
fracLost := uint8(lossRate * 256.0)
if proxyFracLost > fracLost {
fracLost = proxyFracLost
}
totalLost := r.packetsLost
if totalLost > 0xffffff { // 24-bits max
totalLost = 0xffffff
}
lastSR := uint32(0)
dlsr := uint32(0)
if r.srNewest != nil {
lastSR = uint32(r.srNewest.NtpTimestamp >> 16)
if r.srNewest.At != 0 {
delayUS := time.Since(time.Unix(0, r.srNewest.At)).Microseconds()
dlsr = uint32(delayUS * 65536 / 1e6)
}
}
return &rtcp.ReceptionReport{
SSRC: ssrc,
FractionLost: fracLost,
TotalLost: uint32(totalLost),
LastSequenceNumber: uint32(now.extStartSN),
Jitter: uint32(r.jitter),
LastSenderReport: lastSR,
Delay: dlsr,
}
}
func (r *RTPStatsReceiver) DeltaInfo(snapshotID uint32) *RTPDeltaInfo {
r.lock.Lock()
defer r.lock.Unlock()
deltaInfo, err, loggingFields := r.deltaInfo(
snapshotID,
r.sequenceNumber.GetExtendedStart(),
r.sequenceNumber.GetExtendedHighest(),
)
if err != nil {
r.logger.Infow(err.Error(), append(loggingFields, "rtpStats", lockedRTPStatsReceiverLogEncoder{r})...)
}
return deltaInfo
}
func (r *RTPStatsReceiver) MarshalLogObject(e zapcore.ObjectEncoder) error {
if r == nil {
return nil
}
r.lock.RLock()
defer r.lock.RUnlock()
return lockedRTPStatsReceiverLogEncoder{r}.MarshalLogObject(e)
}
func (r *RTPStatsReceiver) ToProto() *livekit.RTPStats {
r.lock.RLock()
defer r.lock.RUnlock()
extStartSN, extHighestSN := r.sequenceNumber.GetExtendedStart(), r.sequenceNumber.GetExtendedHighest()
return r.toProto(
getPacketsExpected(extStartSN, extHighestSN),
r.getPacketsSeenMinusPadding(extStartSN, extHighestSN),
r.packetsLost,
r.timestamp.GetExtendedStart(),
r.timestamp.GetExtendedHighest(),
r.jitter,
r.maxJitter,
)
}
func (r *RTPStatsReceiver) isInRange(esn uint64, ehsn uint64) bool {
diff := int64(ehsn - esn)
return diff >= 0 && diff < cHistorySize
}
func (r *RTPStatsReceiver) HighestTimestamp() uint32 {
r.lock.RLock()
defer r.lock.RUnlock()
return r.timestamp.GetHighest()
}
// for testing only
func (r *RTPStatsReceiver) HighestSequenceNumber() uint16 {
r.lock.RLock()
defer r.lock.RUnlock()
return r.sequenceNumber.GetHighest()
}
// for testing only
func (r *RTPStatsReceiver) ExtendedHighestSequenceNumber() uint64 {
r.lock.RLock()
defer r.lock.RUnlock()
return r.sequenceNumber.GetExtendedHighest()
}
// ----------------------------------
type lockedRTPStatsReceiverLogEncoder struct {
*RTPStatsReceiver
}
func (r lockedRTPStatsReceiverLogEncoder) MarshalLogObject(e zapcore.ObjectEncoder) error {
if r.RTPStatsReceiver == nil {
return nil
}
extStartSN, extHighestSN := r.sequenceNumber.GetExtendedStart(), r.sequenceNumber.GetExtendedHighest()
extStartTS, extHighestTS := r.timestamp.GetExtendedStart(), r.timestamp.GetExtendedHighest()
if _, err := r.rtpStatsBase.marshalLogObject(
e,
getPacketsExpected(extStartSN, extHighestSN),
r.getPacketsSeenMinusPadding(extStartSN, extHighestSN),
extStartTS,
extHighestTS,
); err != nil {
return err
}
e.AddUint64("extStartSN", extStartSN)
e.AddUint64("extHighestSN", extHighestSN)
e.AddUint64("extStartTS", extStartTS)
e.AddUint64("extHighestTS", extHighestTS)
e.AddObject("propagationDelayEstimator", r.propagationDelayEstimator)
return nil
}
// ----------------------------------
livekit-server/pkg/sfu/rtpstats/rtpstats_receiver_lite.go
+177
View File
@@ -0,0 +1,177 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package rtpstats
import (
"go.uber.org/zap/zapcore"
"github.com/livekit/livekit-server/pkg/sfu/utils"
"github.com/livekit/protocol/livekit"
"github.com/livekit/protocol/utils/mono"
)
type RTPFlowStateLite struct {
IsNotHandled bool
LossStartInclusive uint64
LossEndExclusive uint64
ExtSequenceNumber uint64
}
func (r *RTPFlowStateLite) MarshalLogObject(e zapcore.ObjectEncoder) error {
if r == nil {
return nil
}
e.AddBool("IsNotHandled", r.IsNotHandled)
e.AddUint64("LossStartInclusive", r.LossStartInclusive)
e.AddUint64("LossEndExclusive", r.LossEndExclusive)
e.AddUint64("ExtSequenceNumber", r.ExtSequenceNumber)
return nil
}
// ---------------------------------------------------------------------
type RTPStatsReceiverLite struct {
*rtpStatsBaseLite
sequenceNumber *utils.WrapAround[uint16, uint64]
}
func NewRTPStatsReceiverLite(params RTPStatsParams) *RTPStatsReceiverLite {
return &RTPStatsReceiverLite{
rtpStatsBaseLite: newRTPStatsBaseLite(params),
sequenceNumber: utils.NewWrapAround[uint16, uint64](utils.WrapAroundParams{IsRestartAllowed: false}),
}
}
func (r *RTPStatsReceiverLite) NewSnapshotLiteId() uint32 {
r.lock.Lock()
defer r.lock.Unlock()
return r.newSnapshotLiteID(r.sequenceNumber.GetExtendedHighest())
}
func (r *RTPStatsReceiverLite) Update(packetTime int64, packetSize int, sequenceNumber uint16) (flowStateLite RTPFlowStateLite) {
r.lock.Lock()
defer r.lock.Unlock()
if r.endTime != 0 {
flowStateLite.IsNotHandled = true
return
}
var resSN utils.WrapAroundUpdateResult[uint64]
if !r.initialized {
r.initialized = true
r.startTime = mono.UnixNano()
resSN = r.sequenceNumber.Update(sequenceNumber)
// initialize lite snapshots if any
for i := uint32(0); i < r.nextSnapshotLiteID-cFirstSnapshotID; i++ {
r.snapshotLites[i] = initSnapshotLite(r.startTime, r.sequenceNumber.GetExtendedStart())
}
r.logger.Debugw(
"rtp receiver lite stream start",
"rtpStats", lockedRTPStatsReceiverLiteLogEncoder{r},
)
} else {
resSN = r.sequenceNumber.Update(sequenceNumber)
if resSN.IsUnhandled {
flowStateLite.IsNotHandled = true
return
}
}
gapSN := int64(resSN.ExtendedVal - resSN.PreExtendedHighest)
if gapSN <= 0 { // duplicate OR out-of-order
r.packetsOutOfOrder++ // counting duplicate as out-of-order
r.packetsLost--
} else { // in-order
r.updateGapHistogram(int(gapSN))
r.packetsLost += uint64(gapSN - 1)
flowStateLite.LossStartInclusive = resSN.PreExtendedHighest + 1
flowStateLite.LossEndExclusive = resSN.ExtendedVal
}
flowStateLite.ExtSequenceNumber = resSN.ExtendedVal
r.bytes += uint64(packetSize)
return
}
func (r *RTPStatsReceiverLite) DeltaInfoLite(snapshotLiteID uint32) *RTPDeltaInfoLite {
r.lock.Lock()
defer r.lock.Unlock()
deltaInfoLite, err, loggingFields := r.deltaInfoLite(
snapshotLiteID,
r.sequenceNumber.GetExtendedStart(),
r.sequenceNumber.GetExtendedHighest(),
)
if err != nil {
r.logger.Infow(err.Error(), append(loggingFields, "rtpStats", lockedRTPStatsReceiverLiteLogEncoder{r})...)
}
return deltaInfoLite
}
func (r *RTPStatsReceiverLite) MarshalLogObject(e zapcore.ObjectEncoder) error {
if r == nil {
return nil
}
r.lock.RLock()
defer r.lock.RUnlock()
return lockedRTPStatsReceiverLiteLogEncoder{r}.MarshalLogObject(e)
}
func (r *RTPStatsReceiverLite) ToProto() *livekit.RTPStats {
r.lock.RLock()
defer r.lock.RUnlock()
return r.rtpStatsBaseLite.toProto(r.sequenceNumber.GetExtendedStart(), r.sequenceNumber.GetExtendedHighest(), r.packetsLost)
}
// ----------------------------------
type lockedRTPStatsReceiverLiteLogEncoder struct {
*RTPStatsReceiverLite
}
func (r lockedRTPStatsReceiverLiteLogEncoder) MarshalLogObject(e zapcore.ObjectEncoder) error {
if r.RTPStatsReceiverLite == nil {
return nil
}
extStartSN, extHighestSN := r.sequenceNumber.GetExtendedStart(), r.sequenceNumber.GetExtendedHighest()
if _, err := r.rtpStatsBaseLite.marshalLogObject(
e,
getPacketsExpected(extStartSN, extHighestSN),
getPacketsExpected(extStartSN, extHighestSN),
); err != nil {
return err
}
e.AddUint64("extStartSN", r.sequenceNumber.GetExtendedStart())
e.AddUint64("extHighestSN", r.sequenceNumber.GetExtendedHighest())
return nil
}
// ----------------------------------
livekit-server/pkg/sfu/rtpstats/rtpstats_receiver_test.go
+228
View File
@@ -0,0 +1,228 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package rtpstats
import (
"math/rand"
"testing"
"time"
"github.com/pion/rtp"
"github.com/stretchr/testify/require"
"github.com/livekit/protocol/logger"
)
func getPacket(sn uint16, ts uint32, payloadSize int) *rtp.Packet {
return &rtp.Packet{
Header: rtp.Header{
SequenceNumber: sn,
Timestamp: ts,
},
Payload: make([]byte, payloadSize),
}
}
func Test_RTPStatsReceiver_Update(t *testing.T) {
clockRate := uint32(90000)
r := NewRTPStatsReceiver(RTPStatsParams{
ClockRate: clockRate,
Logger: logger.GetLogger(),
})
sequenceNumber := uint16(rand.Float64() * float64(1<<16))
timestamp := uint32(rand.Float64() * float64(1<<32))
packet := getPacket(sequenceNumber, timestamp, 1000)
flowState := r.Update(
time.Now().UnixNano(),
packet.Header.SequenceNumber,
packet.Header.Timestamp,
packet.Header.Marker,
packet.Header.MarshalSize(),
len(packet.Payload),
0,
)
require.True(t, r.initialized)
require.Equal(t, sequenceNumber, r.sequenceNumber.GetHighest())
require.Equal(t, sequenceNumber, uint16(r.sequenceNumber.GetExtendedHighest()))
require.Equal(t, timestamp, r.timestamp.GetHighest())
require.Equal(t, timestamp, uint32(r.timestamp.GetExtendedHighest()))
// in-order, no loss
sequenceNumber++
timestamp += 3000
packet = getPacket(sequenceNumber, timestamp, 1000)
flowState = r.Update(
time.Now().UnixNano(),
packet.Header.SequenceNumber,
packet.Header.Timestamp,
packet.Header.Marker,
packet.Header.MarshalSize(),
len(packet.Payload),
0,
)
require.Equal(t, sequenceNumber, r.sequenceNumber.GetHighest())
require.Equal(t, sequenceNumber, uint16(r.sequenceNumber.GetExtendedHighest()))
require.Equal(t, timestamp, r.timestamp.GetHighest())
require.Equal(t, timestamp, uint32(r.timestamp.GetExtendedHighest()))
// out-of-order, would cause a restart which is disallowed
packet = getPacket(sequenceNumber-10, timestamp-30000, 1000)
flowState = r.Update(
time.Now().UnixNano(),
packet.Header.SequenceNumber,
packet.Header.Timestamp,
packet.Header.Marker,
packet.Header.MarshalSize(),
len(packet.Payload),
0,
)
require.True(t, flowState.IsNotHandled)
require.Equal(t, sequenceNumber, r.sequenceNumber.GetHighest())
require.Equal(t, sequenceNumber, uint16(r.sequenceNumber.GetExtendedHighest()))
require.Equal(t, timestamp, r.timestamp.GetHighest())
require.Equal(t, timestamp, uint32(r.timestamp.GetExtendedHighest()))
require.Equal(t, uint64(0), r.packetsOutOfOrder)
require.Equal(t, uint64(0), r.packetsDuplicate)
// duplicate of the above out-of-order packet, but would not be handled as it causes a restart
packet = getPacket(sequenceNumber-10, timestamp-30000, 1000)
flowState = r.Update(
time.Now().UnixNano(),
packet.Header.SequenceNumber,
packet.Header.Timestamp,
packet.Header.Marker,
packet.Header.MarshalSize(),
len(packet.Payload),
0,
)
require.True(t, flowState.IsNotHandled)
require.Equal(t, sequenceNumber, r.sequenceNumber.GetHighest())
require.Equal(t, sequenceNumber, uint16(r.sequenceNumber.GetExtendedHighest()))
require.Equal(t, timestamp, r.timestamp.GetHighest())
require.Equal(t, timestamp, uint32(r.timestamp.GetExtendedHighest()))
require.Equal(t, uint64(0), r.packetsOutOfOrder)
require.Equal(t, uint64(0), r.packetsDuplicate)
// loss
sequenceNumber += 10
timestamp += 30000
packet = getPacket(sequenceNumber, timestamp, 1000)
flowState = r.Update(
time.Now().UnixNano(),
packet.Header.SequenceNumber,
packet.Header.Timestamp,
packet.Header.Marker,
packet.Header.MarshalSize(),
len(packet.Payload),
0,
)
require.Equal(t, uint64(sequenceNumber-9), flowState.LossStartInclusive)
require.Equal(t, uint64(sequenceNumber), flowState.LossEndExclusive)
require.Equal(t, uint64(9), r.packetsLost)
// out-of-order should decrement number of lost packets
packet = getPacket(sequenceNumber-6, timestamp-18000, 1000)
flowState = r.Update(
time.Now().UnixNano(),
packet.Header.SequenceNumber,
packet.Header.Timestamp,
packet.Header.Marker,
packet.Header.MarshalSize(),
len(packet.Payload),
0,
)
require.Equal(t, sequenceNumber, r.sequenceNumber.GetHighest())
require.Equal(t, sequenceNumber, uint16(r.sequenceNumber.GetExtendedHighest()))
require.Equal(t, timestamp, r.timestamp.GetHighest())
require.Equal(t, timestamp, uint32(r.timestamp.GetExtendedHighest()))
require.Equal(t, uint64(1), r.packetsOutOfOrder)
require.Equal(t, uint64(0), r.packetsDuplicate)
require.Equal(t, uint64(8), r.packetsLost)
// test sequence number history
// with a gap
sequenceNumber += 2
timestamp += 6000
packet = getPacket(sequenceNumber, timestamp, 1000)
flowState = r.Update(
time.Now().UnixNano(),
packet.Header.SequenceNumber,
packet.Header.Timestamp,
packet.Header.Marker,
packet.Header.MarshalSize(),
len(packet.Payload),
0,
)
require.Equal(t, uint64(sequenceNumber-1), flowState.LossStartInclusive)
require.Equal(t, uint64(sequenceNumber), flowState.LossEndExclusive)
require.Equal(t, uint64(9), r.packetsLost)
require.False(t, r.history.IsSet(uint64(sequenceNumber)-1))
// out-of-order
sequenceNumber--
timestamp -= 3000
packet = getPacket(sequenceNumber, timestamp, 999)
flowState = r.Update(
time.Now().UnixNano(),
packet.Header.SequenceNumber,
packet.Header.Timestamp,
packet.Header.Marker,
packet.Header.MarshalSize(),
len(packet.Payload),
0,
)
require.Equal(t, uint64(8), r.packetsLost)
require.Equal(t, uint64(2), r.packetsOutOfOrder)
require.True(t, r.history.IsSet(uint64(sequenceNumber)))
// padding only
sequenceNumber += 2
timestamp += 3000
packet = getPacket(sequenceNumber, timestamp, 0)
flowState = r.Update(
time.Now().UnixNano(),
packet.Header.SequenceNumber,
packet.Header.Timestamp,
packet.Header.Marker,
packet.Header.MarshalSize(),
len(packet.Payload),
25,
)
require.Equal(t, uint64(8), r.packetsLost)
require.Equal(t, uint64(2), r.packetsOutOfOrder)
require.True(t, r.history.IsSet(uint64(sequenceNumber)))
require.True(t, r.history.IsSet(uint64(sequenceNumber)-1))
require.True(t, r.history.IsSet(uint64(sequenceNumber)-2))
// old packet, but simulating increasing sequence number after roll over
packet = getPacket(sequenceNumber+400, timestamp-6000, 300)
flowState = r.Update(
time.Now().UnixNano(),
packet.Header.SequenceNumber,
packet.Header.Timestamp,
packet.Header.Marker,
packet.Header.MarshalSize(),
len(packet.Payload),
0,
)
require.True(t, flowState.IsNotHandled)
require.Equal(t, sequenceNumber, r.sequenceNumber.GetHighest())
require.Equal(t, sequenceNumber, uint16(r.sequenceNumber.GetExtendedHighest()))
require.Equal(t, timestamp, r.timestamp.GetHighest())
require.Equal(t, timestamp, uint32(r.timestamp.GetExtendedHighest()))
r.Stop()
}
livekit-server/pkg/sfu/rtpstats/rtpstats_sender.go
+1392
View File
File diff suppressed because it is too large Load Diff
livekit-server/pkg/sfu/rtpstats/rtpstats_sender_lite.go
+121
View File
@@ -0,0 +1,121 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package rtpstats
import (
"github.com/livekit/protocol/livekit"
"github.com/livekit/protocol/utils/mono"
"go.uber.org/zap/zapcore"
)
type RTPStatsSenderLite struct {
*rtpStatsBaseLite
extStartSN uint64
extHighestSN uint64
}
func NewRTPStatsSenderLite(params RTPStatsParams) *RTPStatsSenderLite {
return &RTPStatsSenderLite{
rtpStatsBaseLite: newRTPStatsBaseLite(params),
}
}
func (r *RTPStatsSenderLite) Update(packetTime int64, packetSize int, extSequenceNumber uint64) {
r.lock.Lock()
defer r.lock.Unlock()
if r.endTime != 0 {
return
}
if !r.initialized {
r.initialized = true
r.startTime = mono.UnixNano()
r.extStartSN = extSequenceNumber
r.extHighestSN = extSequenceNumber - 1
r.logger.Debugw(
"rtp sender lite stream start",
"rtpStats", lockedRTPStatsSenderLiteLogEncoder{r},
)
}
gapSN := int64(extSequenceNumber - r.extHighestSN)
if gapSN <= 0 { // duplicate OR out-of-order
r.packetsOutOfOrder++ // counting duplicate as out-of-order
r.packetsLost--
} else { // in-order
r.updateGapHistogram(int(gapSN))
r.packetsLost += uint64(gapSN - 1)
r.extHighestSN = extSequenceNumber
}
r.bytes += uint64(packetSize)
}
func (r *RTPStatsSenderLite) MarshalLogObject(e zapcore.ObjectEncoder) error {
if r == nil {
return nil
}
r.lock.RLock()
defer r.lock.RUnlock()
return lockedRTPStatsSenderLiteLogEncoder{r}.MarshalLogObject(e)
}
func (r *RTPStatsSenderLite) ToProto() *livekit.RTPStats {
r.lock.RLock()
defer r.lock.RUnlock()
return r.rtpStatsBaseLite.toProto(r.extStartSN, r.extHighestSN, r.packetsLost)
}
func (r *RTPStatsSenderLite) ExtHighestSequenceNumber() uint64 {
r.lock.RLock()
defer r.lock.RUnlock()
return r.extHighestSN
}
// -------------------------------------------------------------------
type lockedRTPStatsSenderLiteLogEncoder struct {
*RTPStatsSenderLite
}
func (r lockedRTPStatsSenderLiteLogEncoder) MarshalLogObject(e zapcore.ObjectEncoder) error {
if r.RTPStatsSenderLite == nil {
return nil
}
if _, err := r.rtpStatsBaseLite.marshalLogObject(
e,
getPacketsExpected(r.extStartSN, r.extHighestSN),
getPacketsExpected(r.extStartSN, r.extHighestSN),
); err != nil {
return err
}
e.AddUint64("extStartSN", r.extStartSN)
e.AddUint64("extHighestSN", r.extHighestSN)
return nil
}
// -------------------------------------------------------------------
livekit-server/pkg/sfu/sequencer.go
+459
View File
@@ -0,0 +1,459 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package sfu
import (
"math"
"sync"
"time"
"github.com/livekit/livekit-server/pkg/sfu/utils"
"github.com/livekit/protocol/logger"
"go.uber.org/zap/zapcore"
)
const (
defaultRtt = 70
ignoreRetransmission = 100 // Ignore packet retransmission after ignoreRetransmission milliseconds
maxAck = 3
)
type packetMeta struct {
// Original extended sequence number from stream.
// The original extended sequence number is used to find the original
// packet from publisher
sourceSeqNo uint64
// Modified sequence number after offset.
// This sequence number is used for the associated
// down track, is modified according the offsets, and
// must not be shared
targetSeqNo uint16
// Modified timestamp for current associated
// down track.
timestamp uint32
// Modified marker
marker bool
// The last time this packet was nack requested.
// Sometimes clients request the same packet more than once, so keep
// track of the requested packets helps to avoid writing multiple times
// the same packet.
// The resolution is 1 ms counting after the sequencer start time.
lastNack uint32
// number of NACKs this packet has received
nacked uint8
// Spatial layer of packet
layer int8
// Information that differs depending on the codec
codecBytes [8]byte
numCodecBytesIn uint8
numCodecBytesOut uint8
codecBytesSlice []byte
// Dependency Descriptor of packet
ddBytes [8]byte
ddBytesSize uint8
ddBytesSlice []byte
// abs-capture-time of packet
actBytes []byte
}
func (pm packetMeta) MarshalLogObject(e zapcore.ObjectEncoder) error {
e.AddUint64("sourceSeqNo", pm.sourceSeqNo)
e.AddUint16("targetSeqNo", pm.targetSeqNo)
e.AddInt8("layer", pm.layer)
e.AddUint8("nacked", pm.nacked)
e.AddUint8("numCodecBytesIn", pm.numCodecBytesIn)
if len(pm.codecBytesSlice) != 0 {
e.AddInt("codecBytesSlice", len(pm.codecBytesSlice))
} else {
e.AddUint8("numCodecBytesOut", pm.numCodecBytesOut)
}
if len(pm.ddBytesSlice) != 0 {
e.AddInt("ddBytesSlice", len(pm.ddBytesSlice))
} else {
e.AddUint8("ddBytesSize", pm.ddBytesSize)
}
if len(pm.actBytes) != 0 {
e.AddInt("actBytes", len(pm.actBytes))
}
return nil
}
type extPacketMeta struct {
packetMeta
extSequenceNumber uint64
extTimestamp uint64
}
func (epm extPacketMeta) MarshalLogObject(e zapcore.ObjectEncoder) error {
e.AddObject("packetMeta", epm.packetMeta)
e.AddUint64("extSequenceNumber", epm.extSequenceNumber)
return nil
}
// Sequencer stores the packet sequence received by the down track
type sequencer struct {
sync.Mutex
size int
startTime int64
initialized bool
extStartSN uint64
extHighestSN uint64
snOffset uint64
extHighestTS uint64
meta []packetMeta
snRangeMap *utils.RangeMap[uint64, uint64]
rtt uint32
logger logger.Logger
}
func newSequencer(size int, maybeSparse bool, logger logger.Logger) *sequencer {
s := &sequencer{
size: size,
startTime: time.Now().UnixNano(),
meta: make([]packetMeta, size),
rtt: defaultRtt,
logger: logger,
}
if maybeSparse {
s.snRangeMap = utils.NewRangeMap[uint64, uint64]((size + 1) / 2) // assume run lengths of at least 2 in between padding bursts
}
return s
}
func (s *sequencer) setRTT(rtt uint32) {
s.Lock()
defer s.Unlock()
if rtt == 0 {
s.rtt = defaultRtt
} else {
s.rtt = rtt
}
}
func (s *sequencer) push(
packetTime int64,
extIncomingSN, extModifiedSN uint64,
extModifiedTS uint64,
marker bool,
layer int8,
codecBytes []byte,
numCodecBytesIn int,
ddBytes []byte,
actBytes []byte,
) {
s.Lock()
defer s.Unlock()
if !s.initialized {
s.initialized = true
s.extStartSN = extModifiedSN
s.extHighestSN = extModifiedSN
s.extHighestTS = extModifiedTS
s.updateSNOffset()
}
if extModifiedSN < s.extStartSN {
// old packet, should not happen
return
}
extHighestSNAdjusted := s.extHighestSN - s.snOffset
extModifiedSNAdjusted := extModifiedSN - s.snOffset
if extModifiedSN < s.extHighestSN {
if s.snRangeMap != nil {
snOffset, err := s.snRangeMap.GetValue(extModifiedSN)
if err != nil {
s.logger.Errorw(
"could not get sequence number offset", err,
"extStartSN", s.extStartSN,
"extHighestSN", s.extHighestSN,
"extIncomingSN", extIncomingSN,
"extModifiedSN", extModifiedSN,
"snOffset", s.snOffset,
)
return
}
extModifiedSNAdjusted = extModifiedSN - snOffset
}
}
if int64(extModifiedSNAdjusted-extHighestSNAdjusted) <= -int64(s.size) {
s.logger.Warnw(
"old packet, cannot be sequenced", nil,
"extHighestSN", s.extHighestSN,
"extIncomingSN", extIncomingSN,
"extModifiedSN", extModifiedSN,
)
return
}
// invalidate missing sequence numbers
if extModifiedSNAdjusted > extHighestSNAdjusted {
numInvalidated := 0
for esn := extHighestSNAdjusted + 1; esn != extModifiedSNAdjusted; esn++ {
s.invalidateSlot(int(esn % uint64(s.size)))
numInvalidated++
if numInvalidated >= s.size {
break
}
}
}
slot := extModifiedSNAdjusted % uint64(s.size)
s.meta[slot] = packetMeta{
sourceSeqNo: extIncomingSN,
targetSeqNo: uint16(extModifiedSN),
timestamp: uint32(extModifiedTS),
marker: marker,
layer: layer,
numCodecBytesIn: uint8(numCodecBytesIn),
lastNack: s.getRefTime(packetTime), // delay retransmissions after the original transmission
}
pm := &s.meta[slot]
pm.numCodecBytesOut = uint8(len(codecBytes))
if len(codecBytes) > len(pm.codecBytes) {
pm.codecBytesSlice = append([]byte{}, codecBytes...)
} else {
copy(pm.codecBytes[:pm.numCodecBytesOut], codecBytes)
}
pm.ddBytesSize = uint8(len(ddBytes))
if len(ddBytes) > len(pm.ddBytes) {
pm.ddBytesSlice = append([]byte{}, ddBytes...)
} else {
copy(pm.ddBytes[:pm.ddBytesSize], ddBytes)
}
pm.actBytes = append([]byte{}, actBytes...)
if extModifiedSN > s.extHighestSN {
s.extHighestSN = extModifiedSN
}
if extModifiedTS > s.extHighestTS {
s.extHighestTS = extModifiedTS
}
}
func (s *sequencer) pushPadding(extStartSNInclusive uint64, extEndSNInclusive uint64) {
s.Lock()
defer s.Unlock()
if s.snRangeMap == nil || !s.initialized {
return
}
if extStartSNInclusive <= s.extHighestSN {
// a higher sequence number has already been recorded with an offset,
// adding an exclusion range before the highest means the offset of sequence numbers
// after the exclusion range will be affected and all those higher sequence numbers
// need to be patched.
//
// Not recording exclusion range means a few slots (of the size of exclusion range)
// are wasted in this cycle. That should be fine as the exclusion ranges should be
// a few packets at a time.
if extEndSNInclusive >= s.extHighestSN {
s.logger.Errorw("cannot exclude overlapping range", nil, "extHighestSN", s.extHighestSN, "startSN", extStartSNInclusive, "endSN", extEndSNInclusive)
} else {
s.logger.Warnw("cannot exclude old range", nil, "extHighestSN", s.extHighestSN, "startSN", extStartSNInclusive, "endSN", extEndSNInclusive)
}
// if exclusion range is before what has already been sequenced, invalidate exclusion range slots
for esn := extStartSNInclusive; esn != extEndSNInclusive+1; esn++ {
diff := int64(esn - s.extHighestSN)
if diff >= 0 || diff < -int64(s.size) {
// too old OR too new (too new should not happen, just be safe)
continue
}
snOffset, err := s.snRangeMap.GetValue(esn)
if err != nil {
s.logger.Errorw("could not get sequence number offset", err, "sn", esn)
continue
}
slot := (esn - snOffset) % uint64(s.size)
s.invalidateSlot(int(slot))
}
return
}
if err := s.snRangeMap.ExcludeRange(extStartSNInclusive, extEndSNInclusive+1); err != nil {
s.logger.Errorw("could not exclude range", err, "startSN", extStartSNInclusive, "endSN", extEndSNInclusive)
return
}
s.extHighestSN = extEndSNInclusive
s.updateSNOffset()
}
func (s *sequencer) getExtPacketMetas(seqNo []uint16) []extPacketMeta {
s.Lock()
defer s.Unlock()
if !s.initialized {
return nil
}
snOffset := uint64(0)
var err error
extPacketMetas := make([]extPacketMeta, 0, len(seqNo))
refTime := s.getRefTime(time.Now().UnixNano())
highestSN := uint16(s.extHighestSN)
highestTS := uint32(s.extHighestTS)
for _, sn := range seqNo {
diff := highestSN - sn
if diff > (1 << 15) {
// out-of-order from head (should not happen, just be safe)
continue
}
// find slot by adjusting for padding only packets that were not recorded in sequencer
extSN := uint64(sn) + (s.extHighestSN & 0xFFFF_FFFF_FFFF_0000)
if sn > highestSN {
extSN -= (1 << 16)
}
if s.snRangeMap != nil {
snOffset, err = s.snRangeMap.GetValue(extSN)
if err != nil {
// could be padding packet which is excluded and will not have value
continue
}
}
extSNAdjusted := extSN - snOffset
extHighestSNAdjusted := s.extHighestSN - s.snOffset
if extHighestSNAdjusted-extSNAdjusted >= uint64(s.size) {
// too old
continue
}
slot := extSNAdjusted % uint64(s.size)
meta := &s.meta[slot]
if meta.targetSeqNo != sn || s.isInvalidSlot(int(slot)) {
// invalid slot access could happen if padding packets exclusion range could not be recorded
continue
}
if meta.nacked < maxAck && refTime-meta.lastNack > uint32(math.Min(float64(ignoreRetransmission), float64(2*s.rtt))) {
meta.nacked++
meta.lastNack = refTime
extTS := uint64(meta.timestamp) + (s.extHighestTS & 0xFFFF_FFFF_0000_0000)
if meta.timestamp > highestTS {
extTS -= (1 << 32)
}
epm := extPacketMeta{
packetMeta: *meta,
extSequenceNumber: extSN,
extTimestamp: extTS,
}
epm.codecBytesSlice = append([]byte{}, meta.codecBytesSlice...)
epm.ddBytesSlice = append([]byte{}, meta.ddBytesSlice...)
epm.actBytes = append([]byte{}, meta.actBytes...)
extPacketMetas = append(extPacketMetas, epm)
}
}
return extPacketMetas
}
func (s *sequencer) lookupExtPacketMeta(extSN uint64) *extPacketMeta {
s.Lock()
defer s.Unlock()
if !s.initialized {
return nil
}
snOffset := uint64(0)
var err error
if s.snRangeMap != nil {
snOffset, err = s.snRangeMap.GetValue(extSN)
if err != nil {
return nil
}
}
extSNAdjusted := extSN - snOffset
extHighestSNAdjusted := s.extHighestSN - s.snOffset
if extHighestSNAdjusted-extSNAdjusted >= uint64(s.size) {
// too old
return nil
}
slot := extSNAdjusted % uint64(s.size)
meta := &s.meta[slot]
if s.isInvalidSlot(int(slot)) {
// invalid slot access could happen if padding packets exclusion range could not be recorded
return nil
}
extTS := uint64(meta.timestamp) + (s.extHighestTS & 0xFFFF_FFFF_0000_0000)
if meta.timestamp > uint32(s.extHighestTS) {
extTS -= (1 << 32)
}
epm := extPacketMeta{
packetMeta: *meta,
extSequenceNumber: extSN,
extTimestamp: extTS,
}
epm.codecBytesSlice = append([]byte{}, meta.codecBytesSlice...)
epm.ddBytesSlice = append([]byte{}, meta.ddBytesSlice...)
epm.actBytes = append([]byte{}, meta.actBytes...)
return &epm
}
func (s *sequencer) getRefTime(at int64) uint32 {
return uint32((at - s.startTime) / 1e6)
}
func (s *sequencer) updateSNOffset() {
if s.snRangeMap == nil {
return
}
snOffset, err := s.snRangeMap.GetValue(s.extHighestSN + 1)
if err != nil {
s.logger.Errorw("could not update sequence number offset", err, "extHighestSN", s.extHighestSN)
return
}
s.snOffset = snOffset
}
func (s *sequencer) invalidateSlot(slot int) {
if slot >= len(s.meta) {
return
}
s.meta[slot] = packetMeta{
sourceSeqNo: 0,
targetSeqNo: 0,
lastNack: 0,
}
}
func (s *sequencer) isInvalidSlot(slot int) bool {
if slot >= len(s.meta) {
return true
}
meta := &s.meta[slot]
return meta.sourceSeqNo == 0 && meta.targetSeqNo == 0 && meta.lastNack == 0
}
livekit-server/pkg/sfu/sequencer_test.go
+368
View File
@@ -0,0 +1,368 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package sfu
import (
"reflect"
"testing"
"time"
"github.com/stretchr/testify/require"
"github.com/livekit/protocol/logger"
)
func Test_sequencer(t *testing.T) {
seq := newSequencer(500, false, logger.GetLogger())
off := uint16(15)
for i := uint64(1); i < 518; i++ {
seq.push(time.Now().UnixNano(), i, i+uint64(off), 123, true, 2, nil, 0, nil, nil)
}
// send the last two out-of-order
seq.push(time.Now().UnixNano(), 519, 519+uint64(off), 123, false, 2, nil, 0, nil, nil)
seq.push(time.Now().UnixNano(), 518, 518+uint64(off), 123, true, 2, nil, 0, nil, nil)
req := []uint16{57, 58, 62, 63, 513, 514, 515, 516, 517}
res := seq.getExtPacketMetas(req)
// nothing should be returned as not enough time has elapsed since sending packet
require.Equal(t, 0, len(res))
time.Sleep((ignoreRetransmission + 10) * time.Millisecond)
res = seq.getExtPacketMetas(req)
require.Equal(t, len(req), len(res))
for i, val := range res {
require.Equal(t, val.targetSeqNo, req[i])
require.Equal(t, val.sourceSeqNo, uint64(req[i]-off))
require.Equal(t, val.layer, int8(2))
require.Equal(t, val.extSequenceNumber, uint64(req[i]))
require.Equal(t, val.extTimestamp, uint64(123))
}
res = seq.getExtPacketMetas(req)
require.Equal(t, 0, len(res))
time.Sleep((ignoreRetransmission + 10) * time.Millisecond)
res = seq.getExtPacketMetas(req)
require.Equal(t, len(req), len(res))
for i, val := range res {
require.Equal(t, val.targetSeqNo, req[i])
require.Equal(t, val.sourceSeqNo, uint64(req[i]-off))
require.Equal(t, val.layer, int8(2))
require.Equal(t, val.extSequenceNumber, uint64(req[i]))
require.Equal(t, val.extTimestamp, uint64(123))
}
seq.push(time.Now().UnixNano(), 521, 521+uint64(off), 123, true, 1, nil, 0, nil, nil)
m := seq.getExtPacketMetas([]uint16{521 + off})
require.Equal(t, 0, len(m))
time.Sleep((ignoreRetransmission + 10) * time.Millisecond)
m = seq.getExtPacketMetas([]uint16{521 + off})
require.Equal(t, 1, len(m))
seq.push(time.Now().UnixNano(), 505, 505+uint64(off), 123, false, 1, nil, 0, nil, nil)
m = seq.getExtPacketMetas([]uint16{505 + off})
require.Equal(t, 0, len(m))
time.Sleep((ignoreRetransmission + 10) * time.Millisecond)
m = seq.getExtPacketMetas([]uint16{505 + off})
require.Equal(t, 1, len(m))
}
func Test_sequencer_getNACKSeqNo_exclusion(t *testing.T) {
type args struct {
seqNo []uint16
}
type input struct {
seqNo uint64
isPadding bool
}
type fields struct {
inputs []input
offset uint64
markerOdd bool
markerEven bool
codecBytesOdd []byte
numCodecBytesInOdd int
codecBytesEven []byte
numCodecBytesInEven int
codecBytesOversized []byte
ddBytesOdd []byte
ddBytesEven []byte
ddBytesOversized []byte
actBytesOdd []byte
actBytesEven []byte
}
tests := []struct {
name string
fields fields
args args
want []uint16
}{
{
name: "Should get correct seq numbers",
fields: fields{
inputs: []input{
{65526, false},
{65524, false},
{65525, false},
{65529, false},
{65530, false},
{65531, true},
{65533, false},
{65532, true},
{65534, false},
},
offset: 5,
markerOdd: true,
markerEven: false,
codecBytesOdd: []byte{1, 2, 3, 4},
numCodecBytesInOdd: 3,
codecBytesEven: []byte{5, 6, 7},
numCodecBytesInEven: 4,
codecBytesOversized: []byte{1, 2, 3, 4, 5, 6, 7, 8, 9},
ddBytesOdd: []byte{8, 9, 10},
ddBytesEven: []byte{11, 12},
ddBytesOversized: []byte{11, 12, 13, 14, 15, 16, 17, 18, 19},
actBytesOdd: []byte{0, 1, 2, 3, 4, 5, 6, 7},
actBytesEven: []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15},
},
args: args{
seqNo: []uint16{65526 + 5, 65527 + 5, 65530 + 5, 0 /* 65531 input */, 1 /* 65532 input */, 2 /* 65533 input */, 3 /* 65534 input */},
},
// although 65526 is originally pushed, that would have been reset by 65532 (padding only packet)
// because of trying to add an exclusion range before highest sequence number which will fail
// and the resulting fix up of the exclusion range slots
want: []uint16{65530, 65533, 65534},
},
}
for _, tt := range tests {
tt := tt
t.Run(tt.name, func(t *testing.T) {
n := newSequencer(5, true, logger.GetLogger())
for _, i := range tt.fields.inputs {
if i.isPadding {
n.pushPadding(i.seqNo+tt.fields.offset, i.seqNo+tt.fields.offset)
} else {
if i.seqNo%5 == 0 {
n.push(
time.Now().UnixNano(),
i.seqNo,
i.seqNo+tt.fields.offset,
123,
tt.fields.markerOdd,
3,
tt.fields.codecBytesOversized,
len(tt.fields.codecBytesOversized),
tt.fields.ddBytesOversized,
tt.fields.actBytesOdd,
)
} else {
if i.seqNo%2 == 0 {
n.push(
time.Now().UnixNano(),
i.seqNo,
i.seqNo+tt.fields.offset,
123,
tt.fields.markerEven,
3,
tt.fields.codecBytesEven,
tt.fields.numCodecBytesInEven,
tt.fields.ddBytesEven,
tt.fields.actBytesEven,
)
} else {
n.push(
time.Now().UnixNano(),
i.seqNo,
i.seqNo+tt.fields.offset,
123,
tt.fields.markerOdd,
3,
tt.fields.codecBytesOdd,
tt.fields.numCodecBytesInOdd,
tt.fields.ddBytesOdd,
tt.fields.actBytesOdd,
)
}
}
}
}
time.Sleep((ignoreRetransmission + 10) * time.Millisecond)
g := n.getExtPacketMetas(tt.args.seqNo)
var got []uint16
for _, sn := range g {
got = append(got, uint16(sn.sourceSeqNo))
if sn.sourceSeqNo%5 == 0 {
require.Equal(t, tt.fields.markerOdd, sn.marker)
require.Equal(t, tt.fields.codecBytesOversized, sn.codecBytesSlice)
require.Equal(t, uint8(len(tt.fields.codecBytesOversized)), sn.numCodecBytesIn)
require.Equal(t, tt.fields.ddBytesOversized, sn.ddBytesSlice)
require.Equal(t, uint8(len(tt.fields.codecBytesOversized)), sn.ddBytesSize)
require.Equal(t, tt.fields.actBytesOdd, sn.actBytes)
} else {
if sn.sourceSeqNo%2 == 0 {
require.Equal(t, tt.fields.markerEven, sn.marker)
require.Equal(t, tt.fields.codecBytesEven, sn.codecBytes[:sn.numCodecBytesOut])
require.Equal(t, uint8(tt.fields.numCodecBytesInEven), sn.numCodecBytesIn)
require.Equal(t, tt.fields.ddBytesEven, sn.ddBytes[:sn.ddBytesSize])
require.Equal(t, uint8(len(tt.fields.ddBytesEven)), sn.ddBytesSize)
require.Equal(t, tt.fields.actBytesEven, sn.actBytes)
} else {
require.Equal(t, tt.fields.markerOdd, sn.marker)
require.Equal(t, tt.fields.codecBytesOdd, sn.codecBytes[:sn.numCodecBytesOut])
require.Equal(t, uint8(tt.fields.numCodecBytesInOdd), sn.numCodecBytesIn)
require.Equal(t, tt.fields.ddBytesOdd, sn.ddBytes[:sn.ddBytesSize])
require.Equal(t, uint8(len(tt.fields.ddBytesOdd)), sn.ddBytesSize)
require.Equal(t, tt.fields.actBytesOdd, sn.actBytes)
}
}
}
if !reflect.DeepEqual(got, tt.want) {
t.Errorf("getExtPacketMetas() = %v, want %v", got, tt.want)
}
})
}
}
func Test_sequencer_getNACKSeqNo_no_exclusion(t *testing.T) {
type args struct {
seqNo []uint16
}
type input struct {
seqNo uint64
isPadding bool
}
type fields struct {
inputs []input
offset uint64
markerOdd bool
markerEven bool
codecBytesOdd []byte
numCodecBytesInOdd int
codecBytesEven []byte
numCodecBytesInEven int
ddBytesOdd []byte
ddBytesEven []byte
actBytesOdd []byte
actBytesEven []byte
}
tests := []struct {
name string
fields fields
args args
want []uint16
}{
{
name: "Should get correct seq numbers",
fields: fields{
inputs: []input{
{2, false},
{3, false},
{4, false},
{7, false},
{8, false},
{9, true},
{11, false},
{10, true},
{12, false},
{13, false},
},
offset: 5,
markerOdd: true,
markerEven: false,
codecBytesOdd: []byte{1, 2, 3, 4},
numCodecBytesInOdd: 3,
codecBytesEven: []byte{5, 6, 7},
numCodecBytesInEven: 4,
ddBytesOdd: []byte{8, 9, 10},
ddBytesEven: []byte{11, 12},
actBytesOdd: []byte{8, 9, 10},
actBytesEven: []byte{11, 12},
},
args: args{
seqNo: []uint16{4 + 5, 5 + 5, 8 + 5, 9 + 5, 10 + 5, 11 + 5, 12 + 5},
},
// although 4 and 8 were originally added, they would be too old after a cycle of sequencer buffer
want: []uint16{11, 12},
},
}
for _, tt := range tests {
tt := tt
t.Run(tt.name, func(t *testing.T) {
n := newSequencer(5, false, logger.GetLogger())
for _, i := range tt.fields.inputs {
if i.isPadding {
n.pushPadding(i.seqNo+tt.fields.offset, i.seqNo+tt.fields.offset)
} else {
if i.seqNo%2 == 0 {
n.push(
time.Now().UnixNano(),
i.seqNo,
i.seqNo+tt.fields.offset,
123,
tt.fields.markerEven,
3,
tt.fields.codecBytesEven,
tt.fields.numCodecBytesInEven,
tt.fields.ddBytesEven,
tt.fields.actBytesEven,
)
} else {
n.push(
time.Now().UnixNano(),
i.seqNo,
i.seqNo+tt.fields.offset,
123,
tt.fields.markerOdd,
3,
tt.fields.codecBytesOdd,
tt.fields.numCodecBytesInOdd,
tt.fields.ddBytesOdd,
tt.fields.actBytesOdd,
)
}
}
}
time.Sleep((ignoreRetransmission + 10) * time.Millisecond)
g := n.getExtPacketMetas(tt.args.seqNo)
var got []uint16
for _, sn := range g {
got = append(got, uint16(sn.sourceSeqNo))
if sn.sourceSeqNo%2 == 0 {
require.Equal(t, tt.fields.markerEven, sn.marker)
require.Equal(t, tt.fields.codecBytesEven, sn.codecBytes[:sn.numCodecBytesOut])
require.Equal(t, uint8(tt.fields.numCodecBytesInEven), sn.numCodecBytesIn)
require.Equal(t, tt.fields.ddBytesEven, sn.ddBytes[:sn.ddBytesSize])
require.Equal(t, uint8(len(tt.fields.ddBytesEven)), sn.ddBytesSize)
require.Equal(t, tt.fields.actBytesEven, sn.actBytes)
} else {
require.Equal(t, tt.fields.markerOdd, sn.marker)
require.Equal(t, tt.fields.codecBytesOdd, sn.codecBytes[:sn.numCodecBytesOut])
require.Equal(t, uint8(tt.fields.numCodecBytesInOdd), sn.numCodecBytesIn)
require.Equal(t, tt.fields.ddBytesOdd, sn.ddBytes[:sn.ddBytesSize])
require.Equal(t, uint8(len(tt.fields.ddBytesOdd)), sn.ddBytesSize)
require.Equal(t, tt.fields.actBytesOdd, sn.actBytes)
}
}
if !reflect.DeepEqual(got, tt.want) {
t.Errorf("getExtPacketMetas() = %v, want %v", got, tt.want)
}
})
}
}
livekit-server/pkg/sfu/sfu.go
+33
View File
@@ -0,0 +1,33 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package sfu
import (
"sync"
)
var (
PacketFactory *sync.Pool
)
func init() {
// Init packet factory
PacketFactory = &sync.Pool{
New: func() interface{} {
b := make([]byte, 1460)
return &b
},
}
}
livekit-server/pkg/sfu/streamallocator/streamallocator.go
+1447
View File
File diff suppressed because it is too large Load Diff
livekit-server/pkg/sfu/streamallocator/streamstateupdate.go
+85
View File
@@ -0,0 +1,85 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package streamallocator
import (
"fmt"
"github.com/livekit/protocol/livekit"
)
// ------------------------------------------------
type StreamState int
const (
StreamStateInactive StreamState = iota
StreamStateActive
StreamStatePaused
)
func (s StreamState) String() string {
switch s {
case StreamStateInactive:
return "INACTIVE"
case StreamStateActive:
return "ACTIVE"
case StreamStatePaused:
return "PAUSED"
default:
return fmt.Sprintf("UNKNOWN: %d", int(s))
}
}
// ------------------------------------------------
type StreamStateInfo struct {
ParticipantID livekit.ParticipantID
TrackID livekit.TrackID
State StreamState
}
type StreamStateUpdate struct {
StreamStates []*StreamStateInfo
}
func NewStreamStateUpdate() *StreamStateUpdate {
return &StreamStateUpdate{}
}
func (s *StreamStateUpdate) HandleStreamingChange(track *Track, streamState StreamState) {
switch streamState {
case StreamStateInactive:
// inactive is not a notification, could get into this state because of mute
case StreamStateActive:
s.StreamStates = append(s.StreamStates, &StreamStateInfo{
ParticipantID: track.PublisherID(),
TrackID: track.ID(),
State: StreamStateActive,
})
case StreamStatePaused:
s.StreamStates = append(s.StreamStates, &StreamStateInfo{
ParticipantID: track.PublisherID(),
TrackID: track.ID(),
State: StreamStatePaused,
})
}
}
func (s *StreamStateUpdate) Empty() bool {
return len(s.StreamStates) == 0
}
// ------------------------------------------------
livekit-server/pkg/sfu/streamallocator/track.go
+281
View File
@@ -0,0 +1,281 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package streamallocator
import (
"github.com/livekit/protocol/livekit"
"github.com/livekit/protocol/logger"
"github.com/livekit/livekit-server/pkg/sfu"
"github.com/livekit/livekit-server/pkg/sfu/buffer"
)
type Track struct {
downTrack *sfu.DownTrack
source livekit.TrackSource
isSimulcast bool
priority uint8
publisherID livekit.ParticipantID
logger logger.Logger
maxLayer buffer.VideoLayer
totalPackets uint32
totalRepeatedNacks uint32
isDirty bool
streamState StreamState
}
func NewTrack(
downTrack *sfu.DownTrack,
source livekit.TrackSource,
isSimulcast bool,
publisherID livekit.ParticipantID,
logger logger.Logger,
) *Track {
t := &Track{
downTrack: downTrack,
source: source,
isSimulcast: isSimulcast,
publisherID: publisherID,
logger: logger,
streamState: StreamStateInactive,
}
t.SetPriority(0)
t.SetMaxLayer(downTrack.MaxLayer())
return t
}
func (t *Track) SetDirty(isDirty bool) bool {
if t.isDirty == isDirty {
return false
}
t.isDirty = isDirty
return true
}
func (t *Track) SetStreamState(streamState StreamState) bool {
if t.streamState == streamState {
return false
}
t.streamState = streamState
return true
}
func (t *Track) IsSubscribeMutable() bool {
return t.streamState != StreamStatePaused
}
func (t *Track) SetPriority(priority uint8) bool {
if priority == 0 {
switch t.source {
case livekit.TrackSource_SCREEN_SHARE:
priority = PriorityDefaultScreenshare
default:
priority = PriorityDefaultVideo
}
}
if t.priority == priority {
return false
}
t.priority = priority
return true
}
func (t *Track) Priority() uint8 {
return t.priority
}
func (t *Track) DownTrack() *sfu.DownTrack {
return t.downTrack
}
func (t *Track) IsManaged() bool {
return t.source != livekit.TrackSource_SCREEN_SHARE || t.isSimulcast
}
func (t *Track) ID() livekit.TrackID {
return livekit.TrackID(t.downTrack.ID())
}
func (t *Track) PublisherID() livekit.ParticipantID {
return t.publisherID
}
func (t *Track) SetMaxLayer(layer buffer.VideoLayer) bool {
if t.maxLayer == layer {
return false
}
t.maxLayer = layer
return true
}
func (t *Track) WritePaddingRTP(bytesToSend int) int {
return t.downTrack.WritePaddingRTP(bytesToSend, false, false)
}
func (t *Track) WriteProbePackets(bytesToSend int) int {
return t.downTrack.WriteProbePackets(bytesToSend, false)
}
func (t *Track) AllocateOptimal(allowOvershoot bool, hold bool) sfu.VideoAllocation {
return t.downTrack.AllocateOptimal(allowOvershoot, hold)
}
func (t *Track) ProvisionalAllocatePrepare() {
t.downTrack.ProvisionalAllocatePrepare()
}
func (t *Track) ProvisionalAllocateReset() {
t.downTrack.ProvisionalAllocateReset()
}
func (t *Track) ProvisionalAllocate(availableChannelCapacity int64, layer buffer.VideoLayer, allowPause bool, allowOvershoot bool) (bool, int64) {
return t.downTrack.ProvisionalAllocate(availableChannelCapacity, layer, allowPause, allowOvershoot)
}
func (t *Track) ProvisionalAllocateGetCooperativeTransition(allowOvershoot bool) sfu.VideoTransition {
return t.downTrack.ProvisionalAllocateGetCooperativeTransition(allowOvershoot)
}
func (t *Track) ProvisionalAllocateGetBestWeightedTransition() sfu.VideoTransition {
return t.downTrack.ProvisionalAllocateGetBestWeightedTransition()
}
func (t *Track) ProvisionalAllocateCommit() sfu.VideoAllocation {
return t.downTrack.ProvisionalAllocateCommit()
}
func (t *Track) AllocateNextHigher(availableChannelCapacity int64, allowOvershoot bool) (sfu.VideoAllocation, bool) {
return t.downTrack.AllocateNextHigher(availableChannelCapacity, allowOvershoot)
}
func (t *Track) GetNextHigherTransition(allowOvershoot bool) (sfu.VideoTransition, bool) {
return t.downTrack.GetNextHigherTransition(allowOvershoot)
}
func (t *Track) Pause() sfu.VideoAllocation {
return t.downTrack.Pause()
}
func (t *Track) IsDeficient() bool {
return t.downTrack.IsDeficient()
}
func (t *Track) BandwidthRequested() int64 {
return t.downTrack.BandwidthRequested()
}
func (t *Track) DistanceToDesired() float64 {
return t.downTrack.DistanceToDesired()
}
func (t *Track) GetNackDelta() (uint32, uint32) {
totalPackets, totalRepeatedNacks := t.downTrack.GetNackStats()
packetDelta := totalPackets - t.totalPackets
t.totalPackets = totalPackets
nackDelta := totalRepeatedNacks - t.totalRepeatedNacks
t.totalRepeatedNacks = totalRepeatedNacks
return packetDelta, nackDelta
}
// ------------------------------------------------
type TrackSorter []*Track
func (t TrackSorter) Len() int {
return len(t)
}
func (t TrackSorter) Swap(i, j int) {
t[i], t[j] = t[j], t[i]
}
func (t TrackSorter) Less(i, j int) bool {
//
// TrackSorter is used to allocate layer-by-layer.
// So, higher priority track should come earlier so that it gets an earlier shot at each layer
//
if t[i].priority != t[j].priority {
return t[i].priority > t[j].priority
}
if t[i].maxLayer.Spatial != t[j].maxLayer.Spatial {
return t[i].maxLayer.Spatial > t[j].maxLayer.Spatial
}
return t[i].maxLayer.Temporal > t[j].maxLayer.Temporal
}
// ------------------------------------------------
type MaxDistanceSorter []*Track
func (m MaxDistanceSorter) Len() int {
return len(m)
}
func (m MaxDistanceSorter) Swap(i, j int) {
m[i], m[j] = m[j], m[i]
}
func (m MaxDistanceSorter) Less(i, j int) bool {
//
// MaxDistanceSorter is used to find a deficient track to use for probing during recovery from congestion.
// So, higher priority track should come earlier so that they have a chance to recover sooner.
//
if m[i].priority != m[j].priority {
return m[i].priority > m[j].priority
}
return m[i].DistanceToDesired() > m[j].DistanceToDesired()
}
// ------------------------------------------------
type MinDistanceSorter []*Track
func (m MinDistanceSorter) Len() int {
return len(m)
}
func (m MinDistanceSorter) Swap(i, j int) {
m[i], m[j] = m[j], m[i]
}
func (m MinDistanceSorter) Less(i, j int) bool {
//
// MinDistanceSorter is used to find excess bandwidth in cooperative allocation.
// So, lower priority track should come earlier so that they contribute bandwidth to higher priority tracks.
//
if m[i].priority != m[j].priority {
return m[i].priority < m[j].priority
}
return m[i].DistanceToDesired() < m[j].DistanceToDesired()
}
// ------------------------------------------------
livekit-server/pkg/sfu/streamtracker/interfaces.go
+70
View File
@@ -0,0 +1,70 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package streamtracker
import (
"fmt"
"time"
"github.com/livekit/livekit-server/pkg/sfu/buffer"
)
// ------------------------------------------------------------
type StreamStatusChange int32
func (s StreamStatusChange) String() string {
switch s {
case StreamStatusChangeNone:
return "none"
case StreamStatusChangeStopped:
return "stopped"
case StreamStatusChangeActive:
return "active"
default:
return fmt.Sprintf("unknown: %d", int(s))
}
}
const (
StreamStatusChangeNone StreamStatusChange = iota
StreamStatusChangeStopped
StreamStatusChangeActive
)
// ------------------------------------------------------------
type StreamTrackerImpl interface {
Start()
Stop()
Reset()
GetCheckInterval() time.Duration
Observe(hasMarker bool, ts uint32) StreamStatusChange
CheckStatus() StreamStatusChange
}
type StreamTrackerWorker interface {
Start()
Stop()
Reset()
OnStatusChanged(f func(status StreamStatus))
OnBitrateAvailable(f func())
Status() StreamStatus
BitrateTemporalCumulative() []int64
SetPaused(paused bool)
Observe(temporalLayer int32, pktSize int, payloadSize int, hasMarker bool, ts uint32, dd *buffer.ExtDependencyDescriptor)
}
livekit-server/pkg/sfu/streamtracker/streamtracker.go
+307
View File
@@ -0,0 +1,307 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package streamtracker
import (
"fmt"
"sync"
"time"
"go.uber.org/atomic"
"github.com/livekit/livekit-server/pkg/sfu/buffer"
"github.com/livekit/protocol/logger"
)
// ------------------------------------------------------------
type StreamStatus int32
func (s StreamStatus) String() string {
switch s {
case StreamStatusStopped:
return "stopped"
case StreamStatusActive:
return "active"
default:
return fmt.Sprintf("unknown: %d", int(s))
}
}
const (
StreamStatusStopped StreamStatus = iota
StreamStatusActive
)
// ------------------------------------------------------------
type StreamTrackerParams struct {
StreamTrackerImpl StreamTrackerImpl
BitrateReportInterval time.Duration
Logger logger.Logger
}
type StreamTracker struct {
params StreamTrackerParams
onStatusChanged func(status StreamStatus)
onBitrateAvailable func()
lock sync.RWMutex
paused bool
generation atomic.Uint32
status StreamStatus
lastNotifiedStatus StreamStatus
lastBitrateReport time.Time
bytesForBitrate [4]int64
bitrate [4]int64
isStopped bool
}
func NewStreamTracker(params StreamTrackerParams) *StreamTracker {
return &StreamTracker{
params: params,
status: StreamStatusStopped,
}
}
func (s *StreamTracker) OnStatusChanged(f func(status StreamStatus)) {
s.onStatusChanged = f
}
func (s *StreamTracker) OnBitrateAvailable(f func()) {
s.onBitrateAvailable = f
}
func (s *StreamTracker) Status() StreamStatus {
s.lock.RLock()
defer s.lock.RUnlock()
return s.status
}
func (s *StreamTracker) setStatusLocked(status StreamStatus) {
s.status = status
}
func (s *StreamTracker) maybeNotifyStatus() {
var status StreamStatus
notify := false
s.lock.Lock()
if s.status != s.lastNotifiedStatus {
notify = true
status = s.status
s.lastNotifiedStatus = s.status
}
s.lock.Unlock()
if notify && s.onStatusChanged != nil {
s.onStatusChanged(status)
}
}
func (s *StreamTracker) Start() {
s.lock.Lock()
defer s.lock.Unlock()
s.params.StreamTrackerImpl.Start()
}
func (s *StreamTracker) Stop() {
s.lock.Lock()
defer s.lock.Unlock()
if s.isStopped {
return
}
s.isStopped = true
// bump generation to trigger exit of worker
s.generation.Inc()
s.params.StreamTrackerImpl.Stop()
}
func (s *StreamTracker) Reset() {
s.lock.Lock()
if s.isStopped {
s.lock.Unlock()
return
}
s.resetLocked()
s.lock.Unlock()
s.maybeNotifyStatus()
}
func (s *StreamTracker) resetLocked() {
// bump generation to trigger exit of current worker
s.generation.Inc()
s.setStatusLocked(StreamStatusStopped)
for i := 0; i < len(s.bytesForBitrate); i++ {
s.bytesForBitrate[i] = 0
}
for i := 0; i < len(s.bitrate); i++ {
s.bitrate[i] = 0
}
s.params.StreamTrackerImpl.Reset()
}
func (s *StreamTracker) SetPaused(paused bool) {
s.lock.Lock()
s.paused = paused
if !paused {
s.resetLocked()
} else {
// bump generation to trigger exit of current worker
s.generation.Inc()
s.setStatusLocked(StreamStatusStopped)
}
s.lock.Unlock()
s.maybeNotifyStatus()
}
func (s *StreamTracker) Observe(
temporalLayer int32,
pktSize int,
payloadSize int,
hasMarker bool,
ts uint32,
_ *buffer.ExtDependencyDescriptor,
) {
s.lock.Lock()
if s.isStopped || s.paused || payloadSize == 0 {
s.lock.Unlock()
return
}
statusChange := s.params.StreamTrackerImpl.Observe(hasMarker, ts)
if statusChange == StreamStatusChangeActive {
s.setStatusLocked(StreamStatusActive)
s.lastBitrateReport = time.Now()
go s.worker(s.generation.Load())
}
if temporalLayer >= 0 {
s.bytesForBitrate[temporalLayer] += int64(pktSize)
}
s.lock.Unlock()
if statusChange != StreamStatusChangeNone {
s.maybeNotifyStatus()
}
}
// BitrateTemporalCumulative returns the current stream bitrate temporal layer accumulated with lower temporal layers.
func (s *StreamTracker) BitrateTemporalCumulative() []int64 {
s.lock.RLock()
defer s.lock.RUnlock()
// copy and process
brs := make([]int64, len(s.bitrate))
copy(brs, s.bitrate[:])
for i := len(brs) - 1; i >= 1; i-- {
if brs[i] != 0 {
for j := i - 1; j >= 0; j-- {
brs[i] += brs[j]
}
}
}
// clear higher layers
for i := 0; i < len(brs); i++ {
if brs[i] == 0 {
for j := i + 1; j < len(brs); j++ {
brs[j] = 0
}
}
}
return brs
}
func (s *StreamTracker) worker(generation uint32) {
ticker := time.NewTicker(s.params.StreamTrackerImpl.GetCheckInterval())
defer ticker.Stop()
tickerBitrate := time.NewTicker(s.params.BitrateReportInterval)
defer tickerBitrate.Stop()
for {
select {
case <-ticker.C:
if generation != s.generation.Load() {
return
}
s.updateStatus()
case <-tickerBitrate.C:
if generation != s.generation.Load() {
return
}
s.bitrateReport()
}
}
}
func (s *StreamTracker) updateStatus() {
s.lock.Lock()
switch s.params.StreamTrackerImpl.CheckStatus() {
case StreamStatusChangeStopped:
s.setStatusLocked(StreamStatusStopped)
case StreamStatusChangeActive:
s.setStatusLocked(StreamStatusActive)
}
s.lock.Unlock()
s.maybeNotifyStatus()
}
func (s *StreamTracker) bitrateReport() {
// run this even if paused to drain out bitrate if there are no packets coming in
s.lock.Lock()
now := time.Now()
diff := now.Sub(s.lastBitrateReport)
s.lastBitrateReport = now
bitrateAvailabilityChanged := false
for i := 0; i < len(s.bytesForBitrate); i++ {
bitrate := int64(float64(s.bytesForBitrate[i]*8) / diff.Seconds())
if (s.bitrate[i] == 0 && bitrate > 0) || (s.bitrate[i] > 0 && bitrate == 0) {
bitrateAvailabilityChanged = true
}
s.bitrate[i] = bitrate
s.bytesForBitrate[i] = 0
}
s.lock.Unlock()
if bitrateAvailabilityChanged && s.onBitrateAvailable != nil {
s.onBitrateAvailable()
}
}
livekit-server/pkg/sfu/streamtracker/streamtracker_dd.go
+303
View File
@@ -0,0 +1,303 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package streamtracker
import (
"sync"
"time"
"go.uber.org/atomic"
"github.com/livekit/livekit-server/pkg/sfu/buffer"
dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor"
)
type StreamTrackerDependencyDescriptor struct {
lock sync.RWMutex
paused bool
generation atomic.Uint32
params StreamTrackerParams
maxSpatialLayer int32
maxTemporalLayer int32
onStatusChanged [buffer.DefaultMaxLayerSpatial + 1]func(status StreamStatus)
onBitrateAvailable [buffer.DefaultMaxLayerSpatial + 1]func()
lastBitrateReport time.Time
bytesForBitrate [buffer.DefaultMaxLayerSpatial + 1][buffer.DefaultMaxLayerTemporal + 1]int64
bitrate [buffer.DefaultMaxLayerSpatial + 1][buffer.DefaultMaxLayerTemporal + 1]int64
isStopped bool
}
func NewStreamTrackerDependencyDescriptor(params StreamTrackerParams) *StreamTrackerDependencyDescriptor {
return &StreamTrackerDependencyDescriptor{
params: params,
maxSpatialLayer: buffer.InvalidLayerSpatial,
maxTemporalLayer: buffer.InvalidLayerTemporal,
}
}
func (s *StreamTrackerDependencyDescriptor) Start() {
}
func (s *StreamTrackerDependencyDescriptor) Stop() {
s.lock.Lock()
defer s.lock.Unlock()
if s.isStopped {
return
}
s.isStopped = true
// bump generation to trigger exit of worker
s.generation.Inc()
}
func (s *StreamTrackerDependencyDescriptor) OnStatusChanged(layer int32, f func(status StreamStatus)) {
s.lock.Lock()
s.onStatusChanged[layer] = f
s.lock.Unlock()
}
func (s *StreamTrackerDependencyDescriptor) OnBitrateAvailable(layer int32, f func()) {
s.lock.Lock()
s.onBitrateAvailable[layer] = f
s.lock.Unlock()
}
func (s *StreamTrackerDependencyDescriptor) Status(layer int32) StreamStatus {
s.lock.RLock()
defer s.lock.RUnlock()
if layer > s.maxSpatialLayer {
return StreamStatusStopped
}
return StreamStatusActive
}
func (s *StreamTrackerDependencyDescriptor) BitrateTemporalCumulative(layer int32) []int64 {
s.lock.RLock()
defer s.lock.RUnlock()
if layer > s.maxSpatialLayer {
brs := make([]int64, len(s.bitrate[0]))
return brs
}
return s.bitrate[layer][:]
}
func (s *StreamTrackerDependencyDescriptor) Reset() {
}
func (s *StreamTrackerDependencyDescriptor) resetLocked() {
// bump generation to trigger exit of current worker
s.generation.Inc()
for i := 0; i < len(s.bytesForBitrate); i++ {
for j := 0; j < len(s.bytesForBitrate[i]); j++ {
s.bytesForBitrate[i][j] = 0
}
}
for i := 0; i < len(s.bitrate); i++ {
for j := 0; j < len(s.bitrate[i]); j++ {
s.bitrate[i][j] = 0
}
}
s.maxSpatialLayer = buffer.InvalidLayerSpatial
s.maxTemporalLayer = buffer.InvalidLayerTemporal
}
func (s *StreamTrackerDependencyDescriptor) SetPaused(paused bool) {
s.lock.Lock()
if s.paused == paused {
s.lock.Unlock()
return
}
s.paused = paused
var notifyFns []func(status StreamStatus)
var notifyStatus StreamStatus
if !paused {
s.resetLocked()
notifyStatus = StreamStatusStopped
notifyFns = append(notifyFns, s.onStatusChanged[:]...)
} else {
s.lastBitrateReport = time.Now()
go s.worker(s.generation.Inc())
}
s.lock.Unlock()
for _, fn := range notifyFns {
if fn != nil {
fn(notifyStatus)
}
}
}
func (s *StreamTrackerDependencyDescriptor) Observe(temporalLayer int32, pktSize int, payloadSize int, hasMarker bool, ts uint32, ddVal *buffer.ExtDependencyDescriptor) {
s.lock.Lock()
if s.isStopped || s.paused || payloadSize == 0 || ddVal == nil {
s.lock.Unlock()
return
}
var notifyFns []func(status StreamStatus)
var notifyStatus StreamStatus
if mask := ddVal.Descriptor.ActiveDecodeTargetsBitmask; mask != nil && ddVal.ActiveDecodeTargetsUpdated {
var maxSpatial, maxTemporal int32
for _, dt := range ddVal.DecodeTargets {
if *mask&(1<<dt.Target) != uint32(dd.DecodeTargetNotPresent) {
if maxSpatial < dt.Layer.Spatial {
maxSpatial = dt.Layer.Spatial
}
if maxTemporal < dt.Layer.Temporal {
maxTemporal = dt.Layer.Temporal
}
}
}
if maxSpatial > buffer.DefaultMaxLayerSpatial {
maxSpatial = buffer.DefaultMaxLayerSpatial
s.params.Logger.Warnw("max spatial layer exceeded", nil, "maxSpatial", maxSpatial)
}
if maxTemporal > buffer.DefaultMaxLayerTemporal {
maxTemporal = buffer.DefaultMaxLayerTemporal
s.params.Logger.Warnw("max temporal layer exceeded", nil, "maxTemporal", maxTemporal)
}
s.params.Logger.Debugw("max layer changed", "maxSpatial", maxSpatial, "maxTemporal", maxTemporal)
oldMaxSpatial := s.maxSpatialLayer
s.maxSpatialLayer, s.maxTemporalLayer = maxSpatial, maxTemporal
if oldMaxSpatial == -1 {
s.lastBitrateReport = time.Now()
go s.worker(s.generation.Inc())
}
if oldMaxSpatial > s.maxSpatialLayer {
notifyStatus = StreamStatusStopped
for i := s.maxSpatialLayer + 1; i <= oldMaxSpatial; i++ {
notifyFns = append(notifyFns, s.onStatusChanged[i])
}
} else if oldMaxSpatial < s.maxSpatialLayer {
notifyStatus = StreamStatusActive
for i := oldMaxSpatial + 1; i <= s.maxSpatialLayer; i++ {
notifyFns = append(notifyFns, s.onStatusChanged[i])
}
}
}
dtis := ddVal.Descriptor.FrameDependencies.DecodeTargetIndications
for _, dt := range ddVal.DecodeTargets {
if len(dtis) <= dt.Target {
s.params.Logger.Errorw("len(dtis) less than target", nil, "target", dt.Target, "dtis", dtis)
continue
}
// we are not dropping discardable frames now, so only ingore not present frames
if dtis[dt.Target] == dd.DecodeTargetNotPresent {
continue
}
s.bytesForBitrate[dt.Layer.Spatial][dt.Layer.Temporal] += int64(pktSize)
}
s.lock.Unlock()
for _, fn := range notifyFns {
if fn != nil {
fn(notifyStatus)
}
}
}
func (s *StreamTrackerDependencyDescriptor) worker(generation uint32) {
tickerBitrate := time.NewTicker(s.params.BitrateReportInterval)
defer tickerBitrate.Stop()
for {
<-tickerBitrate.C
if generation != s.generation.Load() {
return
}
s.bitrateReport()
}
}
func (s *StreamTrackerDependencyDescriptor) bitrateReport() {
// run this even if paused to drain out bitrate if there are no packets coming in
s.lock.Lock()
now := time.Now()
diff := now.Sub(s.lastBitrateReport)
s.lastBitrateReport = now
var availableChangedFns []func()
for spatial := 0; spatial < len(s.bytesForBitrate); spatial++ {
bytesForBitrate := s.bytesForBitrate[spatial][:]
bitrateAvailabilityChanged := false
bitrates := s.bitrate[spatial][:]
for i := 0; i < len(bytesForBitrate); i++ {
bitrate := int64(float64(bytesForBitrate[i]*8) / diff.Seconds())
if (bitrates[i] == 0 && bitrate > 0) || (bitrates[i] > 0 && bitrate == 0) {
bitrateAvailabilityChanged = true
}
bitrates[i] = bitrate
bytesForBitrate[i] = 0
}
if bitrateAvailabilityChanged && s.onBitrateAvailable[spatial] != nil {
availableChangedFns = append(availableChangedFns, s.onBitrateAvailable[spatial])
}
}
s.lock.Unlock()
for _, fn := range availableChangedFns {
fn()
}
}
func (s *StreamTrackerDependencyDescriptor) LayeredTracker(layer int32) *StreamTrackerDependencyDescriptorLayered {
return &StreamTrackerDependencyDescriptorLayered{
StreamTrackerDependencyDescriptor: s,
layer: layer,
}
}
// ----------------------------
// Layered wrapper for StreamTrackerWorker
type StreamTrackerDependencyDescriptorLayered struct {
*StreamTrackerDependencyDescriptor
layer int32
}
func (s *StreamTrackerDependencyDescriptorLayered) OnStatusChanged(f func(status StreamStatus)) {
s.StreamTrackerDependencyDescriptor.OnStatusChanged(s.layer, f)
}
func (s *StreamTrackerDependencyDescriptorLayered) OnBitrateAvailable(f func()) {
s.StreamTrackerDependencyDescriptor.OnBitrateAvailable(s.layer, f)
}
func (s *StreamTrackerDependencyDescriptorLayered) Status() StreamStatus {
return s.StreamTrackerDependencyDescriptor.Status(s.layer)
}
func (s *StreamTrackerDependencyDescriptorLayered) BitrateTemporalCumulative() []int64 {
return s.StreamTrackerDependencyDescriptor.BitrateTemporalCumulative(s.layer)
}
livekit-server/pkg/sfu/streamtracker/streamtracker_dd_test.go
+98
View File
@@ -0,0 +1,98 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package streamtracker
import (
"testing"
"time"
"github.com/stretchr/testify/require"
"github.com/livekit/livekit-server/pkg/sfu/buffer"
dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor"
"github.com/livekit/protocol/logger"
)
func createDescriptorDependencyForTargets(maxSpatial, maxTemporal int) *buffer.ExtDependencyDescriptor {
var targets []buffer.DependencyDescriptorDecodeTarget
var mask uint32
for i := 0; i <= maxSpatial; i++ {
for j := 0; j <= maxTemporal; j++ {
targets = append(targets, buffer.DependencyDescriptorDecodeTarget{Target: len(targets), Layer: buffer.VideoLayer{Spatial: int32(i), Temporal: int32(j)}})
mask |= 1 << uint32(len(targets)-1)
}
}
dtis := make([]dd.DecodeTargetIndication, len(targets))
for _, t := range targets {
dtis[t.Target] = dd.DecodeTargetRequired
}
return &buffer.ExtDependencyDescriptor{
Descriptor: &dd.DependencyDescriptor{
ActiveDecodeTargetsBitmask: &mask,
FrameDependencies: &dd.FrameDependencyTemplate{
DecodeTargetIndications: dtis,
},
},
DecodeTargets: targets,
ActiveDecodeTargetsUpdated: true,
}
}
func checkStatues(t *testing.T, statuses []StreamStatus, expected StreamStatus, maxSpatial int) {
for i := 0; i <= maxSpatial; i++ {
require.Equal(t, expected, statuses[i])
}
for i := maxSpatial + 1; i < len(statuses); i++ {
require.NotEqual(t, expected, statuses[i])
}
}
func TestStreamTrackerDD(t *testing.T) {
ddTracker := NewStreamTrackerDependencyDescriptor(StreamTrackerParams{
BitrateReportInterval: 1 * time.Second,
Logger: logger.GetLogger(),
})
layeredTrackers := make([]StreamTrackerWorker, buffer.DefaultMaxLayerSpatial+1)
statuses := make([]StreamStatus, buffer.DefaultMaxLayerSpatial+1)
for i := 0; i <= int(buffer.DefaultMaxLayerSpatial); i++ {
layeredTrack := ddTracker.LayeredTracker(int32(i))
layer := i
layeredTrack.OnStatusChanged(func(status StreamStatus) {
statuses[layer] = status
})
layeredTrack.Start()
layeredTrackers[i] = layeredTrack
}
defer ddTracker.Stop()
// no active layers
ddTracker.Observe(0, 1000, 1000, false, 0, nil)
checkStatues(t, statuses, StreamStatusActive, int(buffer.InvalidLayerSpatial))
// layer seen [0,1]
ddTracker.Observe(0, 1000, 1000, false, 0, createDescriptorDependencyForTargets(1, 1))
checkStatues(t, statuses, StreamStatusActive, 1)
// layer seen [0,1,2]
ddTracker.Observe(0, 1000, 1000, false, 0, createDescriptorDependencyForTargets(2, 1))
checkStatues(t, statuses, StreamStatusActive, 2)
// layer 2 gone, layer seen [0,1]
ddTracker.Observe(0, 1000, 1000, false, 0, createDescriptorDependencyForTargets(1, 1))
checkStatues(t, statuses, StreamStatusActive, 1)
}
livekit-server/pkg/sfu/streamtracker/streamtracker_frame.go
+244
View File
@@ -0,0 +1,244 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package streamtracker
import (
"math"
"time"
"github.com/livekit/protocol/logger"
)
const (
checkInterval = 500 * time.Millisecond
statusCheckTolerance = 0.98
frameRateResolution = 0.01 // 1 frame every 100 seconds
frameRateIncreaseFactor = 0.6 // slow increase
frameRateDecreaseFactor = 0.9 // fast decrease
)
// -------------------------------------------------------
type StreamTrackerFrameConfig struct {
MinFPS float64 `yaml:"min_fps,omitempty"`
}
var (
DefaultStreamTrackerFrameConfigVideo = map[int32]StreamTrackerFrameConfig{
0: {
MinFPS: 5.0,
},
1: {
MinFPS: 5.0,
},
2: {
MinFPS: 5.0,
},
}
DefaultStreamTrackerFrameConfigScreenshare = map[int32]StreamTrackerFrameConfig{
0: {
MinFPS: 0.5,
},
1: {
MinFPS: 0.5,
},
2: {
MinFPS: 0.5,
},
}
)
// -------------------------------------------------------
type StreamTrackerFrameParams struct {
Config StreamTrackerFrameConfig
ClockRate uint32
Logger logger.Logger
}
type StreamTrackerFrame struct {
params StreamTrackerFrameParams
initialized bool
tsInitialized bool
oldestTS uint32
newestTS uint32
numFrames int
estimatedFrameRate float64
evalInterval time.Duration
lastStatusCheckAt time.Time
}
func NewStreamTrackerFrame(params StreamTrackerFrameParams) StreamTrackerImpl {
s := &StreamTrackerFrame{
params: params,
}
s.Reset()
return s
}
func (s *StreamTrackerFrame) Start() {
}
func (s *StreamTrackerFrame) Stop() {
}
func (s *StreamTrackerFrame) Reset() {
s.initialized = false
s.resetFPSCalculator()
s.lastStatusCheckAt = time.Time{}
}
func (s *StreamTrackerFrame) resetFPSCalculator() {
s.tsInitialized = false
s.oldestTS = 0
s.newestTS = 0
s.numFrames = 0
s.estimatedFrameRate = 0.0
s.updateEvalInterval()
}
func (s *StreamTrackerFrame) GetCheckInterval() time.Duration {
return checkInterval
}
func (s *StreamTrackerFrame) Observe(hasMarker bool, ts uint32) StreamStatusChange {
if hasMarker {
if !s.tsInitialized {
s.tsInitialized = true
s.oldestTS = ts
s.newestTS = ts
s.numFrames = 1
} else {
diff := ts - s.oldestTS
if diff > (1 << 31) {
s.oldestTS = ts
}
diff = ts - s.newestTS
if diff < (1 << 31) {
s.newestTS = ts
}
s.numFrames++
}
}
// When starting up, check for first packet and declare active.
// Happens under following conditions
// 1. Start up
// 2. Unmute (stream restarting)
// 3. Layer starting after dynacast pause
if !s.initialized {
s.initialized = true
s.lastStatusCheckAt = time.Now()
return StreamStatusChangeActive
}
return StreamStatusChangeNone
}
func (s *StreamTrackerFrame) CheckStatus() StreamStatusChange {
if !s.initialized {
// should not be getting called when not initialized, but be safe
return StreamStatusChangeNone
}
if !s.updateStatusCheckTime() {
return StreamStatusChangeNone
}
if s.updateEstimatedFrameRate() == 0.0 {
// when stream is stopped, reset FPS calculator to ensure re-start is not done until at least two frames are available,
// i. e. enough frames available to be able to calculate FPS
s.resetFPSCalculator()
return StreamStatusChangeStopped
}
return StreamStatusChangeActive
}
func (s *StreamTrackerFrame) updateStatusCheckTime() bool {
// check only at intervals based on estimated frame rate
if s.lastStatusCheckAt.IsZero() {
s.lastStatusCheckAt = time.Now()
}
if time.Since(s.lastStatusCheckAt) < time.Duration(statusCheckTolerance*float64(s.evalInterval)) {
return false
}
s.lastStatusCheckAt = time.Now()
return true
}
func (s *StreamTrackerFrame) updateEstimatedFrameRate() float64 {
diff := s.newestTS - s.oldestTS
if diff == 0 || s.numFrames < 2 {
return 0.0
}
frameRate := roundFrameRate(float64(s.params.ClockRate) / float64(diff) * float64(s.numFrames-1))
// reset for next evaluation interval
s.oldestTS = s.newestTS
s.numFrames = 1
factor := 1.0
switch {
case s.estimatedFrameRate < frameRate:
// slow increase, prevents shortening eval interval too quickly on frame rate going up
factor = frameRateIncreaseFactor
case s.estimatedFrameRate > frameRate:
// fast decrease, prevents declaring stream stop too quickly on frame rate going down
factor = frameRateDecreaseFactor
}
estimatedFrameRate := roundFrameRate(frameRate*factor + s.estimatedFrameRate*(1.0-factor))
if s.estimatedFrameRate != estimatedFrameRate {
s.estimatedFrameRate = estimatedFrameRate
s.updateEvalInterval()
s.params.Logger.Debugw("updating estimated frame rate", "estimatedFPS", estimatedFrameRate, "evalInterval", s.evalInterval)
}
return frameRate
}
func (s *StreamTrackerFrame) updateEvalInterval() {
// STREAM-TRACKER-FRAME-TODO: This will run into challenges for frame rate falling steeply, How to address that?
// Maybe, look at some referential rules (between layers) for possibilities to solve it. Currently, this is addressed
// by setting a source aware min FPS to ensure evaluation window is long enough to avoid declaring stop too quickly.
s.evalInterval = checkInterval
if s.estimatedFrameRate > 0.0 {
estimatedFrameRateInterval := time.Duration(float64(time.Second) / s.estimatedFrameRate)
if estimatedFrameRateInterval > s.evalInterval {
s.evalInterval = estimatedFrameRateInterval
}
}
if s.params.Config.MinFPS > 0.0 {
minFPSInterval := time.Duration(float64(time.Second) / s.params.Config.MinFPS)
if minFPSInterval > s.evalInterval {
s.evalInterval = minFPSInterval
}
}
}
// ------------------------------------------------------------------------------
func roundFrameRate(frameRate float64) float64 {
return math.Round(frameRate/frameRateResolution) * frameRateResolution
}
livekit-server/pkg/sfu/streamtracker/streamtracker_packet.go
+141
View File
@@ -0,0 +1,141 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package streamtracker
import (
"time"
"github.com/livekit/protocol/logger"
)
// --------------------------------------------
type StreamTrackerPacketConfig struct {
SamplesRequired uint32 `yaml:"samples_required,omitempty"` // number of samples needed per cycle
CyclesRequired uint32 `yaml:"cycles_required,omitempty"` // number of cycles needed to be active
CycleDuration time.Duration `yaml:"cycle_duration,omitempty"`
}
var (
DefaultStreamTrackerPacketConfigVideo = map[int32]StreamTrackerPacketConfig{
0: {SamplesRequired: 1,
CyclesRequired: 4,
CycleDuration: 500 * time.Millisecond,
},
1: {SamplesRequired: 5,
CyclesRequired: 20,
CycleDuration: 500 * time.Millisecond,
},
2: {SamplesRequired: 5,
CyclesRequired: 20,
CycleDuration: 500 * time.Millisecond,
},
}
DefaultStreamTrackerPacketConfigScreenshare = map[int32]StreamTrackerPacketConfig{
0: {
SamplesRequired: 1,
CyclesRequired: 1,
CycleDuration: 2 * time.Second,
},
1: {
SamplesRequired: 1,
CyclesRequired: 1,
CycleDuration: 2 * time.Second,
},
2: {
SamplesRequired: 1,
CyclesRequired: 1,
CycleDuration: 2 * time.Second,
},
}
)
// --------------------------------------------
type StreamTrackerPacketParams struct {
Config StreamTrackerPacketConfig
Logger logger.Logger
}
type StreamTrackerPacket struct {
params StreamTrackerPacketParams
countSinceLast uint32 // number of packets received since last check
initialized bool
cycleCount uint32
}
func NewStreamTrackerPacket(params StreamTrackerPacketParams) StreamTrackerImpl {
return &StreamTrackerPacket{
params: params,
}
}
func (s *StreamTrackerPacket) Start() {
}
func (s *StreamTrackerPacket) Stop() {
}
func (s *StreamTrackerPacket) Reset() {
s.countSinceLast = 0
s.cycleCount = 0
s.initialized = false
}
func (s *StreamTrackerPacket) GetCheckInterval() time.Duration {
return s.params.Config.CycleDuration
}
func (s *StreamTrackerPacket) Observe(_hasMarker bool, _ts uint32) StreamStatusChange {
if !s.initialized {
// first packet
s.initialized = true
s.countSinceLast = 1
return StreamStatusChangeActive
}
s.countSinceLast++
return StreamStatusChangeNone
}
func (s *StreamTrackerPacket) CheckStatus() StreamStatusChange {
if !s.initialized {
// should not be getting called when not initialized, but be safe
return StreamStatusChangeNone
}
if s.countSinceLast >= s.params.Config.SamplesRequired {
s.cycleCount++
} else {
s.cycleCount = 0
}
statusChange := StreamStatusChangeNone
if s.cycleCount == 0 {
// no packets seen for a period, flip to stopped
statusChange = StreamStatusChangeStopped
} else if s.cycleCount >= s.params.Config.CyclesRequired {
// packets seen for some time after resume, flip to active
statusChange = StreamStatusChangeActive
}
s.countSinceLast = 0
return statusChange
}

Some files were not shown because too many files have changed in this diff Show More