Squashed 'livekit-server/' content from commit 154b4d26
git-subtree-dir: livekit-server git-subtree-split: 154b4d26b769c68a03c096124094b97bf61a996f
This commit is contained in:
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,435 @@
|
||||
// Copyright 2023 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package buffer
|
||||
|
||||
import (
|
||||
"math"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/pion/rtcp"
|
||||
"github.com/pion/rtp"
|
||||
"github.com/pion/webrtc/v4"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/livekit/mediatransportutil/pkg/nack"
|
||||
)
|
||||
|
||||
var h265Codec = webrtc.RTPCodecParameters{
|
||||
RTPCodecCapability: webrtc.RTPCodecCapability{
|
||||
MimeType: "video/h265",
|
||||
ClockRate: 90000,
|
||||
RTCPFeedback: []webrtc.RTCPFeedback{{
|
||||
Type: "nack",
|
||||
}},
|
||||
},
|
||||
PayloadType: 116,
|
||||
}
|
||||
|
||||
var vp8Codec = webrtc.RTPCodecParameters{
|
||||
RTPCodecCapability: webrtc.RTPCodecCapability{
|
||||
MimeType: "video/vp8",
|
||||
ClockRate: 90000,
|
||||
RTCPFeedback: []webrtc.RTCPFeedback{{
|
||||
Type: "nack",
|
||||
}},
|
||||
},
|
||||
PayloadType: 96,
|
||||
}
|
||||
|
||||
var opusCodec = webrtc.RTPCodecParameters{
|
||||
RTPCodecCapability: webrtc.RTPCodecCapability{
|
||||
MimeType: "audio/opus",
|
||||
ClockRate: 48000,
|
||||
},
|
||||
PayloadType: 111,
|
||||
}
|
||||
|
||||
func TestNack(t *testing.T) {
|
||||
t.Run("nack normal", func(t *testing.T) {
|
||||
buff := NewBuffer(123, 1, 1)
|
||||
buff.codecType = webrtc.RTPCodecTypeVideo
|
||||
require.NotNil(t, buff)
|
||||
var wg sync.WaitGroup
|
||||
// 5 tries
|
||||
wg.Add(5)
|
||||
buff.OnRtcpFeedback(func(fb []rtcp.Packet) {
|
||||
for _, pkt := range fb {
|
||||
switch p := pkt.(type) {
|
||||
case *rtcp.TransportLayerNack:
|
||||
if p.Nacks[0].PacketList()[0] == 1 && p.MediaSSRC == 123 {
|
||||
wg.Done()
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
buff.Bind(webrtc.RTPParameters{
|
||||
HeaderExtensions: nil,
|
||||
Codecs: []webrtc.RTPCodecParameters{vp8Codec},
|
||||
}, vp8Codec.RTPCodecCapability, 0)
|
||||
rtt := uint32(20)
|
||||
buff.nacker.SetRTT(rtt)
|
||||
for i := 0; i < 15; i++ {
|
||||
if i == 1 {
|
||||
continue
|
||||
}
|
||||
if i < 14 {
|
||||
time.Sleep(time.Duration(float64(rtt)*math.Pow(nack.NackQueueParamsDefault.BackoffFactor, float64(i))+10) * time.Millisecond)
|
||||
} else {
|
||||
time.Sleep(500 * time.Millisecond) // even a long wait should not exceed max retries
|
||||
}
|
||||
pkt := rtp.Packet{
|
||||
Header: rtp.Header{
|
||||
Version: 2,
|
||||
PayloadType: 96,
|
||||
SequenceNumber: uint16(i),
|
||||
Timestamp: uint32(i),
|
||||
SSRC: 123,
|
||||
},
|
||||
Payload: []byte{0xff, 0xff, 0xff, 0xfd, 0xb4, 0x9f, 0x94, 0x1},
|
||||
}
|
||||
b, err := pkt.Marshal()
|
||||
require.NoError(t, err)
|
||||
_, err = buff.Write(b)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
})
|
||||
|
||||
t.Run("nack with seq wrap", func(t *testing.T) {
|
||||
buff := NewBuffer(123, 1, 1)
|
||||
buff.codecType = webrtc.RTPCodecTypeVideo
|
||||
require.NotNil(t, buff)
|
||||
var wg sync.WaitGroup
|
||||
expects := map[uint16]int{
|
||||
65534: 0,
|
||||
65535: 0,
|
||||
0: 0,
|
||||
1: 0,
|
||||
}
|
||||
wg.Add(5 * len(expects)) // retry 5 times
|
||||
buff.OnRtcpFeedback(func(fb []rtcp.Packet) {
|
||||
for _, pkt := range fb {
|
||||
switch p := pkt.(type) {
|
||||
case *rtcp.TransportLayerNack:
|
||||
if p.MediaSSRC == 123 {
|
||||
for _, v := range p.Nacks {
|
||||
v.Range(func(seq uint16) bool {
|
||||
if _, ok := expects[seq]; ok {
|
||||
wg.Done()
|
||||
} else {
|
||||
require.Fail(t, "unexpected nack seq ", seq)
|
||||
}
|
||||
return true
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
buff.Bind(webrtc.RTPParameters{
|
||||
HeaderExtensions: nil,
|
||||
Codecs: []webrtc.RTPCodecParameters{vp8Codec},
|
||||
}, vp8Codec.RTPCodecCapability, 0)
|
||||
rtt := uint32(30)
|
||||
buff.nacker.SetRTT(rtt)
|
||||
for i := 0; i < 15; i++ {
|
||||
if i > 0 && i < 5 {
|
||||
continue
|
||||
}
|
||||
if i < 14 {
|
||||
time.Sleep(time.Duration(float64(rtt)*math.Pow(nack.NackQueueParamsDefault.BackoffFactor, float64(i))+10) * time.Millisecond)
|
||||
} else {
|
||||
time.Sleep(500 * time.Millisecond) // even a long wait should not exceed max retries
|
||||
}
|
||||
pkt := rtp.Packet{
|
||||
Header: rtp.Header{
|
||||
Version: 2,
|
||||
PayloadType: 96,
|
||||
SequenceNumber: uint16(i + 65533),
|
||||
Timestamp: uint32(i),
|
||||
SSRC: 123,
|
||||
},
|
||||
Payload: []byte{0xff, 0xff, 0xff, 0xfd, 0xb4, 0x9f, 0x94, 0x1},
|
||||
}
|
||||
b, err := pkt.Marshal()
|
||||
require.NoError(t, err)
|
||||
_, err = buff.Write(b)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
})
|
||||
}
|
||||
|
||||
func TestNewBuffer(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
}{
|
||||
{
|
||||
name: "Must not be nil and add packets in sequence",
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
var TestPackets = []*rtp.Packet{
|
||||
{
|
||||
Header: rtp.Header{
|
||||
Version: 2,
|
||||
PayloadType: 96,
|
||||
SequenceNumber: 65533,
|
||||
SSRC: 123,
|
||||
},
|
||||
},
|
||||
{
|
||||
Header: rtp.Header{
|
||||
Version: 2,
|
||||
PayloadType: 96,
|
||||
SequenceNumber: 65534,
|
||||
SSRC: 123,
|
||||
},
|
||||
Payload: []byte{1},
|
||||
},
|
||||
{
|
||||
Header: rtp.Header{
|
||||
Version: 2,
|
||||
PayloadType: 96,
|
||||
SequenceNumber: 2,
|
||||
SSRC: 123,
|
||||
},
|
||||
},
|
||||
{
|
||||
Header: rtp.Header{
|
||||
Version: 2,
|
||||
PayloadType: 96,
|
||||
SequenceNumber: 65535,
|
||||
SSRC: 123,
|
||||
},
|
||||
},
|
||||
}
|
||||
buff := NewBuffer(123, 1, 1)
|
||||
buff.codecType = webrtc.RTPCodecTypeVideo
|
||||
require.NotNil(t, buff)
|
||||
buff.OnRtcpFeedback(func(_ []rtcp.Packet) {})
|
||||
buff.Bind(webrtc.RTPParameters{
|
||||
HeaderExtensions: nil,
|
||||
Codecs: []webrtc.RTPCodecParameters{vp8Codec},
|
||||
}, vp8Codec.RTPCodecCapability, 0)
|
||||
|
||||
for _, p := range TestPackets {
|
||||
buf, _ := p.Marshal()
|
||||
_, _ = buff.Write(buf)
|
||||
}
|
||||
require.Equal(t, uint16(2), buff.rtpStats.HighestSequenceNumber())
|
||||
require.Equal(t, uint64(65536+2), buff.rtpStats.ExtendedHighestSequenceNumber())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFractionLostReport(t *testing.T) {
|
||||
buff := NewBuffer(123, 1, 1)
|
||||
require.NotNil(t, buff)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
|
||||
// with loss proxying
|
||||
wg.Add(1)
|
||||
buff.SetAudioLossProxying(true)
|
||||
buff.SetLastFractionLostReport(55)
|
||||
buff.OnRtcpFeedback(func(fb []rtcp.Packet) {
|
||||
for _, pkt := range fb {
|
||||
switch p := pkt.(type) {
|
||||
case *rtcp.ReceiverReport:
|
||||
for _, v := range p.Reports {
|
||||
require.EqualValues(t, 55, v.FractionLost)
|
||||
}
|
||||
wg.Done()
|
||||
}
|
||||
}
|
||||
})
|
||||
buff.Bind(webrtc.RTPParameters{
|
||||
HeaderExtensions: nil,
|
||||
Codecs: []webrtc.RTPCodecParameters{opusCodec},
|
||||
}, opusCodec.RTPCodecCapability, 0)
|
||||
for i := 0; i < 15; i++ {
|
||||
pkt := rtp.Packet{
|
||||
Header: rtp.Header{
|
||||
Version: 2,
|
||||
PayloadType: 111,
|
||||
SequenceNumber: uint16(i),
|
||||
Timestamp: uint32(i),
|
||||
SSRC: 123,
|
||||
},
|
||||
Payload: []byte{0xff, 0xff, 0xff, 0xfd, 0xb4, 0x9f, 0x94, 0x1},
|
||||
}
|
||||
b, err := pkt.Marshal()
|
||||
require.NoError(t, err)
|
||||
if i == 1 {
|
||||
time.Sleep(1 * time.Second)
|
||||
}
|
||||
_, err = buff.Write(b)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
wg.Add(1)
|
||||
buff.SetAudioLossProxying(false)
|
||||
buff.OnRtcpFeedback(func(fb []rtcp.Packet) {
|
||||
for _, pkt := range fb {
|
||||
switch p := pkt.(type) {
|
||||
case *rtcp.ReceiverReport:
|
||||
for _, v := range p.Reports {
|
||||
require.EqualValues(t, 0, v.FractionLost)
|
||||
}
|
||||
wg.Done()
|
||||
}
|
||||
}
|
||||
})
|
||||
buff.Bind(webrtc.RTPParameters{
|
||||
HeaderExtensions: nil,
|
||||
Codecs: []webrtc.RTPCodecParameters{opusCodec},
|
||||
}, opusCodec.RTPCodecCapability, 0)
|
||||
for i := 0; i < 15; i++ {
|
||||
pkt := rtp.Packet{
|
||||
Header: rtp.Header{
|
||||
Version: 2,
|
||||
PayloadType: 111,
|
||||
SequenceNumber: uint16(i),
|
||||
Timestamp: uint32(i),
|
||||
SSRC: 123,
|
||||
},
|
||||
Payload: []byte{0xff, 0xff, 0xff, 0xfd, 0xb4, 0x9f, 0x94, 0x1},
|
||||
}
|
||||
b, err := pkt.Marshal()
|
||||
require.NoError(t, err)
|
||||
if i == 1 {
|
||||
time.Sleep(1 * time.Second)
|
||||
}
|
||||
_, err = buff.Write(b)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func TestCodecChange(t *testing.T) {
|
||||
// codec change before bind
|
||||
buff := NewBuffer(123, 1, 1)
|
||||
require.NotNil(t, buff)
|
||||
changedCodec := make(chan webrtc.RTPCodecParameters, 1)
|
||||
buff.OnCodecChange(func(rp webrtc.RTPCodecParameters) {
|
||||
select {
|
||||
case changedCodec <- rp:
|
||||
default:
|
||||
t.Fatalf("codec change not consumed")
|
||||
}
|
||||
})
|
||||
|
||||
h265Pkt := rtp.Packet{
|
||||
Header: rtp.Header{
|
||||
Version: 2,
|
||||
PayloadType: 116,
|
||||
SequenceNumber: 1,
|
||||
Timestamp: 1,
|
||||
SSRC: 123,
|
||||
},
|
||||
Payload: []byte{0xff, 0xff, 0xff, 0xfd, 0xb4, 0x9f, 0x94, 0x1},
|
||||
}
|
||||
buf, err := h265Pkt.Marshal()
|
||||
require.NoError(t, err)
|
||||
_, err = buff.Write(buf)
|
||||
require.NoError(t, err)
|
||||
|
||||
select {
|
||||
case <-changedCodec:
|
||||
t.Fatalf("unexpected codec change")
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
}
|
||||
|
||||
buff.Bind(webrtc.RTPParameters{
|
||||
HeaderExtensions: nil,
|
||||
Codecs: []webrtc.RTPCodecParameters{vp8Codec, h265Codec},
|
||||
}, vp8Codec.RTPCodecCapability, 0)
|
||||
|
||||
select {
|
||||
case c := <-changedCodec:
|
||||
require.Equal(t, h265Codec, c)
|
||||
|
||||
case <-time.After(1 * time.Second):
|
||||
t.Fatalf("expected codec change")
|
||||
}
|
||||
|
||||
// codec change after bind
|
||||
vp8Pkt := rtp.Packet{
|
||||
Header: rtp.Header{
|
||||
Version: 2,
|
||||
PayloadType: 96,
|
||||
SequenceNumber: 3,
|
||||
Timestamp: 3,
|
||||
SSRC: 123,
|
||||
},
|
||||
Payload: []byte{0xff, 0xff, 0xff, 0xfd, 0xb4, 0x9f, 0x94, 0x1},
|
||||
}
|
||||
buf, err = vp8Pkt.Marshal()
|
||||
require.NoError(t, err)
|
||||
_, err = buff.Write(buf)
|
||||
require.NoError(t, err)
|
||||
|
||||
select {
|
||||
case c := <-changedCodec:
|
||||
require.Equal(t, vp8Codec, c)
|
||||
|
||||
case <-time.After(1 * time.Second):
|
||||
t.Fatalf("expected codec change")
|
||||
}
|
||||
|
||||
// out of order pkts can't cause codec change
|
||||
h265Pkt.SequenceNumber = 2
|
||||
h265Pkt.Timestamp = 2
|
||||
buf, err = h265Pkt.Marshal()
|
||||
require.NoError(t, err)
|
||||
_, err = buff.Write(buf)
|
||||
require.NoError(t, err)
|
||||
select {
|
||||
case <-changedCodec:
|
||||
t.Fatalf("unexpected codec change")
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
}
|
||||
|
||||
// unknown codec should not cause change
|
||||
h265Pkt.SequenceNumber = 4
|
||||
h265Pkt.Timestamp = 4
|
||||
h265Pkt.PayloadType = 117
|
||||
buf, err = h265Pkt.Marshal()
|
||||
require.NoError(t, err)
|
||||
_, err = buff.Write(buf)
|
||||
require.NoError(t, err)
|
||||
select {
|
||||
case <-changedCodec:
|
||||
t.Fatalf("unexpected codec change")
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkMemcpu(b *testing.B) {
|
||||
buf := make([]byte, 1500*1500*10)
|
||||
buf2 := make([]byte, 1500*1500*20)
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
copy(buf2, buf)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
// Copyright 2023 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package buffer
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
|
||||
"github.com/livekit/protocol/livekit"
|
||||
)
|
||||
|
||||
type DataStatsParam struct {
|
||||
WindowDuration time.Duration
|
||||
}
|
||||
|
||||
type DataStats struct {
|
||||
params DataStatsParam
|
||||
lock sync.RWMutex
|
||||
totalBytes int64
|
||||
startTime time.Time
|
||||
endTime time.Time
|
||||
windowStart int64
|
||||
windowBytes int64
|
||||
}
|
||||
|
||||
func NewDataStats(params DataStatsParam) *DataStats {
|
||||
return &DataStats{
|
||||
params: params,
|
||||
startTime: time.Now(),
|
||||
windowStart: time.Now().UnixNano(),
|
||||
}
|
||||
}
|
||||
|
||||
func (s *DataStats) Update(bytes int, time int64) {
|
||||
s.lock.Lock()
|
||||
defer s.lock.Unlock()
|
||||
s.totalBytes += int64(bytes)
|
||||
|
||||
if s.params.WindowDuration > 0 && time-s.windowStart > s.params.WindowDuration.Nanoseconds() {
|
||||
s.windowBytes = 0
|
||||
s.windowStart = time
|
||||
}
|
||||
s.windowBytes += int64(bytes)
|
||||
}
|
||||
|
||||
func (s *DataStats) ToProtoActive() *livekit.RTPStats {
|
||||
if s.params.WindowDuration == 0 {
|
||||
return &livekit.RTPStats{}
|
||||
}
|
||||
s.lock.RLock()
|
||||
defer s.lock.RUnlock()
|
||||
now := time.Now().UnixNano()
|
||||
duration := now - s.windowStart
|
||||
if duration > s.params.WindowDuration.Nanoseconds() {
|
||||
return &livekit.RTPStats{}
|
||||
}
|
||||
|
||||
return &livekit.RTPStats{
|
||||
StartTime: timestamppb.New(time.Unix(s.windowStart/1e9, s.windowStart%1e9)),
|
||||
EndTime: timestamppb.New(time.Unix(0, now)),
|
||||
Duration: float64(duration / 1e9),
|
||||
Bytes: uint64(s.windowBytes),
|
||||
Bitrate: float64(s.windowBytes) * 8 / float64(duration) / 1e9,
|
||||
}
|
||||
}
|
||||
|
||||
func (s *DataStats) Stop() {
|
||||
s.lock.Lock()
|
||||
s.endTime = time.Now()
|
||||
s.lock.Unlock()
|
||||
}
|
||||
|
||||
func (s *DataStats) ToProtoAggregateOnly() *livekit.RTPStats {
|
||||
s.lock.RLock()
|
||||
defer s.lock.RUnlock()
|
||||
|
||||
end := s.endTime
|
||||
if end.IsZero() {
|
||||
end = time.Now()
|
||||
}
|
||||
return &livekit.RTPStats{
|
||||
StartTime: timestamppb.New(s.startTime),
|
||||
EndTime: timestamppb.New(end),
|
||||
Duration: end.Sub(s.startTime).Seconds(),
|
||||
Bytes: uint64(s.totalBytes),
|
||||
Bitrate: float64(s.totalBytes) * 8 / end.Sub(s.startTime).Seconds(),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
// Copyright 2023 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package buffer
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
"google.golang.org/protobuf/proto"
|
||||
|
||||
"github.com/livekit/protocol/livekit"
|
||||
)
|
||||
|
||||
func TestDataStats(t *testing.T) {
|
||||
stats := NewDataStats(DataStatsParam{WindowDuration: time.Second})
|
||||
|
||||
time.Sleep(time.Millisecond)
|
||||
r := stats.ToProtoAggregateOnly()
|
||||
require.Equal(t, r.StartTime.AsTime().UnixNano(), stats.startTime.UnixNano())
|
||||
require.NotZero(t, r.EndTime)
|
||||
require.NotZero(t, r.Duration)
|
||||
r.StartTime = nil
|
||||
r.EndTime = nil
|
||||
r.Duration = 0
|
||||
require.True(t, proto.Equal(r, &livekit.RTPStats{}))
|
||||
|
||||
stats.Update(100, time.Now().UnixNano())
|
||||
r = stats.ToProtoActive()
|
||||
require.EqualValues(t, 100, r.Bytes)
|
||||
require.NotZero(t, r.Bitrate)
|
||||
|
||||
// wait for window duration
|
||||
time.Sleep(time.Second)
|
||||
r = stats.ToProtoActive()
|
||||
require.True(t, proto.Equal(r, &livekit.RTPStats{}))
|
||||
stats.Stop()
|
||||
r = stats.ToProtoAggregateOnly()
|
||||
require.EqualValues(t, 100, r.Bytes)
|
||||
require.NotZero(t, r.Bitrate)
|
||||
}
|
||||
@@ -0,0 +1,218 @@
|
||||
// Copyright 2023 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package buffer
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
|
||||
"github.com/pion/rtp"
|
||||
"go.uber.org/atomic"
|
||||
|
||||
dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor"
|
||||
"github.com/livekit/livekit-server/pkg/sfu/utils"
|
||||
|
||||
"github.com/livekit/protocol/logger"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrFrameEarlierThanKeyFrame = fmt.Errorf("frame is earlier than current keyframe")
|
||||
ErrDDStructureAttachedToNonFirstPacket = fmt.Errorf("dependency descriptor structure is attached to non-first packet of a frame")
|
||||
)
|
||||
|
||||
type DependencyDescriptorParser struct {
|
||||
structure *dd.FrameDependencyStructure
|
||||
ddExtID uint8
|
||||
logger logger.Logger
|
||||
onMaxLayerChanged func(int32, int32)
|
||||
decodeTargets []DependencyDescriptorDecodeTarget
|
||||
|
||||
seqWrapAround *utils.WrapAround[uint16, uint64]
|
||||
frameWrapAround *utils.WrapAround[uint16, uint64]
|
||||
structureExtFrameNum uint64
|
||||
activeDecodeTargetsExtSeq uint64
|
||||
activeDecodeTargetsMask uint32
|
||||
frameChecker *FrameIntegrityChecker
|
||||
|
||||
ddNotFoundCount atomic.Uint32
|
||||
}
|
||||
|
||||
func NewDependencyDescriptorParser(ddExtID uint8, logger logger.Logger, onMaxLayerChanged func(int32, int32)) *DependencyDescriptorParser {
|
||||
return &DependencyDescriptorParser{
|
||||
ddExtID: ddExtID,
|
||||
logger: logger,
|
||||
onMaxLayerChanged: onMaxLayerChanged,
|
||||
seqWrapAround: utils.NewWrapAround[uint16, uint64](utils.WrapAroundParams{IsRestartAllowed: false}),
|
||||
frameWrapAround: utils.NewWrapAround[uint16, uint64](utils.WrapAroundParams{IsRestartAllowed: false}),
|
||||
frameChecker: NewFrameIntegrityChecker(180, 1024), // 2seconds for L3T3 30fps video
|
||||
}
|
||||
}
|
||||
|
||||
type ExtDependencyDescriptor struct {
|
||||
Descriptor *dd.DependencyDescriptor
|
||||
|
||||
DecodeTargets []DependencyDescriptorDecodeTarget
|
||||
StructureUpdated bool
|
||||
ActiveDecodeTargetsUpdated bool
|
||||
Integrity bool
|
||||
ExtFrameNum uint64
|
||||
// the frame number of the keyframe which the current frame depends on
|
||||
ExtKeyFrameNum uint64
|
||||
}
|
||||
|
||||
func (r *DependencyDescriptorParser) Parse(pkt *rtp.Packet) (*ExtDependencyDescriptor, VideoLayer, error) {
|
||||
var videoLayer VideoLayer
|
||||
ddBuf := pkt.GetExtension(r.ddExtID)
|
||||
if ddBuf == nil {
|
||||
ddNotFoundCount := r.ddNotFoundCount.Inc()
|
||||
if ddNotFoundCount%100 == 0 {
|
||||
r.logger.Warnw("dependency descriptor extension is not present", nil, "seq", pkt.SequenceNumber, "count", ddNotFoundCount)
|
||||
}
|
||||
return nil, videoLayer, nil
|
||||
}
|
||||
|
||||
var ddVal dd.DependencyDescriptor
|
||||
ext := &dd.DependencyDescriptorExtension{
|
||||
Descriptor: &ddVal,
|
||||
Structure: r.structure,
|
||||
}
|
||||
_, err := ext.Unmarshal(ddBuf)
|
||||
if err != nil {
|
||||
if err != dd.ErrDDReaderNoStructure && err != dd.ErrDDReaderInvalidTemplateIndex {
|
||||
r.logger.Infow("failed to parse generic dependency descriptor", err, "payload", pkt.PayloadType, "ddbufLen", len(ddBuf))
|
||||
}
|
||||
return nil, videoLayer, err
|
||||
}
|
||||
|
||||
extSeq := r.seqWrapAround.Update(pkt.SequenceNumber).ExtendedVal
|
||||
|
||||
if ddVal.FrameDependencies != nil {
|
||||
videoLayer.Spatial, videoLayer.Temporal = int32(ddVal.FrameDependencies.SpatialId), int32(ddVal.FrameDependencies.TemporalId)
|
||||
}
|
||||
|
||||
unwrapped := r.frameWrapAround.Update(ddVal.FrameNumber)
|
||||
extFN := unwrapped.ExtendedVal
|
||||
|
||||
if extFN < r.structureExtFrameNum {
|
||||
r.logger.Debugw("drop frame which is earlier than current structure", "frameNum", extFN, "structureFrameNum", r.structureExtFrameNum)
|
||||
return nil, videoLayer, ErrFrameEarlierThanKeyFrame
|
||||
}
|
||||
|
||||
r.frameChecker.AddPacket(extSeq, extFN, &ddVal)
|
||||
|
||||
extDD := &ExtDependencyDescriptor{
|
||||
Descriptor: &ddVal,
|
||||
ExtFrameNum: extFN,
|
||||
Integrity: r.frameChecker.FrameIntegrity(extFN),
|
||||
}
|
||||
|
||||
if ddVal.AttachedStructure != nil {
|
||||
if !ddVal.FirstPacketInFrame {
|
||||
r.logger.Warnw("attached structure is not the first packet in frame", nil, "extSeq", extSeq, "extFN", extFN)
|
||||
return nil, videoLayer, ErrDDStructureAttachedToNonFirstPacket
|
||||
}
|
||||
|
||||
if r.structure == nil || ddVal.AttachedStructure.StructureId != r.structure.StructureId {
|
||||
r.logger.Debugw("structure updated", "structureID", ddVal.AttachedStructure.StructureId, "extSeq", extSeq, "extFN", extFN, "descriptor", ddVal.String())
|
||||
}
|
||||
r.structure = ddVal.AttachedStructure
|
||||
r.decodeTargets = ProcessFrameDependencyStructure(ddVal.AttachedStructure)
|
||||
if extFN > unwrapped.PreExtendedHighest && extFN-unwrapped.PreExtendedHighest > 1000 {
|
||||
r.logger.Debugw("large frame number jump on structure updating", "extFN", extFN, "preExtendedHighest", unwrapped.PreExtendedHighest, "structureExtFrameNum", r.structureExtFrameNum)
|
||||
}
|
||||
r.structureExtFrameNum = extFN
|
||||
extDD.StructureUpdated = true
|
||||
extDD.ActiveDecodeTargetsUpdated = true
|
||||
// The dependency descriptor reader will always set ActiveDecodeTargetsBitmask for TemplateDependencyStructure is present,
|
||||
// so don't need to notify max layer change here.
|
||||
}
|
||||
|
||||
if mask := ddVal.ActiveDecodeTargetsBitmask; mask != nil && extSeq > r.activeDecodeTargetsExtSeq {
|
||||
r.activeDecodeTargetsExtSeq = extSeq
|
||||
if *mask != r.activeDecodeTargetsMask {
|
||||
r.activeDecodeTargetsMask = *mask
|
||||
extDD.ActiveDecodeTargetsUpdated = true
|
||||
var maxSpatial, maxTemporal int32
|
||||
for _, dt := range r.decodeTargets {
|
||||
if *mask&(1<<dt.Target) != uint32(dd.DecodeTargetNotPresent) {
|
||||
if maxSpatial < dt.Layer.Spatial {
|
||||
maxSpatial = dt.Layer.Spatial
|
||||
}
|
||||
if maxTemporal < dt.Layer.Temporal {
|
||||
maxTemporal = dt.Layer.Temporal
|
||||
}
|
||||
}
|
||||
}
|
||||
r.logger.Debugw("max layer changed", "maxSpatial", maxSpatial, "maxTemporal", maxTemporal)
|
||||
r.onMaxLayerChanged(maxSpatial, maxTemporal)
|
||||
}
|
||||
}
|
||||
|
||||
extDD.DecodeTargets = r.decodeTargets
|
||||
extDD.ExtKeyFrameNum = r.structureExtFrameNum
|
||||
|
||||
return extDD, videoLayer, nil
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------------------
|
||||
|
||||
type DependencyDescriptorDecodeTarget struct {
|
||||
Target int
|
||||
Layer VideoLayer
|
||||
}
|
||||
|
||||
func (dt *DependencyDescriptorDecodeTarget) String() string {
|
||||
return fmt.Sprintf("DecodeTarget{t: %d, l: %+v}", dt.Target, dt.Layer)
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------------------
|
||||
|
||||
func ProcessFrameDependencyStructure(structure *dd.FrameDependencyStructure) []DependencyDescriptorDecodeTarget {
|
||||
decodeTargets := make([]DependencyDescriptorDecodeTarget, 0, structure.NumDecodeTargets)
|
||||
for target := 0; target < structure.NumDecodeTargets; target++ {
|
||||
layer := VideoLayer{Spatial: 0, Temporal: 0}
|
||||
for _, t := range structure.Templates {
|
||||
if t.DecodeTargetIndications[target] != dd.DecodeTargetNotPresent {
|
||||
if layer.Spatial < int32(t.SpatialId) {
|
||||
layer.Spatial = int32(t.SpatialId)
|
||||
}
|
||||
if layer.Temporal < int32(t.TemporalId) {
|
||||
layer.Temporal = int32(t.TemporalId)
|
||||
}
|
||||
}
|
||||
}
|
||||
decodeTargets = append(decodeTargets, DependencyDescriptorDecodeTarget{target, layer})
|
||||
}
|
||||
|
||||
// sort decode target layer by spatial and temporal from high to low
|
||||
sort.Slice(decodeTargets, func(i, j int) bool {
|
||||
return decodeTargets[i].Layer.GreaterThan(decodeTargets[j].Layer)
|
||||
})
|
||||
|
||||
return decodeTargets
|
||||
}
|
||||
|
||||
func GetActiveDecodeTargetBitmask(layer VideoLayer, decodeTargets []DependencyDescriptorDecodeTarget) *uint32 {
|
||||
activeBitMask := uint32(0)
|
||||
for _, dt := range decodeTargets {
|
||||
if dt.Layer.Spatial <= layer.Spatial && dt.Layer.Temporal <= layer.Temporal {
|
||||
activeBitMask |= 1 << dt.Target
|
||||
}
|
||||
}
|
||||
|
||||
return &activeBitMask
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------------------
|
||||
@@ -0,0 +1,131 @@
|
||||
// Copyright 2023 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package buffer
|
||||
|
||||
import (
|
||||
"io"
|
||||
"sync"
|
||||
|
||||
"github.com/pion/transport/v3/packetio"
|
||||
)
|
||||
|
||||
type FactoryOfBufferFactory struct {
|
||||
trackingPacketsVideo int
|
||||
trackingPacketsAudio int
|
||||
}
|
||||
|
||||
func NewFactoryOfBufferFactory(trackingPacketsVideo int, trackingPacketsAudio int) *FactoryOfBufferFactory {
|
||||
return &FactoryOfBufferFactory{
|
||||
trackingPacketsVideo: trackingPacketsVideo,
|
||||
trackingPacketsAudio: trackingPacketsAudio,
|
||||
}
|
||||
}
|
||||
|
||||
func (f *FactoryOfBufferFactory) CreateBufferFactory() *Factory {
|
||||
return &Factory{
|
||||
trackingPacketsVideo: f.trackingPacketsVideo,
|
||||
trackingPacketsAudio: f.trackingPacketsAudio,
|
||||
rtpBuffers: make(map[uint32]*Buffer),
|
||||
rtcpReaders: make(map[uint32]*RTCPReader),
|
||||
rtxPair: make(map[uint32]uint32),
|
||||
}
|
||||
}
|
||||
|
||||
type Factory struct {
|
||||
sync.RWMutex
|
||||
trackingPacketsVideo int
|
||||
trackingPacketsAudio int
|
||||
rtpBuffers map[uint32]*Buffer
|
||||
rtcpReaders map[uint32]*RTCPReader
|
||||
rtxPair map[uint32]uint32 // repair -> base
|
||||
}
|
||||
|
||||
func (f *Factory) GetOrNew(packetType packetio.BufferPacketType, ssrc uint32) io.ReadWriteCloser {
|
||||
f.Lock()
|
||||
defer f.Unlock()
|
||||
switch packetType {
|
||||
case packetio.RTCPBufferPacket:
|
||||
if reader, ok := f.rtcpReaders[ssrc]; ok {
|
||||
return reader
|
||||
}
|
||||
reader := NewRTCPReader(ssrc)
|
||||
f.rtcpReaders[ssrc] = reader
|
||||
reader.OnClose(func() {
|
||||
f.Lock()
|
||||
delete(f.rtcpReaders, ssrc)
|
||||
f.Unlock()
|
||||
})
|
||||
return reader
|
||||
case packetio.RTPBufferPacket:
|
||||
if reader, ok := f.rtpBuffers[ssrc]; ok {
|
||||
return reader
|
||||
}
|
||||
buffer := NewBuffer(ssrc, f.trackingPacketsVideo, f.trackingPacketsAudio)
|
||||
f.rtpBuffers[ssrc] = buffer
|
||||
for repair, base := range f.rtxPair {
|
||||
if repair == ssrc {
|
||||
baseBuffer, ok := f.rtpBuffers[base]
|
||||
if ok {
|
||||
buffer.SetPrimaryBufferForRTX(baseBuffer)
|
||||
}
|
||||
break
|
||||
} else if base == ssrc {
|
||||
repairBuffer, ok := f.rtpBuffers[repair]
|
||||
if ok {
|
||||
repairBuffer.SetPrimaryBufferForRTX(buffer)
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
buffer.OnClose(func() {
|
||||
f.Lock()
|
||||
delete(f.rtpBuffers, ssrc)
|
||||
delete(f.rtxPair, ssrc)
|
||||
f.Unlock()
|
||||
})
|
||||
return buffer
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *Factory) GetBufferPair(ssrc uint32) (*Buffer, *RTCPReader) {
|
||||
f.RLock()
|
||||
defer f.RUnlock()
|
||||
return f.rtpBuffers[ssrc], f.rtcpReaders[ssrc]
|
||||
}
|
||||
|
||||
func (f *Factory) GetBuffer(ssrc uint32) *Buffer {
|
||||
f.RLock()
|
||||
defer f.RUnlock()
|
||||
return f.rtpBuffers[ssrc]
|
||||
}
|
||||
|
||||
func (f *Factory) GetRTCPReader(ssrc uint32) *RTCPReader {
|
||||
f.RLock()
|
||||
defer f.RUnlock()
|
||||
return f.rtcpReaders[ssrc]
|
||||
}
|
||||
|
||||
func (f *Factory) SetRTXPair(repair, base uint32) {
|
||||
f.Lock()
|
||||
repairBuffer, baseBuffer := f.rtpBuffers[repair], f.rtpBuffers[base]
|
||||
if repairBuffer == nil || baseBuffer == nil {
|
||||
f.rtxPair[repair] = base
|
||||
}
|
||||
f.Unlock()
|
||||
if repairBuffer != nil && baseBuffer != nil {
|
||||
repairBuffer.SetPrimaryBufferForRTX(baseBuffer)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,732 @@
|
||||
// Copyright 2023 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package buffer
|
||||
|
||||
import (
|
||||
"container/list"
|
||||
|
||||
"github.com/pion/rtp/codecs"
|
||||
|
||||
"github.com/livekit/protocol/logger"
|
||||
)
|
||||
|
||||
var minFramesForCalculation = [...]int{8, 15, 40, 60}
|
||||
|
||||
type frameInfo struct {
|
||||
startSeq uint16
|
||||
endSeq uint16
|
||||
ts uint32
|
||||
fn uint16
|
||||
spatial int32
|
||||
temporal int32
|
||||
frameDiff []int
|
||||
}
|
||||
|
||||
type FrameRateCalculator interface {
|
||||
RecvPacket(ep *ExtPacket) bool
|
||||
GetFrameRate() []float32
|
||||
Completed() bool
|
||||
}
|
||||
|
||||
// -----------------------------
|
||||
|
||||
// FrameRateCalculator based on PictureID in VPx
|
||||
type frameRateCalculatorVPx struct {
|
||||
frameRates [DefaultMaxLayerTemporal + 1]float32
|
||||
clockRate uint32
|
||||
logger logger.Logger
|
||||
firstFrames [DefaultMaxLayerTemporal + 1]*frameInfo
|
||||
secondFrames [DefaultMaxLayerTemporal + 1]*frameInfo
|
||||
fnReceived [64]*frameInfo
|
||||
baseFrame *frameInfo
|
||||
completed bool
|
||||
}
|
||||
|
||||
func newFrameRateCalculatorVPx(clockRate uint32, logger logger.Logger) *frameRateCalculatorVPx {
|
||||
return &frameRateCalculatorVPx{
|
||||
clockRate: clockRate,
|
||||
logger: logger,
|
||||
}
|
||||
}
|
||||
|
||||
func (f *frameRateCalculatorVPx) Completed() bool {
|
||||
return f.completed
|
||||
}
|
||||
|
||||
func (f *frameRateCalculatorVPx) RecvPacket(ep *ExtPacket, fn uint16) bool {
|
||||
if f.completed {
|
||||
return true
|
||||
}
|
||||
|
||||
if ep.Temporal >= int32(len(f.frameRates)) {
|
||||
f.logger.Warnw("invalid temporal layer", nil, "temporal", ep.Temporal)
|
||||
return false
|
||||
}
|
||||
|
||||
temporal := ep.Temporal
|
||||
if temporal < 0 {
|
||||
temporal = 0
|
||||
}
|
||||
|
||||
if f.baseFrame == nil {
|
||||
f.baseFrame = &frameInfo{ts: ep.Packet.Timestamp, fn: fn}
|
||||
f.fnReceived[0] = f.baseFrame
|
||||
f.firstFrames[temporal] = f.baseFrame
|
||||
return false
|
||||
}
|
||||
|
||||
baseDiff := fn - f.baseFrame.fn
|
||||
if baseDiff == 0 || baseDiff > 0x4000 {
|
||||
return false
|
||||
}
|
||||
|
||||
if baseDiff >= uint16(len(f.fnReceived)) {
|
||||
// frame number is not continuous, reset
|
||||
f.reset()
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
if f.fnReceived[baseDiff] != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
fi := &frameInfo{
|
||||
ts: ep.Packet.Timestamp,
|
||||
fn: fn,
|
||||
temporal: temporal,
|
||||
}
|
||||
f.fnReceived[baseDiff] = fi
|
||||
|
||||
firstFrame := f.firstFrames[temporal]
|
||||
secondFrame := f.secondFrames[temporal]
|
||||
if firstFrame == nil {
|
||||
f.firstFrames[temporal] = fi
|
||||
firstFrame = fi
|
||||
} else {
|
||||
if (secondFrame == nil || secondFrame.fn < fn) && fn != firstFrame.fn && (fn-firstFrame.fn) < 0x4000 {
|
||||
f.secondFrames[temporal] = fi
|
||||
}
|
||||
}
|
||||
|
||||
return f.calc()
|
||||
}
|
||||
|
||||
func (f *frameRateCalculatorVPx) calc() bool {
|
||||
var rateCounter int
|
||||
for currentTemporal := int32(0); currentTemporal <= DefaultMaxLayerTemporal; currentTemporal++ {
|
||||
if f.frameRates[currentTemporal] > 0 {
|
||||
rateCounter++
|
||||
continue
|
||||
}
|
||||
|
||||
ff := f.firstFrames[currentTemporal]
|
||||
sf := f.secondFrames[currentTemporal]
|
||||
// lower temporal layer has been calculated, but higher layer has not received any frames, it should not exist
|
||||
if rateCounter > 0 && ff == nil {
|
||||
rateCounter++
|
||||
continue
|
||||
}
|
||||
if ff == nil || sf == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
var frameCount int
|
||||
lastTs := ff.ts
|
||||
for j := ff.fn - f.baseFrame.fn + 1; j < sf.fn-f.baseFrame.fn+1; j++ {
|
||||
if f := f.fnReceived[j]; f == nil {
|
||||
break
|
||||
} else if f.temporal <= currentTemporal {
|
||||
frameCount++
|
||||
lastTs = f.ts
|
||||
}
|
||||
}
|
||||
if frameCount >= minFramesForCalculation[currentTemporal] {
|
||||
f.frameRates[currentTemporal] = float32(f.clockRate) / float32(lastTs-ff.ts) * float32(frameCount)
|
||||
rateCounter++
|
||||
}
|
||||
}
|
||||
|
||||
if rateCounter == len(f.frameRates) {
|
||||
f.completed = true
|
||||
|
||||
// normalize frame rates, Microsoft Edge use 3 temporal layers for vp8 but the middle layer has chance to
|
||||
// get a very low frame rate, so we need to normalize the frame rate(use fixed ration 1:2 of highest layer for that layer)
|
||||
if f.frameRates[2] > 0 && f.frameRates[2] > f.frameRates[1]*3 {
|
||||
f.frameRates[1] = f.frameRates[2] / 2
|
||||
}
|
||||
f.reset()
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (f *frameRateCalculatorVPx) reset() {
|
||||
for i := range f.firstFrames {
|
||||
f.firstFrames[i] = nil
|
||||
f.secondFrames[i] = nil
|
||||
}
|
||||
|
||||
for i := range f.fnReceived {
|
||||
f.fnReceived[i] = nil
|
||||
}
|
||||
f.baseFrame = nil
|
||||
}
|
||||
|
||||
func (f *frameRateCalculatorVPx) GetFrameRate() []float32 {
|
||||
return f.frameRates[:]
|
||||
}
|
||||
|
||||
// -----------------------------
|
||||
|
||||
// FrameRateCalculator based on PictureID in VP8
|
||||
type FrameRateCalculatorVP8 struct {
|
||||
*frameRateCalculatorVPx
|
||||
logger logger.Logger
|
||||
}
|
||||
|
||||
func NewFrameRateCalculatorVP8(clockRate uint32, logger logger.Logger) *FrameRateCalculatorVP8 {
|
||||
return &FrameRateCalculatorVP8{
|
||||
frameRateCalculatorVPx: newFrameRateCalculatorVPx(clockRate, logger),
|
||||
logger: logger,
|
||||
}
|
||||
}
|
||||
|
||||
func (f *FrameRateCalculatorVP8) RecvPacket(ep *ExtPacket) bool {
|
||||
if f.frameRateCalculatorVPx.Completed() {
|
||||
return true
|
||||
}
|
||||
|
||||
vp8, ok := ep.Payload.(VP8)
|
||||
if !ok {
|
||||
f.logger.Debugw("no vp8 payload", "sn", ep.Packet.SequenceNumber)
|
||||
return false
|
||||
}
|
||||
success := f.frameRateCalculatorVPx.RecvPacket(ep, vp8.PictureID)
|
||||
|
||||
if f.frameRateCalculatorVPx.Completed() {
|
||||
f.logger.Debugw("frame rate calculated", "rate", f.frameRateCalculatorVPx.GetFrameRate())
|
||||
}
|
||||
|
||||
return success
|
||||
}
|
||||
|
||||
// -----------------------------
|
||||
|
||||
// FrameRateCalculator based on PictureID in VP9
|
||||
type FrameRateCalculatorVP9 struct {
|
||||
logger logger.Logger
|
||||
completed bool
|
||||
|
||||
// VP9-TODO - this is assuming three spatial layers. As `completed` marker relies on all layers being finished, have to assume this. FIX.
|
||||
// Maybe look at number of layers in livekit.TrackInfo and declare completed once advertised layers are measured
|
||||
frameRateCalculatorsVPx [DefaultMaxLayerSpatial + 1]*frameRateCalculatorVPx
|
||||
}
|
||||
|
||||
func NewFrameRateCalculatorVP9(clockRate uint32, logger logger.Logger) *FrameRateCalculatorVP9 {
|
||||
f := &FrameRateCalculatorVP9{
|
||||
logger: logger,
|
||||
}
|
||||
|
||||
for i := range f.frameRateCalculatorsVPx {
|
||||
f.frameRateCalculatorsVPx[i] = newFrameRateCalculatorVPx(clockRate, logger)
|
||||
}
|
||||
|
||||
return f
|
||||
}
|
||||
|
||||
func (f *FrameRateCalculatorVP9) Completed() bool {
|
||||
return f.completed
|
||||
}
|
||||
|
||||
func (f *FrameRateCalculatorVP9) RecvPacket(ep *ExtPacket) bool {
|
||||
if f.completed {
|
||||
return true
|
||||
}
|
||||
|
||||
vp9, ok := ep.Payload.(codecs.VP9Packet)
|
||||
if !ok {
|
||||
f.logger.Debugw("no vp9 payload", "sn", ep.Packet.SequenceNumber)
|
||||
return false
|
||||
}
|
||||
|
||||
if ep.Spatial < 0 || ep.Spatial >= int32(len(f.frameRateCalculatorsVPx)) || f.frameRateCalculatorsVPx[ep.Spatial] == nil {
|
||||
f.logger.Debugw("invalid spatial layer", "sn", ep.Packet.SequenceNumber, "spatial", ep.Spatial)
|
||||
return false
|
||||
}
|
||||
|
||||
success := f.frameRateCalculatorsVPx[ep.Spatial].RecvPacket(ep, vp9.PictureID)
|
||||
|
||||
completed := true
|
||||
for _, frc := range f.frameRateCalculatorsVPx {
|
||||
if !frc.Completed() {
|
||||
completed = false
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if completed {
|
||||
f.completed = true
|
||||
|
||||
var frameRates [DefaultMaxLayerSpatial + 1][]float32
|
||||
for i := range f.frameRateCalculatorsVPx {
|
||||
frameRates[i] = f.frameRateCalculatorsVPx[i].GetFrameRate()
|
||||
}
|
||||
f.logger.Debugw("frame rate calculated", "rate", frameRates)
|
||||
}
|
||||
|
||||
return success
|
||||
}
|
||||
|
||||
func (f *FrameRateCalculatorVP9) GetFrameRateForSpatial(spatial int32) []float32 {
|
||||
if spatial < 0 || spatial >= int32(len(f.frameRateCalculatorsVPx)) || f.frameRateCalculatorsVPx[spatial] == nil {
|
||||
return nil
|
||||
}
|
||||
return f.frameRateCalculatorsVPx[spatial].GetFrameRate()
|
||||
}
|
||||
|
||||
func (f *FrameRateCalculatorVP9) GetFrameRateCalculatorForSpatial(spatial int32) *FrameRateCalculatorForVP9Layer {
|
||||
return &FrameRateCalculatorForVP9Layer{
|
||||
FrameRateCalculatorVP9: f,
|
||||
spatial: spatial,
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------
|
||||
|
||||
type FrameRateCalculatorForVP9Layer struct {
|
||||
*FrameRateCalculatorVP9
|
||||
spatial int32
|
||||
}
|
||||
|
||||
func (f *FrameRateCalculatorForVP9Layer) GetFrameRate() []float32 {
|
||||
return f.FrameRateCalculatorVP9.GetFrameRateForSpatial(f.spatial)
|
||||
}
|
||||
|
||||
// -----------------------------------------------
|
||||
|
||||
// FrameRateCalculator based on Dependency descriptor
|
||||
type FrameRateCalculatorDD struct {
|
||||
frameRates [DefaultMaxLayerSpatial + 1][DefaultMaxLayerTemporal + 1]float32
|
||||
clockRate uint32
|
||||
logger logger.Logger
|
||||
firstFrames [DefaultMaxLayerSpatial + 1][DefaultMaxLayerTemporal + 1]*frameInfo
|
||||
secondFrames [DefaultMaxLayerSpatial + 1][DefaultMaxLayerTemporal + 1]*frameInfo
|
||||
fnReceived [256]*frameInfo
|
||||
baseFrame *frameInfo
|
||||
completed bool
|
||||
|
||||
// frames for each decode target
|
||||
targetFrames [DefaultMaxLayerSpatial + 1][DefaultMaxLayerTemporal + 1]list.List
|
||||
|
||||
maxSpatial, maxTemporal int32
|
||||
}
|
||||
|
||||
func NewFrameRateCalculatorDD(clockRate uint32, logger logger.Logger) *FrameRateCalculatorDD {
|
||||
return &FrameRateCalculatorDD{
|
||||
clockRate: clockRate,
|
||||
logger: logger,
|
||||
maxSpatial: DefaultMaxLayerSpatial,
|
||||
maxTemporal: DefaultMaxLayerTemporal,
|
||||
}
|
||||
}
|
||||
|
||||
func (f *FrameRateCalculatorDD) Completed() bool {
|
||||
return f.completed
|
||||
}
|
||||
|
||||
func (f *FrameRateCalculatorDD) SetMaxLayer(spatial, temporal int32) {
|
||||
f.maxSpatial, f.maxTemporal = spatial, temporal
|
||||
}
|
||||
|
||||
func (f *FrameRateCalculatorDD) RecvPacket(ep *ExtPacket) bool {
|
||||
if f.completed {
|
||||
return true
|
||||
}
|
||||
|
||||
if ep.DependencyDescriptor == nil {
|
||||
f.logger.Debugw("dependency descriptor is nil")
|
||||
return false
|
||||
}
|
||||
|
||||
spatial := ep.Spatial
|
||||
// non-SVC codec will set spatial to -1
|
||||
if spatial < 0 {
|
||||
spatial = 0
|
||||
}
|
||||
temporal := ep.Temporal
|
||||
if temporal < 0 || temporal > DefaultMaxLayerTemporal || spatial > DefaultMaxLayerSpatial {
|
||||
f.logger.Warnw("invalid spatial or temporal", nil, "spatial", spatial, "temporal", temporal, "sn", ep.Packet.SequenceNumber)
|
||||
return false
|
||||
}
|
||||
|
||||
fn := ep.DependencyDescriptor.Descriptor.FrameNumber
|
||||
if f.baseFrame == nil {
|
||||
f.baseFrame = &frameInfo{ts: ep.Packet.Timestamp, fn: fn}
|
||||
f.fnReceived[0] = f.baseFrame
|
||||
f.firstFrames[spatial][temporal] = f.baseFrame
|
||||
f.secondFrames[spatial][temporal] = f.baseFrame
|
||||
return false
|
||||
}
|
||||
|
||||
baseDiff := fn - f.baseFrame.fn
|
||||
if baseDiff == 0 || baseDiff > 0x8000 {
|
||||
return false
|
||||
}
|
||||
|
||||
if baseDiff >= uint16(len(f.fnReceived)) {
|
||||
// frame number is not continuous, reset
|
||||
f.baseFrame = nil
|
||||
for i := range f.firstFrames {
|
||||
for j := range f.firstFrames[i] {
|
||||
f.firstFrames[i][j] = nil
|
||||
f.secondFrames[i][j] = nil
|
||||
f.targetFrames[i][j].Init()
|
||||
}
|
||||
}
|
||||
for i := range f.fnReceived {
|
||||
f.fnReceived[i] = nil
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
if f.fnReceived[baseDiff] != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
fi := &frameInfo{
|
||||
ts: ep.Packet.Timestamp,
|
||||
fn: fn,
|
||||
temporal: temporal,
|
||||
spatial: spatial,
|
||||
frameDiff: ep.DependencyDescriptor.Descriptor.FrameDependencies.FrameDiffs,
|
||||
}
|
||||
f.fnReceived[baseDiff] = fi
|
||||
|
||||
if f.firstFrames[spatial][temporal] == nil {
|
||||
f.firstFrames[spatial][temporal] = fi
|
||||
f.secondFrames[spatial][temporal] = fi
|
||||
return false
|
||||
}
|
||||
|
||||
chain := &f.targetFrames[spatial][temporal]
|
||||
if chain.Len() == 0 {
|
||||
chain.PushBack(fn)
|
||||
}
|
||||
for _, fdiff := range ep.DependencyDescriptor.Descriptor.FrameDependencies.FrameDiffs {
|
||||
dependFrame := fn - uint16(fdiff)
|
||||
// frame too old, ignore
|
||||
if dependFrame-f.secondFrames[spatial][temporal].fn > 0x8000 {
|
||||
continue
|
||||
}
|
||||
|
||||
insertFrame:
|
||||
for e := chain.Back(); e != nil; e = e.Prev() {
|
||||
val := e.Value.(uint16)
|
||||
switch {
|
||||
case val == dependFrame:
|
||||
break insertFrame
|
||||
case sn16LT(val, dependFrame):
|
||||
chain.InsertAfter(dependFrame, e)
|
||||
break insertFrame
|
||||
default:
|
||||
if e == chain.Front() {
|
||||
chain.PushFront(dependFrame)
|
||||
break insertFrame
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return f.calc()
|
||||
}
|
||||
|
||||
func (f *FrameRateCalculatorDD) calc() bool {
|
||||
var rateCounter int
|
||||
for currentSpatial := int32(0); currentSpatial <= f.maxSpatial; currentSpatial++ {
|
||||
var currentSpatialRateCounter int
|
||||
for currentTemporal := int32(0); currentTemporal <= f.maxTemporal; currentTemporal++ {
|
||||
if f.frameRates[currentSpatial][currentTemporal] > 0 {
|
||||
rateCounter++
|
||||
currentSpatialRateCounter++
|
||||
continue
|
||||
}
|
||||
|
||||
firstFrame := f.firstFrames[currentSpatial][currentTemporal]
|
||||
// lower temporal layer has been calculated, but higher layer has not received any frames, it should not exist
|
||||
if currentSpatialRateCounter > 0 && firstFrame == nil {
|
||||
currentSpatialRateCounter++
|
||||
rateCounter++
|
||||
continue
|
||||
}
|
||||
|
||||
chain := &f.targetFrames[currentSpatial][currentTemporal]
|
||||
|
||||
// find last decodable frame (no dependency frame is lost)
|
||||
var lastFrame *frameInfo
|
||||
for e := chain.Front(); e != nil; e = e.Next() {
|
||||
diff := e.Value.(uint16) - f.baseFrame.fn
|
||||
if diff >= uint16(len(f.fnReceived)) {
|
||||
continue
|
||||
}
|
||||
|
||||
fi := f.fnReceived[diff]
|
||||
if fi == nil {
|
||||
break
|
||||
} else {
|
||||
lastFrame = fi
|
||||
if firstFrame == nil && fi.spatial == currentSpatial && fi.temporal == currentTemporal {
|
||||
firstFrame = fi
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if lastFrame != nil && lastFrame.fn > f.secondFrames[currentSpatial][currentTemporal].fn {
|
||||
f.secondFrames[currentSpatial][currentTemporal] = lastFrame
|
||||
} else {
|
||||
continue
|
||||
}
|
||||
|
||||
frameCount := 0
|
||||
for i := firstFrame.fn - f.baseFrame.fn; i <= lastFrame.fn-f.baseFrame.fn; i++ {
|
||||
fi := f.fnReceived[i]
|
||||
if fi == nil {
|
||||
continue
|
||||
}
|
||||
if fi.spatial == currentSpatial && fi.temporal <= currentTemporal {
|
||||
frameCount++
|
||||
}
|
||||
}
|
||||
|
||||
if frameCount >= minFramesForCalculation[currentTemporal] && lastFrame.ts > firstFrame.ts {
|
||||
f.frameRates[currentSpatial][currentTemporal] = float32(f.clockRate) / float32(lastFrame.ts-firstFrame.ts) * float32(frameCount)
|
||||
rateCounter++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if rateCounter == int(f.maxSpatial+1)*int(f.maxTemporal+1) {
|
||||
f.completed = true
|
||||
f.close()
|
||||
|
||||
f.logger.Debugw("frame rate calculated", "rate", f.frameRates)
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (f *FrameRateCalculatorDD) GetFrameRateForSpatial(spatial int32) []float32 {
|
||||
if spatial < 0 || spatial >= int32(len(f.frameRates)) {
|
||||
return nil
|
||||
}
|
||||
return f.frameRates[spatial][:]
|
||||
}
|
||||
|
||||
func (f *FrameRateCalculatorDD) close() {
|
||||
f.baseFrame = nil
|
||||
for i := range f.firstFrames {
|
||||
for j := range f.firstFrames[i] {
|
||||
f.firstFrames[i][j] = nil
|
||||
f.secondFrames[i][j] = nil
|
||||
}
|
||||
}
|
||||
|
||||
for i := range f.fnReceived {
|
||||
f.fnReceived[i] = nil
|
||||
}
|
||||
for i := range f.targetFrames {
|
||||
for j := range f.targetFrames[i] {
|
||||
f.targetFrames[i][j].Init()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (f *FrameRateCalculatorDD) GetFrameRateCalculatorForSpatial(spatial int32) *FrameRateCalculatorForDDLayer {
|
||||
return &FrameRateCalculatorForDDLayer{
|
||||
FrameRateCalculatorDD: f,
|
||||
spatial: spatial,
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------
|
||||
|
||||
type FrameRateCalculatorForDDLayer struct {
|
||||
*FrameRateCalculatorDD
|
||||
spatial int32
|
||||
}
|
||||
|
||||
func (f *FrameRateCalculatorForDDLayer) GetFrameRate() []float32 {
|
||||
return f.FrameRateCalculatorDD.GetFrameRateForSpatial(f.spatial)
|
||||
}
|
||||
|
||||
// -----------------------------------------------
|
||||
|
||||
type FrameRateCalculatorH26x struct {
|
||||
frameRates [DefaultMaxLayerTemporal + 1]float32
|
||||
clockRate uint32
|
||||
logger logger.Logger
|
||||
fnReceived *list.List
|
||||
baseFrame *frameInfo
|
||||
completed bool
|
||||
}
|
||||
|
||||
func NewFrameRateCalculatorH26x(clockRate uint32, logger logger.Logger) *FrameRateCalculatorH26x {
|
||||
return &FrameRateCalculatorH26x{
|
||||
clockRate: clockRate,
|
||||
logger: logger,
|
||||
}
|
||||
}
|
||||
|
||||
func (f *FrameRateCalculatorH26x) Completed() bool {
|
||||
return f.completed
|
||||
}
|
||||
|
||||
func (f *FrameRateCalculatorH26x) RecvPacket(ep *ExtPacket) bool {
|
||||
if f.completed {
|
||||
return true
|
||||
}
|
||||
|
||||
if ep.Temporal >= int32(len(f.frameRates)) {
|
||||
f.logger.Warnw("invalid temporal layer", nil, "temporal", ep.Temporal)
|
||||
return false
|
||||
}
|
||||
|
||||
temporal := ep.Temporal
|
||||
if temporal < 0 {
|
||||
temporal = 0
|
||||
}
|
||||
|
||||
if f.baseFrame == nil {
|
||||
f.baseFrame = &frameInfo{
|
||||
startSeq: ep.Packet.SequenceNumber,
|
||||
endSeq: ep.Packet.SequenceNumber,
|
||||
ts: ep.Packet.Timestamp,
|
||||
temporal: temporal,
|
||||
}
|
||||
f.fnReceived = list.New()
|
||||
f.fnReceived.PushBack(f.baseFrame)
|
||||
return false
|
||||
}
|
||||
|
||||
if sn16LTOrEqual(ep.Packet.SequenceNumber, f.baseFrame.startSeq) {
|
||||
return false
|
||||
}
|
||||
|
||||
insertFrame:
|
||||
for e := f.fnReceived.Back(); e != nil; e = e.Prev() {
|
||||
frame := e.Value.(*frameInfo)
|
||||
switch {
|
||||
case frame.ts == ep.Packet.Timestamp:
|
||||
if sn16LT(frame.endSeq, ep.Packet.SequenceNumber) {
|
||||
frame.endSeq = ep.Packet.SequenceNumber
|
||||
}
|
||||
if sn16LT(ep.Packet.SequenceNumber, frame.startSeq) {
|
||||
frame.startSeq = ep.Packet.SequenceNumber
|
||||
}
|
||||
break insertFrame
|
||||
case sn32LT(frame.ts, ep.Packet.Timestamp):
|
||||
f.fnReceived.InsertAfter(&frameInfo{
|
||||
startSeq: ep.Packet.SequenceNumber,
|
||||
endSeq: ep.Packet.SequenceNumber,
|
||||
ts: ep.Packet.Timestamp,
|
||||
temporal: temporal,
|
||||
}, e)
|
||||
break insertFrame
|
||||
default:
|
||||
if e == f.fnReceived.Front() {
|
||||
f.fnReceived.PushFront(&frameInfo{
|
||||
startSeq: ep.Packet.SequenceNumber,
|
||||
endSeq: ep.Packet.SequenceNumber,
|
||||
ts: ep.Packet.Timestamp,
|
||||
temporal: temporal,
|
||||
})
|
||||
break insertFrame
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return f.calc()
|
||||
}
|
||||
|
||||
func (f *FrameRateCalculatorH26x) calc() bool {
|
||||
frameCounts := make([]int, DefaultMaxLayerTemporal+1)
|
||||
var totalFrameCount int
|
||||
var tsDuration int
|
||||
cur := f.fnReceived.Front()
|
||||
for {
|
||||
next := cur.Next()
|
||||
if next == nil {
|
||||
break
|
||||
}
|
||||
ff := cur.Value.(*frameInfo)
|
||||
nf := next.Value.(*frameInfo)
|
||||
if nf.startSeq-ff.endSeq == 1 {
|
||||
totalFrameCount++
|
||||
tsDuration += int(nf.ts - ff.ts)
|
||||
for i := int(nf.temporal); i < len(frameCounts); i++ {
|
||||
frameCounts[i]++
|
||||
}
|
||||
} else {
|
||||
// reset to find continuous frames
|
||||
totalFrameCount = 0
|
||||
for i := range frameCounts {
|
||||
frameCounts[i] = 0
|
||||
}
|
||||
tsDuration = 0
|
||||
}
|
||||
|
||||
// received enough continuous frames, calculate fps
|
||||
if totalFrameCount >= minFramesForCalculation[DefaultMaxLayerTemporal] {
|
||||
for currentTemporal := int32(0); currentTemporal <= DefaultMaxLayerTemporal; currentTemporal++ {
|
||||
count := frameCounts[currentTemporal]
|
||||
if currentTemporal > 0 && count == frameCounts[currentTemporal-1] {
|
||||
// no frames for this temporal layer
|
||||
f.frameRates[currentTemporal] = 0
|
||||
} else {
|
||||
f.frameRates[currentTemporal] = float32(f.clockRate) / float32(tsDuration) * float32(count)
|
||||
}
|
||||
}
|
||||
f.logger.Debugw("fps changed", "fps", f.GetFrameRate())
|
||||
f.completed = true
|
||||
f.reset()
|
||||
return true
|
||||
}
|
||||
|
||||
cur = next
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func (f *FrameRateCalculatorH26x) reset() {
|
||||
f.fnReceived.Init()
|
||||
f.baseFrame = nil
|
||||
}
|
||||
|
||||
func (f *FrameRateCalculatorH26x) GetFrameRate() []float32 {
|
||||
return f.frameRates[:]
|
||||
}
|
||||
|
||||
// -----------------------------------------------
|
||||
func sn16LT(a, b uint16) bool {
|
||||
return a-b > 0x8000
|
||||
}
|
||||
|
||||
func sn16LTOrEqual(a, b uint16) bool {
|
||||
return a == b || a-b > 0x8000
|
||||
}
|
||||
|
||||
func sn32LT(a, b uint32) bool {
|
||||
return a-b > 0x80000000
|
||||
}
|
||||
@@ -0,0 +1,437 @@
|
||||
// Copyright 2023 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package buffer
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/pion/rtp"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor"
|
||||
"github.com/livekit/protocol/logger"
|
||||
)
|
||||
|
||||
type testFrameInfo struct {
|
||||
header rtp.Header
|
||||
framenumber uint16
|
||||
spatial int
|
||||
temporal int
|
||||
frameDiff []int
|
||||
}
|
||||
|
||||
func (f *testFrameInfo) toVP8() *ExtPacket {
|
||||
return &ExtPacket{
|
||||
Packet: &rtp.Packet{Header: f.header},
|
||||
Payload: VP8{
|
||||
PictureID: f.framenumber,
|
||||
},
|
||||
VideoLayer: VideoLayer{Spatial: InvalidLayerSpatial, Temporal: int32(f.temporal)},
|
||||
}
|
||||
}
|
||||
|
||||
func (f *testFrameInfo) toDD() *ExtPacket {
|
||||
return &ExtPacket{
|
||||
Packet: &rtp.Packet{Header: f.header},
|
||||
DependencyDescriptor: &ExtDependencyDescriptor{
|
||||
Descriptor: &dd.DependencyDescriptor{
|
||||
FrameNumber: f.framenumber,
|
||||
FrameDependencies: &dd.FrameDependencyTemplate{
|
||||
FrameDiffs: f.frameDiff,
|
||||
},
|
||||
},
|
||||
},
|
||||
VideoLayer: VideoLayer{Spatial: int32(f.spatial), Temporal: int32(f.temporal)},
|
||||
}
|
||||
}
|
||||
|
||||
func (f *testFrameInfo) toH26x() *ExtPacket {
|
||||
return &ExtPacket{
|
||||
Packet: &rtp.Packet{Header: f.header},
|
||||
VideoLayer: VideoLayer{Spatial: InvalidLayerSpatial, Temporal: int32(f.temporal)},
|
||||
}
|
||||
}
|
||||
|
||||
func createFrames(startFrameNumber uint16, startTs uint32, startSeq uint16, totalFramesPerSpatial int, fps [][]float32, spatialDependency bool) [][]*testFrameInfo {
|
||||
spatials := len(fps)
|
||||
temporals := len(fps[0])
|
||||
frames := make([][]*testFrameInfo, spatials)
|
||||
for s := 0; s < spatials; s++ {
|
||||
frames[s] = make([]*testFrameInfo, 0, totalFramesPerSpatial)
|
||||
}
|
||||
fn := startFrameNumber
|
||||
|
||||
nextTs := make([][]uint32, spatials)
|
||||
tsStep := make([][]uint32, spatials)
|
||||
for i := 0; i < spatials; i++ {
|
||||
nextTs[i] = make([]uint32, temporals)
|
||||
tsStep[i] = make([]uint32, temporals)
|
||||
for j := 0; j < temporals; j++ {
|
||||
nextTs[i][j] = startTs
|
||||
tsStep[i][j] = uint32(90000 / fps[i][j])
|
||||
}
|
||||
}
|
||||
|
||||
currentTs := make([]uint32, spatials)
|
||||
for i := 0; i < spatials; i++ {
|
||||
currentTs[i] = startTs
|
||||
}
|
||||
for i := 0; i < totalFramesPerSpatial; i++ {
|
||||
for s := 0; s < spatials; s++ {
|
||||
frame := &testFrameInfo{
|
||||
header: rtp.Header{Timestamp: currentTs[s], SequenceNumber: startSeq},
|
||||
framenumber: fn,
|
||||
spatial: s,
|
||||
}
|
||||
for t := 0; t < temporals; t++ {
|
||||
if currentTs[s] >= nextTs[s][t] {
|
||||
frame.temporal = t
|
||||
for nt := t; nt < temporals; nt++ {
|
||||
nextTs[s][nt] += tsStep[s][nt]
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
currentTs[s] += tsStep[s][temporals-1]
|
||||
frames[s] = append(frames[s], frame)
|
||||
fn++
|
||||
startSeq++
|
||||
|
||||
for fidx := len(frames[s]) - 1; fidx >= 0; fidx-- {
|
||||
cf := frames[s][fidx]
|
||||
if cf.header.Timestamp-frame.header.Timestamp > 0x80000000 {
|
||||
frame.frameDiff = append(frame.frameDiff, int(frame.framenumber-cf.framenumber))
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if spatialDependency && frame.spatial > 0 {
|
||||
for fidx := len(frames[frame.spatial-1]) - 1; fidx >= 0; fidx-- {
|
||||
cf := frames[frame.spatial-1][fidx]
|
||||
if cf.header.Timestamp == frame.header.Timestamp {
|
||||
frame.frameDiff = append(frame.frameDiff, int(frame.framenumber-cf.framenumber))
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return frames
|
||||
}
|
||||
|
||||
func verifyFps(t *testing.T, expect, got []float32) {
|
||||
require.Equal(t, len(expect), len(got))
|
||||
for i := 0; i < len(expect); i++ {
|
||||
require.GreaterOrEqual(t, got[i], expect[i]*0.9, "expect %v, got %v", expect, got)
|
||||
require.LessOrEqual(t, got[i], expect[i]*1.1, "expect %v, got %v", expect, got)
|
||||
}
|
||||
}
|
||||
|
||||
type testcase struct {
|
||||
startTs uint32
|
||||
startSeq uint16
|
||||
startFrameNumber uint16
|
||||
fps [][]float32
|
||||
spatialDependency bool
|
||||
}
|
||||
|
||||
func TestFpsVP8(t *testing.T) {
|
||||
cases := map[string]testcase{
|
||||
"normal": {
|
||||
startTs: 12345678,
|
||||
startFrameNumber: 100,
|
||||
fps: [][]float32{{5, 10, 15}, {5, 10, 15}, {7.5, 15, 30}},
|
||||
},
|
||||
"frame number and timestamp wrap": {
|
||||
startTs: (uint32(1) << 31) - 10,
|
||||
startFrameNumber: (uint16(1) << 15) - 10,
|
||||
fps: [][]float32{{5, 10, 15}, {5, 10, 15}, {7.5, 15, 30}},
|
||||
},
|
||||
"2 temporal layers": {
|
||||
startTs: 12345678,
|
||||
startFrameNumber: 100,
|
||||
fps: [][]float32{{7.5, 15}, {7.5, 15}, {15, 30}},
|
||||
},
|
||||
}
|
||||
|
||||
for name, c := range cases {
|
||||
testCase := c
|
||||
t.Run(name, func(t *testing.T) {
|
||||
fps := testCase.fps
|
||||
frames := make([][]*testFrameInfo, 0)
|
||||
vp8calcs := make([]*FrameRateCalculatorVP8, len(fps))
|
||||
for i := range vp8calcs {
|
||||
vp8calcs[i] = NewFrameRateCalculatorVP8(90000, logger.GetLogger())
|
||||
frames = append(frames, createFrames(c.startFrameNumber, c.startTs, 10, 200, [][]float32{fps[i]}, false)[0])
|
||||
}
|
||||
|
||||
var frameratesGot bool
|
||||
for s, fs := range frames {
|
||||
for _, f := range fs {
|
||||
if vp8calcs[s].RecvPacket(f.toVP8()) {
|
||||
frameratesGot = true
|
||||
for _, calc := range vp8calcs {
|
||||
if !calc.Completed() {
|
||||
frameratesGot = false
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
require.True(t, frameratesGot)
|
||||
for i, calc := range vp8calcs {
|
||||
fpsExpected := fps[i]
|
||||
fpsGot := calc.GetFrameRate()
|
||||
verifyFps(t, fpsExpected, fpsGot[:len(fpsExpected)])
|
||||
}
|
||||
})
|
||||
}
|
||||
t.Run("packet lost and duplicate", func(t *testing.T) {
|
||||
fps := [][]float32{{7.5, 15}, {7.5, 15}, {15, 30}}
|
||||
frames := make([][]*testFrameInfo, 0)
|
||||
vp8calcs := make([]*FrameRateCalculatorVP8, len(fps))
|
||||
for i := range vp8calcs {
|
||||
vp8calcs[i] = NewFrameRateCalculatorVP8(90000, logger.GetLogger())
|
||||
frames = append(frames, createFrames(100, 12345678, 10, 300, [][]float32{fps[i]}, false)[0])
|
||||
for j := 5; j < 130; j++ {
|
||||
if j%2 == 0 {
|
||||
frames[i][j] = frames[i][j-1]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var frameratesGot bool
|
||||
for s, fs := range frames {
|
||||
for _, f := range fs {
|
||||
if vp8calcs[s].RecvPacket(f.toVP8()) {
|
||||
frameratesGot = true
|
||||
for _, calc := range vp8calcs {
|
||||
if !calc.Completed() {
|
||||
frameratesGot = false
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
require.True(t, frameratesGot)
|
||||
for i, calc := range vp8calcs {
|
||||
fpsExpected := fps[i]
|
||||
fpsGot := calc.GetFrameRate()
|
||||
verifyFps(t, fpsExpected, fpsGot[:len(fpsExpected)])
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestFpsDD(t *testing.T) {
|
||||
cases := map[string]testcase{
|
||||
"normal": {
|
||||
startTs: 12345678,
|
||||
startFrameNumber: 100,
|
||||
fps: [][]float32{{5.1, 10.1, 16}, {5.1, 10.1, 16}, {8, 15, 30.1}},
|
||||
spatialDependency: true,
|
||||
},
|
||||
"frame number and timestamp wrap": {
|
||||
startTs: (uint32(1) << 31) - 10,
|
||||
startFrameNumber: (uint16(1) << 15) - 10,
|
||||
fps: [][]float32{{7.5, 15, 30}, {7.5, 15, 30}, {7.5, 15, 30}},
|
||||
spatialDependency: true,
|
||||
},
|
||||
"vp8": {
|
||||
startTs: 12345678,
|
||||
startFrameNumber: 100,
|
||||
fps: [][]float32{{7.5, 15}, {7.5, 15}, {15, 30}},
|
||||
spatialDependency: false,
|
||||
},
|
||||
}
|
||||
|
||||
for name, c := range cases {
|
||||
testCase := c
|
||||
t.Run(name, func(t *testing.T) {
|
||||
fps := testCase.fps
|
||||
frames := createFrames(c.startFrameNumber, c.startTs, 10, 500, fps, testCase.spatialDependency)
|
||||
ddcalc := NewFrameRateCalculatorDD(90000, logger.GetLogger())
|
||||
ddcalc.SetMaxLayer(int32(len(fps)-1), int32(len(fps[0])-1))
|
||||
ddcalcs := make([]FrameRateCalculator, len(fps))
|
||||
for i := range fps {
|
||||
ddcalcs[i] = ddcalc.GetFrameRateCalculatorForSpatial(int32(i))
|
||||
}
|
||||
|
||||
var frameratesGot bool
|
||||
for s, fs := range frames {
|
||||
for _, f := range fs {
|
||||
if ddcalcs[s].RecvPacket(f.toDD()) {
|
||||
frameratesGot = true
|
||||
for _, calc := range ddcalcs {
|
||||
if !calc.Completed() {
|
||||
frameratesGot = false
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
require.True(t, frameratesGot)
|
||||
for i, calc := range ddcalcs {
|
||||
fpsExpected := fps[i]
|
||||
fpsGot := calc.GetFrameRate()
|
||||
verifyFps(t, fpsExpected, fpsGot[:len(fpsExpected)])
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("packet lost and duplicate", func(t *testing.T) {
|
||||
fps := [][]float32{{7.5, 15, 30}, {7.5, 15, 30}, {7.5, 15, 30}}
|
||||
frames := createFrames(100, 12345678, 10, 500, fps, true)
|
||||
ddcalc := NewFrameRateCalculatorDD(90000, logger.GetLogger())
|
||||
ddcalc.SetMaxLayer(int32(len(fps)-1), int32(len(fps[0])-1))
|
||||
ddcalcs := make([]FrameRateCalculator, len(fps))
|
||||
for i := range fps {
|
||||
ddcalcs[i] = ddcalc.GetFrameRateCalculatorForSpatial(int32(i))
|
||||
for j := 5; j < 130; j++ {
|
||||
if j%2 == 0 {
|
||||
frames[i][j] = frames[i][j-1]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var frameratesGot bool
|
||||
for s, fs := range frames {
|
||||
for _, f := range fs {
|
||||
if ddcalcs[s].RecvPacket(f.toDD()) {
|
||||
frameratesGot = true
|
||||
for _, calc := range ddcalcs {
|
||||
if !calc.Completed() {
|
||||
frameratesGot = false
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
require.True(t, frameratesGot)
|
||||
for i, calc := range ddcalcs {
|
||||
fpsExpected := fps[i]
|
||||
fpsGot := calc.GetFrameRate()
|
||||
verifyFps(t, fpsExpected, fpsGot[:len(fpsExpected)])
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestFpsH26x(t *testing.T) {
|
||||
cases := map[string]testcase{
|
||||
"normal": {
|
||||
startTs: 12345678,
|
||||
startSeq: 100,
|
||||
startFrameNumber: 100,
|
||||
fps: [][]float32{{5, 10, 15}, {5, 10, 15}, {7.5, 15, 30}},
|
||||
},
|
||||
"frame number and timestamp wrap": {
|
||||
startTs: (uint32(1) << 31) - 10,
|
||||
startSeq: (uint16(1) << 15) - 10,
|
||||
startFrameNumber: (uint16(1) << 15) - 10,
|
||||
fps: [][]float32{{5, 10, 15}, {5, 10, 15}, {7.5, 15, 30}},
|
||||
},
|
||||
"2 temporal layers": {
|
||||
startTs: 12345678,
|
||||
startFrameNumber: 100,
|
||||
fps: [][]float32{{7.5, 15}, {7.5, 15}, {15, 30}},
|
||||
},
|
||||
}
|
||||
|
||||
for name, c := range cases {
|
||||
testCase := c
|
||||
t.Run(name, func(t *testing.T) {
|
||||
fps := testCase.fps
|
||||
frames := make([][]*testFrameInfo, 0)
|
||||
h26xcalcs := make([]*FrameRateCalculatorH26x, len(fps))
|
||||
for i := range h26xcalcs {
|
||||
h26xcalcs[i] = NewFrameRateCalculatorH26x(90000, logger.GetLogger())
|
||||
frames = append(frames, createFrames(c.startFrameNumber, c.startTs, c.startSeq, 200, [][]float32{fps[i]}, false)[0])
|
||||
}
|
||||
|
||||
var frameratesGot bool
|
||||
for s, fs := range frames {
|
||||
for _, f := range fs {
|
||||
if h26xcalcs[s].RecvPacket(f.toH26x()) {
|
||||
frameratesGot = true
|
||||
for _, calc := range h26xcalcs {
|
||||
if !calc.Completed() {
|
||||
frameratesGot = false
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
require.True(t, frameratesGot)
|
||||
for i, calc := range h26xcalcs {
|
||||
fpsExpected := fps[i]
|
||||
fpsGot := calc.GetFrameRate()
|
||||
verifyFps(t, fpsExpected, fpsGot[:len(fpsExpected)])
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("packet lost and duplicate", func(t *testing.T) {
|
||||
fps := [][]float32{{7.5, 15, 30}, {7.5, 15, 30}, {7.5, 15, 30}}
|
||||
frames := make([][]*testFrameInfo, 0, len(fps))
|
||||
h26xcalcs := make([]FrameRateCalculator, len(fps))
|
||||
for i := range fps {
|
||||
frames = append(frames, createFrames(100, 12345678, 10, 500, [][]float32{fps[i]}, false)[0])
|
||||
h26xcalcs[i] = NewFrameRateCalculatorH26x(90000, logger.GetLogger())
|
||||
for j := 5; j < 130; j++ {
|
||||
if j%2 == 0 {
|
||||
frames[i][j] = frames[i][j-1]
|
||||
}
|
||||
}
|
||||
for j := 130; j < 230; j++ {
|
||||
if j%3 == 0 {
|
||||
frames[i][j] = nil
|
||||
}
|
||||
}
|
||||
for j := 230; j < 330; j++ {
|
||||
if j%2 == 0 {
|
||||
frames[i][j], frames[i][j-1] = frames[i][j-1], frames[i][j]
|
||||
}
|
||||
}
|
||||
}
|
||||
var frameratesGot bool
|
||||
for s, fs := range frames {
|
||||
for _, f := range fs {
|
||||
if f == nil {
|
||||
continue
|
||||
}
|
||||
if h26xcalcs[s].RecvPacket(f.toH26x()) {
|
||||
frameratesGot = true
|
||||
for _, calc := range h26xcalcs {
|
||||
if !calc.Completed() {
|
||||
frameratesGot = false
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
require.True(t, frameratesGot)
|
||||
for i, calc := range h26xcalcs {
|
||||
fpsExpected := fps[i]
|
||||
fpsGot := calc.GetFrameRate()
|
||||
verifyFps(t, fpsExpected, fpsGot[:len(fpsExpected)])
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,225 @@
|
||||
// Copyright 2024 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package buffer
|
||||
|
||||
import (
|
||||
dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor"
|
||||
)
|
||||
|
||||
type FrameEntity struct {
|
||||
startSeq *uint64
|
||||
endSeq *uint64
|
||||
integrity bool
|
||||
|
||||
pktHistory *PacketHistory
|
||||
}
|
||||
|
||||
func (fe *FrameEntity) AddPacket(extSeq uint64, ddVal *dd.DependencyDescriptor) {
|
||||
// duplicate packet
|
||||
if fe.integrity {
|
||||
return
|
||||
}
|
||||
|
||||
if fe.startSeq == nil && ddVal.FirstPacketInFrame {
|
||||
fe.startSeq = &extSeq
|
||||
}
|
||||
if fe.endSeq == nil && ddVal.LastPacketInFrame {
|
||||
fe.endSeq = &extSeq
|
||||
}
|
||||
|
||||
if fe.startSeq != nil && fe.endSeq != nil {
|
||||
if fe.pktHistory.PacketsConsecutive(*fe.startSeq, *fe.endSeq) {
|
||||
fe.integrity = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (fe *FrameEntity) Reset() {
|
||||
fe.integrity = false
|
||||
fe.startSeq, fe.endSeq = nil, nil
|
||||
}
|
||||
|
||||
func (fe *FrameEntity) Integrity() bool {
|
||||
return fe.integrity
|
||||
}
|
||||
|
||||
// ------------------------------
|
||||
|
||||
type PacketHistory struct {
|
||||
base uint64
|
||||
last uint64
|
||||
bits []uint64
|
||||
packetCount int
|
||||
inited bool
|
||||
}
|
||||
|
||||
func NewPacketHistory(packetCount int) *PacketHistory {
|
||||
packetCount = (packetCount + 63) / 64 * 64
|
||||
return &PacketHistory{
|
||||
bits: make([]uint64, packetCount/64),
|
||||
packetCount: packetCount,
|
||||
}
|
||||
}
|
||||
|
||||
func (ph *PacketHistory) AddPacket(extSeq uint64) {
|
||||
if !ph.inited {
|
||||
ph.inited = true
|
||||
ph.base = uint64(extSeq)
|
||||
// set base to extSeq-100 to avoid out-of-order packets belongs to first frame to be dropped
|
||||
if ph.base > 100 {
|
||||
ph.base -= 100
|
||||
} else {
|
||||
ph.base = 0
|
||||
}
|
||||
ph.last = uint64(extSeq)
|
||||
ph.set(extSeq, true)
|
||||
return
|
||||
}
|
||||
|
||||
if extSeq <= ph.base {
|
||||
// too old
|
||||
return
|
||||
}
|
||||
|
||||
if extSeq <= ph.last {
|
||||
if ph.last-extSeq < uint64(ph.packetCount) {
|
||||
ph.set(extSeq, true)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
for i := ph.last + 1; i < extSeq; i++ {
|
||||
ph.set(i, false)
|
||||
}
|
||||
|
||||
ph.set(extSeq, true)
|
||||
ph.last = extSeq
|
||||
}
|
||||
|
||||
func (ph *PacketHistory) getPos(seq uint64) (index, offset int) {
|
||||
idx := (seq - ph.base) % uint64(ph.packetCount)
|
||||
return int(idx >> 6), int(idx % 64)
|
||||
}
|
||||
|
||||
func (ph *PacketHistory) set(seq uint64, received bool) {
|
||||
idx, offset := ph.getPos(seq)
|
||||
if !received {
|
||||
ph.bits[idx] &= ^(1 << offset)
|
||||
} else {
|
||||
ph.bits[idx] |= 1 << (offset)
|
||||
}
|
||||
}
|
||||
|
||||
func (ph *PacketHistory) PacketsConsecutive(start, end uint64) bool {
|
||||
if start > end {
|
||||
return false
|
||||
}
|
||||
|
||||
if end-start >= uint64(ph.packetCount) {
|
||||
return false
|
||||
}
|
||||
|
||||
startIndex, startOffset := ph.getPos(start)
|
||||
endIndex, endOffset := ph.getPos(end)
|
||||
|
||||
if startIndex == endIndex && end-start <= 64 {
|
||||
testBits := uint64((1<<(endOffset-startOffset+1))-1) << startOffset
|
||||
return ph.bits[startIndex]&testBits == testBits
|
||||
}
|
||||
|
||||
if (ph.bits[startIndex]>>(startOffset))+1 != 1<<(64-startOffset) {
|
||||
return false
|
||||
}
|
||||
|
||||
for i := startIndex + 1; i != endIndex; i++ {
|
||||
if i == len(ph.bits) {
|
||||
i = 0
|
||||
if i == endIndex {
|
||||
break
|
||||
}
|
||||
}
|
||||
if ph.bits[i]+1 != 0 {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
testBits := uint64((1 << (endOffset + 1)) - 1)
|
||||
return ph.bits[endIndex]&testBits == testBits
|
||||
}
|
||||
|
||||
// ------------------------------
|
||||
|
||||
type FrameIntegrityChecker struct {
|
||||
frameCount int
|
||||
frames []FrameEntity
|
||||
base uint64
|
||||
last uint64
|
||||
|
||||
pktHistory *PacketHistory
|
||||
inited bool
|
||||
}
|
||||
|
||||
func NewFrameIntegrityChecker(frameCount, packetCount int) *FrameIntegrityChecker {
|
||||
fc := &FrameIntegrityChecker{
|
||||
frames: make([]FrameEntity, frameCount),
|
||||
pktHistory: NewPacketHistory(packetCount),
|
||||
frameCount: frameCount,
|
||||
}
|
||||
|
||||
for i := range fc.frames {
|
||||
fc.frames[i].pktHistory = fc.pktHistory
|
||||
fc.frames[i].Reset()
|
||||
}
|
||||
return fc
|
||||
}
|
||||
|
||||
func (fc *FrameIntegrityChecker) AddPacket(extSeq uint64, extFrameNum uint64, ddVal *dd.DependencyDescriptor) {
|
||||
fc.pktHistory.AddPacket(extSeq)
|
||||
|
||||
if !fc.inited {
|
||||
fc.inited = true
|
||||
fc.base = extFrameNum
|
||||
fc.last = extFrameNum
|
||||
}
|
||||
|
||||
if extFrameNum < fc.base {
|
||||
// frame too old
|
||||
return
|
||||
}
|
||||
|
||||
if extFrameNum <= fc.last {
|
||||
if fc.last-extFrameNum >= uint64(fc.frameCount) {
|
||||
// frame too old
|
||||
return
|
||||
}
|
||||
fc.frames[int(extFrameNum-fc.base)%fc.frameCount].AddPacket(extSeq, ddVal)
|
||||
return
|
||||
}
|
||||
|
||||
// reset missing frames
|
||||
for i := fc.last + 1; i <= extFrameNum; i++ {
|
||||
fc.frames[int(i-fc.base)%fc.frameCount].Reset()
|
||||
}
|
||||
fc.frames[int(extFrameNum-fc.base)%fc.frameCount].AddPacket(extSeq, ddVal)
|
||||
fc.last = extFrameNum
|
||||
}
|
||||
|
||||
func (fc *FrameIntegrityChecker) FrameIntegrity(extFrameNum uint64) bool {
|
||||
if extFrameNum < fc.base || extFrameNum > fc.last || fc.last-extFrameNum >= uint64(fc.frameCount) {
|
||||
return false
|
||||
}
|
||||
|
||||
return fc.frames[int(extFrameNum-fc.base)%fc.frameCount].Integrity()
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
// Copyright 2024 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package buffer
|
||||
|
||||
import (
|
||||
"math/rand"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor"
|
||||
)
|
||||
|
||||
func TestFrameIntegrityChecker(t *testing.T) {
|
||||
fc := NewFrameIntegrityChecker(100, 1000)
|
||||
|
||||
// first frame out of order
|
||||
fc.AddPacket(10, 10, &dd.DependencyDescriptor{})
|
||||
require.False(t, fc.FrameIntegrity(10))
|
||||
fc.AddPacket(9, 10, &dd.DependencyDescriptor{FirstPacketInFrame: true})
|
||||
require.False(t, fc.FrameIntegrity(10))
|
||||
fc.AddPacket(11, 10, &dd.DependencyDescriptor{LastPacketInFrame: true})
|
||||
require.True(t, fc.FrameIntegrity(10))
|
||||
|
||||
// single packet frame
|
||||
fc.AddPacket(100, 100, &dd.DependencyDescriptor{FirstPacketInFrame: true, LastPacketInFrame: true})
|
||||
require.True(t, fc.FrameIntegrity(100))
|
||||
require.False(t, fc.FrameIntegrity(101))
|
||||
require.False(t, fc.FrameIntegrity(99))
|
||||
|
||||
// frame too old than first frame
|
||||
fc.AddPacket(99, 99, &dd.DependencyDescriptor{FirstPacketInFrame: true, LastPacketInFrame: true})
|
||||
|
||||
// multiple packet frame, out of order
|
||||
fc.AddPacket(2001, 2001, &dd.DependencyDescriptor{})
|
||||
require.False(t, fc.FrameIntegrity(2001))
|
||||
require.False(t, fc.FrameIntegrity(1999))
|
||||
// out of frame count(100)
|
||||
require.False(t, fc.FrameIntegrity(100))
|
||||
require.False(t, fc.FrameIntegrity(1900))
|
||||
|
||||
fc.AddPacket(2000, 2001, &dd.DependencyDescriptor{FirstPacketInFrame: true})
|
||||
require.False(t, fc.FrameIntegrity(2001))
|
||||
fc.AddPacket(2002, 2001, &dd.DependencyDescriptor{LastPacketInFrame: true})
|
||||
require.True(t, fc.FrameIntegrity(2001))
|
||||
// duplicate packet
|
||||
fc.AddPacket(2001, 2001, &dd.DependencyDescriptor{})
|
||||
require.True(t, fc.FrameIntegrity(2001))
|
||||
|
||||
// frame too old
|
||||
fc.AddPacket(900, 1900, &dd.DependencyDescriptor{FirstPacketInFrame: true, LastPacketInFrame: true})
|
||||
require.False(t, fc.FrameIntegrity(1900))
|
||||
|
||||
for frame := uint64(2002); frame < 2102; frame++ {
|
||||
// large frame (1000 packets) out of order / retransmitted
|
||||
firstFrame := uint64(3000 + (frame-2002)*1000)
|
||||
lastFrame := uint64(3999 + (frame-2002)*1000)
|
||||
frames := make([]uint64, 0, lastFrame-firstFrame+1)
|
||||
for i := firstFrame; i <= lastFrame; i++ {
|
||||
frames = append(frames, i)
|
||||
}
|
||||
require.False(t, fc.FrameIntegrity(frame))
|
||||
rand.Seed(int64(frame))
|
||||
rand.Shuffle(len(frames), func(i, j int) { frames[i], frames[j] = frames[j], frames[i] })
|
||||
for i, f := range frames {
|
||||
fc.AddPacket(f, frame, &dd.DependencyDescriptor{
|
||||
FirstPacketInFrame: f == firstFrame,
|
||||
LastPacketInFrame: f == lastFrame,
|
||||
})
|
||||
require.Equal(t, i == len(frames)-1, fc.FrameIntegrity(frame), i)
|
||||
}
|
||||
require.True(t, fc.FrameIntegrity(frame))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,455 @@
|
||||
// Copyright 2023 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package buffer
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
|
||||
"github.com/pion/rtp/codecs"
|
||||
|
||||
"github.com/livekit/protocol/logger"
|
||||
)
|
||||
|
||||
var (
|
||||
errShortPacket = errors.New("packet is not large enough")
|
||||
errNilPacket = errors.New("invalid nil packet")
|
||||
errInvalidPacket = errors.New("invalid packet")
|
||||
)
|
||||
|
||||
// VP8 is a helper to get temporal data from VP8 packet header
|
||||
/*
|
||||
VP8 Payload Descriptor
|
||||
0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7
|
||||
+-+-+-+-+-+-+-+-+ +-+-+-+-+-+-+-+-+
|
||||
|X|R|N|S|R| PID | (REQUIRED) |X|R|N|S|R| PID | (REQUIRED)
|
||||
+-+-+-+-+-+-+-+-+ +-+-+-+-+-+-+-+-+
|
||||
X: |I|L|T|K| RSV | (OPTIONAL) X: |I|L|T|K| RSV | (OPTIONAL)
|
||||
+-+-+-+-+-+-+-+-+ +-+-+-+-+-+-+-+-+
|
||||
I: |M| PictureID | (OPTIONAL) I: |M| PictureID | (OPTIONAL)
|
||||
+-+-+-+-+-+-+-+-+ +-+-+-+-+-+-+-+-+
|
||||
L: | TL0PICIDX | (OPTIONAL) | PictureID |
|
||||
+-+-+-+-+-+-+-+-+ +-+-+-+-+-+-+-+-+
|
||||
T/K:|TID|Y| KEYIDX | (OPTIONAL) L: | TL0PICIDX | (OPTIONAL)
|
||||
+-+-+-+-+-+-+-+-+ +-+-+-+-+-+-+-+-+
|
||||
T/K:|TID|Y| KEYIDX | (OPTIONAL)
|
||||
+-+-+-+-+-+-+-+-+
|
||||
*/
|
||||
type VP8 struct {
|
||||
FirstByte byte
|
||||
S bool
|
||||
|
||||
I bool
|
||||
M bool
|
||||
PictureID uint16 /* 7 or 15 bits, picture ID */
|
||||
|
||||
L bool
|
||||
TL0PICIDX uint8 /* 8 bits temporal level zero index */
|
||||
|
||||
// Optional Header If either of the T or K bits are set to 1,
|
||||
// the TID/Y/KEYIDX extension field MUST be present.
|
||||
T bool
|
||||
TID uint8 /* 2 bits temporal layer idx */
|
||||
Y bool
|
||||
|
||||
K bool
|
||||
KEYIDX uint8 /* 5 bits of key frame idx */
|
||||
|
||||
HeaderSize int
|
||||
|
||||
// IsKeyFrame is a helper to detect if current packet is a keyframe
|
||||
IsKeyFrame bool
|
||||
}
|
||||
|
||||
// Unmarshal parses the passed byte slice and stores the result in the VP8 this method is called upon
|
||||
func (v *VP8) Unmarshal(payload []byte) error {
|
||||
if payload == nil {
|
||||
return errNilPacket
|
||||
}
|
||||
|
||||
payloadLen := len(payload)
|
||||
if payloadLen < 1 {
|
||||
return errShortPacket
|
||||
}
|
||||
|
||||
idx := 0
|
||||
v.FirstByte = payload[idx]
|
||||
v.S = payload[idx]&0x10 > 0
|
||||
// Check for extended bit control
|
||||
if payload[idx]&0x80 > 0 {
|
||||
idx++
|
||||
if payloadLen < idx+1 {
|
||||
return errShortPacket
|
||||
}
|
||||
v.I = payload[idx]&0x80 > 0
|
||||
v.L = payload[idx]&0x40 > 0
|
||||
v.T = payload[idx]&0x20 > 0
|
||||
v.K = payload[idx]&0x10 > 0
|
||||
if v.L && !v.T {
|
||||
return errInvalidPacket
|
||||
}
|
||||
|
||||
if v.I {
|
||||
idx++
|
||||
if payloadLen < idx+1 {
|
||||
return errShortPacket
|
||||
}
|
||||
pid := payload[idx] & 0x7f
|
||||
// if m is 1, then Picture ID is 15 bits
|
||||
v.M = payload[idx]&0x80 > 0
|
||||
if v.M {
|
||||
idx++
|
||||
if payloadLen < idx+1 {
|
||||
return errShortPacket
|
||||
}
|
||||
v.PictureID = binary.BigEndian.Uint16([]byte{pid, payload[idx]})
|
||||
} else {
|
||||
v.PictureID = uint16(pid)
|
||||
}
|
||||
}
|
||||
|
||||
if v.L {
|
||||
idx++
|
||||
if payloadLen < idx+1 {
|
||||
return errShortPacket
|
||||
}
|
||||
v.TL0PICIDX = payload[idx]
|
||||
}
|
||||
|
||||
if v.T || v.K {
|
||||
idx++
|
||||
if payloadLen < idx+1 {
|
||||
return errShortPacket
|
||||
}
|
||||
|
||||
if v.T {
|
||||
v.TID = (payload[idx] & 0xc0) >> 6
|
||||
v.Y = (payload[idx] & 0x20) > 0
|
||||
}
|
||||
|
||||
if v.K {
|
||||
v.KEYIDX = payload[idx] & 0x1f
|
||||
}
|
||||
}
|
||||
idx++
|
||||
if payloadLen < idx+1 {
|
||||
return errShortPacket
|
||||
}
|
||||
|
||||
// Check is packet is a keyframe by looking at P bit in vp8 payload
|
||||
v.IsKeyFrame = payload[idx]&0x01 == 0 && v.S
|
||||
} else {
|
||||
idx++
|
||||
if payloadLen < idx+1 {
|
||||
return errShortPacket
|
||||
}
|
||||
// Check is packet is a keyframe by looking at P bit in vp8 payload
|
||||
v.IsKeyFrame = payload[idx]&0x01 == 0 && v.S
|
||||
}
|
||||
v.HeaderSize = idx
|
||||
return nil
|
||||
}
|
||||
|
||||
func (v *VP8) Marshal() ([]byte, error) {
|
||||
buf := make([]byte, v.HeaderSize)
|
||||
n, err := v.MarshalTo(buf)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return buf[:n], err
|
||||
}
|
||||
|
||||
func (v *VP8) MarshalTo(buf []byte) (int, error) {
|
||||
if len(buf) < v.HeaderSize {
|
||||
return 0, errShortPacket
|
||||
}
|
||||
|
||||
idx := 0
|
||||
buf[idx] = v.FirstByte
|
||||
if v.I || v.L || v.T || v.K {
|
||||
buf[idx] |= 0x80 // X bit
|
||||
idx++
|
||||
|
||||
xpos := idx
|
||||
xval := byte(0)
|
||||
|
||||
idx++
|
||||
if v.I {
|
||||
xval |= (1 << 7)
|
||||
if v.M {
|
||||
buf[idx] = 0x80 | byte((v.PictureID>>8)&0x7f)
|
||||
buf[idx+1] = byte(v.PictureID & 0xff)
|
||||
idx += 2
|
||||
} else {
|
||||
buf[idx] = byte(v.PictureID)
|
||||
idx++
|
||||
}
|
||||
}
|
||||
|
||||
if v.L {
|
||||
xval |= (1 << 6)
|
||||
buf[idx] = v.TL0PICIDX
|
||||
idx++
|
||||
}
|
||||
|
||||
if v.T || v.K {
|
||||
buf[idx] = 0
|
||||
if v.T {
|
||||
xval |= (1 << 5)
|
||||
buf[idx] = v.TID << 6
|
||||
if v.Y {
|
||||
buf[idx] |= (1 << 5)
|
||||
}
|
||||
}
|
||||
|
||||
if v.K {
|
||||
xval |= (1 << 4)
|
||||
buf[idx] |= v.KEYIDX & 0x1f
|
||||
}
|
||||
idx++
|
||||
}
|
||||
|
||||
buf[xpos] = xval
|
||||
} else {
|
||||
buf[idx] &^= 0x80 // X bit
|
||||
idx++
|
||||
}
|
||||
|
||||
return idx, nil
|
||||
}
|
||||
|
||||
// -------------------------------------
|
||||
|
||||
func VPxPictureIdSizeDiff(mBit1 bool, mBit2 bool) int {
|
||||
if mBit1 == mBit2 {
|
||||
return 0
|
||||
}
|
||||
|
||||
if mBit1 {
|
||||
return 1
|
||||
}
|
||||
|
||||
return -1
|
||||
}
|
||||
|
||||
// -------------------------------------
|
||||
|
||||
// IsH264KeyFrame detects if h264 payload is a keyframe
|
||||
// this code was taken from https://github.com/jech/galene/blob/codecs/rtpconn/rtpreader.go#L45
|
||||
// all credits belongs to Juliusz Chroboczek @jech and the awesome Galene SFU
|
||||
func IsH264KeyFrame(payload []byte) bool {
|
||||
if len(payload) < 1 {
|
||||
return false
|
||||
}
|
||||
nalu := payload[0] & 0x1F
|
||||
if nalu == 0 {
|
||||
// reserved
|
||||
return false
|
||||
} else if nalu <= 23 {
|
||||
// simple NALU
|
||||
return nalu == 7
|
||||
} else if nalu == 24 || nalu == 25 || nalu == 26 || nalu == 27 {
|
||||
// STAP-A, STAP-B, MTAP16 or MTAP24
|
||||
i := 1
|
||||
if nalu == 25 || nalu == 26 || nalu == 27 {
|
||||
// skip DON
|
||||
i += 2
|
||||
}
|
||||
for i < len(payload) {
|
||||
if i+2 > len(payload) {
|
||||
return false
|
||||
}
|
||||
length := uint16(payload[i])<<8 |
|
||||
uint16(payload[i+1])
|
||||
i += 2
|
||||
if i+int(length) > len(payload) {
|
||||
return false
|
||||
}
|
||||
offset := 0
|
||||
if nalu == 26 {
|
||||
offset = 3
|
||||
} else if nalu == 27 {
|
||||
offset = 4
|
||||
}
|
||||
if offset >= int(length) {
|
||||
return false
|
||||
}
|
||||
n := payload[i+offset] & 0x1F
|
||||
if n == 7 {
|
||||
return true
|
||||
} else if n >= 24 {
|
||||
// is this legal?
|
||||
logger.Debugw("Non-simple NALU within a STAP")
|
||||
}
|
||||
i += int(length)
|
||||
}
|
||||
if i == len(payload) {
|
||||
return false
|
||||
}
|
||||
return false
|
||||
} else if nalu == 28 || nalu == 29 {
|
||||
// FU-A or FU-B
|
||||
if len(payload) < 2 {
|
||||
return false
|
||||
}
|
||||
if (payload[1] & 0x80) == 0 {
|
||||
// not a starting fragment
|
||||
return false
|
||||
}
|
||||
return payload[1]&0x1F == 7
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// -------------------------------------
|
||||
|
||||
// IsVP9KeyFrame detects if vp9 payload is a keyframe
|
||||
// taken from https://github.com/jech/galene/blob/master/codecs/codecs.go
|
||||
// all credits belongs to Juliusz Chroboczek @jech and the awesome Galene SFU
|
||||
func IsVP9KeyFrame(payload []byte) bool {
|
||||
var vp9 codecs.VP9Packet
|
||||
_, err := vp9.Unmarshal(payload)
|
||||
if err != nil || len(vp9.Payload) < 1 {
|
||||
return false
|
||||
}
|
||||
if !vp9.B {
|
||||
return false
|
||||
}
|
||||
|
||||
if (vp9.Payload[0] & 0xc0) != 0x80 {
|
||||
return false
|
||||
}
|
||||
|
||||
profile := (vp9.Payload[0] >> 4) & 0x3
|
||||
if profile != 3 {
|
||||
return (vp9.Payload[0] & 0xC) == 0
|
||||
}
|
||||
return (vp9.Payload[0] & 0x6) == 0
|
||||
}
|
||||
|
||||
// -------------------------------------
|
||||
|
||||
// IsAV1KeyFrame detects if av1 payload is a keyframe
|
||||
// taken from https://github.com/jech/galene/blob/master/codecs/codecs.go
|
||||
// all credits belongs to Juliusz Chroboczek @jech and the awesome Galene SFU
|
||||
func IsAV1KeyFrame(payload []byte) bool {
|
||||
if len(payload) < 2 {
|
||||
return false
|
||||
}
|
||||
// Z=0, N=1
|
||||
if (payload[0] & 0x88) != 0x08 {
|
||||
return false
|
||||
}
|
||||
w := (payload[0] & 0x30) >> 4
|
||||
|
||||
getObu := func(data []byte, last bool) ([]byte, int, bool) {
|
||||
if last {
|
||||
return data, len(data), false
|
||||
}
|
||||
offset := 0
|
||||
length := 0
|
||||
for {
|
||||
if len(data) <= offset {
|
||||
return nil, offset, offset > 0
|
||||
}
|
||||
l := data[offset]
|
||||
length |= int(l&0x7f) << (offset * 7)
|
||||
offset++
|
||||
if (l & 0x80) == 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
if len(data) < offset+length {
|
||||
return data[offset:], len(data), true
|
||||
}
|
||||
return data[offset : offset+length],
|
||||
offset + length, false
|
||||
}
|
||||
offset := 1
|
||||
i := 0
|
||||
for {
|
||||
obu, length, truncated :=
|
||||
getObu(payload[offset:], int(w) == i+1)
|
||||
if len(obu) < 1 {
|
||||
return false
|
||||
}
|
||||
tpe := (obu[0] & 0x38) >> 3
|
||||
switch i {
|
||||
case 0:
|
||||
// OBU_SEQUENCE_HEADER
|
||||
if tpe != 1 {
|
||||
return false
|
||||
}
|
||||
default:
|
||||
// OBU_FRAME_HEADER or OBU_FRAME
|
||||
if tpe == 3 || tpe == 6 {
|
||||
if len(obu) < 2 {
|
||||
return false
|
||||
}
|
||||
// show_existing_frame == 0
|
||||
if (obu[1] & 0x80) != 0 {
|
||||
return false
|
||||
}
|
||||
// frame_type == KEY_FRAME
|
||||
return (obu[1] & 0x60) == 0
|
||||
}
|
||||
}
|
||||
if truncated || i >= int(w) {
|
||||
// the first frame header is in a second
|
||||
// packet, give up.
|
||||
return false
|
||||
}
|
||||
offset += length
|
||||
i++
|
||||
}
|
||||
}
|
||||
|
||||
func IsH265KeyFrame(payload []byte) (kf bool) {
|
||||
if len(payload) < 2 {
|
||||
return false
|
||||
}
|
||||
naluType := (payload[0] & 0x7E) >> 1
|
||||
switch {
|
||||
case naluType == 33 || naluType == 34:
|
||||
return true
|
||||
case naluType == 48: // AP
|
||||
idx := 2
|
||||
for idx < len(payload)-2 {
|
||||
// TODO: check the DONL field (controled by sprop-max-don-diff)
|
||||
size := binary.BigEndian.Uint16(payload[idx:])
|
||||
idx += 2
|
||||
if idx >= len(payload) {
|
||||
return false
|
||||
}
|
||||
naluType = (payload[idx] & 0x7E) >> 1
|
||||
if naluType == 33 || naluType == 34 {
|
||||
return true
|
||||
}
|
||||
idx += int(size)
|
||||
}
|
||||
return false
|
||||
|
||||
case naluType == 49: // FU
|
||||
if len(payload) < 3 {
|
||||
return false
|
||||
}
|
||||
naluType = (payload[2] & 0x7E) >> 1
|
||||
return naluType == 33 || naluType == 34
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// -------------------------------------
|
||||
@@ -0,0 +1,110 @@
|
||||
// Copyright 2023 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package buffer
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestVP8Helper_Unmarshal(t *testing.T) {
|
||||
type args struct {
|
||||
payload []byte
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
args args
|
||||
wantErr bool
|
||||
checkTemporal bool
|
||||
temporalSupport bool
|
||||
checkKeyFrame bool
|
||||
keyFrame bool
|
||||
checkPictureID bool
|
||||
pictureID uint16
|
||||
checkTlzIdx bool
|
||||
tlzIdx uint8
|
||||
checkTempID bool
|
||||
temporalID uint8
|
||||
}{
|
||||
{
|
||||
name: "Empty or nil payload must return error",
|
||||
args: args{payload: []byte{}},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "Temporal must be supported by setting T bit to 1",
|
||||
args: args{payload: []byte{0xff, 0x20, 0x1, 0x2, 0x3, 0x4}},
|
||||
checkTemporal: true,
|
||||
temporalSupport: true,
|
||||
},
|
||||
{
|
||||
name: "Picture must be ID 7 bits by setting M bit to 0 and present by I bit set to 1",
|
||||
args: args{payload: []byte{0xff, 0xff, 0x11, 0x2, 0x3, 0x4}},
|
||||
checkPictureID: true,
|
||||
pictureID: 17,
|
||||
},
|
||||
{
|
||||
name: "Picture ID must be 15 bits by setting M bit to 1 and present by I bit set to 1",
|
||||
args: args{payload: []byte{0xff, 0xff, 0x92, 0x67, 0x3, 0x4, 0x5}},
|
||||
checkPictureID: true,
|
||||
pictureID: 4711,
|
||||
},
|
||||
{
|
||||
name: "Temporal level zero index must be present if L set to 1",
|
||||
args: args{payload: []byte{0xff, 0xff, 0xff, 0xfd, 0xb4, 0x4, 0x5}},
|
||||
checkTlzIdx: true,
|
||||
tlzIdx: 180,
|
||||
},
|
||||
{
|
||||
name: "Temporal index must be present and used if T bit set to 1",
|
||||
args: args{payload: []byte{0xff, 0xff, 0xff, 0xfd, 0xb4, 0x9f, 0x5, 0x6}},
|
||||
checkTempID: true,
|
||||
temporalID: 2,
|
||||
},
|
||||
{
|
||||
name: "Check if packet is a keyframe by looking at P bit set to 0",
|
||||
args: args{payload: []byte{0xff, 0xff, 0xff, 0xfd, 0xb4, 0x9f, 0x94, 0x1}},
|
||||
checkKeyFrame: true,
|
||||
keyFrame: true,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
tt := tt
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
p := &VP8{}
|
||||
if err := p.Unmarshal(tt.args.payload); (err != nil) != tt.wantErr {
|
||||
t.Errorf("Unmarshal() error = %v, wantErr %v", err, tt.wantErr)
|
||||
}
|
||||
if tt.checkTemporal {
|
||||
require.Equal(t, tt.temporalSupport, p.T)
|
||||
}
|
||||
if tt.checkKeyFrame {
|
||||
require.Equal(t, tt.keyFrame, p.IsKeyFrame)
|
||||
}
|
||||
if tt.checkPictureID {
|
||||
require.Equal(t, tt.pictureID, p.PictureID)
|
||||
}
|
||||
if tt.checkTlzIdx {
|
||||
require.Equal(t, tt.tlzIdx, p.TL0PICIDX)
|
||||
}
|
||||
if tt.checkTempID {
|
||||
require.Equal(t, tt.temporalID, p.TID)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// ------------------------------------------
|
||||
@@ -0,0 +1,62 @@
|
||||
// Copyright 2023 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package buffer
|
||||
|
||||
import (
|
||||
"io"
|
||||
|
||||
"go.uber.org/atomic"
|
||||
)
|
||||
|
||||
type RTCPReader struct {
|
||||
ssrc uint32
|
||||
closed atomic.Bool
|
||||
onPacket atomic.Value // func([]byte)
|
||||
onClose func()
|
||||
}
|
||||
|
||||
func NewRTCPReader(ssrc uint32) *RTCPReader {
|
||||
return &RTCPReader{ssrc: ssrc}
|
||||
}
|
||||
|
||||
func (r *RTCPReader) Write(p []byte) (n int, err error) {
|
||||
if r.closed.Load() {
|
||||
err = io.EOF
|
||||
return
|
||||
}
|
||||
if f, ok := r.onPacket.Load().(func([]byte)); ok && f != nil {
|
||||
f(p)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (r *RTCPReader) OnClose(fn func()) {
|
||||
r.onClose = fn
|
||||
}
|
||||
|
||||
func (r *RTCPReader) Close() error {
|
||||
if r.closed.Swap(true) {
|
||||
return nil
|
||||
}
|
||||
|
||||
r.onClose()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *RTCPReader) OnPacket(f func([]byte)) {
|
||||
r.onPacket.Store(f)
|
||||
}
|
||||
|
||||
func (r *RTCPReader) Read(_ []byte) (n int, err error) { return }
|
||||
@@ -0,0 +1,24 @@
|
||||
// Copyright 2023 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package buffer
|
||||
|
||||
import "github.com/livekit/livekit-server/pkg/sfu/rtpstats"
|
||||
|
||||
type StreamStatsWithLayers struct {
|
||||
RTPStats *rtpstats.RTPDeltaInfo
|
||||
Layers map[int32]*rtpstats.RTPDeltaInfo
|
||||
|
||||
RTPStatsRemoteView *rtpstats.RTPDeltaInfo
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
// Copyright 2023 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package buffer
|
||||
|
||||
import "fmt"
|
||||
|
||||
const (
|
||||
InvalidLayerSpatial = int32(-1)
|
||||
InvalidLayerTemporal = int32(-1)
|
||||
|
||||
DefaultMaxLayerSpatial = int32(2)
|
||||
DefaultMaxLayerTemporal = int32(3)
|
||||
)
|
||||
|
||||
var (
|
||||
InvalidLayer = VideoLayer{
|
||||
Spatial: InvalidLayerSpatial,
|
||||
Temporal: InvalidLayerTemporal,
|
||||
}
|
||||
|
||||
DefaultMaxLayer = VideoLayer{
|
||||
Spatial: DefaultMaxLayerSpatial,
|
||||
Temporal: DefaultMaxLayerTemporal,
|
||||
}
|
||||
)
|
||||
|
||||
type VideoLayer struct {
|
||||
Spatial int32
|
||||
Temporal int32
|
||||
}
|
||||
|
||||
func (v VideoLayer) String() string {
|
||||
return fmt.Sprintf("VideoLayer{s: %d, t: %d}", v.Spatial, v.Temporal)
|
||||
}
|
||||
|
||||
func (v VideoLayer) GreaterThan(v2 VideoLayer) bool {
|
||||
return v.Spatial > v2.Spatial || (v.Spatial == v2.Spatial && v.Temporal > v2.Temporal)
|
||||
}
|
||||
|
||||
func (v VideoLayer) SpatialGreaterThanOrEqual(v2 VideoLayer) bool {
|
||||
return v.Spatial >= v2.Spatial
|
||||
}
|
||||
|
||||
func (v VideoLayer) IsValid() bool {
|
||||
return v.Spatial != InvalidLayerSpatial && v.Temporal != InvalidLayerTemporal
|
||||
}
|
||||
@@ -0,0 +1,326 @@
|
||||
// Copyright 2023 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package buffer
|
||||
|
||||
import (
|
||||
"github.com/livekit/protocol/livekit"
|
||||
"github.com/livekit/protocol/logger"
|
||||
)
|
||||
|
||||
const (
|
||||
QuarterResolution = "q"
|
||||
HalfResolution = "h"
|
||||
FullResolution = "f"
|
||||
)
|
||||
|
||||
// SIMULCAST-CODEC-TODO: these need to be codec mime aware if and when each codec suppports different layers
|
||||
func LayerPresenceFromTrackInfo(trackInfo *livekit.TrackInfo) *[livekit.VideoQuality_HIGH + 1]bool {
|
||||
if trackInfo == nil || len(trackInfo.Layers) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
var layerPresence [livekit.VideoQuality_HIGH + 1]bool
|
||||
for _, layer := range trackInfo.Layers {
|
||||
// WARNING: comparing protobuf enum
|
||||
if layer.Quality <= livekit.VideoQuality_HIGH {
|
||||
layerPresence[layer.Quality] = true
|
||||
} else {
|
||||
logger.Warnw("unexpected quality in track info", nil, "trackID", trackInfo.Sid, "trackInfo", logger.Proto(trackInfo))
|
||||
}
|
||||
}
|
||||
|
||||
return &layerPresence
|
||||
}
|
||||
|
||||
func RidToSpatialLayer(rid string, trackInfo *livekit.TrackInfo) int32 {
|
||||
lp := LayerPresenceFromTrackInfo(trackInfo)
|
||||
if lp == nil {
|
||||
switch rid {
|
||||
case QuarterResolution:
|
||||
return 0
|
||||
case HalfResolution:
|
||||
return 1
|
||||
case FullResolution:
|
||||
return 2
|
||||
default:
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
switch rid {
|
||||
case QuarterResolution:
|
||||
switch {
|
||||
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
|
||||
fallthrough
|
||||
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM]:
|
||||
fallthrough
|
||||
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_HIGH]:
|
||||
fallthrough
|
||||
case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
|
||||
return 0
|
||||
|
||||
default:
|
||||
// only one quality published, could be any
|
||||
return 0
|
||||
}
|
||||
|
||||
case HalfResolution:
|
||||
switch {
|
||||
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
|
||||
fallthrough
|
||||
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM]:
|
||||
fallthrough
|
||||
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_HIGH]:
|
||||
fallthrough
|
||||
case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
|
||||
return 1
|
||||
|
||||
default:
|
||||
// only one quality published, could be any
|
||||
return 0
|
||||
}
|
||||
|
||||
case FullResolution:
|
||||
switch {
|
||||
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
|
||||
return 2
|
||||
|
||||
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM]:
|
||||
logger.Warnw("unexpected rid f with only two qualities, low and medium", nil, "trackID", trackInfo.Sid, "trackInfo", logger.Proto(trackInfo))
|
||||
return 1
|
||||
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_HIGH]:
|
||||
logger.Warnw("unexpected rid f with only two qualities, low and high", nil, "trackID", trackInfo.Sid, "trackInfo", logger.Proto(trackInfo))
|
||||
return 1
|
||||
case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
|
||||
logger.Warnw("unexpected rid f with only two qualities, medium and high", nil, "trackID", trackInfo.Sid, "trackInfo", logger.Proto(trackInfo))
|
||||
return 1
|
||||
|
||||
default:
|
||||
// only one quality published, could be any
|
||||
return 0
|
||||
}
|
||||
|
||||
default:
|
||||
// no rid, should be single layer
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
func SpatialLayerToRid(layer int32, trackInfo *livekit.TrackInfo) string {
|
||||
lp := LayerPresenceFromTrackInfo(trackInfo)
|
||||
if lp == nil {
|
||||
switch layer {
|
||||
case 0:
|
||||
return QuarterResolution
|
||||
case 1:
|
||||
return HalfResolution
|
||||
case 2:
|
||||
return FullResolution
|
||||
default:
|
||||
return QuarterResolution
|
||||
}
|
||||
}
|
||||
|
||||
switch layer {
|
||||
case 0:
|
||||
switch {
|
||||
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
|
||||
fallthrough
|
||||
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM]:
|
||||
fallthrough
|
||||
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_HIGH]:
|
||||
fallthrough
|
||||
case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
|
||||
return QuarterResolution
|
||||
|
||||
default:
|
||||
return QuarterResolution
|
||||
}
|
||||
|
||||
case 1:
|
||||
switch {
|
||||
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
|
||||
fallthrough
|
||||
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM]:
|
||||
fallthrough
|
||||
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_HIGH]:
|
||||
fallthrough
|
||||
case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
|
||||
return HalfResolution
|
||||
|
||||
default:
|
||||
return QuarterResolution
|
||||
}
|
||||
|
||||
case 2:
|
||||
switch {
|
||||
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
|
||||
return FullResolution
|
||||
|
||||
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM]:
|
||||
logger.Warnw("unexpected layer 2 with only two qualities, low and medium", nil, "trackID", trackInfo.Sid, "trackInfo", logger.Proto(trackInfo))
|
||||
return HalfResolution
|
||||
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_HIGH]:
|
||||
logger.Warnw("unexpected layer 2 with only two qualities, low and high", nil, "trackID", trackInfo.Sid, "trackInfo", logger.Proto(trackInfo))
|
||||
return HalfResolution
|
||||
case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
|
||||
logger.Warnw("unexpected layer 2 with only two qualities, medium and high", nil, "trackID", trackInfo.Sid, "trackInfo", logger.Proto(trackInfo))
|
||||
return HalfResolution
|
||||
|
||||
default:
|
||||
return QuarterResolution
|
||||
}
|
||||
|
||||
default:
|
||||
return QuarterResolution
|
||||
}
|
||||
}
|
||||
|
||||
func VideoQualityToRid(quality livekit.VideoQuality, trackInfo *livekit.TrackInfo) string {
|
||||
return SpatialLayerToRid(VideoQualityToSpatialLayer(quality, trackInfo), trackInfo)
|
||||
}
|
||||
|
||||
func SpatialLayerToVideoQuality(layer int32, trackInfo *livekit.TrackInfo) livekit.VideoQuality {
|
||||
lp := LayerPresenceFromTrackInfo(trackInfo)
|
||||
if lp == nil {
|
||||
switch layer {
|
||||
case 0:
|
||||
return livekit.VideoQuality_LOW
|
||||
case 1:
|
||||
return livekit.VideoQuality_MEDIUM
|
||||
case 2:
|
||||
return livekit.VideoQuality_HIGH
|
||||
default:
|
||||
return livekit.VideoQuality_OFF
|
||||
}
|
||||
}
|
||||
|
||||
switch layer {
|
||||
case 0:
|
||||
switch {
|
||||
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
|
||||
fallthrough
|
||||
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM]:
|
||||
fallthrough
|
||||
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_HIGH]:
|
||||
fallthrough
|
||||
case lp[livekit.VideoQuality_LOW]:
|
||||
return livekit.VideoQuality_LOW
|
||||
|
||||
case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
|
||||
fallthrough
|
||||
case lp[livekit.VideoQuality_MEDIUM]:
|
||||
return livekit.VideoQuality_MEDIUM
|
||||
|
||||
default:
|
||||
return livekit.VideoQuality_HIGH
|
||||
}
|
||||
|
||||
case 1:
|
||||
switch {
|
||||
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
|
||||
fallthrough
|
||||
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM]:
|
||||
return livekit.VideoQuality_MEDIUM
|
||||
|
||||
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_HIGH]:
|
||||
fallthrough
|
||||
case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
|
||||
return livekit.VideoQuality_HIGH
|
||||
|
||||
default:
|
||||
logger.Errorw("invalid layer", nil, "trackID", trackInfo.Sid, "layer", layer, "trackInfo", logger.Proto(trackInfo))
|
||||
return livekit.VideoQuality_HIGH
|
||||
}
|
||||
|
||||
case 2:
|
||||
switch {
|
||||
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
|
||||
return livekit.VideoQuality_HIGH
|
||||
|
||||
default:
|
||||
logger.Errorw("invalid layer", nil, "trackID", trackInfo.Sid, "layer", layer, "trackInfo", logger.Proto(trackInfo))
|
||||
return livekit.VideoQuality_HIGH
|
||||
}
|
||||
}
|
||||
|
||||
return livekit.VideoQuality_OFF
|
||||
}
|
||||
|
||||
func VideoQualityToSpatialLayer(quality livekit.VideoQuality, trackInfo *livekit.TrackInfo) int32 {
|
||||
lp := LayerPresenceFromTrackInfo(trackInfo)
|
||||
if lp == nil {
|
||||
switch quality {
|
||||
case livekit.VideoQuality_LOW:
|
||||
return 0
|
||||
case livekit.VideoQuality_MEDIUM:
|
||||
return 1
|
||||
case livekit.VideoQuality_HIGH:
|
||||
return 2
|
||||
default:
|
||||
return InvalidLayerSpatial
|
||||
}
|
||||
}
|
||||
|
||||
switch quality {
|
||||
case livekit.VideoQuality_LOW:
|
||||
switch {
|
||||
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
|
||||
fallthrough
|
||||
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM]:
|
||||
fallthrough
|
||||
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_HIGH]:
|
||||
fallthrough
|
||||
case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
|
||||
fallthrough
|
||||
default: // only one quality published, could be any
|
||||
return 0
|
||||
}
|
||||
|
||||
case livekit.VideoQuality_MEDIUM:
|
||||
switch {
|
||||
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
|
||||
fallthrough
|
||||
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM]:
|
||||
fallthrough
|
||||
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_HIGH]:
|
||||
return 1
|
||||
|
||||
case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
|
||||
return 0
|
||||
|
||||
default: // only one quality published, could be any
|
||||
return 0
|
||||
}
|
||||
|
||||
case livekit.VideoQuality_HIGH:
|
||||
switch {
|
||||
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
|
||||
return 2
|
||||
|
||||
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_MEDIUM]:
|
||||
fallthrough
|
||||
case lp[livekit.VideoQuality_LOW] && lp[livekit.VideoQuality_HIGH]:
|
||||
fallthrough
|
||||
case lp[livekit.VideoQuality_MEDIUM] && lp[livekit.VideoQuality_HIGH]:
|
||||
return 1
|
||||
|
||||
default: // only one quality published, could be any
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
return InvalidLayerSpatial
|
||||
}
|
||||
@@ -0,0 +1,442 @@
|
||||
// Copyright 2023 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package buffer
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/livekit/protocol/livekit"
|
||||
)
|
||||
|
||||
func TestRidConversion(t *testing.T) {
|
||||
type RidAndLayer struct {
|
||||
rid string
|
||||
layer int32
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
trackInfo *livekit.TrackInfo
|
||||
ridToLayer map[string]RidAndLayer
|
||||
}{
|
||||
{
|
||||
"no track info",
|
||||
nil,
|
||||
map[string]RidAndLayer{
|
||||
"": {rid: QuarterResolution, layer: 0},
|
||||
QuarterResolution: {rid: QuarterResolution, layer: 0},
|
||||
HalfResolution: {rid: HalfResolution, layer: 1},
|
||||
FullResolution: {rid: FullResolution, layer: 2},
|
||||
},
|
||||
},
|
||||
{
|
||||
"no layers",
|
||||
&livekit.TrackInfo{},
|
||||
map[string]RidAndLayer{
|
||||
"": {rid: QuarterResolution, layer: 0},
|
||||
QuarterResolution: {rid: QuarterResolution, layer: 0},
|
||||
HalfResolution: {rid: HalfResolution, layer: 1},
|
||||
FullResolution: {rid: FullResolution, layer: 2},
|
||||
},
|
||||
},
|
||||
{
|
||||
"single layer, low",
|
||||
&livekit.TrackInfo{
|
||||
Layers: []*livekit.VideoLayer{
|
||||
{Quality: livekit.VideoQuality_LOW},
|
||||
},
|
||||
},
|
||||
map[string]RidAndLayer{
|
||||
"": {rid: QuarterResolution, layer: 0},
|
||||
QuarterResolution: {rid: QuarterResolution, layer: 0},
|
||||
HalfResolution: {rid: QuarterResolution, layer: 0},
|
||||
FullResolution: {rid: QuarterResolution, layer: 0},
|
||||
},
|
||||
},
|
||||
{
|
||||
"single layer, medium",
|
||||
&livekit.TrackInfo{
|
||||
Layers: []*livekit.VideoLayer{
|
||||
{Quality: livekit.VideoQuality_MEDIUM},
|
||||
},
|
||||
},
|
||||
map[string]RidAndLayer{
|
||||
"": {rid: QuarterResolution, layer: 0},
|
||||
QuarterResolution: {rid: QuarterResolution, layer: 0},
|
||||
HalfResolution: {rid: QuarterResolution, layer: 0},
|
||||
FullResolution: {rid: QuarterResolution, layer: 0},
|
||||
},
|
||||
},
|
||||
{
|
||||
"single layer, high",
|
||||
&livekit.TrackInfo{
|
||||
Layers: []*livekit.VideoLayer{
|
||||
{Quality: livekit.VideoQuality_HIGH},
|
||||
},
|
||||
},
|
||||
map[string]RidAndLayer{
|
||||
"": {rid: QuarterResolution, layer: 0},
|
||||
QuarterResolution: {rid: QuarterResolution, layer: 0},
|
||||
HalfResolution: {rid: QuarterResolution, layer: 0},
|
||||
FullResolution: {rid: QuarterResolution, layer: 0},
|
||||
},
|
||||
},
|
||||
{
|
||||
"two layers, low and medium",
|
||||
&livekit.TrackInfo{
|
||||
Layers: []*livekit.VideoLayer{
|
||||
{Quality: livekit.VideoQuality_LOW},
|
||||
{Quality: livekit.VideoQuality_MEDIUM},
|
||||
},
|
||||
},
|
||||
map[string]RidAndLayer{
|
||||
"": {rid: QuarterResolution, layer: 0},
|
||||
QuarterResolution: {rid: QuarterResolution, layer: 0},
|
||||
HalfResolution: {rid: HalfResolution, layer: 1},
|
||||
FullResolution: {rid: HalfResolution, layer: 1},
|
||||
},
|
||||
},
|
||||
{
|
||||
"two layers, low and high",
|
||||
&livekit.TrackInfo{
|
||||
Layers: []*livekit.VideoLayer{
|
||||
{Quality: livekit.VideoQuality_LOW},
|
||||
{Quality: livekit.VideoQuality_HIGH},
|
||||
},
|
||||
},
|
||||
map[string]RidAndLayer{
|
||||
"": {rid: QuarterResolution, layer: 0},
|
||||
QuarterResolution: {rid: QuarterResolution, layer: 0},
|
||||
HalfResolution: {rid: HalfResolution, layer: 1},
|
||||
FullResolution: {rid: HalfResolution, layer: 1},
|
||||
},
|
||||
},
|
||||
{
|
||||
"two layers, medium and high",
|
||||
&livekit.TrackInfo{
|
||||
Layers: []*livekit.VideoLayer{
|
||||
{Quality: livekit.VideoQuality_MEDIUM},
|
||||
{Quality: livekit.VideoQuality_HIGH},
|
||||
},
|
||||
},
|
||||
map[string]RidAndLayer{
|
||||
"": {rid: QuarterResolution, layer: 0},
|
||||
QuarterResolution: {rid: QuarterResolution, layer: 0},
|
||||
HalfResolution: {rid: HalfResolution, layer: 1},
|
||||
FullResolution: {rid: HalfResolution, layer: 1},
|
||||
},
|
||||
},
|
||||
{
|
||||
"three layers",
|
||||
&livekit.TrackInfo{
|
||||
Layers: []*livekit.VideoLayer{
|
||||
{Quality: livekit.VideoQuality_LOW},
|
||||
{Quality: livekit.VideoQuality_MEDIUM},
|
||||
{Quality: livekit.VideoQuality_HIGH},
|
||||
},
|
||||
},
|
||||
map[string]RidAndLayer{
|
||||
"": {rid: QuarterResolution, layer: 0},
|
||||
QuarterResolution: {rid: QuarterResolution, layer: 0},
|
||||
HalfResolution: {rid: HalfResolution, layer: 1},
|
||||
FullResolution: {rid: FullResolution, layer: 2},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
for testRid, expectedResult := range test.ridToLayer {
|
||||
actualLayer := RidToSpatialLayer(testRid, test.trackInfo)
|
||||
require.Equal(t, expectedResult.layer, actualLayer)
|
||||
|
||||
actualRid := SpatialLayerToRid(actualLayer, test.trackInfo)
|
||||
require.Equal(t, expectedResult.rid, actualRid)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestQualityConversion(t *testing.T) {
|
||||
type QualityAndLayer struct {
|
||||
quality livekit.VideoQuality
|
||||
layer int32
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
trackInfo *livekit.TrackInfo
|
||||
qualityToLayer map[livekit.VideoQuality]QualityAndLayer
|
||||
}{
|
||||
{
|
||||
"no track info",
|
||||
nil,
|
||||
map[livekit.VideoQuality]QualityAndLayer{
|
||||
livekit.VideoQuality_LOW: {quality: livekit.VideoQuality_LOW, layer: 0},
|
||||
livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_MEDIUM, layer: 1},
|
||||
livekit.VideoQuality_HIGH: {quality: livekit.VideoQuality_HIGH, layer: 2},
|
||||
},
|
||||
},
|
||||
{
|
||||
"no layers",
|
||||
&livekit.TrackInfo{},
|
||||
map[livekit.VideoQuality]QualityAndLayer{
|
||||
livekit.VideoQuality_LOW: {quality: livekit.VideoQuality_LOW, layer: 0},
|
||||
livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_MEDIUM, layer: 1},
|
||||
livekit.VideoQuality_HIGH: {quality: livekit.VideoQuality_HIGH, layer: 2},
|
||||
},
|
||||
},
|
||||
{
|
||||
"single layer, low",
|
||||
&livekit.TrackInfo{
|
||||
Layers: []*livekit.VideoLayer{
|
||||
{Quality: livekit.VideoQuality_LOW},
|
||||
},
|
||||
},
|
||||
map[livekit.VideoQuality]QualityAndLayer{
|
||||
livekit.VideoQuality_LOW: {quality: livekit.VideoQuality_LOW, layer: 0},
|
||||
livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_LOW, layer: 0},
|
||||
livekit.VideoQuality_HIGH: {quality: livekit.VideoQuality_LOW, layer: 0},
|
||||
},
|
||||
},
|
||||
{
|
||||
"single layer, medium",
|
||||
&livekit.TrackInfo{
|
||||
Layers: []*livekit.VideoLayer{
|
||||
{Quality: livekit.VideoQuality_MEDIUM},
|
||||
},
|
||||
},
|
||||
map[livekit.VideoQuality]QualityAndLayer{
|
||||
livekit.VideoQuality_LOW: {quality: livekit.VideoQuality_MEDIUM, layer: 0},
|
||||
livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_MEDIUM, layer: 0},
|
||||
livekit.VideoQuality_HIGH: {quality: livekit.VideoQuality_MEDIUM, layer: 0},
|
||||
},
|
||||
},
|
||||
{
|
||||
"single layer, high",
|
||||
&livekit.TrackInfo{
|
||||
Layers: []*livekit.VideoLayer{
|
||||
{Quality: livekit.VideoQuality_HIGH},
|
||||
},
|
||||
},
|
||||
map[livekit.VideoQuality]QualityAndLayer{
|
||||
livekit.VideoQuality_LOW: {quality: livekit.VideoQuality_HIGH, layer: 0},
|
||||
livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_HIGH, layer: 0},
|
||||
livekit.VideoQuality_HIGH: {quality: livekit.VideoQuality_HIGH, layer: 0},
|
||||
},
|
||||
},
|
||||
{
|
||||
"two layers, low and medium",
|
||||
&livekit.TrackInfo{
|
||||
Layers: []*livekit.VideoLayer{
|
||||
{Quality: livekit.VideoQuality_LOW},
|
||||
{Quality: livekit.VideoQuality_MEDIUM},
|
||||
},
|
||||
},
|
||||
map[livekit.VideoQuality]QualityAndLayer{
|
||||
livekit.VideoQuality_LOW: {quality: livekit.VideoQuality_LOW, layer: 0},
|
||||
livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_MEDIUM, layer: 1},
|
||||
livekit.VideoQuality_HIGH: {quality: livekit.VideoQuality_MEDIUM, layer: 1},
|
||||
},
|
||||
},
|
||||
{
|
||||
"two layers, low and high",
|
||||
&livekit.TrackInfo{
|
||||
Layers: []*livekit.VideoLayer{
|
||||
{Quality: livekit.VideoQuality_LOW},
|
||||
{Quality: livekit.VideoQuality_HIGH},
|
||||
},
|
||||
},
|
||||
map[livekit.VideoQuality]QualityAndLayer{
|
||||
livekit.VideoQuality_LOW: {quality: livekit.VideoQuality_LOW, layer: 0},
|
||||
livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_HIGH, layer: 1},
|
||||
livekit.VideoQuality_HIGH: {quality: livekit.VideoQuality_HIGH, layer: 1},
|
||||
},
|
||||
},
|
||||
{
|
||||
"two layers, medium and high",
|
||||
&livekit.TrackInfo{
|
||||
Layers: []*livekit.VideoLayer{
|
||||
{Quality: livekit.VideoQuality_MEDIUM},
|
||||
{Quality: livekit.VideoQuality_HIGH},
|
||||
},
|
||||
},
|
||||
map[livekit.VideoQuality]QualityAndLayer{
|
||||
livekit.VideoQuality_LOW: {quality: livekit.VideoQuality_MEDIUM, layer: 0},
|
||||
livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_MEDIUM, layer: 0},
|
||||
livekit.VideoQuality_HIGH: {quality: livekit.VideoQuality_HIGH, layer: 1},
|
||||
},
|
||||
},
|
||||
{
|
||||
"three layers",
|
||||
&livekit.TrackInfo{
|
||||
Layers: []*livekit.VideoLayer{
|
||||
{Quality: livekit.VideoQuality_LOW},
|
||||
{Quality: livekit.VideoQuality_MEDIUM},
|
||||
{Quality: livekit.VideoQuality_HIGH},
|
||||
},
|
||||
},
|
||||
map[livekit.VideoQuality]QualityAndLayer{
|
||||
livekit.VideoQuality_LOW: {quality: livekit.VideoQuality_LOW, layer: 0},
|
||||
livekit.VideoQuality_MEDIUM: {quality: livekit.VideoQuality_MEDIUM, layer: 1},
|
||||
livekit.VideoQuality_HIGH: {quality: livekit.VideoQuality_HIGH, layer: 2},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
for testQuality, expectedResult := range test.qualityToLayer {
|
||||
actualLayer := VideoQualityToSpatialLayer(testQuality, test.trackInfo)
|
||||
require.Equal(t, expectedResult.layer, actualLayer)
|
||||
|
||||
actualQuality := SpatialLayerToVideoQuality(actualLayer, test.trackInfo)
|
||||
require.Equal(t, expectedResult.quality, actualQuality)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestVideoQualityToRidConversion(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
trackInfo *livekit.TrackInfo
|
||||
qualityToRid map[livekit.VideoQuality]string
|
||||
}{
|
||||
{
|
||||
"no track info",
|
||||
nil,
|
||||
map[livekit.VideoQuality]string{
|
||||
livekit.VideoQuality_LOW: QuarterResolution,
|
||||
livekit.VideoQuality_MEDIUM: HalfResolution,
|
||||
livekit.VideoQuality_HIGH: FullResolution,
|
||||
},
|
||||
},
|
||||
{
|
||||
"no layers",
|
||||
&livekit.TrackInfo{},
|
||||
map[livekit.VideoQuality]string{
|
||||
livekit.VideoQuality_LOW: QuarterResolution,
|
||||
livekit.VideoQuality_MEDIUM: HalfResolution,
|
||||
livekit.VideoQuality_HIGH: FullResolution,
|
||||
},
|
||||
},
|
||||
{
|
||||
"single layer, low",
|
||||
&livekit.TrackInfo{
|
||||
Layers: []*livekit.VideoLayer{
|
||||
{Quality: livekit.VideoQuality_LOW},
|
||||
},
|
||||
},
|
||||
map[livekit.VideoQuality]string{
|
||||
livekit.VideoQuality_LOW: QuarterResolution,
|
||||
livekit.VideoQuality_MEDIUM: QuarterResolution,
|
||||
livekit.VideoQuality_HIGH: QuarterResolution,
|
||||
},
|
||||
},
|
||||
{
|
||||
"single layer, medium",
|
||||
&livekit.TrackInfo{
|
||||
Layers: []*livekit.VideoLayer{
|
||||
{Quality: livekit.VideoQuality_MEDIUM},
|
||||
},
|
||||
},
|
||||
map[livekit.VideoQuality]string{
|
||||
livekit.VideoQuality_LOW: QuarterResolution,
|
||||
livekit.VideoQuality_MEDIUM: QuarterResolution,
|
||||
livekit.VideoQuality_HIGH: QuarterResolution,
|
||||
},
|
||||
},
|
||||
{
|
||||
"single layer, high",
|
||||
&livekit.TrackInfo{
|
||||
Layers: []*livekit.VideoLayer{
|
||||
{Quality: livekit.VideoQuality_HIGH},
|
||||
},
|
||||
},
|
||||
map[livekit.VideoQuality]string{
|
||||
livekit.VideoQuality_LOW: QuarterResolution,
|
||||
livekit.VideoQuality_MEDIUM: QuarterResolution,
|
||||
livekit.VideoQuality_HIGH: QuarterResolution,
|
||||
},
|
||||
},
|
||||
{
|
||||
"two layers, low and medium",
|
||||
&livekit.TrackInfo{
|
||||
Layers: []*livekit.VideoLayer{
|
||||
{Quality: livekit.VideoQuality_LOW},
|
||||
{Quality: livekit.VideoQuality_MEDIUM},
|
||||
},
|
||||
},
|
||||
map[livekit.VideoQuality]string{
|
||||
livekit.VideoQuality_LOW: QuarterResolution,
|
||||
livekit.VideoQuality_MEDIUM: HalfResolution,
|
||||
livekit.VideoQuality_HIGH: HalfResolution,
|
||||
},
|
||||
},
|
||||
{
|
||||
"two layers, low and high",
|
||||
&livekit.TrackInfo{
|
||||
Layers: []*livekit.VideoLayer{
|
||||
{Quality: livekit.VideoQuality_LOW},
|
||||
{Quality: livekit.VideoQuality_HIGH},
|
||||
},
|
||||
},
|
||||
map[livekit.VideoQuality]string{
|
||||
livekit.VideoQuality_LOW: QuarterResolution,
|
||||
livekit.VideoQuality_MEDIUM: HalfResolution,
|
||||
livekit.VideoQuality_HIGH: HalfResolution,
|
||||
},
|
||||
},
|
||||
{
|
||||
"two layers, medium and high",
|
||||
&livekit.TrackInfo{
|
||||
Layers: []*livekit.VideoLayer{
|
||||
{Quality: livekit.VideoQuality_MEDIUM},
|
||||
{Quality: livekit.VideoQuality_HIGH},
|
||||
},
|
||||
},
|
||||
map[livekit.VideoQuality]string{
|
||||
livekit.VideoQuality_LOW: QuarterResolution,
|
||||
livekit.VideoQuality_MEDIUM: QuarterResolution,
|
||||
livekit.VideoQuality_HIGH: HalfResolution,
|
||||
},
|
||||
},
|
||||
{
|
||||
"three layers",
|
||||
&livekit.TrackInfo{
|
||||
Layers: []*livekit.VideoLayer{
|
||||
{Quality: livekit.VideoQuality_LOW},
|
||||
{Quality: livekit.VideoQuality_MEDIUM},
|
||||
{Quality: livekit.VideoQuality_HIGH},
|
||||
},
|
||||
},
|
||||
map[livekit.VideoQuality]string{
|
||||
livekit.VideoQuality_LOW: QuarterResolution,
|
||||
livekit.VideoQuality_MEDIUM: HalfResolution,
|
||||
livekit.VideoQuality_HIGH: FullResolution,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
for testQuality, expectedRid := range test.qualityToRid {
|
||||
actualRid := VideoQualityToRid(testQuality, test.trackInfo)
|
||||
require.Equal(t, expectedRid, actualRid)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user