Squashed 'livekit-server/' content from commit 154b4d26

git-subtree-dir: livekit-server
git-subtree-split: 154b4d26b769c68a03c096124094b97bf61a996f
This commit is contained in:
2026-06-25 14:35:28 +09:00
commit 0da97ebd21
339 changed files with 114111 additions and 0 deletions
+70
View File
@@ -0,0 +1,70 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package streamtracker
import (
"fmt"
"time"
"github.com/livekit/livekit-server/pkg/sfu/buffer"
)
// ------------------------------------------------------------
type StreamStatusChange int32
func (s StreamStatusChange) String() string {
switch s {
case StreamStatusChangeNone:
return "none"
case StreamStatusChangeStopped:
return "stopped"
case StreamStatusChangeActive:
return "active"
default:
return fmt.Sprintf("unknown: %d", int(s))
}
}
const (
StreamStatusChangeNone StreamStatusChange = iota
StreamStatusChangeStopped
StreamStatusChangeActive
)
// ------------------------------------------------------------
type StreamTrackerImpl interface {
Start()
Stop()
Reset()
GetCheckInterval() time.Duration
Observe(hasMarker bool, ts uint32) StreamStatusChange
CheckStatus() StreamStatusChange
}
type StreamTrackerWorker interface {
Start()
Stop()
Reset()
OnStatusChanged(f func(status StreamStatus))
OnBitrateAvailable(f func())
Status() StreamStatus
BitrateTemporalCumulative() []int64
SetPaused(paused bool)
Observe(temporalLayer int32, pktSize int, payloadSize int, hasMarker bool, ts uint32, dd *buffer.ExtDependencyDescriptor)
}
+307
View File
@@ -0,0 +1,307 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package streamtracker
import (
"fmt"
"sync"
"time"
"go.uber.org/atomic"
"github.com/livekit/livekit-server/pkg/sfu/buffer"
"github.com/livekit/protocol/logger"
)
// ------------------------------------------------------------
type StreamStatus int32
func (s StreamStatus) String() string {
switch s {
case StreamStatusStopped:
return "stopped"
case StreamStatusActive:
return "active"
default:
return fmt.Sprintf("unknown: %d", int(s))
}
}
const (
StreamStatusStopped StreamStatus = iota
StreamStatusActive
)
// ------------------------------------------------------------
type StreamTrackerParams struct {
StreamTrackerImpl StreamTrackerImpl
BitrateReportInterval time.Duration
Logger logger.Logger
}
type StreamTracker struct {
params StreamTrackerParams
onStatusChanged func(status StreamStatus)
onBitrateAvailable func()
lock sync.RWMutex
paused bool
generation atomic.Uint32
status StreamStatus
lastNotifiedStatus StreamStatus
lastBitrateReport time.Time
bytesForBitrate [4]int64
bitrate [4]int64
isStopped bool
}
func NewStreamTracker(params StreamTrackerParams) *StreamTracker {
return &StreamTracker{
params: params,
status: StreamStatusStopped,
}
}
func (s *StreamTracker) OnStatusChanged(f func(status StreamStatus)) {
s.onStatusChanged = f
}
func (s *StreamTracker) OnBitrateAvailable(f func()) {
s.onBitrateAvailable = f
}
func (s *StreamTracker) Status() StreamStatus {
s.lock.RLock()
defer s.lock.RUnlock()
return s.status
}
func (s *StreamTracker) setStatusLocked(status StreamStatus) {
s.status = status
}
func (s *StreamTracker) maybeNotifyStatus() {
var status StreamStatus
notify := false
s.lock.Lock()
if s.status != s.lastNotifiedStatus {
notify = true
status = s.status
s.lastNotifiedStatus = s.status
}
s.lock.Unlock()
if notify && s.onStatusChanged != nil {
s.onStatusChanged(status)
}
}
func (s *StreamTracker) Start() {
s.lock.Lock()
defer s.lock.Unlock()
s.params.StreamTrackerImpl.Start()
}
func (s *StreamTracker) Stop() {
s.lock.Lock()
defer s.lock.Unlock()
if s.isStopped {
return
}
s.isStopped = true
// bump generation to trigger exit of worker
s.generation.Inc()
s.params.StreamTrackerImpl.Stop()
}
func (s *StreamTracker) Reset() {
s.lock.Lock()
if s.isStopped {
s.lock.Unlock()
return
}
s.resetLocked()
s.lock.Unlock()
s.maybeNotifyStatus()
}
func (s *StreamTracker) resetLocked() {
// bump generation to trigger exit of current worker
s.generation.Inc()
s.setStatusLocked(StreamStatusStopped)
for i := 0; i < len(s.bytesForBitrate); i++ {
s.bytesForBitrate[i] = 0
}
for i := 0; i < len(s.bitrate); i++ {
s.bitrate[i] = 0
}
s.params.StreamTrackerImpl.Reset()
}
func (s *StreamTracker) SetPaused(paused bool) {
s.lock.Lock()
s.paused = paused
if !paused {
s.resetLocked()
} else {
// bump generation to trigger exit of current worker
s.generation.Inc()
s.setStatusLocked(StreamStatusStopped)
}
s.lock.Unlock()
s.maybeNotifyStatus()
}
func (s *StreamTracker) Observe(
temporalLayer int32,
pktSize int,
payloadSize int,
hasMarker bool,
ts uint32,
_ *buffer.ExtDependencyDescriptor,
) {
s.lock.Lock()
if s.isStopped || s.paused || payloadSize == 0 {
s.lock.Unlock()
return
}
statusChange := s.params.StreamTrackerImpl.Observe(hasMarker, ts)
if statusChange == StreamStatusChangeActive {
s.setStatusLocked(StreamStatusActive)
s.lastBitrateReport = time.Now()
go s.worker(s.generation.Load())
}
if temporalLayer >= 0 {
s.bytesForBitrate[temporalLayer] += int64(pktSize)
}
s.lock.Unlock()
if statusChange != StreamStatusChangeNone {
s.maybeNotifyStatus()
}
}
// BitrateTemporalCumulative returns the current stream bitrate temporal layer accumulated with lower temporal layers.
func (s *StreamTracker) BitrateTemporalCumulative() []int64 {
s.lock.RLock()
defer s.lock.RUnlock()
// copy and process
brs := make([]int64, len(s.bitrate))
copy(brs, s.bitrate[:])
for i := len(brs) - 1; i >= 1; i-- {
if brs[i] != 0 {
for j := i - 1; j >= 0; j-- {
brs[i] += brs[j]
}
}
}
// clear higher layers
for i := 0; i < len(brs); i++ {
if brs[i] == 0 {
for j := i + 1; j < len(brs); j++ {
brs[j] = 0
}
}
}
return brs
}
func (s *StreamTracker) worker(generation uint32) {
ticker := time.NewTicker(s.params.StreamTrackerImpl.GetCheckInterval())
defer ticker.Stop()
tickerBitrate := time.NewTicker(s.params.BitrateReportInterval)
defer tickerBitrate.Stop()
for {
select {
case <-ticker.C:
if generation != s.generation.Load() {
return
}
s.updateStatus()
case <-tickerBitrate.C:
if generation != s.generation.Load() {
return
}
s.bitrateReport()
}
}
}
func (s *StreamTracker) updateStatus() {
s.lock.Lock()
switch s.params.StreamTrackerImpl.CheckStatus() {
case StreamStatusChangeStopped:
s.setStatusLocked(StreamStatusStopped)
case StreamStatusChangeActive:
s.setStatusLocked(StreamStatusActive)
}
s.lock.Unlock()
s.maybeNotifyStatus()
}
func (s *StreamTracker) bitrateReport() {
// run this even if paused to drain out bitrate if there are no packets coming in
s.lock.Lock()
now := time.Now()
diff := now.Sub(s.lastBitrateReport)
s.lastBitrateReport = now
bitrateAvailabilityChanged := false
for i := 0; i < len(s.bytesForBitrate); i++ {
bitrate := int64(float64(s.bytesForBitrate[i]*8) / diff.Seconds())
if (s.bitrate[i] == 0 && bitrate > 0) || (s.bitrate[i] > 0 && bitrate == 0) {
bitrateAvailabilityChanged = true
}
s.bitrate[i] = bitrate
s.bytesForBitrate[i] = 0
}
s.lock.Unlock()
if bitrateAvailabilityChanged && s.onBitrateAvailable != nil {
s.onBitrateAvailable()
}
}
+303
View File
@@ -0,0 +1,303 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package streamtracker
import (
"sync"
"time"
"go.uber.org/atomic"
"github.com/livekit/livekit-server/pkg/sfu/buffer"
dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor"
)
type StreamTrackerDependencyDescriptor struct {
lock sync.RWMutex
paused bool
generation atomic.Uint32
params StreamTrackerParams
maxSpatialLayer int32
maxTemporalLayer int32
onStatusChanged [buffer.DefaultMaxLayerSpatial + 1]func(status StreamStatus)
onBitrateAvailable [buffer.DefaultMaxLayerSpatial + 1]func()
lastBitrateReport time.Time
bytesForBitrate [buffer.DefaultMaxLayerSpatial + 1][buffer.DefaultMaxLayerTemporal + 1]int64
bitrate [buffer.DefaultMaxLayerSpatial + 1][buffer.DefaultMaxLayerTemporal + 1]int64
isStopped bool
}
func NewStreamTrackerDependencyDescriptor(params StreamTrackerParams) *StreamTrackerDependencyDescriptor {
return &StreamTrackerDependencyDescriptor{
params: params,
maxSpatialLayer: buffer.InvalidLayerSpatial,
maxTemporalLayer: buffer.InvalidLayerTemporal,
}
}
func (s *StreamTrackerDependencyDescriptor) Start() {
}
func (s *StreamTrackerDependencyDescriptor) Stop() {
s.lock.Lock()
defer s.lock.Unlock()
if s.isStopped {
return
}
s.isStopped = true
// bump generation to trigger exit of worker
s.generation.Inc()
}
func (s *StreamTrackerDependencyDescriptor) OnStatusChanged(layer int32, f func(status StreamStatus)) {
s.lock.Lock()
s.onStatusChanged[layer] = f
s.lock.Unlock()
}
func (s *StreamTrackerDependencyDescriptor) OnBitrateAvailable(layer int32, f func()) {
s.lock.Lock()
s.onBitrateAvailable[layer] = f
s.lock.Unlock()
}
func (s *StreamTrackerDependencyDescriptor) Status(layer int32) StreamStatus {
s.lock.RLock()
defer s.lock.RUnlock()
if layer > s.maxSpatialLayer {
return StreamStatusStopped
}
return StreamStatusActive
}
func (s *StreamTrackerDependencyDescriptor) BitrateTemporalCumulative(layer int32) []int64 {
s.lock.RLock()
defer s.lock.RUnlock()
if layer > s.maxSpatialLayer {
brs := make([]int64, len(s.bitrate[0]))
return brs
}
return s.bitrate[layer][:]
}
func (s *StreamTrackerDependencyDescriptor) Reset() {
}
func (s *StreamTrackerDependencyDescriptor) resetLocked() {
// bump generation to trigger exit of current worker
s.generation.Inc()
for i := 0; i < len(s.bytesForBitrate); i++ {
for j := 0; j < len(s.bytesForBitrate[i]); j++ {
s.bytesForBitrate[i][j] = 0
}
}
for i := 0; i < len(s.bitrate); i++ {
for j := 0; j < len(s.bitrate[i]); j++ {
s.bitrate[i][j] = 0
}
}
s.maxSpatialLayer = buffer.InvalidLayerSpatial
s.maxTemporalLayer = buffer.InvalidLayerTemporal
}
func (s *StreamTrackerDependencyDescriptor) SetPaused(paused bool) {
s.lock.Lock()
if s.paused == paused {
s.lock.Unlock()
return
}
s.paused = paused
var notifyFns []func(status StreamStatus)
var notifyStatus StreamStatus
if !paused {
s.resetLocked()
notifyStatus = StreamStatusStopped
notifyFns = append(notifyFns, s.onStatusChanged[:]...)
} else {
s.lastBitrateReport = time.Now()
go s.worker(s.generation.Inc())
}
s.lock.Unlock()
for _, fn := range notifyFns {
if fn != nil {
fn(notifyStatus)
}
}
}
func (s *StreamTrackerDependencyDescriptor) Observe(temporalLayer int32, pktSize int, payloadSize int, hasMarker bool, ts uint32, ddVal *buffer.ExtDependencyDescriptor) {
s.lock.Lock()
if s.isStopped || s.paused || payloadSize == 0 || ddVal == nil {
s.lock.Unlock()
return
}
var notifyFns []func(status StreamStatus)
var notifyStatus StreamStatus
if mask := ddVal.Descriptor.ActiveDecodeTargetsBitmask; mask != nil && ddVal.ActiveDecodeTargetsUpdated {
var maxSpatial, maxTemporal int32
for _, dt := range ddVal.DecodeTargets {
if *mask&(1<<dt.Target) != uint32(dd.DecodeTargetNotPresent) {
if maxSpatial < dt.Layer.Spatial {
maxSpatial = dt.Layer.Spatial
}
if maxTemporal < dt.Layer.Temporal {
maxTemporal = dt.Layer.Temporal
}
}
}
if maxSpatial > buffer.DefaultMaxLayerSpatial {
maxSpatial = buffer.DefaultMaxLayerSpatial
s.params.Logger.Warnw("max spatial layer exceeded", nil, "maxSpatial", maxSpatial)
}
if maxTemporal > buffer.DefaultMaxLayerTemporal {
maxTemporal = buffer.DefaultMaxLayerTemporal
s.params.Logger.Warnw("max temporal layer exceeded", nil, "maxTemporal", maxTemporal)
}
s.params.Logger.Debugw("max layer changed", "maxSpatial", maxSpatial, "maxTemporal", maxTemporal)
oldMaxSpatial := s.maxSpatialLayer
s.maxSpatialLayer, s.maxTemporalLayer = maxSpatial, maxTemporal
if oldMaxSpatial == -1 {
s.lastBitrateReport = time.Now()
go s.worker(s.generation.Inc())
}
if oldMaxSpatial > s.maxSpatialLayer {
notifyStatus = StreamStatusStopped
for i := s.maxSpatialLayer + 1; i <= oldMaxSpatial; i++ {
notifyFns = append(notifyFns, s.onStatusChanged[i])
}
} else if oldMaxSpatial < s.maxSpatialLayer {
notifyStatus = StreamStatusActive
for i := oldMaxSpatial + 1; i <= s.maxSpatialLayer; i++ {
notifyFns = append(notifyFns, s.onStatusChanged[i])
}
}
}
dtis := ddVal.Descriptor.FrameDependencies.DecodeTargetIndications
for _, dt := range ddVal.DecodeTargets {
if len(dtis) <= dt.Target {
s.params.Logger.Errorw("len(dtis) less than target", nil, "target", dt.Target, "dtis", dtis)
continue
}
// we are not dropping discardable frames now, so only ingore not present frames
if dtis[dt.Target] == dd.DecodeTargetNotPresent {
continue
}
s.bytesForBitrate[dt.Layer.Spatial][dt.Layer.Temporal] += int64(pktSize)
}
s.lock.Unlock()
for _, fn := range notifyFns {
if fn != nil {
fn(notifyStatus)
}
}
}
func (s *StreamTrackerDependencyDescriptor) worker(generation uint32) {
tickerBitrate := time.NewTicker(s.params.BitrateReportInterval)
defer tickerBitrate.Stop()
for {
<-tickerBitrate.C
if generation != s.generation.Load() {
return
}
s.bitrateReport()
}
}
func (s *StreamTrackerDependencyDescriptor) bitrateReport() {
// run this even if paused to drain out bitrate if there are no packets coming in
s.lock.Lock()
now := time.Now()
diff := now.Sub(s.lastBitrateReport)
s.lastBitrateReport = now
var availableChangedFns []func()
for spatial := 0; spatial < len(s.bytesForBitrate); spatial++ {
bytesForBitrate := s.bytesForBitrate[spatial][:]
bitrateAvailabilityChanged := false
bitrates := s.bitrate[spatial][:]
for i := 0; i < len(bytesForBitrate); i++ {
bitrate := int64(float64(bytesForBitrate[i]*8) / diff.Seconds())
if (bitrates[i] == 0 && bitrate > 0) || (bitrates[i] > 0 && bitrate == 0) {
bitrateAvailabilityChanged = true
}
bitrates[i] = bitrate
bytesForBitrate[i] = 0
}
if bitrateAvailabilityChanged && s.onBitrateAvailable[spatial] != nil {
availableChangedFns = append(availableChangedFns, s.onBitrateAvailable[spatial])
}
}
s.lock.Unlock()
for _, fn := range availableChangedFns {
fn()
}
}
func (s *StreamTrackerDependencyDescriptor) LayeredTracker(layer int32) *StreamTrackerDependencyDescriptorLayered {
return &StreamTrackerDependencyDescriptorLayered{
StreamTrackerDependencyDescriptor: s,
layer: layer,
}
}
// ----------------------------
// Layered wrapper for StreamTrackerWorker
type StreamTrackerDependencyDescriptorLayered struct {
*StreamTrackerDependencyDescriptor
layer int32
}
func (s *StreamTrackerDependencyDescriptorLayered) OnStatusChanged(f func(status StreamStatus)) {
s.StreamTrackerDependencyDescriptor.OnStatusChanged(s.layer, f)
}
func (s *StreamTrackerDependencyDescriptorLayered) OnBitrateAvailable(f func()) {
s.StreamTrackerDependencyDescriptor.OnBitrateAvailable(s.layer, f)
}
func (s *StreamTrackerDependencyDescriptorLayered) Status() StreamStatus {
return s.StreamTrackerDependencyDescriptor.Status(s.layer)
}
func (s *StreamTrackerDependencyDescriptorLayered) BitrateTemporalCumulative() []int64 {
return s.StreamTrackerDependencyDescriptor.BitrateTemporalCumulative(s.layer)
}
@@ -0,0 +1,98 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package streamtracker
import (
"testing"
"time"
"github.com/stretchr/testify/require"
"github.com/livekit/livekit-server/pkg/sfu/buffer"
dd "github.com/livekit/livekit-server/pkg/sfu/rtpextension/dependencydescriptor"
"github.com/livekit/protocol/logger"
)
func createDescriptorDependencyForTargets(maxSpatial, maxTemporal int) *buffer.ExtDependencyDescriptor {
var targets []buffer.DependencyDescriptorDecodeTarget
var mask uint32
for i := 0; i <= maxSpatial; i++ {
for j := 0; j <= maxTemporal; j++ {
targets = append(targets, buffer.DependencyDescriptorDecodeTarget{Target: len(targets), Layer: buffer.VideoLayer{Spatial: int32(i), Temporal: int32(j)}})
mask |= 1 << uint32(len(targets)-1)
}
}
dtis := make([]dd.DecodeTargetIndication, len(targets))
for _, t := range targets {
dtis[t.Target] = dd.DecodeTargetRequired
}
return &buffer.ExtDependencyDescriptor{
Descriptor: &dd.DependencyDescriptor{
ActiveDecodeTargetsBitmask: &mask,
FrameDependencies: &dd.FrameDependencyTemplate{
DecodeTargetIndications: dtis,
},
},
DecodeTargets: targets,
ActiveDecodeTargetsUpdated: true,
}
}
func checkStatues(t *testing.T, statuses []StreamStatus, expected StreamStatus, maxSpatial int) {
for i := 0; i <= maxSpatial; i++ {
require.Equal(t, expected, statuses[i])
}
for i := maxSpatial + 1; i < len(statuses); i++ {
require.NotEqual(t, expected, statuses[i])
}
}
func TestStreamTrackerDD(t *testing.T) {
ddTracker := NewStreamTrackerDependencyDescriptor(StreamTrackerParams{
BitrateReportInterval: 1 * time.Second,
Logger: logger.GetLogger(),
})
layeredTrackers := make([]StreamTrackerWorker, buffer.DefaultMaxLayerSpatial+1)
statuses := make([]StreamStatus, buffer.DefaultMaxLayerSpatial+1)
for i := 0; i <= int(buffer.DefaultMaxLayerSpatial); i++ {
layeredTrack := ddTracker.LayeredTracker(int32(i))
layer := i
layeredTrack.OnStatusChanged(func(status StreamStatus) {
statuses[layer] = status
})
layeredTrack.Start()
layeredTrackers[i] = layeredTrack
}
defer ddTracker.Stop()
// no active layers
ddTracker.Observe(0, 1000, 1000, false, 0, nil)
checkStatues(t, statuses, StreamStatusActive, int(buffer.InvalidLayerSpatial))
// layer seen [0,1]
ddTracker.Observe(0, 1000, 1000, false, 0, createDescriptorDependencyForTargets(1, 1))
checkStatues(t, statuses, StreamStatusActive, 1)
// layer seen [0,1,2]
ddTracker.Observe(0, 1000, 1000, false, 0, createDescriptorDependencyForTargets(2, 1))
checkStatues(t, statuses, StreamStatusActive, 2)
// layer 2 gone, layer seen [0,1]
ddTracker.Observe(0, 1000, 1000, false, 0, createDescriptorDependencyForTargets(1, 1))
checkStatues(t, statuses, StreamStatusActive, 1)
}
@@ -0,0 +1,244 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package streamtracker
import (
"math"
"time"
"github.com/livekit/protocol/logger"
)
const (
checkInterval = 500 * time.Millisecond
statusCheckTolerance = 0.98
frameRateResolution = 0.01 // 1 frame every 100 seconds
frameRateIncreaseFactor = 0.6 // slow increase
frameRateDecreaseFactor = 0.9 // fast decrease
)
// -------------------------------------------------------
type StreamTrackerFrameConfig struct {
MinFPS float64 `yaml:"min_fps,omitempty"`
}
var (
DefaultStreamTrackerFrameConfigVideo = map[int32]StreamTrackerFrameConfig{
0: {
MinFPS: 5.0,
},
1: {
MinFPS: 5.0,
},
2: {
MinFPS: 5.0,
},
}
DefaultStreamTrackerFrameConfigScreenshare = map[int32]StreamTrackerFrameConfig{
0: {
MinFPS: 0.5,
},
1: {
MinFPS: 0.5,
},
2: {
MinFPS: 0.5,
},
}
)
// -------------------------------------------------------
type StreamTrackerFrameParams struct {
Config StreamTrackerFrameConfig
ClockRate uint32
Logger logger.Logger
}
type StreamTrackerFrame struct {
params StreamTrackerFrameParams
initialized bool
tsInitialized bool
oldestTS uint32
newestTS uint32
numFrames int
estimatedFrameRate float64
evalInterval time.Duration
lastStatusCheckAt time.Time
}
func NewStreamTrackerFrame(params StreamTrackerFrameParams) StreamTrackerImpl {
s := &StreamTrackerFrame{
params: params,
}
s.Reset()
return s
}
func (s *StreamTrackerFrame) Start() {
}
func (s *StreamTrackerFrame) Stop() {
}
func (s *StreamTrackerFrame) Reset() {
s.initialized = false
s.resetFPSCalculator()
s.lastStatusCheckAt = time.Time{}
}
func (s *StreamTrackerFrame) resetFPSCalculator() {
s.tsInitialized = false
s.oldestTS = 0
s.newestTS = 0
s.numFrames = 0
s.estimatedFrameRate = 0.0
s.updateEvalInterval()
}
func (s *StreamTrackerFrame) GetCheckInterval() time.Duration {
return checkInterval
}
func (s *StreamTrackerFrame) Observe(hasMarker bool, ts uint32) StreamStatusChange {
if hasMarker {
if !s.tsInitialized {
s.tsInitialized = true
s.oldestTS = ts
s.newestTS = ts
s.numFrames = 1
} else {
diff := ts - s.oldestTS
if diff > (1 << 31) {
s.oldestTS = ts
}
diff = ts - s.newestTS
if diff < (1 << 31) {
s.newestTS = ts
}
s.numFrames++
}
}
// When starting up, check for first packet and declare active.
// Happens under following conditions
// 1. Start up
// 2. Unmute (stream restarting)
// 3. Layer starting after dynacast pause
if !s.initialized {
s.initialized = true
s.lastStatusCheckAt = time.Now()
return StreamStatusChangeActive
}
return StreamStatusChangeNone
}
func (s *StreamTrackerFrame) CheckStatus() StreamStatusChange {
if !s.initialized {
// should not be getting called when not initialized, but be safe
return StreamStatusChangeNone
}
if !s.updateStatusCheckTime() {
return StreamStatusChangeNone
}
if s.updateEstimatedFrameRate() == 0.0 {
// when stream is stopped, reset FPS calculator to ensure re-start is not done until at least two frames are available,
// i. e. enough frames available to be able to calculate FPS
s.resetFPSCalculator()
return StreamStatusChangeStopped
}
return StreamStatusChangeActive
}
func (s *StreamTrackerFrame) updateStatusCheckTime() bool {
// check only at intervals based on estimated frame rate
if s.lastStatusCheckAt.IsZero() {
s.lastStatusCheckAt = time.Now()
}
if time.Since(s.lastStatusCheckAt) < time.Duration(statusCheckTolerance*float64(s.evalInterval)) {
return false
}
s.lastStatusCheckAt = time.Now()
return true
}
func (s *StreamTrackerFrame) updateEstimatedFrameRate() float64 {
diff := s.newestTS - s.oldestTS
if diff == 0 || s.numFrames < 2 {
return 0.0
}
frameRate := roundFrameRate(float64(s.params.ClockRate) / float64(diff) * float64(s.numFrames-1))
// reset for next evaluation interval
s.oldestTS = s.newestTS
s.numFrames = 1
factor := 1.0
switch {
case s.estimatedFrameRate < frameRate:
// slow increase, prevents shortening eval interval too quickly on frame rate going up
factor = frameRateIncreaseFactor
case s.estimatedFrameRate > frameRate:
// fast decrease, prevents declaring stream stop too quickly on frame rate going down
factor = frameRateDecreaseFactor
}
estimatedFrameRate := roundFrameRate(frameRate*factor + s.estimatedFrameRate*(1.0-factor))
if s.estimatedFrameRate != estimatedFrameRate {
s.estimatedFrameRate = estimatedFrameRate
s.updateEvalInterval()
s.params.Logger.Debugw("updating estimated frame rate", "estimatedFPS", estimatedFrameRate, "evalInterval", s.evalInterval)
}
return frameRate
}
func (s *StreamTrackerFrame) updateEvalInterval() {
// STREAM-TRACKER-FRAME-TODO: This will run into challenges for frame rate falling steeply, How to address that?
// Maybe, look at some referential rules (between layers) for possibilities to solve it. Currently, this is addressed
// by setting a source aware min FPS to ensure evaluation window is long enough to avoid declaring stop too quickly.
s.evalInterval = checkInterval
if s.estimatedFrameRate > 0.0 {
estimatedFrameRateInterval := time.Duration(float64(time.Second) / s.estimatedFrameRate)
if estimatedFrameRateInterval > s.evalInterval {
s.evalInterval = estimatedFrameRateInterval
}
}
if s.params.Config.MinFPS > 0.0 {
minFPSInterval := time.Duration(float64(time.Second) / s.params.Config.MinFPS)
if minFPSInterval > s.evalInterval {
s.evalInterval = minFPSInterval
}
}
}
// ------------------------------------------------------------------------------
func roundFrameRate(frameRate float64) float64 {
return math.Round(frameRate/frameRateResolution) * frameRateResolution
}
@@ -0,0 +1,141 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package streamtracker
import (
"time"
"github.com/livekit/protocol/logger"
)
// --------------------------------------------
type StreamTrackerPacketConfig struct {
SamplesRequired uint32 `yaml:"samples_required,omitempty"` // number of samples needed per cycle
CyclesRequired uint32 `yaml:"cycles_required,omitempty"` // number of cycles needed to be active
CycleDuration time.Duration `yaml:"cycle_duration,omitempty"`
}
var (
DefaultStreamTrackerPacketConfigVideo = map[int32]StreamTrackerPacketConfig{
0: {SamplesRequired: 1,
CyclesRequired: 4,
CycleDuration: 500 * time.Millisecond,
},
1: {SamplesRequired: 5,
CyclesRequired: 20,
CycleDuration: 500 * time.Millisecond,
},
2: {SamplesRequired: 5,
CyclesRequired: 20,
CycleDuration: 500 * time.Millisecond,
},
}
DefaultStreamTrackerPacketConfigScreenshare = map[int32]StreamTrackerPacketConfig{
0: {
SamplesRequired: 1,
CyclesRequired: 1,
CycleDuration: 2 * time.Second,
},
1: {
SamplesRequired: 1,
CyclesRequired: 1,
CycleDuration: 2 * time.Second,
},
2: {
SamplesRequired: 1,
CyclesRequired: 1,
CycleDuration: 2 * time.Second,
},
}
)
// --------------------------------------------
type StreamTrackerPacketParams struct {
Config StreamTrackerPacketConfig
Logger logger.Logger
}
type StreamTrackerPacket struct {
params StreamTrackerPacketParams
countSinceLast uint32 // number of packets received since last check
initialized bool
cycleCount uint32
}
func NewStreamTrackerPacket(params StreamTrackerPacketParams) StreamTrackerImpl {
return &StreamTrackerPacket{
params: params,
}
}
func (s *StreamTrackerPacket) Start() {
}
func (s *StreamTrackerPacket) Stop() {
}
func (s *StreamTrackerPacket) Reset() {
s.countSinceLast = 0
s.cycleCount = 0
s.initialized = false
}
func (s *StreamTrackerPacket) GetCheckInterval() time.Duration {
return s.params.Config.CycleDuration
}
func (s *StreamTrackerPacket) Observe(_hasMarker bool, _ts uint32) StreamStatusChange {
if !s.initialized {
// first packet
s.initialized = true
s.countSinceLast = 1
return StreamStatusChangeActive
}
s.countSinceLast++
return StreamStatusChangeNone
}
func (s *StreamTrackerPacket) CheckStatus() StreamStatusChange {
if !s.initialized {
// should not be getting called when not initialized, but be safe
return StreamStatusChangeNone
}
if s.countSinceLast >= s.params.Config.SamplesRequired {
s.cycleCount++
} else {
s.cycleCount = 0
}
statusChange := StreamStatusChangeNone
if s.cycleCount == 0 {
// no packets seen for a period, flip to stopped
statusChange = StreamStatusChangeStopped
} else if s.cycleCount >= s.params.Config.CyclesRequired {
// packets seen for some time after resume, flip to active
statusChange = StreamStatusChangeActive
}
s.countSinceLast = 0
return statusChange
}
@@ -0,0 +1,222 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package streamtracker
import (
"fmt"
"sync"
"testing"
"time"
"github.com/stretchr/testify/require"
"go.uber.org/atomic"
"github.com/livekit/livekit-server/pkg/testutils"
"github.com/livekit/protocol/logger"
)
func newStreamTrackerPacket(samplesRequired uint32, cyclesRequired uint32, cycleDuration time.Duration) *StreamTracker {
stp := NewStreamTrackerPacket(StreamTrackerPacketParams{
Config: StreamTrackerPacketConfig{
SamplesRequired: samplesRequired,
CyclesRequired: cyclesRequired,
CycleDuration: cycleDuration,
},
Logger: logger.GetLogger(),
})
return NewStreamTracker(StreamTrackerParams{
StreamTrackerImpl: stp,
BitrateReportInterval: 1 * time.Second,
Logger: logger.GetLogger(),
})
}
func TestStreamTracker(t *testing.T) {
t.Run("flips to active on first observe", func(t *testing.T) {
callbackCalled := atomic.NewBool(false)
tracker := newStreamTrackerPacket(5, 60, 500*time.Millisecond)
tracker.Start()
tracker.OnStatusChanged(func(status StreamStatus) {
callbackCalled.Store(true)
})
require.Equal(t, StreamStatusStopped, tracker.Status())
// observe first packet
tracker.Observe(0, 20, 10, false, 0, nil)
testutils.WithTimeout(t, func() string {
if callbackCalled.Load() {
return ""
}
return "first packet didn't activate stream"
})
require.Equal(t, StreamStatusActive, tracker.Status())
require.True(t, callbackCalled.Load())
tracker.Stop()
})
t.Run("flips to inactive immediately", func(t *testing.T) {
tracker := newStreamTrackerPacket(5, 60, 500*time.Millisecond)
tracker.Start()
require.Equal(t, StreamStatusStopped, tracker.Status())
callbackStatusMu := sync.RWMutex{}
callbackStatusMu.Lock()
callbackStatus := StreamStatusStopped
callbackStatusMu.Unlock()
tracker.OnStatusChanged(func(status StreamStatus) {
callbackStatusMu.Lock()
callbackStatus = status
callbackStatusMu.Unlock()
})
tracker.Observe(0, 20, 10, false, 0, nil)
testutils.WithTimeout(t, func() string {
callbackStatusMu.RLock()
defer callbackStatusMu.RUnlock()
if callbackStatus == StreamStatusActive {
return ""
}
return "first packet did not activate stream"
})
require.Equal(t, StreamStatusActive, tracker.Status())
// run a single iteration
tracker.updateStatus()
testutils.WithTimeout(t, func() string {
callbackStatusMu.RLock()
defer callbackStatusMu.RUnlock()
if callbackStatus == StreamStatusStopped {
return ""
}
return "inactive cycle did not declare stream stopped"
})
require.Equal(t, StreamStatusStopped, tracker.Status())
require.Equal(t, StreamStatusStopped, callbackStatus)
tracker.Stop()
})
t.Run("flips back to active after iterations", func(t *testing.T) {
tracker := newStreamTrackerPacket(1, 2, 500*time.Millisecond)
tracker.Start()
require.Equal(t, StreamStatusStopped, tracker.Status())
tracker.Observe(0, 20, 10, false, 0, nil)
testutils.WithTimeout(t, func() string {
if tracker.Status() == StreamStatusActive {
return ""
}
return "first packet did not activate stream"
})
tracker.setStatusLocked(StreamStatusStopped)
tracker.Observe(0, 20, 10, false, 0, nil)
tracker.updateStatus()
require.Equal(t, StreamStatusStopped, tracker.Status())
tracker.Observe(0, 20, 10, false, 0, nil)
tracker.updateStatus()
require.Equal(t, StreamStatusActive, tracker.Status())
tracker.Stop()
})
t.Run("changes to inactive when paused", func(t *testing.T) {
tracker := newStreamTrackerPacket(5, 60, 500*time.Millisecond)
tracker.Start()
tracker.Observe(0, 20, 10, false, 0, nil)
testutils.WithTimeout(t, func() string {
if tracker.Status() == StreamStatusActive {
return ""
}
return "first packet did not activate stream"
})
tracker.SetPaused(true)
tracker.updateStatus()
require.Equal(t, StreamStatusStopped, tracker.Status())
tracker.Stop()
})
t.Run("flips back to active on first observe after reset", func(t *testing.T) {
callbackCalled := atomic.NewUint32(0)
tracker := newStreamTrackerPacket(5, 60, 500*time.Millisecond)
tracker.Start()
tracker.OnStatusChanged(func(status StreamStatus) {
callbackCalled.Inc()
})
require.Equal(t, StreamStatusStopped, tracker.Status())
// observe first packet
tracker.Observe(0, 20, 10, false, 0, nil)
testutils.WithTimeout(t, func() string {
if callbackCalled.Load() == 1 {
return ""
}
return fmt.Sprintf("expected onStatusChanged to be called once, actual: %d", callbackCalled.Load())
})
require.Equal(t, StreamStatusActive, tracker.Status())
require.Equal(t, uint32(1), callbackCalled.Load())
// observe a few more
tracker.Observe(0, 20, 10, false, 0, nil)
tracker.Observe(0, 20, 10, false, 0, nil)
tracker.Observe(0, 20, 10, false, 0, nil)
tracker.Observe(0, 20, 10, false, 0, nil)
tracker.updateStatus()
// should still be active
require.Equal(t, StreamStatusActive, tracker.Status())
require.Equal(t, uint32(1), callbackCalled.Load())
// Reset. The first packet after reset should flip state again
tracker.Reset()
require.Equal(t, StreamStatusStopped, tracker.Status())
require.Equal(t, uint32(2), callbackCalled.Load())
// first packet after reset
tracker.Observe(0, 20, 10, false, 0, nil)
testutils.WithTimeout(t, func() string {
if callbackCalled.Load() == 3 {
return ""
}
return fmt.Sprintf("expected onStatusChanged to be called thrice, actual %d", callbackCalled.Load())
})
require.Equal(t, StreamStatusActive, tracker.Status())
require.Equal(t, uint32(3), callbackCalled.Load())
tracker.Stop()
})
}