Squashed 'livekit-server/' content from commit 154b4d26
git-subtree-dir: livekit-server git-subtree-split: 154b4d26b769c68a03c096124094b97bf61a996f
This commit is contained in:
@@ -0,0 +1,31 @@
|
||||
// Copyright 2023 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package metric
|
||||
|
||||
// ------------------------------------------------
|
||||
|
||||
type MetricConfig struct {
|
||||
Timestamper MetricTimestamperConfig `yaml:"timestamper_config,omitempty"`
|
||||
Collector MetricsCollectorConfig `yaml:"collector,omitempty"`
|
||||
Reporter MetricsReporterConfig `yaml:"reporter,omitempty"`
|
||||
}
|
||||
|
||||
var (
|
||||
DefaultMetricConfig = MetricConfig{
|
||||
Timestamper: DefaultMetricTimestamperConfig,
|
||||
Collector: DefaultMetricsCollectorConfig,
|
||||
Reporter: DefaultMetricsReporterConfig,
|
||||
}
|
||||
)
|
||||
@@ -0,0 +1,119 @@
|
||||
// Copyright 2023 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package metric
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/livekit/livekit-server/pkg/sfu/utils"
|
||||
"github.com/livekit/protocol/livekit"
|
||||
"github.com/livekit/protocol/logger"
|
||||
"github.com/livekit/protocol/utils/mono"
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
)
|
||||
|
||||
// ------------------------------------------------
|
||||
|
||||
type MetricTimestamperConfig struct {
|
||||
OneWayDelayEstimatorMinInterval time.Duration `yaml:"one_way_delay_estimator_min_interval,omitempty"`
|
||||
OneWayDelayEstimatorMaxBatch int `yaml:"one_way_delay_estimator_max_batch,omitempty"`
|
||||
}
|
||||
|
||||
var (
|
||||
DefaultMetricTimestamperConfig = MetricTimestamperConfig{
|
||||
OneWayDelayEstimatorMinInterval: 5 * time.Second,
|
||||
OneWayDelayEstimatorMaxBatch: 100,
|
||||
}
|
||||
)
|
||||
|
||||
// ------------------------------------------------
|
||||
|
||||
type MetricTimestamperParams struct {
|
||||
Config MetricTimestamperConfig
|
||||
Logger logger.Logger
|
||||
}
|
||||
|
||||
type MetricTimestamper struct {
|
||||
params MetricTimestamperParams
|
||||
lock sync.Mutex
|
||||
owdEstimator *utils.OWDEstimator
|
||||
lastOWDEstimatorRunAt time.Time
|
||||
batchesSinceLastOWDEstimatorRun int
|
||||
}
|
||||
|
||||
func NewMetricTimestamper(params MetricTimestamperParams) *MetricTimestamper {
|
||||
return &MetricTimestamper{
|
||||
params: params,
|
||||
owdEstimator: utils.NewOWDEstimator(utils.OWDEstimatorParamsDefault),
|
||||
lastOWDEstimatorRunAt: time.Now().Add(-params.Config.OneWayDelayEstimatorMinInterval),
|
||||
}
|
||||
}
|
||||
|
||||
func (m *MetricTimestamper) Process(batch *livekit.MetricsBatch) {
|
||||
if m == nil {
|
||||
return
|
||||
}
|
||||
|
||||
// run OWD estimation periodically
|
||||
estimatedOWDNanos := m.maybeRunOWDEstimator(batch)
|
||||
|
||||
// normalize all time stamps and add estimated OWD
|
||||
// NOTE: all timestamps will be re-mapped. If the time series or event happened some time
|
||||
// in the past and the OWD estimation has changed since, those samples will get the updated
|
||||
// OWD estimation applied. So, they may have more uncertainty in addition to the uncertainty
|
||||
// of OWD estimation process.
|
||||
batch.NormalizedTimestamp = timestamppb.New(time.Unix(0, batch.TimestampMs*1e6+estimatedOWDNanos))
|
||||
|
||||
for _, ts := range batch.TimeSeries {
|
||||
for _, sample := range ts.Samples {
|
||||
sample.NormalizedTimestamp = timestamppb.New(time.Unix(0, sample.TimestampMs*1e6+estimatedOWDNanos))
|
||||
}
|
||||
}
|
||||
|
||||
for _, ev := range batch.Events {
|
||||
ev.NormalizedStartTimestamp = timestamppb.New(time.Unix(0, ev.StartTimestampMs*1e6+estimatedOWDNanos))
|
||||
|
||||
endTimestampMs := ev.GetEndTimestampMs()
|
||||
if endTimestampMs != 0 {
|
||||
ev.NormalizedEndTimestamp = timestamppb.New(time.Unix(0, endTimestampMs*1e6+estimatedOWDNanos))
|
||||
}
|
||||
}
|
||||
|
||||
m.params.Logger.Debugw("timestamped metrics batch", "batch", logger.Proto(batch))
|
||||
}
|
||||
|
||||
func (m *MetricTimestamper) maybeRunOWDEstimator(batch *livekit.MetricsBatch) int64 {
|
||||
m.lock.Lock()
|
||||
defer m.lock.Unlock()
|
||||
|
||||
if time.Since(m.lastOWDEstimatorRunAt) < m.params.Config.OneWayDelayEstimatorMinInterval &&
|
||||
m.batchesSinceLastOWDEstimatorRun < m.params.Config.OneWayDelayEstimatorMaxBatch {
|
||||
m.batchesSinceLastOWDEstimatorRun++
|
||||
return m.owdEstimator.EstimatedPropagationDelay()
|
||||
}
|
||||
|
||||
senderClockTime := batch.GetTimestampMs()
|
||||
if senderClockTime == 0 {
|
||||
m.batchesSinceLastOWDEstimatorRun++
|
||||
return m.owdEstimator.EstimatedPropagationDelay()
|
||||
}
|
||||
|
||||
m.lastOWDEstimatorRunAt = time.Now()
|
||||
m.batchesSinceLastOWDEstimatorRun = 1
|
||||
|
||||
estimatedOWDNs, _ := m.owdEstimator.Update(senderClockTime*1e6, mono.UnixNano())
|
||||
return estimatedOWDNs
|
||||
}
|
||||
@@ -0,0 +1,225 @@
|
||||
// Copyright 2023 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package metric
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/frostbyte73/core"
|
||||
"github.com/livekit/protocol/livekit"
|
||||
"github.com/livekit/protocol/logger"
|
||||
"github.com/livekit/protocol/utils/mono"
|
||||
|
||||
"github.com/livekit/protocol/utils"
|
||||
)
|
||||
|
||||
type MetricsCollectorProvider interface {
|
||||
MetricsCollectorTimeToCollectMetrics()
|
||||
MetricsCollectorBatchReady(mb *livekit.MetricsBatch)
|
||||
}
|
||||
|
||||
// --------------------------------------------------------
|
||||
|
||||
type MetricsCollectorConfig struct {
|
||||
SamplingIntervalMs uint32 `yaml:"sampling_interval_ms,omitempty" json:"sampling_interval_ms,omitempty"`
|
||||
BatchIntervalMs uint32 `yaml:"batch_interval_ms,omitempty" json:"batch_interval_ms,omitempty"`
|
||||
}
|
||||
|
||||
var (
|
||||
DefaultMetricsCollectorConfig = MetricsCollectorConfig{
|
||||
SamplingIntervalMs: 3 * 1000,
|
||||
BatchIntervalMs: 10 * 1000,
|
||||
}
|
||||
)
|
||||
|
||||
// --------------------------------------------------------
|
||||
|
||||
type MetricsCollectorParams struct {
|
||||
ParticipantIdentity livekit.ParticipantIdentity
|
||||
Config MetricsCollectorConfig
|
||||
Provider MetricsCollectorProvider
|
||||
Logger logger.Logger
|
||||
}
|
||||
|
||||
type MetricsCollector struct {
|
||||
params MetricsCollectorParams
|
||||
|
||||
lock sync.RWMutex
|
||||
mbb *utils.MetricsBatchBuilder
|
||||
publisherRTTMetricId map[livekit.ParticipantIdentity]int
|
||||
subscriberRTTMetricId int
|
||||
relayRTTMetricId map[livekit.ParticipantIdentity]int
|
||||
|
||||
stop core.Fuse
|
||||
}
|
||||
|
||||
func NewMetricsCollector(params MetricsCollectorParams) *MetricsCollector {
|
||||
mc := &MetricsCollector{
|
||||
params: params,
|
||||
}
|
||||
mc.reset()
|
||||
|
||||
go mc.worker()
|
||||
return mc
|
||||
}
|
||||
|
||||
func (mc *MetricsCollector) Stop() {
|
||||
if mc != nil {
|
||||
mc.stop.Break()
|
||||
}
|
||||
}
|
||||
|
||||
func (mc *MetricsCollector) AddPublisherRTT(participantIdentity livekit.ParticipantIdentity, rtt float32) {
|
||||
mc.lock.Lock()
|
||||
defer mc.lock.Unlock()
|
||||
|
||||
metricId, ok := mc.publisherRTTMetricId[participantIdentity]
|
||||
if !ok {
|
||||
var err error
|
||||
metricId, err = mc.createTimeSeriesMetric(livekit.MetricLabel_PUBLISHER_RTT, participantIdentity)
|
||||
if err != nil {
|
||||
mc.params.Logger.Warnw("could not add time series metric for publisher RTT", err)
|
||||
return
|
||||
}
|
||||
|
||||
mc.publisherRTTMetricId[participantIdentity] = metricId
|
||||
}
|
||||
|
||||
mc.addTimeSeriesMetricSample(metricId, rtt)
|
||||
}
|
||||
|
||||
func (mc *MetricsCollector) AddSubscriberRTT(rtt float32) {
|
||||
mc.lock.Lock()
|
||||
defer mc.lock.Unlock()
|
||||
|
||||
if mc.subscriberRTTMetricId == utils.MetricsBatchBuilderInvalidTimeSeriesMetricId {
|
||||
var err error
|
||||
mc.subscriberRTTMetricId, err = mc.createTimeSeriesMetric(livekit.MetricLabel_SUBSCRIBER_RTT, mc.params.ParticipantIdentity)
|
||||
if err != nil {
|
||||
mc.params.Logger.Warnw("could not add time series metric for publisher RTT", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
mc.addTimeSeriesMetricSample(mc.subscriberRTTMetricId, rtt)
|
||||
}
|
||||
|
||||
func (mc *MetricsCollector) AddRelayRTT(participantIdentity livekit.ParticipantIdentity, rtt float32) {
|
||||
mc.lock.Lock()
|
||||
defer mc.lock.Unlock()
|
||||
|
||||
metricId, ok := mc.relayRTTMetricId[participantIdentity]
|
||||
if !ok {
|
||||
var err error
|
||||
metricId, err = mc.createTimeSeriesMetric(livekit.MetricLabel_SERVER_MESH_RTT, participantIdentity)
|
||||
if err != nil {
|
||||
mc.params.Logger.Warnw("could not add time series metric for server mesh RTT", err)
|
||||
return
|
||||
}
|
||||
|
||||
mc.relayRTTMetricId[participantIdentity] = metricId
|
||||
}
|
||||
|
||||
mc.addTimeSeriesMetricSample(metricId, rtt)
|
||||
}
|
||||
|
||||
func (mc *MetricsCollector) getMetricsBatchAndReset() *livekit.MetricsBatch {
|
||||
mc.lock.Lock()
|
||||
mbb := mc.mbb
|
||||
|
||||
mc.reset()
|
||||
mc.lock.Unlock()
|
||||
|
||||
if mbb.IsEmpty() {
|
||||
return nil
|
||||
}
|
||||
|
||||
now := mono.Now()
|
||||
mbb.SetTime(now, now)
|
||||
return mbb.ToProto()
|
||||
}
|
||||
|
||||
func (mc *MetricsCollector) reset() {
|
||||
mc.mbb = utils.NewMetricsBatchBuilder()
|
||||
mc.mbb.SetRestrictedLabels(utils.MetricRestrictedLabels{
|
||||
LabelRanges: []utils.MetricLabelRange{
|
||||
{
|
||||
StartInclusive: livekit.MetricLabel_CLIENT_VIDEO_SUBSCRIBER_FREEZE_COUNT,
|
||||
EndInclusive: livekit.MetricLabel_CLIENT_VIDEO_PUBLISHER_QUALITY_LIMITATION_DURATION_OTHER,
|
||||
},
|
||||
},
|
||||
ParticipantIdentity: mc.params.ParticipantIdentity,
|
||||
})
|
||||
|
||||
mc.publisherRTTMetricId = make(map[livekit.ParticipantIdentity]int)
|
||||
mc.subscriberRTTMetricId = utils.MetricsBatchBuilderInvalidTimeSeriesMetricId
|
||||
mc.relayRTTMetricId = make(map[livekit.ParticipantIdentity]int)
|
||||
}
|
||||
|
||||
func (mc *MetricsCollector) createTimeSeriesMetric(
|
||||
label livekit.MetricLabel,
|
||||
participantIdentity livekit.ParticipantIdentity,
|
||||
) (int, error) {
|
||||
return mc.mbb.AddTimeSeriesMetric(utils.TimeSeriesMetric{
|
||||
MetricLabel: label,
|
||||
ParticipantIdentity: participantIdentity,
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
func (mc *MetricsCollector) addTimeSeriesMetricSample(metricId int, value float32) {
|
||||
now := mono.Now()
|
||||
if err := mc.mbb.AddMetricSamplesToTimeSeriesMetric(metricId, []utils.MetricSample{
|
||||
{
|
||||
At: now,
|
||||
NormalizedAt: now,
|
||||
Value: value,
|
||||
},
|
||||
}); err != nil {
|
||||
mc.params.Logger.Warnw("could not add metric sample", err, "metricId", metricId)
|
||||
}
|
||||
}
|
||||
|
||||
func (mc *MetricsCollector) worker() {
|
||||
samplingIntervalMs := mc.params.Config.SamplingIntervalMs
|
||||
if samplingIntervalMs == 0 {
|
||||
samplingIntervalMs = DefaultMetricsCollectorConfig.SamplingIntervalMs
|
||||
}
|
||||
samplingTicker := time.NewTicker(time.Duration(samplingIntervalMs) * time.Millisecond)
|
||||
defer samplingTicker.Stop()
|
||||
|
||||
batchIntervalMs := mc.params.Config.BatchIntervalMs
|
||||
if batchIntervalMs < samplingIntervalMs {
|
||||
batchIntervalMs = samplingIntervalMs
|
||||
}
|
||||
batchTicker := time.NewTicker(time.Duration(batchIntervalMs) * time.Millisecond)
|
||||
defer batchTicker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-samplingTicker.C:
|
||||
mc.params.Provider.MetricsCollectorTimeToCollectMetrics()
|
||||
|
||||
case <-batchTicker.C:
|
||||
if mb := mc.getMetricsBatchAndReset(); mb != nil {
|
||||
mc.params.Provider.MetricsCollectorBatchReady(mb)
|
||||
}
|
||||
|
||||
case <-mc.stop.Watch():
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
// Copyright 2023 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package metric
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/frostbyte73/core"
|
||||
"github.com/livekit/protocol/livekit"
|
||||
"github.com/livekit/protocol/logger"
|
||||
"github.com/livekit/protocol/utils/mono"
|
||||
|
||||
"github.com/livekit/protocol/utils"
|
||||
)
|
||||
|
||||
type MetricsReporterConsumer interface {
|
||||
MetricsReporterBatchReady(mb *livekit.MetricsBatch)
|
||||
}
|
||||
|
||||
// --------------------------------------------------------
|
||||
|
||||
type MetricsReporterConfig struct {
|
||||
ReportingIntervalMs uint32 `yaml:"reporting_interval_ms,omitempty" json:"reporting_interval_ms,omitempty"`
|
||||
}
|
||||
|
||||
var (
|
||||
DefaultMetricsReporterConfig = MetricsReporterConfig{
|
||||
ReportingIntervalMs: 10 * 1000,
|
||||
}
|
||||
)
|
||||
|
||||
// --------------------------------------------------------
|
||||
|
||||
type MetricsReporterParams struct {
|
||||
ParticipantIdentity livekit.ParticipantIdentity
|
||||
Config MetricsReporterConfig
|
||||
Consumer MetricsReporterConsumer
|
||||
Logger logger.Logger
|
||||
}
|
||||
|
||||
type MetricsReporter struct {
|
||||
params MetricsReporterParams
|
||||
|
||||
lock sync.RWMutex
|
||||
mbb *utils.MetricsBatchBuilder
|
||||
|
||||
stop core.Fuse
|
||||
}
|
||||
|
||||
func NewMetricsReporter(params MetricsReporterParams) *MetricsReporter {
|
||||
mr := &MetricsReporter{
|
||||
params: params,
|
||||
}
|
||||
mr.reset()
|
||||
|
||||
go mr.worker()
|
||||
return mr
|
||||
}
|
||||
|
||||
func (mr *MetricsReporter) Stop() {
|
||||
if mr != nil {
|
||||
mr.stop.Break()
|
||||
}
|
||||
}
|
||||
|
||||
func (mr *MetricsReporter) Merge(other *livekit.MetricsBatch) {
|
||||
if mr == nil {
|
||||
return
|
||||
}
|
||||
|
||||
mr.lock.Lock()
|
||||
defer mr.lock.Unlock()
|
||||
|
||||
mr.mbb.Merge(other)
|
||||
}
|
||||
|
||||
func (mr *MetricsReporter) getMetricsBatchAndReset() *livekit.MetricsBatch {
|
||||
mr.lock.Lock()
|
||||
mbb := mr.mbb
|
||||
|
||||
mr.reset()
|
||||
mr.lock.Unlock()
|
||||
|
||||
if mbb.IsEmpty() {
|
||||
return nil
|
||||
}
|
||||
|
||||
now := mono.Now()
|
||||
mbb.SetTime(now, now)
|
||||
return mbb.ToProto()
|
||||
}
|
||||
|
||||
func (mr *MetricsReporter) reset() {
|
||||
mr.mbb = utils.NewMetricsBatchBuilder()
|
||||
mr.mbb.SetRestrictedLabels(utils.MetricRestrictedLabels{
|
||||
LabelRanges: []utils.MetricLabelRange{
|
||||
{
|
||||
StartInclusive: livekit.MetricLabel_CLIENT_VIDEO_SUBSCRIBER_FREEZE_COUNT,
|
||||
EndInclusive: livekit.MetricLabel_CLIENT_VIDEO_PUBLISHER_QUALITY_LIMITATION_DURATION_OTHER,
|
||||
},
|
||||
},
|
||||
ParticipantIdentity: mr.params.ParticipantIdentity,
|
||||
})
|
||||
}
|
||||
|
||||
func (mr *MetricsReporter) worker() {
|
||||
reportingIntervalMs := mr.params.Config.ReportingIntervalMs
|
||||
if reportingIntervalMs == 0 {
|
||||
reportingIntervalMs = DefaultMetricsReporterConfig.ReportingIntervalMs
|
||||
}
|
||||
reportingTicker := time.NewTicker(time.Duration(reportingIntervalMs) * time.Millisecond)
|
||||
defer reportingTicker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-reportingTicker.C:
|
||||
if mb := mr.getMetricsBatchAndReset(); mb != nil {
|
||||
mr.params.Consumer.MetricsReporterBatchReady(mb)
|
||||
}
|
||||
|
||||
case <-mr.stop.Watch():
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user