Squashed 'livekit-server/' content from commit 154b4d26
git-subtree-dir: livekit-server git-subtree-split: 154b4d26b769c68a03c096124094b97bf61a996f
This commit is contained in:
@@ -0,0 +1,75 @@
|
||||
// Copyright 2023 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package prometheus
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
|
||||
"github.com/livekit/protocol/livekit"
|
||||
)
|
||||
|
||||
var (
|
||||
promDataPacketStreamLabels = []string{"type", "mime_type"}
|
||||
promDataPacketStreamMimeTypes = []string{"text", "image", "application", "audio", "video"}
|
||||
|
||||
promDataPacketStreamDestCount *prometheus.HistogramVec
|
||||
promDataPacketStreamSize *prometheus.HistogramVec
|
||||
)
|
||||
|
||||
func initDataPacketStats(nodeID string, nodeType livekit.NodeType) {
|
||||
promDataPacketStreamDestCount = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "datapacket_stream",
|
||||
Name: "dest_count",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
Buckets: []float64{1, 2, 3, 4, 5, 10, 15, 25, 50},
|
||||
}, promDataPacketStreamLabels)
|
||||
promDataPacketStreamSize = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "datapacket_stream",
|
||||
Name: "bytes",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
Buckets: []float64{128, 512, 2048, 8192, 32768, 131072, 524288, 2097152, 8388608, 33554432},
|
||||
}, promDataPacketStreamLabels)
|
||||
|
||||
prometheus.MustRegister(promDataPacketStreamDestCount)
|
||||
prometheus.MustRegister(promDataPacketStreamSize)
|
||||
}
|
||||
|
||||
func RecordDataPacketStream(h *livekit.DataStream_Header, destCount int) {
|
||||
streamType := "unknown"
|
||||
switch h.ContentHeader.(type) {
|
||||
case *livekit.DataStream_Header_TextHeader:
|
||||
streamType = "text"
|
||||
case *livekit.DataStream_Header_ByteHeader:
|
||||
streamType = "bytes"
|
||||
}
|
||||
|
||||
mimeType := strings.ToLower(h.MimeType)
|
||||
if i := strings.IndexByte(mimeType, '/'); i != -1 {
|
||||
mimeType = mimeType[:i]
|
||||
}
|
||||
if !slices.Contains(promDataPacketStreamMimeTypes, mimeType) {
|
||||
mimeType = "unknown"
|
||||
}
|
||||
|
||||
promDataPacketStreamDestCount.WithLabelValues(streamType, mimeType).Observe(float64(destCount))
|
||||
if h.TotalLength != nil {
|
||||
promDataPacketStreamSize.WithLabelValues(streamType, mimeType).Observe(float64(*h.TotalLength))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,279 @@
|
||||
// Copyright 2023 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package prometheus
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/mackerelio/go-osstat/memory"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"go.uber.org/atomic"
|
||||
|
||||
"github.com/livekit/protocol/livekit"
|
||||
"github.com/livekit/protocol/rpc"
|
||||
"github.com/livekit/protocol/utils/hwstats"
|
||||
)
|
||||
|
||||
const (
|
||||
livekitNamespace string = "livekit"
|
||||
|
||||
statsUpdateInterval = time.Second * 10
|
||||
)
|
||||
|
||||
var (
|
||||
initialized atomic.Bool
|
||||
|
||||
MessageCounter *prometheus.CounterVec
|
||||
MessageBytes *prometheus.CounterVec
|
||||
ServiceOperationCounter *prometheus.CounterVec
|
||||
TwirpRequestStatusCounter *prometheus.CounterVec
|
||||
|
||||
sysPacketsStart uint32
|
||||
sysDroppedPacketsStart uint32
|
||||
promSysPacketGauge *prometheus.GaugeVec
|
||||
promSysDroppedPacketPctGauge prometheus.Gauge
|
||||
|
||||
cpuStats *hwstats.CPUStats
|
||||
)
|
||||
|
||||
func Init(nodeID string, nodeType livekit.NodeType) error {
|
||||
if initialized.Swap(true) {
|
||||
return nil
|
||||
}
|
||||
|
||||
MessageCounter = prometheus.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "node",
|
||||
Name: "messages",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
},
|
||||
[]string{"type", "status"},
|
||||
)
|
||||
|
||||
MessageBytes = prometheus.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "node",
|
||||
Name: "message_bytes",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
},
|
||||
[]string{"type", "message_type"},
|
||||
)
|
||||
|
||||
ServiceOperationCounter = prometheus.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "node",
|
||||
Name: "service_operation",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
},
|
||||
[]string{"type", "status", "error_type"},
|
||||
)
|
||||
|
||||
TwirpRequestStatusCounter = prometheus.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "node",
|
||||
Name: "twirp_request_status",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
},
|
||||
[]string{"service", "method", "status", "code"},
|
||||
)
|
||||
|
||||
promSysPacketGauge = prometheus.NewGaugeVec(
|
||||
prometheus.GaugeOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "node",
|
||||
Name: "packet_total",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
Help: "System level packet count. Count starts at 0 when service is first started.",
|
||||
},
|
||||
[]string{"type"},
|
||||
)
|
||||
|
||||
promSysDroppedPacketPctGauge = prometheus.NewGauge(
|
||||
prometheus.GaugeOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "node",
|
||||
Name: "dropped_packets",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
Help: "System level dropped outgoing packet percentage.",
|
||||
},
|
||||
)
|
||||
|
||||
prometheus.MustRegister(MessageCounter)
|
||||
prometheus.MustRegister(MessageBytes)
|
||||
prometheus.MustRegister(ServiceOperationCounter)
|
||||
prometheus.MustRegister(TwirpRequestStatusCounter)
|
||||
prometheus.MustRegister(promSysPacketGauge)
|
||||
prometheus.MustRegister(promSysDroppedPacketPctGauge)
|
||||
|
||||
sysPacketsStart, sysDroppedPacketsStart, _ = getTCStats()
|
||||
|
||||
initPacketStats(nodeID, nodeType)
|
||||
initRoomStats(nodeID, nodeType)
|
||||
rpc.InitPSRPCStats(prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()})
|
||||
initQualityStats(nodeID, nodeType)
|
||||
initDataPacketStats(nodeID, nodeType)
|
||||
|
||||
var err error
|
||||
cpuStats, err = hwstats.NewCPUStats(nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func GetUpdatedNodeStats(prev *livekit.NodeStats, prevAverage *livekit.NodeStats) (*livekit.NodeStats, bool, error) {
|
||||
loadAvg, err := getLoadAvg()
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
|
||||
var cpuLoad float64
|
||||
cpuIdle := cpuStats.GetCPUIdle()
|
||||
if cpuIdle > 0 {
|
||||
cpuLoad = 1 - (cpuIdle / cpuStats.NumCPU())
|
||||
}
|
||||
|
||||
// On MacOS, get "\"vm_stat\": executable file not found in $PATH" although it is in /usr/bin
|
||||
// So, do not error out. Use the information if it is available.
|
||||
memTotal := uint64(0)
|
||||
memUsed := uint64(0)
|
||||
memInfo, _ := memory.Get()
|
||||
if memInfo != nil {
|
||||
memTotal = memInfo.Total
|
||||
memUsed = memInfo.Used
|
||||
}
|
||||
|
||||
// do not error out, and use the information if it is available
|
||||
sysPackets, sysDroppedPackets, _ := getTCStats()
|
||||
promSysPacketGauge.WithLabelValues("out").Set(float64(sysPackets - sysPacketsStart))
|
||||
promSysPacketGauge.WithLabelValues("dropped").Set(float64(sysDroppedPackets - sysDroppedPacketsStart))
|
||||
|
||||
bytesInNow := bytesIn.Load()
|
||||
bytesOutNow := bytesOut.Load()
|
||||
packetsInNow := packetsIn.Load()
|
||||
packetsOutNow := packetsOut.Load()
|
||||
nackTotalNow := nackTotal.Load()
|
||||
retransmitBytesNow := retransmitBytes.Load()
|
||||
retransmitPacketsNow := retransmitPackets.Load()
|
||||
participantSignalConnectedNow := participantSignalConnected.Load()
|
||||
participantRTCInitNow := participantRTCInit.Load()
|
||||
participantRTConnectedCNow := participantRTCConnected.Load()
|
||||
trackPublishAttemptsNow := trackPublishAttempts.Load()
|
||||
trackPublishSuccessNow := trackPublishSuccess.Load()
|
||||
trackSubscribeAttemptsNow := trackSubscribeAttempts.Load()
|
||||
trackSubscribeSuccessNow := trackSubscribeSuccess.Load()
|
||||
forwardLatencyNow := forwardLatency.Load()
|
||||
forwardJitterNow := forwardJitter.Load()
|
||||
|
||||
updatedAt := time.Now().Unix()
|
||||
elapsed := updatedAt - prevAverage.UpdatedAt
|
||||
// include sufficient buffer to be sure a stats update had taken place
|
||||
computeAverage := elapsed > int64(statsUpdateInterval.Seconds()+2)
|
||||
if bytesInNow != prevAverage.BytesIn ||
|
||||
bytesOutNow != prevAverage.BytesOut ||
|
||||
packetsInNow != prevAverage.PacketsIn ||
|
||||
packetsOutNow != prevAverage.PacketsOut ||
|
||||
retransmitBytesNow != prevAverage.RetransmitBytesOut ||
|
||||
retransmitPacketsNow != prevAverage.RetransmitPacketsOut {
|
||||
computeAverage = true
|
||||
}
|
||||
|
||||
stats := &livekit.NodeStats{
|
||||
StartedAt: prev.StartedAt,
|
||||
UpdatedAt: updatedAt,
|
||||
NumRooms: roomCurrent.Load(),
|
||||
NumClients: participantCurrent.Load(),
|
||||
NumTracksIn: trackPublishedCurrent.Load(),
|
||||
NumTracksOut: trackSubscribedCurrent.Load(),
|
||||
NumTrackPublishAttempts: trackPublishAttemptsNow,
|
||||
NumTrackPublishSuccess: trackPublishSuccessNow,
|
||||
NumTrackSubscribeAttempts: trackSubscribeAttemptsNow,
|
||||
NumTrackSubscribeSuccess: trackSubscribeSuccessNow,
|
||||
BytesIn: bytesInNow,
|
||||
BytesOut: bytesOutNow,
|
||||
PacketsIn: packetsInNow,
|
||||
PacketsOut: packetsOutNow,
|
||||
RetransmitBytesOut: retransmitBytesNow,
|
||||
RetransmitPacketsOut: retransmitPacketsNow,
|
||||
NackTotal: nackTotalNow,
|
||||
ParticipantSignalConnected: participantSignalConnectedNow,
|
||||
ParticipantRtcInit: participantRTCInitNow,
|
||||
ParticipantRtcConnected: participantRTConnectedCNow,
|
||||
BytesInPerSec: prevAverage.BytesInPerSec,
|
||||
BytesOutPerSec: prevAverage.BytesOutPerSec,
|
||||
PacketsInPerSec: prevAverage.PacketsInPerSec,
|
||||
PacketsOutPerSec: prevAverage.PacketsOutPerSec,
|
||||
RetransmitBytesOutPerSec: prevAverage.RetransmitBytesOutPerSec,
|
||||
RetransmitPacketsOutPerSec: prevAverage.RetransmitPacketsOutPerSec,
|
||||
NackPerSec: prevAverage.NackPerSec,
|
||||
ForwardLatency: forwardLatencyNow,
|
||||
ForwardJitter: forwardJitterNow,
|
||||
ParticipantSignalConnectedPerSec: prevAverage.ParticipantSignalConnectedPerSec,
|
||||
ParticipantRtcInitPerSec: prevAverage.ParticipantRtcInitPerSec,
|
||||
ParticipantRtcConnectedPerSec: prevAverage.ParticipantRtcConnectedPerSec,
|
||||
NumCpus: uint32(cpuStats.NumCPU()), // this will round down to the nearest integer
|
||||
CpuLoad: float32(cpuLoad),
|
||||
MemoryTotal: memTotal,
|
||||
MemoryUsed: memUsed,
|
||||
LoadAvgLast1Min: float32(loadAvg.Loadavg1),
|
||||
LoadAvgLast5Min: float32(loadAvg.Loadavg5),
|
||||
LoadAvgLast15Min: float32(loadAvg.Loadavg15),
|
||||
SysPacketsOut: sysPackets,
|
||||
SysPacketsDropped: sysDroppedPackets,
|
||||
TrackPublishAttemptsPerSec: prevAverage.TrackPublishAttemptsPerSec,
|
||||
TrackPublishSuccessPerSec: prevAverage.TrackPublishSuccessPerSec,
|
||||
TrackSubscribeAttemptsPerSec: prevAverage.TrackSubscribeAttemptsPerSec,
|
||||
TrackSubscribeSuccessPerSec: prevAverage.TrackSubscribeSuccessPerSec,
|
||||
}
|
||||
|
||||
// update stats
|
||||
if computeAverage {
|
||||
stats.BytesInPerSec = perSec(prevAverage.BytesIn, bytesInNow, elapsed)
|
||||
stats.BytesOutPerSec = perSec(prevAverage.BytesOut, bytesOutNow, elapsed)
|
||||
stats.PacketsInPerSec = perSec(prevAverage.PacketsIn, packetsInNow, elapsed)
|
||||
stats.PacketsOutPerSec = perSec(prevAverage.PacketsOut, packetsOutNow, elapsed)
|
||||
stats.RetransmitBytesOutPerSec = perSec(prevAverage.RetransmitBytesOut, retransmitBytesNow, elapsed)
|
||||
stats.RetransmitPacketsOutPerSec = perSec(prevAverage.RetransmitPacketsOut, retransmitPacketsNow, elapsed)
|
||||
stats.NackPerSec = perSec(prevAverage.NackTotal, nackTotalNow, elapsed)
|
||||
stats.ParticipantSignalConnectedPerSec = perSec(prevAverage.ParticipantSignalConnected, participantSignalConnectedNow, elapsed)
|
||||
stats.ParticipantRtcInitPerSec = perSec(prevAverage.ParticipantRtcInit, participantRTCInitNow, elapsed)
|
||||
stats.ParticipantRtcConnectedPerSec = perSec(prevAverage.ParticipantRtcConnected, participantRTConnectedCNow, elapsed)
|
||||
stats.SysPacketsOutPerSec = perSec(uint64(prevAverage.SysPacketsOut), uint64(sysPackets), elapsed)
|
||||
stats.SysPacketsDroppedPerSec = perSec(uint64(prevAverage.SysPacketsDropped), uint64(sysDroppedPackets), elapsed)
|
||||
stats.TrackPublishAttemptsPerSec = perSec(uint64(prevAverage.NumTrackPublishAttempts), uint64(trackPublishAttemptsNow), elapsed)
|
||||
stats.TrackPublishSuccessPerSec = perSec(uint64(prevAverage.NumTrackPublishSuccess), uint64(trackPublishSuccessNow), elapsed)
|
||||
stats.TrackSubscribeAttemptsPerSec = perSec(uint64(prevAverage.NumTrackSubscribeAttempts), uint64(trackSubscribeAttemptsNow), elapsed)
|
||||
stats.TrackSubscribeSuccessPerSec = perSec(uint64(prevAverage.NumTrackSubscribeSuccess), uint64(trackSubscribeSuccessNow), elapsed)
|
||||
|
||||
packetTotal := stats.SysPacketsOutPerSec + stats.SysPacketsDroppedPerSec
|
||||
if packetTotal == 0 {
|
||||
stats.SysPacketsDroppedPctPerSec = 0
|
||||
} else {
|
||||
stats.SysPacketsDroppedPctPerSec = stats.SysPacketsDroppedPerSec / packetTotal
|
||||
}
|
||||
promSysDroppedPacketPctGauge.Set(float64(stats.SysPacketsDroppedPctPerSec))
|
||||
}
|
||||
|
||||
return stats, computeAverage, nil
|
||||
}
|
||||
|
||||
func perSec(prev, curr uint64, secs int64) float32 {
|
||||
return float32(curr-prev) / float32(secs)
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
// Copyright 2023 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//go:build linux
|
||||
// +build linux
|
||||
|
||||
package prometheus
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/florianl/go-tc"
|
||||
)
|
||||
|
||||
func getTCStats() (packets, drops uint32, err error) {
|
||||
rtnl, err := tc.Open(&tc.Config{})
|
||||
if err != nil {
|
||||
err = fmt.Errorf("could not open rtnetlink socket: %v", err)
|
||||
return
|
||||
}
|
||||
defer rtnl.Close()
|
||||
|
||||
qdiscs, err := rtnl.Qdisc().Get()
|
||||
if err != nil {
|
||||
err = fmt.Errorf("could not get qdiscs: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
for _, qdisc := range qdiscs {
|
||||
packets = packets + qdisc.Stats.Packets
|
||||
drops = drops + qdisc.Stats.Drops
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
// Copyright 2023 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//go:build !linux
|
||||
|
||||
package prometheus
|
||||
|
||||
func getTCStats() (packets, drops uint32, err error) {
|
||||
// linux only
|
||||
return
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
//go:build !windows
|
||||
|
||||
/*
|
||||
* Copyright 2023 LiveKit, Inc
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package prometheus
|
||||
|
||||
import (
|
||||
"runtime"
|
||||
"sync"
|
||||
|
||||
"github.com/mackerelio/go-osstat/cpu"
|
||||
"github.com/mackerelio/go-osstat/loadavg"
|
||||
)
|
||||
|
||||
var (
|
||||
cpuStatsLock sync.RWMutex
|
||||
lastCPUTotal, lastCPUIdle uint64
|
||||
)
|
||||
|
||||
func getLoadAvg() (*loadavg.Stats, error) {
|
||||
return loadavg.Get()
|
||||
}
|
||||
|
||||
func getCPUStats() (cpuLoad float32, numCPUs uint32, err error) {
|
||||
cpuInfo, err := cpu.Get()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
cpuStatsLock.Lock()
|
||||
if lastCPUTotal > 0 && lastCPUTotal < cpuInfo.Total {
|
||||
cpuLoad = 1 - float32(cpuInfo.Idle-lastCPUIdle)/float32(cpuInfo.Total-lastCPUTotal)
|
||||
}
|
||||
|
||||
lastCPUTotal = cpuInfo.Total
|
||||
lastCPUIdle = cpuInfo.Idle
|
||||
cpuStatsLock.Unlock()
|
||||
|
||||
numCPUs = uint32(runtime.NumCPU())
|
||||
|
||||
return
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
//go:build windows
|
||||
|
||||
/*
|
||||
* Copyright 2023 LiveKit, Inc
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package prometheus
|
||||
|
||||
import "github.com/mackerelio/go-osstat/loadavg"
|
||||
|
||||
func getLoadAvg() (*loadavg.Stats, error) {
|
||||
return &loadavg.Stats{}, nil
|
||||
}
|
||||
|
||||
func getCPUStats() (cpuLoad float32, numCPUs uint32, err error) {
|
||||
return 1, 1, nil
|
||||
}
|
||||
@@ -0,0 +1,336 @@
|
||||
// Copyright 2023 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package prometheus
|
||||
|
||||
import (
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"go.uber.org/atomic"
|
||||
|
||||
"github.com/livekit/protocol/livekit"
|
||||
)
|
||||
|
||||
type Direction string
|
||||
|
||||
const (
|
||||
Incoming Direction = "incoming"
|
||||
Outgoing Direction = "outgoing"
|
||||
transmissionInitial = "initial"
|
||||
transmissionRetransmit = "retransmit"
|
||||
)
|
||||
|
||||
var (
|
||||
bytesIn atomic.Uint64
|
||||
bytesOut atomic.Uint64
|
||||
packetsIn atomic.Uint64
|
||||
packetsOut atomic.Uint64
|
||||
nackTotal atomic.Uint64
|
||||
retransmitBytes atomic.Uint64
|
||||
retransmitPackets atomic.Uint64
|
||||
participantSignalConnected atomic.Uint64
|
||||
participantRTCConnected atomic.Uint64
|
||||
participantRTCInit atomic.Uint64
|
||||
forwardLatency atomic.Uint32
|
||||
forwardJitter atomic.Uint32
|
||||
|
||||
promPacketLabels = []string{"direction", "transmission"}
|
||||
promPacketTotal *prometheus.CounterVec
|
||||
promPacketBytes *prometheus.CounterVec
|
||||
promRTCPLabels = []string{"direction"}
|
||||
promStreamLabels = []string{"direction", "source", "type"}
|
||||
promNackTotal *prometheus.CounterVec
|
||||
promPliTotal *prometheus.CounterVec
|
||||
promFirTotal *prometheus.CounterVec
|
||||
promPacketLossTotal *prometheus.CounterVec
|
||||
promPacketLoss *prometheus.HistogramVec
|
||||
promPacketOutOfOrderTotal *prometheus.CounterVec
|
||||
promPacketOutOfOrder *prometheus.HistogramVec
|
||||
promJitter *prometheus.HistogramVec
|
||||
promRTT *prometheus.HistogramVec
|
||||
promParticipantJoin *prometheus.CounterVec
|
||||
promConnections *prometheus.GaugeVec
|
||||
promForwardLatency prometheus.Gauge
|
||||
promForwardJitter prometheus.Gauge
|
||||
|
||||
promPacketTotalIncomingInitial prometheus.Counter
|
||||
promPacketTotalIncomingRetransmit prometheus.Counter
|
||||
promPacketTotalOutgoingInitial prometheus.Counter
|
||||
promPacketTotalOutgoingRetransmit prometheus.Counter
|
||||
promPacketBytesIncomingInitial prometheus.Counter
|
||||
promPacketBytesIncomingRetransmit prometheus.Counter
|
||||
promPacketBytesOutgoingInitial prometheus.Counter
|
||||
promPacketBytesOutgoingRetransmit prometheus.Counter
|
||||
)
|
||||
|
||||
func initPacketStats(nodeID string, nodeType livekit.NodeType) {
|
||||
promPacketTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "packet",
|
||||
Name: "total",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
}, promPacketLabels)
|
||||
promPacketBytes = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "packet",
|
||||
Name: "bytes",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
}, promPacketLabels)
|
||||
promNackTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "nack",
|
||||
Name: "total",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
}, promRTCPLabels)
|
||||
promPliTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "pli",
|
||||
Name: "total",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
}, promRTCPLabels)
|
||||
promFirTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "fir",
|
||||
Name: "total",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
}, promRTCPLabels)
|
||||
promPacketLossTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "packet_loss",
|
||||
Name: "total",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
}, promStreamLabels)
|
||||
promPacketLoss = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "packet_loss",
|
||||
Name: "percent",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
Buckets: []float64{0.0, 0.1, 0.3, 0.5, 0.7, 1, 5, 10, 40, 100},
|
||||
}, promStreamLabels)
|
||||
promPacketOutOfOrderTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "packet_out_of_order",
|
||||
Name: "total",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
}, promStreamLabels)
|
||||
promPacketOutOfOrder = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "packet_out_of_order",
|
||||
Name: "percent",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
Buckets: []float64{0.0, 0.1, 0.3, 0.5, 0.7, 1, 5, 10, 40, 100},
|
||||
}, promStreamLabels)
|
||||
promJitter = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "jitter",
|
||||
Name: "us",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
|
||||
// 1ms, 10ms, 30ms, 50ms, 70ms, 100ms, 300ms, 600ms, 1s
|
||||
Buckets: []float64{1000, 10000, 30000, 50000, 70000, 100000, 300000, 600000, 1000000},
|
||||
}, promStreamLabels)
|
||||
promRTT = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "rtt",
|
||||
Name: "ms",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
Buckets: []float64{50, 100, 150, 200, 250, 500, 750, 1000, 5000, 10000},
|
||||
}, promStreamLabels)
|
||||
promParticipantJoin = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "participant_join",
|
||||
Name: "total",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
}, []string{"state"})
|
||||
promConnections = prometheus.NewGaugeVec(prometheus.GaugeOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "connection",
|
||||
Name: "total",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
}, []string{"kind"})
|
||||
promForwardLatency = prometheus.NewGauge(prometheus.GaugeOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "forward",
|
||||
Name: "latency",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
})
|
||||
promForwardJitter = prometheus.NewGauge(prometheus.GaugeOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "forward",
|
||||
Name: "jitter",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
})
|
||||
|
||||
prometheus.MustRegister(promPacketTotal)
|
||||
prometheus.MustRegister(promPacketBytes)
|
||||
prometheus.MustRegister(promNackTotal)
|
||||
prometheus.MustRegister(promPliTotal)
|
||||
prometheus.MustRegister(promFirTotal)
|
||||
prometheus.MustRegister(promPacketLossTotal)
|
||||
prometheus.MustRegister(promPacketLoss)
|
||||
prometheus.MustRegister(promPacketOutOfOrderTotal)
|
||||
prometheus.MustRegister(promPacketOutOfOrder)
|
||||
prometheus.MustRegister(promJitter)
|
||||
prometheus.MustRegister(promRTT)
|
||||
prometheus.MustRegister(promParticipantJoin)
|
||||
prometheus.MustRegister(promConnections)
|
||||
prometheus.MustRegister(promForwardLatency)
|
||||
prometheus.MustRegister(promForwardJitter)
|
||||
|
||||
promPacketTotalIncomingInitial = promPacketTotal.WithLabelValues(string(Incoming), transmissionInitial)
|
||||
promPacketTotalIncomingRetransmit = promPacketTotal.WithLabelValues(string(Incoming), transmissionRetransmit)
|
||||
promPacketTotalOutgoingInitial = promPacketTotal.WithLabelValues(string(Outgoing), transmissionInitial)
|
||||
promPacketTotalOutgoingRetransmit = promPacketTotal.WithLabelValues(string(Outgoing), transmissionRetransmit)
|
||||
promPacketBytesIncomingInitial = promPacketBytes.WithLabelValues(string(Incoming), transmissionInitial)
|
||||
promPacketBytesIncomingRetransmit = promPacketBytes.WithLabelValues(string(Incoming), transmissionRetransmit)
|
||||
promPacketBytesOutgoingInitial = promPacketBytes.WithLabelValues(string(Outgoing), transmissionInitial)
|
||||
promPacketBytesOutgoingRetransmit = promPacketBytes.WithLabelValues(string(Outgoing), transmissionRetransmit)
|
||||
}
|
||||
|
||||
func IncrementPackets(direction Direction, count uint64, retransmit bool) {
|
||||
if direction == Incoming {
|
||||
if retransmit {
|
||||
promPacketTotalIncomingRetransmit.Add(float64(count))
|
||||
} else {
|
||||
promPacketTotalIncomingInitial.Add(float64(count))
|
||||
}
|
||||
} else {
|
||||
if retransmit {
|
||||
promPacketTotalOutgoingRetransmit.Add(float64(count))
|
||||
} else {
|
||||
promPacketTotalOutgoingInitial.Add(float64(count))
|
||||
}
|
||||
}
|
||||
|
||||
if direction == Incoming {
|
||||
packetsIn.Add(count)
|
||||
} else {
|
||||
packetsOut.Add(count)
|
||||
if retransmit {
|
||||
retransmitPackets.Add(count)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func IncrementBytes(direction Direction, count uint64, retransmit bool) {
|
||||
if direction == Incoming {
|
||||
if retransmit {
|
||||
promPacketBytesIncomingRetransmit.Add(float64(count))
|
||||
} else {
|
||||
promPacketBytesIncomingInitial.Add(float64(count))
|
||||
}
|
||||
} else {
|
||||
if retransmit {
|
||||
promPacketBytesOutgoingRetransmit.Add(float64(count))
|
||||
} else {
|
||||
promPacketBytesOutgoingInitial.Add(float64(count))
|
||||
}
|
||||
}
|
||||
|
||||
if direction == Incoming {
|
||||
bytesIn.Add(count)
|
||||
} else {
|
||||
bytesOut.Add(count)
|
||||
if retransmit {
|
||||
retransmitBytes.Add(count)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func IncrementRTCP(direction Direction, nack, pli, fir uint32) {
|
||||
if nack > 0 {
|
||||
promNackTotal.WithLabelValues(string(direction)).Add(float64(nack))
|
||||
nackTotal.Add(uint64(nack))
|
||||
}
|
||||
if pli > 0 {
|
||||
promPliTotal.WithLabelValues(string(direction)).Add(float64(pli))
|
||||
}
|
||||
if fir > 0 {
|
||||
promFirTotal.WithLabelValues(string(direction)).Add(float64(fir))
|
||||
}
|
||||
}
|
||||
|
||||
func RecordPacketLoss(direction Direction, trackSource livekit.TrackSource, trackType livekit.TrackType, lost, total uint32) {
|
||||
if total > 0 {
|
||||
promPacketLoss.WithLabelValues(string(direction), trackSource.String(), trackType.String()).Observe(float64(lost) / float64(total) * 100)
|
||||
}
|
||||
if lost > 0 {
|
||||
promPacketLossTotal.WithLabelValues(string(direction), trackSource.String(), trackType.String()).Add(float64(lost))
|
||||
}
|
||||
}
|
||||
|
||||
func RecordPacketOutOfOrder(direction Direction, trackSource livekit.TrackSource, trackType livekit.TrackType, ooo, total uint32) {
|
||||
if total > 0 {
|
||||
promPacketOutOfOrder.WithLabelValues(string(direction), trackSource.String(), trackType.String()).Observe(float64(ooo) / float64(total) * 100)
|
||||
}
|
||||
if ooo > 0 {
|
||||
promPacketOutOfOrderTotal.WithLabelValues(string(direction), trackSource.String(), trackType.String()).Add(float64(ooo))
|
||||
}
|
||||
}
|
||||
|
||||
func RecordJitter(direction Direction, trackSource livekit.TrackSource, trackType livekit.TrackType, jitter uint32) {
|
||||
if jitter > 0 {
|
||||
promJitter.WithLabelValues(string(direction), trackSource.String(), trackType.String()).Observe(float64(jitter))
|
||||
}
|
||||
}
|
||||
|
||||
func RecordRTT(direction Direction, trackSource livekit.TrackSource, trackType livekit.TrackType, rtt uint32) {
|
||||
if rtt > 0 {
|
||||
promRTT.WithLabelValues(string(direction), trackSource.String(), trackType.String()).Observe(float64(rtt))
|
||||
}
|
||||
}
|
||||
|
||||
func IncrementParticipantJoin(join uint32) {
|
||||
if join > 0 {
|
||||
participantSignalConnected.Add(uint64(join))
|
||||
promParticipantJoin.WithLabelValues("signal_connected").Add(float64(join))
|
||||
}
|
||||
}
|
||||
|
||||
func IncrementParticipantJoinFail(join uint32) {
|
||||
if join > 0 {
|
||||
promParticipantJoin.WithLabelValues("signal_failed").Add(float64(join))
|
||||
}
|
||||
}
|
||||
|
||||
func IncrementParticipantRtcInit(join uint32) {
|
||||
if join > 0 {
|
||||
participantRTCInit.Add(uint64(join))
|
||||
promParticipantJoin.WithLabelValues("rtc_init").Add(float64(join))
|
||||
}
|
||||
}
|
||||
|
||||
func IncrementParticipantRtcConnected(join uint32) {
|
||||
if join > 0 {
|
||||
participantRTCConnected.Add(uint64(join))
|
||||
promParticipantJoin.WithLabelValues("rtc_connected").Add(float64(join))
|
||||
}
|
||||
}
|
||||
|
||||
func AddConnection(direction Direction) {
|
||||
promConnections.WithLabelValues(string(direction)).Add(1)
|
||||
}
|
||||
|
||||
func SubConnection(direction Direction) {
|
||||
promConnections.WithLabelValues(string(direction)).Sub(1)
|
||||
}
|
||||
|
||||
func RecordForwardLatency(_, latencyAvg uint32) {
|
||||
forwardLatency.Store(latencyAvg)
|
||||
promForwardLatency.Set(float64(latencyAvg))
|
||||
}
|
||||
|
||||
func RecordForwardJitter(_, jitterAvg uint32) {
|
||||
forwardJitter.Store(jitterAvg)
|
||||
promForwardJitter.Set(float64(jitterAvg))
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
// Copyright 2023 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package prometheus
|
||||
|
||||
import (
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
|
||||
"github.com/livekit/protocol/livekit"
|
||||
)
|
||||
|
||||
var (
|
||||
qualityRating prometheus.Histogram
|
||||
qualityScore prometheus.Histogram
|
||||
)
|
||||
|
||||
func initQualityStats(nodeID string, nodeType livekit.NodeType) {
|
||||
qualityRating = prometheus.NewHistogram(prometheus.HistogramOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "quality",
|
||||
Name: "rating",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
Buckets: []float64{0, 1, 2},
|
||||
})
|
||||
qualityScore = prometheus.NewHistogram(prometheus.HistogramOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "quality",
|
||||
Name: "score",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
Buckets: []float64{1.0, 2.0, 2.5, 3.0, 3.25, 3.5, 3.75, 4.0, 4.25, 4.5},
|
||||
})
|
||||
|
||||
prometheus.MustRegister(qualityRating)
|
||||
prometheus.MustRegister(qualityScore)
|
||||
}
|
||||
|
||||
func RecordQuality(rating livekit.ConnectionQuality, score float32) {
|
||||
qualityRating.Observe(float64(rating))
|
||||
qualityScore.Observe(float64(score))
|
||||
}
|
||||
@@ -0,0 +1,226 @@
|
||||
// Copyright 2023 LiveKit, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package prometheus
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"go.uber.org/atomic"
|
||||
|
||||
"github.com/livekit/protocol/livekit"
|
||||
)
|
||||
|
||||
var (
|
||||
roomCurrent atomic.Int32
|
||||
participantCurrent atomic.Int32
|
||||
trackPublishedCurrent atomic.Int32
|
||||
trackSubscribedCurrent atomic.Int32
|
||||
trackPublishAttempts atomic.Int32
|
||||
trackPublishSuccess atomic.Int32
|
||||
trackSubscribeAttempts atomic.Int32
|
||||
trackSubscribeSuccess atomic.Int32
|
||||
// count the number of failures that are due to user error (permissions, track doesn't exist), so we could compute
|
||||
// success rate by subtracting this from total attempts
|
||||
trackSubscribeUserError atomic.Int32
|
||||
|
||||
promRoomCurrent prometheus.Gauge
|
||||
promRoomDuration prometheus.Histogram
|
||||
promParticipantCurrent prometheus.Gauge
|
||||
promTrackPublishedCurrent *prometheus.GaugeVec
|
||||
promTrackSubscribedCurrent *prometheus.GaugeVec
|
||||
promTrackPublishCounter *prometheus.CounterVec
|
||||
promTrackSubscribeCounter *prometheus.CounterVec
|
||||
promSessionStartTime *prometheus.HistogramVec
|
||||
promSessionDuration *prometheus.HistogramVec
|
||||
promPubSubTime *prometheus.HistogramVec
|
||||
)
|
||||
|
||||
func initRoomStats(nodeID string, nodeType livekit.NodeType) {
|
||||
promRoomCurrent = prometheus.NewGauge(prometheus.GaugeOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "room",
|
||||
Name: "total",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
})
|
||||
promRoomDuration = prometheus.NewHistogram(prometheus.HistogramOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "room",
|
||||
Name: "duration_seconds",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
Buckets: []float64{
|
||||
5, 10, 60, 5 * 60, 10 * 60, 30 * 60, 60 * 60, 2 * 60 * 60, 5 * 60 * 60, 10 * 60 * 60,
|
||||
},
|
||||
})
|
||||
promParticipantCurrent = prometheus.NewGauge(prometheus.GaugeOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "participant",
|
||||
Name: "total",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
})
|
||||
promTrackPublishedCurrent = prometheus.NewGaugeVec(prometheus.GaugeOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "track",
|
||||
Name: "published_total",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
}, []string{"kind"})
|
||||
promTrackSubscribedCurrent = prometheus.NewGaugeVec(prometheus.GaugeOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "track",
|
||||
Name: "subscribed_total",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
}, []string{"kind"})
|
||||
promTrackPublishCounter = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "track",
|
||||
Name: "publish_counter",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
}, []string{"kind", "state"})
|
||||
promTrackSubscribeCounter = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "track",
|
||||
Name: "subscribe_counter",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
}, []string{"state", "error"})
|
||||
promSessionStartTime = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "session",
|
||||
Name: "start_time_ms",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
Buckets: prometheus.ExponentialBucketsRange(100, 10000, 15),
|
||||
}, []string{"protocol_version"})
|
||||
promSessionDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "session",
|
||||
Name: "duration_ms",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
Buckets: prometheus.ExponentialBucketsRange(100, 4*60*60*1000, 15),
|
||||
}, []string{"protocol_version"})
|
||||
promPubSubTime = prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Namespace: livekitNamespace,
|
||||
Subsystem: "pubsubtime",
|
||||
Name: "ms",
|
||||
ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
|
||||
Buckets: []float64{100, 200, 500, 700, 1000, 5000, 10000},
|
||||
}, append(promStreamLabels, "sdk", "kind", "count"))
|
||||
|
||||
prometheus.MustRegister(promRoomCurrent)
|
||||
prometheus.MustRegister(promRoomDuration)
|
||||
prometheus.MustRegister(promParticipantCurrent)
|
||||
prometheus.MustRegister(promTrackPublishedCurrent)
|
||||
prometheus.MustRegister(promTrackSubscribedCurrent)
|
||||
prometheus.MustRegister(promTrackPublishCounter)
|
||||
prometheus.MustRegister(promTrackSubscribeCounter)
|
||||
prometheus.MustRegister(promSessionStartTime)
|
||||
prometheus.MustRegister(promSessionDuration)
|
||||
prometheus.MustRegister(promPubSubTime)
|
||||
}
|
||||
|
||||
func RoomStarted() {
|
||||
promRoomCurrent.Add(1)
|
||||
roomCurrent.Inc()
|
||||
}
|
||||
|
||||
func RoomEnded(startedAt time.Time) {
|
||||
if !startedAt.IsZero() {
|
||||
promRoomDuration.Observe(float64(time.Since(startedAt)) / float64(time.Second))
|
||||
}
|
||||
promRoomCurrent.Sub(1)
|
||||
roomCurrent.Dec()
|
||||
}
|
||||
|
||||
func AddParticipant() {
|
||||
promParticipantCurrent.Add(1)
|
||||
participantCurrent.Inc()
|
||||
}
|
||||
|
||||
func SubParticipant() {
|
||||
promParticipantCurrent.Sub(1)
|
||||
participantCurrent.Dec()
|
||||
}
|
||||
|
||||
func AddPublishedTrack(kind string) {
|
||||
promTrackPublishedCurrent.WithLabelValues(kind).Add(1)
|
||||
trackPublishedCurrent.Inc()
|
||||
}
|
||||
|
||||
func SubPublishedTrack(kind string) {
|
||||
promTrackPublishedCurrent.WithLabelValues(kind).Sub(1)
|
||||
trackPublishedCurrent.Dec()
|
||||
}
|
||||
|
||||
func AddPublishAttempt(kind string) {
|
||||
trackPublishAttempts.Inc()
|
||||
promTrackPublishCounter.WithLabelValues(kind, "attempt").Inc()
|
||||
}
|
||||
|
||||
func AddPublishSuccess(kind string) {
|
||||
trackPublishSuccess.Inc()
|
||||
promTrackPublishCounter.WithLabelValues(kind, "success").Inc()
|
||||
}
|
||||
|
||||
func RecordPublishTime(source livekit.TrackSource, trackType livekit.TrackType, d time.Duration, sdk livekit.ClientInfo_SDK, kind livekit.ParticipantInfo_Kind) {
|
||||
recordPubSubTime(true, source, trackType, d, sdk, kind, 1)
|
||||
}
|
||||
|
||||
func RecordSubscribeTime(source livekit.TrackSource, trackType livekit.TrackType, d time.Duration, sdk livekit.ClientInfo_SDK, kind livekit.ParticipantInfo_Kind, count int) {
|
||||
recordPubSubTime(false, source, trackType, d, sdk, kind, count)
|
||||
}
|
||||
|
||||
func recordPubSubTime(isPublish bool, source livekit.TrackSource, trackType livekit.TrackType, d time.Duration, sdk livekit.ClientInfo_SDK, kind livekit.ParticipantInfo_Kind, count int) {
|
||||
direction := "subscribe"
|
||||
if isPublish {
|
||||
direction = "publish"
|
||||
}
|
||||
promPubSubTime.WithLabelValues(direction, source.String(), trackType.String(), sdk.String(), kind.String(), strconv.Itoa(count)).Observe(float64(d.Milliseconds()))
|
||||
}
|
||||
|
||||
func RecordTrackSubscribeSuccess(kind string) {
|
||||
// modify both current and total counters
|
||||
promTrackSubscribedCurrent.WithLabelValues(kind).Add(1)
|
||||
trackSubscribedCurrent.Inc()
|
||||
|
||||
promTrackSubscribeCounter.WithLabelValues("success", "").Inc()
|
||||
trackSubscribeSuccess.Inc()
|
||||
}
|
||||
|
||||
func RecordTrackUnsubscribed(kind string) {
|
||||
// unsubscribed modifies current counter, but we leave the total values alone since they
|
||||
// are used to compute rate
|
||||
promTrackSubscribedCurrent.WithLabelValues(kind).Sub(1)
|
||||
trackSubscribedCurrent.Dec()
|
||||
}
|
||||
|
||||
func RecordTrackSubscribeAttempt() {
|
||||
trackSubscribeAttempts.Inc()
|
||||
promTrackSubscribeCounter.WithLabelValues("attempt", "").Inc()
|
||||
}
|
||||
|
||||
func RecordTrackSubscribeFailure(err error, isUserError bool) {
|
||||
promTrackSubscribeCounter.WithLabelValues("failure", err.Error()).Inc()
|
||||
|
||||
if isUserError {
|
||||
trackSubscribeUserError.Inc()
|
||||
}
|
||||
}
|
||||
|
||||
func RecordSessionStartTime(protocolVersion int, d time.Duration) {
|
||||
promSessionStartTime.WithLabelValues(strconv.Itoa(protocolVersion)).Observe(float64(d.Milliseconds()))
|
||||
}
|
||||
|
||||
func RecordSessionDuration(protocolVersion int, d time.Duration) {
|
||||
promSessionDuration.WithLabelValues(strconv.Itoa(protocolVersion)).Observe(float64(d.Milliseconds()))
|
||||
}
|
||||
Reference in New Issue
Block a user