Merge commit '0da97ebd2149ec2a0412a273b17270f540431d81' as 'livekit-server'

livekit-server/pkg/telemetry/analyticsservice.go

@@ -0,0 +1,97 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package telemetry
+
+import (
+	"context"
+
+	"go.uber.org/atomic"
+	"google.golang.org/protobuf/types/known/timestamppb"
+
+	"github.com/livekit/protocol/livekit"
+	"github.com/livekit/protocol/logger"
+	"github.com/livekit/protocol/rpc"
+	"github.com/livekit/protocol/utils/guid"
+
+	"github.com/livekit/livekit-server/pkg/config"
+	"github.com/livekit/livekit-server/pkg/routing"
+)
+
+//counterfeiter:generate . AnalyticsService
+type AnalyticsService interface {
+	SendStats(ctx context.Context, stats []*livekit.AnalyticsStat)
+	SendEvent(ctx context.Context, events *livekit.AnalyticsEvent)
+	SendNodeRoomStates(ctx context.Context, nodeRooms *livekit.AnalyticsNodeRooms)
+}
+
+type analyticsService struct {
+	analyticsKey   string
+	nodeID         string
+	sequenceNumber atomic.Uint64
+
+	events    rpc.AnalyticsRecorderService_IngestEventsClient
+	stats     rpc.AnalyticsRecorderService_IngestStatsClient
+	nodeRooms rpc.AnalyticsRecorderService_IngestNodeRoomStatesClient
+}
+
+func NewAnalyticsService(_ *config.Config, currentNode routing.LocalNode) AnalyticsService {
+	return &analyticsService{
+		analyticsKey: "", // TODO: conf.AnalyticsKey
+		nodeID:       string(currentNode.NodeID()),
+	}
+}
+
+func (a *analyticsService) SendStats(_ context.Context, stats []*livekit.AnalyticsStat) {
+	if a.stats == nil {
+		return
+	}
+
+	for _, stat := range stats {
+		stat.Id = guid.New("AS_")
+		stat.AnalyticsKey = a.analyticsKey
+		stat.Node = a.nodeID
+	}
+	if err := a.stats.Send(&livekit.AnalyticsStats{Stats: stats}); err != nil {
+		logger.Errorw("failed to send stats", err)
+	}
+}
+
+func (a *analyticsService) SendEvent(_ context.Context, event *livekit.AnalyticsEvent) {
+	if a.events == nil {
+		return
+	}
+
+	event.Id = guid.New("AE_")
+	event.NodeId = a.nodeID
+	event.AnalyticsKey = a.analyticsKey
+	if err := a.events.Send(&livekit.AnalyticsEvents{
+		Events: []*livekit.AnalyticsEvent{event},
+	}); err != nil {
+		logger.Errorw("failed to send event", err, "eventType", event.Type.String())
+	}
+}
+
+func (a *analyticsService) SendNodeRoomStates(_ context.Context, nodeRooms *livekit.AnalyticsNodeRooms) {
+	if a.nodeRooms == nil {
+		return
+	}
+
+	nodeRooms.NodeId = a.nodeID
+	nodeRooms.SequenceNumber = a.sequenceNumber.Add(1)
+	nodeRooms.Timestamp = timestamppb.Now()
+	if err := a.nodeRooms.Send(nodeRooms); err != nil {
+		logger.Errorw("failed to send node room states", err)
+	}
+}

livekit-server/pkg/telemetry/events.go

@@ -0,0 +1,598 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package telemetry
+
+import (
+	"context"
+	"time"
+
+	"google.golang.org/protobuf/types/known/timestamppb"
+
+	"github.com/livekit/livekit-server/pkg/sfu/mime"
+	"github.com/livekit/livekit-server/pkg/telemetry/prometheus"
+	"github.com/livekit/protocol/livekit"
+	"github.com/livekit/protocol/logger"
+	"github.com/livekit/protocol/utils/guid"
+	"github.com/livekit/protocol/webhook"
+)
+
+func (t *telemetryService) NotifyEvent(ctx context.Context, event *livekit.WebhookEvent) {
+	if t.notifier == nil {
+		return
+	}
+
+	event.CreatedAt = time.Now().Unix()
+	event.Id = guid.New("EV_")
+
+	if err := t.notifier.QueueNotify(ctx, event); err != nil {
+		logger.Warnw("failed to notify webhook", err, "event", event.Event)
+	}
+}
+
+func (t *telemetryService) RoomStarted(ctx context.Context, room *livekit.Room) {
+	t.enqueue(func() {
+		t.NotifyEvent(ctx, &livekit.WebhookEvent{
+			Event: webhook.EventRoomStarted,
+			Room:  room,
+		})
+
+		t.SendEvent(ctx, &livekit.AnalyticsEvent{
+			Type:      livekit.AnalyticsEventType_ROOM_CREATED,
+			Timestamp: &timestamppb.Timestamp{Seconds: room.CreationTime},
+			Room:      room,
+		})
+	})
+}
+
+func (t *telemetryService) RoomEnded(ctx context.Context, room *livekit.Room) {
+	t.enqueue(func() {
+		t.NotifyEvent(ctx, &livekit.WebhookEvent{
+			Event: webhook.EventRoomFinished,
+			Room:  room,
+		})
+
+		t.SendEvent(ctx, &livekit.AnalyticsEvent{
+			Type:      livekit.AnalyticsEventType_ROOM_ENDED,
+			Timestamp: timestamppb.Now(),
+			RoomId:    room.Sid,
+			Room:      room,
+		})
+	})
+}
+
+func (t *telemetryService) ParticipantJoined(
+	ctx context.Context,
+	room *livekit.Room,
+	participant *livekit.ParticipantInfo,
+	clientInfo *livekit.ClientInfo,
+	clientMeta *livekit.AnalyticsClientMeta,
+	shouldSendEvent bool,
+) {
+	t.enqueue(func() {
+		_, found := t.getOrCreateWorker(
+			ctx,
+			livekit.RoomID(room.Sid),
+			livekit.RoomName(room.Name),
+			livekit.ParticipantID(participant.Sid),
+			livekit.ParticipantIdentity(participant.Identity),
+		)
+		if !found {
+			prometheus.IncrementParticipantRtcConnected(1)
+			prometheus.AddParticipant()
+		}
+
+		if shouldSendEvent {
+			ev := newParticipantEvent(livekit.AnalyticsEventType_PARTICIPANT_JOINED, room, participant)
+			ev.ClientInfo = clientInfo
+			ev.ClientMeta = clientMeta
+			t.SendEvent(ctx, ev)
+		}
+	})
+}
+
+func (t *telemetryService) ParticipantActive(
+	ctx context.Context,
+	room *livekit.Room,
+	participant *livekit.ParticipantInfo,
+	clientMeta *livekit.AnalyticsClientMeta,
+	isMigration bool,
+) {
+	t.enqueue(func() {
+		if !isMigration {
+			// consider participant joined only when they became active
+			t.NotifyEvent(ctx, &livekit.WebhookEvent{
+				Event:       webhook.EventParticipantJoined,
+				Room:        room,
+				Participant: participant,
+			})
+		}
+
+		worker, found := t.getOrCreateWorker(
+			ctx,
+			livekit.RoomID(room.Sid),
+			livekit.RoomName(room.Name),
+			livekit.ParticipantID(participant.Sid),
+			livekit.ParticipantIdentity(participant.Identity),
+		)
+		if !found {
+			// need to also account for participant count
+			prometheus.AddParticipant()
+		}
+		worker.SetConnected()
+
+		ev := newParticipantEvent(livekit.AnalyticsEventType_PARTICIPANT_ACTIVE, room, participant)
+		ev.ClientMeta = clientMeta
+		t.SendEvent(ctx, ev)
+	})
+}
+
+func (t *telemetryService) ParticipantResumed(
+	ctx context.Context,
+	room *livekit.Room,
+	participant *livekit.ParticipantInfo,
+	nodeID livekit.NodeID,
+	reason livekit.ReconnectReason,
+) {
+	t.enqueue(func() {
+		// create a worker if needed.
+		//
+		// Signalling channel stats collector and media channel stats collector could both call
+		// ParticipantJoined and ParticipantLeft.
+		//
+		// On a resume, the signalling channel collector would call `ParticipantLeft` which would close
+		// the corresponding participant's stats worker.
+		//
+		// So, on a successful resume, create the worker if needed.
+		_, found := t.getOrCreateWorker(
+			ctx,
+			livekit.RoomID(room.Sid),
+			livekit.RoomName(room.Name),
+			livekit.ParticipantID(participant.Sid),
+			livekit.ParticipantIdentity(participant.Identity),
+		)
+		if !found {
+			prometheus.AddParticipant()
+		}
+
+		ev := newParticipantEvent(livekit.AnalyticsEventType_PARTICIPANT_RESUMED, room, participant)
+		ev.ClientMeta = &livekit.AnalyticsClientMeta{
+			Node:            string(nodeID),
+			ReconnectReason: reason,
+		}
+		t.SendEvent(ctx, ev)
+	})
+}
+
+func (t *telemetryService) ParticipantLeft(ctx context.Context,
+	room *livekit.Room,
+	participant *livekit.ParticipantInfo,
+	shouldSendEvent bool,
+) {
+	t.enqueue(func() {
+		isConnected := false
+		if worker, ok := t.getWorker(livekit.ParticipantID(participant.Sid)); ok {
+			isConnected = worker.IsConnected()
+			if worker.Close() {
+				prometheus.SubParticipant()
+			}
+		}
+
+		if isConnected && shouldSendEvent {
+			t.NotifyEvent(ctx, &livekit.WebhookEvent{
+				Event:       webhook.EventParticipantLeft,
+				Room:        room,
+				Participant: participant,
+			})
+
+			t.SendEvent(ctx, newParticipantEvent(livekit.AnalyticsEventType_PARTICIPANT_LEFT, room, participant))
+		}
+	})
+}
+
+func (t *telemetryService) TrackPublishRequested(
+	ctx context.Context,
+	participantID livekit.ParticipantID,
+	identity livekit.ParticipantIdentity,
+	track *livekit.TrackInfo,
+) {
+	t.enqueue(func() {
+		prometheus.AddPublishAttempt(track.Type.String())
+		room := t.getRoomDetails(participantID)
+		ev := newTrackEvent(livekit.AnalyticsEventType_TRACK_PUBLISH_REQUESTED, room, participantID, track)
+		if ev.Participant != nil {
+			ev.Participant.Identity = string(identity)
+		}
+		t.SendEvent(ctx, ev)
+	})
+}
+
+func (t *telemetryService) TrackPublished(
+	ctx context.Context,
+	participantID livekit.ParticipantID,
+	identity livekit.ParticipantIdentity,
+	track *livekit.TrackInfo,
+) {
+	t.enqueue(func() {
+		prometheus.AddPublishedTrack(track.Type.String())
+		prometheus.AddPublishSuccess(track.Type.String())
+
+		room := t.getRoomDetails(participantID)
+		participant := &livekit.ParticipantInfo{
+			Sid:      string(participantID),
+			Identity: string(identity),
+		}
+		t.NotifyEvent(ctx, &livekit.WebhookEvent{
+			Event:       webhook.EventTrackPublished,
+			Room:        room,
+			Participant: participant,
+			Track:       track,
+		})
+
+		ev := newTrackEvent(livekit.AnalyticsEventType_TRACK_PUBLISHED, room, participantID, track)
+		ev.Participant = participant
+		t.SendEvent(ctx, ev)
+	})
+}
+
+func (t *telemetryService) TrackPublishedUpdate(ctx context.Context, participantID livekit.ParticipantID, track *livekit.TrackInfo) {
+	t.enqueue(func() {
+		room := t.getRoomDetails(participantID)
+		t.SendEvent(ctx, newTrackEvent(livekit.AnalyticsEventType_TRACK_PUBLISHED_UPDATE, room, participantID, track))
+	})
+}
+
+func (t *telemetryService) TrackMaxSubscribedVideoQuality(
+	ctx context.Context,
+	participantID livekit.ParticipantID,
+	track *livekit.TrackInfo,
+	mime mime.MimeType,
+	maxQuality livekit.VideoQuality,
+) {
+	t.enqueue(func() {
+		room := t.getRoomDetails(participantID)
+		ev := newTrackEvent(livekit.AnalyticsEventType_TRACK_MAX_SUBSCRIBED_VIDEO_QUALITY, room, participantID, track)
+		ev.MaxSubscribedVideoQuality = maxQuality
+		ev.Mime = mime.String()
+		t.SendEvent(ctx, ev)
+	})
+}
+
+func (t *telemetryService) TrackSubscribeRequested(
+	ctx context.Context,
+	participantID livekit.ParticipantID,
+	track *livekit.TrackInfo,
+) {
+	t.enqueue(func() {
+		prometheus.RecordTrackSubscribeAttempt()
+
+		room := t.getRoomDetails(participantID)
+		ev := newTrackEvent(livekit.AnalyticsEventType_TRACK_SUBSCRIBE_REQUESTED, room, participantID, track)
+		t.SendEvent(ctx, ev)
+	})
+}
+
+func (t *telemetryService) TrackSubscribed(
+	ctx context.Context,
+	participantID livekit.ParticipantID,
+	track *livekit.TrackInfo,
+	publisher *livekit.ParticipantInfo,
+	shouldSendEvent bool,
+) {
+	t.enqueue(func() {
+		prometheus.RecordTrackSubscribeSuccess(track.Type.String())
+
+		if !shouldSendEvent {
+			return
+		}
+
+		room := t.getRoomDetails(participantID)
+		ev := newTrackEvent(livekit.AnalyticsEventType_TRACK_SUBSCRIBED, room, participantID, track)
+		ev.Publisher = publisher
+		t.SendEvent(ctx, ev)
+	})
+}
+
+func (t *telemetryService) TrackSubscribeFailed(
+	ctx context.Context,
+	participantID livekit.ParticipantID,
+	trackID livekit.TrackID,
+	err error,
+	isUserError bool,
+) {
+	t.enqueue(func() {
+		prometheus.RecordTrackSubscribeFailure(err, isUserError)
+
+		room := t.getRoomDetails(participantID)
+		ev := newTrackEvent(livekit.AnalyticsEventType_TRACK_SUBSCRIBE_FAILED, room, participantID, &livekit.TrackInfo{
+			Sid: string(trackID),
+		})
+		ev.Error = err.Error()
+		t.SendEvent(ctx, ev)
+	})
+}
+
+func (t *telemetryService) TrackUnsubscribed(
+	ctx context.Context,
+	participantID livekit.ParticipantID,
+	track *livekit.TrackInfo,
+	shouldSendEvent bool,
+) {
+	t.enqueue(func() {
+		prometheus.RecordTrackUnsubscribed(track.Type.String())
+
+		if shouldSendEvent {
+			room := t.getRoomDetails(participantID)
+			t.SendEvent(ctx, newTrackEvent(livekit.AnalyticsEventType_TRACK_UNSUBSCRIBED, room, participantID, track))
+		}
+	})
+}
+
+func (t *telemetryService) TrackUnpublished(
+	ctx context.Context,
+	participantID livekit.ParticipantID,
+	identity livekit.ParticipantIdentity,
+	track *livekit.TrackInfo,
+	shouldSendEvent bool,
+) {
+	t.enqueue(func() {
+		prometheus.SubPublishedTrack(track.Type.String())
+		if !shouldSendEvent {
+			return
+		}
+
+		room := t.getRoomDetails(participantID)
+		participant := &livekit.ParticipantInfo{
+			Sid:      string(participantID),
+			Identity: string(identity),
+		}
+		t.NotifyEvent(ctx, &livekit.WebhookEvent{
+			Event:       webhook.EventTrackUnpublished,
+			Room:        room,
+			Participant: participant,
+			Track:       track,
+		})
+
+		t.SendEvent(ctx, newTrackEvent(livekit.AnalyticsEventType_TRACK_UNPUBLISHED, room, participantID, track))
+	})
+}
+
+func (t *telemetryService) TrackMuted(
+	ctx context.Context,
+	participantID livekit.ParticipantID,
+	track *livekit.TrackInfo,
+) {
+	t.enqueue(func() {
+		room := t.getRoomDetails(participantID)
+		t.SendEvent(ctx, newTrackEvent(livekit.AnalyticsEventType_TRACK_MUTED, room, participantID, track))
+	})
+}
+
+func (t *telemetryService) TrackUnmuted(
+	ctx context.Context,
+	participantID livekit.ParticipantID,
+	track *livekit.TrackInfo,
+) {
+	t.enqueue(func() {
+		room := t.getRoomDetails(participantID)
+		t.SendEvent(ctx, newTrackEvent(livekit.AnalyticsEventType_TRACK_UNMUTED, room, participantID, track))
+	})
+}
+
+func (t *telemetryService) TrackPublishRTPStats(
+	ctx context.Context,
+	participantID livekit.ParticipantID,
+	trackID livekit.TrackID,
+	mimeType mime.MimeType,
+	layer int,
+	stats *livekit.RTPStats,
+) {
+	t.enqueue(func() {
+		room := t.getRoomDetails(participantID)
+		ev := newRoomEvent(livekit.AnalyticsEventType_TRACK_PUBLISH_STATS, room)
+		ev.ParticipantId = string(participantID)
+		ev.TrackId = string(trackID)
+		ev.Mime = mimeType.String()
+		ev.VideoLayer = int32(layer)
+		ev.RtpStats = stats
+		t.SendEvent(ctx, ev)
+	})
+}
+
+func (t *telemetryService) TrackSubscribeRTPStats(
+	ctx context.Context,
+	participantID livekit.ParticipantID,
+	trackID livekit.TrackID,
+	mimeType mime.MimeType,
+	stats *livekit.RTPStats,
+) {
+	t.enqueue(func() {
+		room := t.getRoomDetails(participantID)
+		ev := newRoomEvent(livekit.AnalyticsEventType_TRACK_SUBSCRIBE_STATS, room)
+		ev.ParticipantId = string(participantID)
+		ev.TrackId = string(trackID)
+		ev.Mime = mimeType.String()
+		ev.RtpStats = stats
+		t.SendEvent(ctx, ev)
+	})
+}
+
+func (t *telemetryService) EgressStarted(ctx context.Context, info *livekit.EgressInfo) {
+	t.enqueue(func() {
+		t.NotifyEvent(ctx, &livekit.WebhookEvent{
+			Event:      webhook.EventEgressStarted,
+			EgressInfo: info,
+		})
+
+		t.SendEvent(ctx, newEgressEvent(livekit.AnalyticsEventType_EGRESS_STARTED, info))
+	})
+}
+
+func (t *telemetryService) EgressUpdated(ctx context.Context, info *livekit.EgressInfo) {
+	t.enqueue(func() {
+		t.NotifyEvent(ctx, &livekit.WebhookEvent{
+			Event:      webhook.EventEgressUpdated,
+			EgressInfo: info,
+		})
+		t.SendEvent(ctx, newEgressEvent(livekit.AnalyticsEventType_EGRESS_UPDATED, info))
+	})
+}
+
+func (t *telemetryService) EgressEnded(ctx context.Context, info *livekit.EgressInfo) {
+	t.enqueue(func() {
+		t.NotifyEvent(ctx, &livekit.WebhookEvent{
+			Event:      webhook.EventEgressEnded,
+			EgressInfo: info,
+		})
+
+		t.SendEvent(ctx, newEgressEvent(livekit.AnalyticsEventType_EGRESS_ENDED, info))
+	})
+}
+
+func (t *telemetryService) IngressCreated(ctx context.Context, info *livekit.IngressInfo) {
+	t.enqueue(func() {
+		t.SendEvent(ctx, newIngressEvent(livekit.AnalyticsEventType_INGRESS_CREATED, info))
+	})
+}
+
+func (t *telemetryService) IngressDeleted(ctx context.Context, info *livekit.IngressInfo) {
+	t.enqueue(func() {
+		t.SendEvent(ctx, newIngressEvent(livekit.AnalyticsEventType_INGRESS_DELETED, info))
+	})
+}
+
+func (t *telemetryService) IngressStarted(ctx context.Context, info *livekit.IngressInfo) {
+	t.enqueue(func() {
+		t.NotifyEvent(ctx, &livekit.WebhookEvent{
+			Event:       webhook.EventIngressStarted,
+			IngressInfo: info,
+		})
+
+		t.SendEvent(ctx, newIngressEvent(livekit.AnalyticsEventType_INGRESS_STARTED, info))
+	})
+}
+
+func (t *telemetryService) IngressUpdated(ctx context.Context, info *livekit.IngressInfo) {
+	t.enqueue(func() {
+		t.SendEvent(ctx, newIngressEvent(livekit.AnalyticsEventType_INGRESS_UPDATED, info))
+	})
+}
+
+func (t *telemetryService) IngressEnded(ctx context.Context, info *livekit.IngressInfo) {
+	t.enqueue(func() {
+		t.NotifyEvent(ctx, &livekit.WebhookEvent{
+			Event:       webhook.EventIngressEnded,
+			IngressInfo: info,
+		})
+
+		t.SendEvent(ctx, newIngressEvent(livekit.AnalyticsEventType_INGRESS_ENDED, info))
+	})
+}
+
+func (t *telemetryService) Report(ctx context.Context, reportInfo *livekit.ReportInfo) {
+	t.enqueue(func() {
+		ev := &livekit.AnalyticsEvent{
+			Type:      livekit.AnalyticsEventType_REPORT,
+			Timestamp: timestamppb.Now(),
+			Report:    reportInfo,
+		}
+		t.SendEvent(ctx, ev)
+	})
+}
+
+func (t *telemetryService) APICall(ctx context.Context, apiCallInfo *livekit.APICallInfo) {
+	t.enqueue(func() {
+		ev := &livekit.AnalyticsEvent{
+			Type:      livekit.AnalyticsEventType_API_CALL,
+			Timestamp: timestamppb.Now(),
+			ApiCall:   apiCallInfo,
+		}
+		t.SendEvent(ctx, ev)
+	})
+}
+
+func (t *telemetryService) Webhook(ctx context.Context, webhookInfo *livekit.WebhookInfo) {
+	t.enqueue(func() {
+		ev := &livekit.AnalyticsEvent{
+			Type:      livekit.AnalyticsEventType_WEBHOOK,
+			Timestamp: timestamppb.Now(),
+			Webhook:   webhookInfo,
+		}
+		t.SendEvent(ctx, ev)
+	})
+}
+
+// returns a livekit.Room with only name and sid filled out
+// returns nil if room is not found
+func (t *telemetryService) getRoomDetails(participantID livekit.ParticipantID) *livekit.Room {
+	if worker, ok := t.getWorker(participantID); ok {
+		return &livekit.Room{
+			Sid:  string(worker.roomID),
+			Name: string(worker.roomName),
+		}
+	}
+
+	return nil
+}
+
+func newRoomEvent(event livekit.AnalyticsEventType, room *livekit.Room) *livekit.AnalyticsEvent {
+	ev := &livekit.AnalyticsEvent{
+		Type:      event,
+		Timestamp: timestamppb.Now(),
+	}
+	if room != nil {
+		ev.Room = room
+		ev.RoomId = room.Sid
+	}
+	return ev
+}
+
+func newParticipantEvent(event livekit.AnalyticsEventType, room *livekit.Room, participant *livekit.ParticipantInfo) *livekit.AnalyticsEvent {
+	ev := newRoomEvent(event, room)
+	if participant != nil {
+		ev.ParticipantId = participant.Sid
+		ev.Participant = participant
+	}
+	return ev
+}
+
+func newTrackEvent(event livekit.AnalyticsEventType, room *livekit.Room, participantID livekit.ParticipantID, track *livekit.TrackInfo) *livekit.AnalyticsEvent {
+	ev := newParticipantEvent(event, room, &livekit.ParticipantInfo{
+		Sid: string(participantID),
+	})
+	if track != nil {
+		ev.TrackId = track.Sid
+		ev.Track = track
+	}
+	return ev
+}
+
+func newEgressEvent(event livekit.AnalyticsEventType, egress *livekit.EgressInfo) *livekit.AnalyticsEvent {
+	return &livekit.AnalyticsEvent{
+		Type:      event,
+		Timestamp: timestamppb.Now(),
+		EgressId:  egress.EgressId,
+		RoomId:    egress.RoomId,
+		Egress:    egress,
+	}
+}
+
+func newIngressEvent(event livekit.AnalyticsEventType, ingress *livekit.IngressInfo) *livekit.AnalyticsEvent {
+	return &livekit.AnalyticsEvent{
+		Type:      event,
+		Timestamp: timestamppb.Now(),
+		IngressId: ingress.IngressId,
+		Ingress:   ingress,
+	}
+}

livekit-server/pkg/telemetry/events_test.go

@@ -0,0 +1,238 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package telemetry_test
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/livekit/protocol/livekit"
+)
+
+func Test_OnParticipantJoin_EventIsSent(t *testing.T) {
+	fixture := createFixture()
+
+	// prepare
+	room := &livekit.Room{Sid: "RoomSid", Name: "RoomName"}
+	partSID := "part1"
+	clientInfo := &livekit.ClientInfo{
+		Sdk:            2,
+		Version:        "v1",
+		Os:             "mac",
+		OsVersion:      "v1",
+		DeviceModel:    "DM1",
+		Browser:        "chrome",
+		BrowserVersion: "97.0.1",
+	}
+	clientMeta := &livekit.AnalyticsClientMeta{
+		Region:            "dark-side",
+		Node:              "moon",
+		ClientAddr:        "127.0.0.1",
+		ClientConnectTime: 420,
+	}
+	participantInfo := &livekit.ParticipantInfo{Sid: partSID}
+
+	// do
+	fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, clientInfo, clientMeta, true)
+	time.Sleep(time.Millisecond * 500)
+
+	// test
+	require.Equal(t, 1, fixture.analytics.SendEventCallCount())
+	_, event := fixture.analytics.SendEventArgsForCall(0)
+	require.Equal(t, livekit.AnalyticsEventType_PARTICIPANT_JOINED, event.Type)
+	require.Equal(t, partSID, event.ParticipantId)
+	require.Equal(t, participantInfo, event.Participant)
+	require.Equal(t, room.Sid, event.RoomId)
+	require.Equal(t, room, event.Room)
+
+	require.Equal(t, clientInfo.Sdk, event.ClientInfo.Sdk)
+	require.Equal(t, clientInfo.Version, event.ClientInfo.Version)
+	require.Equal(t, clientInfo.Os, event.ClientInfo.Os)
+	require.Equal(t, clientInfo.OsVersion, event.ClientInfo.OsVersion)
+	require.Equal(t, clientInfo.DeviceModel, event.ClientInfo.DeviceModel)
+	require.Equal(t, clientInfo.Browser, event.ClientInfo.Browser)
+	require.Equal(t, clientInfo.BrowserVersion, event.ClientInfo.BrowserVersion)
+
+	require.Equal(t, clientMeta.Region, event.ClientMeta.Region)
+	require.Equal(t, clientMeta.Node, event.ClientMeta.Node)
+	require.Equal(t, clientMeta.ClientAddr, event.ClientMeta.ClientAddr)
+	require.Equal(t, clientMeta.ClientConnectTime, event.ClientMeta.ClientConnectTime)
+}
+
+func Test_OnParticipantLeft_EventIsSent(t *testing.T) {
+	fixture := createFixture()
+
+	// prepare
+	room := &livekit.Room{Sid: "RoomSid", Name: "RoomName"}
+	partSID := "part1"
+	participantInfo := &livekit.ParticipantInfo{Sid: partSID}
+
+	// do
+	fixture.sut.ParticipantActive(context.Background(), room, participantInfo, &livekit.AnalyticsClientMeta{}, false)
+	fixture.sut.ParticipantLeft(context.Background(), room, participantInfo, true)
+	time.Sleep(time.Millisecond * 500)
+
+	// test
+	require.Equal(t, 2, fixture.analytics.SendEventCallCount())
+	_, event := fixture.analytics.SendEventArgsForCall(1)
+	require.Equal(t, livekit.AnalyticsEventType_PARTICIPANT_LEFT, event.Type)
+	require.Equal(t, partSID, event.ParticipantId)
+	require.Equal(t, room.Sid, event.RoomId)
+	require.Equal(t, room, event.Room)
+}
+
+func Test_OnTrackUpdate_EventIsSent(t *testing.T) {
+	fixture := createFixture()
+
+	// prepare
+	partID := "part1"
+	trackID := "track1"
+	layer := &livekit.VideoLayer{
+		Quality: livekit.VideoQuality_HIGH,
+		Width:   uint32(360),
+		Height:  uint32(720),
+		Bitrate: 2048,
+	}
+
+	trackInfo := &livekit.TrackInfo{
+		Sid:        trackID,
+		Type:       livekit.TrackType_VIDEO,
+		Muted:      false,
+		Simulcast:  false,
+		DisableDtx: false,
+		Layers:     []*livekit.VideoLayer{layer},
+	}
+
+	// do
+	fixture.sut.TrackPublishedUpdate(context.Background(), livekit.ParticipantID(partID), trackInfo)
+	time.Sleep(time.Millisecond * 500)
+
+	// test
+	require.Equal(t, 1, fixture.analytics.SendEventCallCount())
+	_, event := fixture.analytics.SendEventArgsForCall(0)
+	require.Equal(t, livekit.AnalyticsEventType_TRACK_PUBLISHED_UPDATE, event.Type)
+	require.Equal(t, partID, event.ParticipantId)
+
+	require.Equal(t, trackID, event.Track.Sid)
+	require.NotNil(t, event.Track.Layers)
+	require.Equal(t, layer.Width, event.Track.Layers[0].Width)
+	require.Equal(t, layer.Height, event.Track.Layers[0].Height)
+	require.Equal(t, layer.Quality, event.Track.Layers[0].Quality)
+
+}
+
+func Test_OnParticipantActive_EventIsSent(t *testing.T) {
+	fixture := createFixture()
+
+	// prepare participant to change status
+	room := &livekit.Room{Sid: "RoomSid", Name: "RoomName"}
+	partSID := "part1"
+
+	clientInfo := &livekit.ClientInfo{
+		Sdk:            2,
+		Version:        "v1",
+		Os:             "mac",
+		OsVersion:      "v1",
+		DeviceModel:    "DM1",
+		Browser:        "chrome",
+		BrowserVersion: "97.0.1",
+	}
+	clientMeta := &livekit.AnalyticsClientMeta{
+		Region:     "dark-side",
+		Node:       "moon",
+		ClientAddr: "127.0.0.1",
+	}
+	participantInfo := &livekit.ParticipantInfo{Sid: partSID}
+
+	// do
+	fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, clientInfo, clientMeta, true)
+	time.Sleep(time.Millisecond * 500)
+
+	// test
+	require.Equal(t, 1, fixture.analytics.SendEventCallCount())
+	_, event := fixture.analytics.SendEventArgsForCall(0)
+
+	// test
+	// do
+	clientMetaConnect := &livekit.AnalyticsClientMeta{
+		ClientConnectTime: 420,
+	}
+
+	fixture.sut.ParticipantActive(context.Background(), room, participantInfo, clientMetaConnect, false)
+	time.Sleep(time.Millisecond * 500)
+
+	require.Equal(t, 2, fixture.analytics.SendEventCallCount())
+	_, eventActive := fixture.analytics.SendEventArgsForCall(1)
+	require.Equal(t, livekit.AnalyticsEventType_PARTICIPANT_ACTIVE, eventActive.Type)
+	require.Equal(t, partSID, eventActive.ParticipantId)
+	require.Equal(t, room.Sid, eventActive.RoomId)
+	require.Equal(t, room, event.Room)
+
+	require.Equal(t, clientMetaConnect.ClientConnectTime, eventActive.ClientMeta.ClientConnectTime)
+}
+
+func Test_OnTrackSubscribed_EventIsSent(t *testing.T) {
+	fixture := createFixture()
+
+	// prepare participant to change status
+	room := &livekit.Room{Sid: "RoomSid", Name: "RoomName"}
+	partSID := "part1"
+	publisherInfo := &livekit.ParticipantInfo{Sid: "pub1", Identity: "publisher"}
+	trackInfo := &livekit.TrackInfo{Sid: "tr1", Type: livekit.TrackType_VIDEO}
+
+	clientInfo := &livekit.ClientInfo{
+		Sdk:            2,
+		Version:        "v1",
+		Os:             "mac",
+		OsVersion:      "v1",
+		DeviceModel:    "DM1",
+		Browser:        "chrome",
+		BrowserVersion: "97.0.1",
+	}
+	clientMeta := &livekit.AnalyticsClientMeta{
+		Region:     "dark-side",
+		Node:       "moon",
+		ClientAddr: "127.0.0.1",
+	}
+	participantInfo := &livekit.ParticipantInfo{Sid: partSID}
+
+	// do
+	fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, clientInfo, clientMeta, true)
+	time.Sleep(time.Millisecond * 500)
+
+	// test
+	require.Equal(t, 1, fixture.analytics.SendEventCallCount())
+	_, event := fixture.analytics.SendEventArgsForCall(0)
+	require.Equal(t, room, event.Room)
+
+	// do
+	fixture.sut.TrackSubscribed(context.Background(), livekit.ParticipantID(partSID), trackInfo, publisherInfo, true)
+	time.Sleep(time.Millisecond * 500)
+
+	require.Eventually(t, func() bool {
+		return fixture.analytics.SendEventCallCount() == 2
+	}, time.Second, time.Millisecond*50, "expected send event to be called twice")
+	_, eventTrackSubscribed := fixture.analytics.SendEventArgsForCall(1)
+	require.Equal(t, livekit.AnalyticsEventType_TRACK_SUBSCRIBED, eventTrackSubscribed.Type)
+	require.Equal(t, partSID, eventTrackSubscribed.ParticipantId)
+	require.Equal(t, trackInfo.Sid, eventTrackSubscribed.Track.Sid)
+	require.Equal(t, trackInfo.Type, eventTrackSubscribed.Track.Type)
+	require.Equal(t, publisherInfo.Sid, eventTrackSubscribed.Publisher.Sid)
+	require.Equal(t, publisherInfo.Identity, eventTrackSubscribed.Publisher.Identity)
+
+}

livekit-server/pkg/telemetry/prometheus/datapacket.go

@@ -0,0 +1,75 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package prometheus
+
+import (
+	"slices"
+	"strings"
+
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/livekit/protocol/livekit"
+)
+
+var (
+	promDataPacketStreamLabels    = []string{"type", "mime_type"}
+	promDataPacketStreamMimeTypes = []string{"text", "image", "application", "audio", "video"}
+
+	promDataPacketStreamDestCount *prometheus.HistogramVec
+	promDataPacketStreamSize      *prometheus.HistogramVec
+)
+
+func initDataPacketStats(nodeID string, nodeType livekit.NodeType) {
+	promDataPacketStreamDestCount = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Namespace:   livekitNamespace,
+		Subsystem:   "datapacket_stream",
+		Name:        "dest_count",
+		ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
+		Buckets:     []float64{1, 2, 3, 4, 5, 10, 15, 25, 50},
+	}, promDataPacketStreamLabels)
+	promDataPacketStreamSize = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Namespace:   livekitNamespace,
+		Subsystem:   "datapacket_stream",
+		Name:        "bytes",
+		ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
+		Buckets:     []float64{128, 512, 2048, 8192, 32768, 131072, 524288, 2097152, 8388608, 33554432},
+	}, promDataPacketStreamLabels)
+
+	prometheus.MustRegister(promDataPacketStreamDestCount)
+	prometheus.MustRegister(promDataPacketStreamSize)
+}
+
+func RecordDataPacketStream(h *livekit.DataStream_Header, destCount int) {
+	streamType := "unknown"
+	switch h.ContentHeader.(type) {
+	case *livekit.DataStream_Header_TextHeader:
+		streamType = "text"
+	case *livekit.DataStream_Header_ByteHeader:
+		streamType = "bytes"
+	}
+
+	mimeType := strings.ToLower(h.MimeType)
+	if i := strings.IndexByte(mimeType, '/'); i != -1 {
+		mimeType = mimeType[:i]
+	}
+	if !slices.Contains(promDataPacketStreamMimeTypes, mimeType) {
+		mimeType = "unknown"
+	}
+
+	promDataPacketStreamDestCount.WithLabelValues(streamType, mimeType).Observe(float64(destCount))
+	if h.TotalLength != nil {
+		promDataPacketStreamSize.WithLabelValues(streamType, mimeType).Observe(float64(*h.TotalLength))
+	}
+}

livekit-server/pkg/telemetry/prometheus/node.go

@@ -0,0 +1,279 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package prometheus
+
+import (
+	"time"
+
+	"github.com/mackerelio/go-osstat/memory"
+	"github.com/prometheus/client_golang/prometheus"
+	"go.uber.org/atomic"
+
+	"github.com/livekit/protocol/livekit"
+	"github.com/livekit/protocol/rpc"
+	"github.com/livekit/protocol/utils/hwstats"
+)
+
+const (
+	livekitNamespace string = "livekit"
+
+	statsUpdateInterval = time.Second * 10
+)
+
+var (
+	initialized atomic.Bool
+
+	MessageCounter            *prometheus.CounterVec
+	MessageBytes              *prometheus.CounterVec
+	ServiceOperationCounter   *prometheus.CounterVec
+	TwirpRequestStatusCounter *prometheus.CounterVec
+
+	sysPacketsStart              uint32
+	sysDroppedPacketsStart       uint32
+	promSysPacketGauge           *prometheus.GaugeVec
+	promSysDroppedPacketPctGauge prometheus.Gauge
+
+	cpuStats *hwstats.CPUStats
+)
+
+func Init(nodeID string, nodeType livekit.NodeType) error {
+	if initialized.Swap(true) {
+		return nil
+	}
+
+	MessageCounter = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace:   livekitNamespace,
+			Subsystem:   "node",
+			Name:        "messages",
+			ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
+		},
+		[]string{"type", "status"},
+	)
+
+	MessageBytes = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace:   livekitNamespace,
+			Subsystem:   "node",
+			Name:        "message_bytes",
+			ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
+		},
+		[]string{"type", "message_type"},
+	)
+
+	ServiceOperationCounter = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace:   livekitNamespace,
+			Subsystem:   "node",
+			Name:        "service_operation",
+			ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
+		},
+		[]string{"type", "status", "error_type"},
+	)
+
+	TwirpRequestStatusCounter = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace:   livekitNamespace,
+			Subsystem:   "node",
+			Name:        "twirp_request_status",
+			ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
+		},
+		[]string{"service", "method", "status", "code"},
+	)
+
+	promSysPacketGauge = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace:   livekitNamespace,
+			Subsystem:   "node",
+			Name:        "packet_total",
+			ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
+			Help:        "System level packet count. Count starts at 0 when service is first started.",
+		},
+		[]string{"type"},
+	)
+
+	promSysDroppedPacketPctGauge = prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Namespace:   livekitNamespace,
+			Subsystem:   "node",
+			Name:        "dropped_packets",
+			ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
+			Help:        "System level dropped outgoing packet percentage.",
+		},
+	)
+
+	prometheus.MustRegister(MessageCounter)
+	prometheus.MustRegister(MessageBytes)
+	prometheus.MustRegister(ServiceOperationCounter)
+	prometheus.MustRegister(TwirpRequestStatusCounter)
+	prometheus.MustRegister(promSysPacketGauge)
+	prometheus.MustRegister(promSysDroppedPacketPctGauge)
+
+	sysPacketsStart, sysDroppedPacketsStart, _ = getTCStats()
+
+	initPacketStats(nodeID, nodeType)
+	initRoomStats(nodeID, nodeType)
+	rpc.InitPSRPCStats(prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()})
+	initQualityStats(nodeID, nodeType)
+	initDataPacketStats(nodeID, nodeType)
+
+	var err error
+	cpuStats, err = hwstats.NewCPUStats(nil)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func GetUpdatedNodeStats(prev *livekit.NodeStats, prevAverage *livekit.NodeStats) (*livekit.NodeStats, bool, error) {
+	loadAvg, err := getLoadAvg()
+	if err != nil {
+		return nil, false, err
+	}
+
+	var cpuLoad float64
+	cpuIdle := cpuStats.GetCPUIdle()
+	if cpuIdle > 0 {
+		cpuLoad = 1 - (cpuIdle / cpuStats.NumCPU())
+	}
+
+	// On MacOS, get "\"vm_stat\": executable file not found in $PATH" although it is in /usr/bin
+	// So, do not error out. Use the information if it is available.
+	memTotal := uint64(0)
+	memUsed := uint64(0)
+	memInfo, _ := memory.Get()
+	if memInfo != nil {
+		memTotal = memInfo.Total
+		memUsed = memInfo.Used
+	}
+
+	// do not error out, and use the information if it is available
+	sysPackets, sysDroppedPackets, _ := getTCStats()
+	promSysPacketGauge.WithLabelValues("out").Set(float64(sysPackets - sysPacketsStart))
+	promSysPacketGauge.WithLabelValues("dropped").Set(float64(sysDroppedPackets - sysDroppedPacketsStart))
+
+	bytesInNow := bytesIn.Load()
+	bytesOutNow := bytesOut.Load()
+	packetsInNow := packetsIn.Load()
+	packetsOutNow := packetsOut.Load()
+	nackTotalNow := nackTotal.Load()
+	retransmitBytesNow := retransmitBytes.Load()
+	retransmitPacketsNow := retransmitPackets.Load()
+	participantSignalConnectedNow := participantSignalConnected.Load()
+	participantRTCInitNow := participantRTCInit.Load()
+	participantRTConnectedCNow := participantRTCConnected.Load()
+	trackPublishAttemptsNow := trackPublishAttempts.Load()
+	trackPublishSuccessNow := trackPublishSuccess.Load()
+	trackSubscribeAttemptsNow := trackSubscribeAttempts.Load()
+	trackSubscribeSuccessNow := trackSubscribeSuccess.Load()
+	forwardLatencyNow := forwardLatency.Load()
+	forwardJitterNow := forwardJitter.Load()
+
+	updatedAt := time.Now().Unix()
+	elapsed := updatedAt - prevAverage.UpdatedAt
+	// include sufficient buffer to be sure a stats update had taken place
+	computeAverage := elapsed > int64(statsUpdateInterval.Seconds()+2)
+	if bytesInNow != prevAverage.BytesIn ||
+		bytesOutNow != prevAverage.BytesOut ||
+		packetsInNow != prevAverage.PacketsIn ||
+		packetsOutNow != prevAverage.PacketsOut ||
+		retransmitBytesNow != prevAverage.RetransmitBytesOut ||
+		retransmitPacketsNow != prevAverage.RetransmitPacketsOut {
+		computeAverage = true
+	}
+
+	stats := &livekit.NodeStats{
+		StartedAt:                        prev.StartedAt,
+		UpdatedAt:                        updatedAt,
+		NumRooms:                         roomCurrent.Load(),
+		NumClients:                       participantCurrent.Load(),
+		NumTracksIn:                      trackPublishedCurrent.Load(),
+		NumTracksOut:                     trackSubscribedCurrent.Load(),
+		NumTrackPublishAttempts:          trackPublishAttemptsNow,
+		NumTrackPublishSuccess:           trackPublishSuccessNow,
+		NumTrackSubscribeAttempts:        trackSubscribeAttemptsNow,
+		NumTrackSubscribeSuccess:         trackSubscribeSuccessNow,
+		BytesIn:                          bytesInNow,
+		BytesOut:                         bytesOutNow,
+		PacketsIn:                        packetsInNow,
+		PacketsOut:                       packetsOutNow,
+		RetransmitBytesOut:               retransmitBytesNow,
+		RetransmitPacketsOut:             retransmitPacketsNow,
+		NackTotal:                        nackTotalNow,
+		ParticipantSignalConnected:       participantSignalConnectedNow,
+		ParticipantRtcInit:               participantRTCInitNow,
+		ParticipantRtcConnected:          participantRTConnectedCNow,
+		BytesInPerSec:                    prevAverage.BytesInPerSec,
+		BytesOutPerSec:                   prevAverage.BytesOutPerSec,
+		PacketsInPerSec:                  prevAverage.PacketsInPerSec,
+		PacketsOutPerSec:                 prevAverage.PacketsOutPerSec,
+		RetransmitBytesOutPerSec:         prevAverage.RetransmitBytesOutPerSec,
+		RetransmitPacketsOutPerSec:       prevAverage.RetransmitPacketsOutPerSec,
+		NackPerSec:                       prevAverage.NackPerSec,
+		ForwardLatency:                   forwardLatencyNow,
+		ForwardJitter:                    forwardJitterNow,
+		ParticipantSignalConnectedPerSec: prevAverage.ParticipantSignalConnectedPerSec,
+		ParticipantRtcInitPerSec:         prevAverage.ParticipantRtcInitPerSec,
+		ParticipantRtcConnectedPerSec:    prevAverage.ParticipantRtcConnectedPerSec,
+		NumCpus:                          uint32(cpuStats.NumCPU()), // this will round down to the nearest integer
+		CpuLoad:                          float32(cpuLoad),
+		MemoryTotal:                      memTotal,
+		MemoryUsed:                       memUsed,
+		LoadAvgLast1Min:                  float32(loadAvg.Loadavg1),
+		LoadAvgLast5Min:                  float32(loadAvg.Loadavg5),
+		LoadAvgLast15Min:                 float32(loadAvg.Loadavg15),
+		SysPacketsOut:                    sysPackets,
+		SysPacketsDropped:                sysDroppedPackets,
+		TrackPublishAttemptsPerSec:       prevAverage.TrackPublishAttemptsPerSec,
+		TrackPublishSuccessPerSec:        prevAverage.TrackPublishSuccessPerSec,
+		TrackSubscribeAttemptsPerSec:     prevAverage.TrackSubscribeAttemptsPerSec,
+		TrackSubscribeSuccessPerSec:      prevAverage.TrackSubscribeSuccessPerSec,
+	}
+
+	// update stats
+	if computeAverage {
+		stats.BytesInPerSec = perSec(prevAverage.BytesIn, bytesInNow, elapsed)
+		stats.BytesOutPerSec = perSec(prevAverage.BytesOut, bytesOutNow, elapsed)
+		stats.PacketsInPerSec = perSec(prevAverage.PacketsIn, packetsInNow, elapsed)
+		stats.PacketsOutPerSec = perSec(prevAverage.PacketsOut, packetsOutNow, elapsed)
+		stats.RetransmitBytesOutPerSec = perSec(prevAverage.RetransmitBytesOut, retransmitBytesNow, elapsed)
+		stats.RetransmitPacketsOutPerSec = perSec(prevAverage.RetransmitPacketsOut, retransmitPacketsNow, elapsed)
+		stats.NackPerSec = perSec(prevAverage.NackTotal, nackTotalNow, elapsed)
+		stats.ParticipantSignalConnectedPerSec = perSec(prevAverage.ParticipantSignalConnected, participantSignalConnectedNow, elapsed)
+		stats.ParticipantRtcInitPerSec = perSec(prevAverage.ParticipantRtcInit, participantRTCInitNow, elapsed)
+		stats.ParticipantRtcConnectedPerSec = perSec(prevAverage.ParticipantRtcConnected, participantRTConnectedCNow, elapsed)
+		stats.SysPacketsOutPerSec = perSec(uint64(prevAverage.SysPacketsOut), uint64(sysPackets), elapsed)
+		stats.SysPacketsDroppedPerSec = perSec(uint64(prevAverage.SysPacketsDropped), uint64(sysDroppedPackets), elapsed)
+		stats.TrackPublishAttemptsPerSec = perSec(uint64(prevAverage.NumTrackPublishAttempts), uint64(trackPublishAttemptsNow), elapsed)
+		stats.TrackPublishSuccessPerSec = perSec(uint64(prevAverage.NumTrackPublishSuccess), uint64(trackPublishSuccessNow), elapsed)
+		stats.TrackSubscribeAttemptsPerSec = perSec(uint64(prevAverage.NumTrackSubscribeAttempts), uint64(trackSubscribeAttemptsNow), elapsed)
+		stats.TrackSubscribeSuccessPerSec = perSec(uint64(prevAverage.NumTrackSubscribeSuccess), uint64(trackSubscribeSuccessNow), elapsed)
+
+		packetTotal := stats.SysPacketsOutPerSec + stats.SysPacketsDroppedPerSec
+		if packetTotal == 0 {
+			stats.SysPacketsDroppedPctPerSec = 0
+		} else {
+			stats.SysPacketsDroppedPctPerSec = stats.SysPacketsDroppedPerSec / packetTotal
+		}
+		promSysDroppedPacketPctGauge.Set(float64(stats.SysPacketsDroppedPctPerSec))
+	}
+
+	return stats, computeAverage, nil
+}
+
+func perSec(prev, curr uint64, secs int64) float32 {
+	return float32(curr-prev) / float32(secs)
+}

livekit-server/pkg/telemetry/prometheus/node_linux.go

@@ -0,0 +1,46 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build linux
+// +build linux
+
+package prometheus
+
+import (
+	"fmt"
+
+	"github.com/florianl/go-tc"
+)
+
+func getTCStats() (packets, drops uint32, err error) {
+	rtnl, err := tc.Open(&tc.Config{})
+	if err != nil {
+		err = fmt.Errorf("could not open rtnetlink socket: %v", err)
+		return
+	}
+	defer rtnl.Close()
+
+	qdiscs, err := rtnl.Qdisc().Get()
+	if err != nil {
+		err = fmt.Errorf("could not get qdiscs: %v", err)
+		return
+	}
+
+	for _, qdisc := range qdiscs {
+		packets = packets + qdisc.Stats.Packets
+		drops = drops + qdisc.Stats.Drops
+	}
+
+	return
+}

livekit-server/pkg/telemetry/prometheus/node_nonlinux.go

@@ -0,0 +1,22 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build !linux
+
+package prometheus
+
+func getTCStats() (packets, drops uint32, err error) {
+	// linux only
+	return
+}

livekit-server/pkg/telemetry/prometheus/node_nonwindows.go

@@ -0,0 +1,56 @@
+//go:build !windows
+
+/*
+ * Copyright 2023 LiveKit, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package prometheus
+
+import (
+	"runtime"
+	"sync"
+
+	"github.com/mackerelio/go-osstat/cpu"
+	"github.com/mackerelio/go-osstat/loadavg"
+)
+
+var (
+	cpuStatsLock              sync.RWMutex
+	lastCPUTotal, lastCPUIdle uint64
+)
+
+func getLoadAvg() (*loadavg.Stats, error) {
+	return loadavg.Get()
+}
+
+func getCPUStats() (cpuLoad float32, numCPUs uint32, err error) {
+	cpuInfo, err := cpu.Get()
+	if err != nil {
+		return
+	}
+
+	cpuStatsLock.Lock()
+	if lastCPUTotal > 0 && lastCPUTotal < cpuInfo.Total {
+		cpuLoad = 1 - float32(cpuInfo.Idle-lastCPUIdle)/float32(cpuInfo.Total-lastCPUTotal)
+	}
+
+	lastCPUTotal = cpuInfo.Total
+	lastCPUIdle = cpuInfo.Idle
+	cpuStatsLock.Unlock()
+
+	numCPUs = uint32(runtime.NumCPU())
+
+	return
+}

livekit-server/pkg/telemetry/prometheus/node_windows.go

@@ -0,0 +1,29 @@
+//go:build windows
+
+/*
+ * Copyright 2023 LiveKit, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package prometheus
+
+import "github.com/mackerelio/go-osstat/loadavg"
+
+func getLoadAvg() (*loadavg.Stats, error) {
+	return &loadavg.Stats{}, nil
+}
+
+func getCPUStats() (cpuLoad float32, numCPUs uint32, err error) {
+	return 1, 1, nil
+}

livekit-server/pkg/telemetry/prometheus/packets.go

@@ -0,0 +1,336 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package prometheus
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"go.uber.org/atomic"
+
+	"github.com/livekit/protocol/livekit"
+)
+
+type Direction string
+
+const (
+	Incoming               Direction = "incoming"
+	Outgoing               Direction = "outgoing"
+	transmissionInitial              = "initial"
+	transmissionRetransmit           = "retransmit"
+)
+
+var (
+	bytesIn                    atomic.Uint64
+	bytesOut                   atomic.Uint64
+	packetsIn                  atomic.Uint64
+	packetsOut                 atomic.Uint64
+	nackTotal                  atomic.Uint64
+	retransmitBytes            atomic.Uint64
+	retransmitPackets          atomic.Uint64
+	participantSignalConnected atomic.Uint64
+	participantRTCConnected    atomic.Uint64
+	participantRTCInit         atomic.Uint64
+	forwardLatency             atomic.Uint32
+	forwardJitter              atomic.Uint32
+
+	promPacketLabels          = []string{"direction", "transmission"}
+	promPacketTotal           *prometheus.CounterVec
+	promPacketBytes           *prometheus.CounterVec
+	promRTCPLabels            = []string{"direction"}
+	promStreamLabels          = []string{"direction", "source", "type"}
+	promNackTotal             *prometheus.CounterVec
+	promPliTotal              *prometheus.CounterVec
+	promFirTotal              *prometheus.CounterVec
+	promPacketLossTotal       *prometheus.CounterVec
+	promPacketLoss            *prometheus.HistogramVec
+	promPacketOutOfOrderTotal *prometheus.CounterVec
+	promPacketOutOfOrder      *prometheus.HistogramVec
+	promJitter                *prometheus.HistogramVec
+	promRTT                   *prometheus.HistogramVec
+	promParticipantJoin       *prometheus.CounterVec
+	promConnections           *prometheus.GaugeVec
+	promForwardLatency        prometheus.Gauge
+	promForwardJitter         prometheus.Gauge
+
+	promPacketTotalIncomingInitial    prometheus.Counter
+	promPacketTotalIncomingRetransmit prometheus.Counter
+	promPacketTotalOutgoingInitial    prometheus.Counter
+	promPacketTotalOutgoingRetransmit prometheus.Counter
+	promPacketBytesIncomingInitial    prometheus.Counter
+	promPacketBytesIncomingRetransmit prometheus.Counter
+	promPacketBytesOutgoingInitial    prometheus.Counter
+	promPacketBytesOutgoingRetransmit prometheus.Counter
+)
+
+func initPacketStats(nodeID string, nodeType livekit.NodeType) {
+	promPacketTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Namespace:   livekitNamespace,
+		Subsystem:   "packet",
+		Name:        "total",
+		ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
+	}, promPacketLabels)
+	promPacketBytes = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Namespace:   livekitNamespace,
+		Subsystem:   "packet",
+		Name:        "bytes",
+		ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
+	}, promPacketLabels)
+	promNackTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Namespace:   livekitNamespace,
+		Subsystem:   "nack",
+		Name:        "total",
+		ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
+	}, promRTCPLabels)
+	promPliTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Namespace:   livekitNamespace,
+		Subsystem:   "pli",
+		Name:        "total",
+		ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
+	}, promRTCPLabels)
+	promFirTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Namespace:   livekitNamespace,
+		Subsystem:   "fir",
+		Name:        "total",
+		ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
+	}, promRTCPLabels)
+	promPacketLossTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Namespace:   livekitNamespace,
+		Subsystem:   "packet_loss",
+		Name:        "total",
+		ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
+	}, promStreamLabels)
+	promPacketLoss = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Namespace:   livekitNamespace,
+		Subsystem:   "packet_loss",
+		Name:        "percent",
+		ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
+		Buckets:     []float64{0.0, 0.1, 0.3, 0.5, 0.7, 1, 5, 10, 40, 100},
+	}, promStreamLabels)
+	promPacketOutOfOrderTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Namespace:   livekitNamespace,
+		Subsystem:   "packet_out_of_order",
+		Name:        "total",
+		ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
+	}, promStreamLabels)
+	promPacketOutOfOrder = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Namespace:   livekitNamespace,
+		Subsystem:   "packet_out_of_order",
+		Name:        "percent",
+		ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
+		Buckets:     []float64{0.0, 0.1, 0.3, 0.5, 0.7, 1, 5, 10, 40, 100},
+	}, promStreamLabels)
+	promJitter = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Namespace:   livekitNamespace,
+		Subsystem:   "jitter",
+		Name:        "us",
+		ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
+
+		// 1ms, 10ms, 30ms, 50ms, 70ms, 100ms, 300ms, 600ms, 1s
+		Buckets: []float64{1000, 10000, 30000, 50000, 70000, 100000, 300000, 600000, 1000000},
+	}, promStreamLabels)
+	promRTT = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Namespace:   livekitNamespace,
+		Subsystem:   "rtt",
+		Name:        "ms",
+		ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
+		Buckets:     []float64{50, 100, 150, 200, 250, 500, 750, 1000, 5000, 10000},
+	}, promStreamLabels)
+	promParticipantJoin = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Namespace:   livekitNamespace,
+		Subsystem:   "participant_join",
+		Name:        "total",
+		ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
+	}, []string{"state"})
+	promConnections = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace:   livekitNamespace,
+		Subsystem:   "connection",
+		Name:        "total",
+		ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
+	}, []string{"kind"})
+	promForwardLatency = prometheus.NewGauge(prometheus.GaugeOpts{
+		Namespace:   livekitNamespace,
+		Subsystem:   "forward",
+		Name:        "latency",
+		ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
+	})
+	promForwardJitter = prometheus.NewGauge(prometheus.GaugeOpts{
+		Namespace:   livekitNamespace,
+		Subsystem:   "forward",
+		Name:        "jitter",
+		ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
+	})
+
+	prometheus.MustRegister(promPacketTotal)
+	prometheus.MustRegister(promPacketBytes)
+	prometheus.MustRegister(promNackTotal)
+	prometheus.MustRegister(promPliTotal)
+	prometheus.MustRegister(promFirTotal)
+	prometheus.MustRegister(promPacketLossTotal)
+	prometheus.MustRegister(promPacketLoss)
+	prometheus.MustRegister(promPacketOutOfOrderTotal)
+	prometheus.MustRegister(promPacketOutOfOrder)
+	prometheus.MustRegister(promJitter)
+	prometheus.MustRegister(promRTT)
+	prometheus.MustRegister(promParticipantJoin)
+	prometheus.MustRegister(promConnections)
+	prometheus.MustRegister(promForwardLatency)
+	prometheus.MustRegister(promForwardJitter)
+
+	promPacketTotalIncomingInitial = promPacketTotal.WithLabelValues(string(Incoming), transmissionInitial)
+	promPacketTotalIncomingRetransmit = promPacketTotal.WithLabelValues(string(Incoming), transmissionRetransmit)
+	promPacketTotalOutgoingInitial = promPacketTotal.WithLabelValues(string(Outgoing), transmissionInitial)
+	promPacketTotalOutgoingRetransmit = promPacketTotal.WithLabelValues(string(Outgoing), transmissionRetransmit)
+	promPacketBytesIncomingInitial = promPacketBytes.WithLabelValues(string(Incoming), transmissionInitial)
+	promPacketBytesIncomingRetransmit = promPacketBytes.WithLabelValues(string(Incoming), transmissionRetransmit)
+	promPacketBytesOutgoingInitial = promPacketBytes.WithLabelValues(string(Outgoing), transmissionInitial)
+	promPacketBytesOutgoingRetransmit = promPacketBytes.WithLabelValues(string(Outgoing), transmissionRetransmit)
+}
+
+func IncrementPackets(direction Direction, count uint64, retransmit bool) {
+	if direction == Incoming {
+		if retransmit {
+			promPacketTotalIncomingRetransmit.Add(float64(count))
+		} else {
+			promPacketTotalIncomingInitial.Add(float64(count))
+		}
+	} else {
+		if retransmit {
+			promPacketTotalOutgoingRetransmit.Add(float64(count))
+		} else {
+			promPacketTotalOutgoingInitial.Add(float64(count))
+		}
+	}
+
+	if direction == Incoming {
+		packetsIn.Add(count)
+	} else {
+		packetsOut.Add(count)
+		if retransmit {
+			retransmitPackets.Add(count)
+		}
+	}
+}
+
+func IncrementBytes(direction Direction, count uint64, retransmit bool) {
+	if direction == Incoming {
+		if retransmit {
+			promPacketBytesIncomingRetransmit.Add(float64(count))
+		} else {
+			promPacketBytesIncomingInitial.Add(float64(count))
+		}
+	} else {
+		if retransmit {
+			promPacketBytesOutgoingRetransmit.Add(float64(count))
+		} else {
+			promPacketBytesOutgoingInitial.Add(float64(count))
+		}
+	}
+
+	if direction == Incoming {
+		bytesIn.Add(count)
+	} else {
+		bytesOut.Add(count)
+		if retransmit {
+			retransmitBytes.Add(count)
+		}
+	}
+}
+
+func IncrementRTCP(direction Direction, nack, pli, fir uint32) {
+	if nack > 0 {
+		promNackTotal.WithLabelValues(string(direction)).Add(float64(nack))
+		nackTotal.Add(uint64(nack))
+	}
+	if pli > 0 {
+		promPliTotal.WithLabelValues(string(direction)).Add(float64(pli))
+	}
+	if fir > 0 {
+		promFirTotal.WithLabelValues(string(direction)).Add(float64(fir))
+	}
+}
+
+func RecordPacketLoss(direction Direction, trackSource livekit.TrackSource, trackType livekit.TrackType, lost, total uint32) {
+	if total > 0 {
+		promPacketLoss.WithLabelValues(string(direction), trackSource.String(), trackType.String()).Observe(float64(lost) / float64(total) * 100)
+	}
+	if lost > 0 {
+		promPacketLossTotal.WithLabelValues(string(direction), trackSource.String(), trackType.String()).Add(float64(lost))
+	}
+}
+
+func RecordPacketOutOfOrder(direction Direction, trackSource livekit.TrackSource, trackType livekit.TrackType, ooo, total uint32) {
+	if total > 0 {
+		promPacketOutOfOrder.WithLabelValues(string(direction), trackSource.String(), trackType.String()).Observe(float64(ooo) / float64(total) * 100)
+	}
+	if ooo > 0 {
+		promPacketOutOfOrderTotal.WithLabelValues(string(direction), trackSource.String(), trackType.String()).Add(float64(ooo))
+	}
+}
+
+func RecordJitter(direction Direction, trackSource livekit.TrackSource, trackType livekit.TrackType, jitter uint32) {
+	if jitter > 0 {
+		promJitter.WithLabelValues(string(direction), trackSource.String(), trackType.String()).Observe(float64(jitter))
+	}
+}
+
+func RecordRTT(direction Direction, trackSource livekit.TrackSource, trackType livekit.TrackType, rtt uint32) {
+	if rtt > 0 {
+		promRTT.WithLabelValues(string(direction), trackSource.String(), trackType.String()).Observe(float64(rtt))
+	}
+}
+
+func IncrementParticipantJoin(join uint32) {
+	if join > 0 {
+		participantSignalConnected.Add(uint64(join))
+		promParticipantJoin.WithLabelValues("signal_connected").Add(float64(join))
+	}
+}
+
+func IncrementParticipantJoinFail(join uint32) {
+	if join > 0 {
+		promParticipantJoin.WithLabelValues("signal_failed").Add(float64(join))
+	}
+}
+
+func IncrementParticipantRtcInit(join uint32) {
+	if join > 0 {
+		participantRTCInit.Add(uint64(join))
+		promParticipantJoin.WithLabelValues("rtc_init").Add(float64(join))
+	}
+}
+
+func IncrementParticipantRtcConnected(join uint32) {
+	if join > 0 {
+		participantRTCConnected.Add(uint64(join))
+		promParticipantJoin.WithLabelValues("rtc_connected").Add(float64(join))
+	}
+}
+
+func AddConnection(direction Direction) {
+	promConnections.WithLabelValues(string(direction)).Add(1)
+}
+
+func SubConnection(direction Direction) {
+	promConnections.WithLabelValues(string(direction)).Sub(1)
+}
+
+func RecordForwardLatency(_, latencyAvg uint32) {
+	forwardLatency.Store(latencyAvg)
+	promForwardLatency.Set(float64(latencyAvg))
+}
+
+func RecordForwardJitter(_, jitterAvg uint32) {
+	forwardJitter.Store(jitterAvg)
+	promForwardJitter.Set(float64(jitterAvg))
+}

livekit-server/pkg/telemetry/prometheus/quality.go

@@ -0,0 +1,51 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package prometheus
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/livekit/protocol/livekit"
+)
+
+var (
+	qualityRating prometheus.Histogram
+	qualityScore  prometheus.Histogram
+)
+
+func initQualityStats(nodeID string, nodeType livekit.NodeType) {
+	qualityRating = prometheus.NewHistogram(prometheus.HistogramOpts{
+		Namespace:   livekitNamespace,
+		Subsystem:   "quality",
+		Name:        "rating",
+		ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
+		Buckets:     []float64{0, 1, 2},
+	})
+	qualityScore = prometheus.NewHistogram(prometheus.HistogramOpts{
+		Namespace:   livekitNamespace,
+		Subsystem:   "quality",
+		Name:        "score",
+		ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
+		Buckets:     []float64{1.0, 2.0, 2.5, 3.0, 3.25, 3.5, 3.75, 4.0, 4.25, 4.5},
+	})
+
+	prometheus.MustRegister(qualityRating)
+	prometheus.MustRegister(qualityScore)
+}
+
+func RecordQuality(rating livekit.ConnectionQuality, score float32) {
+	qualityRating.Observe(float64(rating))
+	qualityScore.Observe(float64(score))
+}

livekit-server/pkg/telemetry/prometheus/rooms.go

@@ -0,0 +1,226 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package prometheus
+
+import (
+	"strconv"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"go.uber.org/atomic"
+
+	"github.com/livekit/protocol/livekit"
+)
+
+var (
+	roomCurrent            atomic.Int32
+	participantCurrent     atomic.Int32
+	trackPublishedCurrent  atomic.Int32
+	trackSubscribedCurrent atomic.Int32
+	trackPublishAttempts   atomic.Int32
+	trackPublishSuccess    atomic.Int32
+	trackSubscribeAttempts atomic.Int32
+	trackSubscribeSuccess  atomic.Int32
+	// count the number of failures that are due to user error (permissions, track doesn't exist), so we could compute
+	// success rate by subtracting this from total attempts
+	trackSubscribeUserError atomic.Int32
+
+	promRoomCurrent            prometheus.Gauge
+	promRoomDuration           prometheus.Histogram
+	promParticipantCurrent     prometheus.Gauge
+	promTrackPublishedCurrent  *prometheus.GaugeVec
+	promTrackSubscribedCurrent *prometheus.GaugeVec
+	promTrackPublishCounter    *prometheus.CounterVec
+	promTrackSubscribeCounter  *prometheus.CounterVec
+	promSessionStartTime       *prometheus.HistogramVec
+	promSessionDuration        *prometheus.HistogramVec
+	promPubSubTime             *prometheus.HistogramVec
+)
+
+func initRoomStats(nodeID string, nodeType livekit.NodeType) {
+	promRoomCurrent = prometheus.NewGauge(prometheus.GaugeOpts{
+		Namespace:   livekitNamespace,
+		Subsystem:   "room",
+		Name:        "total",
+		ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
+	})
+	promRoomDuration = prometheus.NewHistogram(prometheus.HistogramOpts{
+		Namespace:   livekitNamespace,
+		Subsystem:   "room",
+		Name:        "duration_seconds",
+		ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
+		Buckets: []float64{
+			5, 10, 60, 5 * 60, 10 * 60, 30 * 60, 60 * 60, 2 * 60 * 60, 5 * 60 * 60, 10 * 60 * 60,
+		},
+	})
+	promParticipantCurrent = prometheus.NewGauge(prometheus.GaugeOpts{
+		Namespace:   livekitNamespace,
+		Subsystem:   "participant",
+		Name:        "total",
+		ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
+	})
+	promTrackPublishedCurrent = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace:   livekitNamespace,
+		Subsystem:   "track",
+		Name:        "published_total",
+		ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
+	}, []string{"kind"})
+	promTrackSubscribedCurrent = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace:   livekitNamespace,
+		Subsystem:   "track",
+		Name:        "subscribed_total",
+		ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
+	}, []string{"kind"})
+	promTrackPublishCounter = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Namespace:   livekitNamespace,
+		Subsystem:   "track",
+		Name:        "publish_counter",
+		ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
+	}, []string{"kind", "state"})
+	promTrackSubscribeCounter = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Namespace:   livekitNamespace,
+		Subsystem:   "track",
+		Name:        "subscribe_counter",
+		ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
+	}, []string{"state", "error"})
+	promSessionStartTime = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Namespace:   livekitNamespace,
+		Subsystem:   "session",
+		Name:        "start_time_ms",
+		ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
+		Buckets:     prometheus.ExponentialBucketsRange(100, 10000, 15),
+	}, []string{"protocol_version"})
+	promSessionDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Namespace:   livekitNamespace,
+		Subsystem:   "session",
+		Name:        "duration_ms",
+		ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
+		Buckets:     prometheus.ExponentialBucketsRange(100, 4*60*60*1000, 15),
+	}, []string{"protocol_version"})
+	promPubSubTime = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Namespace:   livekitNamespace,
+		Subsystem:   "pubsubtime",
+		Name:        "ms",
+		ConstLabels: prometheus.Labels{"node_id": nodeID, "node_type": nodeType.String()},
+		Buckets:     []float64{100, 200, 500, 700, 1000, 5000, 10000},
+	}, append(promStreamLabels, "sdk", "kind", "count"))
+
+	prometheus.MustRegister(promRoomCurrent)
+	prometheus.MustRegister(promRoomDuration)
+	prometheus.MustRegister(promParticipantCurrent)
+	prometheus.MustRegister(promTrackPublishedCurrent)
+	prometheus.MustRegister(promTrackSubscribedCurrent)
+	prometheus.MustRegister(promTrackPublishCounter)
+	prometheus.MustRegister(promTrackSubscribeCounter)
+	prometheus.MustRegister(promSessionStartTime)
+	prometheus.MustRegister(promSessionDuration)
+	prometheus.MustRegister(promPubSubTime)
+}
+
+func RoomStarted() {
+	promRoomCurrent.Add(1)
+	roomCurrent.Inc()
+}
+
+func RoomEnded(startedAt time.Time) {
+	if !startedAt.IsZero() {
+		promRoomDuration.Observe(float64(time.Since(startedAt)) / float64(time.Second))
+	}
+	promRoomCurrent.Sub(1)
+	roomCurrent.Dec()
+}
+
+func AddParticipant() {
+	promParticipantCurrent.Add(1)
+	participantCurrent.Inc()
+}
+
+func SubParticipant() {
+	promParticipantCurrent.Sub(1)
+	participantCurrent.Dec()
+}
+
+func AddPublishedTrack(kind string) {
+	promTrackPublishedCurrent.WithLabelValues(kind).Add(1)
+	trackPublishedCurrent.Inc()
+}
+
+func SubPublishedTrack(kind string) {
+	promTrackPublishedCurrent.WithLabelValues(kind).Sub(1)
+	trackPublishedCurrent.Dec()
+}
+
+func AddPublishAttempt(kind string) {
+	trackPublishAttempts.Inc()
+	promTrackPublishCounter.WithLabelValues(kind, "attempt").Inc()
+}
+
+func AddPublishSuccess(kind string) {
+	trackPublishSuccess.Inc()
+	promTrackPublishCounter.WithLabelValues(kind, "success").Inc()
+}
+
+func RecordPublishTime(source livekit.TrackSource, trackType livekit.TrackType, d time.Duration, sdk livekit.ClientInfo_SDK, kind livekit.ParticipantInfo_Kind) {
+	recordPubSubTime(true, source, trackType, d, sdk, kind, 1)
+}
+
+func RecordSubscribeTime(source livekit.TrackSource, trackType livekit.TrackType, d time.Duration, sdk livekit.ClientInfo_SDK, kind livekit.ParticipantInfo_Kind, count int) {
+	recordPubSubTime(false, source, trackType, d, sdk, kind, count)
+}
+
+func recordPubSubTime(isPublish bool, source livekit.TrackSource, trackType livekit.TrackType, d time.Duration, sdk livekit.ClientInfo_SDK, kind livekit.ParticipantInfo_Kind, count int) {
+	direction := "subscribe"
+	if isPublish {
+		direction = "publish"
+	}
+	promPubSubTime.WithLabelValues(direction, source.String(), trackType.String(), sdk.String(), kind.String(), strconv.Itoa(count)).Observe(float64(d.Milliseconds()))
+}
+
+func RecordTrackSubscribeSuccess(kind string) {
+	// modify both current and total counters
+	promTrackSubscribedCurrent.WithLabelValues(kind).Add(1)
+	trackSubscribedCurrent.Inc()
+
+	promTrackSubscribeCounter.WithLabelValues("success", "").Inc()
+	trackSubscribeSuccess.Inc()
+}
+
+func RecordTrackUnsubscribed(kind string) {
+	// unsubscribed modifies current counter, but we leave the total values alone since they
+	// are used to compute rate
+	promTrackSubscribedCurrent.WithLabelValues(kind).Sub(1)
+	trackSubscribedCurrent.Dec()
+}
+
+func RecordTrackSubscribeAttempt() {
+	trackSubscribeAttempts.Inc()
+	promTrackSubscribeCounter.WithLabelValues("attempt", "").Inc()
+}
+
+func RecordTrackSubscribeFailure(err error, isUserError bool) {
+	promTrackSubscribeCounter.WithLabelValues("failure", err.Error()).Inc()
+
+	if isUserError {
+		trackSubscribeUserError.Inc()
+	}
+}
+
+func RecordSessionStartTime(protocolVersion int, d time.Duration) {
+	promSessionStartTime.WithLabelValues(strconv.Itoa(protocolVersion)).Observe(float64(d.Milliseconds()))
+}
+
+func RecordSessionDuration(protocolVersion int, d time.Duration) {
+	promSessionDuration.WithLabelValues(strconv.Itoa(protocolVersion)).Observe(float64(d.Milliseconds()))
+}

livekit-server/pkg/telemetry/signalanddatastats.go

@@ -0,0 +1,205 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package telemetry
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/frostbyte73/core"
+	"go.uber.org/atomic"
+
+	"github.com/livekit/protocol/livekit"
+	"github.com/livekit/protocol/utils"
+)
+
+type BytesTrackType string
+
+const (
+	BytesTrackTypeData   BytesTrackType = "DT"
+	BytesTrackTypeSignal BytesTrackType = "SG"
+)
+
+// -------------------------------
+
+type TrafficTotals struct {
+	At           time.Time
+	SendBytes    uint64
+	SendMessages uint32
+	RecvBytes    uint64
+	RecvMessages uint32
+}
+
+// --------------------------------
+
+// stats for signal and data channel
+type BytesTrackStats struct {
+	trackID                              livekit.TrackID
+	pID                                  livekit.ParticipantID
+	send, recv                           atomic.Uint64
+	sendMessages, recvMessages           atomic.Uint32
+	totalSendBytes, totalRecvBytes       atomic.Uint64
+	totalSendMessages, totalRecvMessages atomic.Uint32
+	telemetry                            TelemetryService
+	done                                 core.Fuse
+}
+
+func NewBytesTrackStats(trackID livekit.TrackID, pID livekit.ParticipantID, telemetry TelemetryService) *BytesTrackStats {
+	s := &BytesTrackStats{
+		trackID:   trackID,
+		pID:       pID,
+		telemetry: telemetry,
+	}
+	go s.reporter()
+	return s
+}
+
+func (s *BytesTrackStats) AddBytes(bytes uint64, isSend bool) {
+	if isSend {
+		s.send.Add(bytes)
+		s.sendMessages.Inc()
+		s.totalSendBytes.Add(bytes)
+		s.totalSendMessages.Inc()
+	} else {
+		s.recv.Add(bytes)
+		s.recvMessages.Inc()
+		s.totalRecvBytes.Add(bytes)
+		s.totalRecvMessages.Inc()
+	}
+}
+
+func (s *BytesTrackStats) GetTrafficTotals() *TrafficTotals {
+	return &TrafficTotals{
+		At:           time.Now(),
+		SendBytes:    s.totalSendBytes.Load(),
+		SendMessages: s.totalSendMessages.Load(),
+		RecvBytes:    s.totalRecvBytes.Load(),
+		RecvMessages: s.totalRecvMessages.Load(),
+	}
+}
+
+func (s *BytesTrackStats) Stop() {
+	s.done.Break()
+}
+
+func (s *BytesTrackStats) report() {
+	if recv := s.recv.Swap(0); recv > 0 {
+		s.telemetry.TrackStats(StatsKeyForData(livekit.StreamType_UPSTREAM, s.pID, s.trackID), &livekit.AnalyticsStat{
+			Streams: []*livekit.AnalyticsStream{
+				{
+					PrimaryBytes:   recv,
+					PrimaryPackets: s.recvMessages.Swap(0),
+				},
+			},
+		})
+	}
+
+	if send := s.send.Swap(0); send > 0 {
+		s.telemetry.TrackStats(StatsKeyForData(livekit.StreamType_DOWNSTREAM, s.pID, s.trackID), &livekit.AnalyticsStat{
+			Streams: []*livekit.AnalyticsStream{
+				{
+					PrimaryBytes:   send,
+					PrimaryPackets: s.sendMessages.Swap(0),
+				},
+			},
+		})
+	}
+}
+
+func (s *BytesTrackStats) reporter() {
+	ticker := time.NewTicker(telemetryNonMediaStatsUpdateInterval)
+	defer func() {
+		ticker.Stop()
+		s.report()
+	}()
+
+	for {
+		select {
+		case <-s.done.Watch():
+			return
+		case <-ticker.C:
+			s.report()
+		}
+	}
+}
+
+// -----------------------------------------------------------------------
+
+type BytesSignalStats struct {
+	BytesTrackStats
+	ctx context.Context
+
+	mu sync.Mutex
+	ri *livekit.Room
+	pi *livekit.ParticipantInfo
+}
+
+func NewBytesSignalStats(ctx context.Context, telemetry TelemetryService) *BytesSignalStats {
+	return &BytesSignalStats{
+		BytesTrackStats: BytesTrackStats{
+			telemetry: telemetry,
+		},
+		ctx: ctx,
+	}
+}
+
+func (s *BytesSignalStats) ResolveRoom(ri *livekit.Room) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if s.ri == nil && ri.GetSid() != "" {
+		s.ri = &livekit.Room{
+			Sid:  ri.Sid,
+			Name: ri.Name,
+		}
+		s.maybeStart()
+	}
+}
+
+func (s *BytesSignalStats) ResolveParticipant(pi *livekit.ParticipantInfo) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if s.pi == nil && pi != nil {
+		s.pi = &livekit.ParticipantInfo{
+			Sid:      pi.Sid,
+			Identity: pi.Identity,
+		}
+		s.maybeStart()
+	}
+}
+
+func (s *BytesSignalStats) maybeStart() {
+	if s.ri == nil || s.pi == nil {
+		return
+	}
+
+	s.pID = livekit.ParticipantID(s.pi.Sid)
+	s.trackID = BytesTrackIDForParticipantID(BytesTrackTypeSignal, s.pID)
+
+	s.telemetry.ParticipantJoined(s.ctx, s.ri, s.pi, nil, nil, false)
+	go s.reporter()
+}
+
+func (s *BytesSignalStats) reporter() {
+	s.BytesTrackStats.reporter()
+	s.telemetry.ParticipantLeft(s.ctx, s.ri, s.pi, false)
+}
+
+// -----------------------------------------------------------------------
+
+func BytesTrackIDForParticipantID(typ BytesTrackType, participantID livekit.ParticipantID) livekit.TrackID {
+	return livekit.TrackID(fmt.Sprintf("%s_%s%s", utils.TrackPrefix, string(typ), participantID))
+}

livekit-server/pkg/telemetry/stats.go

@@ -0,0 +1,99 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package telemetry
+
+import (
+	"github.com/livekit/livekit-server/pkg/telemetry/prometheus"
+	"github.com/livekit/protocol/livekit"
+)
+
+type StatsKey struct {
+	streamType    livekit.StreamType
+	participantID livekit.ParticipantID
+	trackID       livekit.TrackID
+	trackSource   livekit.TrackSource
+	trackType     livekit.TrackType
+	track         bool
+}
+
+func StatsKeyForTrack(streamType livekit.StreamType, participantID livekit.ParticipantID, trackID livekit.TrackID, trackSource livekit.TrackSource, trackType livekit.TrackType) StatsKey {
+	return StatsKey{
+		streamType:    streamType,
+		participantID: participantID,
+		trackID:       trackID,
+		trackSource:   trackSource,
+		trackType:     trackType,
+		track:         true,
+	}
+}
+
+func StatsKeyForData(streamType livekit.StreamType, participantID livekit.ParticipantID, trackID livekit.TrackID) StatsKey {
+	return StatsKey{
+		streamType:    streamType,
+		participantID: participantID,
+		trackID:       trackID,
+	}
+}
+
+func (t *telemetryService) TrackStats(key StatsKey, stat *livekit.AnalyticsStat) {
+	t.enqueue(func() {
+		direction := prometheus.Incoming
+		if key.streamType == livekit.StreamType_DOWNSTREAM {
+			direction = prometheus.Outgoing
+		}
+
+		nacks := uint32(0)
+		plis := uint32(0)
+		firs := uint32(0)
+		packets := uint32(0)
+		bytes := uint64(0)
+		retransmitBytes := uint64(0)
+		retransmitPackets := uint32(0)
+		for _, stream := range stat.Streams {
+			nacks += stream.Nacks
+			plis += stream.Plis
+			firs += stream.Firs
+			packets += stream.PrimaryPackets + stream.PaddingPackets
+			bytes += stream.PrimaryBytes + stream.PaddingBytes
+			if key.streamType == livekit.StreamType_DOWNSTREAM {
+				retransmitPackets += stream.RetransmitPackets
+				retransmitBytes += stream.RetransmitBytes
+			} else {
+				// for upstream, we don't account for these separately for now
+				packets += stream.RetransmitPackets
+				bytes += stream.RetransmitBytes
+			}
+			if key.track {
+				prometheus.RecordPacketLoss(direction, key.trackSource, key.trackType, stream.PacketsLost, stream.PrimaryPackets+stream.PaddingPackets)
+				prometheus.RecordPacketOutOfOrder(direction, key.trackSource, key.trackType, stream.PacketsOutOfOrder, stream.PrimaryPackets+stream.PaddingPackets)
+				prometheus.RecordRTT(direction, key.trackSource, key.trackType, stream.Rtt)
+				prometheus.RecordJitter(direction, key.trackSource, key.trackType, stream.Jitter)
+			}
+		}
+		prometheus.IncrementRTCP(direction, nacks, plis, firs)
+		prometheus.IncrementPackets(direction, uint64(packets), false)
+		prometheus.IncrementBytes(direction, bytes, false)
+		if retransmitPackets != 0 {
+			prometheus.IncrementPackets(direction, uint64(retransmitPackets), true)
+		}
+		if retransmitBytes != 0 {
+			prometheus.IncrementBytes(direction, retransmitBytes, true)
+		}
+
+		if worker, ok := t.getWorker(key.participantID); ok {
+			worker.OnTrackStat(key.trackID, key.streamType, stat)
+		}
+	})
+}

livekit-server/pkg/telemetry/stats_test.go

@@ -0,0 +1,620 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package telemetry_test
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/livekit/livekit-server/pkg/telemetry"
+	"github.com/livekit/protocol/livekit"
+
+	"github.com/livekit/livekit-server/pkg/telemetry/prometheus"
+	"github.com/livekit/livekit-server/pkg/telemetry/telemetryfakes"
+)
+
+func init() {
+	prometheus.Init("test", livekit.NodeType_SERVER)
+}
+
+type telemetryServiceFixture struct {
+	sut       telemetry.TelemetryService
+	analytics *telemetryfakes.FakeAnalyticsService
+}
+
+func createFixture() *telemetryServiceFixture {
+	fixture := &telemetryServiceFixture{}
+	fixture.analytics = &telemetryfakes.FakeAnalyticsService{}
+	fixture.sut = telemetry.NewTelemetryService(nil, fixture.analytics)
+	return fixture
+}
+
+func Test_ParticipantAndRoomDataAreSentWithAnalytics(t *testing.T) {
+	fixture := createFixture()
+
+	// prepare
+	room := &livekit.Room{Sid: "RoomSid", Name: "RoomName"}
+	partSID := livekit.ParticipantID("part1")
+	clientInfo := &livekit.ClientInfo{Sdk: 2}
+	participantInfo := &livekit.ParticipantInfo{Sid: string(partSID)}
+	fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, clientInfo, nil, true)
+
+	// do
+	packet := 33
+	stat := &livekit.AnalyticsStat{Streams: []*livekit.AnalyticsStream{{PrimaryBytes: uint64(packet)}}}
+	fixture.sut.TrackStats(telemetry.StatsKeyForData(livekit.StreamType_DOWNSTREAM, partSID, ""), stat)
+
+	// flush
+	fixture.flush()
+
+	// test
+	require.Equal(t, 1, fixture.analytics.SendStatsCallCount())
+	_, stats := fixture.analytics.SendStatsArgsForCall(0)
+	require.Equal(t, 1, len(stats))
+	require.Equal(t, livekit.StreamType_DOWNSTREAM, stats[0].Kind)
+	require.Equal(t, string(partSID), stats[0].ParticipantId)
+	require.Equal(t, room.Sid, stats[0].RoomId)
+	require.Equal(t, room.Name, stats[0].RoomName)
+}
+
+func Test_OnDownstreamPackets(t *testing.T) {
+	fixture := createFixture()
+
+	// prepare
+	room := &livekit.Room{}
+	partSID := livekit.ParticipantID("part1")
+	clientInfo := &livekit.ClientInfo{Sdk: 2}
+	participantInfo := &livekit.ParticipantInfo{Sid: string(partSID)}
+	fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, clientInfo, nil, true)
+
+	// do
+	packets := []int{33, 23}
+	totalBytes := packets[0] + packets[1]
+	totalPackets := len(packets)
+	trackID := livekit.TrackID("trackID")
+	for i := range packets {
+		stat := &livekit.AnalyticsStat{Streams: []*livekit.AnalyticsStream{{PrimaryBytes: uint64(packets[i]), PrimaryPackets: uint32(1)}}}
+		fixture.sut.TrackStats(telemetry.StatsKeyForData(livekit.StreamType_DOWNSTREAM, partSID, trackID), stat)
+	}
+
+	// flush
+	fixture.flush()
+
+	// test
+	require.Equal(t, 1, fixture.analytics.SendStatsCallCount())
+	_, stats := fixture.analytics.SendStatsArgsForCall(0)
+	require.Equal(t, 1, len(stats))
+	require.Equal(t, livekit.StreamType_DOWNSTREAM, stats[0].Kind)
+	require.Equal(t, totalBytes, int(stats[0].Streams[0].PrimaryBytes))
+	require.Equal(t, totalPackets, int(stats[0].Streams[0].PrimaryPackets))
+	require.Equal(t, string(trackID), stats[0].TrackId)
+}
+
+func Test_OnDownstreamPackets_SeveralTracks(t *testing.T) {
+	fixture := createFixture()
+
+	// prepare
+	room := &livekit.Room{}
+	partSID := livekit.ParticipantID("part1")
+	clientInfo := &livekit.ClientInfo{Sdk: 2}
+	participantInfo := &livekit.ParticipantInfo{Sid: string(partSID)}
+	fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, clientInfo, nil, true)
+
+	// do
+	packet1 := 33
+	trackID1 := livekit.TrackID("trackID1")
+	stat1 := &livekit.AnalyticsStat{Streams: []*livekit.AnalyticsStream{{PrimaryBytes: uint64(packet1), PrimaryPackets: 1}}}
+	fixture.sut.TrackStats(telemetry.StatsKeyForData(livekit.StreamType_DOWNSTREAM, partSID, trackID1), stat1)
+
+	packet2 := 23
+	trackID2 := livekit.TrackID("trackID2")
+	stat2 := &livekit.AnalyticsStat{Streams: []*livekit.AnalyticsStream{{PrimaryBytes: uint64(packet2), PrimaryPackets: 1}}}
+	fixture.sut.TrackStats(telemetry.StatsKeyForData(livekit.StreamType_DOWNSTREAM, partSID, trackID2), stat2)
+
+	// flush
+	fixture.flush()
+
+	// test
+	require.Equal(t, 1, fixture.analytics.SendStatsCallCount())
+	_, stats := fixture.analytics.SendStatsArgsForCall(0)
+	require.Equal(t, 2, len(stats))
+
+	found1 := false
+	found2 := false
+	for _, sentStat := range stats {
+		if livekit.TrackID(sentStat.TrackId) == trackID1 {
+			found1 = true
+			require.Equal(t, packet1, int(sentStat.Streams[0].PrimaryBytes))
+			require.Equal(t, 1, int(sentStat.Streams[0].PrimaryPackets))
+		} else if livekit.TrackID(sentStat.TrackId) == trackID2 {
+			found2 = true
+			require.Equal(t, packet2, int(sentStat.Streams[0].PrimaryBytes))
+			require.Equal(t, 1, int(sentStat.Streams[0].PrimaryPackets))
+		}
+	}
+	require.True(t, found1)
+	require.True(t, found2)
+}
+
+func Test_OnDownStreamStat(t *testing.T) {
+	fixture := createFixture()
+
+	// prepare
+	room := &livekit.Room{}
+	partSID := livekit.ParticipantID("part1")
+	participantInfo := &livekit.ParticipantInfo{Sid: string(partSID)}
+	fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, nil, nil, true)
+
+	// do
+	stat1 := &livekit.AnalyticsStat{
+		Streams: []*livekit.AnalyticsStream{
+			{
+				PrimaryBytes:   1,
+				PrimaryPackets: 1,
+				PacketsLost:    3,
+				Nacks:          1,
+				Plis:           1,
+				Rtt:            23,
+				Jitter:         3,
+			},
+		},
+	}
+	trackID := livekit.TrackID("trackID1")
+	fixture.sut.TrackStats(telemetry.StatsKeyForData(livekit.StreamType_DOWNSTREAM, partSID, trackID), stat1)
+
+	stat2 := &livekit.AnalyticsStat{
+		Streams: []*livekit.AnalyticsStream{
+			{
+				PrimaryBytes:   2,
+				PrimaryPackets: 2,
+				PacketsLost:    4,
+				Nacks:          1,
+				Plis:           1,
+				Firs:           1,
+				Rtt:            10,
+				Jitter:         5,
+			},
+		},
+	}
+	fixture.sut.TrackStats(telemetry.StatsKeyForData(livekit.StreamType_DOWNSTREAM, partSID, trackID), stat2)
+
+	// flush
+	fixture.flush()
+
+	// test
+	require.Equal(t, 1, fixture.analytics.SendStatsCallCount())
+	_, stats := fixture.analytics.SendStatsArgsForCall(0)
+	require.Equal(t, 1, len(stats))
+	require.Equal(t, livekit.StreamType_DOWNSTREAM, stats[0].Kind)
+	require.Equal(t, 2, int(stats[0].Streams[0].Nacks))
+	require.Equal(t, 2, int(stats[0].Streams[0].Plis))
+	require.Equal(t, 1, int(stats[0].Streams[0].Firs))
+	require.Equal(t, 23, int(stats[0].Streams[0].Rtt))        // max of RTT
+	require.Equal(t, 5, int(stats[0].Streams[0].Jitter))      // max of jitter
+	require.Equal(t, 7, int(stats[0].Streams[0].PacketsLost)) // coalesced delta packet losses
+	require.Equal(t, string(trackID), stats[0].TrackId)
+}
+
+func Test_PacketLostDiffShouldBeSentToTelemetry(t *testing.T) {
+	fixture := createFixture()
+
+	// prepare
+	room := &livekit.Room{}
+	partSID := livekit.ParticipantID("part1")
+	participantInfo := &livekit.ParticipantInfo{Sid: string(partSID)}
+	fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, nil, nil, true)
+
+	// do
+	trackID := livekit.TrackID("trackID1")
+	stat1 := &livekit.AnalyticsStat{
+		Streams: []*livekit.AnalyticsStream{
+			{
+				PrimaryBytes:   1,
+				PrimaryPackets: 1,
+				PacketsLost:    1,
+			},
+		},
+	}
+	fixture.sut.TrackStats(telemetry.StatsKeyForData(livekit.StreamType_DOWNSTREAM, partSID, trackID), stat1) // there should be bytes reported so that stats are sent
+
+	// flush
+	fixture.flush()
+
+	stat2 := &livekit.AnalyticsStat{
+		Streams: []*livekit.AnalyticsStream{
+			{
+				PrimaryBytes:   2,
+				PrimaryPackets: 2,
+				PacketsLost:    4,
+			},
+		},
+	}
+	fixture.sut.TrackStats(telemetry.StatsKeyForData(livekit.StreamType_DOWNSTREAM, partSID, trackID), stat2)
+
+	// flush
+	fixture.flush()
+
+	// test
+	require.Equal(t, 2, fixture.analytics.SendStatsCallCount()) // 2 calls to fixture.sut.FlushStats()
+	_, stats := fixture.analytics.SendStatsArgsForCall(0)
+	require.Equal(t, 1, len(stats))
+	require.Equal(t, livekit.StreamType_DOWNSTREAM, stats[0].Kind)
+	require.Equal(t, 1, int(stats[0].Streams[0].PacketsLost)) // see pkts1
+
+	_, stats = fixture.analytics.SendStatsArgsForCall(1)
+	require.Equal(t, 1, len(stats))
+	require.Equal(t, livekit.StreamType_DOWNSTREAM, stats[0].Kind)
+	require.Equal(t, 4, int(stats[0].Streams[0].PacketsLost)) // delta loss should be sent as is
+}
+
+func Test_OnDownStreamRTCP_SeveralTracks(t *testing.T) {
+	fixture := createFixture()
+
+	// prepare
+	room := &livekit.Room{}
+	partSID := livekit.ParticipantID("part1")
+	participantInfo := &livekit.ParticipantInfo{Sid: string(partSID)}
+	fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, nil, nil, true)
+
+	// do
+	trackID1 := livekit.TrackID("trackID1")
+	trackID2 := livekit.TrackID("trackID2")
+	stat1 := &livekit.AnalyticsStat{
+		Streams: []*livekit.AnalyticsStream{
+			{
+				PrimaryBytes:   1,
+				PrimaryPackets: 1,
+			},
+		},
+	}
+	fixture.sut.TrackStats(telemetry.StatsKeyForData(livekit.StreamType_DOWNSTREAM, partSID, trackID1), stat1) // there should be bytes reported so that stats are sent
+
+	stat2 := &livekit.AnalyticsStat{
+		Streams: []*livekit.AnalyticsStream{
+			{
+				PrimaryBytes:   2,
+				PrimaryPackets: 2,
+				Nacks:          1,
+			},
+		},
+	}
+	fixture.sut.TrackStats(telemetry.StatsKeyForData(livekit.StreamType_DOWNSTREAM, partSID, trackID1), stat2)
+
+	stat3 := &livekit.AnalyticsStat{
+		Streams: []*livekit.AnalyticsStream{
+			{
+				PrimaryBytes:   3,
+				PrimaryPackets: 3,
+				Firs:           1,
+			},
+		},
+	}
+	fixture.sut.TrackStats(telemetry.StatsKeyForData(livekit.StreamType_DOWNSTREAM, partSID, trackID2), stat3)
+
+	// flush
+	fixture.flush()
+
+	// test
+	require.Equal(t, 1, fixture.analytics.SendStatsCallCount())
+	_, stats := fixture.analytics.SendStatsArgsForCall(0)
+	require.Equal(t, 2, len(stats))
+
+	found1 := false
+	found2 := false
+	for _, sentStat := range stats {
+		if livekit.TrackID(sentStat.TrackId) == trackID1 {
+			found1 = true
+			require.Equal(t, livekit.StreamType_DOWNSTREAM, sentStat.Kind)
+			require.Equal(t, 1, int(sentStat.Streams[0].Nacks)) // see pkts1 above
+		} else if livekit.TrackID(sentStat.TrackId) == trackID2 {
+			found2 = true
+			require.Equal(t, livekit.StreamType_DOWNSTREAM, sentStat.Kind)
+			require.Equal(t, 1, int(sentStat.Streams[0].Firs)) // see pkts2 above
+		}
+	}
+	require.True(t, found1)
+	require.True(t, found2)
+}
+
+func Test_OnUpstreamStat(t *testing.T) {
+	fixture := createFixture()
+
+	// prepare
+	room := &livekit.Room{}
+	partSID := livekit.ParticipantID("part1")
+	participantInfo := &livekit.ParticipantInfo{Sid: string(partSID)}
+	fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, nil, nil, true)
+
+	// do
+	stat1 := &livekit.AnalyticsStat{
+		Streams: []*livekit.AnalyticsStream{
+			{
+				PrimaryBytes:   1,
+				PrimaryPackets: 1,
+				PacketsLost:    3,
+				Nacks:          1,
+				Plis:           1,
+				Firs:           1,
+				Rtt:            13,
+				Jitter:         5,
+			},
+		},
+	}
+	trackID := livekit.TrackID("trackID")
+
+	fixture.sut.TrackStats(telemetry.StatsKeyForData(livekit.StreamType_UPSTREAM, partSID, trackID), stat1)
+
+	stat2 := &livekit.AnalyticsStat{
+		Streams: []*livekit.AnalyticsStream{
+			{
+				PrimaryBytes:   2,
+				PrimaryPackets: 2,
+				PacketsLost:    4,
+				Nacks:          1,
+				Plis:           1,
+				Firs:           1,
+				Rtt:            33,
+				Jitter:         2,
+			},
+		},
+	}
+	fixture.sut.TrackStats(telemetry.StatsKeyForData(livekit.StreamType_UPSTREAM, partSID, trackID), stat2)
+
+	// flush
+	fixture.flush()
+
+	// test
+	require.Equal(t, 1, fixture.analytics.SendStatsCallCount())
+	_, stats := fixture.analytics.SendStatsArgsForCall(0)
+	require.Equal(t, 1, len(stats))
+	require.Equal(t, livekit.StreamType_UPSTREAM, stats[0].Kind)
+	require.Equal(t, 2, int(stats[0].Streams[0].Nacks))
+	require.Equal(t, 2, int(stats[0].Streams[0].Plis))
+	require.Equal(t, 2, int(stats[0].Streams[0].Firs))
+	require.Equal(t, 33, int(stats[0].Streams[0].Rtt))        // max of RTT
+	require.Equal(t, 5, int(stats[0].Streams[0].Jitter))      // max of jitter
+	require.Equal(t, 7, int(stats[0].Streams[0].PacketsLost)) // coalesced delta packet losses
+	require.Equal(t, string(trackID), stats[0].TrackId)
+}
+
+func Test_OnUpstreamRTCP_SeveralTracks(t *testing.T) {
+	fixture := createFixture()
+
+	// prepare
+	room := &livekit.Room{}
+	partSID := livekit.ParticipantID("part1")
+	identity := livekit.ParticipantIdentity("part1Identity")
+	participantInfo := &livekit.ParticipantInfo{Sid: string(partSID), Identity: string(identity)}
+	fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, nil, nil, true)
+
+	// there should be bytes reported so that stats are sent
+	totalBytes := 1
+	totalPackets := 1
+	trackID1 := livekit.TrackID("trackID1")
+	trackID2 := livekit.TrackID("trackID2")
+
+	stat1 := &livekit.AnalyticsStat{
+		Streams: []*livekit.AnalyticsStream{
+			{
+				PrimaryBytes:   uint64(totalBytes),
+				PrimaryPackets: uint32(totalPackets),
+			},
+		},
+	}
+	fixture.sut.TrackStats(telemetry.StatsKeyForData(livekit.StreamType_UPSTREAM, partSID, trackID1), stat1)
+	fixture.sut.TrackStats(telemetry.StatsKeyForData(livekit.StreamType_UPSTREAM, partSID, trackID2), stat1) // using same buffer is not correct but for test it is fine
+
+	// do
+	totalBytes++
+	totalPackets++
+	stat2 := &livekit.AnalyticsStat{
+		Streams: []*livekit.AnalyticsStream{
+			{
+				PrimaryBytes:   uint64(totalBytes),
+				PrimaryPackets: uint32(totalPackets),
+				Nacks:          1,
+			},
+		},
+	}
+	fixture.sut.TrackStats(telemetry.StatsKeyForData(livekit.StreamType_UPSTREAM, partSID, trackID1), stat2)
+
+	stat3 := &livekit.AnalyticsStat{
+		Streams: []*livekit.AnalyticsStream{
+			{
+				PrimaryBytes:   uint64(totalBytes),
+				PrimaryPackets: uint32(totalPackets),
+				Firs:           1,
+			},
+		},
+	}
+	fixture.sut.TrackStats(telemetry.StatsKeyForData(livekit.StreamType_UPSTREAM, partSID, trackID2), stat3)
+
+	// flush
+	fixture.flush()
+
+	// test
+	require.Equal(t, 1, fixture.analytics.SendStatsCallCount())
+	_, stats := fixture.analytics.SendStatsArgsForCall(0)
+	require.Equal(t, 2, len(stats))
+
+	found1 := false
+	found2 := false
+	for _, sentStat := range stats {
+		if livekit.TrackID(sentStat.TrackId) == trackID1 {
+			found1 = true
+			require.Equal(t, livekit.StreamType_UPSTREAM, sentStat.Kind)
+			require.Equal(t, 1, int(sentStat.Streams[0].Nacks)) // see pkts1 above
+		} else if livekit.TrackID(sentStat.TrackId) == trackID2 {
+			found2 = true
+			require.Equal(t, livekit.StreamType_UPSTREAM, sentStat.Kind)
+			require.Equal(t, 1, int(sentStat.Streams[0].Firs)) // see pkts2 above
+		}
+		require.Equal(t, 3, int(sentStat.Streams[0].PrimaryBytes))
+		require.Equal(t, 3, int(sentStat.Streams[0].PrimaryPackets))
+	}
+	require.True(t, found1)
+	require.True(t, found2)
+
+	// remove 1 track - track stats were flushed above, so no more calls to SendStats
+	fixture.sut.TrackUnpublished(context.Background(), partSID, identity, &livekit.TrackInfo{Sid: string(trackID2)}, true)
+
+	// flush
+	fixture.flush()
+
+	require.Equal(t, 1, fixture.analytics.SendStatsCallCount())
+}
+
+func Test_AnalyticsSentWhenParticipantLeaves(t *testing.T) {
+	fixture := createFixture()
+
+	// prepare
+	room := &livekit.Room{}
+	partSID := "part1"
+	participantInfo := &livekit.ParticipantInfo{Sid: partSID}
+	fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, nil, nil, true)
+
+	// do
+	fixture.sut.ParticipantLeft(context.Background(), room, participantInfo, true)
+
+	// should not be called if there are no track stats
+	time.Sleep(time.Millisecond * 500)
+	require.Equal(t, 0, fixture.analytics.SendStatsCallCount())
+}
+
+func Test_AddUpTrack(t *testing.T) {
+	fixture := createFixture()
+
+	// prepare
+	room := &livekit.Room{}
+	partSID := livekit.ParticipantID("part1")
+	participantInfo := &livekit.ParticipantInfo{Sid: string(partSID)}
+	fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, nil, nil, true)
+
+	// do
+	var totalBytes uint64 = 3
+	var totalPackets uint32 = 3
+
+	stat := &livekit.AnalyticsStat{
+		Streams: []*livekit.AnalyticsStream{
+			{
+				PrimaryBytes:   totalBytes,
+				PrimaryPackets: totalPackets,
+			},
+		},
+	}
+	trackID := livekit.TrackID("trackID")
+	fixture.sut.TrackStats(telemetry.StatsKeyForData(livekit.StreamType_UPSTREAM, partSID, trackID), stat)
+
+	// flush
+	fixture.flush()
+
+	// test
+	require.Equal(t, 1, fixture.analytics.SendStatsCallCount())
+	_, stats := fixture.analytics.SendStatsArgsForCall(0)
+	require.Equal(t, 1, len(stats))
+	require.Equal(t, livekit.StreamType_UPSTREAM, stats[0].Kind)
+	require.Equal(t, totalBytes, stats[0].Streams[0].PrimaryBytes)
+	require.Equal(t, totalPackets, stats[0].Streams[0].PrimaryPackets)
+	require.Equal(t, string(trackID), stats[0].TrackId)
+}
+
+func Test_AddUpTrack_SeveralBuffers_Simulcast(t *testing.T) {
+	fixture := createFixture()
+
+	// prepare
+	room := &livekit.Room{}
+	partSID := livekit.ParticipantID("part1")
+	participantInfo := &livekit.ParticipantInfo{Sid: string(partSID)}
+	fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, nil, nil, true)
+
+	// do
+	trackID := livekit.TrackID("trackID")
+	stat1 := &livekit.AnalyticsStat{
+		Streams: []*livekit.AnalyticsStream{
+			{
+				PrimaryBytes:   1,
+				PrimaryPackets: 1,
+			},
+			{
+				PrimaryBytes:   2,
+				PrimaryPackets: 2,
+			},
+		},
+	}
+	fixture.sut.TrackStats(telemetry.StatsKeyForData(livekit.StreamType_UPSTREAM, partSID, trackID), stat1)
+
+	// flush
+	fixture.flush()
+
+	// test
+	require.Equal(t, 1, fixture.analytics.SendStatsCallCount())
+	_, stats := fixture.analytics.SendStatsArgsForCall(0)
+	require.Equal(t, 1, len(stats))
+	require.Equal(t, livekit.StreamType_UPSTREAM, stats[0].Kind)
+	// should be a consolidated stream
+	require.Equal(t, stat1.Streams[0].PrimaryBytes+stat1.Streams[1].PrimaryBytes, stats[0].Streams[0].PrimaryBytes)
+	require.Equal(t, stat1.Streams[0].PrimaryPackets+stat1.Streams[1].PrimaryPackets, stats[0].Streams[0].PrimaryPackets)
+	require.Equal(t, string(trackID), stats[0].TrackId)
+}
+
+func Test_BothDownstreamAndUpstreamStatsAreSentTogether(t *testing.T) {
+	fixture := createFixture()
+
+	// prepare
+	room := &livekit.Room{}
+	partSID := livekit.ParticipantID("part1")
+	participantInfo := &livekit.ParticipantInfo{Sid: string(partSID)}
+	fixture.sut.ParticipantJoined(context.Background(), room, participantInfo, nil, nil, true)
+
+	// do
+	// upstream bytes
+	stat1 := &livekit.AnalyticsStat{
+		Streams: []*livekit.AnalyticsStream{
+			{
+				PrimaryBytes:   3,
+				PrimaryPackets: 3,
+			},
+		},
+	}
+	fixture.sut.TrackStats(telemetry.StatsKeyForData(livekit.StreamType_UPSTREAM, partSID, "trackID"), stat1)
+	// downstream bytes
+	stat2 := &livekit.AnalyticsStat{
+		Streams: []*livekit.AnalyticsStream{
+			{
+				PrimaryBytes:   1,
+				PrimaryPackets: 1,
+			},
+		},
+	}
+	fixture.sut.TrackStats(telemetry.StatsKeyForData(livekit.StreamType_DOWNSTREAM, partSID, "trackID1"), stat2)
+
+	// flush
+	fixture.flush()
+
+	// test
+	require.Equal(t, 1, fixture.analytics.SendStatsCallCount())
+	_, stats := fixture.analytics.SendStatsArgsForCall(0)
+	require.Equal(t, 2, len(stats))
+	require.Equal(t, livekit.StreamType_UPSTREAM, stats[0].Kind)
+	require.Equal(t, livekit.StreamType_DOWNSTREAM, stats[1].Kind)
+}
+
+func (f *telemetryServiceFixture) flush() {
+	time.Sleep(time.Millisecond * 500)
+	f.sut.FlushStats()
+}

livekit-server/pkg/telemetry/statsconn.go

@@ -0,0 +1,130 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package telemetry
+
+import (
+	"net"
+
+	"github.com/pion/turn/v4"
+
+	"github.com/livekit/livekit-server/pkg/telemetry/prometheus"
+)
+
+type Listener struct {
+	net.Listener
+}
+
+func NewListener(l net.Listener) *Listener {
+	return &Listener{Listener: l}
+}
+
+func (l *Listener) Accept() (net.Conn, error) {
+	conn, err := l.Listener.Accept()
+	if err != nil {
+		return nil, err
+	}
+
+	return NewConn(conn, prometheus.Incoming), nil
+}
+
+type Conn struct {
+	net.Conn
+	direction prometheus.Direction
+}
+
+func NewConn(c net.Conn, direction prometheus.Direction) *Conn {
+	prometheus.AddConnection(direction)
+	return &Conn{Conn: c, direction: direction}
+}
+
+func (c *Conn) Read(b []byte) (n int, err error) {
+	n, err = c.Conn.Read(b)
+	if n > 0 {
+		prometheus.IncrementBytes(prometheus.Incoming, uint64(n), false)
+	}
+	return
+}
+
+func (c *Conn) Write(b []byte) (n int, err error) {
+	n, err = c.Conn.Write(b)
+	if n > 0 {
+		prometheus.IncrementBytes(prometheus.Outgoing, uint64(n), false)
+	}
+	return
+}
+
+func (c *Conn) Close() error {
+	prometheus.SubConnection(c.direction)
+	return c.Conn.Close()
+}
+
+type PacketConn struct {
+	net.PacketConn
+	direction prometheus.Direction
+}
+
+func NewPacketConn(c net.PacketConn, direction prometheus.Direction) *PacketConn {
+	prometheus.AddConnection(direction)
+	return &PacketConn{PacketConn: c, direction: direction}
+}
+
+func (c *PacketConn) ReadFrom(p []byte) (n int, addr net.Addr, err error) {
+	n, addr, err = c.PacketConn.ReadFrom(p)
+	if n > 0 {
+		prometheus.IncrementBytes(prometheus.Incoming, uint64(n), false)
+		prometheus.IncrementPackets(prometheus.Incoming, 1, false)
+	}
+	return
+}
+
+func (c *PacketConn) WriteTo(p []byte, addr net.Addr) (n int, err error) {
+	n, err = c.PacketConn.WriteTo(p, addr)
+	if n > 0 {
+		prometheus.IncrementBytes(prometheus.Outgoing, uint64(n), false)
+		prometheus.IncrementPackets(prometheus.Outgoing, 1, false)
+	}
+	return
+}
+
+func (c *PacketConn) Close() error {
+	prometheus.SubConnection(c.direction)
+	return c.PacketConn.Close()
+}
+
+type RelayAddressGenerator struct {
+	turn.RelayAddressGenerator
+}
+
+func NewRelayAddressGenerator(g turn.RelayAddressGenerator) *RelayAddressGenerator {
+	return &RelayAddressGenerator{RelayAddressGenerator: g}
+}
+
+func (g *RelayAddressGenerator) AllocatePacketConn(network string, requestedPort int) (net.PacketConn, net.Addr, error) {
+	conn, addr, err := g.RelayAddressGenerator.AllocatePacketConn(network, requestedPort)
+	if err != nil {
+		return nil, addr, err
+	}
+
+	return NewPacketConn(conn, prometheus.Outgoing), addr, err
+}
+
+func (g *RelayAddressGenerator) AllocateConn(network string, requestedPort int) (net.Conn, net.Addr, error) {
+	conn, addr, err := g.RelayAddressGenerator.AllocateConn(network, requestedPort)
+	if err != nil {
+		return nil, addr, err
+	}
+
+	return NewConn(conn, prometheus.Outgoing), addr, err
+}

livekit-server/pkg/telemetry/statsworker.go

@@ -0,0 +1,293 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package telemetry
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	"google.golang.org/protobuf/types/known/timestamppb"
+
+	"github.com/livekit/livekit-server/pkg/utils"
+	"github.com/livekit/protocol/livekit"
+	"github.com/livekit/protocol/logger"
+	protoutils "github.com/livekit/protocol/utils"
+)
+
+// StatsWorker handles participant stats
+type StatsWorker struct {
+	next *StatsWorker
+
+	ctx                 context.Context
+	t                   TelemetryService
+	roomID              livekit.RoomID
+	roomName            livekit.RoomName
+	participantID       livekit.ParticipantID
+	participantIdentity livekit.ParticipantIdentity
+	isConnected         bool
+
+	lock             sync.RWMutex
+	outgoingPerTrack map[livekit.TrackID][]*livekit.AnalyticsStat
+	incomingPerTrack map[livekit.TrackID][]*livekit.AnalyticsStat
+	closedAt         time.Time
+}
+
+func newStatsWorker(
+	ctx context.Context,
+	t TelemetryService,
+	roomID livekit.RoomID,
+	roomName livekit.RoomName,
+	participantID livekit.ParticipantID,
+	identity livekit.ParticipantIdentity,
+) *StatsWorker {
+	s := &StatsWorker{
+		ctx:                 ctx,
+		t:                   t,
+		roomID:              roomID,
+		roomName:            roomName,
+		participantID:       participantID,
+		participantIdentity: identity,
+		outgoingPerTrack:    make(map[livekit.TrackID][]*livekit.AnalyticsStat),
+		incomingPerTrack:    make(map[livekit.TrackID][]*livekit.AnalyticsStat),
+	}
+	return s
+}
+
+func (s *StatsWorker) OnTrackStat(trackID livekit.TrackID, direction livekit.StreamType, stat *livekit.AnalyticsStat) {
+	s.lock.Lock()
+	if direction == livekit.StreamType_DOWNSTREAM {
+		s.outgoingPerTrack[trackID] = append(s.outgoingPerTrack[trackID], stat)
+	} else {
+		s.incomingPerTrack[trackID] = append(s.incomingPerTrack[trackID], stat)
+	}
+	s.lock.Unlock()
+}
+
+func (s *StatsWorker) ParticipantID() livekit.ParticipantID {
+	return s.participantID
+}
+
+func (s *StatsWorker) SetConnected() {
+	s.lock.Lock()
+	s.isConnected = true
+	s.lock.Unlock()
+}
+
+func (s *StatsWorker) IsConnected() bool {
+	s.lock.RLock()
+	defer s.lock.RUnlock()
+
+	return s.isConnected
+}
+
+func (s *StatsWorker) Flush(now time.Time) bool {
+	ts := timestamppb.New(now)
+
+	s.lock.Lock()
+	stats := make([]*livekit.AnalyticsStat, 0, len(s.incomingPerTrack)+len(s.outgoingPerTrack))
+
+	incomingPerTrack := s.incomingPerTrack
+	s.incomingPerTrack = make(map[livekit.TrackID][]*livekit.AnalyticsStat)
+
+	outgoingPerTrack := s.outgoingPerTrack
+	s.outgoingPerTrack = make(map[livekit.TrackID][]*livekit.AnalyticsStat)
+
+	closed := !s.closedAt.IsZero() && now.Sub(s.closedAt) > workerCleanupWait
+	s.lock.Unlock()
+
+	stats = s.collectStats(ts, livekit.StreamType_UPSTREAM, incomingPerTrack, stats)
+	stats = s.collectStats(ts, livekit.StreamType_DOWNSTREAM, outgoingPerTrack, stats)
+	if len(stats) > 0 {
+		s.t.SendStats(s.ctx, stats)
+	}
+
+	return closed
+}
+
+func (s *StatsWorker) Close() bool {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	ok := s.closedAt.IsZero()
+	if ok {
+		s.closedAt = time.Now()
+	}
+	return ok
+}
+
+func (s *StatsWorker) Closed() bool {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+	return !s.closedAt.IsZero()
+}
+
+func (s *StatsWorker) collectStats(
+	ts *timestamppb.Timestamp,
+	streamType livekit.StreamType,
+	perTrack map[livekit.TrackID][]*livekit.AnalyticsStat,
+	stats []*livekit.AnalyticsStat,
+) []*livekit.AnalyticsStat {
+	for trackID, analyticsStats := range perTrack {
+		coalesced := coalesce(analyticsStats)
+		if coalesced == nil {
+			continue
+		}
+
+		coalesced.TimeStamp = ts
+		coalesced.TrackId = string(trackID)
+		coalesced.Kind = streamType
+		coalesced.RoomId = string(s.roomID)
+		coalesced.ParticipantId = string(s.participantID)
+		coalesced.RoomName = string(s.roomName)
+		stats = append(stats, coalesced)
+	}
+	return stats
+}
+
+// -------------------------------------------------------------------------
+
+// create a single stream and single video layer post aggregation
+func coalesce(stats []*livekit.AnalyticsStat) *livekit.AnalyticsStat {
+	if len(stats) == 0 {
+		return nil
+	}
+
+	// find aggregates across streams
+	startTime := time.Time{}
+	endTime := time.Time{}
+	scoreSum := float32(0.0) // used for average
+	minScore := float32(0.0) // min score in batched stats
+	var scores []float32     // used for median
+	maxRtt := uint32(0)
+	maxJitter := uint32(0)
+	coalescedVideoLayers := make(map[int32]*livekit.AnalyticsVideoLayer)
+	coalescedStream := &livekit.AnalyticsStream{}
+	for _, stat := range stats {
+		if !isValid(stat) {
+			logger.Warnw("telemetry skipping invalid stat", nil, "stat", stat)
+			continue
+		}
+
+		// only consider non-zero scores
+		if stat.Score > 0 {
+			if minScore == 0 {
+				minScore = stat.Score
+			} else if stat.Score < minScore {
+				minScore = stat.Score
+			}
+			scoreSum += stat.Score
+			scores = append(scores, stat.Score)
+		}
+
+		for _, analyticsStream := range stat.Streams {
+			start := analyticsStream.StartTime.AsTime()
+			if startTime.IsZero() || startTime.After(start) {
+				startTime = start
+			}
+
+			end := analyticsStream.EndTime.AsTime()
+			if endTime.IsZero() || endTime.Before(end) {
+				endTime = end
+			}
+
+			if analyticsStream.Rtt > maxRtt {
+				maxRtt = analyticsStream.Rtt
+			}
+
+			if analyticsStream.Jitter > maxJitter {
+				maxJitter = analyticsStream.Jitter
+			}
+
+			coalescedStream.PrimaryPackets += analyticsStream.PrimaryPackets
+			coalescedStream.PrimaryBytes += analyticsStream.PrimaryBytes
+			coalescedStream.RetransmitPackets += analyticsStream.RetransmitPackets
+			coalescedStream.RetransmitBytes += analyticsStream.RetransmitBytes
+			coalescedStream.PaddingPackets += analyticsStream.PaddingPackets
+			coalescedStream.PaddingBytes += analyticsStream.PaddingBytes
+			coalescedStream.PacketsLost += analyticsStream.PacketsLost
+			coalescedStream.PacketsOutOfOrder += analyticsStream.PacketsOutOfOrder
+			coalescedStream.Frames += analyticsStream.Frames
+			coalescedStream.Nacks += analyticsStream.Nacks
+			coalescedStream.Plis += analyticsStream.Plis
+			coalescedStream.Firs += analyticsStream.Firs
+
+			for _, videoLayer := range analyticsStream.VideoLayers {
+				coalescedVideoLayer := coalescedVideoLayers[videoLayer.Layer]
+				if coalescedVideoLayer == nil {
+					coalescedVideoLayer = protoutils.CloneProto(videoLayer)
+					coalescedVideoLayers[videoLayer.Layer] = coalescedVideoLayer
+				} else {
+					coalescedVideoLayer.Packets += videoLayer.Packets
+					coalescedVideoLayer.Bytes += videoLayer.Bytes
+					coalescedVideoLayer.Frames += videoLayer.Frames
+				}
+			}
+		}
+	}
+	coalescedStream.StartTime = timestamppb.New(startTime)
+	coalescedStream.EndTime = timestamppb.New(endTime)
+	coalescedStream.Rtt = maxRtt
+	coalescedStream.Jitter = maxJitter
+
+	// whittle it down to one video layer, just the max available layer
+	maxVideoLayer := int32(-1)
+	for _, coalescedVideoLayer := range coalescedVideoLayers {
+		if maxVideoLayer == -1 || maxVideoLayer < coalescedVideoLayer.Layer {
+			maxVideoLayer = coalescedVideoLayer.Layer
+			coalescedStream.VideoLayers = []*livekit.AnalyticsVideoLayer{coalescedVideoLayer}
+		}
+	}
+
+	stat := &livekit.AnalyticsStat{
+		MinScore:    minScore,
+		MedianScore: utils.MedianFloat32(scores),
+		Streams:     []*livekit.AnalyticsStream{coalescedStream},
+	}
+	numScores := len(scores)
+	if numScores > 0 {
+		stat.Score = scoreSum / float32(numScores)
+	}
+	return stat
+}
+
+func isValid(stat *livekit.AnalyticsStat) bool {
+	for _, analyticsStream := range stat.Streams {
+		if int32(analyticsStream.PrimaryPackets) < 0 ||
+			int64(analyticsStream.PrimaryBytes) < 0 ||
+			int32(analyticsStream.RetransmitPackets) < 0 ||
+			int64(analyticsStream.RetransmitBytes) < 0 ||
+			int32(analyticsStream.PaddingPackets) < 0 ||
+			int64(analyticsStream.PaddingBytes) < 0 ||
+			int32(analyticsStream.PacketsLost) < 0 ||
+			int32(analyticsStream.PacketsOutOfOrder) < 0 ||
+			int32(analyticsStream.Frames) < 0 ||
+			int32(analyticsStream.Nacks) < 0 ||
+			int32(analyticsStream.Plis) < 0 ||
+			int32(analyticsStream.Firs) < 0 {
+			return false
+		}
+
+		for _, videoLayer := range analyticsStream.VideoLayers {
+			if int32(videoLayer.Packets) < 0 ||
+				int64(videoLayer.Bytes) < 0 ||
+				int32(videoLayer.Frames) < 0 {
+				return false
+			}
+		}
+	}
+
+	return true
+}

livekit-server/pkg/telemetry/telemetryfakes/fake_analytics_service.go

@@ -0,0 +1,167 @@
+// Code generated by counterfeiter. DO NOT EDIT.
+package telemetryfakes
+
+import (
+	"context"
+	"sync"
+
+	"github.com/livekit/livekit-server/pkg/telemetry"
+	"github.com/livekit/protocol/livekit"
+)
+
+type FakeAnalyticsService struct {
+	SendEventStub        func(context.Context, *livekit.AnalyticsEvent)
+	sendEventMutex       sync.RWMutex
+	sendEventArgsForCall []struct {
+		arg1 context.Context
+		arg2 *livekit.AnalyticsEvent
+	}
+	SendNodeRoomStatesStub        func(context.Context, *livekit.AnalyticsNodeRooms)
+	sendNodeRoomStatesMutex       sync.RWMutex
+	sendNodeRoomStatesArgsForCall []struct {
+		arg1 context.Context
+		arg2 *livekit.AnalyticsNodeRooms
+	}
+	SendStatsStub        func(context.Context, []*livekit.AnalyticsStat)
+	sendStatsMutex       sync.RWMutex
+	sendStatsArgsForCall []struct {
+		arg1 context.Context
+		arg2 []*livekit.AnalyticsStat
+	}
+	invocations      map[string][][]interface{}
+	invocationsMutex sync.RWMutex
+}
+
+func (fake *FakeAnalyticsService) SendEvent(arg1 context.Context, arg2 *livekit.AnalyticsEvent) {
+	fake.sendEventMutex.Lock()
+	fake.sendEventArgsForCall = append(fake.sendEventArgsForCall, struct {
+		arg1 context.Context
+		arg2 *livekit.AnalyticsEvent
+	}{arg1, arg2})
+	stub := fake.SendEventStub
+	fake.recordInvocation("SendEvent", []interface{}{arg1, arg2})
+	fake.sendEventMutex.Unlock()
+	if stub != nil {
+		fake.SendEventStub(arg1, arg2)
+	}
+}
+
+func (fake *FakeAnalyticsService) SendEventCallCount() int {
+	fake.sendEventMutex.RLock()
+	defer fake.sendEventMutex.RUnlock()
+	return len(fake.sendEventArgsForCall)
+}
+
+func (fake *FakeAnalyticsService) SendEventCalls(stub func(context.Context, *livekit.AnalyticsEvent)) {
+	fake.sendEventMutex.Lock()
+	defer fake.sendEventMutex.Unlock()
+	fake.SendEventStub = stub
+}
+
+func (fake *FakeAnalyticsService) SendEventArgsForCall(i int) (context.Context, *livekit.AnalyticsEvent) {
+	fake.sendEventMutex.RLock()
+	defer fake.sendEventMutex.RUnlock()
+	argsForCall := fake.sendEventArgsForCall[i]
+	return argsForCall.arg1, argsForCall.arg2
+}
+
+func (fake *FakeAnalyticsService) SendNodeRoomStates(arg1 context.Context, arg2 *livekit.AnalyticsNodeRooms) {
+	fake.sendNodeRoomStatesMutex.Lock()
+	fake.sendNodeRoomStatesArgsForCall = append(fake.sendNodeRoomStatesArgsForCall, struct {
+		arg1 context.Context
+		arg2 *livekit.AnalyticsNodeRooms
+	}{arg1, arg2})
+	stub := fake.SendNodeRoomStatesStub
+	fake.recordInvocation("SendNodeRoomStates", []interface{}{arg1, arg2})
+	fake.sendNodeRoomStatesMutex.Unlock()
+	if stub != nil {
+		fake.SendNodeRoomStatesStub(arg1, arg2)
+	}
+}
+
+func (fake *FakeAnalyticsService) SendNodeRoomStatesCallCount() int {
+	fake.sendNodeRoomStatesMutex.RLock()
+	defer fake.sendNodeRoomStatesMutex.RUnlock()
+	return len(fake.sendNodeRoomStatesArgsForCall)
+}
+
+func (fake *FakeAnalyticsService) SendNodeRoomStatesCalls(stub func(context.Context, *livekit.AnalyticsNodeRooms)) {
+	fake.sendNodeRoomStatesMutex.Lock()
+	defer fake.sendNodeRoomStatesMutex.Unlock()
+	fake.SendNodeRoomStatesStub = stub
+}
+
+func (fake *FakeAnalyticsService) SendNodeRoomStatesArgsForCall(i int) (context.Context, *livekit.AnalyticsNodeRooms) {
+	fake.sendNodeRoomStatesMutex.RLock()
+	defer fake.sendNodeRoomStatesMutex.RUnlock()
+	argsForCall := fake.sendNodeRoomStatesArgsForCall[i]
+	return argsForCall.arg1, argsForCall.arg2
+}
+
+func (fake *FakeAnalyticsService) SendStats(arg1 context.Context, arg2 []*livekit.AnalyticsStat) {
+	var arg2Copy []*livekit.AnalyticsStat
+	if arg2 != nil {
+		arg2Copy = make([]*livekit.AnalyticsStat, len(arg2))
+		copy(arg2Copy, arg2)
+	}
+	fake.sendStatsMutex.Lock()
+	fake.sendStatsArgsForCall = append(fake.sendStatsArgsForCall, struct {
+		arg1 context.Context
+		arg2 []*livekit.AnalyticsStat
+	}{arg1, arg2Copy})
+	stub := fake.SendStatsStub
+	fake.recordInvocation("SendStats", []interface{}{arg1, arg2Copy})
+	fake.sendStatsMutex.Unlock()
+	if stub != nil {
+		fake.SendStatsStub(arg1, arg2)
+	}
+}
+
+func (fake *FakeAnalyticsService) SendStatsCallCount() int {
+	fake.sendStatsMutex.RLock()
+	defer fake.sendStatsMutex.RUnlock()
+	return len(fake.sendStatsArgsForCall)
+}
+
+func (fake *FakeAnalyticsService) SendStatsCalls(stub func(context.Context, []*livekit.AnalyticsStat)) {
+	fake.sendStatsMutex.Lock()
+	defer fake.sendStatsMutex.Unlock()
+	fake.SendStatsStub = stub
+}
+
+func (fake *FakeAnalyticsService) SendStatsArgsForCall(i int) (context.Context, []*livekit.AnalyticsStat) {
+	fake.sendStatsMutex.RLock()
+	defer fake.sendStatsMutex.RUnlock()
+	argsForCall := fake.sendStatsArgsForCall[i]
+	return argsForCall.arg1, argsForCall.arg2
+}
+
+func (fake *FakeAnalyticsService) Invocations() map[string][][]interface{} {
+	fake.invocationsMutex.RLock()
+	defer fake.invocationsMutex.RUnlock()
+	fake.sendEventMutex.RLock()
+	defer fake.sendEventMutex.RUnlock()
+	fake.sendNodeRoomStatesMutex.RLock()
+	defer fake.sendNodeRoomStatesMutex.RUnlock()
+	fake.sendStatsMutex.RLock()
+	defer fake.sendStatsMutex.RUnlock()
+	copiedInvocations := map[string][][]interface{}{}
+	for key, value := range fake.invocations {
+		copiedInvocations[key] = value
+	}
+	return copiedInvocations
+}
+
+func (fake *FakeAnalyticsService) recordInvocation(key string, args []interface{}) {
+	fake.invocationsMutex.Lock()
+	defer fake.invocationsMutex.Unlock()
+	if fake.invocations == nil {
+		fake.invocations = map[string][][]interface{}{}
+	}
+	if fake.invocations[key] == nil {
+		fake.invocations[key] = [][]interface{}{}
+	}
+	fake.invocations[key] = append(fake.invocations[key], args)
+}
+
+var _ telemetry.AnalyticsService = new(FakeAnalyticsService)

livekit-server/pkg/telemetry/telemetryservice.go

@@ -0,0 +1,246 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package telemetry
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	"github.com/livekit/livekit-server/pkg/sfu/mime"
+	"github.com/livekit/livekit-server/pkg/utils"
+	"github.com/livekit/protocol/livekit"
+	"github.com/livekit/protocol/logger"
+	"github.com/livekit/protocol/webhook"
+)
+
+//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -generate
+
+//counterfeiter:generate . TelemetryService
+type TelemetryService interface {
+	// TrackStats is called periodically for each track in both directions (published/subscribed)
+	TrackStats(key StatsKey, stat *livekit.AnalyticsStat)
+
+	// events
+	RoomStarted(ctx context.Context, room *livekit.Room)
+	RoomEnded(ctx context.Context, room *livekit.Room)
+	// ParticipantJoined - a participant establishes signal connection to a room
+	ParticipantJoined(ctx context.Context, room *livekit.Room, participant *livekit.ParticipantInfo, clientInfo *livekit.ClientInfo, clientMeta *livekit.AnalyticsClientMeta, shouldSendEvent bool)
+	// ParticipantActive - a participant establishes media connection
+	ParticipantActive(ctx context.Context, room *livekit.Room, participant *livekit.ParticipantInfo, clientMeta *livekit.AnalyticsClientMeta, isMigration bool)
+	// ParticipantResumed - there has been an ICE restart or connection resume attempt, and we've received their signal connection
+	ParticipantResumed(ctx context.Context, room *livekit.Room, participant *livekit.ParticipantInfo, nodeID livekit.NodeID, reason livekit.ReconnectReason)
+	// ParticipantLeft - the participant leaves the room, only sent if ParticipantActive has been called before
+	ParticipantLeft(ctx context.Context, room *livekit.Room, participant *livekit.ParticipantInfo, shouldSendEvent bool)
+	// TrackPublishRequested - a publication attempt has been received
+	TrackPublishRequested(ctx context.Context, participantID livekit.ParticipantID, identity livekit.ParticipantIdentity, track *livekit.TrackInfo)
+	// TrackPublished - a publication attempt has been successful
+	TrackPublished(ctx context.Context, participantID livekit.ParticipantID, identity livekit.ParticipantIdentity, track *livekit.TrackInfo)
+	// TrackUnpublished - a participant unpublished a track
+	TrackUnpublished(ctx context.Context, participantID livekit.ParticipantID, identity livekit.ParticipantIdentity, track *livekit.TrackInfo, shouldSendEvent bool)
+	// TrackSubscribeRequested - a participant requested to subscribe to a track
+	TrackSubscribeRequested(ctx context.Context, participantID livekit.ParticipantID, track *livekit.TrackInfo)
+	// TrackSubscribed - a participant subscribed to a track successfully
+	TrackSubscribed(ctx context.Context, participantID livekit.ParticipantID, track *livekit.TrackInfo, publisher *livekit.ParticipantInfo, shouldSendEvent bool)
+	// TrackUnsubscribed - a participant unsubscribed from a track successfully
+	TrackUnsubscribed(ctx context.Context, participantID livekit.ParticipantID, track *livekit.TrackInfo, shouldSendEvent bool)
+	// TrackSubscribeFailed - failure to subscribe to a track
+	TrackSubscribeFailed(ctx context.Context, participantID livekit.ParticipantID, trackID livekit.TrackID, err error, isUserError bool)
+	// TrackMuted - the publisher has muted the Track
+	TrackMuted(ctx context.Context, participantID livekit.ParticipantID, track *livekit.TrackInfo)
+	// TrackUnmuted - the publisher has muted the Track
+	TrackUnmuted(ctx context.Context, participantID livekit.ParticipantID, track *livekit.TrackInfo)
+	// TrackPublishedUpdate - track metadata has been updated
+	TrackPublishedUpdate(ctx context.Context, participantID livekit.ParticipantID, track *livekit.TrackInfo)
+	// TrackMaxSubscribedVideoQuality - publisher is notified of the max quality subscribers desire
+	TrackMaxSubscribedVideoQuality(ctx context.Context, participantID livekit.ParticipantID, track *livekit.TrackInfo, mime mime.MimeType, maxQuality livekit.VideoQuality)
+	TrackPublishRTPStats(ctx context.Context, participantID livekit.ParticipantID, trackID livekit.TrackID, mimeType mime.MimeType, layer int, stats *livekit.RTPStats)
+	TrackSubscribeRTPStats(ctx context.Context, participantID livekit.ParticipantID, trackID livekit.TrackID, mimeType mime.MimeType, stats *livekit.RTPStats)
+	EgressStarted(ctx context.Context, info *livekit.EgressInfo)
+	EgressUpdated(ctx context.Context, info *livekit.EgressInfo)
+	EgressEnded(ctx context.Context, info *livekit.EgressInfo)
+	IngressCreated(ctx context.Context, info *livekit.IngressInfo)
+	IngressDeleted(ctx context.Context, info *livekit.IngressInfo)
+	IngressStarted(ctx context.Context, info *livekit.IngressInfo)
+	IngressUpdated(ctx context.Context, info *livekit.IngressInfo)
+	IngressEnded(ctx context.Context, info *livekit.IngressInfo)
+	LocalRoomState(ctx context.Context, info *livekit.AnalyticsNodeRooms)
+	Report(ctx context.Context, reportInfo *livekit.ReportInfo)
+	APICall(ctx context.Context, apiCallInfo *livekit.APICallInfo)
+	Webhook(ctx context.Context, webhookInfo *livekit.WebhookInfo)
+
+	// helpers
+	AnalyticsService
+	NotifyEvent(ctx context.Context, event *livekit.WebhookEvent)
+	FlushStats()
+}
+
+const (
+	workerCleanupWait = 3 * time.Minute
+	jobsQueueMinSize  = 2048
+
+	telemetryStatsUpdateInterval         = time.Second * 30
+	telemetryNonMediaStatsUpdateInterval = time.Second * 30
+)
+
+type telemetryService struct {
+	AnalyticsService
+
+	notifier  webhook.QueuedNotifier
+	jobsQueue *utils.OpsQueue
+
+	workersMu  sync.RWMutex
+	workers    map[livekit.ParticipantID]*StatsWorker
+	workerList *StatsWorker
+
+	flushMu sync.Mutex
+}
+
+func NewTelemetryService(notifier webhook.QueuedNotifier, analytics AnalyticsService) TelemetryService {
+	t := &telemetryService{
+		AnalyticsService: analytics,
+		notifier:         notifier,
+		jobsQueue: utils.NewOpsQueue(utils.OpsQueueParams{
+			Name:        "telemetry",
+			MinSize:     jobsQueueMinSize,
+			FlushOnStop: true,
+			Logger:      logger.GetLogger(),
+		}),
+		workers: make(map[livekit.ParticipantID]*StatsWorker),
+	}
+	if t.notifier != nil {
+		t.notifier.RegisterProcessedHook(func(ctx context.Context, whi *livekit.WebhookInfo) {
+			t.Webhook(ctx, whi)
+		})
+	}
+
+	t.jobsQueue.Start()
+	go t.run()
+
+	return t
+}
+
+func (t *telemetryService) FlushStats() {
+	t.flushMu.Lock()
+	defer t.flushMu.Unlock()
+
+	t.workersMu.RLock()
+	worker := t.workerList
+	t.workersMu.RUnlock()
+
+	now := time.Now()
+	var prev, reap *StatsWorker
+	for worker != nil {
+		next := worker.next
+		if closed := worker.Flush(now); closed {
+			if prev == nil {
+				// this worker was at the head of the list
+				t.workersMu.Lock()
+				p := &t.workerList
+				for *p != worker {
+					// new workers have been added. scan until we find the one
+					// immediately before this
+					prev = *p
+					p = &prev.next
+				}
+				*p = worker.next
+				t.workersMu.Unlock()
+			} else {
+				prev.next = worker.next
+			}
+
+			worker.next = reap
+			reap = worker
+		} else {
+			prev = worker
+		}
+		worker = next
+	}
+
+	if reap != nil {
+		t.workersMu.Lock()
+		for reap != nil {
+			if reap == t.workers[reap.participantID] {
+				delete(t.workers, reap.participantID)
+			}
+			reap = reap.next
+		}
+		t.workersMu.Unlock()
+	}
+}
+
+func (t *telemetryService) run() {
+	for range time.Tick(telemetryStatsUpdateInterval) {
+		t.FlushStats()
+	}
+}
+
+func (t *telemetryService) enqueue(op func()) {
+	t.jobsQueue.Enqueue(op)
+}
+
+func (t *telemetryService) getWorker(participantID livekit.ParticipantID) (worker *StatsWorker, ok bool) {
+	t.workersMu.RLock()
+	defer t.workersMu.RUnlock()
+
+	worker, ok = t.workers[participantID]
+	return
+}
+
+func (t *telemetryService) getOrCreateWorker(
+	ctx context.Context,
+	roomID livekit.RoomID,
+	roomName livekit.RoomName,
+	participantID livekit.ParticipantID,
+	participantIdentity livekit.ParticipantIdentity,
+) (*StatsWorker, bool) {
+	t.workersMu.Lock()
+	defer t.workersMu.Unlock()
+
+	worker, ok := t.workers[participantID]
+	if ok && !worker.Closed() {
+		return worker, true
+	}
+
+	existingIsConnected := false
+	if ok {
+		existingIsConnected = worker.IsConnected()
+	}
+
+	worker = newStatsWorker(
+		ctx,
+		t,
+		roomID,
+		roomName,
+		participantID,
+		participantIdentity,
+	)
+	if existingIsConnected {
+		worker.SetConnected()
+	}
+
+	t.workers[participantID] = worker
+
+	worker.next = t.workerList
+	t.workerList = worker
+
+	return worker, false
+}
+
+func (t *telemetryService) LocalRoomState(ctx context.Context, info *livekit.AnalyticsNodeRooms) {
+	t.enqueue(func() {
+		t.SendNodeRoomStates(ctx, info)
+	})
+}