Merge commit '6bd7fac875d9e9009915053d8a590abb372c5679' into feature/livekit-upgrade-1.13.1

livekit-server/test/agent_test.go

@@ -32,194 +32,205 @@ var (
 )
 
 func TestAgents(t *testing.T) {
-	_, finish := setupSingleNodeTest("TestAgents")
-	defer finish()
+	for _, testRTCServicePath := range testRTCServicePaths {
+		t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) {
+			_, finish := setupSingleNodeTest("TestAgents")
+			defer finish()
 
-	ac1, err := newAgentClient(agentToken(), defaultServerPort)
-	require.NoError(t, err)
-	ac2, err := newAgentClient(agentToken(), defaultServerPort)
-	require.NoError(t, err)
-	ac3, err := newAgentClient(agentToken(), defaultServerPort)
-	require.NoError(t, err)
-	ac4, err := newAgentClient(agentToken(), defaultServerPort)
-	require.NoError(t, err)
-	ac5, err := newAgentClient(agentToken(), defaultServerPort)
-	require.NoError(t, err)
-	ac6, err := newAgentClient(agentToken(), defaultServerPort)
-	require.NoError(t, err)
-	defer ac1.close()
-	defer ac2.close()
-	defer ac3.close()
-	defer ac4.close()
-	defer ac5.close()
-	defer ac6.close()
-	ac1.Run(livekit.JobType_JT_ROOM, "default")
-	ac2.Run(livekit.JobType_JT_ROOM, "default")
-	ac3.Run(livekit.JobType_JT_PUBLISHER, "default")
-	ac4.Run(livekit.JobType_JT_PUBLISHER, "default")
-	ac5.Run(livekit.JobType_JT_PARTICIPANT, "default")
-	ac6.Run(livekit.JobType_JT_PARTICIPANT, "default")
+			ac1, err := newAgentClient(agentToken(), defaultServerPort)
+			require.NoError(t, err)
+			ac2, err := newAgentClient(agentToken(), defaultServerPort)
+			require.NoError(t, err)
+			ac3, err := newAgentClient(agentToken(), defaultServerPort)
+			require.NoError(t, err)
+			ac4, err := newAgentClient(agentToken(), defaultServerPort)
+			require.NoError(t, err)
+			ac5, err := newAgentClient(agentToken(), defaultServerPort)
+			require.NoError(t, err)
+			ac6, err := newAgentClient(agentToken(), defaultServerPort)
+			require.NoError(t, err)
+			defer ac1.close()
+			defer ac2.close()
+			defer ac3.close()
+			defer ac4.close()
+			defer ac5.close()
+			defer ac6.close()
+			ac1.Run(livekit.JobType_JT_ROOM, "default")
+			ac2.Run(livekit.JobType_JT_ROOM, "default")
+			ac3.Run(livekit.JobType_JT_PUBLISHER, "default")
+			ac4.Run(livekit.JobType_JT_PUBLISHER, "default")
+			ac5.Run(livekit.JobType_JT_PARTICIPANT, "default")
+			ac6.Run(livekit.JobType_JT_PARTICIPANT, "default")
 
-	testutils.WithTimeout(t, func() string {
-		if ac1.registered.Load() != 1 || ac2.registered.Load() != 1 || ac3.registered.Load() != 1 || ac4.registered.Load() != 1 || ac5.registered.Load() != 1 || ac6.registered.Load() != 1 {
-			return "worker not registered"
-		}
+			testutils.WithTimeout(t, func() string {
+				if ac1.registered.Load() != 1 || ac2.registered.Load() != 1 || ac3.registered.Load() != 1 || ac4.registered.Load() != 1 || ac5.registered.Load() != 1 || ac6.registered.Load() != 1 {
+					return "worker not registered"
+				}
 
-		return ""
-	}, RegisterTimeout)
+				return ""
+			}, RegisterTimeout)
 
-	c1 := createRTCClient("c1", defaultServerPort, nil)
-	c2 := createRTCClient("c2", defaultServerPort, nil)
-	waitUntilConnected(t, c1, c2)
+			c1 := createRTCClient("c1", defaultServerPort, testRTCServicePath, nil)
+			c2 := createRTCClient("c2", defaultServerPort, testRTCServicePath, nil)
+			waitUntilConnected(t, c1, c2)
 
-	// publish 2 tracks
-	t1, err := c1.AddStaticTrack("audio/opus", "audio", "micro")
-	require.NoError(t, err)
-	defer t1.Stop()
-	t2, err := c1.AddStaticTrack("video/vp8", "video", "webcam")
-	require.NoError(t, err)
-	defer t2.Stop()
+			// publish 2 tracks
+			t1, err := c1.AddStaticTrack("audio/opus", "audio", "micro")
+			require.NoError(t, err)
+			defer t1.Stop()
+			t2, err := c1.AddStaticTrack("video/vp8", "video", "webcam")
+			require.NoError(t, err)
+			defer t2.Stop()
 
-	testutils.WithTimeout(t, func() string {
-		if ac1.roomJobs.Load()+ac2.roomJobs.Load() != 1 {
-			return "room job not assigned"
-		}
+			testutils.WithTimeout(t, func() string {
+				if ac1.roomJobs.Load()+ac2.roomJobs.Load() != 1 {
+					return "room job not assigned"
+				}
 
-		if ac3.publisherJobs.Load()+ac4.publisherJobs.Load() != 1 {
-			return fmt.Sprintf("publisher jobs not assigned, ac3: %d, ac4: %d", ac3.publisherJobs.Load(), ac4.publisherJobs.Load())
-		}
+				if ac3.publisherJobs.Load()+ac4.publisherJobs.Load() != 1 {
+					return fmt.Sprintf("publisher jobs not assigned, ac3: %d, ac4: %d", ac3.publisherJobs.Load(), ac4.publisherJobs.Load())
+				}
 
-		if ac5.participantJobs.Load()+ac6.participantJobs.Load() != 2 {
-			return fmt.Sprintf("participant jobs not assigned, ac5: %d, ac6: %d", ac5.participantJobs.Load(), ac6.participantJobs.Load())
-		}
+				if ac5.participantJobs.Load()+ac6.participantJobs.Load() != 2 {
+					return fmt.Sprintf("participant jobs not assigned, ac5: %d, ac6: %d", ac5.participantJobs.Load(), ac6.participantJobs.Load())
+				}
 
-		return ""
-	}, 6*time.Second)
+				return ""
+			}, 6*time.Second)
 
-	// publish 2 tracks
-	t3, err := c2.AddStaticTrack("audio/opus", "audio", "micro")
-	require.NoError(t, err)
-	defer t3.Stop()
-	t4, err := c2.AddStaticTrack("video/vp8", "video", "webcam")
-	require.NoError(t, err)
-	defer t4.Stop()
+			// publish 2 tracks
+			t3, err := c2.AddStaticTrack("audio/opus", "audio", "micro")
+			require.NoError(t, err)
+			defer t3.Stop()
+			t4, err := c2.AddStaticTrack("video/vp8", "video", "webcam")
+			require.NoError(t, err)
+			defer t4.Stop()
 
-	testutils.WithTimeout(t, func() string {
-		if ac1.roomJobs.Load()+ac2.roomJobs.Load() != 1 {
-			return "room job must be assigned 1 time"
-		}
+			testutils.WithTimeout(t, func() string {
+				if ac1.roomJobs.Load()+ac2.roomJobs.Load() != 1 {
+					return "room job must be assigned 1 time"
+				}
 
-		if ac3.publisherJobs.Load()+ac4.publisherJobs.Load() != 2 {
-			return "2 publisher jobs must assigned"
-		}
+				if ac3.publisherJobs.Load()+ac4.publisherJobs.Load() != 2 {
+					return "2 publisher jobs must assigned"
+				}
 
-		if ac5.participantJobs.Load()+ac6.participantJobs.Load() != 2 {
-			return "2 participant jobs must assigned"
-		}
+				if ac5.participantJobs.Load()+ac6.participantJobs.Load() != 2 {
+					return "2 participant jobs must assigned"
+				}
 
-		return ""
-	}, AssignJobTimeout)
+				return ""
+			}, AssignJobTimeout)
+		})
+	}
 }
 
 func TestAgentNamespaces(t *testing.T) {
-	_, finish := setupSingleNodeTest("TestAgentNamespaces")
-	defer finish()
+	for _, testRTCServicePath := range testRTCServicePaths {
+		t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) {
+			_, finish := setupSingleNodeTest("TestAgentNamespaces")
+			defer finish()
 
-	ac1, err := newAgentClient(agentToken(), defaultServerPort)
-	require.NoError(t, err)
-	ac2, err := newAgentClient(agentToken(), defaultServerPort)
-	require.NoError(t, err)
-	defer ac1.close()
-	defer ac2.close()
-	ac1.Run(livekit.JobType_JT_ROOM, "namespace1")
-	ac2.Run(livekit.JobType_JT_ROOM, "namespace2")
+			ac1, err := newAgentClient(agentToken(), defaultServerPort)
+			require.NoError(t, err)
+			ac2, err := newAgentClient(agentToken(), defaultServerPort)
+			require.NoError(t, err)
+			defer ac1.close()
+			defer ac2.close()
+			ac1.Run(livekit.JobType_JT_ROOM, "namespace1")
+			ac2.Run(livekit.JobType_JT_ROOM, "namespace2")
 
-	_, err = roomClient.CreateRoom(contextWithToken(createRoomToken()), &livekit.CreateRoomRequest{
-		Name: testRoom,
-		Agents: []*livekit.RoomAgentDispatch{
-			{},
-			{
-				AgentName: "ag",
-			},
-		},
-	})
-	require.NoError(t, err)
+			_, err = roomClient.CreateRoom(contextWithToken(createRoomToken()), &livekit.CreateRoomRequest{
+				Name: testRoom,
+				Agents: []*livekit.RoomAgentDispatch{
+					{},
+					{
+						AgentName: "ag",
+					},
+				},
+			})
+			require.NoError(t, err)
 
-	testutils.WithTimeout(t, func() string {
-		if ac1.registered.Load() != 1 || ac2.registered.Load() != 1 {
-			return "worker not registered"
-		}
-		return ""
-	}, RegisterTimeout)
+			testutils.WithTimeout(t, func() string {
+				if ac1.registered.Load() != 1 || ac2.registered.Load() != 1 {
+					return "worker not registered"
+				}
+				return ""
+			}, RegisterTimeout)
 
-	c1 := createRTCClient("c1", defaultServerPort, nil)
-	waitUntilConnected(t, c1)
+			c1 := createRTCClient("c1", defaultServerPort, testRTCServicePath, nil)
+			waitUntilConnected(t, c1)
 
-	testutils.WithTimeout(t, func() string {
-		if ac1.roomJobs.Load() != 1 || ac2.roomJobs.Load() != 1 {
-			return "room job not assigned"
-		}
+			testutils.WithTimeout(t, func() string {
+				if ac1.roomJobs.Load() != 1 || ac2.roomJobs.Load() != 1 {
+					return "room job not assigned"
+				}
 
-		job1 := <-ac1.requestedJobs
-		job2 := <-ac2.requestedJobs
+				job1 := <-ac1.requestedJobs
+				job2 := <-ac2.requestedJobs
 
-		if job1.Namespace != "namespace1" {
-			return "namespace is not 'namespace'"
-		}
+				if job1.Namespace != "namespace1" {
+					return "namespace is not 'namespace'"
+				}
 
-		if job2.Namespace != "namespace2" {
-			return "namespace is not 'namespace2'"
-		}
+				if job2.Namespace != "namespace2" {
+					return "namespace is not 'namespace2'"
+				}
 
-		if job1.Id == job2.Id {
-			return "job ids are the same"
-		}
-
-		return ""
-	}, AssignJobTimeout)
+				if job1.Id == job2.Id {
+					return "job ids are the same"
+				}
 
+				return ""
+			}, AssignJobTimeout)
+		})
+	}
 }
 
 func TestAgentMultiNode(t *testing.T) {
-	_, _, finish := setupMultiNodeTest("TestAgentMultiNode")
-	defer finish()
+	for _, testRTCServicePath := range testRTCServicePaths {
+		t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) {
+			_, _, finish := setupMultiNodeTest("TestAgentMultiNode")
+			defer finish()
 
-	ac1, err := newAgentClient(agentToken(), defaultServerPort)
-	require.NoError(t, err)
-	ac2, err := newAgentClient(agentToken(), defaultServerPort)
-	require.NoError(t, err)
-	defer ac1.close()
-	defer ac2.close()
-	ac1.Run(livekit.JobType_JT_ROOM, "default")
-	ac2.Run(livekit.JobType_JT_PUBLISHER, "default")
+			ac1, err := newAgentClient(agentToken(), defaultServerPort)
+			require.NoError(t, err)
+			ac2, err := newAgentClient(agentToken(), defaultServerPort)
+			require.NoError(t, err)
+			defer ac1.close()
+			defer ac2.close()
+			ac1.Run(livekit.JobType_JT_ROOM, "default")
+			ac2.Run(livekit.JobType_JT_PUBLISHER, "default")
 
-	testutils.WithTimeout(t, func() string {
-		if ac1.registered.Load() != 1 || ac2.registered.Load() != 1 {
-			return "worker not registered"
-		}
-		return ""
-	}, RegisterTimeout)
+			testutils.WithTimeout(t, func() string {
+				if ac1.registered.Load() != 1 || ac2.registered.Load() != 1 {
+					return "worker not registered"
+				}
+				return ""
+			}, RegisterTimeout)
 
-	c1 := createRTCClient("c1", secondServerPort, nil) // Create a room on the second node
-	waitUntilConnected(t, c1)
+			c1 := createRTCClient("c1", secondServerPort, testRTCServicePath, nil) // Create a room on the second node
+			waitUntilConnected(t, c1)
 
-	t1, err := c1.AddStaticTrack("audio/opus", "audio", "micro")
-	require.NoError(t, err)
-	defer t1.Stop()
+			t1, err := c1.AddStaticTrack("audio/opus", "audio", "micro")
+			require.NoError(t, err)
+			defer t1.Stop()
 
-	time.Sleep(time.Second * 10)
+			time.Sleep(time.Second * 10)
 
-	testutils.WithTimeout(t, func() string {
-		if ac1.roomJobs.Load() != 1 {
-			return "room job not assigned"
-		}
+			testutils.WithTimeout(t, func() string {
+				if ac1.roomJobs.Load() != 1 {
+					return "room job not assigned"
+				}
 
-		if ac2.publisherJobs.Load() != 1 {
-			return "participant job not assigned"
-		}
+				if ac2.publisherJobs.Load() != 1 {
+					return "participant job not assigned"
+				}
 
-		return ""
-	}, AssignJobTimeout)
+				return ""
+			}, AssignJobTimeout)
+		})
+	}
 }
 
 func agentToken() string {

livekit-server/test/client/datatrack_remote.go

@@ -0,0 +1,84 @@
+package client
+
+import (
+	"github.com/frostbyte73/core"
+	"github.com/livekit/livekit-server/pkg/rtc/datatrack"
+	"github.com/livekit/protocol/livekit"
+	"github.com/livekit/protocol/logger"
+	"go.uber.org/atomic"
+)
+
+type DataTrackRemote struct {
+	publisherIdentity  livekit.ParticipantIdentity
+	publisherID        livekit.ParticipantID
+	handle             uint16
+	trackID            livekit.TrackID
+	logger             logger.Logger
+	numReceivedPackets atomic.Uint32
+
+	closed core.Fuse
+}
+
+func NewDataTrackRemote(
+	publisherIdentity livekit.ParticipantIdentity,
+	publisherID livekit.ParticipantID,
+	handle uint16,
+	trackID livekit.TrackID,
+	logger logger.Logger,
+) *DataTrackRemote {
+	logger.Infow(
+		"creating data track remote",
+		"publisherIdentity", publisherIdentity,
+		"publisherID", publisherID,
+		"handle", handle,
+		"trackID", trackID,
+	)
+	return &DataTrackRemote{
+		publisherIdentity: publisherIdentity,
+		publisherID:       publisherID,
+		handle:            handle,
+		trackID:           trackID,
+		logger:            logger,
+	}
+}
+
+func (d *DataTrackRemote) Close() {
+	d.logger.Infow(
+		"closing data track remote",
+		"publisherIdentity", d.publisherIdentity,
+		"publisherID", d.publisherID,
+		"handle", d.handle,
+		"trackID", d.trackID,
+	)
+	d.closed.Break()
+}
+
+func (d *DataTrackRemote) Handle() uint16 {
+	return d.handle
+}
+
+func (d *DataTrackRemote) ID() livekit.TrackID {
+	return d.trackID
+}
+
+func (d *DataTrackRemote) PacketReceived(packet *datatrack.Packet) {
+	if d.closed.IsBroken() {
+		return
+	}
+
+	valid := len(packet.Payload) != 0
+
+	for i := range packet.Payload {
+		if packet.Payload[i] != byte(255-i) {
+			valid = false
+			break
+		}
+	}
+	if valid {
+		d.numReceivedPackets.Inc()
+	}
+}
+
+func (d *DataTrackRemote) NumReceivedPackets() uint32 {
+	return d.numReceivedPackets.Load()
+}

livekit-server/test/client/datatrack_writer.go

@@ -0,0 +1,79 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package client
+
+import (
+	"context"
+	"math/rand"
+	"time"
+
+	"github.com/livekit/livekit-server/pkg/rtc/datatrack"
+	"github.com/livekit/livekit-server/pkg/rtc/types"
+	"github.com/livekit/protocol/logger"
+)
+
+type dataTrackWriter struct {
+	ctx       context.Context
+	cancel    context.CancelFunc
+	handle    uint16
+	transport types.DataTrackTransport
+}
+
+func NewDataTrackWriter(ctx context.Context, handle uint16, transport types.DataTrackTransport) TrackWriter {
+	ctx, cancel := context.WithCancel(ctx)
+	return &dataTrackWriter{
+		ctx:       ctx,
+		cancel:    cancel,
+		handle:    handle,
+		transport: transport,
+	}
+}
+
+func (d *dataTrackWriter) Start() error {
+	go d.writeFrames()
+	return nil
+}
+
+func (d *dataTrackWriter) Stop() {
+	d.cancel()
+}
+
+func (d *dataTrackWriter) writeFrames() {
+	seqNum := uint16(0)
+	frameNum := uint16(0)
+	for {
+		select {
+		case <-d.ctx.Done():
+			return
+
+		default:
+			packets := datatrack.GenerateRawDataPackets(d.handle, seqNum, frameNum, 1, rand.Intn(2048)+1, 100*time.Millisecond)
+			for _, packet := range packets {
+				if err := d.transport.SendDataTrackMessage(packet); err != nil {
+					logger.Errorw("could not send data track packet", err)
+				}
+			}
+
+			if len(packets) != 0 {
+				var lastPacket datatrack.Packet
+				if err := lastPacket.Unmarshal(packets[len(packets)-1]); err == nil {
+					seqNum = lastPacket.SequenceNumber + 1
+					frameNum = lastPacket.FrameNumber + 1
+				}
+			}
+			time.Sleep(100 * time.Millisecond)
+		}
+	}
+}

livekit-server/test/client/trackwriter.go

@@ -26,13 +26,18 @@ import (
 	"github.com/pion/webrtc/v4/pkg/media/ivfreader"
 	"github.com/pion/webrtc/v4/pkg/media/oggreader"
 
-	"github.com/livekit/livekit-server/pkg/sfu/mime"
+	"github.com/livekit/protocol/codecs/mime"
 	"github.com/livekit/protocol/logger"
 )
 
+type TrackWriter interface {
+	Start() error
+	Stop()
+}
+
 // Writes a file to an RTP track.
 // makes it easier to debug and create RTP streams
-type TrackWriter struct {
+type trackWriter struct {
 	ctx      context.Context
 	cancel   context.CancelFunc
 	track    *webrtc.TrackLocalStaticSample
@@ -45,9 +50,9 @@ type TrackWriter struct {
 	h264      *h264reader.H264Reader
 }
 
-func NewTrackWriter(ctx context.Context, track *webrtc.TrackLocalStaticSample, filePath string) *TrackWriter {
+func NewTrackWriter(ctx context.Context, track *webrtc.TrackLocalStaticSample, filePath string) TrackWriter {
 	ctx, cancel := context.WithCancel(ctx)
-	return &TrackWriter{
+	return &trackWriter{
 		ctx:      ctx,
 		cancel:   cancel,
 		track:    track,
@@ -56,7 +61,7 @@ func NewTrackWriter(ctx context.Context, track *webrtc.TrackLocalStaticSample, f
 	}
 }
 
-func (w *TrackWriter) Start() error {
+func (w *trackWriter) Start() error {
 	if w.filePath == "" {
 		go w.writeNull()
 		return nil
@@ -95,11 +100,11 @@ func (w *TrackWriter) Start() error {
 	return nil
 }
 
-func (w *TrackWriter) Stop() {
+func (w *trackWriter) Stop() {
 	w.cancel()
 }
 
-func (w *TrackWriter) writeNull() {
+func (w *trackWriter) writeNull() {
 	defer w.onWriteComplete()
 	sample := media.Sample{Data: []byte{0x0, 0xff, 0xff, 0xff, 0xff}, Duration: 30 * time.Millisecond}
 	h264Sample := media.Sample{Data: []byte{0x00, 0x00, 0x00, 0x01, 0x7, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, 0x8, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, 0x5, 0xff, 0xff, 0xff, 0xff}, Duration: 30 * time.Millisecond}
@@ -117,7 +122,7 @@ func (w *TrackWriter) writeNull() {
 	}
 }
 
-func (w *TrackWriter) writeOgg() {
+func (w *trackWriter) writeOgg() {
 	// Keep track of last granule, the difference is the amount of samples in the buffer
 	var lastGranule uint64
 	for {
@@ -150,7 +155,7 @@ func (w *TrackWriter) writeOgg() {
 	}
 }
 
-func (w *TrackWriter) writeVP8() {
+func (w *trackWriter) writeVP8() {
 	// Send our video file frame at a time. Pace our sending such that we send it at the same speed it should be played back as.
 	// This isn't required since the video is timestamped, but we will such much higher loss if we send all at once.
 	sleepTime := time.Millisecond * time.Duration((float32(w.ivfheader.TimebaseNumerator)/float32(w.ivfheader.TimebaseDenominator))*1000)
@@ -178,10 +183,9 @@ func (w *TrackWriter) writeVP8() {
 	}
 }
 
-func (w *TrackWriter) writeH264() {
+func (w *trackWriter) writeH264() {
 	// TODO: this is harder
 }
 
-func (w *TrackWriter) onWriteComplete() {
-
+func (w *trackWriter) onWriteComplete() {
 }

livekit-server/test/integration_helpers.go

@@ -25,17 +25,18 @@ import (
 	"github.com/redis/go-redis/v9"
 	"github.com/twitchtv/twirp"
 
+	"github.com/livekit/mediatransportutil/pkg/rtcconfig"
+	"github.com/livekit/protocol/auth"
+	"github.com/livekit/protocol/livekit"
+	"github.com/livekit/protocol/logger"
+	"github.com/livekit/protocol/utils/guid"
+
 	"github.com/livekit/livekit-server/pkg/config"
 	"github.com/livekit/livekit-server/pkg/routing"
 	"github.com/livekit/livekit-server/pkg/service"
 	"github.com/livekit/livekit-server/pkg/telemetry/prometheus"
 	"github.com/livekit/livekit-server/pkg/testutils"
 	testclient "github.com/livekit/livekit-server/test/client"
-	"github.com/livekit/mediatransportutil/pkg/rtcconfig"
-	"github.com/livekit/protocol/auth"
-	"github.com/livekit/protocol/livekit"
-	"github.com/livekit/protocol/logger"
-	"github.com/livekit/protocol/utils/guid"
 )
 
 const (
@@ -131,14 +132,30 @@ func waitUntilConnected(t *testing.T, clients ...*testclient.RTCClient) {
 	wg := sync.WaitGroup{}
 	for i := range clients {
 		c := clients[i]
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			err := c.WaitUntilConnected()
+		wg.Go(func() {
+			err := c.WaitUntilConnected(5 * time.Second)
 			if err != nil {
 				t.Error(err)
 			}
-		}()
+		})
+	}
+	wg.Wait()
+	if t.Failed() {
+		t.FailNow()
+	}
+}
+
+func ensureNotConnected(t *testing.T, clients ...*testclient.RTCClient) {
+	logger.Infow("checking if clients connect")
+	wg := sync.WaitGroup{}
+	for i := range clients {
+		c := clients[i]
+		wg.Go(func() {
+			err := c.WaitUntilConnected(5 * time.Second)
+			if err == nil {
+				t.Error(fmt.Errorf("expected client to not connect: %s", c.ID()))
+			}
+		})
 	}
 	wg.Wait()
 	if t.Failed() {
@@ -153,6 +170,7 @@ func createSingleNodeServer(configUpdater func(*config.Config)) *service.Livekit
 		panic(fmt.Sprintf("could not create config: %v", err))
 	}
 	conf.Keys = map[string]string{testApiKey: testApiSecret}
+	conf.EnableDataTracks = true
 	if configUpdater != nil {
 		configUpdater(conf)
 	}
@@ -183,6 +201,7 @@ func createMultiNodeServer(nodeID string, port uint32) *service.LivekitServer {
 	conf.RTC.TCPPort = port + 2
 	conf.Redis.Address = "localhost:6379"
 	conf.Keys = map[string]string{testApiKey: testApiSecret}
+	conf.EnableDataTracks = true
 
 	currentNode, err := routing.NewLocalNode(conf)
 	if err != nil {
@@ -200,36 +219,77 @@ func createMultiNodeServer(nodeID string, port uint32) *service.LivekitServer {
 	return s
 }
 
+type testRTCServicePath int
+
+const (
+	testRTCServicePathv0 testRTCServicePath = iota
+	testRTCServicePathv0SinglePeerConnection
+	testRTCServicePathv1
+)
+
+func (t testRTCServicePath) String() string {
+	switch t {
+	case testRTCServicePathv0:
+		return "v0"
+	case testRTCServicePathv0SinglePeerConnection:
+		return "v0-single-peer-connection"
+	case testRTCServicePathv1:
+		return "v1"
+	default:
+		return fmt.Sprintf("unknown: %d", t)
+	}
+}
+
+var testRTCServicePaths = []testRTCServicePath{
+	testRTCServicePathv0,
+	testRTCServicePathv0SinglePeerConnection,
+	testRTCServicePathv1,
+}
+
+func testRTCServicePathToTestClientOptions(testRTCServicePath testRTCServicePath, opts *testclient.Options) {
+	if opts == nil {
+		return
+	}
+
+	switch testRTCServicePath {
+	case testRTCServicePathv0:
+		opts.RTCServicePath = "/rtc"
+	case testRTCServicePathv0SinglePeerConnection:
+		opts.RTCServicePath = "/rtc"
+		opts.UseJoinRequestQueryParam = true
+	case testRTCServicePathv1:
+		opts.RTCServicePath = "/rtc/v1"
+		opts.UseJoinRequestQueryParam = true
+	default:
+		opts.RTCServicePath = "/rtc"
+	}
+}
+
 // creates a client and runs against server
-func createRTCClient(name string, port int, opts *testclient.Options) *testclient.RTCClient {
+func createRTCClient(name string, port int, testRTCServicePath testRTCServicePath, opts *testclient.Options) *testclient.RTCClient {
 	var customizer func(token *auth.AccessToken, grants *auth.VideoGrant)
 	if opts != nil {
 		customizer = opts.TokenCustomizer
 	}
 	token := joinToken(testRoom, name, customizer)
-	ws, err := testclient.NewWebSocketConn(fmt.Sprintf("ws://localhost:%d", port), token, opts)
-	if err != nil {
-		panic(err)
-	}
 
-	c, err := testclient.NewRTCClient(ws, opts)
-	if err != nil {
-		panic(err)
-	}
-
-	go c.Run()
-
-	return c
+	return createRTCClientWithToken(token, port, testRTCServicePath, opts)
 }
 
 // creates a client and runs against server
-func createRTCClientWithToken(token string, port int, opts *testclient.Options) *testclient.RTCClient {
+func createRTCClientWithToken(token string, port int, testRTCServicePath testRTCServicePath, opts *testclient.Options) *testclient.RTCClient {
+	if opts == nil {
+		opts = &testclient.Options{
+			AutoSubscribe: true,
+		}
+	}
+	testRTCServicePathToTestClientOptions(testRTCServicePath, opts)
 	ws, err := testclient.NewWebSocketConn(fmt.Sprintf("ws://localhost:%d", port), token, opts)
 	if err != nil {
 		panic(err)
 	}
 
-	c, err := testclient.NewRTCClient(ws, opts)
+	c, err := testclient.NewRTCClient(ws, opts.UseJoinRequestQueryParam, opts)
 	if err != nil {
 		panic(err)
 	}
@@ -304,7 +364,7 @@ func listRoomToken() string {
 	return t
 }
 
-func stopWriters(writers ...*testclient.TrackWriter) {
+func stopWriters(writers ...testclient.TrackWriter) {
 	for _, w := range writers {
 		w.Stop()
 	}

livekit-server/test/multinode_roomservice_test.go

@@ -61,24 +61,28 @@ func TestMultiNodeUpdateRoomMetadata(t *testing.T) {
 	})
 
 	t.Run("when room has a participant", func(t *testing.T) {
-		_, _, finish := setupMultiNodeTest("TestMultiNodeUpdateRoomMetadata_with_participant")
-		defer finish()
+		for _, testRTCServicePath := range testRTCServicePaths {
+			t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) {
+				_, _, finish := setupMultiNodeTest("TestMultiNodeUpdateRoomMetadata_with_participant")
+				defer finish()
 
-		c1 := createRTCClient("c1", defaultServerPort, nil)
-		waitUntilConnected(t, c1)
-		defer c1.Stop()
+				c1 := createRTCClient("c1", defaultServerPort, testRTCServicePath, nil)
+				waitUntilConnected(t, c1)
+				defer c1.Stop()
 
-		_, err := roomClient.CreateRoom(contextWithToken(createRoomToken()), &livekit.CreateRoomRequest{
-			Name: "emptyRoom",
-		})
-		require.NoError(t, err)
+				_, err := roomClient.CreateRoom(contextWithToken(createRoomToken()), &livekit.CreateRoomRequest{
+					Name: "emptyRoom",
+				})
+				require.NoError(t, err)
 
-		rm, err := roomClient.UpdateRoomMetadata(contextWithToken(adminRoomToken("emptyRoom")), &livekit.UpdateRoomMetadataRequest{
-			Room:     "emptyRoom",
-			Metadata: "updated metadata",
-		})
-		require.NoError(t, err)
-		require.Equal(t, "updated metadata", rm.Metadata)
+				rm, err := roomClient.UpdateRoomMetadata(contextWithToken(adminRoomToken("emptyRoom")), &livekit.UpdateRoomMetadataRequest{
+					Room:     "emptyRoom",
+					Metadata: "updated metadata",
+				})
+				require.NoError(t, err)
+				require.Equal(t, "updated metadata", rm.Metadata)
+			})
+		}
 	})
 }
 
@@ -89,85 +93,97 @@ func TestMultiNodeRemoveParticipant(t *testing.T) {
 		return
 	}
 
-	_, _, finish := setupMultiNodeTest("TestMultiNodeRemoveParticipant")
-	defer finish()
+	for _, testRTCServicePath := range testRTCServicePaths {
+		t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) {
+			_, _, finish := setupMultiNodeTest("TestMultiNodeRemoveParticipant")
+			defer finish()
 
-	c1 := createRTCClient("mn_remove_participant", defaultServerPort, nil)
-	defer c1.Stop()
-	waitUntilConnected(t, c1)
+			c1 := createRTCClient("mn_remove_participant", defaultServerPort, testRTCServicePath, nil)
+			defer c1.Stop()
+			waitUntilConnected(t, c1)
 
-	ctx := contextWithToken(adminRoomToken(testRoom))
-	_, err := roomClient.RemoveParticipant(ctx, &livekit.RoomParticipantIdentity{
-		Room:     testRoom,
-		Identity: "mn_remove_participant",
-	})
-	require.NoError(t, err)
+			ctx := contextWithToken(adminRoomToken(testRoom))
+			_, err := roomClient.RemoveParticipant(ctx, &livekit.RoomParticipantIdentity{
+				Room:     testRoom,
+				Identity: "mn_remove_participant",
+			})
+			require.NoError(t, err)
 
-	// participant list doesn't show the participant
-	listRes, err := roomClient.ListParticipants(ctx, &livekit.ListParticipantsRequest{
-		Room: testRoom,
-	})
-	require.NoError(t, err)
-	require.Len(t, listRes.Participants, 0)
+			// participant list doesn't show the participant
+			listRes, err := roomClient.ListParticipants(ctx, &livekit.ListParticipantsRequest{
+				Room: testRoom,
+			})
+			require.NoError(t, err)
+			require.Len(t, listRes.Participants, 0)
+		})
+	}
 }
 
 // update participant metadata
 func TestMultiNodeUpdateParticipantMetadata(t *testing.T) {
-	_, _, finish := setupMultiNodeTest("TestMultiNodeUpdateParticipantMetadata")
-	defer finish()
+	for _, testRTCServicePath := range testRTCServicePaths {
+		t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) {
+			_, _, finish := setupMultiNodeTest("TestMultiNodeUpdateParticipantMetadata")
+			defer finish()
 
-	c1 := createRTCClient("update_participant_metadata", defaultServerPort, nil)
-	defer c1.Stop()
-	waitUntilConnected(t, c1)
+			c1 := createRTCClient("update_participant_metadata", defaultServerPort, testRTCServicePath, nil)
+			defer c1.Stop()
+			waitUntilConnected(t, c1)
 
-	ctx := contextWithToken(adminRoomToken(testRoom))
-	res, err := roomClient.UpdateParticipant(ctx, &livekit.UpdateParticipantRequest{
-		Room:     testRoom,
-		Identity: "update_participant_metadata",
-		Metadata: "the new metadata",
-	})
-	require.NoError(t, err)
-	require.Equal(t, "the new metadata", res.Metadata)
+			ctx := contextWithToken(adminRoomToken(testRoom))
+			res, err := roomClient.UpdateParticipant(ctx, &livekit.UpdateParticipantRequest{
+				Room:     testRoom,
+				Identity: "update_participant_metadata",
+				Metadata: "the new metadata",
+			})
+			require.NoError(t, err)
+			require.Equal(t, "the new metadata", res.Metadata)
+		})
+	}
 }
 
 // admin mute published track
 func TestMultiNodeMutePublishedTrack(t *testing.T) {
-	_, _, finish := setupMultiNodeTest("TestMultiNodeMutePublishedTrack")
-	defer finish()
+	for _, testRTCServicePath := range testRTCServicePaths {
+		t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) {
+			_, _, finish := setupMultiNodeTest("TestMultiNodeMutePublishedTrack")
+			defer finish()
 
-	identity := "mute_published_track"
-	c1 := createRTCClient(identity, defaultServerPort, nil)
-	defer c1.Stop()
-	waitUntilConnected(t, c1)
+			identity := "mute_published_track"
+			c1 := createRTCClient(identity, defaultServerPort, testRTCServicePath, nil)
+			defer c1.Stop()
+			waitUntilConnected(t, c1)
 
-	writers := publishTracksForClients(t, c1)
-	defer stopWriters(writers...)
+			writers := publishTracksForClients(t, c1)
+			defer stopWriters(writers...)
 
-	trackIDs := c1.GetPublishedTrackIDs()
-	require.NotEmpty(t, trackIDs)
+			trackIDs := c1.GetPublishedTrackIDs()
+			require.NotEmpty(t, trackIDs)
 
-	ctx := contextWithToken(adminRoomToken(testRoom))
-	// wait for it to be published before
-	testutils.WithTimeout(t, func() string {
-		res, err := roomClient.GetParticipant(ctx, &livekit.RoomParticipantIdentity{
-			Room:     testRoom,
-			Identity: identity,
+			ctx := contextWithToken(adminRoomToken(testRoom))
+			// wait for it to be published before
+			testutils.WithTimeout(t, func() string {
+				res, err := roomClient.GetParticipant(ctx, &livekit.RoomParticipantIdentity{
+					Room:     testRoom,
+					Identity: identity,
+				})
+				require.NoError(t, err)
+				if len(res.Tracks) == 2 {
+					return ""
+				} else {
+					return fmt.Sprintf("expected 2 tracks to be published, actual: %d", len(res.Tracks))
+				}
+			})
+
+			res, err := roomClient.MutePublishedTrack(ctx, &livekit.MuteRoomTrackRequest{
+				Room:     testRoom,
+				Identity: identity,
+				TrackSid: trackIDs[0],
+				Muted:    true,
+			})
+			require.NoError(t, err)
+			require.Equal(t, trackIDs[0], res.Track.Sid)
+			require.True(t, res.Track.Muted)
 		})
-		require.NoError(t, err)
-		if len(res.Tracks) == 2 {
-			return ""
-		} else {
-			return fmt.Sprintf("expected 2 tracks to be published, actual: %d", len(res.Tracks))
-		}
-	})
-
-	res, err := roomClient.MutePublishedTrack(ctx, &livekit.MuteRoomTrackRequest{
-		Room:     testRoom,
-		Identity: identity,
-		TrackSid: trackIDs[0],
-		Muted:    true,
-	})
-	require.NoError(t, err)
-	require.Equal(t, trackIDs[0], res.Track.Sid)
-	require.True(t, res.Track.Muted)
+	}
 }

livekit-server/test/multinode_test.go

@@ -15,6 +15,7 @@
 package test
 
 import (
+	"fmt"
 	"testing"
 	"time"
 
@@ -33,6 +34,7 @@ func TestMultiNodeRouting(t *testing.T) {
 		t.SkipNow()
 		return
 	}
+
 	_, _, finish := setupMultiNodeTest("TestMultiNodeRouting")
 	defer finish()
 
@@ -42,35 +44,39 @@ func TestMultiNodeRouting(t *testing.T) {
 	})
 	require.NoError(t, err)
 
-	// one node connecting to node 1, and another connecting to node 2
-	c1 := createRTCClient("c1", defaultServerPort, nil)
-	c2 := createRTCClient("c2", secondServerPort, nil)
-	waitUntilConnected(t, c1, c2)
-	defer stopClients(c1, c2)
+	for _, testRTCServicePath := range testRTCServicePaths {
+		t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) {
+			// one node connecting to node 1, and another connecting to node 2
+			c1 := createRTCClient("c1", defaultServerPort, testRTCServicePath, nil)
+			c2 := createRTCClient("c2", secondServerPort, testRTCServicePath, nil)
+			waitUntilConnected(t, c1, c2)
+			defer stopClients(c1, c2)
 
-	// c1 publishing, and c2 receiving
-	t1, err := c1.AddStaticTrack("audio/opus", "audio", "webcam")
-	require.NoError(t, err)
-	if t1 != nil {
-		defer t1.Stop()
+			// c1 publishing, and c2 receiving
+			t1, err := c1.AddStaticTrack("audio/opus", "audio", "webcam")
+			require.NoError(t, err)
+			if t1 != nil {
+				defer t1.Stop()
+			}
+
+			testutils.WithTimeout(t, func() string {
+				if len(c2.SubscribedTracks()) == 0 {
+					return "c2 received no tracks"
+				}
+				if len(c2.SubscribedTracks()[c1.ID()]) != 1 {
+					return "c2 didn't receive track published by c1"
+				}
+				tr1 := c2.SubscribedTracks()[c1.ID()][0]
+				streamID, _ := rtc.UnpackStreamID(tr1.StreamID())
+				require.Equal(t, c1.ID(), streamID)
+				return ""
+			})
+
+			remoteC1 := c2.GetRemoteParticipant(c1.ID())
+			require.Equal(t, "c1", remoteC1.Name)
+			require.Equal(t, "metadatac1", remoteC1.Metadata)
+		})
 	}
-
-	testutils.WithTimeout(t, func() string {
-		if len(c2.SubscribedTracks()) == 0 {
-			return "c2 received no tracks"
-		}
-		if len(c2.SubscribedTracks()[c1.ID()]) != 1 {
-			return "c2 didn't receive track published by c1"
-		}
-		tr1 := c2.SubscribedTracks()[c1.ID()][0]
-		streamID, _ := rtc.UnpackStreamID(tr1.StreamID())
-		require.Equal(t, c1.ID(), streamID)
-		return ""
-	})
-
-	remoteC1 := c2.GetRemoteParticipant(c1.ID())
-	require.Equal(t, "c1", remoteC1.Name)
-	require.Equal(t, "metadatac1", remoteC1.Metadata)
 }
 
 func TestConnectWithoutCreation(t *testing.T) {
@@ -82,13 +88,16 @@ func TestConnectWithoutCreation(t *testing.T) {
 	_, _, finish := setupMultiNodeTest("TestConnectWithoutCreation")
 	defer finish()
 
-	c1 := createRTCClient("c1", defaultServerPort, nil)
-	waitUntilConnected(t, c1)
+	for _, testRTCServicePath := range testRTCServicePaths {
+		t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) {
+			c1 := createRTCClient("c1", defaultServerPort, testRTCServicePath, nil)
+			waitUntilConnected(t, c1)
 
-	c1.Stop()
+			c1.Stop()
+		})
+	}
 }
 
-// testing multiple scenarios  rooms
 func TestMultinodePublishingUponJoining(t *testing.T) {
 	if testing.Short() {
 		t.SkipNow()
@@ -118,30 +127,34 @@ func TestMultinodeReconnectAfterNodeShutdown(t *testing.T) {
 		return
 	}
 
-	_, s2, finish := setupMultiNodeTest("TestMultinodeReconnectAfterNodeShutdown")
-	defer finish()
+	for _, testRTCServicePath := range testRTCServicePaths {
+		t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) {
+			_, s2, finish := setupMultiNodeTest("TestMultinodeReconnectAfterNodeShutdown")
+			defer finish()
 
-	// creating room on node 1
-	_, err := roomClient.CreateRoom(contextWithToken(createRoomToken()), &livekit.CreateRoomRequest{
-		Name:   testRoom,
-		NodeId: s2.Node().Id,
-	})
-	require.NoError(t, err)
+			// creating room on node 1
+			_, err := roomClient.CreateRoom(contextWithToken(createRoomToken()), &livekit.CreateRoomRequest{
+				Name:   testRoom,
+				NodeId: s2.Node().Id,
+			})
+			require.NoError(t, err)
 
-	// one node connecting to node 1, and another connecting to node 2
-	c1 := createRTCClient("c1", defaultServerPort, nil)
-	c2 := createRTCClient("c2", secondServerPort, nil)
+			// one node connecting to node 1, and another connecting to node 2
+			c1 := createRTCClient("c1", defaultServerPort, testRTCServicePath, nil)
+			c2 := createRTCClient("c2", secondServerPort, testRTCServicePath, nil)
 
-	waitUntilConnected(t, c1, c2)
-	stopClients(c1, c2)
+			waitUntilConnected(t, c1, c2)
+			stopClients(c1, c2)
 
-	// stop s2, and connect to room again
-	s2.Stop(true)
+			// stop s2, and connect to room again
+			s2.Stop(true)
 
-	time.Sleep(syncDelay)
+			time.Sleep(syncDelay)
 
-	c3 := createRTCClient("c3", defaultServerPort, nil)
-	waitUntilConnected(t, c3)
+			c3 := createRTCClient("c3", defaultServerPort, testRTCServicePath, nil)
+			waitUntilConnected(t, c3)
+		})
+	}
 }
 
 func TestMultinodeDataPublishing(t *testing.T) {
@@ -154,6 +167,8 @@ func TestMultinodeDataPublishing(t *testing.T) {
 	defer finish()
 
 	scenarioDataPublish(t)
+	scenarioDataUnlabeledPublish(t)
+	scenarioDataTracksPublishingUponJoining(t)
 }
 
 func TestMultiNodeJoinAfterClose(t *testing.T) {
@@ -185,48 +200,52 @@ func TestMultiNodeRefreshToken(t *testing.T) {
 	_, _, finish := setupMultiNodeTest("TestMultiNodeJoinAfterClose")
 	defer finish()
 
-	// a participant joining with full permissions
-	c1 := createRTCClient("c1", defaultServerPort, nil)
-	waitUntilConnected(t, c1)
+	for _, testRTCServicePath := range testRTCServicePaths {
+		t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) {
+			// a participant joining with full permissions
+			c1 := createRTCClient("c1", defaultServerPort, testRTCServicePath, nil)
+			waitUntilConnected(t, c1)
 
-	// update permissions and metadata
-	ctx := contextWithToken(adminRoomToken(testRoom))
-	_, err := roomClient.UpdateParticipant(ctx, &livekit.UpdateParticipantRequest{
-		Room:     testRoom,
-		Identity: "c1",
-		Permission: &livekit.ParticipantPermission{
-			CanPublish:   false,
-			CanSubscribe: true,
-		},
-		Metadata: "metadata",
-	})
-	require.NoError(t, err)
+			// update permissions and metadata
+			ctx := contextWithToken(adminRoomToken(testRoom))
+			_, err := roomClient.UpdateParticipant(ctx, &livekit.UpdateParticipantRequest{
+				Room:     testRoom,
+				Identity: "c1",
+				Permission: &livekit.ParticipantPermission{
+					CanPublish:   false,
+					CanSubscribe: true,
+				},
+				Metadata: "metadata",
+			})
+			require.NoError(t, err)
 
-	testutils.WithTimeout(t, func() string {
-		if c1.RefreshToken() == "" {
-			return "did not receive refresh token"
-		}
-		// parse token to ensure it's correct
-		verifier, err := auth.ParseAPIToken(c1.RefreshToken())
-		require.NoError(t, err)
+			testutils.WithTimeout(t, func() string {
+				if c1.RefreshToken() == "" {
+					return "did not receive refresh token"
+				}
+				// parse token to ensure it's correct
+				verifier, err := auth.ParseAPIToken(c1.RefreshToken())
+				require.NoError(t, err)
 
-		grants, err := verifier.Verify(testApiSecret)
-		require.NoError(t, err)
+				_, grants, err := verifier.Verify(testApiSecret)
+				require.NoError(t, err)
 
-		if grants.Metadata != "metadata" {
-			return "metadata did not match"
-		}
-		if *grants.Video.CanPublish {
-			return "canPublish should be false"
-		}
-		if *grants.Video.CanPublishData {
-			return "canPublishData should be false"
-		}
-		if !*grants.Video.CanSubscribe {
-			return "canSubscribe should be true"
-		}
-		return ""
-	})
+				if grants.Metadata != "metadata" {
+					return "metadata did not match"
+				}
+				if *grants.Video.CanPublish {
+					return "canPublish should be false"
+				}
+				if *grants.Video.CanPublishData {
+					return "canPublishData should be false"
+				}
+				if !*grants.Video.CanSubscribe {
+					return "canSubscribe should be true"
+				}
+				return ""
+			})
+		})
+	}
 }
 
 // ensure that token accurately reflects out of band updates
@@ -239,158 +258,170 @@ func TestMultiNodeUpdateAttributes(t *testing.T) {
 	_, _, finish := setupMultiNodeTest("TestMultiNodeUpdateAttributes")
 	defer finish()
 
-	c1 := createRTCClient("au1", defaultServerPort, &client.Options{
-		TokenCustomizer: func(token *auth.AccessToken, grants *auth.VideoGrant) {
-			token.SetAttributes(map[string]string{
-				"mykey": "au1",
+	for _, testRTCServicePath := range testRTCServicePaths {
+		t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) {
+			c1 := createRTCClient("au1", defaultServerPort, testRTCServicePath, &client.Options{
+				TokenCustomizer: func(token *auth.AccessToken, grants *auth.VideoGrant) {
+					token.SetAttributes(map[string]string{
+						"mykey": "au1",
+					})
+				},
 			})
-		},
-	})
-	c2 := createRTCClient("au2", secondServerPort, &client.Options{
-		TokenCustomizer: func(token *auth.AccessToken, grants *auth.VideoGrant) {
-			token.SetAttributes(map[string]string{
-				"mykey": "au2",
+			c2 := createRTCClient("au2", secondServerPort, testRTCServicePath, &client.Options{
+				TokenCustomizer: func(token *auth.AccessToken, grants *auth.VideoGrant) {
+					token.SetAttributes(map[string]string{
+						"mykey": "au2",
+					})
+					grants.SetCanUpdateOwnMetadata(true)
+				},
 			})
-			grants.SetCanUpdateOwnMetadata(true)
-		},
-	})
-	waitUntilConnected(t, c1, c2)
+			waitUntilConnected(t, c1, c2)
 
-	testutils.WithTimeout(t, func() string {
-		rc2 := c1.GetRemoteParticipant(c2.ID())
-		rc1 := c2.GetRemoteParticipant(c1.ID())
-		if rc2 == nil || rc1 == nil {
-			return "participants could not see each other"
-		}
-		if rc1.Attributes == nil || rc1.Attributes["mykey"] != "au1" {
-			return "rc1's initial attributes are incorrect"
-		}
-		if rc2.Attributes == nil || rc2.Attributes["mykey"] != "au2" {
-			return "rc2's initial attributes are incorrect"
-		}
-		return ""
-	})
+			testutils.WithTimeout(t, func() string {
+				rc2 := c1.GetRemoteParticipant(c2.ID())
+				rc1 := c2.GetRemoteParticipant(c1.ID())
+				if rc2 == nil || rc1 == nil {
+					return "participants could not see each other"
+				}
+				if rc1.Attributes == nil || rc1.Attributes["mykey"] != "au1" {
+					return "rc1's initial attributes are incorrect"
+				}
+				if rc2.Attributes == nil || rc2.Attributes["mykey"] != "au2" {
+					return "rc2's initial attributes are incorrect"
+				}
+				return ""
+			})
 
-	// this one should not go through
-	_ = c1.SetAttributes(map[string]string{"mykey": "shouldnotchange"})
-	_ = c2.SetAttributes(map[string]string{"secondkey": "au2"})
+			// this one should not go through
+			_ = c1.SetAttributes(map[string]string{"mykey": "shouldnotchange"})
+			_ = c2.SetAttributes(map[string]string{"secondkey": "au2"})
 
-	// updates using room API should succeed
-	_, err := roomClient.UpdateParticipant(contextWithToken(adminRoomToken(testRoom)), &livekit.UpdateParticipantRequest{
-		Room:     testRoom,
-		Identity: "au1",
-		Attributes: map[string]string{
-			"secondkey": "au1",
-		},
-	})
-	require.NoError(t, err)
+			// updates using room API should succeed
+			_, err := roomClient.UpdateParticipant(contextWithToken(adminRoomToken(testRoom)), &livekit.UpdateParticipantRequest{
+				Room:     testRoom,
+				Identity: "au1",
+				Attributes: map[string]string{
+					"secondkey": "au1",
+				},
+			})
+			require.NoError(t, err)
 
-	testutils.WithTimeout(t, func() string {
-		rc1 := c2.GetRemoteParticipant(c1.ID())
-		rc2 := c1.GetRemoteParticipant(c2.ID())
-		if rc1.Attributes["secondkey"] != "au1" {
-			return "au1's attribute update failed"
-		}
-		if rc2.Attributes["secondkey"] != "au2" {
-			return "au2's attribute update failed"
-		}
-		if rc1.Attributes["mykey"] != "au1" {
-			return "au1's mykey should not change"
-		}
-		if rc2.Attributes["mykey"] != "au2" {
-			return "au2's mykey should not change"
-		}
-		return ""
-	})
+			testutils.WithTimeout(t, func() string {
+				rc1 := c2.GetRemoteParticipant(c1.ID())
+				rc2 := c1.GetRemoteParticipant(c2.ID())
+				if rc1.Attributes["secondkey"] != "au1" {
+					return "au1's attribute update failed"
+				}
+				if rc2.Attributes["secondkey"] != "au2" {
+					return "au2's attribute update failed"
+				}
+				if rc1.Attributes["mykey"] != "au1" {
+					return "au1's mykey should not change"
+				}
+				if rc2.Attributes["mykey"] != "au2" {
+					return "au2's mykey should not change"
+				}
+				return ""
+			})
+		})
+	}
 }
 
 func TestMultiNodeRevokePublishPermission(t *testing.T) {
 	_, _, finish := setupMultiNodeTest("TestMultiNodeRevokePublishPermission")
 	defer finish()
 
-	c1 := createRTCClient("c1", defaultServerPort, nil)
-	c2 := createRTCClient("c2", secondServerPort, nil)
-	waitUntilConnected(t, c1, c2)
+	for _, testRTCServicePath := range testRTCServicePaths {
+		t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) {
+			c1 := createRTCClient("c1", defaultServerPort, testRTCServicePath, nil)
+			c2 := createRTCClient("c2", secondServerPort, testRTCServicePath, nil)
+			waitUntilConnected(t, c1, c2)
 
-	// c1 publishes a track for c2
-	writers := publishTracksForClients(t, c1)
-	defer stopWriters(writers...)
+			// c1 publishes a track for c2
+			writers := publishTracksForClients(t, c1)
+			defer stopWriters(writers...)
 
-	testutils.WithTimeout(t, func() string {
-		if len(c2.SubscribedTracks()[c1.ID()]) != 2 {
-			return "c2 did not receive c1's tracks"
-		}
-		return ""
-	})
+			testutils.WithTimeout(t, func() string {
+				if len(c2.SubscribedTracks()[c1.ID()]) != 2 {
+					return "c2 did not receive c1's tracks"
+				}
+				return ""
+			})
 
-	// revoke permission
-	ctx := contextWithToken(adminRoomToken(testRoom))
-	_, err := roomClient.UpdateParticipant(ctx, &livekit.UpdateParticipantRequest{
-		Room:     testRoom,
-		Identity: "c1",
-		Permission: &livekit.ParticipantPermission{
-			CanPublish:     false,
-			CanPublishData: true,
-			CanSubscribe:   true,
-		},
-	})
-	require.NoError(t, err)
+			// revoke permission
+			ctx := contextWithToken(adminRoomToken(testRoom))
+			_, err := roomClient.UpdateParticipant(ctx, &livekit.UpdateParticipantRequest{
+				Room:     testRoom,
+				Identity: "c1",
+				Permission: &livekit.ParticipantPermission{
+					CanPublish:     false,
+					CanPublishData: true,
+					CanSubscribe:   true,
+				},
+			})
+			require.NoError(t, err)
 
-	// ensure c1 no longer has track published, c2 no longer see track under C1
-	testutils.WithTimeout(t, func() string {
-		if len(c1.GetPublishedTrackIDs()) != 0 {
-			return "c1 did not unpublish tracks"
-		}
-		remoteC1 := c2.GetRemoteParticipant(c1.ID())
-		if remoteC1 == nil {
-			return "c2 doesn't know about c1"
-		}
-		if len(remoteC1.Tracks) != 0 {
-			return "c2 still has c1's tracks"
-		}
-		return ""
-	})
+			// ensure c1 no longer has track published, c2 no longer see track under C1
+			testutils.WithTimeout(t, func() string {
+				if len(c1.GetPublishedTrackIDs()) != 0 {
+					return "c1 did not unpublish tracks"
+				}
+				remoteC1 := c2.GetRemoteParticipant(c1.ID())
+				if remoteC1 == nil {
+					return "c2 doesn't know about c1"
+				}
+				if len(remoteC1.Tracks) != 0 {
+					return "c2 still has c1's tracks"
+				}
+				return ""
+			})
+		})
+	}
 }
 
 func TestCloseDisconnectedParticipantOnSignalClose(t *testing.T) {
 	_, _, finish := setupMultiNodeTest("TestCloseDisconnectedParticipantOnSignalClose")
 	defer finish()
 
-	c1 := createRTCClient("c1", secondServerPort, nil)
-	waitUntilConnected(t, c1)
+	for _, testRTCServicePath := range testRTCServicePaths {
+		t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) {
+			c1 := createRTCClient("c1", secondServerPort, testRTCServicePath, nil)
+			waitUntilConnected(t, c1)
 
-	c2 := createRTCClient("c2", defaultServerPort, &client.Options{
-		SignalRequestInterceptor: func(msg *livekit.SignalRequest, next client.SignalRequestHandler) error {
-			switch msg.Message.(type) {
-			case *livekit.SignalRequest_Offer, *livekit.SignalRequest_Answer, *livekit.SignalRequest_Leave:
-				return nil
-			default:
-				return next(msg)
-			}
-		},
-		SignalResponseInterceptor: func(msg *livekit.SignalResponse, next client.SignalResponseHandler) error {
-			switch msg.Message.(type) {
-			case *livekit.SignalResponse_Offer, *livekit.SignalResponse_Answer:
-				return nil
-			default:
-				return next(msg)
-			}
-		},
-	})
+			c2 := createRTCClient("c2", defaultServerPort, testRTCServicePath, &client.Options{
+				SignalRequestInterceptor: func(msg *livekit.SignalRequest, next client.SignalRequestHandler) error {
+					switch msg.Message.(type) {
+					case *livekit.SignalRequest_Offer, *livekit.SignalRequest_Answer, *livekit.SignalRequest_Leave:
+						return nil
+					default:
+						return next(msg)
+					}
+				},
+				SignalResponseInterceptor: func(msg *livekit.SignalResponse, next client.SignalResponseHandler) error {
+					switch msg.Message.(type) {
+					case *livekit.SignalResponse_Offer, *livekit.SignalResponse_Answer:
+						return nil
+					default:
+						return next(msg)
+					}
+				},
+			})
 
-	testutils.WithTimeout(t, func() string {
-		if len(c1.RemoteParticipants()) != 1 {
-			return "c1 did not see c2 join"
-		}
-		return ""
-	})
+			testutils.WithTimeout(t, func() string {
+				if len(c1.RemoteParticipants()) != 1 {
+					return "c1 did not see c2 join"
+				}
+				return ""
+			})
 
-	c2.Stop()
+			c2.Stop()
 
-	testutils.WithTimeout(t, func() string {
-		if len(c1.RemoteParticipants()) != 0 {
-			return "c1 did not see c2 removed"
-		}
-		return ""
-	})
+			testutils.WithTimeout(t, func() string {
+				if len(c1.RemoteParticipants()) != 0 {
+					return "c1 did not see c2 removed"
+				}
+				return ""
+			})
+		})
+	}
 }

livekit-server/test/scenarios.go

@@ -31,153 +31,286 @@ import (
 
 // a scenario with lots of clients connecting, publishing, and leaving at random periods
 func scenarioPublishingUponJoining(t *testing.T) {
-	c1 := createRTCClient("puj_1", defaultServerPort, nil)
-	c2 := createRTCClient("puj_2", secondServerPort, &testclient.Options{AutoSubscribe: true})
-	c3 := createRTCClient("puj_3", defaultServerPort, &testclient.Options{AutoSubscribe: true})
-	defer stopClients(c1, c2, c3)
+	for _, testRTCServicePath := range testRTCServicePaths {
+		t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) {
+			c1 := createRTCClient("puj_1", defaultServerPort, testRTCServicePath, nil)
+			c2 := createRTCClient("puj_2", secondServerPort, testRTCServicePath, &testclient.Options{AutoSubscribe: true})
+			c3 := createRTCClient("puj_3", defaultServerPort, testRTCServicePath, &testclient.Options{AutoSubscribe: true})
+			defer stopClients(c1, c2, c3)
 
-	waitUntilConnected(t, c1, c2, c3)
+			waitUntilConnected(t, c1, c2, c3)
 
-	// c1 and c2 publishing, c3 just receiving
-	writers := publishTracksForClients(t, c1, c2)
-	defer stopWriters(writers...)
+			// c1 and c2 publishing, c3 just receiving
+			writers := publishTracksForClients(t, c1, c2)
+			defer stopWriters(writers...)
 
-	logger.Infow("waiting to receive tracks from c1 and c2")
-	testutils.WithTimeout(t, func() string {
-		tracks := c3.SubscribedTracks()
-		if len(tracks[c1.ID()]) != 2 {
-			return "did not receive tracks from c1"
-		}
-		if len(tracks[c2.ID()]) != 2 {
-			return "did not receive tracks from c2"
-		}
-		return ""
-	})
+			logger.Infow("waiting to receive tracks from c1 and c2")
+			testutils.WithTimeout(t, func() string {
+				tracks := c3.SubscribedTracks()
+				if len(tracks[c1.ID()]) != 2 {
+					return "did not receive tracks from c1"
+				}
+				if len(tracks[c2.ID()]) != 2 {
+					return "did not receive tracks from c2"
+				}
+				return ""
+			})
 
-	// after a delay, c2 reconnects, then publishing
-	time.Sleep(syncDelay)
-	c2.Stop()
+			// after a delay, c2 reconnects, then publishing
+			time.Sleep(syncDelay)
+			c2.Stop()
 
-	logger.Infow("waiting for c2 tracks to be gone")
-	testutils.WithTimeout(t, func() string {
-		tracks := c3.SubscribedTracks()
+			logger.Infow("waiting for c2 tracks to be gone")
+			testutils.WithTimeout(t, func() string {
+				tracks := c3.SubscribedTracks()
 
-		if len(tracks[c1.ID()]) != 2 {
-			return fmt.Sprintf("c3 should be subscribed to 2 tracks from c1, actual: %d", len(tracks[c1.ID()]))
-		}
-		if len(tracks[c2.ID()]) != 0 {
-			return fmt.Sprintf("c3 should be subscribed to 0 tracks from c2, actual: %d", len(tracks[c2.ID()]))
-		}
-		if len(c1.SubscribedTracks()[c2.ID()]) != 0 {
-			return fmt.Sprintf("c3 should be subscribed to 0 tracks from c2, actual: %d", len(c1.SubscribedTracks()[c2.ID()]))
-		}
-		return ""
-	})
+				if len(tracks[c1.ID()]) != 2 {
+					return fmt.Sprintf("c3 should be subscribed to 2 tracks from c1, actual: %d", len(tracks[c1.ID()]))
+				}
+				if len(tracks[c2.ID()]) != 0 {
+					return fmt.Sprintf("c3 should be subscribed to 0 tracks from c2, actual: %d", len(tracks[c2.ID()]))
+				}
+				if len(c1.SubscribedTracks()[c2.ID()]) != 0 {
+					return fmt.Sprintf("c3 should be subscribed to 0 tracks from c2, actual: %d", len(c1.SubscribedTracks()[c2.ID()]))
+				}
+				return ""
+			})
 
-	logger.Infow("c2 reconnecting")
-	// connect to a diff port
-	c2 = createRTCClient("puj_2", defaultServerPort, nil)
-	defer c2.Stop()
-	waitUntilConnected(t, c2)
-	writers = publishTracksForClients(t, c2)
-	defer stopWriters(writers...)
+			logger.Infow("c2 reconnecting")
+			// connect to a diff port
+			c2 = createRTCClient("puj_2", defaultServerPort, testRTCServicePath, nil)
+			defer c2.Stop()
+			waitUntilConnected(t, c2)
+			writers = publishTracksForClients(t, c2)
+			defer stopWriters(writers...)
 
-	testutils.WithTimeout(t, func() string {
-		tracks := c3.SubscribedTracks()
-		// "new c2 tracks should be published again",
-		if len(tracks[c2.ID()]) != 2 {
-			return fmt.Sprintf("c3 should be subscribed to 2 tracks from c2, actual: %d", len(tracks[c2.ID()]))
-		}
-		if len(c1.SubscribedTracks()[c2.ID()]) != 2 {
-			return fmt.Sprintf("c1 should be subscribed to 2 tracks from c2, actual: %d", len(c1.SubscribedTracks()[c2.ID()]))
-		}
-		return ""
-	})
+			testutils.WithTimeout(t, func() string {
+				tracks := c3.SubscribedTracks()
+				// "new c2 tracks should be published again",
+				if len(tracks[c2.ID()]) != 2 {
+					return fmt.Sprintf("c3 should be subscribed to 2 tracks from c2, actual: %d", len(tracks[c2.ID()]))
+				}
+				if len(c1.SubscribedTracks()[c2.ID()]) != 2 {
+					return fmt.Sprintf("c1 should be subscribed to 2 tracks from c2, actual: %d", len(c1.SubscribedTracks()[c2.ID()]))
+				}
+				return ""
+			})
+		})
+	}
 }
 
 func scenarioReceiveBeforePublish(t *testing.T) {
-	c1 := createRTCClient("rbp_1", defaultServerPort, nil)
-	c2 := createRTCClient("rbp_2", defaultServerPort, nil)
+	for _, testRTCServicePath := range testRTCServicePaths {
+		t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) {
+			c1 := createRTCClient("rbp_1", defaultServerPort, testRTCServicePath, nil)
+			c2 := createRTCClient("rbp_2", defaultServerPort, testRTCServicePath, nil)
 
-	waitUntilConnected(t, c1, c2)
-	defer stopClients(c1, c2)
+			waitUntilConnected(t, c1, c2)
+			defer stopClients(c1, c2)
 
-	// c1 publishes
-	writers := publishTracksForClients(t, c1)
-	defer stopWriters(writers...)
+			// c1 publishes
+			writers := publishTracksForClients(t, c1)
+			defer stopWriters(writers...)
 
-	// c2 should see some bytes flowing through
-	testutils.WithTimeout(t, func() string {
-		if c2.BytesReceived() > 20 {
-			return ""
-		} else {
-			return fmt.Sprintf("c2 only received %d bytes", c2.BytesReceived())
-		}
-	})
+			// c2 should see some bytes flowing through
+			testutils.WithTimeout(t, func() string {
+				if c2.BytesReceived() > 20 {
+					return ""
+				} else {
+					return fmt.Sprintf("c2 only received %d bytes", c2.BytesReceived())
+				}
+			})
 
-	// now publish on C2
-	writers = publishTracksForClients(t, c2)
-	defer stopWriters(writers...)
+			// now publish on C2
+			writers = publishTracksForClients(t, c2)
+			defer stopWriters(writers...)
 
-	testutils.WithTimeout(t, func() string {
-		if len(c1.SubscribedTracks()[c2.ID()]) == 2 {
-			return ""
-		} else {
-			return fmt.Sprintf("expected c1 to receive 2 tracks from c2, actual: %d", len(c1.SubscribedTracks()[c2.ID()]))
-		}
-	})
+			testutils.WithTimeout(t, func() string {
+				if len(c1.SubscribedTracks()[c2.ID()]) == 2 {
+					return ""
+				} else {
+					return fmt.Sprintf("expected c1 to receive 2 tracks from c2, actual: %d", len(c1.SubscribedTracks()[c2.ID()]))
+				}
+			})
 
-	// now leave, and ensure that it's immediate
-	c2.Stop()
+			// now leave, and ensure that it's immediate
+			c2.Stop()
 
-	testutils.WithTimeout(t, func() string {
-		if len(c1.RemoteParticipants()) > 0 {
-			return fmt.Sprintf("expected no remote participants, actual: %v", c1.RemoteParticipants())
-		}
-		return ""
-	})
+			testutils.WithTimeout(t, func() string {
+				if len(c1.RemoteParticipants()) > 0 {
+					return fmt.Sprintf("expected no remote participants, actual: %v", c1.RemoteParticipants())
+				}
+				return ""
+			})
+		})
+	}
 }
 
 func scenarioDataPublish(t *testing.T) {
-	c1 := createRTCClient("dp1", defaultServerPort, nil)
-	c2 := createRTCClient("dp2", secondServerPort, nil)
-	waitUntilConnected(t, c1, c2)
-	defer stopClients(c1, c2)
+	for _, testRTCServicePath := range testRTCServicePaths {
+		t.Run(fmt.Sprintf("scenarioDataPublish/testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) {
+			c1 := createRTCClient("dp1", defaultServerPort, testRTCServicePath, nil)
+			c2 := createRTCClient("dp2", secondServerPort, testRTCServicePath, nil)
+			waitUntilConnected(t, c1, c2)
+			defer stopClients(c1, c2)
 
-	payload := "test bytes"
+			payload := "test bytes"
 
-	received := atomic.NewBool(false)
-	c2.OnDataReceived = func(data []byte, sid string) {
-		if string(data) == payload && livekit.ParticipantID(sid) == c1.ID() {
-			received.Store(true)
-		}
+			received := atomic.NewBool(false)
+			c2.OnDataReceived = func(data []byte, sid string) {
+				if string(data) == payload && livekit.ParticipantID(sid) == c1.ID() {
+					received.Store(true)
+				}
+			}
+
+			require.NoError(t, c1.PublishData([]byte(payload), livekit.DataPacket_RELIABLE))
+
+			testutils.WithTimeout(t, func() string {
+				if received.Load() {
+					return ""
+				} else {
+					return "c2 did not receive published data"
+				}
+			})
+		})
 	}
+}
 
-	require.NoError(t, c1.PublishData([]byte(payload), livekit.DataPacket_RELIABLE))
+func scenarioDataUnlabeledPublish(t *testing.T) {
+	for _, testRTCServicePath := range testRTCServicePaths {
+		t.Run(fmt.Sprintf("scenarioDataUnlabeledPublish/testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) {
+			c1 := createRTCClient("dp1", defaultServerPort, testRTCServicePath, nil)
+			c2 := createRTCClient("dp2", secondServerPort, testRTCServicePath, nil)
+			waitUntilConnected(t, c1, c2)
+			defer stopClients(c1, c2)
 
-	testutils.WithTimeout(t, func() string {
-		if received.Load() {
-			return ""
-		} else {
-			return "c2 did not receive published data"
-		}
-	})
+			payload := "test unlabeled bytes"
+
+			received := atomic.NewBool(false)
+			c2.OnDataReceived = func(data []byte, _sid string) {
+				if string(data) == payload {
+					received.Store(true)
+				}
+			}
+
+			require.NoError(t, c1.PublishDataUnlabeled([]byte(payload)))
+
+			testutils.WithTimeout(t, func() string {
+				if received.Load() {
+					return ""
+				} else {
+					return "c2 did not receive published data unlabeled"
+				}
+			})
+		})
+	}
+}
+
+func scenarioDataTracksPublishingUponJoining(t *testing.T) {
+	for _, testRTCServicePath := range testRTCServicePaths {
+		t.Run(fmt.Sprintf("scenarioDataTracksPublishingUponJoining/testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) {
+			c1 := createRTCClient("dtpuj_1", defaultServerPort, testRTCServicePath, &testclient.Options{AutoSubscribeDataTrack: true})
+			c2 := createRTCClient("dtpuj_2", secondServerPort, testRTCServicePath, &testclient.Options{AutoSubscribeDataTrack: true})
+			c3 := createRTCClient("dtpuj_3", defaultServerPort, testRTCServicePath, &testclient.Options{AutoSubscribeDataTrack: true})
+			defer stopClients(c1, c2, c3)
+
+			waitUntilConnected(t, c1, c2, c3)
+
+			// c1 and c2 publishing, c3 just receiving
+			writers := publishDataTracksForClients(t, c1, c2)
+			defer stopWriters(writers...)
+
+			logger.Infow("waiting to receive data tracks from c1 and c2")
+			testutils.WithTimeout(t, func() string {
+				tracks := c3.SubscribedDataTracks()
+				if len(tracks[c1.ID()]) != 2 {
+					return "did not receive data tracks from c1"
+				}
+				if len(tracks[c2.ID()]) != 2 {
+					return "did not receive data tracks from c2"
+				}
+				for _, dts := range tracks {
+					for _, dt := range dts {
+						if dt.NumReceivedPackets() == 0 {
+							return fmt.Sprintf("no packets received from %s", dt.ID())
+						}
+					}
+				}
+				return ""
+			})
+
+			// after a delay, c2 reconnects, then publishing
+			time.Sleep(syncDelay)
+			c2.Stop()
+
+			logger.Infow("waiting for c2 data tracks to be gone")
+			testutils.WithTimeout(t, func() string {
+				tracks := c3.SubscribedDataTracks()
+
+				if len(tracks[c1.ID()]) != 2 {
+					return fmt.Sprintf("c3 should be subscribed to 2 data tracks from c1, actual: %d", len(tracks[c1.ID()]))
+				}
+				if len(tracks[c2.ID()]) != 0 {
+					return fmt.Sprintf("c3 should be subscribed to 0 data tracks from c2, actual: %d", len(tracks[c2.ID()]))
+				}
+				if len(c1.SubscribedDataTracks()[c2.ID()]) != 0 {
+					return fmt.Sprintf("c3 should be subscribed to 0 data tracks from c2, actual: %d", len(c1.SubscribedTracks()[c2.ID()]))
+				}
+				return ""
+			})
+
+			logger.Infow("c2 reconnecting")
+			// connect to a diff port
+			c2 = createRTCClient("dtpuj_2", defaultServerPort, testRTCServicePath, &testclient.Options{AutoSubscribeDataTrack: true})
+			defer c2.Stop()
+			waitUntilConnected(t, c2)
+			writers = publishDataTracksForClients(t, c2)
+			defer stopWriters(writers...)
+
+			testutils.WithTimeout(t, func() string {
+				tracks := c3.SubscribedDataTracks()
+				// new c2 data tracks should be published again
+				if len(tracks[c2.ID()]) != 2 {
+					return fmt.Sprintf("c3 should be subscribed to 2 data tracks from c2, actual: %d", len(tracks[c2.ID()]))
+				}
+				for _, dt := range tracks[c2.ID()] {
+					if dt.NumReceivedPackets() == 0 {
+						return fmt.Sprintf("c3 did not receive packets from c2 data track after reconnecting %s", dt.ID())
+					}
+				}
+
+				if len(c1.SubscribedDataTracks()[c2.ID()]) != 2 {
+					return fmt.Sprintf("c1 should be subscribed to 2 data tracks from c2, actual: %d", len(c1.SubscribedTracks()[c2.ID()]))
+				}
+				for _, dt := range c1.SubscribedDataTracks()[c2.ID()] {
+					if dt.NumReceivedPackets() == 0 {
+						return fmt.Sprintf("c1 did not receive packets from c2 data track after reconnecting %s", dt.ID())
+					}
+				}
+				return ""
+			})
+		})
+	}
 }
 
 func scenarioJoinClosedRoom(t *testing.T) {
-	c1 := createRTCClient("jcr1", defaultServerPort, nil)
-	waitUntilConnected(t, c1)
+	for _, testRTCServicePath := range testRTCServicePaths {
+		t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) {
+			c1 := createRTCClient("jcr1", defaultServerPort, testRTCServicePath, nil)
+			waitUntilConnected(t, c1)
 
-	// close room with room client
-	_, err := roomClient.DeleteRoom(contextWithToken(createRoomToken()), &livekit.DeleteRoomRequest{
-		Room: testRoom,
-	})
-	require.NoError(t, err)
+			// close room with room client
+			_, err := roomClient.DeleteRoom(contextWithToken(createRoomToken()), &livekit.DeleteRoomRequest{
+				Room: testRoom,
+			})
+			require.NoError(t, err)
 
-	// now join again
-	c2 := createRTCClient("jcr2", defaultServerPort, nil)
-	waitUntilConnected(t, c2)
-	stopClients(c2)
+			// now join again
+			c2 := createRTCClient("jcr2", defaultServerPort, testRTCServicePath, nil)
+			waitUntilConnected(t, c2)
+			stopClients(c2)
+		})
+	}
 }
 
 // close a room that has been created, but no participant has joined
@@ -194,15 +327,15 @@ func closeNonRTCRoom(t *testing.T) {
 	require.NoError(t, err)
 }
 
-func publishTracksForClients(t *testing.T, clients ...*testclient.RTCClient) []*testclient.TrackWriter {
+func publishTracksForClients(t *testing.T, clients ...*testclient.RTCClient) []testclient.TrackWriter {
 	logger.Infow("publishing tracks for clients")
-	var writers []*testclient.TrackWriter
+	var writers []testclient.TrackWriter
 	for i := range clients {
 		c := clients[i]
 		tw, err := c.AddStaticTrack("audio/opus", "audio", "webcam")
 		require.NoError(t, err)
-
 		writers = append(writers, tw)
+
 		tw, err = c.AddStaticTrack("video/vp8", "video", "webcam")
 		require.NoError(t, err)
 		writers = append(writers, tw)
@@ -210,6 +343,20 @@ func publishTracksForClients(t *testing.T, clients ...*testclient.RTCClient) []*
 	return writers
 }
 
+func publishDataTracksForClients(t *testing.T, clients ...*testclient.RTCClient) []testclient.TrackWriter {
+	logger.Infow("publishing data tracks for clients")
+	var writers []testclient.TrackWriter
+	for i := range clients {
+		c := clients[i]
+		for range 2 {
+			dtw, err := c.PublishDataTrack()
+			require.NoError(t, err)
+			writers = append(writers, dtw)
+		}
+	}
+	return writers
+}
+
 // Room service tests
 
 func roomServiceListRoom(t *testing.T) {

livekit-server/test/webhook_test.go

@@ -25,12 +25,12 @@ import (
 	"time"
 
 	"github.com/stretchr/testify/require"
-	"google.golang.org/protobuf/encoding/protojson"
 
 	"github.com/livekit/protocol/auth"
 	"github.com/livekit/protocol/livekit"
 	"github.com/livekit/protocol/logger"
 	"github.com/livekit/protocol/utils/guid"
+	"github.com/livekit/protocol/utils/protojson"
 	"github.com/livekit/protocol/webhook"
 
 	"github.com/livekit/livekit-server/pkg/config"
@@ -45,78 +45,82 @@ func TestWebhooks(t *testing.T) {
 	require.NoError(t, err)
 	defer finish()
 
-	c1 := createRTCClient("c1", defaultServerPort, nil)
-	waitUntilConnected(t, c1)
-	testutils.WithTimeout(t, func() string {
-		if ts.GetEvent(webhook.EventRoomStarted) == nil {
-			return "did not receive RoomStarted"
-		}
-		if ts.GetEvent(webhook.EventParticipantJoined) == nil {
-			return "did not receive ParticipantJoined"
-		}
-		return ""
-	})
+	for _, testRTCServicePath := range testRTCServicePaths {
+		t.Run(fmt.Sprintf("testRTCServicePath=%s", testRTCServicePath.String()), func(t *testing.T) {
+			c1 := createRTCClient("c1", defaultServerPort, testRTCServicePath, nil)
+			waitUntilConnected(t, c1)
+			testutils.WithTimeout(t, func() string {
+				if ts.GetEvent(webhook.EventRoomStarted) == nil {
+					return "did not receive RoomStarted"
+				}
+				if ts.GetEvent(webhook.EventParticipantJoined) == nil {
+					return "did not receive ParticipantJoined"
+				}
+				return ""
+			})
 
-	// first participant join should have started the room
-	started := ts.GetEvent(webhook.EventRoomStarted)
-	require.Equal(t, testRoom, started.Room.Name)
-	require.NotEmpty(t, started.Id)
-	require.Greater(t, started.CreatedAt, time.Now().Unix()-100)
-	require.GreaterOrEqual(t, time.Now().Unix(), started.CreatedAt)
-	joined := ts.GetEvent(webhook.EventParticipantJoined)
-	require.Equal(t, "c1", joined.Participant.Identity)
-	ts.ClearEvents()
+			// first participant join should have started the room
+			started := ts.GetEvent(webhook.EventRoomStarted)
+			require.Equal(t, testRoom, started.Room.Name)
+			require.NotEmpty(t, started.Id)
+			require.Greater(t, started.CreatedAt, time.Now().Unix()-100)
+			require.GreaterOrEqual(t, time.Now().Unix(), started.CreatedAt)
+			joined := ts.GetEvent(webhook.EventParticipantJoined)
+			require.Equal(t, "c1", joined.Participant.Identity)
+			ts.ClearEvents()
 
-	// another participant joins
-	c2 := createRTCClient("c2", defaultServerPort, nil)
-	waitUntilConnected(t, c2)
-	defer c2.Stop()
-	testutils.WithTimeout(t, func() string {
-		if ts.GetEvent(webhook.EventParticipantJoined) == nil {
-			return "did not receive ParticipantJoined"
-		}
-		return ""
-	})
-	joined = ts.GetEvent(webhook.EventParticipantJoined)
-	require.Equal(t, "c2", joined.Participant.Identity)
-	ts.ClearEvents()
+			// another participant joins
+			c2 := createRTCClient("c2", defaultServerPort, testRTCServicePath, nil)
+			waitUntilConnected(t, c2)
+			defer c2.Stop()
+			testutils.WithTimeout(t, func() string {
+				if ts.GetEvent(webhook.EventParticipantJoined) == nil {
+					return "did not receive ParticipantJoined"
+				}
+				return ""
+			})
+			joined = ts.GetEvent(webhook.EventParticipantJoined)
+			require.Equal(t, "c2", joined.Participant.Identity)
+			ts.ClearEvents()
 
-	// track published
-	writers := publishTracksForClients(t, c1)
-	defer stopWriters(writers...)
-	testutils.WithTimeout(t, func() string {
-		ev := ts.GetEvent(webhook.EventTrackPublished)
-		if ev == nil {
-			return "did not receive TrackPublished"
-		}
-		require.NotNil(t, ev.Track, "TrackPublished did not include trackInfo")
-		require.Equal(t, string(c1.ID()), ev.Participant.Sid)
-		return ""
-	})
-	ts.ClearEvents()
+			// track published
+			writers := publishTracksForClients(t, c1)
+			defer stopWriters(writers...)
+			testutils.WithTimeout(t, func() string {
+				ev := ts.GetEvent(webhook.EventTrackPublished)
+				if ev == nil {
+					return "did not receive TrackPublished"
+				}
+				require.NotNil(t, ev.Track, "TrackPublished did not include trackInfo")
+				require.Equal(t, string(c1.ID()), ev.Participant.Sid)
+				return ""
+			})
+			ts.ClearEvents()
 
-	// first participant leaves
-	c1.Stop()
-	testutils.WithTimeout(t, func() string {
-		if ts.GetEvent(webhook.EventParticipantLeft) == nil {
-			return "did not receive ParticipantLeft"
-		}
-		return ""
-	})
-	left := ts.GetEvent(webhook.EventParticipantLeft)
-	require.Equal(t, "c1", left.Participant.Identity)
-	ts.ClearEvents()
+			// first participant leaves
+			c1.Stop()
+			testutils.WithTimeout(t, func() string {
+				if ts.GetEvent(webhook.EventParticipantLeft) == nil {
+					return "did not receive ParticipantLeft"
+				}
+				return ""
+			})
+			left := ts.GetEvent(webhook.EventParticipantLeft)
+			require.Equal(t, "c1", left.Participant.Identity)
+			ts.ClearEvents()
 
-	// room closed
-	rm := server.RoomManager().GetRoom(context.Background(), testRoom)
-	rm.Close(types.ParticipantCloseReasonNone)
-	testutils.WithTimeout(t, func() string {
-		if ts.GetEvent(webhook.EventRoomFinished) == nil {
-			return "did not receive RoomFinished"
-		}
-		return ""
-	})
-	require.Equal(t, testRoom, ts.GetEvent(webhook.EventRoomFinished).Room.Name)
+			// room closed
+			rm := server.RoomManager().GetRoom(context.Background(), testRoom)
+			rm.Close(types.ParticipantCloseReasonNone)
+			testutils.WithTimeout(t, func() string {
+				if ts.GetEvent(webhook.EventRoomFinished) == nil {
+					return "did not receive RoomFinished"
+				}
+				return ""
+			})
+			require.Equal(t, testRoom, ts.GetEvent(webhook.EventRoomFinished).Room.Name)
+		})
+	}
 }
 
 func setupServerWithWebhook() (server *service.LivekitServer, testServer *webhookTestServer, finishFunc func(), err error) {