Unity WebRTC: Building Real-Time Communication in Interactive 3D Applications

1using Unity.WebRTC;
2using UnityEngine;
3using System.Collections.Generic;
4
5public class WebRTCManager : MonoBehaviour
6{
7    RTCPeerConnection localPeerConnection;
8    RTCPeerConnection remotePeerConnection;
9    
10    // Video components
11    public RawImage localVideoImage;
12    public RawImage remoteVideoImage;
13    
14    MediaStream localStream;
15    
16    void Start()
17    {
18        // Initialize WebRTC
19        WebRTC.Initialize();
20        
21        // Create peer connections
22        var configuration = new RTCConfiguration
23        {
24            iceServers = new[] { new RTCIceServer { urls = "stun:stun.l.google.com:19302" } }
25        };
26        
27        localPeerConnection = new RTCPeerConnection(ref configuration);
28        
29        // Set up event handlers
30        localPeerConnection.OnIceCandidate = candidate => {
31            // Send candidate to remote peer via signaling server
32        };
33        
34        localPeerConnection.OnTrack = e => {
35            if (e.Track.Kind == TrackKind.Video)
36            {
37                remoteVideoImage.texture = e.Streams[0].GetVideoTracks()[0].Texture;
38            }
39        };
40    }
41    
42    public void StartLocalStream()
43    {
44        // Create local media stream
45        localStream = new MediaStream();
46        
47        // Add video track
48        var videoTrack = new VideoTrack(WebRTCUtils.CreateVideoSource());
49        localStream.AddTrack(videoTrack);
50        
51        // Display local video
52        localVideoImage.texture = videoTrack.Texture;
53        
54        // Add tracks to peer connection
55        foreach (var track in localStream.GetTracks())
56        {
57            localPeerConnection.AddTrack(track, localStream);
58        }
59    }
60    
61    void OnDestroy()
62    {
63        // Clean up resources
64        localPeerConnection?.Close();
65        remotePeerConnection?.Close();
66        WebRTC.Dispose();
67    }
68}
69

1// Simplified signaling example (would typically use WebSockets or similar)
2public async void CreateOffer()
3{
4    var offerOptions = new RTCOfferOptions
5    {
6        iceRestart = false,
7        offerToReceiveAudio = true,
8        offerToReceiveVideo = true
9    };
10    
11    var offer = await localPeerConnection.CreateOffer(ref offerOptions);
12    await localPeerConnection.SetLocalDescription(ref offer);
13    
14    // Send offer to remote peer via signaling server
15    SendOfferToSignalingServer(offer);
16}
17
18public async void HandleReceivedAnswer(RTCSessionDescription answer)
19{
20    await localPeerConnection.SetRemoteDescription(ref answer);
21}
22
23public void HandleRemoteCandidate(RTCIceCandidate candidate)
24{
25    localPeerConnection.AddIceCandidate(candidate);
26}
27

1// Create a custom rendering path for WebRTC video
2public class OptimizedVideoRenderer : MonoBehaviour
3{
4    public RenderTexture targetTexture;
5    private Camera renderCamera;
6    
7    void Start()
8    {
9        renderCamera = GetComponent<Camera>();
10        renderCamera.targetTexture = targetTexture;
11        
12        // Configure for optimal performance
13        renderCamera.clearFlags = CameraClearFlags.SolidColor;
14        renderCamera.backgroundColor = Color.black;
15        renderCamera.allowHDR = false;
16        renderCamera.allowMSAA = false;
17    }
18    
19    // Control rendering frame rate based on WebRTC needs
20    public void SetRenderFrameRate(int frameRate)
21    {
22        StartCoroutine(RenderFrameRoutine(frameRate));
23    }
24    
25    IEnumerator RenderFrameRoutine(int frameRate)
26    {
27        float frameTime = 1f / frameRate;
28        while (true)
29        {
30            renderCamera.Render();
31            yield return new WaitForSeconds(frameTime);
32        }
33    }
34}
35

1// Implement adaptive quality based on network conditions
2public void SetBitrateLimit(uint bitrate)
3{
4    foreach (var sender in peerConnection.GetSenders())
5    {
6        if (sender.Track.Kind == TrackKind.Video)
7        {
8            RTCRtpSendParameters parameters = sender.GetParameters();
9            foreach (var encoding in parameters.encodings)
10            {
11                encoding.maxBitrate = bitrate;
12            }
13            sender.SetParameters(parameters);
14        }
15    }
16}
17
18// Monitor connection quality
19public void MonitorConnectionQuality()
20{
21    StartCoroutine(ConnectionQualityRoutine());
22}
23
24IEnumerator ConnectionQualityRoutine()
25{
26    while (true)
27    {
28        peerConnection.GetStats(stats => {
29            foreach (var report in stats.Values)
30            {
31                if (report.Type == RTCStatsType.InboundRtp && report.Kind == TrackKind.Video)
32                {
33                    float packetLoss = report.GetFloat("packetsLost") / report.GetFloat("packetsReceived");
34                    
35                    if (packetLoss > 0.05f)
36                    {
37                        // Reduce quality if packet loss is high
38                        SetBitrateLimit(500000); // 500kbps
39                    }
40                    else if (packetLoss < 0.01f)
41                    {
42                        // Increase quality if connection is good
43                        SetBitrateLimit(2000000); // 2Mbps
44                    }
45                }
46            }
47        });
48        
49        yield return new WaitForSeconds(2f);
50    }
51}
52

1RTCDataChannel dataChannel;
2
3public void SetupDataChannel()
4{
5    var dataChannelInit = new RTCDataChannelInit
6    {
7        ordered = true,
8        maxRetransmits = 0
9    };
10    
11    dataChannel = peerConnection.CreateDataChannel("gameData", dataChannelInit);
12    dataChannel.OnOpen = () => Debug.Log("Data channel opened");
13    dataChannel.OnClose = () => Debug.Log("Data channel closed");
14    dataChannel.OnMessage = HandleDataMessage;
15}
16
17// Handle incoming game sync data
18void HandleDataMessage(byte[] bytes)
19{
20    // Deserialize game state update
21    GameStateUpdate update = GameStateUpdate.FromBytes(bytes);
22    
23    // Apply to game objects
24    ApplyGameStateUpdate(update);
25}
26
27// Send game state updates
28public void SendGameState(GameStateUpdate update)
29{
30    if (dataChannel != null && dataChannel.ReadyState == RTCDataChannelState.Open)
31    {
32        byte[] data = update.ToBytes();
33        dataChannel.Send(data);
34    }
35}
36

1// Comprehensive ICE server configuration
2var configuration = new RTCConfiguration
3{
4    iceServers = new[]
5    {
6        new RTCIceServer { urls = "stun:stun.l.google.com:19302" },
7        new RTCIceServer { urls = "stun:stun1.l.google.com:19302" },
8        new RTCIceServer
9        {
10            urls = "turn:your-turn-server.com:443?transport=tcp",
11            username = "yourUsername",
12            credential = "yourPassword"
13        }
14    },
15    iceTransportPolicy = RTCIceTransportPolicy.All,
16    bundlePolicy = RTCBundlePolicy.MaxBundle
17};
18

1// Mobile-specific optimizations
2public void OptimizeForMobile()
3{
4    // Detect if running on mobile
5    if (Application.platform == RuntimePlatform.Android || 
6        Application.platform == RuntimePlatform.IPhonePlayer)
7    {
8        // Reduce resolution for mobile
9        int width = 640;
10        int height = 480;
11        int frameRate = 24;
12        
13        // Configure video constraints
14        var videoTrackConstraints = new RTCRtpEncodingParameters
15        {
16            maxFramerate = frameRate,
17            scaleResolutionDownBy = 1.0
18        };
19        
20        // Apply to all video senders
21        foreach (var sender in peerConnection.GetSenders())
22        {
23            if (sender.Track.Kind == TrackKind.Video)
24            {
25                var parameters = sender.GetParameters();
26                parameters.encodings[0] = videoTrackConstraints;
27                sender.SetParameters(parameters);
28            }
29        }
30        
31        // Monitor battery level
32        StartCoroutine(MonitorBatteryLevel());
33    }
34}
35
36IEnumerator MonitorBatteryLevel()
37{
38    while (true)
39    {
40        // If battery is low, further reduce quality
41        if (SystemInfo.batteryLevel < 0.2f)
42        {
43            SetBitrateLimit(250000); // 250kbps
44        }
45        
46        yield return new WaitForSeconds(30f);
47    }
48}
49

1using System.Collections;
2using Unity.WebRTC;
3using UnityEngine;
4using UnityEngine.UI;
5
6public class WebRTCManager : MonoBehaviour
7{
8    [Header("Video Displays")]
9    public RawImage localVideoDisplay;
10    public RawImage remoteVideoDisplay;
11    
12    [Header("Controls")]
13    public Button startButton;
14    public Button callButton;
15    
16    private RTCPeerConnection localPeerConnection;
17    private RTCPeerConnection remotePeerConnection; // For testing in single app
18    private MediaStream localStream;
19    
20    // Normally this would be handled by a signaling server
21    private RTCSessionDescription offerSdp;
22    private RTCSessionDescription answerSdp;
23    private List<RTCIceCandidate> pendingCandidates = new List<RTCIceCandidate>();
24    
25    void Start()
26    {
27        // Initialize WebRTC
28        WebRTC.Initialize();
29        
30        // Set up button listeners
31        startButton.onClick.AddListener(StartVideo);
32        callButton.onClick.AddListener(StartCall);
33        callButton.interactable = false;
34        
35        // Configure automatic WebRTC lifecycle
36        StartCoroutine(WebRTCUpdateRoutine());
37    }
38    
39    IEnumerator WebRTCUpdateRoutine()
40    {
41        while (true)
42        {
43            yield return new WaitForSeconds(0.1f);
44            WebRTC.Update();
45        }
46    }
47    
48    public void StartVideo()
49    {
50        StartCoroutine(SetupLocalStream());
51    }
52    
53    IEnumerator SetupLocalStream()
54    {
55        // Request camera/mic permissions if needed (platform specific)
56        
57        // Create local stream with video track
58        localStream = new MediaStream();
59        var videoTrack = new VideoStreamTrack();
60        localStream.AddTrack(videoTrack);
61        
62        // Display local video
63        localVideoDisplay.texture = videoTrack.Texture;
64        
65        // Enable call button once video is ready
66        callButton.interactable = true;
67        
68        yield return null;
69    }
70    
71    public void StartCall()
72    {
73        StartCoroutine(EstablishPeerConnection());
74    }
75    
76    IEnumerator EstablishPeerConnection()
77    {
78        // Create peer connection config with STUN server
79        var config = new RTCConfiguration
80        {
81            iceServers = new[] { new RTCIceServer { urls = "stun:stun.l.google.com:19302" } }
82        };
83        
84        // Create local peer connection
85        localPeerConnection = new RTCPeerConnection(ref config);
86        
87        // In a real application, you would only create one peer connection
88        // This is for demonstration in a single app
89        remotePeerConnection = new RTCPeerConnection(ref config);
90        
91        // Set up local connection handlers
92        localPeerConnection.OnIceCandidate = candidate => {
93            Debug.Log($"Local ICE candidate: {candidate.Candidate}");
94            // In production, send to remote peer via signaling
95            StartCoroutine(AddRemoteIceCandidate(candidate));
96        };
97        
98        // Set up remote connection handlers
99        remotePeerConnection.OnIceCandidate = candidate => {
100            Debug.Log($"Remote ICE candidate: {candidate.Candidate}");
101            // In production, these would come from the remote peer
102            StartCoroutine(AddLocalIceCandidate(candidate));
103        };
104        
105        // Handle incoming tracks on remote peer
106        remotePeerConnection.OnTrack = e => {
107            Debug.Log($"Received remote track: {e.Track.Kind}");
108            if (e.Track is VideoStreamTrack videoTrack)
109            {
110                remoteVideoDisplay.texture = videoTrack.Texture;
111            }
112        };
113        
114        // Add tracks from local stream to peer connection
115        foreach (var track in localStream.GetTracks())
116        {
117            localPeerConnection.AddTrack(track, localStream);
118        }
119        
120        // Create and set the offer
121        var offerOptions = new RTCOfferOptions { offerToReceiveVideo = true };
122        var offer = localPeerConnection.CreateOffer(ref offerOptions);
123        yield return new WaitUntil(() => offer.IsDone);
124        
125        // Set local description
126        var offerDesc = offer.Desc;
127        localPeerConnection.SetLocalDescription(ref offerDesc);
128        yield return new WaitUntil(() => localPeerConnection.SignalingState != RTCSignalingState.HaveLocalOffer);
129        
130        Debug.Log("Offer created and set as local description");
131        
132        // In a real application, the offer would be sent to the remote peer through a signaling server
133        // For this example, we'll directly set it on the remote peer connection
134        var remoteOfferDesc = offer.Desc;
135        remotePeerConnection.SetRemoteDescription(ref remoteOfferDesc);
136        yield return new WaitUntil(() => remotePeerConnection.SignalingState != RTCSignalingState.Stable);
137        
138        // Create and set answer
139        var answerOptions = new RTCAnswerOptions();
140        var answer = remotePeerConnection.CreateAnswer(ref answerOptions);
141        yield return new WaitUntil(() => answer.IsDone);
142        
143        var answerDesc = answer.Desc;
144        remotePeerConnection.SetLocalDescription(ref answerDesc);
145        yield return new WaitUntil(() => remotePeerConnection.SignalingState != RTCSignalingState.HaveLocalOffer);
146        
147        // In a real application, the answer would be sent back to the local peer
148        // For this example, we'll directly set it
149        var localAnswerDesc = answer.Desc;
150        localPeerConnection.SetRemoteDescription(ref localAnswerDesc);
151        
152        Debug.Log("WebRTC connection established!");
153    }
154    
155    IEnumerator AddRemoteIceCandidate(RTCIceCandidate candidate)
156    {
157        // Wait for remote description to be set
158        yield return new WaitUntil(() => remotePeerConnection.RemoteDescription != null);
159        remotePeerConnection.AddIceCandidate(candidate);
160    }
161    
162    IEnumerator AddLocalIceCandidate(RTCIceCandidate candidate)
163    {
164        // Wait for local description to be set
165        yield return new WaitUntil(() => localPeerConnection.RemoteDescription != null);
166        localPeerConnection.AddIceCandidate(candidate);
167    }
168    
169    void OnDestroy()
170    {
171        // Clean up WebRTC resources
172        localPeerConnection?.Close();
173        remotePeerConnection?.Close();
174        WebRTC.Dispose();
175    }
176}
177