[BUG]: OnIceCandidate Event Handler Starts Second

[Steve-Morales]

Package version

3.0.0-pre.5

Environment

* OS: Windows 11
* Unity version: 2021.3.22f1

Steps To Reproduce

Install Newsoft-JSON https://github.com/jilleJr/Newtonsoft.Json-for-Unity/wiki/Install-official-via-UPM

Install Native Web Socker https://github.com/endel/NativeWebSocket

In Unity, add the following script through a new empty gameobject and add the main camera (or duplicate camera)

using System.Collections;
using UnityEngine;
using Unity.WebRTC;
using NativeWebSocket;
using Newtonsoft.Json.Linq;
using System;
using UnityEngine.UI;

public class RTC_Peer : MonoBehaviour
{
    public RawImage remoteVideo;
    public Camera capture_camera;

    public string ip = "192.168.1.77";
    public string port = "8080";

    private RTCPeerConnection pc;
    private WebSocket ws;
    private void OnDestroy()
    {
        pc.Close();

        WebRTC.Dispose();
    }

    private void Awake()
    {
        // Initialize WebRTC
        WebRTC.Initialize();

        RTCConfiguration config = new RTCConfiguration
        {
            iceServers = new[] { new RTCIceServer { urls = new[] { "stun:stun.l.google.com:19302" } } }
        };
        pc = new RTCPeerConnection(ref config);

        ws = new WebSocket($"ws://{ip}:{port}");

        pc.OnIceCandidate = (candidate) =>
        {
            Debug.Log("[pc.OnIceCandidate] " + JSON_Stringify(candidate));
            ws.SendText(JSON_Stringify(candidate));
        };

        pc.OnNegotiationNeeded = () => {
            StartCoroutine(PeerNegotiationNeeded());
        };

        var receiveStream = new MediaStream();
        receiveStream.OnAddTrack = e =>
        {
            if (e.Track is VideoStreamTrack video_track)
            {
                // You can access received texture using `track.Texture` property.
                remoteVideo.texture = video_track.Texture;
            }
            else if (e.Track is AudioStreamTrack audio_track)
            {
                // This track is for audio.
            }
        };

        pc.OnTrack = (e) =>
        {
            if (e.Track.Kind == TrackKind.Video)
            {
                // Add track to MediaStream for receiver.
                // This process triggers `OnAddTrack` event of `MediaStream`.
                receiveStream.AddTrack(e.Track);
            }
        };
    }

    IEnumerator PeerNegotiationNeeded()
    {
        var offerOp = pc.CreateOffer();
        yield return offerOp;

        if (!offerOp.IsError)
        {
            if (pc.SignalingState != RTCSignalingState.Stable)
            {
                Debug.LogError($"signaling state is not stable.");
                yield break;
            }

            /* ON SUCCESSFUL OFFER -- SET LOCAL DESCRIPTION*/
            RTCSessionDescription offerDesc = offerOp.Desc;
            var setLocalDescOp = pc.SetLocalDescription(ref offerDesc);
            yield return setLocalDescOp;

            if (!setLocalDescOp.IsError)
            {
                Debug.Log("Local Description Set!");
            }
            else
            {
                var error = setLocalDescOp.Error;
                Debug.LogError($"Error Detail Type: {error.message}");
                // TODO: HangUp()
                yield break;
            }

            /* SEND THIS PEER'S LOCAL DESCRIPTION */
            ws.SendText(JSON_Stringify(pc.LocalDescription));
        }
        else
        {
            var error = offerOp.Error;
            Debug.LogError($"Error Detail Type: {error.message}");
        }
    }

    // Start is called before the first frame update
    async void Start()
    {
        StartCoroutine(WebRTC.Update());
        ws.OnOpen += () =>
        {
            Debug.Log("Connection open!");

            // Add Media
            if (capture_camera == null) { Debug.LogError("[ERROR] There is no camera!"); }
            var track = capture_camera.CaptureStreamTrack(1280, 720);
            pc.AddTrack(track);
            //MediaStream video_stream_track = capture_camera.CaptureStream(1280, 720);
            //foreach (var track in video_stream_track.GetTracks())
            //{
            //    pc.AddTrack(track, video_stream_track);
            //    Debug.Log("[Unity] Video track added to local_connection: " + track.Id);
            //}
        };

        ws.OnError += (e) =>
        {
            Debug.Log("Error! " + e);
        };

        ws.OnClose += (e) =>
        {
            Debug.Log("Connection closed!");
        };

        ws.OnMessage += (bytes) =>
        {
            Debug.Log("OnMessage!");
            Debug.Log(bytes);

            var bytes_data = System.Text.Encoding.UTF8.GetString(bytes);
            Debug.Log("OnMessage! " + bytes_data);

            JObject message = JObject.Parse(bytes_data);

            if (message.ContainsKey("candidate"))
            {
                IceCandidateKeys p = JsonUtility.FromJson<IceCandidateKeys>(message["candidate"].ToString());
                if (p != null) 
                {
                    RTCIceCandidateInit candidate_info = new RTCIceCandidateInit
                    {
                        candidate = p.candidate,
                        sdpMid = p.sdpMid,
                        sdpMLineIndex = p.sdpMLineIndex
                    };
                    RTCIceCandidate candidate = new RTCIceCandidate(candidate_info);
                    StartCoroutine(AddIceCandidate(candidate));

                }
                else 
                {
                    //Debug.LogWarning("parsed candidate is null!");
                    //// Append "a=end-of-candidates" to the SDP
                    //string sdp = pc.LocalDescription.sdp + "a=end-of-candidates\r\n";
                    //Debug.Log("Modified SDP: " + sdp);

                    //RTCSessionDescription desc = new RTCSessionDescription
                    //{
                    //    type = RTCSdpType.Offer,
                    //    sdp = sdp
                    //};
                    //pc.SetLocalDescription(ref desc);
                    ws.SendText("{\"candidate\": null}");
                }

            }
            else if (message.ContainsKey("sdp"))
            {

                string sdp_string = message["sdp"]["sdp"].ToString();
                string type_string = message["sdp"]["type"].ToString();
                //Debug.Log($"Items: {json_string}");
                StartCoroutine(HandleIncomingSDP(sdp_string, type_string));
            }
        };

        await ws.Connect();
    }

    IEnumerator HandleIncomingSDP(string sdp_string, string type_string)
    {
        RTCSessionDescription remoteSDP = new RTCSessionDescription { sdp = sdp_string, type = ToRTCSdpType(type_string) };
        var setRemoteDescOp = pc.SetRemoteDescription(ref remoteSDP);
        yield return setRemoteDescOp;

        if (!setRemoteDescOp.IsError)
        {
            Debug.Log("Remote Description Set!");
        }
        else
        {
            var error = setRemoteDescOp.Error;
            Debug.LogError($"Error Detail Type: {error.message}");
            // TODO: HangUp()
            yield break;
        }

        if (remoteSDP.type == RTCSdpType.Offer)
        {
            var answerOp = pc.CreateAnswer();
            yield return answerOp;

            if (!answerOp.IsError)
            {
                /* ON SUCCESSFUL ANSWER -- SET LOCAL DESCRIPTION*/
                RTCSessionDescription answerDesc = answerOp.Desc;
                var setLocalDescOp = pc.SetLocalDescription(ref answerDesc);
                yield return setLocalDescOp;

                if (!setLocalDescOp.IsError)
                {
                    Debug.Log("Local Description Set!");
                }
                else
                {
                    var error = setLocalDescOp.Error;
                    Debug.LogError($"Error Detail Type: {error.message}");
                    // TODO: HangUp()
                    yield break;
                }

                /* SEND THIS PEER'S LOCAL DESCRIPTION */
                ws.SendText(JSON_Stringify(pc.LocalDescription));
            }
            else
            {
                var error = answerOp.Error;
                Debug.LogError($"Error Detail Type: {error.message}");
            }
        }
    }

    IEnumerator AddIceCandidate(RTCIceCandidate candidate)
    {
        var addOp = pc.AddIceCandidate(candidate);
        yield return addOp;
    }

    // Update is called once per frame
    void Update()
    {
        #if !UNITY_WEBGL || UNITY_EDITOR
                ws.DispatchMessageQueue();
        #endif
    }

    [Serializable]
    public class IceCandidateMessage
    {
        public IceCandidateKeys candidate;
    }

    [Serializable]
    public class IceCandidateKeys
    {
        public string candidate;
        public string sdpMid;
        public int sdpMLineIndex;
        public string usernameFragment;
    }
    private string JSON_Stringify(RTCIceCandidate candidate)
    {
        IceCandidateKeys parse = new IceCandidateKeys
        {
            candidate = candidate.Candidate,
            sdpMid = candidate.SdpMid,
            sdpMLineIndex = (int)candidate.SdpMLineIndex,
            usernameFragment = candidate.UserNameFragment
        };

        IceCandidateMessage message = new IceCandidateMessage
        {
            candidate = parse
        };

        string json = JsonUtility.ToJson(message);

        return json;
    }

    [Serializable]
    public class SDP_Message
    {
        public SDP_Keys sdp;
    }

    [Serializable]
    public class SDP_Keys
    {
        public string type;
        public string sdp;
    }
    private string JSON_Stringify(RTCSessionDescription localDescription)
    {
        SDP_Keys parse = new SDP_Keys
        {
            type = localDescription.type.ToString().ToLower(),
            sdp = localDescription.sdp
        };

        SDP_Message desc = new SDP_Message
        {
            sdp = parse
        };

        string json = JsonUtility.ToJson(desc);

        return json;
    }

    private RTCSdpType ToRTCSdpType(string typeString)
    {
        if (typeString.ToLower() == "offer")
        {
            return RTCSdpType.Offer;
        }
        else if (typeString.ToLower() == "pranswer")
        {
            return RTCSdpType.Pranswer;
        }
        else if (typeString.ToLower() == "rollback")
        {
            return RTCSdpType.Rollback;
        }
        else if (typeString.ToLower() == "answer")
        {
            return RTCSdpType.Answer;
        }

        Debug.LogError($"[ToRTCSdpType] Given String \"{typeString}\" Does Not Match Any Enum -- Returning 0");
        return 0;
    }
}

Create a Nodejs server npm i to install (only do this once) npm start to start server with the following code

const signalingServerUrl = 'ws://192.168.1.77:8080';
const localVideo = document.getElementById('localVideo');
const remoteVideo = document.getElementById('remoteVideo');
const localBtn = document.getElementById('add-local-btn');

const constraints = {
    video: {
        cursor: 'always',
        frameRate: { ideal: 30, max: 30 },
    },
};

// WebRTC configuration
const configuration = {
    iceServers: [
        {
            urls: 'stun:stun.l.google.com:19302',
        },
    ],
};

const pc = new RTCPeerConnection(configuration);
const ws = new WebSocket(signalingServerUrl);

// Send any ice candidates to the other peer
pc.onicecandidate = ({ candidate }) => {
    /*
    If the candidate parameter is missing or a value of null is given when 
    calling addIceCandidate(), the added ICE candidate is an "end-of-candidates"
    indicator. The same is the case if the value of the specified object's 
    candidate is either missing or an empty string (""), it signals that all 
    remote candidates have been delivered.
    - https://developer.mozilla.org/en-US/docs/Web/API/RTCPeerConnection/addIceCandidate
    */
   console.log("[pc.onicecandidate] ", candidate);
    ws.send(JSON.stringify({ candidate }));
};

// Let the "negotiationneeded" event trigger offer generation
pc.onnegotiationneeded = async () => {
    const offer = await pc.createOffer();
    await pc.setLocalDescription(offer);
    ws.send(JSON.stringify({ sdp: pc.localDescription }));
};

// Once remote stream arrives, show it in the remote video element
pc.ontrack = (event) => {
    remoteVideo.srcObject = event.streams[0];
};

ws.onmessage = async ({ data }) => {
    let message;

    if (data instanceof Blob) {
        message = await data.text();
    } else {
        message = data;
    }

    console.log("[ws.onmessage]", message)
    const { candidate, sdp } = JSON.parse(message);

    if (candidate) {
        console.log("[ws.onmessage - candidate] ",candidate);
        if(candidate == null){ console.error("Got Null Candidate"); }
        await pc.addIceCandidate(candidate);
    } else if (sdp) {
        const remoteSdp = new RTCSessionDescription(sdp);
        await pc.setRemoteDescription(remoteSdp);

        if (remoteSdp.type === 'offer') {
            const answer = await pc.createAnswer();
            await pc.setLocalDescription(answer);
            ws.send(JSON.stringify({ sdp: pc.localDescription }));
        }
    }
};

ws.onopen = async () => {
    console.log('WebSocket connected');

};

localBtn.addEventListener('click', (e)=>{
    console.log("readystate", ws.readyState, "open:", ws.OPEN, "?", (ws.OPEN == ws.readyState))
    if(ws.OPEN != ws.readyState){ 
        console.log('Still Attempting To Connect, Cannot Add Media!'); 
        return;
    }
    AddLocalMedia();
})

// adds local media
// fires onnegotiationneeded event handler
async function AddLocalMedia(){
    try {
        const stream = await navigator.mediaDevices.getDisplayMedia(constraints);
        localVideo.srcObject = stream;

        stream.getTracks().forEach((track) => {
            pc.addTrack(track, stream);
        });
    } catch (err) {
        console.error('Error accessing screen:', err);
    }
}

With a browser client (HTML/JS) index.html (open this file in your browser)

<!DOCTYPE html>
<html>
  <head>
    <title>WebRTC Screen Sharing</title>
  </head>
  <body>
    <button id="add-local-btn">Add Local Video</button>
    <h1>Local</h1>
    <video id="localVideo" autoplay playsinline width="768" height="432"></video>
    <h1>Remote</h1>    
    <video id="remoteVideo" autoplay playsinline width="768" height="432"></video>

    <script src="client.js"></script>
  </body>
</html>

client.js

const signalingServerUrl = 'ws://192.168.1.77:8080';
const localVideo = document.getElementById('localVideo');
const remoteVideo = document.getElementById('remoteVideo');
const localBtn = document.getElementById('add-local-btn');

const constraints = {
    video: {
        cursor: 'always',
        frameRate: { ideal: 30, max: 30 },
    },
};

// WebRTC configuration
const configuration = {
    iceServers: [
        {
            urls: 'stun:stun.l.google.com:19302',
        },
    ],
};

const pc = new RTCPeerConnection(configuration);
const ws = new WebSocket(signalingServerUrl);

// Send any ice candidates to the other peer
pc.onicecandidate = ({ candidate }) => {
    /*
    If the candidate parameter is missing or a value of null is given when 
    calling addIceCandidate(), the added ICE candidate is an "end-of-candidates"
    indicator. The same is the case if the value of the specified object's 
    candidate is either missing or an empty string (""), it signals that all 
    remote candidates have been delivered.
    - https://developer.mozilla.org/en-US/docs/Web/API/RTCPeerConnection/addIceCandidate
    */
   console.log("[pc.onicecandidate] ", candidate);
    ws.send(JSON.stringify({ candidate }));
};

// Let the "negotiationneeded" event trigger offer generation
pc.onnegotiationneeded = async () => {
    const offer = await pc.createOffer();
    await pc.setLocalDescription(offer);
    ws.send(JSON.stringify({ sdp: pc.localDescription }));
};

// Once remote stream arrives, show it in the remote video element
pc.ontrack = (event) => {
    remoteVideo.srcObject = event.streams[0];
};

ws.onmessage = async ({ data }) => {
    let message;

    if (data instanceof Blob) {
        message = await data.text();
    } else {
        message = data;
    }

    console.log("[ws.onmessage]", message)
    const { candidate, sdp } = JSON.parse(message);

    if (candidate) {
        console.log("[ws.onmessage - candidate] ",candidate);
        if(candidate == null){ console.error("Got Null Candidate"); }
        await pc.addIceCandidate(candidate);
    } else if (sdp) {
        const remoteSdp = new RTCSessionDescription(sdp);
        await pc.setRemoteDescription(remoteSdp);

        if (remoteSdp.type === 'offer') {
            const answer = await pc.createAnswer();
            await pc.setLocalDescription(answer);
            ws.send(JSON.stringify({ sdp: pc.localDescription }));
        }
    }
};

ws.onopen = async () => {
    console.log('WebSocket connected');

};

localBtn.addEventListener('click', (e)=>{
    console.log("readystate", ws.readyState, "open:", ws.OPEN, "?", (ws.OPEN == ws.readyState))
    if(ws.OPEN != ws.readyState){ 
        console.log('Still Attempting To Connect, Cannot Add Media!'); 
        return;
    }
    AddLocalMedia();
})

// adds local media
// fires onnegotiationneeded event handler
async function AddLocalMedia(){
    try {
        const stream = await navigator.mediaDevices.getDisplayMedia(constraints);
        localVideo.srcObject = stream;

        stream.getTracks().forEach((track) => {
            pc.addTrack(track, stream);
        });
    } catch (err) {
        console.error('Error accessing screen:', err);
    }
}

Current Behavior

Unity is sending the SDP first which triggers the OnIceCandidate event handler.

Expected Behavior

Unity should determine ICE Candidates (OnIceCandidate) then send the SDP.

Anything else?

This should be a direct translation of the JavaScript client WebRTC code. The additional libraries make sending message easy. The purpose is for Bidirectional Video sharing.

[sepah99]

Im having the same issue :(

[gtk2k]

If you comment out the OnNegotiationNeeded event handler on the Unity side, it will work for the time being

[Steve-Morales]

If you comment out the OnNegotiationNeeded event handler on the Unity side, it will work for the time being

The issue is that it needs to run to be able to find ICE candidates. Please try setting it up on your end.

[gtk2k]

According to the WebRTC API specification, iceCandidate generation starts after executing setLocalDescription(). So running setLocalDescription() should start IceCandidate generation and trigger the onIceCandidate event.

[kannan-xiao4]

@Steve-Morales Is this issue same as https://github.com/Unity-Technologies/com.unity.webrtc/issues/347 ?

Our stats represent this comment. https://github.com/Unity-Technologies/com.unity.webrtc/issues/347#issuecomment-1189741386