webrtc for unity have question

105 views

Skip to first unread message

Taomeng tao

unread,

Sep 10, 2023, 1:00:32 AM9/10/23

to discuss-webrtc

Hi, i have some question.i use the Node.js to be server.And i'm sure that the two client has exchange the sdp,but why i haven't the remote video?Is there anyone can solve my problem ? i have be crazy for the code.

using UnityEngine;
using UnityEngine.UI;
using Unity.WebRTC;
using WebSocketSharp;
using System;
using System.Collections;
using System.Collections.Generic;

class Client5 : MonoBehaviour
{
[SerializeField] private RawImage localVideoImage;
[SerializeField] private RawImage remoteVideoImage;
[SerializeField] private Button startButton;
[SerializeField] private Button endButton;

[Serializable]
public class CommonMessage
{
public string cmd; // 用于标识消息类型
public string localUid; // 消息中的本地用户 ID
public string remoteUid;
public string msg;
public string mediaType;
// 存储 offer 的 SDP 字符串，可以根据消息类型进行扩展
// 可以根据需要添加其他字段来表示不同消息类型的数据
}
private Queue<CommonMessage> messageQueue = new Queue<CommonMessage>();
private bool isRemoteVideoTextureReady = false;
// 在类级别声明 audioStreamTrack 变量
private AudioStreamTrack audioStreamTrack;
private RTCRtpReceiver audioReceiver; // 在类的顶部定义 RTCRtpReceiver 变量
public AudioSource remoteAudio; // 在类的顶部定义 AudioSource 变量
private RTCPeerConnection pc;
private MediaStream videoStream;
private MediaStream remoteVideoStream;
private WebCamTexture webCamTexture;

private WebSocket ws;
private string localUid;
private string remoteUid;

public Texture localVideoTexture;
public Texture remoteVideoTexture;

private const string serverUrl = "ws://172.30.50.49:8080";

private void Start()
{

localUid = UniqueIdGenerator.GenerateUniqueId();
startButton.onClick.AddListener(StartCall);
endButton.onClick.AddListener(EndCall);
ConnectionSignalingServer();
}
private void StartCall()
{
DoJoin();

}
private void DoJoin()
{
// Send a join message to the server
CommonMessage message = new CommonMessage
{
cmd = "join",
localUid = localUid,
};
string msg = JsonUtility.ToJson(message);
ws.Send(msg);
Debug.Log("Do Join message: " + msg);

InitializeLocalStream();
CreatePeerConnection();
}

private void InitializeLocalStream()
{
videoStream = new MediaStream();
//CaptureAudioStart();
CaptureVideoStart();
}
private void CreatePeerConnection()
{
RTCConfiguration configuration = GetSelectedSdpSemantics();
configuration.iceServers = new[]
{
new RTCIceServer
{
urls = new[] { "stun:172.30.50.49:8080" },

},
};

pc = new RTCPeerConnection(ref configuration);

pc.OnIceCandidate = HandleRemoteOnIceCandidate;
pc.OnTrack = HandleRemoteStreamAdd;
pc.OnIceConnectionChange = HandleConnectionChange;
foreach (var tracks in videoStream.GetTracks())
{
pc.AddTrack(tracks, videoStream);
Debug.Log("Added track to PeerConnection: " + tracks.Kind+"track of id:"+tracks.Id);
Debug.Log("add track value" + tracks.ReadyState);
}

//pc.OnIceCandidate = HandleRemoteOnIceCandidate;
}

private void ConnectionSignalingServer()
{
ws = new WebSocket(serverUrl);
ws.OnOpen += HandleOnOpen;
ws.OnMessage += HandleOnMessage;
ws.OnError += HandleOnError;
ws.OnClose += HandleOnClose;

ws.Connect();
}
private void HandleOnOpen(object sender, EventArgs e)
{
Debug.Log("WebSocket opened");
}
private void HandleOnMessage(object sender, MessageEventArgs e)
{
try
{
CommonMessage commonMessage = JsonUtility.FromJson<CommonMessage>(e.Data);
Debug.Log("onMessage: " + commonMessage);

// 将接收到的消息添加到队列中
messageQueue.Enqueue(commonMessage);

// 然后在这里处理队列中的消息
ProcessMessageQueue();
}
catch (Exception ex)
{
Debug.LogError("Exception during message handling: " + ex.Message);
}
}
private void ProcessMessageQueue()
{
while (messageQueue.Count > 0)
{
CommonMessage message = messageQueue.Dequeue();

switch (message.cmd)
{
case "new-peer":
HandleNewPeer(message);
break;
case "resp-join":
HandleResponseJoin(message);
break;
case "peer-leave":
HandlePeerLeave(message);
break;
case "offer":
HandleRemoteOffer(message);
break;
case "answer":
HandleRemoteAnswer(message);
break;
case "candidate":
HandleRemoteCandidate(message);
break;
// Handle other message types as needed
}
}
}
private void HandleOnError(object sender, ErrorEventArgs e)
{
Debug.LogError("WebSocket Error: " + e.Message);
}

private void HandleOnClose(object sender, CloseEventArgs e)
{
Debug.Log("WebSocket Closed: " + e.Reason);
}

private void HandleConnectionChange(RTCIceConnectionState state)
{
Debug.Log("ICE连接状态变化: " + state);

// 在这里执行与连接状态相关的操作
if (state == RTCIceConnectionState.Connected)
{
// 连接已建立
Debug.Log("ICE连接状态变化:connected " );

}
else if (state == RTCIceConnectionState.Disconnected)
{
Debug.Log("ICE连接状态变化:Disconnected ");
// 连接已断开
}
else if (state == RTCIceConnectionState.Failed)
{
// 连接失败
Debug.LogError("ICE连接状态变化: " + state);
}
// 可以根
}

private RTCConfiguration GetSelectedSdpSemantics()
{
RTCConfiguration config = default;
config.bundlePolicy = RTCBundlePolicy.BundlePolicyMaxBundle;
config.iceTransportPolicy = RTCIceTransportPolicy.All;
config.iceServers = new[]
{ new RTCIceServer
{
urls = new[]
{
"stun:stun.l.google.com:19302"
},

},

};
return config;
}

private void CaptureAudioStart()
{

var audioSource = gameObject.AddComponent<AudioSource>();
audioSource.loop = true;
audioStreamTrack = new AudioStreamTrack(audioSource);
videoStream.AddTrack(audioStreamTrack); // 添加音频轨道到 videoStream
}
private void CaptureVideoStart()
{
if (WebCamTexture.devices.Length == 0)
{
Debug.Log("WebCam device not found");
return;
}

int width = 1280;
int height = 720;
const int fps = 60;
WebCamDevice userCameraDevice = WebCamTexture.devices[0];
webCamTexture = new WebCamTexture(userCameraDevice.name, width, height, fps);
webCamTexture.Play();

videoStream.AddTrack(new VideoStreamTrack(webCamTexture));
localVideoImage.texture = webCamTexture;

}

private void HandleNewPeer(CommonMessage message)
{
remoteUid = message.remoteUid;
GenerateOfferAndSendToServer();
}
private void GenerateOfferAndSendToServer()
{
// Create an Offer and wait for it to be generated
RTCSessionDescriptionAsyncOperation offerOperation = pc.CreateOffer(ref RTCOfferAnswerOptions.Default);
offerOperation.Completed += OnOfferCompleted;
}
private void HandleResponseJoin(CommonMessage message)
{
Debug.Log("HandleResponseJoin , the old one is user:" + message.remoteUid);
remoteUid = message.remoteUid;
}

private void HandlePeerLeave(CommonMessage message)
{
string msg = message.remoteUid;
Debug.Log("handlePeerLeave 收到消息" + msg + "这个用户走了");
}
private void OnOfferCompleted(RTCSessionDescription offer)
{
Debug.Log("Offer created: " + offer.sdp);
pc.SetLocalDescription(ref offer);
SendOfferToServer(offer);
}
private void SendOfferToServer(RTCSessionDescription sessionDescription)
{
CommonMessage sdpJsonMsg = new CommonMessage
{
cmd = "offer",
localUid = localUid,
remoteUid = remoteUid,
msg = sessionDescription.sdp
};
string sdpJsonStringMsg = JsonUtility.ToJson(sdpJsonMsg);
ws.Send(sdpJsonStringMsg);
Debug.Log("SendOfferToServer message: " + sdpJsonMsg);
}
private void HandleRemoteOffer(CommonMessage message)
{

OnRemoteOfferReceived(message);

}

private void OnRemoteOfferReceived(CommonMessage message)
{
RTCSessionDescription desc = new RTCSessionDescription
{
type = RTCSdpType.Offer,
sdp = message.msg
};

if (pc == null)
{
CreatePeerConnection();
}

pc.SetRemoteDescription(ref desc);

RTCSessionDescriptionAsyncOperation answerOperation = pc.CreateAnswer(ref RTCOfferAnswerOptions.Default);

answerOperation.Completed += OnAnswerCompleted;
}
private void OnAnswerCompleted(RTCSessionDescription answer)
{
Debug.Log("Answer created: " + answer.sdp);
pc.SetLocalDescription(ref answer);
SendAnswerToServer(answer);
}

private void SendAnswerToServer(RTCSessionDescription sessionDescription)
{
CommonMessage sdpJsonMsg = new CommonMessage
{
cmd = "answer",
localUid = localUid,
remoteUid = remoteUid,
msg = sessionDescription.sdp
};
string sdpJsonStringMsg = JsonUtility.ToJson(sdpJsonMsg);
ws.Send(sdpJsonStringMsg);
Debug.Log("SendAnswerToServer message: " + sdpJsonMsg);
}

private void HandleRemoteAnswer(CommonMessage message)
{
RTCSessionDescription desc = new RTCSessionDescription
{
type = RTCSdpType.Answer,
sdp = message.msg
};
try
{
pc.SetRemoteDescription(ref desc);
Debug.Log("Received and set remote answer successfully");
// 在这里设置 ICE 候选监听器

}
catch (Exception ex)
{
Debug.LogError("Error while setting remote answer: " + ex.Message);
}
}
private void HandleRemoteCandidate(CommonMessage message)
{
try
{
if (pc != null)
{
RTCIceCandidateInit candidateInit = new RTCIceCandidateInit
{
candidate = message.msg,
sdpMid = message.mediaType,
sdpMLineIndex = 0
};

RTCIceCandidate candidate = new RTCIceCandidate(candidateInit);
pc.AddIceCandidate(candidate);

// 添加调试输出以查看接收到的ICE候选信息
Debug.Log("Received ICE candidate: " + message.msg);

}
else
{
Debug.LogWarning("PeerConnection is null when handling remote candidate.");
}
}
catch (Exception ex)
{
Debug.LogError("Error while handling remote candidate: " + ex.Message);
}
}

private void HandleRemoteOnIceCandidate(RTCIceCandidate candidate)
{
try
{
if (string.IsNullOrEmpty(candidate.Candidate))
{
Debug.LogError("Candidate does not exist");
return;
}

string mediaType = "video";

if (!string.IsNullOrEmpty(candidate.SdpMid))
{
mediaType = candidate.SdpMid;
}

CommonMessage msg = new CommonMessage
{
cmd = "candidate",
localUid = localUid,
remoteUid = remoteUid,
msg = candidate.Candidate,
mediaType = mediaType
};

string jsonMsg = JsonUtility.ToJson(msg);
ws.Send(jsonMsg);
// 添加调试输出以查看发送的ICE候选信息
Debug.Log("Sent ICE candidate message: " + candidate.Candidate);
}
catch (Exception ex)
{
Debug.LogError("Error while handling ICE candidate: " + ex.Message);
}
}
private void HandleRemoteStreamAdd(RTCTrackEvent ev)
{
Debug.Assert(ev.Track != null, "Remote video track is null.");

if (ev.Track != null)
{

if (ev.Track is VideoStreamTrack video)
{
video.OnVideoReceived += tex =>
{
remoteVideoImage.texture = tex;
};
}
else if (ev.Track is AudioStreamTrack audio)
{
remoteAudio.SetTrack(audio);
remoteAudio.loop = true;
remoteAudio.Play();
}
}
else
{
Debug.Log("Received null track in RTCTrackEvent.");
}
}
/* private void StartWaitingForRemoteVideoTexture()
{
StartCoroutine(WaitForRemoteVideoTexture());
}
private IEnumerator WaitForRemoteVideoTexture()
{
float timeout = 60.0f; // 设置一个超时时间，单位为秒
float elapsedTime = 0;
while (!isRemoteVideoTextureReady&&elapsedTime<timeout)
{
var remoteVideoTrack = GetRemoteVideoTrack();
if (remoteVideoTrack != null)
{
var videoTexture = remoteVideoTrack.Texture;
if (videoTexture != null)
{
remoteVideoImage.texture = videoTexture;
isRemoteVideoTextureReady = true;
Debug.Log("Remote video stream added.");
yield break; // 纹理已准备好，退出等待
}
}

// 等待一帧
yield return null;
elapsedTime += Time.deltaTime;
}
if (!isRemoteVideoTextureReady)
{
Debug.LogError("Timeout while waiting for remote video texture.");
}
}*/
private VideoStreamTrack GetRemoteVideoTrack()
{
foreach (var receiver in pc.GetReceivers())
{
var track = receiver.Track;
if (track is VideoStreamTrack)
{
return (VideoStreamTrack)track;
}
}
return null;
}

private void DoLeave()
{
if (pc != null)
{
pc.Close();
pc.Dispose();
pc = null;
}

CommonMessage message = new CommonMessage
{
cmd = "leave",
localUid = localUid
};
string msg = JsonUtility.ToJson(message);
ws.Send(msg);
Debug.Log("do leave message:" + msg);
}
private void EndCall()
{
DoLeave();
if (webCamTexture != null)
{
webCamTexture.Stop();
webCamTexture = null;
}

if (videoStream != null)
{
foreach (var track in videoStream.GetTracks())
{
track.Stop();
track.Dispose();
}
videoStream.Dispose();
videoStream = null;
}

if (remoteVideoStream != null)
{
foreach (var track in remoteVideoStream.GetTracks())
{
track.Stop();
track.Dispose();
}
remoteVideoStream.Dispose();
remoteVideoStream = null;
}

if (pc != null)
{
pc.Close();
pc.Dispose();
pc = null;
}

if (ws != null)
{
ws.Close();
ws = null;
}
}
}

Muhammad Usman Bashir

unread,

Sep 23, 2023, 9:51:23 AM9/23/23

to discuss-webrtc

Please share the console log file here. Add logging around signaling and ICE candidate exchange.