技术背景
我们在对接Unity平台camera场景采集的时候，除了常规的RTMP推送、录像外，还有一些开发者，需要能实现轻量级RTSP服务，对外提供个拉流的RTSP URL。
目前我们在Windows平台Unity下数据源可采集到以下部分：
- 采集Unity camera场景；
- 采集摄像头；
- 采集屏幕；
- 采集Unity声音；
- 采集麦克风；
- 采集扬声器；
- Unity PCM混音；
对外提供的技术能力有：
- RTMP直播推送；
- 轻量级RTSP服务；
- 实时录像、暂停|恢复录像；
- 实时预览。
技术实现
实际上，在实现Unity平台音视频能力之前，我们原生模块已经有非常成熟的技术积累，Unity下还是调用的原生的推送模块，不同的是，数据源需要采集Unity的audio、video，然后高效的投递到底层模块，底层模块负责编码打包，并投递到RTMP或RTSP服务。
先说支持的音视频类型：
public void SelVideoPushType(int type)
{
switch (type)
{
case 0:
video_push_type_ = (uint)NTSmartPublisherDefine.NT_PB_E_VIDEO_OPTION.NT_PB_E_VIDEO_OPTION_LAYER; //采集Unity窗体
break;
case 1:
video_push_type_ = (uint)NTSmartPublisherDefine.NT_PB_E_VIDEO_OPTION.NT_PB_E_VIDEO_OPTION_CAMERA; //采集摄像头
break;
case 2:
video_push_type_ = (uint)NTSmartPublisherDefine.NT_PB_E_VIDEO_OPTION.NT_PB_E_VIDEO_OPTION_SCREEN; //采集屏幕
break;
case 3:
video_push_type_ = (uint)NTSmartPublisherDefine.NT_PB_E_VIDEO_OPTION.NT_PB_E_VIDEO_OPTION_NO_VIDEO; //不采集视频
break;
}
Debug.Log("SelVideoPushType type: " + type + " video_push_type: " + video_push_type_);
}
public void SelAudioPushType(int type)
{
switch (type)
{
case 0:
audio_push_type_ = (uint)NTSmartPublisherDefine.NT_PB_E_AUDIO_OPTION.NT_PB_E_AUDIO_OPTION_EXTERNAL_PCM_DATA; //采集Unity声音
break;
case 1:
audio_push_type_ = (uint)NTSmartPublisherDefine.NT_PB_E_AUDIO_OPTION.NT_PB_E_AUDIO_OPTION_CAPTURE_MIC; //采集麦克风
break;
case 2:
audio_push_type_ = (uint)NTSmartPublisherDefine.NT_PB_E_AUDIO_OPTION.NT_PB_E_AUDIO_OPTION_CAPTURE_SPEAKER; //采集扬声器
break;
case 3:
audio_push_type_ = (uint)NTSmartPublisherDefine.NT_PB_E_AUDIO_OPTION.NT_PB_E_AUDIO_OPTION_TWO_EXTERNAL_PCM_MIXER; //两路Unity AudioClip混音测试
break;
case 4:
audio_push_type_ = (uint)NTSmartPublisherDefine.NT_PB_E_AUDIO_OPTION.NT_PB_E_AUDIO_OPTION_NO_AUDIO; //不采集音频
break;
}
Debug.Log("SelAudioPushType type: " + type + " audio_push_type: " + audio_push_type_);
}采集音视频数据：
private void StartCaptureAvData()
{
if (audio_push_type_ == (uint)NTSmartPublisherDefine.NT_PB_E_AUDIO_OPTION.NT_PB_E_AUDIO_OPTION_EXTERNAL_PCM_DATA
|| audio_push_type_ == (uint)NTSmartPublisherDefine.NT_PB_E_AUDIO_OPTION.NT_PB_E_AUDIO_OPTION_TWO_EXTERNAL_PCM_MIXER)
{
PostUnityAudioClipData();
}
textures_poll_ = new TexturesPool();
post_image_worker_ = new PostImageWorker(textures_poll_, publisher_wrapper_);
post_worker_thread_ = new Thread(post_image_worker_.run);
post_worker_thread_.Start();
}
private void StopCaptureAvData()
{
if (post_image_worker_ != null)
{
post_image_worker_.requestStop();
post_image_worker_ = null;
}
if (post_worker_thread_ != null)
{
post_worker_thread_.Join();
post_worker_thread_ = null;
}
if (textures_poll_ != null)
{
textures_poll_.clear();
textures_poll_ = null;
}
StopAudioSource();
}RTMP推送：
public void btn_start_rtmp_pusher_Click()
{
if (publisher_wrapper_.IsPushingRtmp())
{
StopPushRTMP();
btn_rtmp_pusher_.GetComponentInChildren<Text>().text = "推送RTMP";
return;
}
String url = rtmp_pusher_url_.text;
if (url.Length < 8)
{
publisher_wrapper_.Close();
Debug.LogError("请输入RTMP推送地址");
return;
}
if (!publisher_wrapper_.IsPreviewing() && !publisher_wrapper_.IsRtspPublisherRunning() && !publisher_wrapper_.IsRecording())
{
publisher_wrapper_.SetVideoPushType(video_push_type_);
publisher_wrapper_.SetAudioPushType(audio_push_type_);
}
if (!publisher_wrapper_.StartRtmpPusher(url))
{
Debug.LogError("调用StartPublisher失败..");
return;
}
btn_rtmp_pusher_.GetComponentInChildren<Text>().text = "停止推送";
if (!publisher_wrapper_.IsPreviewing() && !publisher_wrapper_.IsRtspPublisherRunning() && !publisher_wrapper_.IsRecording())
{
StartCaptureAvData();
coroutine_ = StartCoroutine(OnPostVideo());
}
}轻量级RTSP服务相关调用：
public void btn_rtsp_service_Click()
{
if (publisher_wrapper_.IsRtspServiceRunning())
{
publisher_wrapper_.StopRtspService();
btn_rtsp_service_.GetComponentInChildren<Text>().text = "启动RTSP服务";
btn_rtsp_publisher_.interactable = false;
return;
}
if (!publisher_wrapper_.StartRtspService())
{
Debug.LogError("调用StartRtspService失败..");
return;
}
btn_rtsp_publisher_.interactable = true;
btn_rtsp_service_.GetComponentInChildren<Text>().text = "停止RTSP服务";
}
public void btn_rtsp_publisher_Click()
{
if (publisher_wrapper_.IsRtspPublisherRunning())
{
publisher_wrapper_.StopRtspStream();
if (!publisher_wrapper_.IsPreviewing() && !publisher_wrapper_.IsPushingRtmp() && !publisher_wrapper_.IsRecording())
{
StopCaptureAvData();
if (coroutine_ != null)
{
StopCoroutine(coroutine_);
coroutine_ = null;
}
}
btn_rtsp_service_.interactable = true;
btn_rtsp_publisher_.GetComponentInChildren<Text>().text = "发布RTSP";
}
else
{
if (!publisher_wrapper_.IsRtspServiceRunning())
{
Debug.LogError("RTSP service is not running..");
return;
}
if (!publisher_wrapper_.IsPreviewing() && !publisher_wrapper_.IsPushingRtmp() && !publisher_wrapper_.IsRecording())
{
publisher_wrapper_.SetVideoPushType(video_push_type_);
publisher_wrapper_.SetAudioPushType(audio_push_type_);
}
publisher_wrapper_.StartRtspStream();
if (!publisher_wrapper_.IsPreviewing() && !publisher_wrapper_.IsPushingRtmp() && !publisher_wrapper_.IsRecording())
{
StartCaptureAvData();
coroutine_ = StartCoroutine(OnPostVideo());
}
btn_rtsp_publisher_.GetComponentInChildren<Text>().text = "停止RTSP";
btn_rtsp_service_.interactable = false;
}
}
public void btn_get_rtsp_session_numbers_Click()
{
if (publisher_wrapper_.IsRtspServiceRunning())
{
btn_get_rtsp_session_numbers_.GetComponentInChildren<Text>().text = "RTSP会话数:" + publisher_wrapper_.GetRtspSessionNumbers();
}
}实时录像调用：
public void btn_record_Click()
{
if (publisher_wrapper_.IsRecording())
{
StopRecord();
btn_record_.GetComponentInChildren<Text>().text = "开始录像";
return;
}
if (!publisher_wrapper_.IsPreviewing() && !publisher_wrapper_.IsRtspPublisherRunning() && !publisher_wrapper_.IsPushingRtmp())
{
publisher_wrapper_.SetVideoPushType(video_push_type_);
publisher_wrapper_.SetAudioPushType(audio_push_type_);
}
if (!publisher_wrapper_.StartRecorder())
{
Debug.LogError("调用StartRecorder失败..");
return;
}
btn_record_.GetComponentInChildren<Text>().text = "停止录像";
if (!publisher_wrapper_.IsPreviewing() && !publisher_wrapper_.IsRtspPublisherRunning() && !publisher_wrapper_.IsPushingRtmp())
{
StartCaptureAvData();
coroutine_ = StartCoroutine(OnPostVideo());
}
}
public void StopRecord()
{
if (!publisher_wrapper_.IsPreviewing() && !publisher_wrapper_.IsRtspPublisherRunning() && !publisher_wrapper_.IsPushingRtmp())
{
StopCaptureAvData();
if (coroutine_ != null)
{
StopCoroutine(coroutine_);
coroutine_ = null;
}
}
publisher_wrapper_.StopRecorder();
}
private void btn_pause_record_Click()
{
if (!publisher_wrapper_.IsPublisherHandleAvailable())
{
return;
}
String btn_pause_rec_text = btn_pause_record_.GetComponentInChildren<Text>().text;
if ("暂停录像" == btn_pause_rec_text)
{
UInt32 ret = publisher_wrapper_.PauseRecorder(true);
if ((UInt32)NT.NTSmartPublisherDefine.NT_PB_E_ERROR_CODE.NT_ERC_PB_NEED_RETRY == ret)
{
Debug.LogError("暂停录像失败, 请重新尝试!");
return;
}
else if (NTBaseCodeDefine.NT_ERC_OK == ret)
{
btn_pause_record_.GetComponentInChildren<Text>().text = "恢复录像";
}
}
else
{
UInt32 ret = publisher_wrapper_.PauseRecorder(false);
if ((UInt32)NT.NTSmartPublisherDefine.NT_PB_E_ERROR_CODE.NT_ERC_PB_NEED_RETRY == ret)
{
Debug.LogError("恢复录像失败, 请重新尝试!");
return;
}
else if (NTBaseCodeDefine.NT_ERC_OK == ret)
{
btn_pause_record_.GetComponentInChildren<Text>().text = "暂停录像";
}
}
}实时预览相关：
public void btn_preview_Click()
{
if (btn_preview_.GetComponentInChildren<Text>().text.Equals("本地预览"))
{
if (!publisher_wrapper_.IsPushingRtmp() && !publisher_wrapper_.IsRtspPublisherRunning() && !publisher_wrapper_.IsRecording())
{
publisher_wrapper_.SetVideoPushType(video_push_type_);
}
if (publisher_wrapper_.StartPreview())
{
btn_preview_.GetComponentInChildren<Text>().text = "停止预览";
}
if (!publisher_wrapper_.IsPushingRtmp() && !publisher_wrapper_.IsRtspPublisherRunning() && !publisher_wrapper_.IsRecording())
{
StartCaptureAvData();
coroutine_ = StartCoroutine(OnPostVideo());
}
}
else
{
if (!publisher_wrapper_.IsPushingRtmp() && !publisher_wrapper_.IsRtspPublisherRunning() && !publisher_wrapper_.IsRecording())
{
StopCaptureAvData();
if (coroutine_ != null)
{
StopCoroutine(coroutine_);
coroutine_ = null;
}
}
publisher_wrapper_.StopPreview();
btn_preview_.GetComponentInChildren<Text>().text = "本地预览";
}
}总结
Unity平台下RTMP推送、录像、轻量级RTSP服务，在虚拟仿真、医疗、教育等场景下，应用非常广泛。要实现低延迟，除了需要高效率的音视频数据采集，编码和数据投递外，还需要好的直播播放器支持。配合我们的SmartPlayer，可轻松实现毫秒级体验，满足绝大多数应用场景技术诉求。