Android端如何实现拉取RTSP/RTMP流并回调YUV/RGB数据然后注入轻量级RTSP服务？

标签：return libPublisher int RTSP RGB TAG publisherHandle 轻量级

技术背景

我们在对接开发Android平台音视频模块的时候，遇到过这样的问题，厂商希望拉取到海康、大华等摄像机的RTSP流，然后解码后的YUV或RGB数据回给他们，他们做视频分析或处理后，再投递给轻量级RTSP服务模块或RTMP推送模块，实现处理后的数据，二次转发，本文以拉取RTSP流，解析后再注入轻量级RTSP服务为例，介绍下大概的技术实现。

技术实现

废话不多说，无图无真相，下图是测试的时候，Android终端拉取RTSP流，然后把YUV数据回调上来，又通过推送接口，注入到轻量级RTSP服务，然后Windows平台拉取轻量级RTSP的URL，整体下来，毫秒级延迟：

Android端如何实现拉取RTSP/RTMP流并回调YUV/RGB数据然后注入轻量级RTSP服务？_轻量级RTSP服务

先说拉取RTSP流，需要注意的是，如果不要播放的话，可以SetSurface()的时候，第二个参数设置null，如果不需要audio的话，直接SetMute设置1即可，因为需要回调YUV上来，那么设置下I420回调，如果需要RGB的，只要开RGB的回调即可。

	private boolean StartPlay()
	{
		if (!OpenPullHandle())
			return false;

		// 如果第二个参数设置为null，则播放纯音频
		libPlayer.SmartPlayerSetSurface(playerHandle, sSurfaceView);

		libPlayer.SmartPlayerSetRenderScaleMode(playerHandle, 1);

    // libPlayer.SmartPlayerSetExternalRender(playerHandle, new
		// RGBAExternalRender());
		 libPlayer.SmartPlayerSetExternalRender(playerHandle, new
		 I420ExternalRender());

		libPlayer.SmartPlayerSetFastStartup(playerHandle, isFastStartup ? 1 : 0);

		libPlayer.SmartPlayerSetAudioOutputType(playerHandle, 1);

		if (isMute) {
			libPlayer.SmartPlayerSetMute(playerHandle, isMute ? 1
					: 0);
		}

		if (isHardwareDecoder)
		{
			int isSupportH264HwDecoder = libPlayer
					.SetSmartPlayerVideoHWDecoder(playerHandle, 1);

			int isSupportHevcHwDecoder = libPlayer.SetSmartPlayerVideoHevcHWDecoder(playerHandle, 1);

			Log.i(TAG, "isSupportH264HwDecoder: " + isSupportH264HwDecoder + ", isSupportHevcHwDecoder: " + isSupportHevcHwDecoder);
		}

		libPlayer.SmartPlayerSetLowLatencyMode(playerHandle, isLowLatency ? 1
				: 0);

		libPlayer.SmartPlayerSetRotation(playerHandle, rotate_degrees);

		int iPlaybackRet = libPlayer
				.SmartPlayerStartPlay(playerHandle);

		if (iPlaybackRet != 0) {
			Log.e(TAG, "StartPlay failed!");

			if ( !isPulling && !isRecording && !isPushing && !isRTSPPublisherRunning)
			{
				releasePlayerHandle();
			}

			return false;
		}

		isPlaying = true;
		return true;
	}

OpenPullHandle()对应的实现如下：

	/*
   * SmartRelayDemo.java
   * Created: daniusdk.com
   */
  private boolean OpenPullHandle()
	{
		//if (playerHandle != 0) {
		//	return true;
		//}

		if(isPulling || isPlaying || isRecording)
			return true;

		//playbackUrl = "rtsp://xxxx";
    
		if (playbackUrl == null) {
			Log.e(TAG, "playback URL is null...");
			return false;
		}

		playerHandle = libPlayer.SmartPlayerOpen(myContext);

		if (playerHandle == 0) {
			Log.e(TAG, "playerHandle is nil..");
			return false;
		}

		libPlayer.SetSmartPlayerEventCallbackV2(playerHandle,
				new EventHandlePlayerV2());

		libPlayer.SmartPlayerSetBuffer(playerHandle, playBuffer);

		// set report download speed
		libPlayer.SmartPlayerSetReportDownloadSpeed(playerHandle, 1, 5);

		//设置RTSP超时时间
		int rtsp_timeout = 12;
		libPlayer.SmartPlayerSetRTSPTimeout(playerHandle, rtsp_timeout);

		//设置RTSP TCP/UDP模式自动切换
		int is_auto_switch_tcp_udp = 1;
		libPlayer.SmartPlayerSetRTSPAutoSwitchTcpUdp(playerHandle, is_auto_switch_tcp_udp);

		libPlayer.SmartPlayerSaveImageFlag(playerHandle, 1);

		// It only used when playback RTSP stream..
		//libPlayer.SmartPlayerSetRTSPTcpMode(playerHandle, 1);

		libPlayer.SmartPlayerSetUrl(playerHandle, playbackUrl);

		return true;
	}

拉流端的Event回调状态如下，拉流端主要关注的是链接状态，还有实时下载速度：

	class EventHandlePlayerV2 implements NTSmartEventCallbackV2 {
		@Override
		public void onNTSmartEventCallbackV2(long handle, int id, long param1,
											 long param2, String param3, String param4, Object param5) {

			//Log.i(TAG, "EventHandleV2: handle=" + handle + " id:" + id);

			String player_event = "";

			switch (id) {
				case NTSmartEventID.EVENT_DANIULIVE_ERC_PLAYER_STARTED:
					player_event = "开始..";
					break;
				case NTSmartEventID.EVENT_DANIULIVE_ERC_PLAYER_CONNECTING:
					player_event = "连接中..";
					break;
				case NTSmartEventID.EVENT_DANIULIVE_ERC_PLAYER_CONNECTION_FAILED:
					player_event = "连接失败..";
					break;
				case NTSmartEventID.EVENT_DANIULIVE_ERC_PLAYER_CONNECTED:
					player_event = "连接成功..";
					break;
				case NTSmartEventID.EVENT_DANIULIVE_ERC_PLAYER_DISCONNECTED:
					player_event = "连接断开..";
					break;
				case NTSmartEventID.EVENT_DANIULIVE_ERC_PLAYER_STOP:
					player_event = "停止播放..";
					break;
				case NTSmartEventID.EVENT_DANIULIVE_ERC_PLAYER_RESOLUTION_INFO:
					player_event = "分辨率信息: width: " + param1 + ", height: " + param2;
					break;
				case NTSmartEventID.EVENT_DANIULIVE_ERC_PLAYER_NO_MEDIADATA_RECEIVED:
					player_event = "收不到媒体数据，可能是url错误..";
					break;
				case NTSmartEventID.EVENT_DANIULIVE_ERC_PLAYER_SWITCH_URL:
					player_event = "切换播放URL..";
					break;
				case NTSmartEventID.EVENT_DANIULIVE_ERC_PLAYER_CAPTURE_IMAGE:
					player_event = "快照: " + param1 + " 路径：" + param3;

					if (param1 == 0) {
						player_event = player_event + ", 截取快照成功";
					} else {
						player_event = player_event + ", 截取快照失败";
					}
					break;

				case NTSmartEventID.EVENT_DANIULIVE_ERC_PLAYER_RECORDER_START_NEW_FILE:
					player_event = "[record]开始一个新的录像文件 : " + param3;
					break;
				case NTSmartEventID.EVENT_DANIULIVE_ERC_PLAYER_ONE_RECORDER_FILE_FINISHED:
					player_event = "[record]已生成一个录像文件 : " + param3;
					break;

				case NTSmartEventID.EVENT_DANIULIVE_ERC_PLAYER_START_BUFFERING:
					Log.i(TAG, "Start Buffering");
					break;

				case NTSmartEventID.EVENT_DANIULIVE_ERC_PLAYER_BUFFERING:
					Log.i(TAG, "Buffering:" + param1 + "%");
					break;

				case NTSmartEventID.EVENT_DANIULIVE_ERC_PLAYER_STOP_BUFFERING:
					Log.i(TAG, "Stop Buffering");
					break;

				case NTSmartEventID.EVENT_DANIULIVE_ERC_PLAYER_DOWNLOAD_SPEED:
					player_event = "download_speed:" + param1 + "Byte/s" + ", "
							+ (param1 * 8 / 1000) + "kbps" + ", " + (param1 / 1024)
							+ "KB/s";
					break;

				case NTSmartEventID.EVENT_DANIULIVE_ERC_PLAYER_RTSP_STATUS_CODE:
					Log.e(TAG, "RTSP error code received, please make sure username/password is correct, error code:" + param1);
					player_event = "RTSP error code:" + param1;
					break;
			}
		}
	}

下一步，是启动RTSP服务：

	//启动/停止RTSP服务
	class ButtonRtspServiceListener implements OnClickListener {
		public void onClick(View v) {
			if (isRTSPServiceRunning) {
				stopRtspService();

				btnRtspService.setText("启动RTSP服务");
				btnRtspPublisher.setEnabled(false);

				isRTSPServiceRunning = false;
				return;
			}

			if(!OpenPushHandle())
			{
				return;
			}

			Log.i(TAG, "onClick start rtsp service..");

			rtsp_handle_ = libPublisher.OpenRtspServer(0);

			if (rtsp_handle_ == 0) {
				Log.e(TAG, "创建rtsp server实例失败! 请检查SDK有效性");
			} else {
				int port = 8554;
				if (libPublisher.SetRtspServerPort(rtsp_handle_, port) != 0) {
					libPublisher.CloseRtspServer(rtsp_handle_);
					rtsp_handle_ = 0;
					Log.e(TAG, "创建rtsp server端口失败! 请检查端口是否重复或者端口不在范围内!");
				}

				//String user_name = "admin";
				//String password = "12345";
				//libPublisher.SetRtspServerUserNamePassword(rtsp_handle_, user_name, password);

				if (libPublisher.StartRtspServer(rtsp_handle_, 0) == 0) {
					Log.i(TAG, "启动rtsp server 成功!");
				} else {
					libPublisher.CloseRtspServer(rtsp_handle_);
					rtsp_handle_ = 0;
					Log.e(TAG, "启动rtsp server失败! 请检查设置的端口是否被占用!");
				}

				btnRtspService.setText("停止RTSP服务");
				btnRtspPublisher.setEnabled(true);

				isRTSPServiceRunning = true;
			}
		}
	}

如果需要停止服务，对应实现如下：

	//停止RTSP服务
	private void stopRtspService() {
		if(!isRTSPServiceRunning)
			return;

		if (libPublisher != null && rtsp_handle_ != 0) {
			libPublisher.StopRtspServer(rtsp_handle_);
			libPublisher.CloseRtspServer(rtsp_handle_);
			rtsp_handle_ = 0;
		}

		if(!isPushing)
		{
			releasePublisherHandle();
		}

		isRTSPServiceRunning = false;
	}

发布、停止发布RTSP流：

	private boolean StartRtspStream()
	{
		if (isRTSPPublisherRunning)
			return false;

		String rtsp_stream_name = "stream1";
		libPublisher.SetRtspStreamName(publisherHandle, rtsp_stream_name);
		libPublisher.ClearRtspStreamServer(publisherHandle);

		libPublisher.AddRtspStreamServer(publisherHandle, rtsp_handle_, 0);

		if (libPublisher.StartRtspStream(publisherHandle, 0) != 0)
		{
			Log.e(TAG, "调用发布rtsp流接口失败!");

			if (!isPushing)
			{
				libPublisher.SmartPublisherClose(publisherHandle);
				publisherHandle = 0;
			}

			return false;
		}

		isRTSPPublisherRunning = true;
		return true;
	}

	//停止发布RTSP流
	private void stopRtspPublisher()
	{
		if(!isRTSPPublisherRunning)
			return;

		isRTSPPublisherRunning = false;

		if (null == libPublisher || 0 == publisherHandle)
			return;

		libPublisher.StopRtspStream(publisherHandle);

		if (!isPushing && !isRTSPServiceRunning)
		{
			releasePublisherHandle();
		}
	}

因为处理后YUV或RGB数据需要重新编码，这时候需要推送端，设置下编码参数：

	private boolean OpenPushHandle() {

		if(publisherHandle != 0)
		{
			return true;
		}

		publisherHandle = libPublisher.SmartPublisherOpen(myContext, audio_opt, video_opt,
				videoWidth, videoHeight);

		if (publisherHandle == 0) {
			Log.e(TAG, "sdk open failed!");
			return false;
		}

		Log.i(TAG, "publisherHandle=" + publisherHandle);

		int fps = 20;
		int gop = fps * 1;

		int videoEncodeType = 1;	//1: h.264硬编码 2: H.265硬编码

		if(videoEncodeType == 1)  {
			int h264HWKbps = setHardwareEncoderKbps(true, videoWidth, videoHeight);
			h264HWKbps = h264HWKbps*fps/25;

			Log.i(TAG, "h264HWKbps: " + h264HWKbps);

			int isSupportH264HWEncoder = libPublisher
					.SetSmartPublisherVideoHWEncoder(publisherHandle, h264HWKbps);

			if (isSupportH264HWEncoder == 0) {
				libPublisher.SetNativeMediaNDK(publisherHandle, 0);
				libPublisher.SetVideoHWEncoderBitrateMode(publisherHandle, 1); // 0:CQ, 1:VBR, 2:CBR
				libPublisher.SetVideoHWEncoderQuality(publisherHandle, 39);
				libPublisher.SetAVCHWEncoderProfile(publisherHandle, 0x08); // 0x01: Baseline, 0x02: Main, 0x08: High

				// libPublisher.SetAVCHWEncoderLevel(publisherHandle, 0x200); // Level 3.1
				// libPublisher.SetAVCHWEncoderLevel(publisherHandle, 0x400); // Level 3.2
				// libPublisher.SetAVCHWEncoderLevel(publisherHandle, 0x800); // Level 4
				libPublisher.SetAVCHWEncoderLevel(publisherHandle, 0x1000); // Level 4.1 多数情况下,这个够用了
				//libPublisher.SetAVCHWEncoderLevel(publisherHandle, 0x2000); // Level 4.2

				// libPublisher.SetVideoHWEncoderMaxBitrate(publisherHandle, ((long)h264HWKbps)*1300);

				Log.i(TAG, "Great, it supports h.264 hardware encoder!");
			}
		}
		else if (videoEncodeType == 2) {
			int hevcHWKbps = setHardwareEncoderKbps(false, videoWidth, videoHeight);
			hevcHWKbps = hevcHWKbps*fps/25;

			Log.i(TAG, "hevcHWKbps: " + hevcHWKbps);

			int isSupportHevcHWEncoder = libPublisher
					.SetSmartPublisherVideoHevcHWEncoder(publisherHandle, hevcHWKbps);

			if (isSupportHevcHWEncoder == 0) {
				libPublisher.SetNativeMediaNDK(publisherHandle, 0);
				libPublisher.SetVideoHWEncoderBitrateMode(publisherHandle, 0); // 0:CQ, 1:VBR, 2:CBR
				libPublisher.SetVideoHWEncoderQuality(publisherHandle, 39);

				// libPublisher.SetVideoHWEncoderMaxBitrate(publisherHandle, ((long)hevcHWKbps)*1200);

				Log.i(TAG, "Great, it supports hevc hardware encoder!");
			}
		}

		libPublisher.SetSmartPublisherEventCallbackV2(publisherHandle, new EventHandlePublisherV2());

		libPublisher.SmartPublisherSetGopInterval(publisherHandle, gop);

		libPublisher.SmartPublisherSetFPS(publisherHandle, fps);

		return true;
	}

I420ExternalRender实现如下，这里可以拿到拉流的RTSP的YUV数据，然后处理后，可以调用推送端的PostLayerImageI420ByteBuffer()投递到轻量级RTSP服务或RTMP推送端编码发送出去。

	class I420ExternalRender implements NTExternalRender {
		// public static final int NT_FRAME_FORMAT_RGBA = 1;
		// public static final int NT_FRAME_FORMAT_ABGR = 2;
		// public static final int NT_FRAME_FORMAT_I420 = 3;

		private int width_ = 0;
		private int height_ = 0;

		private int y_row_bytes_ = 0;
		private int u_row_bytes_ = 0;
		private int v_row_bytes_ = 0;

		private ByteBuffer y_buffer_ = null;
		private ByteBuffer u_buffer_ = null;
		private ByteBuffer v_buffer_ = null;

		@Override
		public int getNTFrameFormat() {
			Log.i(TAG, "I420ExternalRender::getNTFrameFormat return "
					+ NT_FRAME_FORMAT_I420);
			return NT_FRAME_FORMAT_I420;
		}

		@Override
		public void onNTFrameSizeChanged(int width, int height) {
			width_ = width;
			height_ = height;

			y_row_bytes_ = (width_ + 15) & (~15);
			u_row_bytes_ = ((width_ + 1) / 2 + 15) & (~15);
			v_row_bytes_ = ((width_ + 1) / 2 + 15) & (~15);

			y_buffer_ = ByteBuffer.allocateDirect(y_row_bytes_ * height_);
			u_buffer_ = ByteBuffer.allocateDirect(u_row_bytes_
					* ((height_ + 1) / 2));
			v_buffer_ = ByteBuffer.allocateDirect(v_row_bytes_
					* ((height_ + 1) / 2));

			Log.i(TAG, "I420ExternalRender::onNTFrameSizeChanged width_="
					+ width_ + " height_=" + height_ + " y_row_bytes_="
					+ y_row_bytes_ + " u_row_bytes_=" + u_row_bytes_
					+ " v_row_bytes_=" + v_row_bytes_);
		}

		@Override
		public ByteBuffer getNTPlaneByteBuffer(int index) {
			if (index == 0) {
				return y_buffer_;
			} else if (index == 1) {
				return u_buffer_;
			} else if (index == 2) {
				return v_buffer_;
			} else {
				Log.e(TAG, "I420ExternalRender::getNTPlaneByteBuffer index error:" + index);
				return null;
			}
		}

		@Override
		public int getNTPlanePerRowBytes(int index) {
			if (index == 0) {
				return y_row_bytes_;
			} else if (index == 1) {
				return u_row_bytes_;
			} else if (index == 2) {
				return v_row_bytes_;
			} else {
				Log.e(TAG, "I420ExternalRender::getNTPlanePerRowBytes index error:" + index);
				return 0;
			}
		}

    	public void onNTRenderFrame(int width, int height, long timestamp)
    	{
    		if ( y_buffer_ == null )
    			return;
    		
    		if ( u_buffer_ == null )
    			return;
    		
    		if ( v_buffer_ == null )
    			return;
    		      
    		y_buffer_.rewind();
    		u_buffer_.rewind();
    		v_buffer_.rewind();
    		
    		 if( isPushing || isRTSPPublisherRunning )
         {
            libPublisher.PostLayerImageI420ByteBuffer(publisherHandle, 0, 0, 0,
                y_buffer_, 0, y_row_bytes_,
                u_buffer_, 0, u_row_bytes_,
                v_buffer_, 0, v_row_bytes_,
                width_, height_, 0, 0,
                960, 540, 0,0);
         }
    	}
    }

如果轻量级服务正常启动，会把rtsp的url回调上来：

class EventHandlePublisherV2 implements NTSmartEventCallbackV2 {
   @Override
   public void onNTSmartEventCallbackV2(long handle, int id, long param1, long param2, String param3, String param4, Object param5) {

      Log.i(TAG, "EventHandlePublisherV2: handle=" + handle + " id:" + id);

      String publisher_event = "";

      switch (id) {
         ....
         case NTSmartEventID.EVENT_DANIULIVE_ERC_PUBLISHER_RTSP_URL:
            publisher_event = "RTSP服务URL: " + param3;
            break;
      }
   }
}

技术总结

以上是大概的流程，从RTSP拉流到数据处理后，重新塞给轻量级RTSP服务，然后播放端再从轻量级RTSP服务端拉流，如果针对YUV或RGB算法处理延迟不大的话，整体延迟可轻松达到毫秒级，满足大多数场景的技术诉求。

标签：return,libPublisher,int,RTSP,RGB,TAG,publisherHandle,轻量级
From： https://blog.51cto.com/daniusdk/7553368

Android端如何实现拉取RTSP/RTMP流并回调YUV/RGB数据然后注入轻量级RTSP服务？

技术背景

技术实现

技术总结

相关文章

赞助商

阅读排行