优点：实现视频流的动态处理。
缺点：视频解码（CPU/GPU）后图像处理CPU占用率高。
在Winform中通过LibVLCSharp组件获取视频流中的每一帧图像，需要设置回调函数，主要是SetVideoFormatCallbacks和SetVideoCallbacks，其定义如下所示：
/// <summary>
/// Set decoded video chroma and dimensions.
/// This only works in combination with libvlc_video_set_callbacks().
/// </summary>
/// <param name="formatCb">callback to select the video format (cannot be NULL)</param>
/// <param name="cleanupCb">callback to release any allocated resources (or NULL)</param>
public void SetVideoFormatCallbacks(
MediaPlayer.LibVLCVideoFormatCb formatCb,
MediaPlayer.LibVLCVideoCleanupCb? cleanupCb)
{
this._videoFormatCb = formatCb ?? throw new ArgumentNullException(nameof (formatCb));
this._videoCleanupCb = cleanupCb;
MediaPlayer.Native.LibVLCVideoSetFormatCallbacks(this.NativeReference, MediaPlayer.VideoFormatCallbackHandle, cleanupCb == null ? (MediaPlayer.LibVLCVideoCleanupCb) null : this._videoCleanupCb);
}
/// <summary>
/// Set callbacks and private data to render decoded video to a custom area in memory.
/// Use libvlc_video_set_format() or libvlc_video_set_format_callbacks() to configure the decoded format.
/// Warning
/// Rendering video into custom memory buffers is considerably less efficient than rendering in a custom window as normal.
/// For optimal perfomances, VLC media player renders into a custom window, and does not use this function and associated callbacks.
/// It is highly recommended that other LibVLC-based application do likewise.
/// To embed video in a window, use libvlc_media_player_set_xid() or equivalent depending on the operating system.
/// If window embedding does not fit the application use case, then a custom LibVLC video output display plugin is required to maintain optimal video rendering performances.
/// The following limitations affect performance:
/// Hardware video decoding acceleration will either be disabled completely, or require(relatively slow) copy from video/DSP memory to main memory.
/// Sub-pictures(subtitles, on-screen display, etc.) must be blent into the main picture by the CPU instead of the GPU.
/// Depending on the video format, pixel format conversion, picture scaling, cropping and/or picture re-orientation,
/// must be performed by the CPU instead of the GPU.
/// Memory copying is required between LibVLC reference picture buffers and application buffers (between lock and unlock callbacks).
/// </summary>
/// <param name="lockCb">callback to lock video memory (must not be NULL)</param>
/// <param name="unlockCb">callback to unlock video memory (or NULL if not needed)</param>
/// <param name="displayCb">callback to display video (or NULL if not needed)</param>
public void SetVideoCallbacks(
MediaPlayer.LibVLCVideoLockCb lockCb,
MediaPlayer.LibVLCVideoUnlockCb? unlockCb,
MediaPlayer.LibVLCVideoDisplayCb? displayCb)
{
this._videoLockCb = lockCb ?? throw new ArgumentNullException(nameof (lockCb));
this._videoUnlockCb = unlockCb;
this._videoDisplayCb = displayCb;
MediaPlayer.Native.LibVLCVideoSetCallbacks(this.NativeReference, MediaPlayer.VideoLockCallbackHandle, unlockCb == null ? (MediaPlayer.LibVLCVideoUnlockCb) null : MediaPlayer.VideoUnlockCallbackHandle, displayCb == null ? (MediaPlayer.LibVLCVideoDisplayCb) null : MediaPlayer.VideoDisplayCallbackHandle, GCHandle.ToIntPtr(this._gcHandle));
}
1、回调方法简要说明
SetVideoFormatCallbacks方法中参数formatCb用于初始化图像数据，参数cleanupCb用于清理图像数据。SetVideoFormatCallbacks方法中参数lockCb用于对视频帧进行解码，参数unlockCb用于解锁图片缓冲区，参数displayCb用以显示图片。
2、获取视频帧数据
在Winform中，在LibVLCSharp回调函数中获取视频帧以后，需要将图像数据拷贝进图像缓存中，这可以通过内存映射文件及预设的Bitmap实现。在lockCb委托方法中，通过视频源的图像参数，可以构造出内存映射文件对象MemoryMappedFile和MemoryMappedViewAccessor，同时创建Bitmap相关对象。
var size = pitches * lines;
_memoryMappedFile = MemoryMappedFile.CreateNew(null, size);
_memoryMappedView = _memoryMappedFile.CreateViewAccessor();
var viewHandle = _memoryMappedView.SafeMemoryMappedViewHandle.DangerousGetHandle();
userData = viewHandle;
var args = new
{
// ReSharper disable once RedundantAnonymousTypePropertyName
width = width,
// ReSharper disable once RedundantAnonymousTypePropertyName
height = height,
};
_synchronizationContext.Post((state) =>
{
_bitmap = new Bitmap((int)args.width, (int)args.height, PixelFormat.Format32bppRgb);
_bitmapBuffer = new byte[(int)args.width * (int)args.height * 4];
_graphics = Graphics.FromImage(_bitmap);
}, null);
然后，在对视频帧解码时，将视频数据写入内存映射文件中。
Marshal.WriteIntPtr(planes, userData);
3、显示视频帧
LibVLCVideoDisplayCb类型的回调函数用于显示图像。此时内存映射文件中已经含有视频帧数据，需要将其拷贝到Bitmap中。下面代码在获取内存映射文件中的图像后，通过触发事件对外开放图像绘制接口，实现对图像的二次处理；绘制完成后触发图像渲染完毕事件，通知相关控件加载图像。
try
{
var bitmapData = _bitmap.LockBits(
new Rectangle(0, 0, _videoWidth, _videoHeight),
ImageLockMode.WriteOnly, PixelFormat.Format32bppRgb
);
_memoryMappedView.ReadArray(8, _bitmapBuffer, 0, _bitmapBuffer.Length); //从内存映射视图的第九个字节开始读取图像数据
Marshal.Copy(_bitmapBuffer, 0, bitmapData.Scan0, _bitmapBuffer.Length); //将图像数据写入Bitmap
_bitmap.UnlockBits(bitmapData);
//帧图像已渲染完毕
FrameRendered.Invoke(_graphics);
}
catch (Exception e)
{
Console.WriteLine(e.Message);
}
Task.Run(() =>
{
var image = _bitmap.Clone(
new Rectangle(0, 0, _bitmap.Width, _bitmap.Height),
PixelFormat.Format32bppRgb);
_synchronizationContext.Post((state) =>
{
FrameChanged.Invoke(image); //帧图像已修改
}, null);
});
4、部分代码
4.1 MyVlcVideoSourceProvider类
internal class MyVlcVideoSourceProvider
{
public event Action<Graphics> FrameRendered = (_) => { };
public event Action<Bitmap> FrameChanged = (_) => { };
/// <summary>
/// The memory mapped file that contains the picture data
/// </summary>
private MemoryMappedFile _memoryMappedFile;
/// <summary>
/// The view that contains the pointer to the buffer that contains the picture data
/// </summary>
private MemoryMappedViewAccessor _memoryMappedView;
private int _videoWidth;
private int _videoHeight;
private Bitmap _bitmap;
private Graphics _graphics;
private byte[] _bitmapBuffer;
private readonly MediaPlayer _mediaPlayer;
private readonly SynchronizationContext _synchronizationContext;
public MyVlcVideoSourceProvider(
MediaPlayer mediaPlayer,
SynchronizationContext synchronizationContext
)
{
_mediaPlayer = mediaPlayer;
_mediaPlayer.SetVideoFormatCallbacks(VideoFormat, CleanupVideo);
_mediaPlayer.SetVideoCallbacks(LockVideo, null, DisplayVideo);
_synchronizationContext = synchronizationContext;
}
public void UnInit()
{
RemoveVideo();
}
#region Vlc video callbacks
/// <summary>
/// Called by vlc when the video format is needed. This method allocats the picture buffers for vlc and tells it to set the chroma to RV32
/// </summary>
/// <param name="userData">The user data that will be given to the <see cref="LockVideo"/> callback. It contains the pointer to the buffer</param>
/// <param name="chroma">The chroma</param>
/// <param name="width">The visible width</param>
/// <param name="height">The visible height</param>
/// <param name="pitches">The buffer width</param>
/// <param name="lines">The buffer height</param>
/// <returns>The number of buffers allocated</returns>
private uint VideoFormat(
// ReSharper disable RedundantAssignment
ref IntPtr userData,
IntPtr chroma,
ref uint width,
ref uint height,
ref uint pitches,
ref uint lines
) // ReSharper restore RedundantAssignment
{
FourCCConverter.ToFourCC("RV32", chroma);
//Correct video width and height according to TrackInfo
var media = _mediaPlayer.Media;
if (media != null)
{
foreach (MediaTrack track in media.Tracks)
{
if (track.TrackType == TrackType.Video)
{
var trackInfo = track.Data.Video;
if (trackInfo.Width > 0 && trackInfo.Height > 0)
{
width = trackInfo.Width;
height = trackInfo.Height;
if (trackInfo.SarDen != 0)
{
width = width * trackInfo.SarNum / trackInfo.SarDen;
}
}
break;
}
}
}
pitches = GetAlignedDimension((width * 32) / 8, 32);
lines = GetAlignedDimension(height, 32);
_videoWidth = (int)width;
_videoHeight = (int)height;
var size = pitches * lines;
_memoryMappedFile = MemoryMappedFile.CreateNew(null, size);
var args = new
{
// ReSharper disable once RedundantAnonymousTypePropertyName
width = width,
// ReSharper disable once RedundantAnonymousTypePropertyName
height = height,
};
_synchronizationContext.Post((state) =>
{
_bitmap = new Bitmap((int)args.width, (int)args.height, PixelFormat.Format32bppRgb);
_bitmapBuffer = new byte[(int)args.width * (int)args.height * 4];
_graphics = Graphics.FromImage(_bitmap);
}, null);
_memoryMappedView = _memoryMappedFile.CreateViewAccessor();
var viewHandle = _memoryMappedView.SafeMemoryMappedViewHandle.DangerousGetHandle();
userData = viewHandle;
return 1;
}
/// <summary>
/// Called by Vlc when it requires a cleanup
/// </summary>
/// <param name="userData">The parameter is not used</param>
private void CleanupVideo(ref IntPtr userData)
{
// This callback may be called by Dispose in the Dispatcher thread, in which case it deadlocks if we call RemoveVideo again in the same thread.
_synchronizationContext.Post((state) => { RemoveVideo(); }, null);
}
/// <summary>
/// Called by libvlc when it wants to acquire a buffer where to write
/// </summary>
/// <param name="userData">The pointer to the buffer (the out parameter of the <see cref="VideoFormat"/> callback)</param>
/// <param name="planes">The pointer to the planes array. Since only one plane has been allocated, the array has only one value to be allocated.</param>
/// <returns>The pointer that is passed to the other callbacks as a picture identifier, this is not used</returns>
private IntPtr LockVideo(IntPtr userData, IntPtr planes)
{
Marshal.WriteIntPtr(planes, userData);
return userData;
}
/// <summary>
/// Called by libvlc when the picture has to be displayed.
/// </summary>
/// <param name="userData">The pointer to the buffer (the out parameter of the <see cref="VideoFormat"/> callback)</param>
/// <param name="picture">The pointer returned by the <see cref="LockVideo"/> callback. This is not used.</param>
private void DisplayVideo(IntPtr userData, IntPtr picture)
{
if (_bitmap == null)
{
return;
}
try
{
var bitmapData = _bitmap.LockBits(
new Rectangle(0, 0, _videoWidth, _videoHeight),
ImageLockMode.WriteOnly, PixelFormat.Format32bppRgb
);
_memoryMappedView.ReadArray(8, _bitmapBuffer, 0, _bitmapBuffer.Length); //从内存映射视图的第九个字节开始读取图像数据
Marshal.Copy(_bitmapBuffer, 0, bitmapData.Scan0, _bitmapBuffer.Length); //将图像数据写入Bitmap
_bitmap.UnlockBits(bitmapData);
//帧图像已渲染完毕
FrameRendered.Invoke(_graphics);
}
catch (Exception e)
{
Console.WriteLine(e.Message);
}
Task.Run(() =>
{
var image = _bitmap.Clone(
new Rectangle(0, 0, _bitmap.Width, _bitmap.Height),
PixelFormat.Format32bppRgb);
_synchronizationContext.Post((state) =>
{
FrameChanged.Invoke(image); //帧图像已修改
}, null);
});
}
#endregion
/// <summary>
/// Aligns dimension to the next multiple of mod
/// </summary>
/// <param name="dimension">The dimension to be aligned</param>
/// <param name="mod">The modulus</param>
/// <returns>The aligned dimension</returns>
private uint GetAlignedDimension(uint dimension, uint mod)
{
var modResult = dimension % mod;
if (modResult == 0)
{
return dimension;
}
return dimension + mod - (dimension % mod);
}
/// <summary>
/// Removes the video (must be called from the Dispatcher thread)
/// </summary>
private void RemoveVideo()
{
_memoryMappedView?.Dispose();
_memoryMappedView = null;
_memoryMappedFile?.Dispose();
_memoryMappedFile = null;
_graphics?.Dispose();
_graphics = null;
_bitmap?.Dispose();
_bitmap = null;
}
}
4.2 VideoSourceProviderControl类（基于LibVLCSharp的视频播放代码已省略）
public partial class VideoSourceProviderControl : VideoPlayerControl
{
public override Control VideoView => pictureBoxVideo;
private readonly MyVlcVideoSourceProvider _sourceProvider;
public VideoSourceProviderControl()
{
InitializeComponent();
_sourceProvider = new MyVlcVideoSourceProvider(MediaPlayer,
SynchronizationContext.Current);
_sourceProvider.FrameRendered += OnFrameRendered;
_sourceProvider.FrameChanged += OnFrameChanged;
}
protected override void OnVideoPlayerLoad(object sender, EventArgs e)
{
base.OnVideoPlayerLoad(sender, e);
pictureBoxVideo.SizeMode = PictureBoxSizeMode.StretchImage;
pictureBoxVideo.Dock = DockStyle.Fill;
}
protected override void OnVideoPlayerDestroyed(object sender, EventArgs args)
{
base.OnVideoPlayerDestroyed(sender, args);
_sourceProvider.UnInit();
}
private void OnFrameRendered(Graphics graphics)
{
graphics.DrawString("test测试。", new Font("新宋体", 24), Brushes.Red, 100, 100);
}
private void OnFrameChanged(Bitmap bitmap)
{
var image = pictureBoxVideo.Image;
pictureBoxVideo.Image = bitmap;
image?.Dispose();
}
}
标签：视频,MediaPlayer,private,bitmap,width,video,null,LibVLCSharp,Winform
From： https://www.cnblogs.com/xhubobo/p/17812007.html