使用了很长时间的zlmediakit流媒体服务，一直对其精妙高效的设计实现十分好奇。最好的学习就是去二次开发实现一些小功能，同时摸索框架的代码结构
在参考了zlmediakit的录像功能后，分析模仿它的源码结构，实现定时抽帧的功能。
抽帧之后可以：1）进行算法分析；2）重新编码实现转码功能；3）算法分析之后再编码，实现算法结果视频流程序。
向优秀的流媒体服务zlmediakit致敬！！
--------------------------------
关键代码如下：
1.在installWebApi方法中新增开始转码和停止转码的HTTP接口void installWebApi() {
*****
api_regist("/index/api/startTranscode", [](API_ARGS_MAP_ASYNC) {
CHECK_SECRET();
CHECK_ARGS("type", "vhost", "app", "stream");
auto src = MediaSource::find(allArgs["vhost"], allArgs["app"], allArgs["stream"]);
if (!src) {
throw ApiRetException("can not find the stream", API::NotFound);
}
src->getOwnerPoller()->async([=]() mutable {
auto result = src->setupTranscode((Transcoder::type)allArgs["type"].as<int>(), true, allArgs["customized_path"],
allArgs["max_second"].as<size_t>());
val["result"] = result;
val["code"] = result ? API::Success : API::OtherFailed;
val["msg"] = result ? "success" : "start record failed";
invoker(200, headerOut, val.toStyledString());
});
});
******
api_regist("/index/api/stopTranscode", [](API_ARGS_MAP_ASYNC) {
CHECK_SECRET();
CHECK_ARGS("type", "vhost", "app", "stream");
auto src = MediaSource::find(allArgs["vhost"], allArgs["app"], allArgs["stream"]);
if (!src) {
throw ApiRetException("can not find the stream", API::NotFound);
}
src->getOwnerPoller()->async([=]() mutable {
auto result = src->setupTranscode(
(Transcoder::type)allArgs["type"].as<int>(), false, allArgs["customized_path"],
allArgs["max_second"].as<size_t>());
val["result"] = result;
val["code"] = result ? API::Success : API::OtherFailed;
val["msg"] = result ? "success" : "start record failed";
invoker(200, headerOut, val.toStyledString());
});
});
*****
}
2.在MediaSource、MediaSourceEvent、MediaSourceEventInterceptor、MultiMediaSourceMuxer中模仿setupRecord添加setupTranscode
调用顺序是：MediaSource::setupTranscode ---> MediaSourceEventInterceptor::setupTranscode ---> MultiMediaSourceMuxer::setupTranscode
最终执行创建转码的对象，并赋值给MultiMediaSourceMuxer::_transcode
bool MediaSource::setupTranscode(Transcoder::type type, bool start, const string &custom_path, size_t max_second) {
auto listener = _listener.lock();
if (!listener) {
WarnL << "未设置MediaSource的事件监听者，setupRecord失败:" << getSchema() << "/" << getVhost() << "/"
<< getApp() << "/" << getId();
return false;
}
return listener->setupTranscode(*this, type, start, custom_path, max_second);
}
bool MediaSourceEventInterceptor::setupTranscode(MediaSource &sender, Transcoder::type type, bool start, const string &custom_path, size_t max_second) {
auto listener = _listener.lock();
if (!listener) {
return false;
}
return listener->setupTranscode(sender, type, start, custom_path, max_second);
}
bool MultiMediaSourceMuxer::setupTranscode(MediaSource &sender, Transcoder::type type, bool start, const std::string &custom_path, size_t max_second) {
if (start && !_transcode) {
//开始转码
_transcode = makeTranscoder(sender, getTracks(), Transcoder::type_mp4, custom_path, max_second); //创建转码对象
} else if (!start && _transcode) {
//停止转码
_transcode = nullptr;
}
return true;
}
3.在zlmediakit项目中模拟Record，创建Transcode相关的对象：Transcoder、FFmpegTranscoder、FFmpegMuxer
4.Transcoder::getTranscodePath。用于获取抽帧截图的文件夹目录
std::string Transcoder::getTranscodePath( type type, const std::string &vhost, const std::string &app, const std::string &stream_id, const std::string &customized_path) {
GET_CONFIG(bool, enableVhost, General::kEnableVhost);
GET_CONFIG(string, recordPath, Record::kFilePath);
GET_CONFIG(string, recordAppName, Record::kAppName);
string mp4FilePath;
if (enableVhost) {
mp4FilePath = vhost + "/" + recordAppName + "/" + app + "/" + stream_id + "/";
} else {
mp4FilePath = recordAppName + "/" + app + "/" + stream_id + "/";
}
// Here we use the customized file path.
if (!customized_path.empty()) {
return File::absolutePath(mp4FilePath, customized_path);
}
return File::absolutePath(mp4FilePath, recordPath);
}
5.Transcoder::createTranscoder。用于创建一个FFmpegTranscoder转码对象
std::shared_ptr<MediaSinkInterface> Transcoder::createTranscoder(type type, const std::string &vhost, const std::string &app, const std::string &stream_id,const std::string &customized_path, size_t max_second) {
auto path = Transcoder::getTranscodePath(type, vhost, app, stream_id, customized_path);
return std::make_shared<FFmpegTranscoder>(path, vhost, app, stream_id, max_second);
}
6.FFmpegTranscoder::FFmpegTranscoder。继承自MediaSinkInterface，可以作为一个输出类型的MediaSink。保存info信息，并创建一个FFmpegMuxer封装器对象
FFmpegTranscoder::FFmpegTranscoder(const std::string &path, const std::string &vhost, const std::string &app, const std::string &stream_id,size_t max_second) {
_folder_path = path; _info.app = app;
_info.stream = stream_id;
_info.vhost = vhost;
_info.folder = path;
GET_CONFIG(size_t, recordSec, Record::kFileSecond);
_max_second = max_second ? max_second : recordSec;
_muxer = std::make_shared<FFmpegMuxer>(_folder_path, _max_second);
}
7.FFmpegTranscoder::addTrack。继承自MediaSink::addTrack。添加音视频轨道。最终是向封装器对象_muxer中添加轨道，暂时只关注视频轨道
bool FFmpegTranscoder::addTrack(const Track::Ptr &track) {
_tracks.emplace_back(track);
if (track->getTrackType() == TrackVideo) {
_have_video = true;
_muxer->addTrack(track);
});
}
return true;
}
8.FFmpegTranscoder::inputFrame。继承自MediaSink::inputFrame，接收帧数据。将帧数据写入到了_muxer
bool FFmpegTranscoder::inputFrame(const Frame::Ptr &frame) {
if (_muxer) {
return _muxer->inputFrame(frame);
}
return true;
}
9.FFmpegMuxer::addTrack。创建视频解码器，用的是zlmediakit已经封装好的FFmpegDecoder，是一个基于ffmpeg的多线程异步解码对象，并且可以硬件解码器。
设置解码回调:解析将解码后的YUV数据帧转成图片格式并落地保存。
bool FFmpegMuxer::addTrack(const Track::Ptr &track) {
if (track->getCodecId() == CodecH264) {
_video_dec.reset(new FFmpegDecoder(track));
} else if (track->getCodecId() == CodecH265) {
_video_dec.reset(new FFmpegDecoder(track));
} else {
}
if (_video_dec != nullptr) {
_video_dec->setOnDecode([this](const FFmpegFrame::Ptr &frame) {
time_t now = ::time(NULL);
if (now - _last_time >= _gapTime) {
AVFrame *avFrame = frame->get();
int bufSize = av_image_get_buffer_size(AV_PIX_FMT_BGRA, avFrame->width, avFrame->height, 64);
uint8_t *buf = (uint8_t *)av_malloc(bufSize);
int picSize = frameToImage(avFrame, AV_CODEC_ID_MJPEG, buf, bufSize);
if (picSize > 0) {
auto file_path = _folder_path + getTimeStr("%H-%M-%S_") + std::to_string(_index) + ".jpeg";
auto f = fopen(file_path.c_str(), "wb+");
if (f) {
fwrite(buf, sizeof(uint8_t), bufSize, f);
fclose(f);
}
}
av_free(buf);
_index++;
_last_time = now;
}
});
}
return true;
}
10.FFmpegMuxer::frameToImage。利用FFmpeg将AVFrame的视频帧转成二进制数组输出
int FFmpegMuxer::frameToImage(AVFrame *frame, AVCodecID codecID, uint8_t *outbuf, size_t outbufSize) {
int ret = 0;
AVPacket pkt;
AVCodec *codec;
AVCodecContext *ctx = NULL;
AVFrame *rgbFrame = NULL;
uint8_t *buffer = NULL;
struct SwsContext *swsContext = NULL;
av_init_packet(&pkt);
codec = avcodec_find_encoder(codecID);
if (!codec) {
goto end;
}
if (!codec->pix_fmts) {
goto end;
}
ctx = avcodec_alloc_context3(codec);
ctx->bit_rate = 3000000;
ctx->width = frame->width;
ctx->height = frame->height;
ctx->time_base.num = 1;
ctx->time_base.den = 25;
ctx->gop_size = 10;
ctx->max_b_frames = 0;
ctx->thread_count = 1;
ctx->pix_fmt = *codec->pix_fmts;
ret = avcodec_open2(ctx, codec, NULL);
if (ret < 0) {
printf("avcodec_open2 error %d", ret);
goto end;
}
if (frame->format != ctx->pix_fmt) {
rgbFrame = av_frame_alloc();
if (rgbFrame == NULL) {
printf("av_frame_alloc fail");
goto end;
}
swsContext = sws_getContext(
frame->width, frame->height, (enum AVPixelFormat)frame->format, frame->width, frame->height, ctx->pix_fmt,
1, NULL, NULL, NULL);
if (!swsContext) {
printf("sws_getContext fail");
goto end;
}
int bufferSize = av_image_get_buffer_size(ctx->pix_fmt, frame->width, frame->height, 1) * 2;
buffer = (unsigned char *)av_malloc(bufferSize);
if (buffer == NULL) {
printf("buffer alloc fail:%d", bufferSize);
goto end;
}
av_image_fill_arrays(rgbFrame->data, rgbFrame->linesize, buffer, ctx->pix_fmt, frame->width, frame->height, 1);
if ((ret = sws_scale(
swsContext, frame->data, frame->linesize, 0, frame->height, rgbFrame->data, rgbFrame->linesize))
< 0) {
printf("sws_scale error %d", ret);
}
rgbFrame->format = ctx->pix_fmt;
rgbFrame->width = ctx->width;
rgbFrame->height = ctx->height;
ret = avcodec_send_frame(ctx, rgbFrame);
} else {
ret = avcodec_send_frame(ctx, frame);
}
if (ret < 0) {
printf("avcodec_send_frame error %d", ret);
goto end;
}
ret = avcodec_receive_packet(ctx, &pkt);
if (ret < 0) {
printf("avcodec_receive_packet error %d", ret);
goto end;
}
if (pkt.size > 0 && pkt.size <= outbufSize)
memcpy(outbuf, pkt.data, pkt.size);
ret = pkt.size;
end:
if (swsContext) {
sws_freeContext(swsContext);
}
if (rgbFrame) {
av_frame_unref(rgbFrame);
av_frame_free(&rgbFrame);
}
if (buffer) {
av_free(buffer);
}
av_packet_unref(&pkt);
if (ctx) {
avcodec_close(ctx);
avcodec_free_context(&ctx);
}
return ret;
}
11.FFmpegMuxer::inputFrame。向解码器中塞入H264或H265视频帧
bool FFmpegMuxer::inputFrame(const Frame::Ptr &frame) {
if(frame->getTrackType() == TrackVideo && _video_dec != nullptr) {
_video_dec->inputFrame(frame, true, false, false);
if (_cb) {
_cb(frame);
}
}
return true;
}
12.MultiMediaSourceMuxer::onTrackFrame。MediaSource主对象收到视频帧时会向transcode对象中写入，从而实现抽帧
bool MultiMediaSourceMuxer::onTrackFrame(const Frame::Ptr &frame_in) {
GET_CONFIG(bool, modify_stamp, General::kModifyStamp);
auto frame = frame_in;
if (modify_stamp) {
//开启了时间戳覆盖
frame = std::make_shared<FrameStamp>(frame, _stamp[frame->getTrackType()],true);
}
bool ret = false;
if (_rtmp) {
ret = _rtmp->inputFrame(frame) ? true : ret;
}
if (_rtsp) {
ret = _rtsp->inputFrame(frame) ? true : ret;
}
if (_ts) {
ret = _ts->inputFrame(frame) ? true : ret;
}
//拷贝智能指针，目的是为了防止跨线程调用设置录像相关api导致的线程竞争问题
//此处使用智能指针拷贝来确保线程安全，比互斥锁性能更优
auto hls = _hls;
if (hls) {
ret = hls->inputFrame(frame) ? true : ret;
}
auto mp4 = _mp4;
if (mp4) {
ret = mp4->inputFrame(frame) ? true : ret;
}
auto transcode = _transcode;
if (transcode) {
ret = transcode->inputFrame(frame) ? true : ret;
}
#if defined(ENABLE_MP4)
if (_fmp4) {
ret = _fmp4->inputFrame(frame) ? true : ret;
}
#endif
#if defined(ENABLE_RTPPROXY)
for (auto &pr : _rtp_sender) {
ret = pr.second->inputFrame(frame) ? true : ret;
}
#endif //ENABLE_RTPPROXY
return ret;
}
标签：zlmediakit,string,frame,ctx,ret,抽帧,源码,path,const
From： https://www.cnblogs.com/feixiang-energy/p/17623567.html