基本思想：因为项目中使用mediapipe的检测框架，奈何google对其官方提供的tflite封装解析不开源，只能曲线救国，因此使用visual studio2019进行封装调用
一、先测试python版本的mediapipe
可以使用的拉流地址
CCTV1高清：http://ivi.bupt.edu.cn/hls/cctv1hd.m3u8
CCTV3高清：http://ivi.bupt.edu.cn/hls/cctv3hd.m3u8
CCTV5高清：http://ivi.bupt.edu.cn/hls/cctv5hd.m3u8
CCTV5+高清：http://ivi.bupt.edu.cn/hls/cctv5phd.m3u8
CCTV6高清：http://ivi.bupt.edu.cn/hls/cctv6hd.m3u8
测试了以下google mediapipe代码
import cv2
import mediapipe as mp
mp_drawing = mp.solutions.drawing_utils
mp_face_mesh = mp.solutions.face_mesh
drawing_spec = mp_drawing.DrawingSpec(thickness=1, circle_radius=1)
cap = cv2.VideoCapture("http://ivi.bupt.edu.cn/hls/cctv1hd.m3u8")
with mp_face_mesh.FaceMesh(
min_detection_confidence=0.5,
min_tracking_confidence=0.5) as face_mesh:
while cap.isOpened():
success, image = cap.read()
if not success:
print("Ignoring empty camera frame.")
# If loading a video, use 'break' instead of 'continue'.
continue
# Flip the image horizontally for a later selfie-view display, and convert
# the BGR image to RGB.
image = cv2.cvtColor(cv2.flip(image, 1), cv2.COLOR_BGR2RGB)
scale_percent = 60 # percent of original size
width = int(image.shape[1] * scale_percent / 100)
height = int(image.shape[0] * scale_percent / 100)
dim = (width, height)
# resize image
image = cv2.resize(image, dim, interpolation=cv2.INTER_AREA)
image=cv2.flip(image,1)
# To improve performance, optionally mark the image as not writeable to
# pass by reference.
image.flags.writeable = False
results = face_mesh.process(image)
# Draw the face mesh annotations on the image.
image.flags.writeable = True
image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
if results.multi_face_landmarks:
for face_landmarks in results.multi_face_landmarks:
mp_drawing.draw_landmarks(
image=image,
landmark_list=face_landmarks,
connections=mp_face_mesh.FACE_CONNECTIONS,
landmark_drawing_spec=drawing_spec,
connection_drawing_spec=drawing_spec)
cv2.imshow('MediaPipe FaceMesh', image)
if cv2.waitKey(5) & 0xFF == 27:
break
cap.release()
效果还可以
然后使用visual studio2019进行封装一下，以备后续工程需要使用
二、首先创建一个win32的控制台工程，不在详细叙述
（1）、首先导入opencv，准备使用获取视频帧送给mediapipe处理
测试一下，没啥问题
#include <iostream>
#include "opencv2/opencv.hpp"
using namespace cv;
using namespace std;
int main(int argc, char * argv[])
{
Mat src;
//加载图片
src = imread("F:\\FPGA\\1.png");
//检测是否加载成功
if (!src.data) //or == if(src.empty())
{
cout << "Could not open or find the image" << endl;
return -1;
}
//显示图像
imshow("Display", src);
//暂停，等待按键结束
waitKey(0);
return 0;
}
（2）导入Python库进行与python代码进行交互
注：Python3.9安装过程需要完全安装
测试一下
#include <Python.h>
#include<iostream>
using namespace std;
int main()
{
Py_Initialize();//使用python之前，要调用Py_Initialize();这个函数进行初始化
if (!Py_IsInitialized())
{
printf("初始化失败！");
return 0;
}
PyRun_SimpleString("import sys");
PyRun_SimpleString("sys.path.append('./')");//这一步很重要，修改Python路径
//PyRun_SimpleString("sys.path.append('F:\\Application\\ConsoleApplication1')");//这一步很重要，修改Python路径
PyObject* pModule = NULL;//声明变量
PyObject* pFunc = NULL;// 声明变量
pModule = PyImport_ImportModule("test_demo");//这里是要调用的文件名hello.py
if (pModule == NULL)
{
cout << "没找到" << endl;
}
pFunc = PyObject_GetAttrString(pModule, "add");//这里是要调用的函数名
PyObject* args = Py_BuildValue("(ii)", 28, 103);//给python函数参数赋值
PyObject* pRet = PyObject_CallObject(pFunc, args);//调用函数
int res = 0;
PyArg_Parse(pRet, "i", &res);//转换返回类型
cout << "res:" << res << endl;//输出结果
Py_Finalize();//调用Py_Finalize，这个根Py_Initialize相对应的。
return 0;
}
对应的test_demo.py文件
#coding:utf-8
import os
def add(a,b):
return a+b
测试结果
（3）、开始封装mediapipe在vs2019中调用，
主要流程为 c++进行视频获取，将视频送python代码中，将结果返回给c++进行后续逻辑处理
#include <Python.h>
#include<iostream>
using namespace std;
int main()
{
Py_Initialize();//使用python之前，要调用Py_Initialize();这个函数进行初始化
if (!Py_IsInitialized())
{
printf("初始化失败！");
return 0;
}
PyRun_SimpleString("import sys");
PyRun_SimpleString("sys.path.append('F:\\pythonProject')");//这一步很重要，修改Python路径
PyObject* pModule = NULL;//声明变量
PyObject* pFunc = NULL;// 声明变量
pModule = PyImport_ImportModule("main");//这里是要调用的文件名hello.py
if (pModule == NULL)
{
cout << "没找到" << endl;
}
pFunc = PyObject_GetAttrString(pModule, "fun");//这里是要调用的函数名
//PyObject* args = Py_BuildValue("(ii)", 28, 103);//给python函数参数赋值
PyObject* pRet = PyObject_CallObject(pFunc, NULL);//调用函数
//int res = 0;
//PyArg_Parse(pRet, "i", &res);//转换返回类型
//cout << "res:" << res << endl;//输出结果
Py_Finalize();//调用Py_Finalize，这个根Py_Initialize相对应的。
return 0;
}
将mediapipe 代码引入即可
import cv2
import mediapipe as mp
mp_drawing = mp.solutions.drawing_utils
mp_face_mesh = mp.solutions.face_mesh
def fun():
drawing_spec = mp_drawing.DrawingSpec(thickness=1, circle_radius=1)
cap = cv2.VideoCapture("http://ivi.bupt.edu.cn/hls/cctv1hd.m3u8")
with mp_face_mesh.FaceMesh(
min_detection_confidence=0.5,
min_tracking_confidence=0.5) as face_mesh:
while cap.isOpened():
success, image = cap.read()
if not success:
print("Ignoring empty camera frame.")
# If loading a video, use 'break' instead of 'continue'.
continue
# Flip the image horizontally for a later selfie-view display, and convert
# the BGR image to RGB.
image = cv2.cvtColor(cv2.flip(image, 1), cv2.COLOR_BGR2RGB)
scale_percent = 60 # percent of original size
width = int(image.shape[1] * scale_percent / 100)
height = int(image.shape[0] * scale_percent / 100)
dim = (width, height)
# resize image
image = cv2.resize(image, dim, interpolation=cv2.INTER_AREA)
image=cv2.flip(image,1)
# To improve performance, optionally mark the image as not writeable to
# pass by reference.
image.flags.writeable = False
results = face_mesh.process(image)
# Draw the face mesh annotations on the image.
image.flags.writeable = True
image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
if results.multi_face_landmarks:
for face_landmarks in results.multi_face_landmarks:
mp_drawing.draw_landmarks(
image=image,
landmark_list=face_landmarks,
connections=mp_face_mesh.FACE_CONNECTIONS,
landmark_drawing_spec=drawing_spec,
connection_drawing_spec=drawing_spec)
cv2.imshow('MediaPipe FaceMesh', image)
if cv2.waitKey(5) & 0xFF == 27:
break
cap.release()
fun()
测试结果图
稍微修正一下数据的传递~
#include "Python.h"
#include "object.h"
#include <numpy/arrayobject.h>
#include <opencv2/opencv.hpp>
using namespace std;
using namespace cv;
auto init_numpy()
{
import_array();
}
int main() {
// 初始化Python
//在使用Python系统前，必须使用Py_Initialize对其
//进行初始化。它会载入Python的内建模块并添加系统路
//径到模块搜索路径中。这个函数没有返回值，检查系统
//是否初始化成功需要使用Py_IsInitialized。
Py_Initialize();
init_numpy();
// 检查初始化是否成功
if (!Py_IsInitialized())
{
return -1;
}
PyObject* pName, * pModule, * pDict, * pFunc, * pArgs;
// 载入名为pytest的脚本
//pName = PyString_FromString("testvideo");
PyRun_SimpleString("import sys");
PyRun_SimpleString("import os");
PyRun_SimpleString("sys.path.append('G:\\pythonProject')");
PyRun_SimpleString("print (os.getcwd())");
// PyRun_SimpleString("print sys.path");
//();
pModule = PyImport_ImportModule("main");
if (!pModule)
{
printf("can't find main");
//getchar();
return -1;
}
pDict = PyModule_GetDict(pModule);
if (!pDict)
{
return -1;
}
pFunc = PyDict_GetItemString(pDict, "detectImage");
if (!pFunc || !PyCallable_Check(pFunc))
{
printf("can't find function [detectImage]");
return -1;
}
VideoCapture cap;
cap.open("http://ivi.bupt.edu.cn/hls/cctv1hd.m3u8");
if (!cap.isOpened())//如果视频不能正常打开则返回
return -1;
Mat img;
while (1)
{
cap >> img;//等价于cap.read(frame);
if (img.empty())
return -1;
PyObject* ArgList = PyTuple_New(1);
auto sz = img.size();
int x = sz.width;
int y = sz.height;
int z = img.channels();
uchar* CArrays = new uchar[x * y * z];//这一行申请的内存需要释放指针，否则存在内存泄漏的问题
int iChannels = img.channels();
int iRows = img.rows;
int iCols = img.cols * iChannels;
if (img.isContinuous())
{
iCols *= iRows;
iRows = 1;
}
// uchar* p=NULL;
int id = -1;
for (int i = 0; i < iRows; i++)
{
// get the pointer to the ith row
// p = img.ptr<uchar>(i);
// operates on each pixel
for (int j = 0; j < iCols; j++)
{
CArrays[++id] = img.ptr<uchar>(i)[j];//连续空间
}
}
npy_intp Dims[3] = { y, x, z }; //注意这个维度数据！
PyObject* PyArray = PyArray_SimpleNewFromData(3, Dims, NPY_UBYTE, CArrays);
PyTuple_SetItem(ArgList, 0, PyArray);
PyObject* pReturn = PyObject_CallObject(pFunc, ArgList);
if (PyList_Check(pReturn)) { //检查是否为List对象
printf("返回的结果result： ");
int SizeOfList = PyList_Size(pReturn);//List对象的大小
std::cout << SizeOfList << std::endl;
for (int j = 0; j < SizeOfList; j+=3) {
PyObject* Itemx = PyList_GetItem(pReturn, j);//获取List对象中的每一个元素
double resultx = 0.0;
PyArg_Parse(Itemx, "d", &resultx);
//printf("%lf ", resultx);
//需要将关键点画到图片上
PyObject* Itemy = PyList_GetItem(pReturn, j+1);//获取List对象中的每一个元素
double resulty = 0.0;
PyArg_Parse(Itemy, "d", &resulty);
int pointX = min((int)floor(resultx * x), x - 1);
int pointY = min((int)floor(resulty * y), y - 1);
std::cout << pointX << " " << pointY << std::endl;
cv::circle(img, cv::Point(pointX, pointY), 1, cv::Scalar(0, 0, 255), 1);
}
printf("\n");
}
else {
cout << "Not a List" << endl;
}
imshow("", img);
waitKey(1);
delete[]CArrays;
CArrays = nullptr;
}
cap.release();//释放资源
Py_DECREF(pModule);
// 关闭Python
Py_Finalize();
return 0;
}
py代码 (注意 在C++调用Py文件时，一定要保证py文件单独运行没有一点错误的，即使是独立的函数也要保证运行时没错误的！！！！)
import cv2
import mediapipe as mp
import re
mp_face_mesh = mp.solutions.face_mesh
def detectImage(image):
with mp_face_mesh.FaceMesh(
static_image_mode=True,
max_num_faces=1,
min_detection_confidence=0.5) as face_mesh:
results = face_mesh.process(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
newList2 = []
for item in results.multi_face_landmarks:
pattern = r'x: |y: |z: |\n|landmark|{|}'
result = re.split(pattern, str(item)) # 以pattern的值 分割字符串
newList2 = [eval（x) for x in list(result) if len(x) > 2]
#for idx in range(0, len(newList2), 3):
# cv2.circle(image, (int(newList2[idx] * image.shape[1]), int(newList2[idx + 1] * image.shape[0])), 1, (0, 0, 255))
#cv2.imshow("",image)
#cv2.waitKey(0)
return newList2
#detectImage(cv2.imread("F:\\sxj\\1.jpg"))
