Skip to content

Commit

Permalink
优化和调整人脸跟踪的代码
Browse files Browse the repository at this point in the history
  • Loading branch information
@zq2599
zq2599 committed Jan 9, 2022
1 parent c953c97 commit 6974478
Show file tree
Hide file tree
Showing 3 changed files with 140 additions and 46 deletions.
30 changes: 27 additions & 3 deletions javacv-tutorials/simple-grab-push/src/main/java/com/bolingcavalry/grabpush/Util.java
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,13 @@
package com.bolingcavalry.grabpush;

import org.bytedeco.opencv.opencv_core.Mat;
import org.bytedeco.opencv.opencv_core.Point;
import org.bytedeco.opencv.opencv_core.Scalar;

import static org.bytedeco.opencv.global.opencv_core.CV_8UC1;
import static org.bytedeco.opencv.global.opencv_core.CV_8UC4;
import static org.bytedeco.opencv.global.opencv_imgproc.CV_BGR2RGBA;
import static org.bytedeco.opencv.global.opencv_imgproc.cvtColor;
import static org.bytedeco.opencv.global.opencv_imgproc.*;
import static org.bytedeco.opencv.global.opencv_imgproc.CV_AA;

/**
* @author willzhao
Expand All @@ -14,12 +17,21 @@
*/
public class Util {

/**
* 根据传入的MAT构造相同尺寸的MAT，存放灰度图片用于以后的检测
* @param src 原始图片的MAT对象
* @return 相同尺寸的灰度图片的MAT对象
*/
public static Mat initGrayImageMat(Mat src) {
return new Mat(src.rows(), src.cols(), CV_8UC1);
}

/**
* 根据传入的MAT构造相同尺寸的MAT，存放RGBA图片用于以后的检测
* @param src 原始图片的MAT对象
* @return 相同尺寸的RGBA图片的MAT对象
*/
public static Mat buildRgbaImage(Mat src) {
public static Mat initRgbaImageMat(Mat src) {
return new Mat(src.rows(), src.cols(), CV_8UC4);
}

Expand All @@ -34,4 +46,16 @@ public static org.opencv.core.Mat buildJavacvBGR2OpenCVRGBA(Mat javacvBGR, Mat j
cvtColor(javacvBGR, javacvRGBA, CV_BGR2RGBA);
return new org.opencv.core.Mat(javacvRGBA.address());
}

/**
* 在图片上指定位置画矩形
* @param image
* @param x
* @param y
* @param widht
* @param height
*/
public static void rectOnImage(Mat image, int x, int y, int widht, int height) {
rectangle(image, new Point(x, y), new Point(x + widht, y + height), Scalar.RED, 3, CV_AA, 0);
}
}
96 changes: 59 additions & 37 deletions ...mple-grab-push/src/main/java/com/bolingcavalry/grabpush/extend/CamShiftDetectService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,8 @@
import org.bytedeco.javacv.OpenCVFrameConverter;
import org.bytedeco.opencv.opencv_core.*;
import org.bytedeco.opencv.opencv_objdetect.CascadeClassifier;

import java.io.File;
import java.net.URL;

import static org.bytedeco.opencv.global.opencv_imgproc.*;

/**
Expand Down Expand Up @@ -42,6 +40,26 @@ public class CamShiftDetectService implements DetectService {
*/
private String modelFileUrl;

/**
* 存放RGBA图片Mat
*/
private Mat mRgba;

/**
* 存放灰度图片的Mat，仅用在人脸检测的时候
*/
private Mat mGray;

/**
* 跟踪服务类
*/
private ObjectTracker objectTracker;

/**
* 表示当前是否正在跟踪目标
*/
private boolean isInTracing = false;

/**
* 构造方法，在此指定模型文件的下载地址
* @param modelFileUrl
Expand All @@ -50,11 +68,6 @@ public CamShiftDetectService(String modelFileUrl) {
this.modelFileUrl = modelFileUrl;
}


private Mat mRgba;
private Mat mGray;


/**
* 音频采样对象的初始化
* @throws Exception
Expand All @@ -63,6 +76,7 @@ public CamShiftDetectService(String modelFileUrl) {
public void init() throws Exception {
// 下载模型文件
URL url = new URL(modelFileUrl);

File file = Loader.cacheResource(url);

// 模型文件下载后的完整地址
Expand All @@ -77,84 +91,92 @@ public void init() throws Exception {
}
}

private boolean isDetected = false;

private Rect mTrackWindow;
private ObjectTracker objectTracker;
OpenCVFrameConverter.ToMat converter1 = new OpenCVFrameConverter.ToMat();
OpenCVFrameConverter.ToOrgOpenCvCoreMat converter2 = new OpenCVFrameConverter.ToOrgOpenCvCoreMat();


@Override
public Frame convert(Frame frame) {
// 由帧转为Mat
grabbedImage = converter.convert(frame);

// 灰度Mat
// 初始化灰度Mat
if (null==mGray) {
mGray = DetectService.buildGrayImage(grabbedImage);
mGray = Util.initGrayImageMat(grabbedImage);
}

// RGBA的Mat
// 初始化RGBA的Mat
if (null==mRgba) {
mRgba = Util.buildRgbaImage(grabbedImage);
mRgba = Util.initRgbaImageMat(grabbedImage);
}

// 当前图片转为灰度图片
cvtColor(grabbedImage, mGray, CV_BGR2GRAY);

// 得到opencv的mat，其格式是RGBA
org.opencv.core.Mat openCVRGBAMat = Util.buildJavacvBGR2OpenCVRGBA(grabbedImage, mRgba);

// 如果还没有定位过
if (!isDetected) {
// 如果未在追踪状态
if (!isInTracing) {
// 存放检测结果的容器
RectVector objects = new RectVector();

// 当前图片转为灰度图片
cvtColor(grabbedImage, mGray, CV_BGR2GRAY);

// 开始检测
classifier.detectMultiScale(mGray, objects);

// 检测结果总数
long total = objects.size();

// 如果没有检测到结果，就用原始帧返回
// 当前实例是只追踪一人，因此一旦检测结果不等于一，就不处理，您可以根据自己业务情况修改此处
if (total!=1) {
objects.close();
return frame;
}

log.info("start new trace");

Rect r = objects.get(0);
int x = r.x(), y = r.y(), w = r.width(), h = r.height();

objectTracker = new ObjectTracker(openCVRGBAMat);
// 得到opencv的mat，其格式是RGBA
org.opencv.core.Mat openCVRGBAMat = Util.buildJavacvBGR2OpenCVRGBA(grabbedImage, mRgba);
// 如果第一次追踪，要实例化objectTracker
if (null==objectTracker) {
objectTracker = new ObjectTracker(openCVRGBAMat);
}

// 创建跟踪目标
objectTracker.createTrackedObject(openCVRGBAMat, new org.opencv.core.Rect(x, y, w, h));

rectangle(grabbedImage, new Point(x, y), new Point(x + w, y + h), Scalar.RED, 1, CV_AA, 0);
// 根据本次检测结果给原图标注人脸矩形框
Util.rectOnImage(grabbedImage, x, y, w, h);

// 释放检测结果资源
objects.close();

isDetected = true;
// 修改标志，表示当前正在跟踪
isInTracing = true;

// 将标注过的图片转为帧，返回
return converter.convert(grabbedImage);
}

// 代码走到这里，表示已经在追踪状态了

// 得到opencv的mat，其格式是RGBA
org.opencv.core.Mat openCVRGBAMat = Util.buildJavacvBGR2OpenCVRGBA(grabbedImage, mRgba);

// 基于上一次的检测结果开始跟踪
org.opencv.core.RotatedRect rotatedRect = objectTracker.objectTracking(openCVRGBAMat);
org.opencv.core.Rect rotatedRect = objectTracker.objectTracking(openCVRGBAMat);

if (null!=rotatedRect) {
org.opencv.core.Rect r = rotatedRect.boundingRect();
int x = r.x, y = r.y, w = r.width, h = r.height;
rectangle(grabbedImage, new Point(x, y), new Point(x + w, y + h), Scalar.RED, 1, CV_AA, 0);
return converter.convert(grabbedImage);
// 如果rotatedRect为空，表示跟踪失败，此时要修改状态为"未跟踪"
if (null==rotatedRect) {
isInTracing = false;
// 返回原始帧
return frame;
}

return frame;
// 代码能走到这里，表示跟踪成功，拿到的新的一帧上的目标的位置，此时就在新位置上
Util.rectOnImage(grabbedImage, rotatedRect.x, rotatedRect.y, rotatedRect.width, rotatedRect.width);
return converter.convert(grabbedImage);
}



/**
* 程序结束前，释放人脸识别的资源
*/
Expand Down
60 changes: 54 additions & 6 deletions ...rials/simple-grab-push/src/main/java/com/bolingcavalry/grabpush/extend/ObjectTracker.java
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.bolingcavalry.grabpush.extend;

import lombok.extern.slf4j.Slf4j;
import org.opencv.core.Core;
import org.opencv.core.CvType;
import org.opencv.core.Mat;
Expand All @@ -15,13 +16,13 @@
import java.util.List;
import java.util.Vector;


/**
* @author willzhao
* @version 1.0
* @description TODO
* @date 2022/1/8 21:21
*/
@Slf4j
public class ObjectTracker {

private Mat hsv, hue, mask, prob;
Expand Down Expand Up @@ -99,7 +100,7 @@ public void createTrackedObject(Mat mRgba, Rect region) {

}

public RotatedRect objectTracking(Mat mRgba) {
public Rect objectTracking(Mat mRgba) {

rgba2Hsv(mRgba);

Expand All @@ -114,14 +115,61 @@ public RotatedRect objectTracking(Mat mRgba) {
// 追踪目标
rotatedRect = Video.CamShift(prob, trackRect, new TermCriteria(TermCriteria.EPS, 10, 1));

// 转为Rect对象
Rect camShiftRect = rotatedRect.boundingRect();

// 比较追踪前和追踪后的数据，如果出现太大偏差，就认为追踪失败
if (lostTrace(trackRect, camShiftRect)) {
log.info("lost trace!");
trackRect = null;
return null;
}

// 将本次最终到的目标作为下次追踪的对象
trackRect = rotatedRect.boundingRect();
trackRect = camShiftRect;

rotatedRect.angle = -rotatedRect.angle;
return camShiftRect;
}

private static final double LOST_GATE = 0.8d;

Imgproc.rectangle(prob, trackRect.tl(), trackRect.br(), new Scalar(255, 255, 0, 255), 6);
/**
* 变化率的绝对值
* @param last 变化前
* @param current 变化后
* @return
*/
private static double changeRate(int last, int current) {
return Math.abs((double)(current-last)/(double) last);
}

return rotatedRect;
/**
* 本次和上一次宽度或者高度的变化率，一旦超过阈值就认为跟踪失败
* @param lastRect
* @param currentRect
* @return
*/
private static boolean lostTrace(Rect lastRect, Rect currentRect) {
// 0不能做除数，如果发现0就认跟丢了
if (lastRect.width<1 || lastRect.height<1) {
return true;
}

double widthChangeRate = changeRate(lastRect.width, currentRect.width);

if (widthChangeRate>LOST_GATE) {
log.info("1. lost trace, old [{}], new [{}], rate [{}]", lastRect.width, currentRect.width, widthChangeRate);
return true;
}

double heightChangeRate = changeRate(lastRect.height, currentRect.height);

if (heightChangeRate>LOST_GATE) {
log.info("2. lost trace, old [{}], new [{}], rate [{}]", lastRect.height, currentRect.height, heightChangeRate);
return true;
}

return false;
}

}

0 comments on commit 6974478

Please sign in to comment.