Related
Closed. This question needs to be more focused. It is not currently accepting answers.
Want to improve this question? Update the question so it focuses on one problem only by editing this post.
Closed 10 months ago.
Improve this question
**Requirements: **
(1) Build OpenCV with Cuda and compile in C++
(2) Version = OpenCV Latest Version
(3) Build and Compile OpenCV Link: https://techawarey.com/programming/install-opencv-c-c-in-ubuntu-18-04-lts-step-by-step-guide/#Summary
(4) Library samples_utility is here: https://github.com/opencv/opencv_contrib/blob/master/modules/tracking/samples/samples_utility.hpp
(4) Compile Program Command: g++ test.cpp -o testoutput -std=c++11 'pkg-config --cflags --libs opencv'
(5) Run Program Command: ./testoutput
Code is working fine but not accurate
Step: 1
Read Frame from Camera
Select ROI(Region of Interest)
After that start KCF tracker with Sobal Features Extractor
Tracking the selected object.
Step: 2
Failure detect
After that call template matching function called MatchingMethod()
Run template matching
Get x, y value from template matching
After that reinitialize KCF tracker with Sobal Features Extractor.
This code is fine for still object when the object is moving the tracker false detection. I want to improve accuracy and reduce false detection.
#include <opencv2/core/utility.hpp>
#include <opencv2/tracking.hpp>
#include <opencv2/videoio.hpp>
#include <opencv2/highgui.hpp>
#include <opencv2/core/ocl.hpp>
#include <iostream>
#include <cstring>
#include <unistd.h>
#include "sample_utility.hpp"
#include <thread>
#include <opencv2/cudaimgproc.hpp>
#include <opencv2/imgproc.hpp>
#include <opencv2/calib3d.hpp>
#include <opencv2/cudaarithm.hpp>
#include <iomanip>
#include <stdlib.h>
#include <unistd.h>
//////////////////////////////
using namespace cv;
using namespace std;
////////////////////////////
// Convert to string
#define SSTR( x ) static_cast< std::ostringstream & >( \
( std::ostringstream() << std::dec << x ) ).str()
/// Global Variables
struct Array {
int arr[2];
};
Mat img;
Mat templ;
Mat result_h;
bool flag = true;
int match_method = 5;
int i=0;
int max_Trackbar = 5;
float fps;
int seconds = 0;
// Function Headers
void delay();
// prototype of the functino for feature extractor
void sobelExtractor(const Mat img, const Rect roi, Mat& feat);
struct Array MatchingMethod( int, void* );
int main(int argc, char **argv)
{
TrackerKCF::Params param;
param.compress_feature = true;
param.compressed_size = 2;
param.desc_npca = 0;
param.desc_pca = TrackerKCF::GRAY | TrackerKCF::CN;
param.detect_thresh = 0.8;
// create a tracker object
Ptr<TrackerKCF> tracker = TrackerKCF::create(param);
tracker->setFeatureExtractor(sobelExtractor);
VideoCapture cap(0);
// Exit if video is not opened
if(!cap.isOpened())
{
//cout << "Could not read video file" << endl;
return 1;
}
// Read first frame
Mat frame;
bool ok = cap.read(frame);
// Define initial bounding box
//Rect bbox(x, y, w, h);
// Uncomment the line below to select a different bounding box
Rect bbox = selectROI(frame, false);
// Display bounding box.
rectangle(frame, bbox, Scalar( 255, 0, 0 ), 2, 1 );
///////////////////////////
int H, W, cW, cH;
// print(f"hight {H} , Width {W}")
H = display_height;
W = display_width;
// Center point of the screen
cW = int(W / 2);
cH = int(H / 2);
Point p1(cW, cH);
// get bounding box
Mat imCrop = frame(bbox);
imwrite("1.png", imCrop);
//quit if ROI was not selected
if(bbox.width==0 || bbox.height==0)
return 0;
//////////////////////////
//imshow("Tracking", frame);
tracker->init(frame, bbox);
while(true)
{
Mat frame;
cap >> frame;
circle(frame, p1, 3, Scalar(0,255,0), -1);
// Start timer
if(bbox.width!=0 || bbox.height!=0){
double timer = (double)getTickCount();
// Update the tracking result
/////////////////////////////////////
bool ok = tracker->update(frame, bbox);
//////////////////////////////////////
//ok, bbox = tracker->update(frame);
// Calculate Frames per second (FPS)
fps = getTickFrequency() / ((double)getTickCount() - timer);
if (ok)
{
// Tracking success : Draw the tracked object
rectangle(frame, bbox, Scalar( 255, 0, 0 ), 2, 1 );
///////////////////////////////////////////////////
int xxx, yyy, height, width;
xxx = bbox.x;
yyy = bbox.y;
height = bbox.height;
width = bbox.width;
int diffX, diffY;
float cxROI, cyROI;
cxROI = int((xxx + (xxx + width)) / 2);
cyROI = int((yyy + (yyy + height)) / 2);
diffX = cxROI - cW;
diffY = cH - cyROI;
//cout<<diffX<<"\n";
//cout<<diffY<<"\n";
Point p(cxROI, cyROI);
circle(frame, p, 3, Scalar(128,0,0), -1);
putText(frame, "FPS : " + SSTR(int(fps)), Point(100,20), FONT_HERSHEY_SIMPLEX, 0.75, Scalar(50,170,50), 2);
putText(frame, "Difference From X-Axis: "+SSTR(int(diffX)), Point(100, 50), FONT_HERSHEY_SIMPLEX, 0.6, Scalar(100, 200, 200), 2);
putText(frame, "Difference From Y-Axis: "+SSTR(int(diffY)), Point(100, 80), FONT_HERSHEY_SIMPLEX, 0.6, Scalar(100, 200, 200), 2);
}
else
{
// Tracking failure detected.
putText(frame, "Tracking failure detected", Point(100,110), FONT_HERSHEY_SIMPLEX, 0.75, Scalar(0,0,255),2);
templ = imread( "1.png", 1 );
img=frame.clone();
struct Array a = MatchingMethod( 0, 0 );
cout<<"X: "<<a.arr[0]<<"\n";
cout<<"Y: "<<a.arr[1]<<"\n";
cout<<"Width: "<<w<<"\n";
cout<<"Height: "<<h<<"\n";
int xx, yy, ww, hh;
xx = a.arr[0];
yy = a.arr[1];
ww = w;
hh = h;
Rect bbox(xx, yy, ww, hh);
tracker.release();
tracker = TrackerKCF::create(param);
tracker->setFeatureExtractor(sobelExtractor);
tracker->init(frame, bbox);
//roi.x = MatchingMethod.
//waitKey(30);
rectangle(frame, bbox, Scalar( 255, 0, 0 ), 2, 1 );
////////////////////////////////////////////////////////////////////////
int diffX, diffY;
float cxROI, cyROI;
cxROI = int((xx + (xx + ww)) / 2);
cyROI = int((yy + (yy + hh)) / 2);
diffX = cxROI - cW;
diffY = cH - cyROI;
//cout<<diffX<<"\n";
//cout<<diffY<<"\n";
Point p(cxROI, cyROI);
circle(frame, p, 3, Scalar(128,0,0), -1);
///////////////////////////////////////////////////////////////////////////
}
}
else{
}
// Display frame.
imshow("Tracking", frame);
// Exit if ESC pressed.
int k = waitKey(1);
if(k == 27)
{
break;
}
}
return 0;
}
///////////////////////
void sobelExtractor(const Mat img, const Rect roi, Mat& feat){
Mat sobel[2];
Mat patch;
Rect region=roi;
// extract patch inside the image
if(roi.x<0){region.x=0;region.width+=roi.x;}
if(roi.y<0){region.y=0;region.height+=roi.y;}
if(roi.x+roi.width>img.cols)region.width=img.cols-roi.x;
if(roi.y+roi.height>img.rows)region.height=img.rows-roi.y;
if(region.width>img.cols)region.width=img.cols;
if(region.height>img.rows)region.height=img.rows;
patch=img(region).clone();
cvtColor(patch,patch, COLOR_BGR2GRAY);
// add some padding to compensate when the patch is outside image border
int addTop,addBottom, addLeft, addRight;
addTop=region.y-roi.y;
addBottom=(roi.height+roi.y>img.rows?roi.height+roi.y-img.rows:0);
addLeft=region.x-roi.x;
addRight=(roi.width+roi.x>img.cols?roi.width+roi.x-img.cols:0);
copyMakeBorder(patch,patch,addTop,addBottom,addLeft,addRight,BORDER_REPLICATE);
Sobel(patch, sobel[0], CV_32F,1,0,1);
Sobel(patch, sobel[1], CV_32F,0,1,1);
merge(sobel,2,feat);
feat=feat/255.0-0.5; // normalize to range -0.5 .. 0.5
}
////////////////////////////////////////////////////
struct Array MatchingMethod( int, void* )
{
/// Source image to display
Mat frame;
struct Array a;
/////////
for(int i=1; i<=4; i++){
img.copyTo( frame );
// break;
//}
//////////////////////////
cv::cuda::setDevice(0); // initialize CUDA
// convert from mat to gpumat
cv::cuda::GpuMat image_d(img);
cv::cuda::GpuMat templ_d(templ);
cv::cuda::GpuMat result;
// GPU -> NG
cv::Ptr<cv::cuda::TemplateMatching> alg =
cv::cuda::createTemplateMatching(image_d.type(), cv::TM_CCOEFF_NORMED);
alg->match(image_d, templ_d, result); // no return.
//cv::cuda::normalize(result, result, 0, 1, cv::NORM_MINMAX, -1);
double max_value, min_value;
cv::Point location;
cv::cuda::minMaxLoc(result, &min_value, &max_value, 0, &location);
/////////////////////////
double THRESHOLD = 3e-09; //0.3;
if( min_value <= THRESHOLD) {
//struct Array a;
a.arr[0] = location.x;
a.arr[1] = location.y;
cout<<"Hi"<<endl;
}
}
if(flag==true){
return a;
flag = false;
}
//}
}
Okey here is my answer to your question.
First of all, you are making a mistake by applying template matching when the tracker misses. Because template matching matches the feature if and only if it is totally same with the reference feature. So in your case, there will be shadows, light issues etc. in the environment, and you will never be able to get success results.
Secondly, if you delete the template matching scope, tracker will continue to search the target in the image effectively. Which changements I did in your code is listed below. With these changes, I got better results:
Delete the template matching scope
Decrease the detection threshold(param.detect_thresh) to 0.5
Create more tracker objects to catch the target: This change is the most important part. What I am suggesting is that create more and more tracker objects(in my case I did 4 tracker objects, but you can increase the number). Each tracker should get as input rectangle similar to ROI user chose but not the same coordinates. For example, if user chooses cv::Rect(200,200,400,400) then other tracker should get target as cv::Rect(180,190,400,400) , cv::Rect(220,180,400,400) ... and so on. Why you should do it because, tracker algorithm is feature based, so it will always try to get a similar features to the reference. By doing this, you will increase the feature references.
And here is my code to guide you:
#include <opencv2/core/utility.hpp>
#include <opencv2/tracking.hpp>
#include <opencv2/videoio.hpp>
#include <opencv2/highgui.hpp>
#include <opencv2/core/ocl.hpp>
#include <iostream>
#include <cstring>
#include <unistd.h>
#include <thread>
#include <opencv2/cudaimgproc.hpp>
#include <opencv2/imgproc.hpp>
#include <opencv2/calib3d.hpp>
#include <opencv2/cudaarithm.hpp>
#include <iomanip>
#include <stdlib.h>
#include <unistd.h>
//////////////////////////////
using namespace cv;
using namespace std;
////////////////////////////
// Convert to string
#define SSTR( x ) static_cast< std::ostringstream & >( \
( std::ostringstream() << std::dec << x ) ).str()
/// Global Variables
struct Array {
int arr[2];
};
Mat img;
Mat templ;
Mat result_h;
bool flag = true;
int match_method = 5;
int i=0;
int max_Trackbar = 5;
float fps;
int seconds = 0;
// Function Headers
void delay();
// prototype of the functino for feature extractor
void sobelExtractor(const Mat img, const Rect roi, Mat& feat);
struct Array MatchingMethod( int, void* );
int main(int argc, char **argv)
{
TrackerKCF::Params param;
param.compress_feature = true;
param.compressed_size = 2;
param.desc_npca = 0;
param.desc_pca = TrackerKCF::GRAY | TrackerKCF::CN;
param.detect_thresh = 0.5;
// create a tracker object
Ptr<TrackerKCF> tracker = TrackerKCF::create(param);
tracker->setFeatureExtractor(sobelExtractor);
Ptr<TrackerKCF> tracker2 = TrackerKCF::create(param);
tracker2->setFeatureExtractor(sobelExtractor);
Ptr<TrackerKCF> tracker3 = TrackerKCF::create(param);
tracker3->setFeatureExtractor(sobelExtractor);
Ptr<TrackerKCF> tracker4 = TrackerKCF::create(param);
tracker4->setFeatureExtractor(sobelExtractor);
VideoCapture cap(0);
// Exit if video is not opened
if(!cap.isOpened())
{
//cout << "Could not read video file" << endl;
return 1;
}
cv::imshow("Tracking",0);
// Read first frame
Mat frame;
bool ok = cap.read(frame);
// Define initial bounding box
//Rect bbox(x, y, w, h);
// Uncomment the line below to select a different bounding box
Rect2d bbox = selectROI(frame, false);
// Display bounding box.
rectangle(frame, bbox, Scalar( 255, 0, 0 ), 2, 1 );
///////////////////////////
int H, W, cW, cH;
// print(f"hight {H} , Width {W}")
H = frame.rows;
W = frame.cols;
// Center point of the screen
cW = int(W / 2);
cH = int(H / 2);
Point p1(cW, cH);
//quit if ROI was not selected
if(bbox.width==0 || bbox.height==0)
return 0;
//////////////////////////
//imshow("Tracking", frame);
tracker->init(frame, bbox);
tracker2->init(frame, cv::Rect2d(bbox.x-10,bbox.y-10, bbox.width,bbox.height));
tracker3->init(frame, cv::Rect2d(bbox.x+10,bbox.y+10, bbox.width,bbox.height));
tracker4->init(frame, cv::Rect2d(bbox.x+20,bbox.y+20, bbox.width,bbox.height));
while(true)
{
Mat frame;
cap >> frame;
circle(frame, p1, 3, Scalar(0,255,0), -1);
// Start timer
if(bbox.width!=0 || bbox.height!=0){
double timer = (double)getTickCount();
// Update the tracking result
/////////////////////////////////////
bool ok = tracker->update(frame, bbox);
bool ok2 = tracker->update(frame, bbox);
bool ok3 = tracker->update(frame, bbox);
bool ok4 = tracker->update(frame, bbox);
//////////////////////////////////////
//ok, bbox = tracker->update(frame);
// Calculate Frames per second (FPS)
fps = getTickFrequency() / ((double)getTickCount() - timer);
if (ok || ok2 || ok3 || ok4)
{
// Tracking success : Draw the tracked object
rectangle(frame, bbox, Scalar( 255, 0, 0 ), 2, 1 );
///////////////////////////////////////////////////
int xxx, yyy, height, width;
xxx = bbox.x;
yyy = bbox.y;
height = bbox.height;
width = bbox.width;
int diffX, diffY;
float cxROI, cyROI;
cxROI = int((xxx + (xxx + width)) / 2);
cyROI = int((yyy + (yyy + height)) / 2);
diffX = cxROI - cW;
diffY = cH - cyROI;
//cout<<diffX<<"\n";
//cout<<diffY<<"\n";
Point p(cxROI, cyROI);
circle(frame, p, 3, Scalar(128,0,0), -1);
putText(frame, "FPS : " + SSTR(int(fps)), Point(100,20), FONT_HERSHEY_SIMPLEX, 0.75, Scalar(50,170,50), 2);
putText(frame, "Difference From X-Axis: "+SSTR(int(diffX)), Point(100, 50), FONT_HERSHEY_SIMPLEX, 0.6, Scalar(100, 200, 200), 2);
putText(frame, "Difference From Y-Axis: "+SSTR(int(diffY)), Point(100, 80), FONT_HERSHEY_SIMPLEX, 0.6, Scalar(100, 200, 200), 2);
}
}
else{
}
// Display frame.
imshow("Tracking", frame);
// Exit if ESC pressed.
int k = waitKey(1);
if(k == 27)
{
break;
}
}
return 0;
}
///////////////////////
void sobelExtractor(const Mat img, const Rect roi, Mat& feat){
Mat sobel[2];
Mat patch;
Rect region=roi;
// extract patch inside the image
if(roi.x<0){region.x=0;region.width+=roi.x;}
if(roi.y<0){region.y=0;region.height+=roi.y;}
if(roi.x+roi.width>img.cols)region.width=img.cols-roi.x;
if(roi.y+roi.height>img.rows)region.height=img.rows-roi.y;
if(region.width>img.cols)region.width=img.cols;
if(region.height>img.rows)region.height=img.rows;
patch=img(region).clone();
cvtColor(patch,patch, COLOR_BGR2GRAY);
// add some padding to compensate when the patch is outside image border
int addTop,addBottom, addLeft, addRight;
addTop=region.y-roi.y;
addBottom=(roi.height+roi.y>img.rows?roi.height+roi.y-img.rows:0);
addLeft=region.x-roi.x;
addRight=(roi.width+roi.x>img.cols?roi.width+roi.x-img.cols:0);
copyMakeBorder(patch,patch,addTop,addBottom,addLeft,addRight,BORDER_REPLICATE);
Sobel(patch, sobel[0], CV_32F,1,0,1);
Sobel(patch, sobel[1], CV_32F,0,1,1);
merge(sobel,2,feat);
feat=feat/255.0-0.5; // normalize to range -0.5 .. 0.5
}
////////////////////////////////////////////////////
struct Array MatchingMethod( int, void* )
{
/// Source image to display
Mat frame;
struct Array a;
/////////
for(int i=1; i<=4; i++){
img.copyTo( frame );
// break;
//}
//////////////////////////
cv::cuda::setDevice(0); // initialize CUDA
// convert from mat to gpumat
cv::cuda::GpuMat image_d(img);
cv::cuda::GpuMat templ_d(templ);
cv::cuda::GpuMat result;
// GPU -> NG
cv::Ptr<cv::cuda::TemplateMatching> alg =
cv::cuda::createTemplateMatching(image_d.type(), cv::TM_CCOEFF_NORMED);
alg->match(image_d, templ_d, result); // no return.
//cv::cuda::normalize(result, result, 0, 1, cv::NORM_MINMAX, -1);
double max_value, min_value;
cv::Point location;
cv::cuda::minMaxLoc(result, &min_value, &max_value, 0, &location);
/////////////////////////
double THRESHOLD = 3e-09; //0.3;
if( min_value <= THRESHOLD) {
//struct Array a;
a.arr[0] = location.x;
a.arr[1] = location.y;
cout<<"Hi"<<endl;
}
}
if(flag==true){
return a;
flag = false;
}
//}
}
I write program when window show face and some special points(68).
I use Haar casscade and FaceLandmarkLBF.I have problem in my program. When face have stable position face points are jitter(shaking). How I can fix that? Thanks.
#include <iostream>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include <opencv2/objdetect/objdetect.hpp>
#include <opencv2/face.hpp>
using cv::Scalar;
using cv::Point;
int main(int argc, char** argv)
{
cv::CascadeClassifier faceDetector("haarcascade_frontalface_alt2.xml");
cv::Ptr<cv::face::Facemark>facemark = cv::face::FacemarkLBF::create();
facemark->loadModel("lbfmodel.yaml");
cv::VideoCapture vc(0);
while (true)
{
cv::Mat frame, gray;
vc.read(frame);
cv::cvtColor(frame, gray, cv::COLOR_BGR2GRAY);
//
std::vector<cv::Rect> faces;
faceDetector.detectMultiScale(gray, faces);
std::vector< std::vector<cv::Point2f> > landmarks;
bool success = facemark->fit(frame, faces, landmarks);
for (size_t i = 0; i < landmarks.size(); i++)
{
for (size_t j = 0; j < landmarks[i].size(); j++)
{
cv::circle(frame, cv::Point(landmarks[i][j].x, landmarks[i][j].y), 2, Scalar(255, 0, 0), 2);
}
}
cv::imshow("1", frame);
if ((char)cv::waitKey(20) == 27)
break;
}
return 0;
}
I saw #Nuzhny link : lkdemo.cpp. Not everything is clear for me.
I done rewrite my code but nothing changed:
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include <opencv2/objdetect/objdetect.hpp>
#include "opencv2/video/tracking.hpp"
#include <opencv2/face.hpp>
int main(int argc, char** argv)
{
cv::CascadeClassifier faceDetector("haarcascade_frontalface_alt2.xml");
cv::Ptr<cv::face::Facemark>facemark = cv::face::FacemarkLBF::create();
facemark->loadModel("lbfmodel.yaml");
cv::VideoCapture vc(0);
cv::Mat gray, prevGray, image, frame;
cv::Size subPixWinSize(10, 10), winSize(64, 64);
cv::TermCriteria termcrit(cv::TermCriteria::COUNT | cv::TermCriteria::EPS, 20, 0.03);
std::vector<uchar> status;
std::vector<float> err;
std::vector<cv::Point2f> oldLandmarks;
std::vector< std::vector<cv::Point2f> > landmarks;
bool b = true;
while (true)
{
vc.read(frame);
cv::cvtColor(frame, gray, cv::COLOR_BGR2GRAY);
std::vector<cv::Rect> faces;
faceDetector.detectMultiScale(gray, faces);
bool success = facemark->fit(frame, faces, landmarks);
if (!success)
{
cv::imshow("1", frame);
continue;
}
if (oldLandmarks.empty())
oldLandmarks = landmarks.front();
if (prevGray.empty())
gray.copyTo(prevGray);
calcOpticalFlowPyrLK(prevGray, gray, landmarks.front(), oldLandmarks, status, err, winSize, 3, termcrit, cv::OPTFLOW_LK_GET_MIN_EIGENVALS, 0.001);
for (size_t i = 0; i < oldLandmarks.size(); i++)
{
cv::circle(frame, cv::Point(oldLandmarks[i].x, oldLandmarks[i].y), 2, cv::Scalar(255, 0, 0), 2);
}
cv::imshow("1", frame);
std::swap(oldLandmarks, landmarks.front());
cv::swap(prevGray, gray);
if ((char)cv::waitKey(20) == 27)
break;
}
return 0;
}
Only LK tracking may be not enough. I'm writing some simple application for correcting landmarks after LK with linear Kalman filter (EDIT 2 - remove prev landmarks):
#include <opencv2/opencv.hpp>
#include <opencv2/face.hpp>
///
class PointState
{
public:
PointState(cv::Point2f point)
:
m_point(point),
m_kalman(4, 2, 0, CV_64F)
{
Init();
}
void Update(cv::Point2f point)
{
cv::Mat measurement(2, 1, CV_64FC1);
if (point.x < 0 || point.y < 0)
{
Predict();
measurement.at<double>(0) = m_point.x; //update using prediction
measurement.at<double>(1) = m_point.y;
m_isPredicted = true;
}
else
{
measurement.at<double>(0) = point.x; //update using measurements
measurement.at<double>(1) = point.y;
m_isPredicted = false;
}
// Correction
cv::Mat estimated = m_kalman.correct(measurement);
m_point.x = static_cast<float>(estimated.at<double>(0)); //update using measurements
m_point.y = static_cast<float>(estimated.at<double>(1));
Predict();
}
cv::Point2f GetPoint() const
{
return m_point;
}
bool IsPredicted() const
{
return m_isPredicted;
}
private:
cv::Point2f m_point;
cv::KalmanFilter m_kalman;
double m_deltaTime = 0.2;
double m_accelNoiseMag = 0.3;
bool m_isPredicted = false;
void Init()
{
m_kalman.transitionMatrix = (cv::Mat_<double>(4, 4) <<
1, 0, m_deltaTime, 0,
0, 1, 0, m_deltaTime,
0, 0, 1, 0,
0, 0, 0, 1);
m_kalman.statePre.at<double>(0) = m_point.x; // x
m_kalman.statePre.at<double>(1) = m_point.y; // y
m_kalman.statePre.at<double>(2) = 1; // init velocity x
m_kalman.statePre.at<double>(3) = 1; // init velocity y
m_kalman.statePost.at<double>(0) = m_point.x;
m_kalman.statePost.at<double>(1) = m_point.y;
cv::setIdentity(m_kalman.measurementMatrix);
m_kalman.processNoiseCov = (cv::Mat_<double>(4, 4) <<
pow(m_deltaTime, 4.0) / 4.0, 0, pow(m_deltaTime, 3.0) / 2.0, 0,
0, pow(m_deltaTime, 4.0) / 4.0, 0, pow(m_deltaTime, 3.0) / 2.0,
pow(m_deltaTime, 3.0) / 2.0, 0, pow(m_deltaTime, 2.0), 0,
0, pow(m_deltaTime, 3.0) / 2.0, 0, pow(m_deltaTime, 2.0));
m_kalman.processNoiseCov *= m_accelNoiseMag;
cv::setIdentity(m_kalman.measurementNoiseCov, cv::Scalar::all(0.1));
cv::setIdentity(m_kalman.errorCovPost, cv::Scalar::all(.1));
}
cv::Point2f Predict()
{
cv::Mat prediction = m_kalman.predict();
m_point.x = static_cast<float>(prediction.at<double>(0));
m_point.y = static_cast<float>(prediction.at<double>(1));
return m_point;
}
};
///
void TrackPoints(cv::Mat prevFrame, cv::Mat currFrame,
const std::vector<cv::Point2f>& currLandmarks,
std::vector<PointState>& trackPoints)
{
// Lucas-Kanade
cv::TermCriteria termcrit(cv::TermCriteria::COUNT | cv::TermCriteria::EPS, 30, 0.01);
cv::Size winSize(7, 7);
std::vector<uchar> status(trackPoints.size(), 0);
std::vector<float> err;
std::vector<cv::Point2f> newLandmarks;
std::vector<cv::Point2f> prevLandmarks;
std::for_each(trackPoints.begin(), trackPoints.end(), [&](const PointState& pts) { prevLandmarks.push_back(pts.GetPoint()); });
cv::calcOpticalFlowPyrLK(prevFrame, currFrame, prevLandmarks, newLandmarks, status, err, winSize, 3, termcrit, 0, 0.001);
for (size_t i = 0; i < status.size(); ++i)
{
if (status[i])
{
trackPoints[i].Update((newLandmarks[i] + currLandmarks[i]) / 2);
}
else
{
trackPoints[i].Update(currLandmarks[i]);
}
}
}
///
int main(int argc, char** argv)
{
cv::CascadeClassifier faceDetector("haarcascade_frontalface_alt2.xml");
cv::Ptr<cv::face::Facemark> facemark = cv::face::FacemarkLBF::create();
facemark->loadModel("lbfmodel.yaml");
cv::VideoCapture cam(0, cv::CAP_DSHOW);
cv::namedWindow("Facial Landmark Detection", cv::WINDOW_NORMAL);
cv::Mat frame;
cv::Mat currGray;
cv::Mat prevGray;
std::vector<PointState> trackPoints;
trackPoints.reserve(68);
while (cam.read(frame))
{
std::vector<cv::Rect> faces;
cv::cvtColor(frame, currGray, cv::COLOR_BGR2GRAY);
faceDetector.detectMultiScale(currGray, faces, 1.1, 3, cv::CASCADE_FIND_BIGGEST_OBJECT);
std::vector<std::vector<cv::Point2f>> landmarks;
bool success = facemark->fit(frame, faces, landmarks);
if (success)
{
if (prevGray.empty())
{
trackPoints.clear();
for (cv::Point2f lp : landmarks[0])
{
trackPoints.emplace_back(lp);
}
}
else
{
if (trackPoints.empty())
{
for (cv::Point2f lp : landmarks[0])
{
trackPoints.emplace_back(lp);
}
}
else
{
TrackPoints(prevGray, currGray, landmarks[0], trackPoints);
}
}
for (const PointState& tp : trackPoints)
{
cv::circle(frame, tp.GetPoint(), 3, tp.IsPredicted() ? cv::Scalar(0, 0, 255) : cv::Scalar(0, 255, 0), cv::FILLED);
}
for (cv::Point2f lp : landmarks[0])
{
cv::circle(frame, lp, 2, cv::Scalar(255, 0, 255), cv::FILLED);
}
}
cv::imshow("Facial Landmark Detection", frame);
if (cv::waitKey(1) == 27)
break;
prevGray = currGray;
}
return 0;
}
So, the margenta points - raw landmarks and green points - corrected after LK+Kalman: result video.
You can change Kalman options with 2 constants:
double m_deltaTime = 0.2;
double m_accelNoiseMag = 0.3;
It's latency and noise.
I am trying to code a program on opencv to decide whether a human has approached ahead the camera. After I run the execution file, I get the captured video for few seconds and encounter the segmentation fault error.
The code is like this
Here are headers:
#include "opencv2/objdetect.hpp"
#include "opencv2/videoio.hpp"
#include "opencv2/highgui.hpp"
#include "opencv2/imgproc.hpp"
#include <iostream>
#include <stdio.h>
using namespace std;
using namespace cv;
//define static variable
static int cApp = 0;//number of approached frame
static double last = 0;
//define functions
void detectAndDisplay( Mat frame );
bool computeArea( double width, double height, double lastArea);
double runningAverage(int M);
//define opencv function and classifier
String upperbody_cascade_name = "home/pi/opencv- 3.0.0/data/haarcascades/haarcascade_upperbody.xml";
CascadeClassifier upper_cascade;
String window_name = "Capture - upper body detection";
Here is the main function:
int main( void )
{
//define variable
VideoCapture capture;
Mat frame;
//-- 1. Load the cascades
upper_cascade.load("/home/pi/opencv-3.0.0/data/haarcascades/haarcascade_upperbody.xml");
//-- 2. Read the video stream
capture.open( -1 );
if ( ! capture.isOpened() ) { printf("--(!)Error opening video capture\n"); return -1; }
while ( capture.read(frame) )
{
if( frame.empty() )
{
printf(" --(!) No captured frame -- Break!");
break;
}
//-- 3. Apply the classifier to the frame
detectAndDisplay( frame );
char c = (char)waitKey(10);
if( c == 27 ) { break; } // escape
}
capture.release();
return 0;
}
Here is the detectAndDisplay function:
void detectAndDisplay( Mat frame )
{
std::vector<Rect> upperbodys;
Mat frame_gray;
cvtColor( frame, frame_gray, COLOR_BGR2GRAY );
equalizeHist( frame_gray, frame_gray );
//-- Detect upperbodys
upper_cascade.detectMultiScale( frame_gray, upperbodys, 1.05, 3, 0|CASCADE_SCALE_IMAGE, Size(30, 30) );
Point center( upperbodys[0].x + upperbodys[0].width/2, upperbodys[0].y + upperbodys[0].height/2 );
ellipse( frame, center, Size( upperbodys[0].width/2, upperbodys[0].height/2 ), 0, 0, 360, Scalar( 255, 0, 255 ), 4, 8, 0 );
bool ifApproached = computeArea(upperbodys[0].width/2, upperbodys[0].height/2, last);
if (ifApproached == true) {
cApp++;
}
if (cApp == 3) {
cout << "have approached" << endl;
cApp = cApp - 3;
}
//-- Show what you got
imshow( window_name, frame );
}
Here is the computeArea function:
bool computeArea( double width, double height, double lastArea) {
double newArea = width * height;
bool ifApproached = false;
//double presentArea = newArea;
double presentArea = runningAverage(newArea);
double DifferenceBewteenAreas = presentArea - lastArea;
if (DifferenceBewteenAreas > 1) {//threshold
ifApproached = true;
}
last = presentArea;
return ifApproached;
}
Here is runningAverage function:
double runningAverage(int M) {
//M is measurement
//#define LM_SIZE 5
static int LM[5];
static int index =0;
static long sum = 0;
static int count =0;
//keep sum updated to improve speed
sum = sum - LM[index];
LM[index] = M;
sum = sum + LM[index];
index++;
index = index % 5;
if (count < 5) {
count++;
}
return (double)(sum / (double)count);
}
I have searched many opencv segmentation fault questions, some said this segmentation fault was caused by wrong array used, but my case has little use of array. Others said misused of function characters could also cause this kind of errors, I agree with this, some of my characters could be wrong here.
Actually I found that I should not use upperbodys[0] in the code, because sometimes there are no object being detected at all,so there could be some memory read error happens, I used upperbodys[i] instead and it works well then.
void detectAndDisplay( Mat frame )
{
std::vector<Rect> upperbodys;
Mat frame_gray;
cvtColor( frame, frame_gray, COLOR_BGR2GRAY );
equalizeHist( frame_gray, frame_gray );
//-- Detect upperbodys
upper_cascade.detectMultiScale( frame_gray, upperbodys, 1.05, 3, 0|CASCADE_SCALE_IMAGE, Size(30, 30) );
int size = upperbodys.size();
double newArea = -1;
for (int i = 0 ; i < size; i++) {
Point center( upperbodys[i].x + upperbodys[i].width/2, upperbodys[i].y + upperbodys[i].height/2 );
ellipse( frame, center, Size( upperbodys[i].width/2, upperbodys[i].height/2 ), 0, 0, 360, Scalar( 255, 0, 255 ), 4, 8, 0 );
//bool ifApproached = computeArea(upperbodys[i].width/2, upperbodys[i].height/2, last);
//////////////////////////////////////////
newArea = upperbodys[i].width/2 * upperbodys[i].height/2;
if (newArea != -1) {
cout << "UpperBodys has value, index = " << i << endl;
break;
}
}
bool ifApproached = false;
//double presentArea = runningAverage(newArea);
double presentArea = newArea;
double DifferenceBewteenAreas = presentArea - last;
if (DifferenceBewteenAreas > 1) {//threshold
ifApproached = true;
}
last = presentArea;
//////////////////////////////////////////
if (ifApproached == true) {
cApp++;
}
if (cApp == 3) {
cout << "have approached" << endl;
cApp = cApp - 3;
}
//-- Show what you got
imshow( window_name, frame );
}
I have this image :
And I applied dilation with this code :
int dilation_elem = 0;
int dilation_size = 0;
int const max_elem = 2;
int const max_kernel_size = 21;
void Dilation( int, void* );
int main( int argc, char** argv )
{
src = imread("a18.png");
if( !src.data )
{ return -1; }
namedWindow( "Dilation Demo", CV_WINDOW_AUTOSIZE );
cvMoveWindow( "Dilation Demo", src.cols, 0 );
createTrackbar( "Element:\n 0: Rect \n 1: Cross \n 2: Ellipse", "Dilation Demo",
&dilation_elem, max_elem,
Dilation );
createTrackbar( "Kernel size:\n 2n +1", "Dilation Demo",
&dilation_size, max_kernel_size,
Dilation );
//int dilation_size =7;
/// Default start
Dilation( 0, 0 );
waitKey(0);
return 0;
}
void Dilation( int, void* )
{
int dilation_type;
if( dilation_elem == 0 ){ dilation_type = MORPH_RECT; }
else if( dilation_elem == 1 ){ dilation_type = MORPH_CROSS; }
else if( dilation_elem == 2) { dilation_type = MORPH_ELLIPSE; }
Mat element = getStructuringElement( dilation_type,
Size( 2*dilation_size + 1, 2*dilation_size+1 ),
Point( dilation_size, dilation_size ) );
dilate( src, dilation_dst, element );
imshow( "Dilation Demo", dilation_dst );
imwrite("a18d.png",dilation_dst);
}
And after this step I get this consequent:
And the final step is skeleton :
#include <iostream>
#include "opencv2/highgui/highgui.hpp"
#include "opencv2/imgproc/imgproc.hpp"
int main()
{
cv::Mat img = cv::imread("a18d.png", 0);
cv::threshold(img, img, 127, 255, cv::THRESH_BINARY);
cv::Mat skel(img.size(), CV_8UC1, cv::Scalar(0));
cv::Mat temp;
cv::Mat eroded;
cv::Mat element = cv::getStructuringElement(cv::MORPH_CROSS, cv::Size(3,3));
bool done;
do
{
cv::erode(img, eroded, element);
cv::dilate(eroded, temp, element); // temp = open(img)
cv::subtract(img, temp, temp);
cv::bitwise_or(skel, temp, skel);
eroded.copyTo(img);
done = (cv::countNonZero(img) == 0);
} while (!done);
cv::imshow("Skeleton", skel);
cv::imwrite("18s.png",skel);
cv::waitKey(0);
return 0;
}
code here
And I hve this image :
But I want image like this :
What can I do for this? What is the problem
these steps was done for Image 4 and result is good
Open CV doesn't seem to have a shrink as opposed to an erode operation. At least I can't find it. Try mine
https://github.com/MalcolmMcLean/binaryimagelibrary/blob/master/medialaxistransform.c
Try just using the function "thin" instead of eroding your image.
I am a beginner of using opencv. I using opencv 2.4 with VS2013. I had develop face detection code and success but the decision made is from frame by frame. How can I made final decision by combining all the frame or frame averaging?? For example, when the total face detected is 90% detected, the final decision is 'FACE DETECTED' and vice versa.
Here is my code:
int main(int argc, char** argv)
{
CvCapture* capture;
capture = cvCaptureFromFile("C:/Users/user/Desktop/Face and Motion/45.avi");
//assert(capture != NULL); //terminate if capture is NULL
IplImage* frame;
while (true)
{
frame = cvQueryFrame(capture);
if (!frame)
break;
cvShowImage("original", frame); //show
CvMemStorage* storage = cvCreateMemStorage(0);
CvHaarClassifierCascade* cascade = (CvHaarClassifierCascade*)cvLoad("C:/opencv2410/sources/data/haarcascades/haarcascade_frontalface_alt.xml");
cvClearMemStorage(storage);
CvSeq* faces = cvHaarDetectObjects(frame, cascade, storage, 1.1, 4, 0, cvSize(40, 50));
CvRect* r;
if (faces) //change from (!faces) to (faces)
{
for (int i = 0; i < (faces ? faces->total : 0); i++)
{
r = (CvRect*)cvGetSeqElem(faces, i);
//cvRectangle(frame, cvPoint(100, 50), cvPoint(200, 200), CV_RGB(255, 0, 0), 5, 8);
cvRectangle(frame, cvPoint(r->x, r->y), cvPoint(r->x + r->width, r->y + r->height), CV_RGB(255, 0, 0));
cvShowImage("RESULTS", frame);
char c = cvWaitKey(1000);
}
}
else
{
cvShowImage("RESULT", frame);
}
printf("%d Face Found !\n", faces ? faces->total : 0);
if ((faces ? faces->total : 0) == 0)
{
printf("FACE NOT DETECTED !\n\n");
}
else
{
printf("FACE DETECTED !\n\n");
}
}
return (0);
}
you need two variables like
int frame_count = 0;
int detected_face_count = 0;
you should increase frame_count for every frame and increase detected_face_count when a face detected. finally detected_face_count / frame_count gives your desired value.
however, i did some corrections on your code (i could not test it)
changes explained by comments. i hope it will be useful.
int main(int argc, char** argv)
{
CvCapture* capture;
capture = cvCaptureFromFile("C:/Users/user/Desktop/Face and Motion/45.avi");
//assert(capture != NULL); //terminate if capture is NULL
IplImage* frame;
//you need two variables
int frame_count = 0;
int detected_face_count = 0;
// these lines should not be in the loop
CvMemStorage* storage = cvCreateMemStorage(0);
CvHaarClassifierCascade* cascade = (CvHaarClassifierCascade*)cvLoad("C:/opencv2410/sources/data/haarcascades/haarcascade_frontalface_alt.xml");
while (true)
{
frame = cvQueryFrame(capture);
if (!frame)
break;
frame_count++; // increase frame_count
cvShowImage("original", frame); //show
cvClearMemStorage(storage);
CvSeq* faces = cvHaarDetectObjects(frame, cascade, storage, 1.1, 4, 0, cvSize(40, 50));
CvRect* r;
for (int i = 0; i < (faces ? faces->total : 0); i++)
{
r = (CvRect*)cvGetSeqElem(faces, i);
//cvRectangle(frame, cvPoint(100, 50), cvPoint(200, 200), CV_RGB(255, 0, 0), 5, 8);
cvRectangle(frame, cvPoint(r->x, r->y), cvPoint(r->x + r->width, r->y + r->height), CV_RGB(255, 0, 0));
}
cvShowImage("RESULT", frame);
char c = cvWaitKey(1000);
printf("%d Face Found !\n", faces ? faces->total : 0);
if ((faces ? faces->total : 0) == 0)
{
printf("FACE NOT DETECTED !\n\n");
}
else
{
detected_face_count ++; // increase detected_face_count
printf("FACE DETECTED !\n\n");
}
}
if( (float)(detected_face_count / frame_count) > 0.89 )
{
printf("FACES DETECTED over %90 of frames!\n\n");
}
return (0);
}
as #Miki stated you should not use deprecated C api
take a look at the code below using C++ api (tested with OpenCV 3.1)
#include <opencv2/opencv.hpp>
using namespace cv;
using namespace std;
int main(int argc, char** argv)
{
VideoCapture capture("C:/Users/user/Desktop/Face and Motion/45.avi");
if(!capture.isOpened())
{
printf("Video file could not be opened !\n");
return -1;
}
Mat frame;
//you need two variables
int frame_count = 0;
int detected_face_count = 0;
// these lines should not be in the loop
CascadeClassifier cascade;
if( !cascade.load( "C:/opencv2410/sources/data/haarcascades/haarcascade_frontalface_alt.xml" ) )
{
cerr << "ERROR: Could not load classifier cascade" << endl;
return -1;
}
while (true)
{
capture >>frame;
if (frame.empty())
break;
frame_count++; // increase frame_count
imshow("original", frame); //show
vector<Rect> faces;
cascade.detectMultiScale(frame, faces,1.1, 4, 0, Size(40, 50));
for (int i = 0; i < faces.size(); i++)
{
rectangle(frame, faces[i], Scalar(0, 0, 255));
}
imshow("RESULT", frame);
char c = waitKey(10);
printf("%d Face Found !\n", faces.size());
if ( faces.size() == 0 )
{
printf("FACE NOT DETECTED !\n\n");
}
else
{
detected_face_count ++; // increase detected_face_count
printf("FACE DETECTED !\n\n");
}
}
printf("count of frames : %d \n", frame_count);
printf("count of frames has detected face : %d \n", detected_face_count);
printf("Face Found %d percent of frames!\n", (int)(100 * detected_face_count / frame_count));
if( (float)(detected_face_count / frame_count) > 0.89 )
{
printf("FACES DETECTED over %90 of frames!\n\n");
}
return (0);
}