I am a beginner of using opencv. I using opencv 2.4 with VS2013. I had develop face detection code and success but the decision made is from frame by frame. How can I made final decision by combining all the frame or frame averaging?? For example, when the total face detected is 90% detected, the final decision is 'FACE DETECTED' and vice versa.
Here is my code:
int main(int argc, char** argv)
{
CvCapture* capture;
capture = cvCaptureFromFile("C:/Users/user/Desktop/Face and Motion/45.avi");
//assert(capture != NULL); //terminate if capture is NULL
IplImage* frame;
while (true)
{
frame = cvQueryFrame(capture);
if (!frame)
break;
cvShowImage("original", frame); //show
CvMemStorage* storage = cvCreateMemStorage(0);
CvHaarClassifierCascade* cascade = (CvHaarClassifierCascade*)cvLoad("C:/opencv2410/sources/data/haarcascades/haarcascade_frontalface_alt.xml");
cvClearMemStorage(storage);
CvSeq* faces = cvHaarDetectObjects(frame, cascade, storage, 1.1, 4, 0, cvSize(40, 50));
CvRect* r;
if (faces) //change from (!faces) to (faces)
{
for (int i = 0; i < (faces ? faces->total : 0); i++)
{
r = (CvRect*)cvGetSeqElem(faces, i);
//cvRectangle(frame, cvPoint(100, 50), cvPoint(200, 200), CV_RGB(255, 0, 0), 5, 8);
cvRectangle(frame, cvPoint(r->x, r->y), cvPoint(r->x + r->width, r->y + r->height), CV_RGB(255, 0, 0));
cvShowImage("RESULTS", frame);
char c = cvWaitKey(1000);
}
}
else
{
cvShowImage("RESULT", frame);
}
printf("%d Face Found !\n", faces ? faces->total : 0);
if ((faces ? faces->total : 0) == 0)
{
printf("FACE NOT DETECTED !\n\n");
}
else
{
printf("FACE DETECTED !\n\n");
}
}
return (0);
}
you need two variables like
int frame_count = 0;
int detected_face_count = 0;
you should increase frame_count for every frame and increase detected_face_count when a face detected. finally detected_face_count / frame_count gives your desired value.
however, i did some corrections on your code (i could not test it)
changes explained by comments. i hope it will be useful.
int main(int argc, char** argv)
{
CvCapture* capture;
capture = cvCaptureFromFile("C:/Users/user/Desktop/Face and Motion/45.avi");
//assert(capture != NULL); //terminate if capture is NULL
IplImage* frame;
//you need two variables
int frame_count = 0;
int detected_face_count = 0;
// these lines should not be in the loop
CvMemStorage* storage = cvCreateMemStorage(0);
CvHaarClassifierCascade* cascade = (CvHaarClassifierCascade*)cvLoad("C:/opencv2410/sources/data/haarcascades/haarcascade_frontalface_alt.xml");
while (true)
{
frame = cvQueryFrame(capture);
if (!frame)
break;
frame_count++; // increase frame_count
cvShowImage("original", frame); //show
cvClearMemStorage(storage);
CvSeq* faces = cvHaarDetectObjects(frame, cascade, storage, 1.1, 4, 0, cvSize(40, 50));
CvRect* r;
for (int i = 0; i < (faces ? faces->total : 0); i++)
{
r = (CvRect*)cvGetSeqElem(faces, i);
//cvRectangle(frame, cvPoint(100, 50), cvPoint(200, 200), CV_RGB(255, 0, 0), 5, 8);
cvRectangle(frame, cvPoint(r->x, r->y), cvPoint(r->x + r->width, r->y + r->height), CV_RGB(255, 0, 0));
}
cvShowImage("RESULT", frame);
char c = cvWaitKey(1000);
printf("%d Face Found !\n", faces ? faces->total : 0);
if ((faces ? faces->total : 0) == 0)
{
printf("FACE NOT DETECTED !\n\n");
}
else
{
detected_face_count ++; // increase detected_face_count
printf("FACE DETECTED !\n\n");
}
}
if( (float)(detected_face_count / frame_count) > 0.89 )
{
printf("FACES DETECTED over %90 of frames!\n\n");
}
return (0);
}
as #Miki stated you should not use deprecated C api
take a look at the code below using C++ api (tested with OpenCV 3.1)
#include <opencv2/opencv.hpp>
using namespace cv;
using namespace std;
int main(int argc, char** argv)
{
VideoCapture capture("C:/Users/user/Desktop/Face and Motion/45.avi");
if(!capture.isOpened())
{
printf("Video file could not be opened !\n");
return -1;
}
Mat frame;
//you need two variables
int frame_count = 0;
int detected_face_count = 0;
// these lines should not be in the loop
CascadeClassifier cascade;
if( !cascade.load( "C:/opencv2410/sources/data/haarcascades/haarcascade_frontalface_alt.xml" ) )
{
cerr << "ERROR: Could not load classifier cascade" << endl;
return -1;
}
while (true)
{
capture >>frame;
if (frame.empty())
break;
frame_count++; // increase frame_count
imshow("original", frame); //show
vector<Rect> faces;
cascade.detectMultiScale(frame, faces,1.1, 4, 0, Size(40, 50));
for (int i = 0; i < faces.size(); i++)
{
rectangle(frame, faces[i], Scalar(0, 0, 255));
}
imshow("RESULT", frame);
char c = waitKey(10);
printf("%d Face Found !\n", faces.size());
if ( faces.size() == 0 )
{
printf("FACE NOT DETECTED !\n\n");
}
else
{
detected_face_count ++; // increase detected_face_count
printf("FACE DETECTED !\n\n");
}
}
printf("count of frames : %d \n", frame_count);
printf("count of frames has detected face : %d \n", detected_face_count);
printf("Face Found %d percent of frames!\n", (int)(100 * detected_face_count / frame_count));
if( (float)(detected_face_count / frame_count) > 0.89 )
{
printf("FACES DETECTED over %90 of frames!\n\n");
}
return (0);
}
Related
I write program when window show face and some special points(68).
I use Haar casscade and FaceLandmarkLBF.I have problem in my program. When face have stable position face points are jitter(shaking). How I can fix that? Thanks.
#include <iostream>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include <opencv2/objdetect/objdetect.hpp>
#include <opencv2/face.hpp>
using cv::Scalar;
using cv::Point;
int main(int argc, char** argv)
{
cv::CascadeClassifier faceDetector("haarcascade_frontalface_alt2.xml");
cv::Ptr<cv::face::Facemark>facemark = cv::face::FacemarkLBF::create();
facemark->loadModel("lbfmodel.yaml");
cv::VideoCapture vc(0);
while (true)
{
cv::Mat frame, gray;
vc.read(frame);
cv::cvtColor(frame, gray, cv::COLOR_BGR2GRAY);
//
std::vector<cv::Rect> faces;
faceDetector.detectMultiScale(gray, faces);
std::vector< std::vector<cv::Point2f> > landmarks;
bool success = facemark->fit(frame, faces, landmarks);
for (size_t i = 0; i < landmarks.size(); i++)
{
for (size_t j = 0; j < landmarks[i].size(); j++)
{
cv::circle(frame, cv::Point(landmarks[i][j].x, landmarks[i][j].y), 2, Scalar(255, 0, 0), 2);
}
}
cv::imshow("1", frame);
if ((char)cv::waitKey(20) == 27)
break;
}
return 0;
}
I saw #Nuzhny link : lkdemo.cpp. Not everything is clear for me.
I done rewrite my code but nothing changed:
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include <opencv2/objdetect/objdetect.hpp>
#include "opencv2/video/tracking.hpp"
#include <opencv2/face.hpp>
int main(int argc, char** argv)
{
cv::CascadeClassifier faceDetector("haarcascade_frontalface_alt2.xml");
cv::Ptr<cv::face::Facemark>facemark = cv::face::FacemarkLBF::create();
facemark->loadModel("lbfmodel.yaml");
cv::VideoCapture vc(0);
cv::Mat gray, prevGray, image, frame;
cv::Size subPixWinSize(10, 10), winSize(64, 64);
cv::TermCriteria termcrit(cv::TermCriteria::COUNT | cv::TermCriteria::EPS, 20, 0.03);
std::vector<uchar> status;
std::vector<float> err;
std::vector<cv::Point2f> oldLandmarks;
std::vector< std::vector<cv::Point2f> > landmarks;
bool b = true;
while (true)
{
vc.read(frame);
cv::cvtColor(frame, gray, cv::COLOR_BGR2GRAY);
std::vector<cv::Rect> faces;
faceDetector.detectMultiScale(gray, faces);
bool success = facemark->fit(frame, faces, landmarks);
if (!success)
{
cv::imshow("1", frame);
continue;
}
if (oldLandmarks.empty())
oldLandmarks = landmarks.front();
if (prevGray.empty())
gray.copyTo(prevGray);
calcOpticalFlowPyrLK(prevGray, gray, landmarks.front(), oldLandmarks, status, err, winSize, 3, termcrit, cv::OPTFLOW_LK_GET_MIN_EIGENVALS, 0.001);
for (size_t i = 0; i < oldLandmarks.size(); i++)
{
cv::circle(frame, cv::Point(oldLandmarks[i].x, oldLandmarks[i].y), 2, cv::Scalar(255, 0, 0), 2);
}
cv::imshow("1", frame);
std::swap(oldLandmarks, landmarks.front());
cv::swap(prevGray, gray);
if ((char)cv::waitKey(20) == 27)
break;
}
return 0;
}
Only LK tracking may be not enough. I'm writing some simple application for correcting landmarks after LK with linear Kalman filter (EDIT 2 - remove prev landmarks):
#include <opencv2/opencv.hpp>
#include <opencv2/face.hpp>
///
class PointState
{
public:
PointState(cv::Point2f point)
:
m_point(point),
m_kalman(4, 2, 0, CV_64F)
{
Init();
}
void Update(cv::Point2f point)
{
cv::Mat measurement(2, 1, CV_64FC1);
if (point.x < 0 || point.y < 0)
{
Predict();
measurement.at<double>(0) = m_point.x; //update using prediction
measurement.at<double>(1) = m_point.y;
m_isPredicted = true;
}
else
{
measurement.at<double>(0) = point.x; //update using measurements
measurement.at<double>(1) = point.y;
m_isPredicted = false;
}
// Correction
cv::Mat estimated = m_kalman.correct(measurement);
m_point.x = static_cast<float>(estimated.at<double>(0)); //update using measurements
m_point.y = static_cast<float>(estimated.at<double>(1));
Predict();
}
cv::Point2f GetPoint() const
{
return m_point;
}
bool IsPredicted() const
{
return m_isPredicted;
}
private:
cv::Point2f m_point;
cv::KalmanFilter m_kalman;
double m_deltaTime = 0.2;
double m_accelNoiseMag = 0.3;
bool m_isPredicted = false;
void Init()
{
m_kalman.transitionMatrix = (cv::Mat_<double>(4, 4) <<
1, 0, m_deltaTime, 0,
0, 1, 0, m_deltaTime,
0, 0, 1, 0,
0, 0, 0, 1);
m_kalman.statePre.at<double>(0) = m_point.x; // x
m_kalman.statePre.at<double>(1) = m_point.y; // y
m_kalman.statePre.at<double>(2) = 1; // init velocity x
m_kalman.statePre.at<double>(3) = 1; // init velocity y
m_kalman.statePost.at<double>(0) = m_point.x;
m_kalman.statePost.at<double>(1) = m_point.y;
cv::setIdentity(m_kalman.measurementMatrix);
m_kalman.processNoiseCov = (cv::Mat_<double>(4, 4) <<
pow(m_deltaTime, 4.0) / 4.0, 0, pow(m_deltaTime, 3.0) / 2.0, 0,
0, pow(m_deltaTime, 4.0) / 4.0, 0, pow(m_deltaTime, 3.0) / 2.0,
pow(m_deltaTime, 3.0) / 2.0, 0, pow(m_deltaTime, 2.0), 0,
0, pow(m_deltaTime, 3.0) / 2.0, 0, pow(m_deltaTime, 2.0));
m_kalman.processNoiseCov *= m_accelNoiseMag;
cv::setIdentity(m_kalman.measurementNoiseCov, cv::Scalar::all(0.1));
cv::setIdentity(m_kalman.errorCovPost, cv::Scalar::all(.1));
}
cv::Point2f Predict()
{
cv::Mat prediction = m_kalman.predict();
m_point.x = static_cast<float>(prediction.at<double>(0));
m_point.y = static_cast<float>(prediction.at<double>(1));
return m_point;
}
};
///
void TrackPoints(cv::Mat prevFrame, cv::Mat currFrame,
const std::vector<cv::Point2f>& currLandmarks,
std::vector<PointState>& trackPoints)
{
// Lucas-Kanade
cv::TermCriteria termcrit(cv::TermCriteria::COUNT | cv::TermCriteria::EPS, 30, 0.01);
cv::Size winSize(7, 7);
std::vector<uchar> status(trackPoints.size(), 0);
std::vector<float> err;
std::vector<cv::Point2f> newLandmarks;
std::vector<cv::Point2f> prevLandmarks;
std::for_each(trackPoints.begin(), trackPoints.end(), [&](const PointState& pts) { prevLandmarks.push_back(pts.GetPoint()); });
cv::calcOpticalFlowPyrLK(prevFrame, currFrame, prevLandmarks, newLandmarks, status, err, winSize, 3, termcrit, 0, 0.001);
for (size_t i = 0; i < status.size(); ++i)
{
if (status[i])
{
trackPoints[i].Update((newLandmarks[i] + currLandmarks[i]) / 2);
}
else
{
trackPoints[i].Update(currLandmarks[i]);
}
}
}
///
int main(int argc, char** argv)
{
cv::CascadeClassifier faceDetector("haarcascade_frontalface_alt2.xml");
cv::Ptr<cv::face::Facemark> facemark = cv::face::FacemarkLBF::create();
facemark->loadModel("lbfmodel.yaml");
cv::VideoCapture cam(0, cv::CAP_DSHOW);
cv::namedWindow("Facial Landmark Detection", cv::WINDOW_NORMAL);
cv::Mat frame;
cv::Mat currGray;
cv::Mat prevGray;
std::vector<PointState> trackPoints;
trackPoints.reserve(68);
while (cam.read(frame))
{
std::vector<cv::Rect> faces;
cv::cvtColor(frame, currGray, cv::COLOR_BGR2GRAY);
faceDetector.detectMultiScale(currGray, faces, 1.1, 3, cv::CASCADE_FIND_BIGGEST_OBJECT);
std::vector<std::vector<cv::Point2f>> landmarks;
bool success = facemark->fit(frame, faces, landmarks);
if (success)
{
if (prevGray.empty())
{
trackPoints.clear();
for (cv::Point2f lp : landmarks[0])
{
trackPoints.emplace_back(lp);
}
}
else
{
if (trackPoints.empty())
{
for (cv::Point2f lp : landmarks[0])
{
trackPoints.emplace_back(lp);
}
}
else
{
TrackPoints(prevGray, currGray, landmarks[0], trackPoints);
}
}
for (const PointState& tp : trackPoints)
{
cv::circle(frame, tp.GetPoint(), 3, tp.IsPredicted() ? cv::Scalar(0, 0, 255) : cv::Scalar(0, 255, 0), cv::FILLED);
}
for (cv::Point2f lp : landmarks[0])
{
cv::circle(frame, lp, 2, cv::Scalar(255, 0, 255), cv::FILLED);
}
}
cv::imshow("Facial Landmark Detection", frame);
if (cv::waitKey(1) == 27)
break;
prevGray = currGray;
}
return 0;
}
So, the margenta points - raw landmarks and green points - corrected after LK+Kalman: result video.
You can change Kalman options with 2 constants:
double m_deltaTime = 0.2;
double m_accelNoiseMag = 0.3;
It's latency and noise.
My project is tracking object using opencv and send the coordinate to arduino as tx and read the data using another arduino (rx) with 'SoftwareSerial'. There are no problem with object tracking, knowing the coordinate, and communication between 2 arduino. The problem is i can not send the coordinate while 'tracker' running, but when I close the 'tracker', the data start appearing in the serial com.
opencv code
using namespace cv;
using namespace std;
//default capture width and height
int FRAME_WIDTH = 320; //640 320
int FRAME_HEIGHT = 240; //480 240
int MIN_OBJECT_AREA = 10*10;
int iLowH = 16;
int iHighH = 104;
int iLowS = 110;
int iHighS = 164;
int iLowV = 63;
int iHighV = 255;
int centerX, centerY;
int Xg,Yg;
int Modefilter = 1;
FILE *fp;
bool kirim=false;
string intToString(int number){
std::stringstream ss;
ss << number;
return ss.str();
}
void detect(){
}
int main( int argc, char** argv ){
//open serial
FILE* serial = fopen("\\\\.\\COM3", "w+");
if (serial == 0) {
printf("Failed to open serial port\n");
}
//capture the video from web cam
VideoCapture cap(0);
// if not success, exit program
if ( !cap.isOpened() ){
cout << "Cannot open the web cam" << endl;
return -1;
}
//set height and width of capture frame
cap.set(CV_CAP_PROP_FRAME_WIDTH,FRAME_WIDTH);
cap.set(CV_CAP_PROP_FRAME_HEIGHT,FRAME_HEIGHT);
//create a window called "Control"
namedWindow("Control", CV_WINDOW_AUTOSIZE);
//Create trackbars in "Control" window
cvCreateTrackbar("LowH", "Control", &iLowH, 179); //Hue (0 - 179)
cvCreateTrackbar("HighH", "Control", &iHighH, 179);
cvCreateTrackbar("LowS", "Control", &iLowS, 255); //Saturation (0 - 255)
cvCreateTrackbar("HighS", "Control", &iHighS, 255);
cvCreateTrackbar("LowV", "Control", &iLowV, 255); //Value (0 - 255)
cvCreateTrackbar("HighV", "Control", &iHighV, 255);
string XX,YY,parser1,parser2,result;
while (serial!=0){
Mat imgOriginal;
bool bSuccess = cap.read(imgOriginal); // read a new frame from video
if (!bSuccess){ //if not success, break loop
cout << "Cannot read a frame from video stream" << endl;
break;
}
//Convert the captured frame from BGR to HSV
Mat imgHSV;
cvtColor(imgOriginal, imgHSV, COLOR_BGR2HSV);
//find center point
centerX = FRAME_WIDTH/2;
centerY = FRAME_HEIGHT/2;
putText(imgOriginal, "Tekan", Point(5,10), FONT_HERSHEY_COMPLEX, 0.35, Scalar(0, 255, 0), 0.25, 8);
putText(imgOriginal, "a : Mulai Mengikuti Objek", Point(5,20), FONT_HERSHEY_COMPLEX, 0.35, Scalar(0, 255, 0), 0.25, 8);
putText(imgOriginal, "b : Berhenti Mengikuti Objek", Point(5,30), FONT_HERSHEY_COMPLEX, 0.35, Scalar(0, 255, 0), 0.25, 8);
//create cross line
line(imgOriginal,Point(centerX, centerY-20), Point(centerX, centerY+20), Scalar(0,255,0), 1.5);
line(imgOriginal,Point(centerX-20, centerY), Point(centerX+20, centerY), Scalar(0,255,0), 1.5);
//Threshold the image
Mat imgThresholded;
inRange(imgHSV, Scalar(iLowH, iLowS, iLowV), Scalar(iHighH, iHighS, iHighV), imgThresholded);
//morphological opening (remove small objects from the foreground)
erode(imgThresholded, imgThresholded, getStructuringElement(MORPH_ELLIPSE, Size(5, 5)) );
dilate( imgThresholded, imgThresholded, getStructuringElement(MORPH_ELLIPSE, Size(5, 5)) );
//morphological closing (fill small holes in the foreground)
dilate( imgThresholded, imgThresholded, getStructuringElement(MORPH_ELLIPSE, Size(5, 5)) );
erode(imgThresholded, imgThresholded, getStructuringElement(MORPH_ELLIPSE, Size(5, 5)) );
//these two vectors needed for output of findContours
vector< vector<Point> > contours;
vector<Vec4i> hierarchy;
Mat imgContour;
imgThresholded.copyTo(imgContour);
//find contours of filtered image using openCV findContours function
findContours(imgContour,contours,hierarchy,CV_RETR_CCOMP,CV_CHAIN_APPROX_SIMPLE );
//use moments method to find our filtered object
double refArea = 0;
if (hierarchy.size() > 0) {
int numObjects = hierarchy.size();
for (int index = 0; index >= 0; index = hierarchy[index][0]) {
Moments moment = moments((cv::Mat)contours[index]);
double area = moment.m00;
if(area>MIN_OBJECT_AREA){ //jika area kontur lebih besar dari minimum area object maka gambar lingkaran dan tulis koordinat
kirim=true;
double x = moment.m10/area;
double y = moment.m01/area;
double r = sqrt(area/3.14);
Xg=(int) x;
Yg=(int) y;
circle(imgOriginal, Point(x,y), r, Scalar(0,0,255), 1.5, 8);
line(imgOriginal, Point(x,y-r-5), Point(x,y+r+5), Scalar(0,0,255), 1.5, 8);
line(imgOriginal, Point(x-r-5,y), Point(x+r+5,y), Scalar(0,0,255), 1.5, 8);
putText(imgOriginal, intToString(x) + "," + intToString(y), Point(x,y+10), FONT_HERSHEY_COMPLEX, 0.25, Scalar(0, 255, 0), 0.3, 8);
// send x,y coordinate to arduino
parser1="*"; parser2="!";
ostringstream xxx,yyy ;
xxx << Xg;
yyy << Yg;
XX=xxx.str(); YY=yyy.str();
result=parser1+XX+parser2+YY;
cout << result << endl;
fprintf(serial, "%s\n", result.c_str());
fflush(serial);
}//end if
}//end for
}//end if
//show the thresholded image
Mat dstimgThresholded;
resize(imgThresholded, dstimgThresholded, Size(), 2, 2, INTER_CUBIC);
imshow("Thresholded Image", dstimgThresholded);
//show the original image
Mat dstimgOriginal;
resize(imgOriginal, dstimgOriginal, Size(), 2, 2, INTER_CUBIC);
imshow("Original", dstimgOriginal);
///send data
if (waitKey(5) == 27) {//wait for 'esc' key press for 30ms. If 'esc' key is pressed, break loop
cout << "esc key is pressed by user" << endl;
break;
}
} //end while
return 0;
}
tx arduino
#include <SoftwareSerial.h>
SoftwareSerial SWsend(2, 3); // (rx,tx)
String data;
void setup() {
// put your setup code here, to run once:
Serial.begin(115200);
SWsend.begin(4800);
//pinMode(12,INPUT_PULLUP);
}
void Send_SSIDandPWD_ToESP01() {
while (!SWsend) {
; // wait for serial port to connect.
Serial.println(F("wait for serial port to connect."));
}
while (Serial.available()) {
data = Serial.readString();
//data = Serial.read();
Serial.print(F("Send:"));
Serial.println(data);
SWsend.print(data);
}
}
void loop() {
// put your main code here, to run repeatedly:
Send_SSIDandPWD_ToESP01();
delay(25);
}
rx arduino
#include <SoftwareSerial.h>
SoftwareSerial SWrecv(2, 3); //(rx,tx)
String strSSID = ""; // a string to hold incoming data
String strPWD = "";
bool keepSSID = false;
bool keepPWD = false;
boolean stringComplete = false; // whether the string is complete
void setup() {
// initialize serial:
Serial.begin(115200);
SWrecv.begin(4800);
// Turn on the blacklight and print a message.
Serial.println(F("Hello, world!"));
}
void loop() {
// print the string when a newline arrives:
SWrecvEvent();
if (stringComplete) {
Serial.println("X:" + strSSID + ", Y:" + strPWD);
// clear the string:
// inputString = "";
if (strSSID == "")
{
Serial.println("SSID:not config");
}
strSSID = ""; // a string to hold incoming data
strPWD = "";
stringComplete = false;
}
}
void SWrecvEvent() {
while (SWrecv.available()) {
// get the new byte:
char inChar = (char)SWrecv.read();
//Serial.print(inChar); ///////////////////string asli
// add it to the inputString:
switch (inChar ) {
case '*':
{
keepSSID = true;
keepPWD = false;
}
break;
case '!':
{
keepSSID = false;
keepPWD = true;
}
break;
default:
{
if (inChar == '\n') {
stringComplete = true;
keepSSID = false;
keepPWD = false;
return;
}
if (keepSSID == true )
{
strSSID += inChar;
}
else if ( keepPWD == true )
{
strPWD += inChar;
}
}
break;
}
}
}
what should I do to write coordinate countinously while the tracker is running?
It could be that the IO is being buffered, and only flushed when you exit your tx program. Try calling Flush to force the data to send.
I am trying to code a program on opencv to decide whether a human has approached ahead the camera. After I run the execution file, I get the captured video for few seconds and encounter the segmentation fault error.
The code is like this
Here are headers:
#include "opencv2/objdetect.hpp"
#include "opencv2/videoio.hpp"
#include "opencv2/highgui.hpp"
#include "opencv2/imgproc.hpp"
#include <iostream>
#include <stdio.h>
using namespace std;
using namespace cv;
//define static variable
static int cApp = 0;//number of approached frame
static double last = 0;
//define functions
void detectAndDisplay( Mat frame );
bool computeArea( double width, double height, double lastArea);
double runningAverage(int M);
//define opencv function and classifier
String upperbody_cascade_name = "home/pi/opencv- 3.0.0/data/haarcascades/haarcascade_upperbody.xml";
CascadeClassifier upper_cascade;
String window_name = "Capture - upper body detection";
Here is the main function:
int main( void )
{
//define variable
VideoCapture capture;
Mat frame;
//-- 1. Load the cascades
upper_cascade.load("/home/pi/opencv-3.0.0/data/haarcascades/haarcascade_upperbody.xml");
//-- 2. Read the video stream
capture.open( -1 );
if ( ! capture.isOpened() ) { printf("--(!)Error opening video capture\n"); return -1; }
while ( capture.read(frame) )
{
if( frame.empty() )
{
printf(" --(!) No captured frame -- Break!");
break;
}
//-- 3. Apply the classifier to the frame
detectAndDisplay( frame );
char c = (char)waitKey(10);
if( c == 27 ) { break; } // escape
}
capture.release();
return 0;
}
Here is the detectAndDisplay function:
void detectAndDisplay( Mat frame )
{
std::vector<Rect> upperbodys;
Mat frame_gray;
cvtColor( frame, frame_gray, COLOR_BGR2GRAY );
equalizeHist( frame_gray, frame_gray );
//-- Detect upperbodys
upper_cascade.detectMultiScale( frame_gray, upperbodys, 1.05, 3, 0|CASCADE_SCALE_IMAGE, Size(30, 30) );
Point center( upperbodys[0].x + upperbodys[0].width/2, upperbodys[0].y + upperbodys[0].height/2 );
ellipse( frame, center, Size( upperbodys[0].width/2, upperbodys[0].height/2 ), 0, 0, 360, Scalar( 255, 0, 255 ), 4, 8, 0 );
bool ifApproached = computeArea(upperbodys[0].width/2, upperbodys[0].height/2, last);
if (ifApproached == true) {
cApp++;
}
if (cApp == 3) {
cout << "have approached" << endl;
cApp = cApp - 3;
}
//-- Show what you got
imshow( window_name, frame );
}
Here is the computeArea function:
bool computeArea( double width, double height, double lastArea) {
double newArea = width * height;
bool ifApproached = false;
//double presentArea = newArea;
double presentArea = runningAverage(newArea);
double DifferenceBewteenAreas = presentArea - lastArea;
if (DifferenceBewteenAreas > 1) {//threshold
ifApproached = true;
}
last = presentArea;
return ifApproached;
}
Here is runningAverage function:
double runningAverage(int M) {
//M is measurement
//#define LM_SIZE 5
static int LM[5];
static int index =0;
static long sum = 0;
static int count =0;
//keep sum updated to improve speed
sum = sum - LM[index];
LM[index] = M;
sum = sum + LM[index];
index++;
index = index % 5;
if (count < 5) {
count++;
}
return (double)(sum / (double)count);
}
I have searched many opencv segmentation fault questions, some said this segmentation fault was caused by wrong array used, but my case has little use of array. Others said misused of function characters could also cause this kind of errors, I agree with this, some of my characters could be wrong here.
Actually I found that I should not use upperbodys[0] in the code, because sometimes there are no object being detected at all,so there could be some memory read error happens, I used upperbodys[i] instead and it works well then.
void detectAndDisplay( Mat frame )
{
std::vector<Rect> upperbodys;
Mat frame_gray;
cvtColor( frame, frame_gray, COLOR_BGR2GRAY );
equalizeHist( frame_gray, frame_gray );
//-- Detect upperbodys
upper_cascade.detectMultiScale( frame_gray, upperbodys, 1.05, 3, 0|CASCADE_SCALE_IMAGE, Size(30, 30) );
int size = upperbodys.size();
double newArea = -1;
for (int i = 0 ; i < size; i++) {
Point center( upperbodys[i].x + upperbodys[i].width/2, upperbodys[i].y + upperbodys[i].height/2 );
ellipse( frame, center, Size( upperbodys[i].width/2, upperbodys[i].height/2 ), 0, 0, 360, Scalar( 255, 0, 255 ), 4, 8, 0 );
//bool ifApproached = computeArea(upperbodys[i].width/2, upperbodys[i].height/2, last);
//////////////////////////////////////////
newArea = upperbodys[i].width/2 * upperbodys[i].height/2;
if (newArea != -1) {
cout << "UpperBodys has value, index = " << i << endl;
break;
}
}
bool ifApproached = false;
//double presentArea = runningAverage(newArea);
double presentArea = newArea;
double DifferenceBewteenAreas = presentArea - last;
if (DifferenceBewteenAreas > 1) {//threshold
ifApproached = true;
}
last = presentArea;
//////////////////////////////////////////
if (ifApproached == true) {
cApp++;
}
if (cApp == 3) {
cout << "have approached" << endl;
cApp = cApp - 3;
}
//-- Show what you got
imshow( window_name, frame );
}
Recently I have been trying to learn OpenCV and was trying to detect faces using the Haar Classifier.
I was successful in detecting faces when I obtained the video stream from the default webcam,but when I use Kinect instead of the default webcam, It still detects face but the frame rate drops tremendously.
The code that I've Written is
int main() {
string haar_face = "F:\haarcascade_frontalface_default.xml";
CascadeClassifier haar_cascade;
haar_cascade.load(haar_face);
if (haar_cascade.empty()) {
return -1;
}
vector<Rect_<int>> faces;
bool optionKinect = false;
cout << "Choose Option\n1.) Kinect \n>1.) WebCam\n";
int choice;
cin >> choice;
if (choice == 1) {
optionKinect = true;
}
if (optionKinect) {
CKinectStreamsMat* kinectStream = new CKinectStreamsMat();
kinectStream->initSensor();
while (true) {
Mat original, gray;
Mat face;
Rect face_i;
//cap >> original;
original = kinectStream->getColorFrame();
if (original.data) {
cvtColor(original, gray, CV_BGR2GRAY);
haar_cascade.detectMultiScale(gray, faces);
int size = faces.size();
for (size_t i = 0; i < size; i++) {
face_i = faces[i];
face = gray(face_i);
rectangle(original, face_i, CV_RGB(0, 255, 0), 1);
}
imshow("original", original);
if(waitKey(20) == 27){
break;
}
}
}
}
else {
VideoCapture cap(0);
while (true) {
Mat original, gray;
Mat face;
Rect face_i;
cap >> original;
//original = kinectStream->getColorFrame();
if (original.data) {
cvtColor(original, gray, CV_BGR2GRAY);
haar_cascade.detectMultiScale(gray, faces);
int size = faces.size();
for (size_t i = 0; i < size; i++) {
face_i = faces[i];
face = gray(face_i);
rectangle(original, face_i, CV_RGB(0, 255, 0), 1);
}
imshow("original", original);
if(waitKey(20) == 27){
break;
}
}
}
}
}
And this is how I am obtaining the Color frame from Kinect.
cv::Mat CKinectStreamsMat::getColorFrame()
{
HRESULT hr = E_FAIL;
IColorFrame* frame = NULL;
IFrameDescription* frameDesc;
cv::Mat colorImage;
hr = _color_reader->AcquireLatestFrame(&frame);
if (SUCCEEDED(hr)) {
hr = frame->get_FrameDescription(&frameDesc);
if (SUCCEEDED(hr)) {
int frameWidth = 0, frameHeight = 0;
hr = frameDesc->get_Width(&frameWidth);
if (SUCCEEDED(hr)) {
hr = frameDesc->get_Height(&frameHeight);
}
if (SUCCEEDED(hr)) {
const int imgSize = frameWidth*frameHeight * 4 * sizeof(unsigned char); //4 Channels(BGRA)
colorImage = cv::Mat(1080,1920,CV_8UC4);
hr = frame->CopyConvertedFrameDataToArray(imgSize, reinterpret_cast<BYTE*>(colorImage.data), ColorImageFormat_Bgra);
}
}
SafeRelease(frameDesc);
SafeRelease(frame);
}
return colorImage;
}
I thought the reason for low performance might be the difference in resolution of the frames provided by the WebCam and Kinect, So I also tried scaling down the frame provided by Kinect down to the size that was even less than the frame size of WebCam. But still the performance was very low.
As this is only what I could think of and now I am out of ideas, So could anyone please tell what could be the reason for this low performance?
How can I add roi-based selection in lkdemo.pp( klt optical flow tracker opencv example) source code?
I want select roi in the first frame and track feature point that selected in roi.
#include "opencv2/video/tracking.hpp"
#include "opencv2/imgproc/imgproc.hpp"
#include "opencv2/highgui/highgui.hpp"
#include <iostream>
#include <ctype.h>
using namespace cv;
using namespace std;
static void help()
{
// print a welcome message, and the OpenCV version
cout << "\nThis is a demo of Lukas-Kanade optical flow lkdemo(),\n"
"Using OpenCV version " << CV_VERSION << endl;
}
Point2f point;
bool addRemovePt = false;
static void onMouse( int event, int x, int y, int , void* )
{
if( event == CV_EVENT_LBUTTONDOWN )
{
point = Point2f((float)x, (float)y);
addRemovePt = true;
}
}
int main( int argc, char** argv )
{
help();
VideoCapture cap(CV_CAP_ANY);
TermCriteria termcrit(CV_TERMCRIT_ITER|CV_TERMCRIT_EPS, 20, 0.03);
Size subPixWinSize(10,10), winSize(61,61);
const int MAX_COUNT = 500;
bool needToInit = false;
bool nightMode = false;
//if( argc == 1 || (argc == 2 && strlen(argv[1]) == 1 && isdigit(argv[1][0])))
//cap.open(argc == 2 ? argv[1][0] - '0' : 0);
//else if( argc == 2 )
//cap.open(argv[1]);
if( !cap.isOpened() )
{
cout << "Could not initialize capturing...\n";
return 0;
}
namedWindow( "LK Demo", 1 );
setMouseCallback( "LK Demo", onMouse, 0 );
Mat gray, prevGray, image;
vector<Point2f> points[2];
for(;;)
{
Mat frame;
cap >> frame;
if( frame.empty() )
break;
frame.copyTo(image);
cvtColor(image, gray, COLOR_RGB2GRAY);
if( nightMode )
image = Scalar::all(0);
if( needToInit )
{
// automatic initialization
goodFeaturesToTrack(gray, points[1], MAX_COUNT, 0.01, 10, Mat(), 3, 0, 0.04);
cornerSubPix(gray, points[1], subPixWinSize, Size(-1,-1), termcrit);
addRemovePt = false;
}
else if( !points[0].empty() )
{
vector<uchar> status;
vector<float> err;
if(prevGray.empty())
gray.copyTo(prevGray);
calcOpticalFlowPyrLK(prevGray, gray, points[0], points[1], status, err, winSize,10, termcrit, 0, 0.001);
size_t i, k;
for( i = k = 0; i < points[1].size(); i++ )
{
if( addRemovePt )
{
if( norm(point - points[1][i]) <= 5 )
{
addRemovePt = false;
continue;
}
}
if( !status[i] )
continue;
points[1][k++] = points[1][i];
circle( image, points[1][i], 3, Scalar(0,255,0), -1, 8);
}
points[1].resize(k);
}
if( addRemovePt && points[1].size() < (size_t)MAX_COUNT )
{
vector<Point2f> tmp;
tmp.push_back(point);
cornerSubPix( gray, tmp, winSize, cvSize(-1,-1), termcrit);
points[1].push_back(tmp[0]);
addRemovePt = false;
}
needToInit = false;
imshow("LK Demo", image);
char c = (char)waitKey(10);
if( c == 27 )
break;
switch( c )
{
case 'r':
needToInit = true;
break;
case 'c':
points[0].clear();
points[1].clear();
break;
case 'n':
nightMode = !nightMode;
break;
}
std::swap(points[1], points[0]);
cv::swap(prevGray, gray);
}
return 0;
}
Here is what I use in those cases :
void SelectNewTemplate(int event, int posx, int posy, int flags, void* userdata)
{
if( event == EVENT_MBUTTONDOWN )
{
waitKey();
}
if( event == CV_EVENT_LBUTTONDOWN )
{
x1pt = posx;
y1pt = posy;
}
if( event == CV_EVENT_LBUTTONUP )
{
x2pt = posx;
y2pt = posy;
Rect newTemp(x1pt, y1pt, (x2pt - x1pt), (y2pt - y1pt));
Mat imgROI = frame(newTemp);
}
}
Usage : pausing the video with middle mouse button, then left clic, drag and let go, press any key to keep going.
After that you can compute your features on your new ROI image : imgROI.
Hope that helps,
Thomas