face detection in Videos with OpenCV error close the program - c++

i work on face detection project in c++ with opencv
When the program determines the face
Come out of program
why ?
and the video shows the problem :
http://youtu.be/xYS7udYKiSI
image :
http://i.stack.imgur.com/VdzDX.png
Code :
#include <opencv2/objdetect/objdetect.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include <iostream>
#include <stdio.h>
using namespace std;
using namespace cv;
int main(int argc, const char *argv[]){
if (argc != 4) {
cout << "usage: " << argv[0] << " </path/to/haar_cascade> </path/to/csv.ext> </path/to/device id>" << endl;
cout << "\t </path/to/haar_cascade> -- Path to the Haar Cascade for face detection." << endl;
cout << "\t </path/to/csv.ext> -- Path to the CSV file with the face database." << endl;
cout << "\t <device id> -- The webcam device id to grab frames from." << endl;
// exit(1);
}
CascadeClassifier face_cascade;
String fn="C:\\opencv\\sources\\data\\haarcascades\\haarcascade_frontalface_alt2.xml";
face_cascade.load(fn);
VideoCapture input(0);
if(!input.isOpened()){return -1;}
namedWindow("Mezo",1);
Mat f2;
Mat frame;
while(true){
input>>frame;
waitKey(10);
cvtColor(frame, f2, CV_BGR2GRAY);
equalizeHist(f2, f2);
std::vector<Rect> faces;
face_cascade.detectMultiScale(f2, faces, 1.1, 10, CV_HAAR_SCALE_IMAGE | CV_HAAR_DO_CANNY_PRUNING,Size(150,0),Size(300,300));
//draw a rectangle for all found faces in the vector array on the original image
for(int i = 0; i < faces.size(); i++)
{
Point pt1(faces[i].x + faces[i].width, faces[i].y + faces[i].height);
Point pt2(faces[i].x, faces[i].y);
rectangle(frame, pt1, pt2, cvScalar(0, 255, 0, 0), 2, 8, 0);
}
imshow("Mezo",frame);
waitKey(3);
char c=waitKey(3);
if(c==27){break;}
}
return 0;
}
Any idea ?
Thank all

Related

Debug assertion failed - C++ (OpenCV) - pops up after the program terminates

I am using OpenCV version 2.4.9 and Visual Studio 2015. I am sure all the dependencies between them are working, since other sample programs worked perfectly, using OpenCV libraries.
You can find here my code:
#include <opencv2/opencv.hpp>
#include <stdio.h>
#include "opencv2/imgproc/imgproc.hpp"
#include "opencv2/core/core.hpp"
#include "opencv2/contrib/contrib.hpp"
#include "opencv2/highgui/highgui.hpp"
#include <iostream>
#include <fstream>
#include <sstream>
using namespace cv;
using namespace std;
String face_cascade_name = "C:\\opencv\\sources\\data\\haarcascades\\haarcascade_frontalface_alt.xml";
String eye_cascade_name = "C:\\opencv\\sources\\data\\haarcascades\\haarcascade_eye.xml";
Mat faceDetect(Mat img);
CascadeClassifier face_cascade;
CascadeClassifier eyes_cascade;
using namespace cv;
using namespace std;
enum EmotionState_t {
SERIOUS = 0, // 0
SMILE, // 1
SURPRISED, // 2
};
static void read_csv(const string& filename, vector<Mat>& images,
vector<int>& labels, char separator = ';') {
std::ifstream file(filename.c_str(), ifstream::in);
if (!file) {
string error_message = "No valid input file was given, please check the given filename.";
CV_Error(CV_StsBadArg, error_message);
}
string line, path, classlabel;
while (getline(file, line)) {
stringstream liness(line);
getline(liness, path, separator);
getline(liness, classlabel);
if (!path.empty() && !classlabel.empty()) {
images.push_back(imread(path, 0));
labels.push_back(atoi(classlabel.c_str()));
}
}
}
int main(int argc, const char *argv[])
{
EmotionState_t emotion;
if (!face_cascade.load(face_cascade_name)) {
printf("--(!)Error loading\n"); return -1; };
if (!eyes_cascade.load(eye_cascade_name)) {
printf("--(!)Error loading\n"); return -1; };
// 0 is the ID of the built-in laptop camera, change if you want to useother camera
VideoCapture cap(0);
//check if the file was opened properly
if (!cap.isOpened())
{
std::cout << "Capture could not be opened succesfully" << endl;
return -1;
}
else
{
std::cout << "camera is ok.. Stay 2 ft away from your camera\n" << endl;
}
int w = 432;
int h = 240;
cap.set(CV_CAP_PROP_FRAME_WIDTH, w);
cap.set(CV_CAP_PROP_FRAME_HEIGHT, h);
Mat frame;
cap >> frame;
std::cout << "processing the image...." << endl;
Mat testSample = faceDetect(frame);
// Get the path to your CSV.
string fn_csv = "C:\\Users\\Omar\\Downloads\\test_canny\\my_csv.txt";
// These vectors hold the images and corresponding labels.
vector<Mat>* images;
images = new vector<Mat>;
vector<int>* labels;
labels = new vector<int>;
// Read in the data. This can fail if no valid
// input filename is given.
try
{
read_csv(fn_csv, *images, *labels);
}
catch (cv::Exception& e) {
cerr << "Error opening file \"" << fn_csv << "\". Reason: "
<< e.msg << endl;
// nothing more we can do
exit(1);
}
// Quit if there are not enough images for this demo.
if ((*images).size() <= 1)
{
string error_message = "This demo needs at least 2 images to work.Please add more images to your data set!";
CV_Error(CV_StsError, error_message);
}
// Get the height from the first image. We'll need this
// later in code to reshape the images to their original
// size:
int height = (*images)[0].rows;
// The following lines create an Fisherfaces model for
// face recognition and train it with the images and
// labels read from the given CSV file.
// If you just want to keep 10 Fisherfaces, then call
// the factory method like this:
//
// cv::createFisherFaceRecognizer(10);
//
// However it is not useful to discard Fisherfaces! Please
// always try to use _all_ available Fisherfaces for
// classification.
//
// If you want to create a FaceRecognizer with a
// confidence threshold (e.g. 123.0) and use _all_
// Fisherfaces, then call it with:
//
// cv::createFisherFaceRecognizer(0, 123.0);
//
Ptr<FaceRecognizer> model = createFisherFaceRecognizer();
model->train(*images, *labels);
// The following line predicts the label of a given
// test image:
int predictedLabel = model->predict(testSample);
// To get the confidence of a prediction call the model with:
//
// int predictedLabel = -1;
// double confidence = 0.0;
// model->predict(testSample, predictedLabel, confidence);
//
string result_message = format("Predicted class = %d", predictedLabel);
std::cout << result_message << endl;
// giving the result
switch (predictedLabel)
{
case SMILE:
std::cout << "You are happy!" << endl;
break;
case SURPRISED:
std::cout << "You are surprised!" << endl;
break;
case SERIOUS:
std::cout << "You are serious!" << endl;
break;
}
return 0;
}
Mat faceDetect(Mat img)
{
std::vector<Rect>* faces;
faces = new vector<Rect>;
std::vector<Rect>* eyes;
eyes = new vector<Rect>;
bool two_eyes = false;
bool any_eye_detected = false;
//detecting faces
face_cascade.detectMultiScale(img, *faces, 1.1, 2, 0 | CV_HAAR_SCALE_IMAGE,
Size(30, 30));
if ((*faces).size() == 0)
{
std::cout << "Try again.. I did not dectected any faces..." << endl;
exit(-1); // abort everything
}
Point p1 = Point(0, 0);
for (size_t i = 0; i < (*faces).size(); i++)
{
// we cannot draw in the image !!! otherwise will mess with the prediction
// rectangle( img, faces[i], Scalar( 255, 100, 0 ), 4, 8, 0 );
Mat frame_gray;
cvtColor(img, frame_gray, CV_BGR2GRAY);
// croping only the face in region defined by faces[i]
std::vector<Rect>* eyes;
eyes = new vector<Rect>;
Mat faceROI = frame_gray((*faces)[i]);
//In each face, detect eyes
eyes_cascade.detectMultiScale(faceROI, *eyes, 1.1, 3, 0
| CV_HAAR_SCALE_IMAGE, Size(30, 30));
for (size_t j = 0; j < (*eyes).size(); j++)
{
Point center((*faces)[i].x + (*eyes)[j].x + (*eyes)[j].width*0.5,
(*faces)[i].y + (*eyes)[j].y + (*eyes)[j].height*0.5);
// we cannot draw in the image !!! otherwise will mess with the prediction
// int radius = cvRound( (eyes[j].width + eyes[j].height)*0.25 );
// circle( img, center, radius, Scalar( 255, 0, 0 ), 4, 8, 0 );
if (j == 0)
{
p1 = center;
any_eye_detected = true;
}
else
{
two_eyes = true;
}
}
}
std::cout << "SOME DEBUG" << endl;
std::cout << "-------------------------" << endl;
std::cout << "faces detected:" << (*faces).size() << endl;
std::cout << "x: " << (*faces)[0].x << endl;
std::cout << "y: " << (*faces)[0].y << endl;
std::cout << "w: " << (*faces)[0].width << endl;
std::cout << "h: " << (*faces)[0].height << endl << endl;
Mat imageInRectangle;
imageInRectangle = img((*faces)[0]);
Size recFaceSize = imageInRectangle.size();
std::cout << recFaceSize << endl;
// for debug
imwrite("C:\\Users\\Omar\\Downloads\\test_canny\\imageInRectangle.jpg", imageInRectangle);
int rec_w = 0;
int rec_h = (*faces)[0].height * 0.64;
// checking the (x,y) for cropped rectangle
// based in human anatomy
int px = 0;
int py = 2 * 0.125 * (*faces)[0].height;
Mat cropImage;
std::cout << "faces[0].x:" << (*faces)[0].x << endl;
p1.x = p1.x - (*faces)[0].x;
std::cout << "p1.x:" << p1.x << endl;
if (any_eye_detected)
{
if (two_eyes)
{
std::cout << "two eyes detected" << endl;
// we have detected two eyes
// we have p1 and p2
// left eye
px = p1.x / 1.35;
}
else
{
// only one eye was found.. need to check if the
// left or right eye
// we have only p1
if (p1.x > recFaceSize.width / 2)
{
// right eye
std::cout << "only right eye detected" << endl;
px = p1.x / 1.75;
}
else
{
// left eye
std::cout << "only left eye detected" << endl;
px = p1.x / 1.35;
}
}
}
else
{
// no eyes detected but we have a face
px = 25;
py = 25;
rec_w = recFaceSize.width - 50;
rec_h = recFaceSize.height - 30;
}
rec_w = ((*faces)[0].width - px) * 0.75;
std::cout << "px :" << px << endl;
std::cout << "py :" << py << endl;
std::cout << "rec_w:" << rec_w << endl;
std::cout << "rec_h:" << rec_h << endl;
cropImage = imageInRectangle(Rect(px, py, rec_w, rec_h));
Size dstImgSize(70, 70); // same image size of db
Mat finalSizeImg;
resize(cropImage, finalSizeImg, dstImgSize);
// for debug
imwrite("C:\\Users\\Omar\\Downloads\\test_canny\\onlyface.jpg", finalSizeImg);
cvtColor(finalSizeImg, finalSizeImg, CV_BGR2GRAY);
return finalSizeImg;
}
I've debugged it and the error only pops up when I reach return 0 in the main method.
And here's an image of the error (too large to embed)
Any help will be appreciated.

how to Extract Video Frames and Save them as Images using c++

I am now trying to save the frames of the video file into images on my pc. I am using Visual studio 2010 and opencv 2.3.1. In this code(shown bellow) it can save frames of image sequence but for video file I can not save the frames.
the problem specifically seems to be here:
in the function of vidoeprocessing()
string imageToSave =("output_MOG_" + frameNumberString + ".png");
bool saved = imwrite( imageToSave,fgMaskMOG);
if(!saved) {
cerr << "Unable to save " << imageToSave << endl;
}
any one can help to solve this?
Thanks in advance.
my code is this:
//opencv
#include < opencv2/opencv.hpp>
#include < opencv2/core/core.hpp>
#include < opencv2/highgui/highgui.hpp>
#include < opencv2/video/background_segm.hpp>
//#include <opencv2/imgcodecs.hpp>
#include <opencv2/imgproc/imgproc.hpp>
//#include "opencv2/videoio.hpp"
#include <opencv2/video/video.hpp>
//C
#include <stdio.h>
//C++
#include <iostream>
#include <sstream>
using namespace cv;
using namespace std;
// Global variables
Mat frame; //current frame
Mat fgMaskMOG; //fg mask fg mask generated by MOG method
Ptr<BackgroundSubtractor> pMOG; //MOG Background subtractor
int keyboard; //input from keyboard
/** Function Headers */
void help();
void processVideo(char* videoFilename);
void processImages(char* firstFrameFilename);
void help()
{
cout
<< "--------------------------------------------------------------------- -----" << endl
<< "This program shows how to use background subtraction methods provided by " << endl
<< " OpenCV. You can process both videos (-vid) and images (-img)." << endl
<< endl
<< "Usage:" << endl
<< "./bs {-vid <video filename>|-img <image filename>}" << endl
<< "for example: ./bs -vid video.avi" << endl
<< "or: ./bs -img /data/images/1.png" << endl
<< "--------------------------------------------------------------------------" << endl
<< endl;
}
/**
* #function main
*/
int main(int argc, char* argv[])
{
//print help information
help();
//check for the input parameter correctness
if(argc != 3) {
cerr <<"Incorret input list" << endl;
cerr <<"exiting..." << endl;
return EXIT_FAILURE;
}
//create GUI windows
namedWindow("Frame");
namedWindow("FG Mask MOG ");
//create Background Subtractor objects
pMOG = new BackgroundSubtractorMOG();
if(strcmp(argv[1], "-vid") == 0) {
//input data coming from a video
processVideo(argv[2]);
}
else if(strcmp(argv[1], "-img") == 0) {
//input data coming from a sequence of images
processImages(argv[2]);
}
else {
//error in reading input parameters
cerr <<"Please, check the input parameters." << endl;
cerr <<"Exiting..." << endl;
return EXIT_FAILURE;
}
//destroy GUI windows
destroyAllWindows();
return EXIT_SUCCESS;
}
//function processVideo
void processVideo(char* videoFilename) {
//create the capture object
VideoCapture capture(videoFilename);
if(!capture.isOpened()){
//error in opening the video input
cerr << "Unable to open video file: " << videoFilename << endl;
exit(EXIT_FAILURE);
}
//read input data. ESC or 'q' for quitting
while( (char)keyboard != 'q' && (char)keyboard != 27 ){
//read the current frame
if(!capture.read(frame)) {
cerr << "Unable to read next frame." << endl;
cerr << "Exiting..." << endl;
exit(EXIT_FAILURE);
}
//update the background model
pMOG->operator()(frame, fgMaskMOG,0.9);
//get the frame number and write it on the current frame
stringstream ss;
rectangle(frame, cv::Point(10, 2), cv::Point(100,20),
cv::Scalar(255,255,255), -1);
ss << capture.get(CV_CAP_PROP_POS_FRAMES);
string frameNumberString = ss.str();
putText(frame, frameNumberString.c_str(), cv::Point(15, 15),
FONT_HERSHEY_SIMPLEX, 0.5 , cv::Scalar(0,0,0));
//show the current frame and the fg masks
imshow("Frame", frame);
imshow("FG Mask MOG ", fgMaskMOG);
//get the input from the keyboard
keyboard = waitKey( 30 );
}
/*
string imageToSave =("output_MOG_" + frameNumberString + ".bmp");
bool saved = imwrite( imageToSave,fgMaskMOG);
if(!saved) {
cerr << "Unable to save " << imageToSave << endl;
}
*/
//delete capture object
capture.release();
}
/**
* #function processImages
*/
void processImages(char* fistFrameFilename) {
//read the first file of the sequence
frame = imread(fistFrameFilename);
if(frame.empty()){
//error in opening the first image
cerr << "Unable to open first image frame: " << fistFrameFilename << endl;
exit(EXIT_FAILURE);
}
//current image filename
string fn(fistFrameFilename);
//read input data. ESC or 'q' for quitting
while( (char)keyboard != 'q' && (char)keyboard != 27 ){
//update the background model
pMOG->operator()(frame, fgMaskMOG,0.9);
//get the frame number and write it on the current frame
size_t index = fn.find_last_of("/");
if(index == string::npos) {
index = fn.find_last_of("\\");
}
size_t index2 = fn.find_last_of(".");
string prefix = fn.substr(0,index+1);
string suffix = fn.substr(index2);
string frameNumberString = fn.substr(index+1, index2-index-1);
istringstream iss(frameNumberString);
int frameNumber = 0;
iss >> frameNumber;
rectangle(frame, cv::Point(10, 2), cv::Point(100,20),
cv::Scalar(255,255,255), -1);
putText(frame, frameNumberString.c_str(), cv::Point(15, 15),
FONT_HERSHEY_SIMPLEX, 0.5 , cv::Scalar(0,0,0));
//show the current frame and the fg masks
imshow("Frame", frame);
imshow("FG Mask MOG ", fgMaskMOG);
//get the input from the keyboard
keyboard = waitKey( 30 );
//search for the next image in the sequence
ostringstream oss;
oss << (frameNumber + 1);
string nextFrameNumberString = oss.str();
string nextFrameFilename = prefix + nextFrameNumberString + suffix;
//read the next frame
frame = imread(nextFrameFilename);
if(frame.empty()){
//error in opening the next image in the sequence
cerr << "Unable to open image frame: " << nextFrameFilename << endl;
exit(EXIT_FAILURE);
}
//update the path of the current frame
fn.assign(nextFrameFilename);
// save subtracted images
string imageToSave =("output_MOG_" + frameNumberString + ".png");
bool saved = imwrite( imageToSave,fgMaskMOG);
if(!saved) {
cerr << "Unable to save " << imageToSave << endl;
}
}
}
This is a small sample using your webcam. You can easily adapt it to use a video:
#include "opencv2/opencv.hpp"
using namespace cv;
int main(int, char**)
{
VideoCapture cap(0); // open the default camera
if (!cap.isOpened()) // check if we succeeded
return -1;
Ptr<BackgroundSubtractor> pMOG = new BackgroundSubtractorMOG2();
Mat fg_mask;
Mat frame;
int count = -1;
for (;;)
{
// Get frame
cap >> frame; // get a new frame from camera
// Update counter
++count;
// Background subtraction
pMOG->operator()(frame, fg_mask);
imshow("frame", frame);
imshow("fg_mask", fg_mask);
// Save foreground mask
string name = "mask_" + std::to_string(count) + ".png";
imwrite("D:\\SO\\temp\\" + name, fg_mask);
if (waitKey(1) >= 0) break;
}
// the camera will be deinitialized automatically in VideoCapture destructor
return 0;
}

What's wrong with CascadeClassifier (opencv) that is called many time

Assume:
There are N files on disk that require face detection.
If I do it in "slow" mode all is well (faces are faces at their rects):
for(auto mat : getMyMats()) {
CascadeClassifier facer("haarcascade_frontalface_alt.xml");
vector<Rect> faces;
facer.detectMultiScale(mat, faces);
}
But if i try to cache and reuse CascadeClassifier - near by 50% image are detected very wrong - invalid place, size, count of faces:
CascadeClassifier facer("haarcascade_frontalface_alt.xml");
for(auto mat : getMyMats()) {
vector<Rect> faces;
facer.detectMultiScale(mat, faces);
}
What's wrong with CascadeClassifier reusing
Windows, OpenCV 3.0 both debug and release build.
SAMPLE CODE TO REPRODUCE:
// compiler - opencv/include referenced
// linker - opencv/lib referenced and
// opencv_core300.lib, opencv_imgproc300.lib, opencv_imgcodecs300.lib, opencv_objdetect300.lib (or world or debug libs) are referenced
// 1.png, 2.png, 3.png,
#include <vector>
#include <string>
#include <iostream>
#include "opencv2/opencv.hpp"
using namespace cv;
using namespace std;
//count faces in load-every-time call
int no_reuse_call() {
int cnt = 0;
for (auto n : { "1.png","2.png","3.png" }) {
vector<Rect> faces{};
CascadeClassifier facer("haarcascade_frontalface_alt.xml");
Mat m = imread(n, CV_LOAD_IMAGE_GRAYSCALE);
facer.detectMultiScale(m, faces, 1.1, 3, CASCADE_SCALE_IMAGE, Size(50, 50));
cnt += faces.size(); //vector given to detectMultiScale cleared in detectMultiScale
cout << faces[0] << endl;
}
for (auto n : { "1.png","2.png","3.png" }) {
vector<Rect> faces{};
CascadeClassifier facer("haarcascade_frontalface_alt.xml");
Mat m = imread(n, CV_LOAD_IMAGE_GRAYSCALE);
facer.detectMultiScale(m, faces, 1.1, 3, CASCADE_SCALE_IMAGE, Size(50, 50));
cnt += faces.size(); //vector given to detectMultiScale cleared in detectMultiScale
cout << faces[0] << endl;
}
return cnt;
}
//coutn faces in one-instance call
int reuse_call() {
int cnt = 0;
CascadeClassifier facer("haarcascade_frontalface_alt.xml");
for (auto n : { "1.png","2.png","3.png" }) {
vector<Rect> faces{};
Mat m = imread(n, CV_LOAD_IMAGE_GRAYSCALE);
facer.detectMultiScale(m, faces, 1.1, 3, CASCADE_SCALE_IMAGE, Size(50, 50));
cnt += faces.size(); //vector given to detectMultiScale cleared in detectMultiScale
cout << faces[0] << endl;
}
for (auto n : { "1.png","2.png","3.png" }) {
vector<Rect> faces{};
Mat m = imread(n, CV_LOAD_IMAGE_GRAYSCALE);
facer.detectMultiScale(m, faces, 1.1, 3, CASCADE_SCALE_IMAGE, Size(50, 50));
cnt += faces.size(); //vector given to detectMultiScale cleared in detectMultiScale
cout << faces[0] << endl;
}
return cnt;
}
int reuse_call_reverse() {
int cnt = 0;
CascadeClassifier facer("haarcascade_frontalface_alt.xml");
for (auto n : {"3.png" ,"2.png", "1.png"}) {
vector<Rect> faces{};
Mat m = imread(n, CV_LOAD_IMAGE_GRAYSCALE);
facer.detectMultiScale(m, faces, 1.1, 3, CASCADE_SCALE_IMAGE, Size(50, 50));
cnt += faces.size(); //vector given to detectMultiScale cleared in detectMultiScale
cout << faces[0] << endl;
}
for (auto n : { "3.png" ,"2.png", "1.png" }) {
vector<Rect> faces{};
Mat m = imread(n, CV_LOAD_IMAGE_GRAYSCALE);
facer.detectMultiScale(m, faces, 1.1, 3, CASCADE_SCALE_IMAGE, Size(50, 50));
cnt += faces.size(); //vector given to detectMultiScale cleared in detectMultiScale
cout << faces[0] << endl;
}
return cnt;
}
int main() {
cout << "Created every image 2*3 files" << endl;
cout << no_reuse_call() << endl;
cout << "Created one time and files in alphabet order 2*3 files" << endl;
cout << reuse_call() << endl;
cout << "Created one time and files in contra - alphabet order 2*3 files" << endl;
cout << reuse_call_reverse() << endl;
cout << "You can see:" << endl;
cout << " if cascade is reused - detection in 3.png file is stabily wrong if it is processed after 1.png and 2.png" << endl;
cout << " it's depend on order - if 3.png is first in row it detected right even in second call after 1.png and 2.png" << endl;
cout << " it's random and based somehow on image - 1.png and 2.png are always detected validly" << endl;
return 0;
}
Look at https://github.com/Itseez/opencv/issues/5475
due to some reasones ocl in opencv 3.0.0 is not (stable?)
so author recomend to call setUseOpenCL(false); in programm - with this option all work well

ERROR: Background Subtraction Coding Doesn't Functioning using openCV and VS2013

I had ran a coding for background subtraction from the video for offline. I used openCV 2.4.9 and Visual Studio 2013 to run the coding. The video can display and play but the problem is the coding for background subtraction does not functioning. Can anybody help me what is the wrong for my coding? Someone had tell me the error is
if (strcmp(argv[1], "-vid") == 0)
{
//input data coming from a video
processVideo(argv[2]);
}
so, what i need to do? Help me please...
// BackgroundSubtraction_Success.cpp : Defines the entry point for the console application.
//
#include <stdio.h>
#include "stdafx.h"
#include <iostream>
#include <opencv2\core\core.hpp>
#include <opencv2\flann\flann.hpp>
#include <opencv2\imgproc\imgproc.hpp>
#include <opencv2\photo\photo.hpp>
#include <opencv2\video\video.hpp>
#include <opencv2\features2d\features2d.hpp>
#include <opencv2\objdetect\objdetect.hpp>
#include <opencv2\calib3d\calib3d.hpp>
#include <opencv2\ml\ml.hpp>
#include <opencv2\highgui\highgui.hpp>
#include <opencv2\contrib\contrib.hpp>
//#include "opencv\cv.h"
//#include <opencv2\core\core_c.h>
//#include <opencv2\highgui\highgui_c.h>
//#include <opencv2\imgproc\imgproc_c.h>
using namespace cv;
using namespace std;
//global variables
Mat frame; //current frame
Mat fgMaskMOG; //fg mask generated by MOG method
Ptr <BackgroundSubtractorMOG> pMOG; //MOG Background Subtractor
int keyboard;
//function declarations
void help();
void processVideo(char*Background);
void help()
{
cout
<< "----------------------------------------" << endl
<< endl
<< "This program begins with Motion Detection" << endl
<< "using Background Subtraction" << endl
<< endl
<< "------------------------------------------" << endl;
}
int main(int argc, char*argv[])
{
VideoCapture cap("C:/Users/user/Documents/Visual Studio 2013/Projects/cobaan/NewOpenCV_Success/sample1.avi"); // open the video file for reading
if (!cap.isOpened()) // if not success, exit program
{
cout << "Cannot open the video file" << endl;
return -1;
}
//cap.set(CV_CAP_PROP_POS_MSEC, 300); //start the video at 300ms
double fps = cap.get(CV_CAP_PROP_FPS); //get the frames per seconds of the video
cout << "Frame per seconds : " << fps << endl;
namedWindow("MyVideo", CV_WINDOW_AUTOSIZE); //create a window called "MyVideo"
while (1)
{
Mat frame;
bool bSuccess = cap.read(frame); // read a new frame from video
if (!bSuccess) //if not success, break loop
{
cout << "Cannot read the frame from video file" << endl;
break;
}
imshow("MyVideo", frame); //show the frame in "MyVideo" window
if (waitKey(30) == 27) //wait for 'esc' key press for 30 ms. If 'esc' key is pressed, break loop
{
cout << "esc key is pressed by user" << endl;
break;
}
}
//print help information
help();
//check for the input parameter correctness
if (argc != 3)
{
cerr << "incorrect input list" << endl;
cerr << "exiting..." << endl;
return EXIT_FAILURE;
}
//create GUI windows
namedWindow("Frame");
namedWindow("FG Mask MOG");
//call the constructor
BackgroundSubtractorMOG bgmog;
bgmog(frame, fgMaskMOG);
//create background subtractor objects
//pMOG = createBackgroundSubtractorMOG(); //MOG approach
if (strcmp(argv[1], "-vid") == 0)
{
//input data coming from a video
processVideo(argv[2]);
}
else
{
//error in reading input parameter
cerr << "Please check the input parameters." << endl;
cerr << "Exiting..." << endl;
return EXIT_FAILURE;
}
//destroy GUI windows
destroyAllWindows();
return EXIT_SUCCESS;
}
//call video
void processVideo(char* MyVideo)
{
//create the capture object
VideoCapture capture(MyVideo);
if (!capture.isOpened())
{
//error in opening the video input
cerr << "Unable to open video file: " <<MyVideo << endl;
exit(EXIT_FAILURE);
}
//read input data ESC or 'q' for quitting
while ((char)keyboard != 'q' && (char)keyboard != 27)
{
//read the current frame
if (!capture.read(frame))
{
cerr << "Unable to read next frame" << endl;
cerr << "Exiting..." << endl;
exit(EXIT_FAILURE);
}
//update the background model
//pMOG->apply(frame, fgMaskMOG);
//get the frame number and write it on the current frame
stringstream ss;
rectangle(frame, cv::Point(10, 2), cv::Point(100, 20), cv::Scalar(255, 255, 255), -1);
ss << capture.get(CV_CAP_PROP_POS_FRAMES);
string frameNumberString = ss.str();
putText(frame, frameNumberString.c_str(), cv::Point(15, 15), FONT_HERSHEY_SIMPLEX, 0.5, cv::Scalar(0, 0, 0));
//show the current frame and fg masks
imshow("Frame", frame);
imshow("FG Mask MOG", fgMaskMOG);
//get input from the keyboard
keyboard = waitKey(30);
}
//delete capture object
capture.release();
}

OpenCV Camera Stereo Calibration - Strange results

I am trying to calibrate my Minoru stereo camera with OpenCV. The following code first calibrates the camera and then rectifies the results. When I run the code, the rectification results are somehow strange.
I've attached one of the results.
It would be very helpful, if somebody could look over my code.
Thanks Max
Code:
#include <opencv2/core/core.hpp>
#include <opencv2/calib3d/calib3d.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include <opencv2/contrib/contrib.hpp>
#include "libcam.h"
#include <stdio.h>
#include <iostream>
using namespace cv;
using namespace std;
void loadImagePair(Mat &img1, Mat &img2, int i) {
stringstream ss1, ss2;
ss1 << "data/saves/" << i << "_1.png";
ss2 << "data/saves/" << i << "_2.png";
/*if (i < 10) {
ss1 << "data/martinperris_images/left0" << i << ".ppm";
ss2 << "data/martinperris_images/right0" << i << ".ppm";
} else {
ss1 << "data/martinperris_images/left" << i << ".ppm";
ss2 << "data/martinperris_images/right" << i << ".ppm";
}*/
img1 = imread(ss1.str());
img2 = imread(ss2.str());
}
int main(int argc, char** argv) {
// The camera properties
int w = 640;
int h = 480;
int fps = 20;
// The chessboard properties
CvSize chessboardSize(9, 6);
float squareSize = 1.0f;
// This should contain the physical location of each corner, but since we don't know them, we are assigning constant positions
vector<vector<Point3f> > objPoints;
// The chessboard corner points in the images
vector<vector<Point2f> > imagePoints1, imagePoints2;
vector<Point2f> corners1, corners2;
// The constant positions of each obj points
vector<Point3f> obj;
for (int x = 0; x < chessboardSize.width; x++) {
for (int y = 0; y < chessboardSize.height; y++) {
obj.push_back(Point3f(x * squareSize, y * squareSize, 0));
}
}
/*for (int i = 0; i < chessboardSize.width * chessboardSize.height; i++) {
obj.push_back(Point3f(i / chessboardSize.width, i % chessboardSize.height, 0.0f));
}*/
// The images, which are proceeded
Mat img1, img2;
// The grayscale versions of the images
Mat gray1, gray2;
// Get the image count
int imageCount;
cout << "How much images to load: " << endl;
cin >> imageCount;
// The image number of the current image (nullbased)
int i = 0;
// Whether the chessboard corners in the images were found
bool found1 = false, found2 = false;
while (i < imageCount) {
// Load the images
cout << "Attempting to load image pair " << i << endl;
loadImagePair(img1, img2, i);
cout << "Loaded image pair" << endl;
// Convert to grayscale images
cvtColor(img1, gray1, CV_BGR2GRAY);
cvtColor(img2, gray2, CV_BGR2GRAY);
// Find chessboard corners
found1 = findChessboardCorners(img1, chessboardSize, corners1, CV_CALIB_CB_ADAPTIVE_THRESH | CV_CALIB_CB_FILTER_QUADS);
found2 = findChessboardCorners(img2, chessboardSize, corners2, CV_CALIB_CB_ADAPTIVE_THRESH | CV_CALIB_CB_FILTER_QUADS);
cout << "found 1/2: " << found1 << "/" << found2 << endl;
// Find corners to subpixel accuracy
if (found1) {
cornerSubPix(gray1, corners1, Size(11, 11), Size(-1, -1), TermCriteria(CV_TERMCRIT_EPS | CV_TERMCRIT_ITER, 30, 0.1));
drawChessboardCorners(gray1, chessboardSize, corners1, found1);
}
if (found2) {
cornerSubPix(gray2, corners2, Size(11, 11), Size(-1, -1), TermCriteria(CV_TERMCRIT_EPS | CV_TERMCRIT_ITER, 30, 0.1));
drawChessboardCorners(gray2, chessboardSize, corners2, found2);
}
// Store corners
if (found1 && found2) {
imagePoints1.push_back(corners1);
imagePoints2.push_back(corners2);
objPoints.push_back(obj);
cout << "Corners stored" << endl;
i++;
}
// Error
else {
cout << "Corners not found! Stopping" << endl;
return 0;
}
}
cout << "Starting calibration" << endl;
Mat CM1 = Mat(3, 3, CV_64F);
Mat CM2 = Mat(3, 3, CV_64F);
Mat D1 = Mat(1, 5, CV_64F);
Mat D2 = Mat(1, 5, CV_64F);
Mat R = Mat(3, 3, CV_64F);
Mat T = Mat(3, 1, CV_64F);
Mat E = Mat(3, 3, CV_64F);
Mat F = Mat(3, 3, CV_64F);
//stereoCalibrate(objPoints, imagePoints1, imagePoints2, CM1, D1, CM2, D2, img1.size(), R, T, E, F,
//CV_CALIB_SAME_FOCAL_LENGTH | CV_CALIB_ZERO_TANGENT_DIST, cvTermCriteria(CV_TERMCRIT_ITER + CV_TERMCRIT_EPS, 100, 1e-5));
stereoCalibrate(objPoints, imagePoints1, imagePoints2, CM1, D1, CM2, D2, img1.size(), R, T, E, F, 0,
cvTermCriteria(CV_TERMCRIT_ITER + CV_TERMCRIT_EPS, 100, 1e-5));
cout << "Done calibration" << endl;
cout << "Starting rectification" << endl;
Mat R1 = Mat(3, 3, CV_64F);
Mat R2 = Mat(3, 3, CV_64F);
Mat P1 = Mat(3, 4, CV_64F);
Mat P2 = Mat(3, 4, CV_64F);
Mat Q = Mat(4, 4, CV_64F);
stereoRectify(CM1, D1, CM2, D2, img1.size(), R, T, R1, R2, P1, P2, Q);
cout << "Done rectification" << endl;
cout << "Starting to store results" << endl;
FileStorage fs("stereocalib.yml", FileStorage::WRITE);
fs << "CM1" << CM1;
fs << "CM2" << CM2;
fs << "D1" << D1;
fs << "D2" << D2;
fs << "R" << R;
fs << "T" << T;
fs << "E" << E;
fs << "F" << F;
fs << "R1" << R1;
fs << "R2" << R2;
fs << "P1" << P1;
fs << "P2" << P2;
fs << "Q" << Q;
fs.release();
cout << "Done storing results" << endl;
cout << "Starting to apply undistort" << endl;
Mat map1x = Mat(img1.size().height, img1.size().width, CV_32F);
Mat map1y = Mat(img1.size().height, img1.size().width, CV_32F);
Mat map2x = Mat(img2.size().height, img2.size().width, CV_32F);
Mat map2y = Mat(img2.size().height, img2.size().width, CV_32F);
initUndistortRectifyMap(CM1, D1, R1, P1, img1.size(), CV_32FC1, map1x, map1y);
initUndistortRectifyMap(CM2, D2, R2, P2, img2.size(), CV_32FC1, map2x, map2y);
cout << "Done applying undistort" << endl;
// The rectified images
Mat imgU1 = Mat(img1.size(), img1.type()), imgU2 = Mat(img2.size(), img2.type());
// Show rectified images
i = 0;
while (i < imageCount) {
// Load the images
cout << "Attempting to load image pair " << i << endl;
loadImagePair(img1, img2, i);
cout << "Loaded image pair" << endl;
i++;
remap(img1, imgU1, map1x, map1y, INTER_LINEAR, BORDER_CONSTANT, Scalar());
remap(img2, imgU2, map2x, map2y, INTER_LINEAR, BORDER_CONSTANT, Scalar());
//remap(img1, imgU1, map1x, map1y, INTER_LINEAR, BORDER_DEFAULT);
//remap(img2, imgU2, map2x, map2y, INTER_LINEAR, BORDER_DEFAULT);
imshow("img1", img1);
imshow("img2", img2);
imshow("rec1", imgU1);
imshow("rec2", imgU2);
int key = waitKey(0);
if (key == 'q') {
break;
}
}
return 0;
}
Image:
Finally I found out, what the problem was. The images were good, but I defined the obj Vector the wrong way. It has to be:
// The constant positions of each obj points
vector<Point3f> obj;
for (int y = 0; y < chessboardSize.height; y++) {
for (int x = 0; x < chessboardSize.width; x++) {
obj.push_back(Point3f(y * squareSize, x * squareSize, 0));
}
}
Now I have a calibration result of 1.57324, which is quite good I think.