I am stitching three images. Now I wanted to implement warp perspective on the left side (image A and B) and right side (image B an C). I am able to compute both H matrices, but with some wierd behaving.
When I am computing A + B + C. It will do firstly B + C, then A + B. Here the H matric for AB is incorrect, but I dont know why!
When I will do only A+B then I will recieved "correct values". I tried everything, so I came here. It seems like there is some junk behind after computing B+C, but even I tried .clear() and .release() everything, still same behaving, I also tried to use .clone() everywhere - didnt help. I also checked if correct images are feeded in, yes they are.
I will try to simplify my code as much I can do:
// Stitcher variables
std::vector<std::pair<Mat, Mat>> images_toCalc;
std::vector<Mat> images;
std::vector<Mat> Hs;
//MAIN
images[2] = imread("E:/JOB/StitchingProject/StitchingApp/x64/Debug/i/left.jpg");
images[1] = imread("E:/JOB/StitchingProject/StitchingApp/x64/Debug/i/mid.jpg");
images[0] = imread("E:/JOB/StitchingProject/StitchingApp/x64/Debug/i/right.jpg");
stitcher->setImages(images, 3);
stitcher->runStitching2(false);
// runStitching2
for (int i = 0; i < this->images.size() - 1; i++) {
Mat image1 = this->images.at(i).clone();
Mat image2 = this->images.at(i + 1).clone();
//if (true) {
if (!isHalfAlready(i)) {
flip(image1, image1, 1);
flip(image2, image2, 1);
this->images_toCalc.push_back(std::make_pair(image2.clone(), image1.clone()));
}
else {
this->images_toCalc.push_back(std::make_pair(image1.clone(), image2.clone()));
}
}
for (int i = 0; i < images_toCalc.size(); i++) {
Mat H;
H = this->calcH(images_toCalc.at(i).first, images_toCalc.at(i).second);
this->Hs.push_back(H);
H.release();
if (this->debugLevel[3]) cout << "[" << i << "] H" << endl << Hs.at(i) << endl;
}
Now the buggy part
// this->calcH
Mat heStitcher::calcH(Mat& image1_toCalc, Mat& image2_toCalc) {
std::vector< KeyPoint > keypointsObject, keypointsScene;
Mat descriptorsObject, descriptorsScene;
Mat image1, image2;
image1 = image1_toCalc.clone();
image2 = image2_toCalc.clone();
if (greyCalc) {
cv::cvtColor(image1, image1, cv::COLOR_BGR2GRAY);
cv::cvtColor(image2, image2, cv::COLOR_BGR2GRAY);
}
Ptr<SURF> detector = SURF::create();
detector->detectAndCompute(image1, Mat(), keypointsObject, descriptorsObject);
detector->detectAndCompute(image2, Mat(), keypointsScene, descriptorsScene);
detector->clear();
vector<std::vector<DMatch>> matches;
Ptr<FlannBasedMatcher> matcher = cv::FlannBasedMatcher::create();
matcher->knnMatch(descriptorsObject, descriptorsScene, matches, 2);
matcher->clear();
double min_dist = 100;
double max_dist = 0;
std::vector< DMatch > goodMatches;
for (size_t i = 0; i < matches.size(); i++) {
if (matches[i][0].distance < 0.75f * matches[i][1].distance) {
goodMatches.push_back(matches[i][0]);
}
}
std::vector< Point2f > obj;
std::vector< Point2f > scene;
for (int i = 0; i < goodMatches.size(); i++) {
obj.push_back(keypointsObject[goodMatches[i].queryIdx].pt);
scene.push_back(keypointsScene[goodMatches[i].trainIdx].pt);
}
cv::Mat H;
cv::Mat H2;
if (obj.size() < 4) {
if (this->debugLevel[2]) cout << endl << "!!!!!! not enough similarities (less than 4) !!!!!!" << endl;
}
else {
H = findHomography(obj, scene, RANSAC);
}
image1.release();
image2.release();
descriptorsObject.release();
descriptorsScene.release();
keypointsObject.clear();
keypointsScene.clear();
obj.clear();
scene.clear();
goodMatches.clear();
matches.clear();
H2 = H.clone();
H.release();
return H2;
}
After this warpPerspective, border cutting etc, results with matrices in picture 1 a 2. (from where obviously matrices are incorrect). I cannot understant, why when using same script, with same procedure, I am getting different ressults.
I'm working in a detector for half bodies, in order to improve the performance of a normal people detector. I know there are more ways to deal with occlusion but this is what i was asked for to do in my end of degree project. My problem is that I'm not getting a good performance, more over, I'm getting kind a pattern in wich 4 rectangles that represent the detections are shown in almost the same position, not even representing a half body.
I have a set of images with 414 images of top-half bodies cropped by myself, used as positive samples, and 8520 negative images. All of them sized 64x64. I extracted the HOG descriptors as follows
int i;
string imgname, index;
HOGDescriptor hog (Size(64,64), Size(16,16), Size(8,8), Size(8,8), 9, 1, -1, HOGDescriptor::L2Hys, 0.2,false, HOGDescriptor::DEFAULT_NLEVELS, false);
vector<float> pos_rec_descript;
vector<Point> locations;
size_t SizeDesc;
SizeDesc = hog.getDescriptorSize();
FileStorage fpd ("Pos_Descriptors.yml", FileStorage::WRITE);
for (i = 1; i < 415; i++) { // 2416 images in ./pos_rec
stringstream a;
a << i;
imgname = "./pos_rec3/img" + a.str();
imgname += ".png";
Mat img = imread(imgname, CV_LOAD_IMAGE_COLOR);
hog.compute(img, pos_rec_descript, Size (16,16), Size (0,0),locations);
fpd << "Descriptores" + a.str() << pos_rec_descript;
}
fpd.release();
And I did the same with the negative samples.
Then, I trained a SVM as follows.
#define POS 414
#define NEG 8520
#define TOTAL 8934
#define DESCRIPT 1764
float trainingData[TOTAL][DESCRIPT];
int labels[TOTAL];
fstream doc;
void set_labels(){
int i;
for (i = 0; i < TOTAL; i++){
if (i < POS) {
labels[i] = 1;
}
else{
labels[i] = -1;
}
}
return;
}
int main(int, char**)
{
FileStorage fsv ("supvec.yml", FileStorage::WRITE);
FileStorage ftd ("TrainData.yml", FileStorage::WRITE);
//FileStorage flm ("Labels.yml", FileStorage::WRITE);
FileStorage fpd ("../HOG_descriptors/Pos_Descriptors.yml", FileStorage::READ);
FileStorage fnd ("../HOG_descriptors_neg/Neg_Descriptors.yml", FileStorage::READ);
set_labels();
// Set up training data
vector <float> pos_D, neg_D, train_D ;
int k = 0;
for (int i = 1; i < POS+1; i++) {
stringstream a;
a << i;
fpd["Descriptores" + a.str()] >> pos_D;
for (int j = 0; j < pos_D.size() ; j++){
train_D.push_back(pos_D[j]);
}
}
fpd.release();
for (int i = 1; i < NEG+1; i++) {
stringstream a;
a << i;
fnd["Descriptores" + a.str()] >> neg_D;
for (int j = 0; j < neg_D.size() ; j++){
train_D.push_back(neg_D[j]);
}
}
fnd.release();
for (int i = 0; i < TOTAL; i++){
for (int j = 0; j < DESCRIPT; j++){
trainingData[i][j] = train_D[k];
k++;
}
}
Mat trainingDataMat(TOTAL, DESCRIPT, CV_32FC1, trainingData);
//memcpy(trainingDataMat.data, train_D.data(), train_D.size()*sizeof(float));
Mat labelsMat(TOTAL, 1, CV_32SC1, labels);
//ftd << "trainingDataMat" << trainingDataMat;
//flm << "labelsMat" << labelsMat;
// Train the SVM
Ptr<SVM> svm = SVM::create();
svm->setType(SVM::C_SVC);
svm->setKernel(SVM::LINEAR);
svm->setTermCriteria(TermCriteria(TermCriteria::MAX_ITER, 100, 1e-6));
/*Ptr<TrainData> autoTrainData = TrainData::create(trainingDataMat, ROW_SAMPLE, labelsMat);
ParamGrid Cgrid = SVM::getDefaultGrid(SVM::C);
ParamGrid gammaGrid = SVM::getDefaultGrid(SVM::GAMMA);
ParamGrid pGrid = SVM::getDefaultGrid(SVM::P);
pGrid.logStep = 1;
ParamGrid nuGrid = SVM::getDefaultGrid(SVM::NU);
nuGrid.logStep = 1;
ParamGrid coeffGrid = SVM::getDefaultGrid(SVM::COEF);
coeffGrid.logStep = 1;
ParamGrid degreeGrid = SVM::getDefaultGrid(SVM::DEGREE);
degreeGrid.logStep = 1; */
cout << "Está entrenando..." << endl;
//svm->trainAuto(autoTrainData, 10, Cgrid, gammaGrid, pGrid, nuGrid, coeffGrid, degreeGrid, false);
svm->train(trainingDataMat, ROW_SAMPLE, labelsMat);
svm->save("SVM3_WS16_P0_LINEAR.yml");
I've tried with both LINEAR and RBF kernels (that's why you can see an autotrain part of the code commented that I used to swap between types of SVM) but none of them seems to work. Actually, they give nearly the same responses, something that makes me think that, maybe the training phase or the detection phase (code below) are ruining the whole project.
This is how I load the SVM for the HOG detector and try it over images
using namespace cv;
using namespace std;
using namespace cv::ml;
// static void help()
// {
// printf(
// "\nDemonstrate the use of the HoG descriptor using\n"
// " HOGDescriptor::hog.setSVMDetector(HOGDescriptor::getDefaultPeopleDetector());\n"
// "Usage:\n"
// "./peopledetect (<image_filename> | <image_list>.txt)\n\n");
// }
void get_svm_detector(const Ptr<SVM>& svm, vector< float > & hog_detector );
void get_svm_detector(const Ptr<SVM>& svm, vector< float > & hog_detector )
{
// get the support vectors
Mat sv = svm->getSupportVectors();
const int sv_total = sv.rows;
// get the decision function
Mat alpha, svidx;
double rho = svm->getDecisionFunction(0, alpha, svidx);
CV_Assert( alpha.total() == 1 && svidx.total() == 1 && sv_total == 1 );
CV_Assert( (alpha.type() == CV_64F && alpha.at<double>(0) == 1.) ||
(alpha.type() == CV_32F && alpha.at<float>(0) == 1.f) );
CV_Assert( sv.type() == CV_32F );
hog_detector.clear();
hog_detector.resize(sv.cols + 1);
memcpy(&hog_detector[0], sv.ptr(), sv.cols*sizeof(hog_detector[0]));
hog_detector[sv.cols] = (float)-rho;
}
int main(int argc, char** argv)
{
Mat img;
FILE* f = 0;
char _filename[1024];
if( argc == 1 )
{
printf("Usage: peopledetect (People_imgs | People_imgs.txt)\n");
return 0;
}
img = imread(argv[1]);
if( img.data )
{
strcpy(_filename, argv[1]);
}
else
{
f = fopen(argv[1], "rt");
if(!f)
{
fprintf( stderr, "ERROR: the specified file could not be loaded\n");
return -1;
}
}
// Load SVM
Ptr<SVM> svm = SVM::create();
svm = cv::Algorithm::load<ml::SVM>("../SVM_Train/SVM3_WS16_P0_LINEAR.yml");
HOGDescriptor hog (Size(64,64), Size(16,16), Size(8,8), Size(8,8), 9, 1, -1, HOGDescriptor::L2Hys, 0.2,false, HOGDescriptor::DEFAULT_NLEVELS, false);
vector <float> hog_detector;
get_svm_detector (svm, hog_detector);
hog.setSVMDetector(hog_detector);
namedWindow("people detector", 1);
for(;;)
{
char* filename = _filename;
if(f)
{
if(!fgets(filename, (int)sizeof(_filename)-2, f))
break;
//while(*filename && isspace(*filename))
// ++filename;
if(filename[0] == '#')
continue;
int l = (int)strlen(filename);
while(l > 0 && isspace(filename[l-1]))
--l;
filename[l] = '\0';
img = imread(filename);
}
printf("%s:\n", filename);
if(!img.data)
continue;
fflush(stdout);
vector<Rect> found, found_filtered, searchLocations;
vector<double> found_weights;
double t = (double)getTickCount();
// run the detector with default parameters. to get a higher hit-rate
// (and more false alarms, respectively), decrease the hitThreshold and
// groupThreshold (set groupThreshold to 0 to turn off the grouping completely).
hog.detectMultiScale(img, found, found_weights, 0, Size(16,16), Size(0,0), 1.01, 2);
//hog.detect(img, found, 0, Size(16,16), Size(0,0), searchLocations);
t = (double)getTickCount() - t;
printf("tdetection time = %gms\n", t*1000./cv::getTickFrequency());
size_t i, j;
for( i = 0; i < found.size(); i++ )
{
Rect r = found[i];
for( j = 0; j < found.size(); j++ )
if( j != i && (r & found[j]) == r)
break;
if( j == found.size() )
found_filtered.push_back(r);
}
for( i = 0; i < found_filtered.size(); i++ )
{
Rect r = found_filtered[i];
// the HOG detector returns slightly larger rectangles than the real objects.
// so we slightly shrink the rectangles to get a nicer output.
r.x += cvRound(r.width*0.1);
r.width = cvRound(r.width*0.7);
r.y += cvRound(r.height*0.07);
r.height = cvRound(r.height*0.7);
rectangle(img, r.tl(), r.br(), cv::Scalar(0,255,0), 2);
imshow("people detector", img);
waitKey(0);
}
//imshow("people detector", img);
//string imgname = "./Responses/Win_Stride16_4.png";
//imwrite(imgname, img);
int c = waitKey(0) & 255;
if( c == 'q' || c == 'Q' || !f)
break;
}
if(f)
fclose(f);
return 0;
}
I have checked all dimensions for the descriptors, every Mat seems to be ok. But at the time I use detectMultiScale, it shows things like this:
Image 1: It's strange because is missing lots of detections
Image 2: Here I realized there was a kind of pattern with this 4 rects
My problem is that no matter what I change (descriptors, Kernel, winStride and Padding in detectMultiScale), there is always very similar responses, and nothing indicates that there is a correct detection there.
I'm not very sure about how I'm giving the support vectors to HOG, but is the only way I found to do it (found it in one of the post from StackOverflow).
If any of you has any idea of what is going on here, and why the responses are not changing from one configuration to another, I would be greatly thankfull. This code is giving me headaches since weeks now. I've been changing parametres on fucntions, on HOG, changing Kernels, trying different set of images, but nothing seems to give great changes on the final result.
I am trying to perform camera calibration using openCV. On the same example data from openCV samples this works great, but my source generates an error:
Assertion failed <ni > 0 && ni==ni1>... collectCallibrationData.. calibration.cpp line 3193"
objectPoints and imagePoints are the same size. Changing other inputs dont affect error code.
Can anyone help me deal with it? The code follows:
void main()//enter code here
{
//initial operations, declatarions etc
int *Asize; float *APoints;
float AAPoints[]= {/* My Array */};
int AAsize[] = {4,54,2}; //My array size
Asize = AAsize; APoints = AAPoints; // will be called as dll that is why this wierd attribution
int page = *(Asize); int row = *(Asize+1); int col = *(Asize +2);
Point3f pointBuf;
vector<vector<Point3f>> imagePoints;
vector<Point3f> vectBuf;
Size boardSize;
boardSize.height = 6; boardSize.width = 9;
Mat cameraMatrix, distCoeffs;
float squareSize=50;
//change 1d array to vector<vector<point3f>>
for (int i = 0; i<page; i++)
{
for (int j = 0; j<row; j++)
{
pointBuf.x = *(APoints + (i*(row*col))+j*2);
pointBuf.y = *(APoints + (i*(row*col))+j*2+1);
pointBuf.z = 0;
vectBuf.push_back(pointBuf);
}
imagePoints.push_back(vectBuf);
vectBuf.clear();
}
// create objectPoint vector<vector<Points3f>>
vector<vector<Point3f>> objectPoints;
vectBuf.clear();
for( int i = 0; i < boardSize.height; ++i )
{
for( int j = 0; j < boardSize.width; ++j )
vectBuf.push_back(Point3f(float( j*squareSize ), float( i*squareSize ), 0));
}
objectPoints.resize(imagePoints.size(),vectBuf);
//initialize starting variables for calibration
cameraMatrix = Mat::eye(3, 3, CV_64F);
distCoeffs = Mat::zeros(1, 1, CV_64F);
cameraMatrix.at<double>(0,0) = 1.0;
vector<Mat> rvecs, tvecs;
Size imageSize; imageSize.width = 2040; imageSize.height = 2040;
double rms = calibrateCamera(objectPoints, imagePoints, imageSize, cameraMatrix, distCoeffs, rvecs, tvecs);
}
There is a trouble with the use of cv::calibrateCamera(). It requires std::vector<std::vector<cv::Point3f>> as the first argument and std::vector<std::vector<cv::Point2f>> as the second.
e.g.:
cv::Size imageSize(2040, 2040);
cv::Mat cameraMatrix, distCoeffs;
std::vector<cv::Mat> rvecs, tvecs;
std::vector<std::vector<cv::Point2f> > imagePoints;
std::vector<std::vector<cv::Point3f> > objectPoints;
// Fill imagePoints and objectPoints
...
cv::calibrateCamera(objectPoints, imagePoints, imageSize, cameraMatrix, distCoeffs, rvecs, tvecs);
I met the same issue, too. In my case, there is a bug to convert vector< vector < Point3f > > to InputArrayOfArrays.
You may run the following sample code to check it in your system.
using namespace std;
using namespace cv;
/** #function main */
int main( void )
{
vector<Point3f> objectPoints_tmp(54);
objectPoints_tmp.clear();
for( int i = 0; i < 9; ++i )
for( int j = 0; j < 6; ++j )
objectPoints_tmp.push_back(Point3f(j*50, i*50, 0.0f));
vector<vector<Point3f> > objectPoints(7, objectPoints_tmp);
InputArrayOfArrays OBJPOINT = objectPoints; std::cout << (int) OBJPOINT.total() << std::endl;
for( int i = 0; i < 7; ++i )
std::cout << OBJPOINT.getMat(i).checkVector(3, CV_32F) << std::endl;
waitKey(0);
return 0;
}
Normally, it should print out
7
54
54
54
54
54
54
54
In my NG case, it prints out random numbers.
I guess that the opencv lib is not fit with my system(win7 x64 + vs2012). After rebuild opencv myself, it may work now. You may google keyword "Win7x64 VS2012 OpenCV CMake TBB".
I'm trying to calibrate my two Point Grey (Blackfly) cameras for stereo vision. I'm using the tutorial stereo_calib.cpp that comes with OpenCV (code below). For some reason, I'm getting really bad results (RMS error=4.49756 and average reprojection err = 8.06533) and all my rectified images come out grey. I think my problem is that I'm not picking the right flags for the stereoCalibrate() function, but I've tried many different combinations and at best the rectified images would be warped.
Here's a link to the images I used and a sample rectified pair: https://www.dropbox.com/sh/5wp31o8xcn6vmjl/AAADAfGiaT_NyXEB3zMpcEvVa#/
Any help would be appreciated!!
static void
StereoCalib(const vector<string>& imagelist, Size boardSize, bool useCalibrated=true, bool showRectified=true)
{
if( imagelist.size() % 2 != 0 )
{
cout << "Error: the image list contains odd (non-even) number of elements\n";
return;
}
bool displayCorners = true;//false;//true;
const int maxScale = 1;//2;
const float squareSize = 1.8;
//const float squareSize = 1.f; // Set this to your actual square size
// ARRAY AND VECTOR STORAGE:
vector<vector<Point2f> > imagePoints[2];
vector<vector<Point3f> > objectPoints;
Size imageSize;
//int i, j, k, nimages = (int)imagelist.size()/2;
int i, j, k, nimages = (int)imagelist.size();
cout << "nimages: " << nimages << "\n";
imagePoints[0].resize(nimages);
imagePoints[1].resize(nimages);
vector<string> goodImageList;
for( i = j = 0; i < nimages; i++ )
{
for( k = 0; k < 2; k++ )
{
const string& filename = imagelist[i*2+k];
Mat img = imread(filename, 0);
if(img.empty()) {
break;
}
if( imageSize == Size() ) {
imageSize = img.size();
} else if( img.size() != imageSize )
{
cout << "The image " << filename << " has the size different from the first image size. Skipping the pair\n";
break;
}
bool found = false;
vector<Point2f>& corners = imagePoints[k][j];
for( int scale = 1; scale <= maxScale; scale++ )
{
Mat timg;
if( scale == 1 )
timg = img;
else
resize(img, timg, Size(), scale, scale);
found = findChessboardCorners(timg, boardSize, corners,
CV_CALIB_CB_ADAPTIVE_THRESH | CV_CALIB_CB_NORMALIZE_IMAGE);
if( found )
{
if( scale > 1 )
{
Mat cornersMat(corners);
cornersMat *= 1./scale;
}
break;
}
}
if( displayCorners )
{
cout << filename << endl;
Mat cimg, cimg1;
cvtColor(img, cimg, COLOR_GRAY2BGR);
drawChessboardCorners(cimg, boardSize, corners, found);
double sf = 1280./MAX(img.rows, img.cols);
resize(cimg, cimg1, Size(), sf, sf);
imshow("corners", cimg1);
char c = (char)waitKey(500);
if( c == 27 || c == 'q' || c == 'Q' ) //Allow ESC to quit
exit(-1);
}
else
putchar('.');
if( !found ) {
cout << "!found\n";
break;
}
cornerSubPix(img, corners, Size(11,11), Size(-1,-1),
TermCriteria(CV_TERMCRIT_ITER+CV_TERMCRIT_EPS,
30, 0.01));
}
if( k == 2 )
{
goodImageList.push_back(imagelist[i*2]);
goodImageList.push_back(imagelist[i*2+1]);
j++;
}
}
cout << j << " pairs have been successfully detected.\n";
nimages = j;
if( nimages < 2 )
{
cout << "Error: too little pairs to run the calibration\n";
return;
}
imagePoints[0].resize(nimages);
imagePoints[1].resize(nimages);
objectPoints.resize(nimages);
for( i = 0; i < nimages; i++ )
{
for( j = 0; j < boardSize.height; j++ )
for( k = 0; k < boardSize.width; k++ )
objectPoints[i].push_back(Point3f(j*squareSize, k*squareSize, 0));
}
cout << "Running stereo calibration ...\n";
Mat cameraMatrix[2], distCoeffs[2];
cameraMatrix[0] = Mat::eye(3, 3, CV_64F);
cameraMatrix[1] = Mat::eye(3, 3, CV_64F);
Mat R, T, E, F;
double rms = stereoCalibrate(objectPoints, imagePoints[0], imagePoints[1],
cameraMatrix[0], distCoeffs[0],
cameraMatrix[1], distCoeffs[1],
imageSize, R, T, E, F,
//TermCriteria(CV_TERMCRIT_ITER+CV_TERMCRIT_EPS, 100, 1e-5));
TermCriteria(CV_TERMCRIT_ITER+CV_TERMCRIT_EPS, 100, 1e-5),
CV_CALIB_FIX_ASPECT_RATIO +
//CV_CALIB_ZERO_TANGENT_DIST +
CV_CALIB_SAME_FOCAL_LENGTH +
CV_CALIB_RATIONAL_MODEL +
//CV_CALIB_FIX_K3);
//CV_CALIB_FIX_K2);
CV_CALIB_FIX_K3 + CV_CALIB_FIX_K4 + CV_CALIB_FIX_K5);
//CV_CALIB_FIX_K1 + CV_CALIB_FIX_K2 + CV_CALIB_FIX_K3 + CV_CALIB_FIX_K4 + CV_CALIB_FIX_K5);
cout << "done with RMS error=" << rms << endl;
double err = 0;
int npoints = 0;
vector<Vec3f> lines[2];
for( i = 0; i < nimages; i++ )
{
int npt = (int)imagePoints[0][i].size();
Mat imgpt[2];
for( k = 0; k < 2; k++ )
{
imgpt[k] = Mat(imagePoints[k][i]);
undistortPoints(imgpt[k], imgpt[k], cameraMatrix[k], distCoeffs[k], Mat(), cameraMatrix[k]);
computeCorrespondEpilines(imgpt[k], k+1, F, lines[k]);
}
for( j = 0; j < npt; j++ )
{
double errij = fabs(imagePoints[0][i][j].x*lines[1][j][0] +
imagePoints[0][i][j].y*lines[1][j][1] + lines[1][j][2]) +
fabs(imagePoints[1][i][j].x*lines[0][j][0] +
imagePoints[1][i][j].y*lines[0][j][1] + lines[0][j][2]);
err += errij;
}
npoints += npt;
}
cout << "average reprojection err = " << err/npoints << endl;
// save intrinsic parameters
FileStorage fs("intrinsics.yml", CV_STORAGE_WRITE);
if( fs.isOpened() )
{
fs << "M1" << cameraMatrix[0] << "D1" << distCoeffs[0] <<
"M2" << cameraMatrix[1] << "D2" << distCoeffs[1];
fs.release();
}
else
cout << "Error: can not save the intrinsic parameters\n";
Mat R1, R2, P1, P2, Q;
Rect validRoi[2];
stereoRectify(cameraMatrix[0], distCoeffs[0],
cameraMatrix[1], distCoeffs[1],
imageSize, R, T, R1, R2, P1, P2, Q,
//CALIB_ZERO_DISPARITY, 1, imageSize, &validRoi[0], &validRoi[1]);
CALIB_ZERO_DISPARITY, 0, imageSize, &validRoi[0], &validRoi[1]);
fs.open("extrinsics.yml", CV_STORAGE_WRITE);
if( fs.isOpened() )
{
fs << "R" << R << "T" << T << "R1" << R1 << "R2" << R2 << "P1" << P1 << "P2" << P2 << "Q" << Q;
fs.release();
}
else
cout << "Error: can not save the intrinsic parameters\n";
// OpenCV can handle left-right
// or up-down camera arrangements
//bool isVerticalStereo = fabs(P2.at<double>(1, 3)) > fabs(P2.at<double>(0, 3));
bool isVerticalStereo = false;
// COMPUTE AND DISPLAY RECTIFICATION
if( !showRectified )
return;
Mat rmap[2][2];
// IF BY CALIBRATED (BOUGUET'S METHOD)
if( useCalibrated )
{
// we already computed everything
}
// OR ELSE HARTLEY'S METHOD
else
// use intrinsic parameters of each camera, but
// compute the rectification transformation directly
// from the fundamental matrix
{
vector<Point2f> allimgpt[2];
for( k = 0; k < 2; k++ )
{
for( i = 0; i < nimages; i++ )
std::copy(imagePoints[k][i].begin(), imagePoints[k][i].end(), back_inserter(allimgpt[k]));
}
F = findFundamentalMat(Mat(allimgpt[0]), Mat(allimgpt[1]), FM_8POINT, 0, 0);
Mat H1, H2;
stereoRectifyUncalibrated(Mat(allimgpt[0]), Mat(allimgpt[1]), F, imageSize, H1, H2, 3);
R1 = cameraMatrix[0].inv()*H1*cameraMatrix[0];
R2 = cameraMatrix[1].inv()*H2*cameraMatrix[1];
P1 = cameraMatrix[0];
P2 = cameraMatrix[1];
}
//Precompute maps for cv::remap()
initUndistortRectifyMap(cameraMatrix[0], distCoeffs[0], R1, P1, imageSize, CV_16SC2, rmap[0][0], rmap[0][1]);
initUndistortRectifyMap(cameraMatrix[1], distCoeffs[1], R2, P2, imageSize, CV_16SC2, rmap[1][0], rmap[1][1]);
Mat canvas;
double sf;
int w, h;
if( !isVerticalStereo )
{
sf = 600./MAX(imageSize.width, imageSize.height);
w = cvRound(imageSize.width*sf);
h = cvRound(imageSize.height*sf);
canvas.create(h, w*2, CV_8UC3);
}
else
{
sf = 600./MAX(imageSize.width, imageSize.height);
w = cvRound(imageSize.width*sf);
h = cvRound(imageSize.height*sf);
canvas.create(h*2, w, CV_8UC3);
}
for( i = 0; i < nimages; i++ )
{
for( k = 0; k < 2; k++ )
{
Mat img = imread(goodImageList[i*2+k], 0), rimg, cimg;
remap(img, rimg, rmap[k][0], rmap[k][1], CV_INTER_LINEAR);
cvtColor(rimg, cimg, COLOR_GRAY2BGR);
Mat canvasPart = !isVerticalStereo ? canvas(Rect(w*k, 0, w, h)) : canvas(Rect(0, h*k, w, h));
resize(cimg, canvasPart, canvasPart.size(), 0, 0, CV_INTER_AREA);
if( useCalibrated )
{
Rect vroi(cvRound(validRoi[k].x*sf), cvRound(validRoi[k].y*sf),
cvRound(validRoi[k].width*sf), cvRound(validRoi[k].height*sf));
rectangle(canvasPart, vroi, Scalar(0,0,255), 3, 8);
}
}
if( !isVerticalStereo )
for( j = 0; j < canvas.rows; j += 16 )
line(canvas, Point(0, j), Point(canvas.cols, j), Scalar(0, 255, 0), 1, 8);
else
for( j = 0; j < canvas.cols; j += 16 )
line(canvas, Point(j, 0), Point(j, canvas.rows), Scalar(0, 255, 0), 1, 8);
imshow("rectified", canvas);
char c = (char)waitKey();
if( c == 27 || c == 'q' || c == 'Q' )
break;
}
}
First of all, about your calibration images. I see a few points that could lead to a better calibration :
Use stabler images. Most of your images are blurred a bit, which results in bad accuracy in the corner detection
Vary scale. Most images that you use present the checkerboard approx. at the same distance from the cameras.
Be careful about your checkerboard itself. It appears to be quite badly attached to its support. If you want to achieve a good calibration, you must ensure that your checkerboard is attached tightly on a flat surface.
You have much more detailed advice about how to make a good calibration in this SO answer
Now, about the stereo calibration itself. Best way that I found to achieve a good calibration is to separately calibrate each camera intrinsics (using the calibrateCamera function) then the extrinsics (using stereoCalibrate) using the intrinsics as a guess. Have a look at the stereoCalibrate flags for how to do this.
Outside of this, your flags in the stereoCalibrate function are as such :
CV_CALIB_FIX_ASPECT_RATIO : you force the aspect ratio fx/fy to be fixed
CV_CALIB_SAME_FOCAL_LENGTH : seems OK since you have two identical cameras. You can check whether it is exact by calibrating independently each camera
CV_CALIB_RATIONAL_MODEL : enables K3, k4 and k5 distorsion parameters
CV_CALIB_FIX_K3 + CV_CALIB_FIX_K4 + CV_CALIB_FIX_K5 : fixes those 3 parameters. Since you don't use any uess, you actually put them to 0 here so the option CV_CALIB_RATIONAL_MODEL is no use in your code with those flags
Note that if you calibrate independently each camera and use the intrinsics, you have different levels of use of this data :
With the flag CV_CALIB_FIX_INTRINSIC, the intrinsics will be used as such and only extrinsic parameters will be optimized
With CV_CALIB_USE_INTRINSIC_GUESS, intrinsics will be used as guesses but optimized again
With a combination of CV_CALIB_FIX_PRINCIPAL_POINT, CV_CALIB_FIX_FOCAL_LENGTH and CV_CALIB_FIX_K1,...,CV_CALIB_FIX_K6 you get a little of play about which parameters are fixed and which are optimized again
I had the same problem few weeks back where I tried almost 50 stereo image samples but the rectified image was all blank. So I decided to write my own implementation of stereo calibration code but in real time.
This code detected the chessboard corners in real time and saves those image where chessboard corners in both left and right images is found and then it runs stereo calib code on them.
I hope this helps you and someone in the future.
Source Code: https://github.com/upperwal/opencv/blob/master/samples/cpp/stereo_calib.cpp
Demo Video:
https://www.youtube.com/watch?v=kizO_s-YUDU
All the best
I had similar problem and then I re edited the code . Please make sure you have sufficient number of images at different depth form camera and at different orientation that will lead to less re projection error
#include <opencv2/core/core.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include <opencv2/calib3d/calib3d.hpp>
#include <opencv2/highgui/highgui.hpp>
#include<iostream>
int main()
{
const int CHESSBOARD_WIDTH = 9; //input width of chessboard
const int CHESSBOARD_HEIGHT = 6; //input height of chessboard
const float squareSize = 3.96; //input size of a side of a single square in chessboard
cv::Size corner=cv::Size(CHESSBOARD_WIDTH,CHESSBOARD_HEIGHT);
int counter =30;
int nimages=24;
cv::Size imageSize;
enum{capturing=0,calibrated=1};
int mode=capturing;
char leftfilename[100];
char rightfilename[100];
std::vector<cv::Mat> imagePoints1;
std::vector<cv::Mat> imagePoints2;
std::vector<std::vector<cv::Point3f>> objectPoints;
bool found1=false;
bool found2=false;
int counter2=0;
cv::Mat pointBuf1=cv::Mat::zeros(54,2,CV_32FC1);
cv::Mat pointBuf2=cv::Mat::zeros(54,2,CV_32FC1);
for(int i=1;i<=counter;i++)
{ sprintf(leftfilename,"newleftcheck%d.jpg",i);
sprintf(rightfilename,"newrightcheck%d.jpg",i);
// const int CHESSBOARD_INTERSECTION_COUNT = CHESSBOARD_WIDTH * CHESSBOARD_HEIGHT;
cv::Mat imgleft_frame=cv::imread(leftfilename);
cv::Mat imgright_frame=cv::imread(rightfilename);
//cv::Mat imgleft_frame =cv::Mat(480,640,CV_8UC4,s.leftBuffer,4*imgWidth*sizeof(unsigned char));
//cv::Mat imgright_frame =cv::Mat(480,640,CV_8UC4,s.rightBuffer,4*imgWidth*sizeof(unsigned char));
imageSize=imgleft_frame.size();
found1 = findChessboardCorners(imgleft_frame, corner,pointBuf1,cv::CALIB_CB_ADAPTIVE_THRESH | cv::CALIB_CB_FAST_CHECK | cv::CALIB_CB_NORMALIZE_IMAGE);
found2 = findChessboardCorners(imgright_frame, cv::Size(CHESSBOARD_WIDTH,CHESSBOARD_HEIGHT),pointBuf2,cv::CALIB_CB_ADAPTIVE_THRESH | cv::CALIB_CB_FAST_CHECK | cv::CALIB_CB_NORMALIZE_IMAGE);
if(found1)
{ cv::Mat gray_image1;
cvtColor(imgleft_frame,gray_image1,cv::COLOR_BGRA2GRAY);
cornerSubPix( gray_image1, pointBuf1, cv::Size(11,11),cv::Size(-1,-1), cv::TermCriteria(cv::TermCriteria::EPS+cv::TermCriteria::MAX_ITER, 30, 0.1 ));
drawChessboardCorners( imgleft_frame,cv::Size(CHESSBOARD_WIDTH,CHESSBOARD_HEIGHT), pointBuf1, found1 );
}
if(found2)
{ cv::Mat gray_image2;
cvtColor(imgright_frame,gray_image2,cv::COLOR_BGRA2GRAY);
cornerSubPix( gray_image2, pointBuf2, cv::Size(11,11),cv::Size(-1,-1), cv::TermCriteria(cv::TermCriteria::EPS+cv::TermCriteria::MAX_ITER, 30, 0.1 ));
drawChessboardCorners( imgright_frame,cv::Size(CHESSBOARD_WIDTH,CHESSBOARD_HEIGHT), pointBuf2, found2 );
}
if(found1&&found2)
{ imagePoints1.push_back(pointBuf1);
imagePoints2.push_back(pointBuf2);
//sprintf(leftfilename,"newleftcheck%d.jpg",s.counter);
//sprintf(rightfilename,"newrightcheck%d.jpg",s.counter);
//cv::imwrite(leftfilename,imgleft_frame);
//cv::imwrite(rightfilename,imgright_frame);
counter2=counter2+1;
std::cout<<counter2<<std::endl;
}
nimages=counter2;
objectPoints.resize(nimages);
std::cout<<"countervalue"<<i<<std::endl;
}
for(int i = 0; i <nimages; i++ )
{
for( int j = 0; j < CHESSBOARD_HEIGHT; j++ )
for( int k = 0; k < CHESSBOARD_WIDTH; k++ )
objectPoints[i].push_back(cv::Point3f(j*squareSize, k*squareSize, 0));
}
std::cout<<"check1"<<std::endl;
cv::Mat cameraMatrix[2], distCoeffs[2];
cameraMatrix[0] = cv::Mat::eye(3, 3, CV_64F);
cameraMatrix[1] = cv::Mat::eye(3, 3, CV_64F);
cv::Mat R, T, E, F;
std::cout<<objectPoints.size()<<std::endl;
std::cout<<imagePoints1.size()<<std::endl;
if(imagePoints1.size()==imagePoints2.size())
std::cout<<"samesize"<<std::endl;
if(imagePoints1.size()>=nimages )
{ std::cout<<"check2"<<std::endl;
double rms = stereoCalibrate( objectPoints, imagePoints1, imagePoints2,cameraMatrix[0], distCoeffs[0],
cameraMatrix[1], distCoeffs[1],imageSize, R, T, E, F,
cv::CALIB_FIX_ASPECT_RATIO +cv::CALIB_ZERO_TANGENT_DIST +
cv::CALIB_SAME_FOCAL_LENGTH +cv::CALIB_RATIONAL_MODEL +
cv::CALIB_FIX_K3 +cv::CALIB_FIX_K4 + cv::CALIB_FIX_K5,
cv::TermCriteria(cv::TermCriteria::COUNT+cv::TermCriteria::EPS, 100, 1e-5) );
std::cout<<"check3"<<std::endl;
std::cout << "done with RMS error=" << rms << std::endl;
mode=calibrated;
std::cout<<"calibrated"<<std::endl;
}
if(mode==calibrated)
{
double err = 0;
int npoints = 0;
std::vector<cv::Vec3f> lines[2];
for(int i = 0; i < nimages; i++ )
{
int npt = (int)imagePoints1[i].rows;
std::cout<<npt<<std::endl;
cv:: Mat imgpt1;
cv::Mat imgpt2;
// for(int k = 0; k < 2; k++ )
imgpt1 = cv::Mat(imagePoints1[i]);
undistortPoints(imgpt1, imgpt1, cameraMatrix[0], distCoeffs[0], cv::Mat(), cameraMatrix[0]);
computeCorrespondEpilines(imgpt1, 1, F, lines[0]);
imgpt2 = cv::Mat(imagePoints2[i]);
undistortPoints(imgpt2, imgpt2, cameraMatrix[1], distCoeffs[1], cv::Mat(), cameraMatrix[1]);
computeCorrespondEpilines(imgpt2, 2, F, lines[1]);
std::cout<<"checksdcdufb"<<std::endl;
//std::cout<<"imagepoint"<<imagePoints1[1].at<unsigned int>(1,1)<<std::endl;
/* for(int j = 0; j < npt; j++ )
{
double errij = fabs(imagePoints1[i].at<double>(j,0) *lines[1][j][0] +imagePoints1[i].at<double>(j,1)*lines[1][j][1] + lines[1][j][2]) +fabs(imagePoints2[i].at<double>(j,0)*lines[0][j][0] +
imagePoints2[i].at<double>(j,1)*lines[0][j][1] + lines[0][j][2]);
err += errij;
}
npoints += npt;
}*/
std::cout<<"check8"<<std::endl;
cv::FileStorage fs("intrinsics.xml", cv::FileStorage::WRITE);
if( fs.isOpened() )
{
fs << "M1" << cameraMatrix[0] << "D1" << distCoeffs[0] <<
"M2" << cameraMatrix[1] << "D2" << distCoeffs[1];
fs.release();
}
else
std:: cout << "Error: can not save the intrinsic parameters\n";
cv::Mat R1, R2, P1, P2, Q;
cv::Rect validRoi[2];
stereoRectify(cameraMatrix[0], distCoeffs[0],
cameraMatrix[1], distCoeffs[1],
imageSize, R, T, R1, R2, P1, P2, Q,
cv::CALIB_ZERO_DISPARITY, 1, imageSize, &validRoi[0], &validRoi[1]);
fs.open("extrinsics.xml", cv::FileStorage::WRITE);
if( fs.isOpened() )
{
fs << "R" << R << "T" << T << "R1" << R1 << "R2" << R2 << "P1" << P1 << "P2" << P2 << "Q" << Q;
fs.release();
}
else
std::cout << "Error: can not save the intrinsic parameters\n";
}
//std::cout << "average reprojection err = " << err/npoints <<std::endl;
}
return 0;
}