Histogram matching of two images with c++ and OpenCV 2.4.x - c++

Since I found nothing about Histogram Matching in C++ and OpenCV 2.4 I ask here again a question.
All solutions I found for newer versions.
My code:
void histogramMatching(Mat & reference, Mat & input, Mat & result) {
const float HISMATCH = 0.001;
double min, max;
vector<Mat> reference_channels;
split(reference, reference_channels);
vector<Mat> input_channels;
split(input, input_channels);
int histSize = 256;
float range[] = { 0,256 };
const float* histrange = { range };
bool uniform = true;
for (int i = 0; i < 3; i++) {
Mat reference_histogram, input_histogram;
Mat reference_histogram_accum, input_histogram_accum;
calcHist(&input_channels[i], 1, 0, Mat(), input_histogram, 1, &histSize, &histrange, &uniform);
try {
calcHist(&reference_channels[i], 1, 0, Mat(), reference_histogram, 1, &histSize, &histrange, &uniform);
}
catch (int n) {
cout << "The first element is " << n << endl;
}
minMaxLoc(reference_histogram, &min, &max);
normalize(reference_histogram, reference_histogram, min / max, NORM_MINMAX);
minMaxLoc(input_histogram, &min, &max);
normalize(input_histogram, input_histogram, min / max, NORM_MINMAX);
reference_histogram.copyTo(reference_histogram_accum);
input_histogram.copyTo(input_histogram_accum);
float* src_cdf_data = input_histogram_accum.ptr<float>();
float* dst_cdf_data = reference_histogram_accum.ptr<float>();
for (int j = 1; j < 256; j++) {
src_cdf_data[j] += src_cdf_data[j - 1];
dst_cdf_data[j] += dst_cdf_data[j - 1];
}
minMaxLoc(reference_histogram_accum, &min, &max);
normalize(reference_histogram_accum, reference_histogram_accum, min / max, 1.0, NORM_MINMAX);
minMaxLoc(input_histogram_accum, &min, &max);
normalize(input_histogram_accum, input_histogram_accum, min / max, 1.0, NORM_MINMAX);
//BEGIN Matching
Mat lut(1, 256, CV_8UC1);
uchar* M = lut.ptr<uchar>();
uchar last = 0;
for (int j = 0; j < input_histogram_accum.rows; j++) {
float F1 = dst_cdf_data[j];
int i = 0;
for (uchar k = last; k < reference_histogram_accum.rows; k++) {
i++;
float F2 = src_cdf_data[k];
if (abs(F2 - F1) < HISMATCH || F2 > F1) {
M[j] = k;
last = k;
break;
}
}
}
LUT(input_channels[i], lut, input_channels[i]);
}
merge(input_channels, result);
}
public:int execute() {
Mat input = imread("input.jpg", IMREAD_COLOR);
if (input.empty()) {
cout << "Image is empty" << endl;
return -1;
}
Mat reference = imread("fuchs.jpg", IMREAD_COLOR);
if (reference.empty()) {
cout << "Reference Image is empty" << endl;
return -1;
}
Mat result = input.clone();
namedWindow("Reference", WINDOW_AUTOSIZE);
namedWindow("Input", WINDOW_AUTOSIZE);
namedWindow("Result", WINDOW_AUTOSIZE);
imshow("Reference", reference);
imshow("Input", input);
histogramMatching(reference, input, result);
imshow("Result", result);
waitKey(0);
return 0;
}
All works, but the loop which begins with "for (int j = 0; j < input_histogram_accum.rows; j++) {", get no response and i waited for more than 6 hours and i think it doesn't work. My input image is 500kb and my fuchs.jpg is 180 kb.
Has anybody a solution for histogram matching with C++ and OpenCV 2.4.x?

Related

Color to grayscale conversion

Im using a C++ openCV program for first principles Algorithm development for HDL(Verilog) image object detection. I've finally managed to get HDL version up to the point of canny detection. In order to validate the two, both need to have identical output. I have found their are subtle differences that I thing are being contributed to by the openCV imread colour to grayscale conversion biasing green. The smoothed image is overall brighter in the openCV C++ method. From looking at the rgb2gray method it appears openCV used a bias ie (RX+GY+B*Z)/3 while in HDL I have been using (R+G+B)/3 as I require it to complete Gaussian, Sobel and Canny filters. Human visualisation is secondary and multiplication by a non-int is undesirable.
Is there a standard linear grayscale conversion for conversion or a means to override the existing method?
...
int main()
{
int thold = 15;
clock_t start;
double duration;
const int sobelX[3][3] = { {-1, 0, 1}, {-2, 0, 2}, {-1, 0, 1} }; //Where origionally floats in python
const int sobelY[3][3] = { {-1, -2, -1}, {0, 0, 0}, {1, 2, 1} }; //Where origionally floats in python
const int kernel[5][5] = { {1,6,12,6,1},
{6,42,79,42,6},
{12,79,148,79,12},
{6,42,79,42,6},
{1,6,12,6,1} };// 1/732
// Above normalised kernal for smoothing, see origional python script for method
start = std::clock();
int height, width, intPixel, tSx, tSy, tS, dirE, dirEE, maxDir, curPoint, contDirection, cannyImgPix, nd, tl, tm, tr, mr, br, bm, bl, ml = 0;
int contNum = 128;
int contPixCount = 0;
int curContNum = 0;
int contPlace = 0;
int oldContPlace = 0;
int g = 0;
bool maxPoint;
struct pixel {
int number;
int h;
int w;
};
std::vector<pixel> contourList;
//double floatPixel = 0.0;
int kernalCumulator = 0;
const int mp = 3;
// Scalar color(0, 0, 255);
// duration = ((clock()) - start) / (double)CLOCKS_PER_SEC;
// start = clock();
// cout << "Start image in" << duration << '\n';
// Mat dst;
Mat rawImg = imread("C:\\Users\\&&&\\Documents\\pycode\\paddedGS.png",0);
printf("%d",rawImg.type());
// Mat rawImg = imread("C:\\Users\\&&&\\Documents\\openCV_Master\\openCVexample\\openCVexample\\brace200.jpg ", 0);
height = rawImg.rows;
width = rawImg.cols;
cout << "Height of image " << height << '\n';
cout << "Width of image " << width << '\n';
Mat filteredImg = Mat::zeros(height, width, CV_8U);
printf("%d", filteredImg.type());
Mat sobelImg = Mat::zeros(height, width, CV_8U);
Mat directionImg = Mat::zeros(height, width, CV_8U);
Mat cannyImg = Mat::zeros(height, width, CV_8U);
Mat contourImg = Mat::zeros(height, width, CV_16U);
// rawImg.convertTo(rawImg, CV_8UC1);
duration = ((clock()) - start) / (double)CLOCKS_PER_SEC;
start = clock();
cout << "Start image in" << duration << '\n';
// Loop to threshold already grayscaled image
/*
for (int h = 0; h < (height); h++)
{
for (int w = 0; w < (width); w++)
{
g = (int)rawImg.at<uchar>(h, w,0);
cout << g << "g";
g+= (int)rawImg.at<uchar>(h, w, 1);
cout << g << "g";
g+= (int)rawImg.at<uchar>(h, w, 2);
cout << g << "g";
g = g/3;
rawGImg.at<uchar>(h,w) = g;
}
}
*/
// imshow("thresholded Image", rawImg);
// waitKey();
// Loop to smooth using Gausian 5 x 5 kernal
// imshow("raw Image", rawImg);
for (int h = 3; h < (height - 3); h++)
{
for (int w = 3; w < (width - 3); w++)
{
if (rawImg.at<uchar>(h, w) >=6 )//Thresholding included
{
for (int xk = 0; xk < 5; xk++)
{
for (int yk = 0; yk < 5; yk++)
{
intPixel = rawImg.at<uchar>((h + (xk - mp)), (w + (yk - mp)));
kernalCumulator += intPixel*(kernel[xk][yk]);//Mutiplier required as rounding is making number go above 255, better solution?
}
}
}
else
kernalCumulator = 0;
kernalCumulator = kernalCumulator / 732;
if (kernalCumulator < 0 || kernalCumulator > 255)
{
// cout << "kernal Value: " << kernalCumulator;
// cout << " intPixel:" << intPixel << '\n';
}
filteredImg.at<uchar>(h, w) = (uchar)kernalCumulator;
kernalCumulator = 0;
}
}
Our vision does not perceive linearly the brightness, so it makes sense for usual applications to use some sort of transformation that tries to mimic the human perception.
For your application, you have 2 options: either use a similar transformation in HDL (which might not be easy or desired), or make a custom rgb to grayscale for OpenCV which uses the same transformation you use.
A short snippet (more like pseudocode, you'll have to figure out the details) for this would be something like:
cv::Mat linearRgbToGray(const cv::Mat &color) {
cv::Mat gray(color.size(), CV_8UC1);
for (int i = 0; i < color.rows; i++)
for (int j = 0; j < color.cols; j++)
gray.at(i, j) = (color.at(i, j)[0] + color.at(i, j)[1] + color.at(i, j)[2]) / 3;
}
As per Paul92's advice above
cv::Mat linearRgbToGray(const cv::Mat &color) {
cv::Mat gray(color.size(), CV_8UC1);
for (int i = 0; i < color.rows; i++)
for (int j = 0; j < color.cols; j++)
gray.at<uchar>(i, j) = ((color.at<cv::Vec3b>(i, j)[0] + color.at<cv::Vec3b>(i, j)[1] + color.at<cv::Vec3b>(i, j)[2]) / 3);
return gray;
}
The above code worked and overcame out of bounds errors I experienced earlier. Thank you, Rob.

Spectral clusting in DLIB C++

I would like to cluster an BW image using the k means clustering algorithm that I found in the DLIB library and later the spectral clustering algorithm.
The result that i have at the moment is very strange (at least for me) and I would be grateful for any help.
Original image:
The current result for now is:
cv::Mat inputImage = cv::imread("lixo.png");
cv::cvtColor(inputImage, inputImage, CV_RGB2GRAY);
cv::imshow("Display Image", inputImage);
cv::waitKey(0);
// from cv::mat to dlib vector of points
dlib::matrix<double,2,1> sample_point;
std::vector<dlib::matrix<double,2,1>> samples;
for( long i = 0; i < inputImage.rows; ++i) {
for( long j = 0; j < inputImage.cols; ++j ) {
if (inputImage.at<uchar>(i,j) == (uchar)255) {
sample_point(0) = i;
sample_point(1) = j;
samples.push_back(sample_point);
}
}
}
// typedef for the kind of kernel we want to use
typedef dlib::radial_basis_kernel<dlib::matrix<double,2,1>> kernel_type;
// the kcentroid object
dlib::kcentroid<kernel_type> kc(kernel_type(0.1),0.01, 8);
// kkmeans object and tell it to use kcentroid objects
dlib::kkmeans<kernel_type> test(kc);
// tell the kkmeans we want 3 clusters
int nclus = 3;
test.set_number_of_centers(nclus);
// pick some initial centers for the k-means algorithm
std::vector<dlib::matrix<double,2,1>> initial_centers;
pick_initial_centers(nclus,initial_centers, samples,test.get_kernel());
// now run the k-means algorithm on our set of samples
test.train(samples, initial_centers);
// show result
int r = inputImage.rows;
int c = inputImage.cols;
cv::Mat result1 = cv::Mat::zeros(r, c, CV_8UC1);
cv::Mat result2 = cv::Mat::zeros(r, c, CV_8UC1);
cv::Mat result3 = cv::Mat::zeros(r, c, CV_8UC1);
int n1 = 0;
int n2 = 0;
int n3 = 0;
std::cout << " Result" << std::endl;
for (long i = 0; i < samples.size(); ++i) {
sample_point = samples[i];
int result = test(sample_point);
if(result == 0) {
n1++;
result1.at<uchar>(sample_point(0), sample_point(1)) = (uchar)255;
} else if(result == 1) {
n2++;
result2.at<uchar>(sample_point(0), sample_point(1)) = (uchar)255;
} else if(result == 2) {
n3++;
result3.at<uchar>(sample_point(0), sample_point(1)) = (uchar)255;
}
}
cv::imshow("result1", result1);
cv::imshow("result2", result2);
cv::imshow("result3", result3);
cv::waitKey(0);

Create a RGB image from pixel labels

Given a CV_32SC1 cv::Mat image that contains a label for each pixel (where a label is just an index in 0..N-1), what is the cleanest code in OpenCV to generate a CV_8UC3 image that shows each connected component with a different arbitrary color? If I don't have to specify the colors manually, as with cv::floodFill, the better.
If the max number of labels is 256, you can use applyColorMap, converting the image to CV_8U:
Mat1i img = ...
// Convert to CV_8U
Mat1b img2;
img.convertTo(img2, CV_8U);
// Apply color map
Mat3b out;
applyColorMap(img2, out, COLORMAP_JET);
If the number of labels is higher than 256, you need to do it yourself. Below there is an example that generates a JET colormap (it's based on Matlab implementation of the jet function). Then you can apply the colormap for each element of your matrix.
Please note that if you want a different colormap, or random colors, you just need to modify the //Create JET colormap part:
#include <opencv2/opencv.hpp>
#include <algorithm>
using namespace std;
using namespace cv;
void applyCustomColormap(const Mat1i& src, Mat3b& dst)
{
// Create JET colormap
double m;
minMaxLoc(src, nullptr, &m);
m++;
int n = ceil(m / 4);
Mat1d u(n*3-1, 1, double(1.0));
for (int i = 1; i <= n; ++i) {
u(i-1) = double(i) / n;
u((n*3-1) - i) = double(i) / n;
}
vector<double> g(n * 3 - 1, 1);
vector<double> r(n * 3 - 1, 1);
vector<double> b(n * 3 - 1, 1);
for (int i = 0; i < g.size(); ++i)
{
g[i] = ceil(double(n) / 2) - (int(m)%4 == 1 ? 1 : 0) + i + 1;
r[i] = g[i] + n;
b[i] = g[i] - n;
}
g.erase(remove_if(g.begin(), g.end(), [m](double v){ return v > m;}), g.end());
r.erase(remove_if(r.begin(), r.end(), [m](double v){ return v > m; }), r.end());
b.erase(remove_if(b.begin(), b.end(), [](double v){ return v < 1.0; }), b.end());
Mat1d cmap(m, 3, double(0.0));
for (int i = 0; i < r.size(); ++i) { cmap(int(r[i])-1, 2) = u(i); }
for (int i = 0; i < g.size(); ++i) { cmap(int(g[i])-1, 1) = u(i); }
for (int i = 0; i < b.size(); ++i) { cmap(int(b[i])-1, 0) = u(u.rows - b.size() + i); }
Mat3d cmap3 = cmap.reshape(3);
Mat3b colormap;
cmap3.convertTo(colormap, CV_8U, 255.0);
// Apply color mapping
dst = Mat3b(src.rows, src.cols, Vec3b(0,0,0));
for (int r = 0; r < src.rows; ++r)
{
for (int c = 0; c < src.cols; ++c)
{
dst(r, c) = colormap(src(r,c));
}
}
}
int main()
{
Mat1i img(1000,1000);
randu(img, Scalar(0), Scalar(10));
Mat3b out;
applyCustomColormap(img, out);
imshow("Result", out);
waitKey();
return 0;
}

entropy for a gray image in opencv

I need code to find entropy of an image.
for(int i=0;i<grey_image.rows;i++)
{
for(int j=1;j<grey_image.cols;j++)
{
//cout<<i<<" "<<j<<" "<<(int)grey_image.at<uchar>(i,j)<<endl;
int a=(int)grey_image.at<uchar>(i,j);
int b=(int)grey_image.at<uchar>(i,j-1);
int x=a-b;
if(x<0)
x=0-x;
probability_array[x]++;
//grey_image.at<uchar>(i,j) = 255;
}
}
//calculating probability
int n=rows*cols;
for(int i=0;i<256;i++)
{
probability_array[i]/=n;
//cout<<probability_array[i]<<endl;
}
// galeleo team formula
float entropy=0;
for(int i=0;i<256;i++)
{
if (probability_array[i]>0)
{
float x=probability_array[i]*log(probability_array[i]);
entropy+=x;
}
}
return 0-entropy;
Actually I am using this to dump in a programmable camera to measure entropy. Now I want to use it in windows system. I am getting entropy of a gray image as zero.Please help me out. Where did I go wrong.
Without knowing what image are you using, we cannot know if a zero entropy result is not the right answer (as suggested by #Xocoatzin).
Besides, your code can benefit from some of the latest OpenCV features 😊: Here is a working implementation using OpenCV histograms and matrix expressions:
if (frame.channels()==3) cvtColor(frame,frame,CV_BGR2GRAY);
/// Establish the number of bins
int histSize = 256;
/// Set the ranges ( for B,G,R) )
float range[] = { 0, 256 } ;
const float* histRange = { range };
bool uniform = true; bool accumulate = false;
/// Compute the histograms:
calcHist( &frame, 1, 0, Mat(), hist, 1, &histSize, &histRange, uniform, accumulate );
hist /= frame.total();
hist += 1e-4; //prevent 0
Mat logP;
cv::log(hist,logP);
float entropy = -1*sum(hist.mul(logP)).val[0];
cout << entropy << endl;
here is what i m using, hope it helps; https://github.com/samidalati/OpenCV-Entropy you can find couple of methods to calculate the entropy of colored and grayscaled images using OpenCV
float entropy(Mat seq, Size size, int index)
{
int cnt = 0;
float entr = 0;
float total_size = size.height * size.width; //total size of all symbols in an image
for(int i=0;i<index;i++)
{
float sym_occur = seq.at<float>(0, i); //the number of times a sybmol has occured
if(sym_occur>0) //log of zero goes to infinity
{
cnt++;
entr += (sym_occur/total_size)*(log2(total_size/sym_occur));
}
}
cout<<"cnt: "<<cnt<<endl;
return entr;
}
// myEntropy calculates relative occurrence of different symbols within given input sequence using histogram
Mat myEntropy(Mat seq, int histSize)
{
float range[] = { 0, 256 } ;
const float* histRange = { range };
bool uniform = true; bool accumulate = false;
Mat hist;
/// Compute the histograms:
calcHist( &seq, 1, 0, Mat(), hist, 1, &histSize, &histRange, uniform, accumulate );
return hist;
}
enter code here
//Calculate Entropy of 2D histogram
double Sum_prob_1k = 0, Sum_prob_kl = 0, Sum_prob_ln_1k = 0, Sum_prob_ln_kl = 0;
for (int k = start; k < end; k++)
{
Sum_prob_1k = 0; Sum_prob_kl = 0;
Sum_prob_ln_1k = 0; Sum_prob_ln_kl = 0;
//i=1 need to be start = 1
for (int i = 1; i < k; i++)
{
Sum_prob_1k += HiGreyN[i];
if (HiGreyN[i] != 0)
Sum_prob_ln_1k += (HiGreyN[i] * System.Math.Log(HiGreyN[i]));
}
for (int i = k; i < end; i++)
{
Sum_prob_kl += HiGreyN[i];
if (HiGreyN[i] != 0)
Sum_prob_ln_kl += (HiGreyN[i] * System.Math.Log(HiGreyN[i]));
}
//Final equation of entropy for each K
EiGrey[k] = System.Math.Log(Sum_prob_1k) + System.Math.Log(Sum_prob_kl) -
(Sum_prob_ln_1k / Sum_prob_1k) - (Sum_prob_ln_kl / Sum_prob_kl);
if (EiGrey[k] < 0)
EiGrey[k] = 0;
}
//End calculating 2D Entropy

OpenCV Stereo Camera Calibration/Image Rectification

I'm trying to calibrate my two Point Grey (Blackfly) cameras for stereo vision. I'm using the tutorial stereo_calib.cpp that comes with OpenCV (code below). For some reason, I'm getting really bad results (RMS error=4.49756 and average reprojection err = 8.06533) and all my rectified images come out grey. I think my problem is that I'm not picking the right flags for the stereoCalibrate() function, but I've tried many different combinations and at best the rectified images would be warped.
Here's a link to the images I used and a sample rectified pair: https://www.dropbox.com/sh/5wp31o8xcn6vmjl/AAADAfGiaT_NyXEB3zMpcEvVa#/
Any help would be appreciated!!
static void
StereoCalib(const vector<string>& imagelist, Size boardSize, bool useCalibrated=true, bool showRectified=true)
{
if( imagelist.size() % 2 != 0 )
{
cout << "Error: the image list contains odd (non-even) number of elements\n";
return;
}
bool displayCorners = true;//false;//true;
const int maxScale = 1;//2;
const float squareSize = 1.8;
//const float squareSize = 1.f; // Set this to your actual square size
// ARRAY AND VECTOR STORAGE:
vector<vector<Point2f> > imagePoints[2];
vector<vector<Point3f> > objectPoints;
Size imageSize;
//int i, j, k, nimages = (int)imagelist.size()/2;
int i, j, k, nimages = (int)imagelist.size();
cout << "nimages: " << nimages << "\n";
imagePoints[0].resize(nimages);
imagePoints[1].resize(nimages);
vector<string> goodImageList;
for( i = j = 0; i < nimages; i++ )
{
for( k = 0; k < 2; k++ )
{
const string& filename = imagelist[i*2+k];
Mat img = imread(filename, 0);
if(img.empty()) {
break;
}
if( imageSize == Size() ) {
imageSize = img.size();
} else if( img.size() != imageSize )
{
cout << "The image " << filename << " has the size different from the first image size. Skipping the pair\n";
break;
}
bool found = false;
vector<Point2f>& corners = imagePoints[k][j];
for( int scale = 1; scale <= maxScale; scale++ )
{
Mat timg;
if( scale == 1 )
timg = img;
else
resize(img, timg, Size(), scale, scale);
found = findChessboardCorners(timg, boardSize, corners,
CV_CALIB_CB_ADAPTIVE_THRESH | CV_CALIB_CB_NORMALIZE_IMAGE);
if( found )
{
if( scale > 1 )
{
Mat cornersMat(corners);
cornersMat *= 1./scale;
}
break;
}
}
if( displayCorners )
{
cout << filename << endl;
Mat cimg, cimg1;
cvtColor(img, cimg, COLOR_GRAY2BGR);
drawChessboardCorners(cimg, boardSize, corners, found);
double sf = 1280./MAX(img.rows, img.cols);
resize(cimg, cimg1, Size(), sf, sf);
imshow("corners", cimg1);
char c = (char)waitKey(500);
if( c == 27 || c == 'q' || c == 'Q' ) //Allow ESC to quit
exit(-1);
}
else
putchar('.');
if( !found ) {
cout << "!found\n";
break;
}
cornerSubPix(img, corners, Size(11,11), Size(-1,-1),
TermCriteria(CV_TERMCRIT_ITER+CV_TERMCRIT_EPS,
30, 0.01));
}
if( k == 2 )
{
goodImageList.push_back(imagelist[i*2]);
goodImageList.push_back(imagelist[i*2+1]);
j++;
}
}
cout << j << " pairs have been successfully detected.\n";
nimages = j;
if( nimages < 2 )
{
cout << "Error: too little pairs to run the calibration\n";
return;
}
imagePoints[0].resize(nimages);
imagePoints[1].resize(nimages);
objectPoints.resize(nimages);
for( i = 0; i < nimages; i++ )
{
for( j = 0; j < boardSize.height; j++ )
for( k = 0; k < boardSize.width; k++ )
objectPoints[i].push_back(Point3f(j*squareSize, k*squareSize, 0));
}
cout << "Running stereo calibration ...\n";
Mat cameraMatrix[2], distCoeffs[2];
cameraMatrix[0] = Mat::eye(3, 3, CV_64F);
cameraMatrix[1] = Mat::eye(3, 3, CV_64F);
Mat R, T, E, F;
double rms = stereoCalibrate(objectPoints, imagePoints[0], imagePoints[1],
cameraMatrix[0], distCoeffs[0],
cameraMatrix[1], distCoeffs[1],
imageSize, R, T, E, F,
//TermCriteria(CV_TERMCRIT_ITER+CV_TERMCRIT_EPS, 100, 1e-5));
TermCriteria(CV_TERMCRIT_ITER+CV_TERMCRIT_EPS, 100, 1e-5),
CV_CALIB_FIX_ASPECT_RATIO +
//CV_CALIB_ZERO_TANGENT_DIST +
CV_CALIB_SAME_FOCAL_LENGTH +
CV_CALIB_RATIONAL_MODEL +
//CV_CALIB_FIX_K3);
//CV_CALIB_FIX_K2);
CV_CALIB_FIX_K3 + CV_CALIB_FIX_K4 + CV_CALIB_FIX_K5);
//CV_CALIB_FIX_K1 + CV_CALIB_FIX_K2 + CV_CALIB_FIX_K3 + CV_CALIB_FIX_K4 + CV_CALIB_FIX_K5);
cout << "done with RMS error=" << rms << endl;
double err = 0;
int npoints = 0;
vector<Vec3f> lines[2];
for( i = 0; i < nimages; i++ )
{
int npt = (int)imagePoints[0][i].size();
Mat imgpt[2];
for( k = 0; k < 2; k++ )
{
imgpt[k] = Mat(imagePoints[k][i]);
undistortPoints(imgpt[k], imgpt[k], cameraMatrix[k], distCoeffs[k], Mat(), cameraMatrix[k]);
computeCorrespondEpilines(imgpt[k], k+1, F, lines[k]);
}
for( j = 0; j < npt; j++ )
{
double errij = fabs(imagePoints[0][i][j].x*lines[1][j][0] +
imagePoints[0][i][j].y*lines[1][j][1] + lines[1][j][2]) +
fabs(imagePoints[1][i][j].x*lines[0][j][0] +
imagePoints[1][i][j].y*lines[0][j][1] + lines[0][j][2]);
err += errij;
}
npoints += npt;
}
cout << "average reprojection err = " << err/npoints << endl;
// save intrinsic parameters
FileStorage fs("intrinsics.yml", CV_STORAGE_WRITE);
if( fs.isOpened() )
{
fs << "M1" << cameraMatrix[0] << "D1" << distCoeffs[0] <<
"M2" << cameraMatrix[1] << "D2" << distCoeffs[1];
fs.release();
}
else
cout << "Error: can not save the intrinsic parameters\n";
Mat R1, R2, P1, P2, Q;
Rect validRoi[2];
stereoRectify(cameraMatrix[0], distCoeffs[0],
cameraMatrix[1], distCoeffs[1],
imageSize, R, T, R1, R2, P1, P2, Q,
//CALIB_ZERO_DISPARITY, 1, imageSize, &validRoi[0], &validRoi[1]);
CALIB_ZERO_DISPARITY, 0, imageSize, &validRoi[0], &validRoi[1]);
fs.open("extrinsics.yml", CV_STORAGE_WRITE);
if( fs.isOpened() )
{
fs << "R" << R << "T" << T << "R1" << R1 << "R2" << R2 << "P1" << P1 << "P2" << P2 << "Q" << Q;
fs.release();
}
else
cout << "Error: can not save the intrinsic parameters\n";
// OpenCV can handle left-right
// or up-down camera arrangements
//bool isVerticalStereo = fabs(P2.at<double>(1, 3)) > fabs(P2.at<double>(0, 3));
bool isVerticalStereo = false;
// COMPUTE AND DISPLAY RECTIFICATION
if( !showRectified )
return;
Mat rmap[2][2];
// IF BY CALIBRATED (BOUGUET'S METHOD)
if( useCalibrated )
{
// we already computed everything
}
// OR ELSE HARTLEY'S METHOD
else
// use intrinsic parameters of each camera, but
// compute the rectification transformation directly
// from the fundamental matrix
{
vector<Point2f> allimgpt[2];
for( k = 0; k < 2; k++ )
{
for( i = 0; i < nimages; i++ )
std::copy(imagePoints[k][i].begin(), imagePoints[k][i].end(), back_inserter(allimgpt[k]));
}
F = findFundamentalMat(Mat(allimgpt[0]), Mat(allimgpt[1]), FM_8POINT, 0, 0);
Mat H1, H2;
stereoRectifyUncalibrated(Mat(allimgpt[0]), Mat(allimgpt[1]), F, imageSize, H1, H2, 3);
R1 = cameraMatrix[0].inv()*H1*cameraMatrix[0];
R2 = cameraMatrix[1].inv()*H2*cameraMatrix[1];
P1 = cameraMatrix[0];
P2 = cameraMatrix[1];
}
//Precompute maps for cv::remap()
initUndistortRectifyMap(cameraMatrix[0], distCoeffs[0], R1, P1, imageSize, CV_16SC2, rmap[0][0], rmap[0][1]);
initUndistortRectifyMap(cameraMatrix[1], distCoeffs[1], R2, P2, imageSize, CV_16SC2, rmap[1][0], rmap[1][1]);
Mat canvas;
double sf;
int w, h;
if( !isVerticalStereo )
{
sf = 600./MAX(imageSize.width, imageSize.height);
w = cvRound(imageSize.width*sf);
h = cvRound(imageSize.height*sf);
canvas.create(h, w*2, CV_8UC3);
}
else
{
sf = 600./MAX(imageSize.width, imageSize.height);
w = cvRound(imageSize.width*sf);
h = cvRound(imageSize.height*sf);
canvas.create(h*2, w, CV_8UC3);
}
for( i = 0; i < nimages; i++ )
{
for( k = 0; k < 2; k++ )
{
Mat img = imread(goodImageList[i*2+k], 0), rimg, cimg;
remap(img, rimg, rmap[k][0], rmap[k][1], CV_INTER_LINEAR);
cvtColor(rimg, cimg, COLOR_GRAY2BGR);
Mat canvasPart = !isVerticalStereo ? canvas(Rect(w*k, 0, w, h)) : canvas(Rect(0, h*k, w, h));
resize(cimg, canvasPart, canvasPart.size(), 0, 0, CV_INTER_AREA);
if( useCalibrated )
{
Rect vroi(cvRound(validRoi[k].x*sf), cvRound(validRoi[k].y*sf),
cvRound(validRoi[k].width*sf), cvRound(validRoi[k].height*sf));
rectangle(canvasPart, vroi, Scalar(0,0,255), 3, 8);
}
}
if( !isVerticalStereo )
for( j = 0; j < canvas.rows; j += 16 )
line(canvas, Point(0, j), Point(canvas.cols, j), Scalar(0, 255, 0), 1, 8);
else
for( j = 0; j < canvas.cols; j += 16 )
line(canvas, Point(j, 0), Point(j, canvas.rows), Scalar(0, 255, 0), 1, 8);
imshow("rectified", canvas);
char c = (char)waitKey();
if( c == 27 || c == 'q' || c == 'Q' )
break;
}
}
First of all, about your calibration images. I see a few points that could lead to a better calibration :
Use stabler images. Most of your images are blurred a bit, which results in bad accuracy in the corner detection
Vary scale. Most images that you use present the checkerboard approx. at the same distance from the cameras.
Be careful about your checkerboard itself. It appears to be quite badly attached to its support. If you want to achieve a good calibration, you must ensure that your checkerboard is attached tightly on a flat surface.
You have much more detailed advice about how to make a good calibration in this SO answer
Now, about the stereo calibration itself. Best way that I found to achieve a good calibration is to separately calibrate each camera intrinsics (using the calibrateCamera function) then the extrinsics (using stereoCalibrate) using the intrinsics as a guess. Have a look at the stereoCalibrate flags for how to do this.
Outside of this, your flags in the stereoCalibrate function are as such :
CV_CALIB_FIX_ASPECT_RATIO : you force the aspect ratio fx/fy to be fixed
CV_CALIB_SAME_FOCAL_LENGTH : seems OK since you have two identical cameras. You can check whether it is exact by calibrating independently each camera
CV_CALIB_RATIONAL_MODEL : enables K3, k4 and k5 distorsion parameters
CV_CALIB_FIX_K3 + CV_CALIB_FIX_K4 + CV_CALIB_FIX_K5 : fixes those 3 parameters. Since you don't use any uess, you actually put them to 0 here so the option CV_CALIB_RATIONAL_MODEL is no use in your code with those flags
Note that if you calibrate independently each camera and use the intrinsics, you have different levels of use of this data :
With the flag CV_CALIB_FIX_INTRINSIC, the intrinsics will be used as such and only extrinsic parameters will be optimized
With CV_CALIB_USE_INTRINSIC_GUESS, intrinsics will be used as guesses but optimized again
With a combination of CV_CALIB_FIX_PRINCIPAL_POINT, CV_CALIB_FIX_FOCAL_LENGTH and CV_CALIB_FIX_K1,...,CV_CALIB_FIX_K6 you get a little of play about which parameters are fixed and which are optimized again
I had the same problem few weeks back where I tried almost 50 stereo image samples but the rectified image was all blank. So I decided to write my own implementation of stereo calibration code but in real time.
This code detected the chessboard corners in real time and saves those image where chessboard corners in both left and right images is found and then it runs stereo calib code on them.
I hope this helps you and someone in the future.
Source Code: https://github.com/upperwal/opencv/blob/master/samples/cpp/stereo_calib.cpp
Demo Video:
https://www.youtube.com/watch?v=kizO_s-YUDU
All the best
I had similar problem and then I re edited the code . Please make sure you have sufficient number of images at different depth form camera and at different orientation that will lead to less re projection error
#include <opencv2/core/core.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include <opencv2/calib3d/calib3d.hpp>
#include <opencv2/highgui/highgui.hpp>
#include<iostream>
int main()
{
const int CHESSBOARD_WIDTH = 9; //input width of chessboard
const int CHESSBOARD_HEIGHT = 6; //input height of chessboard
const float squareSize = 3.96; //input size of a side of a single square in chessboard
cv::Size corner=cv::Size(CHESSBOARD_WIDTH,CHESSBOARD_HEIGHT);
int counter =30;
int nimages=24;
cv::Size imageSize;
enum{capturing=0,calibrated=1};
int mode=capturing;
char leftfilename[100];
char rightfilename[100];
std::vector<cv::Mat> imagePoints1;
std::vector<cv::Mat> imagePoints2;
std::vector<std::vector<cv::Point3f>> objectPoints;
bool found1=false;
bool found2=false;
int counter2=0;
cv::Mat pointBuf1=cv::Mat::zeros(54,2,CV_32FC1);
cv::Mat pointBuf2=cv::Mat::zeros(54,2,CV_32FC1);
for(int i=1;i<=counter;i++)
{ sprintf(leftfilename,"newleftcheck%d.jpg",i);
sprintf(rightfilename,"newrightcheck%d.jpg",i);
// const int CHESSBOARD_INTERSECTION_COUNT = CHESSBOARD_WIDTH * CHESSBOARD_HEIGHT;
cv::Mat imgleft_frame=cv::imread(leftfilename);
cv::Mat imgright_frame=cv::imread(rightfilename);
//cv::Mat imgleft_frame =cv::Mat(480,640,CV_8UC4,s.leftBuffer,4*imgWidth*sizeof(unsigned char));
//cv::Mat imgright_frame =cv::Mat(480,640,CV_8UC4,s.rightBuffer,4*imgWidth*sizeof(unsigned char));
imageSize=imgleft_frame.size();
found1 = findChessboardCorners(imgleft_frame, corner,pointBuf1,cv::CALIB_CB_ADAPTIVE_THRESH | cv::CALIB_CB_FAST_CHECK | cv::CALIB_CB_NORMALIZE_IMAGE);
found2 = findChessboardCorners(imgright_frame, cv::Size(CHESSBOARD_WIDTH,CHESSBOARD_HEIGHT),pointBuf2,cv::CALIB_CB_ADAPTIVE_THRESH | cv::CALIB_CB_FAST_CHECK | cv::CALIB_CB_NORMALIZE_IMAGE);
if(found1)
{ cv::Mat gray_image1;
cvtColor(imgleft_frame,gray_image1,cv::COLOR_BGRA2GRAY);
cornerSubPix( gray_image1, pointBuf1, cv::Size(11,11),cv::Size(-1,-1), cv::TermCriteria(cv::TermCriteria::EPS+cv::TermCriteria::MAX_ITER, 30, 0.1 ));
drawChessboardCorners( imgleft_frame,cv::Size(CHESSBOARD_WIDTH,CHESSBOARD_HEIGHT), pointBuf1, found1 );
}
if(found2)
{ cv::Mat gray_image2;
cvtColor(imgright_frame,gray_image2,cv::COLOR_BGRA2GRAY);
cornerSubPix( gray_image2, pointBuf2, cv::Size(11,11),cv::Size(-1,-1), cv::TermCriteria(cv::TermCriteria::EPS+cv::TermCriteria::MAX_ITER, 30, 0.1 ));
drawChessboardCorners( imgright_frame,cv::Size(CHESSBOARD_WIDTH,CHESSBOARD_HEIGHT), pointBuf2, found2 );
}
if(found1&&found2)
{ imagePoints1.push_back(pointBuf1);
imagePoints2.push_back(pointBuf2);
//sprintf(leftfilename,"newleftcheck%d.jpg",s.counter);
//sprintf(rightfilename,"newrightcheck%d.jpg",s.counter);
//cv::imwrite(leftfilename,imgleft_frame);
//cv::imwrite(rightfilename,imgright_frame);
counter2=counter2+1;
std::cout<<counter2<<std::endl;
}
nimages=counter2;
objectPoints.resize(nimages);
std::cout<<"countervalue"<<i<<std::endl;
}
for(int i = 0; i <nimages; i++ )
{
for( int j = 0; j < CHESSBOARD_HEIGHT; j++ )
for( int k = 0; k < CHESSBOARD_WIDTH; k++ )
objectPoints[i].push_back(cv::Point3f(j*squareSize, k*squareSize, 0));
}
std::cout<<"check1"<<std::endl;
cv::Mat cameraMatrix[2], distCoeffs[2];
cameraMatrix[0] = cv::Mat::eye(3, 3, CV_64F);
cameraMatrix[1] = cv::Mat::eye(3, 3, CV_64F);
cv::Mat R, T, E, F;
std::cout<<objectPoints.size()<<std::endl;
std::cout<<imagePoints1.size()<<std::endl;
if(imagePoints1.size()==imagePoints2.size())
std::cout<<"samesize"<<std::endl;
if(imagePoints1.size()>=nimages )
{ std::cout<<"check2"<<std::endl;
double rms = stereoCalibrate( objectPoints, imagePoints1, imagePoints2,cameraMatrix[0], distCoeffs[0],
cameraMatrix[1], distCoeffs[1],imageSize, R, T, E, F,
cv::CALIB_FIX_ASPECT_RATIO +cv::CALIB_ZERO_TANGENT_DIST +
cv::CALIB_SAME_FOCAL_LENGTH +cv::CALIB_RATIONAL_MODEL +
cv::CALIB_FIX_K3 +cv::CALIB_FIX_K4 + cv::CALIB_FIX_K5,
cv::TermCriteria(cv::TermCriteria::COUNT+cv::TermCriteria::EPS, 100, 1e-5) );
std::cout<<"check3"<<std::endl;
std::cout << "done with RMS error=" << rms << std::endl;
mode=calibrated;
std::cout<<"calibrated"<<std::endl;
}
if(mode==calibrated)
{
double err = 0;
int npoints = 0;
std::vector<cv::Vec3f> lines[2];
for(int i = 0; i < nimages; i++ )
{
int npt = (int)imagePoints1[i].rows;
std::cout<<npt<<std::endl;
cv:: Mat imgpt1;
cv::Mat imgpt2;
// for(int k = 0; k < 2; k++ )
imgpt1 = cv::Mat(imagePoints1[i]);
undistortPoints(imgpt1, imgpt1, cameraMatrix[0], distCoeffs[0], cv::Mat(), cameraMatrix[0]);
computeCorrespondEpilines(imgpt1, 1, F, lines[0]);
imgpt2 = cv::Mat(imagePoints2[i]);
undistortPoints(imgpt2, imgpt2, cameraMatrix[1], distCoeffs[1], cv::Mat(), cameraMatrix[1]);
computeCorrespondEpilines(imgpt2, 2, F, lines[1]);
std::cout<<"checksdcdufb"<<std::endl;
//std::cout<<"imagepoint"<<imagePoints1[1].at<unsigned int>(1,1)<<std::endl;
/* for(int j = 0; j < npt; j++ )
{
double errij = fabs(imagePoints1[i].at<double>(j,0) *lines[1][j][0] +imagePoints1[i].at<double>(j,1)*lines[1][j][1] + lines[1][j][2]) +fabs(imagePoints2[i].at<double>(j,0)*lines[0][j][0] +
imagePoints2[i].at<double>(j,1)*lines[0][j][1] + lines[0][j][2]);
err += errij;
}
npoints += npt;
}*/
std::cout<<"check8"<<std::endl;
cv::FileStorage fs("intrinsics.xml", cv::FileStorage::WRITE);
if( fs.isOpened() )
{
fs << "M1" << cameraMatrix[0] << "D1" << distCoeffs[0] <<
"M2" << cameraMatrix[1] << "D2" << distCoeffs[1];
fs.release();
}
else
std:: cout << "Error: can not save the intrinsic parameters\n";
cv::Mat R1, R2, P1, P2, Q;
cv::Rect validRoi[2];
stereoRectify(cameraMatrix[0], distCoeffs[0],
cameraMatrix[1], distCoeffs[1],
imageSize, R, T, R1, R2, P1, P2, Q,
cv::CALIB_ZERO_DISPARITY, 1, imageSize, &validRoi[0], &validRoi[1]);
fs.open("extrinsics.xml", cv::FileStorage::WRITE);
if( fs.isOpened() )
{
fs << "R" << R << "T" << T << "R1" << R1 << "R2" << R2 << "P1" << P1 << "P2" << P2 << "Q" << Q;
fs.release();
}
else
std::cout << "Error: can not save the intrinsic parameters\n";
}
//std::cout << "average reprojection err = " << err/npoints <<std::endl;
}
return 0;
}