fisheyestereocalibration opencv 4.5.5 crashes - c++

Question

i’m desperatly trying to get my fisheye stereo camera working. I've got 2 210° cameras.
So far i’ve tried this code, it doesn’t work with his or mine images.
Whatever i do, it always crashes at cv::fisheye::stereoCalibrate with the error: OpenCV(4.5.5) Error: Assertion failed (abs_max < threshold) in cv::fisheye::stereoCalibrate, file ...\opencv-4.5.5\modules\calib3d\src\fisheye.cpp, line 1051. It tells me the error is in fisheye.cpp # line 1051. I put a breakpoint there to check if the condition comes true... in the first 2 loops abs_max<threshold is true, after the third loop it's false. Why does my code crash then?
I’ve tried different PCs with different openCV configurations (always 4.5.5).
I’ve tried to import stereo-parameters from MATLAB, that crashes as well, no matter if i use cv::fisheye:stereoCalibrate or cv::stereoCalibrate.
i appreciate your help :)
The Code:
#include <opencv2/core/core.hpp>
#include <opencv2/calib3d/calib3d.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include <stdio.h>
#include <filesystem>
#include <string>
#include <vector>
#include <string>
#include <algorithm>
#include <iostream>
#include <iterator>
#include <stdio.h>
#include <stdlib.h>
#include <ctype.h>
using namespace std;
using namespace cv;
static int print_help(char** argv)
{
cout <<
" Given a list of chessboard images, the number of corners (nx, ny)\n"
" on the chessboards, and a flag: useCalibrated for \n"
" calibrated (0) or\n"
" uncalibrated \n"
" (1: use stereoCalibrate(), 2: compute fundamental\n"
" matrix separately) stereo. \n"
" Calibrate the cameras and display the\n"
" rectified results along with the computed disparity images. \n" << endl;
cout << "Usage:\n " << argv[0] << " -w=<board_width default=9> -h=<board_height default=6> -s=<square_size default=1.0> <image list XML/YML file default=stereo_calib.xml>\n" << endl;
return 0;
}
static void StereoCalib(const vector<string>& imagelist, Size boardSize, float squareSize, bool displayCorners = false, bool useCalibrated = true, bool showRectified = true)
{
if (imagelist.size() % 2 != 0)
{
cout << "Error: the image list contains odd (non-even) number of elements\n";
return;
}
const int maxScale = 2;
// ARRAY AND VECTOR STORAGE:
vector<vector<Vec2f> > imagePoints[2];
vector<vector<Vec3f> > objectPoints;
Size imageSize;
int i, j, k, nimages = (int)imagelist.size() / 2;
imagePoints[0].resize(nimages);
imagePoints[1].resize(nimages);
vector<string> goodImageList;
for (i = j = 0; i < nimages; i++)
{
for (k = 0; k < 2; k++)
{
const string& filename = imagelist[i * 2 + k];
Mat img = imread(filename, 0);
if (img.empty())
break;
if (imageSize == Size())
imageSize = img.size();
else if (img.size() != imageSize)
{
cout << "The image " << filename << " has the size different from the first image size. Skipping the pair\n";
break;
}
bool found = false;
vector<Vec2f>& corners = imagePoints[k][j];
for (int scale = 1; scale <= maxScale; scale++)
{
Mat timg;
if (scale == 1)
timg = img;
else
resize(img, timg, Size(), scale, scale, INTER_LINEAR_EXACT);
found = findChessboardCorners(timg, boardSize, corners,
CALIB_CB_ADAPTIVE_THRESH | CALIB_CB_NORMALIZE_IMAGE);
if (found)
{
if (scale > 1)
{
Mat cornersMat(corners);
cornersMat *= 1. / scale;
}
break;
}
}
if (displayCorners)
{
cout << filename << endl;
Mat cimg, cimg1;
cvtColor(img, cimg, COLOR_GRAY2BGR);
drawChessboardCorners(cimg, boardSize, corners, found);
double sf = 640. / MAX(img.rows, img.cols);
resize(cimg, cimg1, Size(), sf, sf, INTER_LINEAR_EXACT);
imshow("corners", cimg1);
char c = (char)waitKey(500);
if (c == 27 || c == 'q' || c == 'Q') //Allow ESC to quit
exit(-1);
}
else
putchar('.');
if (!found)
break;
cornerSubPix(img, corners, Size(11, 11), Size(-1, -1),
TermCriteria(TermCriteria::COUNT + TermCriteria::EPS,
30, 0.01));
}
if (k == 2)
{
goodImageList.push_back(imagelist[i * 2]);
goodImageList.push_back(imagelist[i * 2 + 1]);
j++;
}
}
cout << j << " pairs have been successfully detected.\n";
nimages = j;
if (nimages < 2)
{
cout << "Error: too little pairs to run the calibration\n";
return;
}
imagePoints[0].resize(nimages);
imagePoints[1].resize(nimages);
objectPoints.resize(nimages);
for (i = 0; i < nimages; i++)
{
for (j = 0; j < boardSize.height; j++)
for (k = 0; k < boardSize.width; k++)
objectPoints[i].push_back(Point3f(k * squareSize, j * squareSize, 0));
}
/*TEST mit richtigem Format an Vector Punkte*/
cv::FileStorage fs("omni_calib_data.xml", cv::FileStorage::READ);
std::vector<cv::Mat> objectPoints1;
std::vector<cv::Mat> imagePoints1;
cv::Size imgSize;
fs["objectPoints"] >> objectPoints1;
fs["imagePoints"] >> imagePoints1;
fs["imageSize"] >> imgSize;
cout << "Running stereo calibration ...\n";
Mat testL = imread("C:/Users/Moritz/Documents/Flexybot/Code/fisheyeStereoCalibration/imgs/left2.jpg"),
testR = imread("C:/Users/Moritz/Documents/Flexybot/Code/fisheyeStereoCalibration/imgs/right2.jpg");
printf("Starting Calibration\n");
cv::Matx33d K1, K2, R;
cv::Vec3d T;
cv::Vec4d D1, D2;
int flag = 0;
flag |= cv::fisheye::CALIB_RECOMPUTE_EXTRINSIC;
flag |= cv::fisheye::CALIB_CHECK_COND;
flag |= cv::fisheye::CALIB_FIX_SKEW;
//flag |= cv::fisheye::CALIB_FIX_K2;
//flag |= cv::fisheye::CALIB_FIX_K3;
//flag |= cv::fisheye::CALIB_FIX_K4;
cv::fisheye::stereoCalibrate(objectPoints, imagePoints[0], imagePoints[1],
K1, D1, K2, D2, testR.size(), R, T,0,
cv::TermCriteria(TermCriteria::COUNT + TermCriteria::EPS, 100, DBL_EPSILON));
cv::FileStorage fs1("stereo_params.xml", cv::FileStorage::WRITE);
fs1 << "K1" << Mat(K1);
fs1 << "K2" << Mat(K2);
fs1 << "D1" << D1;
fs1 << "D2" << D2;
fs1 << "R" << Mat(R);
fs1 << "T" << T;
printf("Done Calibration\n");
printf("Starting Rectification\n");
cv::Mat R1, R2, P1, P2, Q;
cv::fisheye::stereoRectify(K1, D1, K2, D2, testR.size(), R, T, R1, R2, P1, P2,
Q, CALIB_ZERO_DISPARITY, testR.size(), 0.0, 1.1);
fs1 << "R1" << R1;
fs1 << "R2" << R2;
fs1 << "P1" << P1;
fs1 << "P2" << P2;
fs1 << "Q" << Q;
printf("Done Rectification\n");
Mat rL, rR, tL, tR, kL, dL, kR, dR,pL,pR;
fisheye::calibrate(objectPoints, imagePoints[0], testL.size(), kL, dL, rL, tL, fisheye::CALIB_ZERO_DISPARITY);
fisheye::calibrate(objectPoints, imagePoints[1], testR.size(), kR, dR, rR, tR, fisheye::CALIB_ZERO_DISPARITY);
fisheye::estimateNewCameraMatrixForUndistortRectify(kL, dL, testL.size(), Matx33d::eye(), pL, 1);
fisheye::estimateNewCameraMatrixForUndistortRectify(kR, dR, testR.size(), Matx33d::eye(), pR, 1);
cv::fisheye::undistortImage(testL, testL, kL, dL,pL);
cv::fisheye::undistortImage(testR, testR, kR, dR,pR);
imshow("Links", testL);
imshow("Rechts", testR);
waitKey(0);
return;
}
static bool readStringList(const string& filename, vector<string>& l)
{
l.resize(0);
FileStorage fs(filename, FileStorage::READ);
if (!fs.isOpened())
return false;
FileNode n = fs.getFirstTopLevelNode();
if (n.type() != FileNode::SEQ)
return false;
FileNodeIterator it = n.begin(), it_end = n.end();
for (; it != it_end; ++it)
l.push_back((string)*it);
return true;
}
int main(int argc, char** argv)
{
Size boardSize;
string imagelistfn;
bool showRectified;
cv::CommandLineParser parser(argc, argv, "{w|9|}{h|6|}{s|20.0|}{nr||}{help||}{#input|stereo_calib.xml|}");
if (parser.has("help"))
return print_help(argv);
showRectified = !parser.has("nr");
imagelistfn = samples::findFile(parser.get<string>("#input"));
boardSize.width = parser.get<int>("w");
boardSize.height = parser.get<int>("h");
float squareSize = parser.get<float>("s");
if (!parser.check())
{
parser.printErrors();
return 1;
}
vector<string> imagelist;
bool ok = readStringList(imagelistfn, imagelist);
if (!ok || imagelist.empty())
{
cout << "can not open " << imagelistfn << " or the string list is empty" << endl;
return print_help(argv);
}
StereoCalib(imagelist, boardSize, squareSize, true, true, showRectified);
return 0;
}