Head pose estimation fails with specific image sizes - c++

I want to find angles of rotation of the head using opencv and dlib. So, I tried to use this code from the tutorial:
cv::Mat im = imread("img.jpg");
matrix<bgr_pixel> dlibImage;
assign_image(dlibImage, cv_image<bgr_pixel>(im));
auto face = detector(dlibImage)[0];
auto shape = sp(dlibImage, face);
// 2D image points.
std::vector<cv::Point2d> image_points;
image_points.push_back(cv::Point2d(shape.part(30).x(), shape.part(30).y())); // Nose tip
image_points.push_back(cv::Point2d(shape.part(8).x(), shape.part(8).y())); // Chin
image_points.push_back(cv::Point2d(shape.part(36).x(), shape.part(36).y())); // Left eye left corner
image_points.push_back(cv::Point2d(shape.part(45).x(), shape.part(45).y())); // Right eye right corner
image_points.push_back(cv::Point2d(shape.part(48).x(), shape.part(48).y())); // Left Mouth corner
image_points.push_back(cv::Point2d(shape.part(54).x(), shape.part(54).y())); // Right mouth corner
// 3D model points.
std::vector<cv::Point3d> model_points;
model_points.push_back(cv::Point3d(0.0f, 0.0f, 0.0f)); // Nose tip
model_points.push_back(cv::Point3d(0.0f, -330.0f, -65.0f)); // Chin
model_points.push_back(cv::Point3d(-225.0f, 170.0f, -135.0f)); // Left eye left corner
model_points.push_back(cv::Point3d(225.0f, 170.0f, -135.0f)); // Right eye right corner
model_points.push_back(cv::Point3d(-150.0f, -150.0f, -125.0f)); // Left Mouth corner
model_points.push_back(cv::Point3d(150.0f, -150.0f, -125.0f)); // Right mouth corner
// Camera internals
double focal_length = im.cols; // Approximate focal length.
Point2d center = cv::Point2d(im.cols/2,im.rows/2);
cv::Mat camera_matrix = (cv::Mat_<double>(3,3) << focal_length, 0, center.x, 0 , focal_length, center.y, 0, 0, 1);
cv::Mat dist_coeffs = cv::Mat::zeros(4,1,cv::DataType<double>::type); // Assuming no lens distortion
cout << "Camera Matrix " << endl << camera_matrix << endl ;
// Output rotation and translation
cv::Mat rotation_vector; // Rotation in axis-angle form
cv::Mat translation_vector;
// Solve for pose
cv::solvePnP(model_points, image_points, camera_matrix, dist_coeffs, rotation_vector, translation_vector);
// Project a 3D point (0, 0, 1000.0) onto the image plane.
// We use this to draw a line sticking out of the nose
std::vector<Point3d> nose_end_point3D;
std::vector<Point2d> nose_end_point2D;
nose_end_point3D.push_back(Point3d(0,0,1000.0));
projectPoints(nose_end_point3D, rotation_vector, translation_vector, camera_matrix, dist_coeffs, nose_end_point2D);
for(int i=0; i < image_points.size(); i++)
{
circle(im, image_points[i], 3, Scalar(0,0,255), -1);
}
cv::line(im,image_points[0], nose_end_point2D[0], cv::Scalar(255,0,0), 2);
cout << "Rotation Vector " << endl << rotation_vector << endl;
cout << "Translation Vector" << endl << translation_vector << endl;
cout << nose_end_point2D << endl;
// Display image.
cv::imshow("Output", im);
cv::waitKey(0);
But, unfortunately, I get completely different results depending on the size of the same image!
If I use this img.jpg which has size 299x299 px(many sizes are ok, but we take the nearest), then all ok and I get right result:
Output:
Rotation Vector
[-0,04450161828760668;
-2,133664002574712;
-0,2208024002827168]
But if I use this img.jpg which has size 298x298 px, then I get absolutely wrong result:
Output:
Rotation Vector
[-2,999117288644056;
0,0777816930911016;
-0,7573144061217354]
I also understood that it's due to the coords of the landmarks, not due to size of image, because result are same for the same hardcoded landmarks while sizes of this image are different.
How can I always get a correct pose estimation, as in the first case?
P.S. also, I want to note this problem has very nondeterministic behaviour - now all ok with 298x298, but I get wrong result with 297x297 size.

Related

Perform calibration on fisheye image - cancelling fisheye effect

I'm currently using opencv library with c++, and my goal is to cancel a fisheye effect on an image ("make it plane")
I'm using the function "undistortImage" to cancel the effect but I need before to perform camera calibration in order to find the parameters K, Knew, and D, but I didn't understand exactly the documentation ( link: http://docs.opencv.org/master/db/d58/group__calib3d__fisheye.html#gga37375a2741e88052ce346884dfc9c6a0a0899eaa2f96d6eed9927c4b4f4464e05).
From my understanding, I should give two lists of points and the function "calibrate" is supposed to return the arrays I need. So my question is the following: given a fisheye image, how am I supposed to pick the two lists of points to get the result ? This is for the moment my code, very basic, just takes the picture, display it, performs the undistortion and displays the new image. The elements in the matrix are random, so currently the result is not as expected. Thanks for the answers.
#include "opencv2\core\core.hpp"
#include "opencv2\highgui\highgui.hpp"
#include "opencv2\calib3d\calib3d.hpp"
#include <stdio.h>
#include <iostream>
using namespace std;
using namespace cv;
int main(){
cout << " Usage: display_image ImageToLoadAndDisplay" << endl;
Mat image;
image = imread("C:/Users/Administrator/Downloads/eiffel.jpg", CV_LOAD_IMAGE_COLOR); // Read the file
if (!image.data) // Check for invalid input
{
cout << "Could not open or find the image" << endl;
return -1;
}
cout << "Input image depth: " << image.depth() << endl;
namedWindow("Display window", WINDOW_AUTOSIZE);// Create a window for display.
imshow("Display window", image); // Show our image inside it.
Mat Ka = Mat::eye(3, 3, CV_64F); // Creating distortion matrix
Mat Da = Mat::ones(1, 4, CV_64F);
Mat dstImage(image.rows, image.cols, CV_32F);
cout << "K matrix depth: " << Ka.depth() << endl;
cout << "D matrix depth: " << Da.depth() << endl;
Mat Knew = Mat::eye(3, 3, CV_64F);
std::vector<cv::Vec3d> rvec;
std::vector<cv::Vec3d> tvec;
int flag = 0;
std::vector<Point3d> objectPoints1 = { Point3d(0,0,0), Point3d(1,1,0), Point3d(2,2,0), Point3d(3,3,0), Point3d(4,4,0), Point3d(5,5,0),
Point3d(6,6,0), Point3d(7,7,0), Point3d(3,0,0), Point3d(4,1,0), Point3d(5,2,0), Point3d(6,3,0), Point3d(7,4,0), Point3d(8,5,0), Point3d(5,4,0), Point3d(0,7,0), Point3d(9,7,0), Point3d(9,0,0), Point3d(4,3,0), Point3d(7,2,0)};
std::vector<Point2d> imagePoints1 = { Point(107,84), Point(110,90), Point(116,96), Point(126,107), Point(142,123), Point(168,147),
Point(202,173), Point(232,192), Point(135,69), Point(148,73), Point(165,81), Point(189,93), Point(219,112), Point(248,133), Point(166,119), Point(96,183), Point(270,174), Point(226,56), Point(144,102), Point(206,75) };
std::vector<std::vector<cv::Point2d> > imagePoints(1);
imagePoints[0] = imagePoints1;
std::vector<std::vector<cv::Point3d> > objectPoints(1);
objectPoints[0] = objectPoints1;
fisheye::calibrate(objectPoints, imagePoints, image.size(), Ka, Da, rvec, tvec, flag); // Calibration
cout << Ka<< endl;
cout << Da << endl;
fisheye::undistortImage(image, dstImage, Ka, Da, Knew); // Performing distortion
namedWindow("Display window 2", WINDOW_AUTOSIZE);// Create a window for display.
imshow("Display window 2", dstImage); // Show our image inside it.
waitKey(0); // Wait for a keystroke in the window
return 0;
}
For calibration with cv::fisheye::calibrate you must provide
objectPoints vector of vectors of calibration pattern points in the calibration pattern coordinate space.
This means to provide KNOWN real-world coordinates of the points (must be corresponding points to the ones in imagePoints), but you can choose the coordinate system positon arbitrarily (but carthesian), so you must know your object - e.g. a planar test pattern.
imagePoints vector of vectors of the projections of calibration pattern points
These must be the same points as in objectPoints, but given in image coordinates, so where the projection of the object points hit your image (read/extract the coordinates from your image).
For example, if your camera did capture this image (taken from here ):
you must know the dimension of your testpattern (up to a scale), for example you could choose the top-left corner of the top-left square to be position (0,0,0), the top-right corner of the top-left square to be (1,0,0), and the bottom-left corner of the top-left square to be (1,1,0), so your whole testpattern would be placed on the xy-plane.
Then you could extract these correspondences:
pixel real-world
(144,103) (4,3,0)
(206,75) (7,2,0)
(109,151) (2,5,0)
(253,159) (8,6,0)
for these points (marked red):
The pixel position could be your imagePoints list while the real-world positions could be your objectPoints list.
Does this answer your question?

How to rotate pixels of contour or approximate polygon of contour in OpenCV?

After finding contours in image, consider I have contours pixels and approximate polygon of it.
I want to rotate contours pixels or the approximate polygon of contour with a given angle. Is it possible in OpenCV?
this is how you can rotate an object within the image
this is the input image with known object/contour position (the colored thing there)
int main()
{
cv::Mat input = cv::imread("rotateObjects_input.png");
std::vector<cv::Point> myContour;
myContour.push_back(cv::Point(100,100));
myContour.push_back(cv::Point(150,100));
myContour.push_back(cv::Point(150,300));
myContour.push_back(cv::Point(100,300));
cv::Point2f cog(0,0);
for(unsigned int i=0; i<myContour.size(); ++i)
{
cog = cog + cv::Point2f(myContour[i].x, myContour[i].y);
}
cog = 1.0f/(float)myContour.size()*cog;
std::cout << "center of gravity: " << cog << std::endl;
// create and draw mask
cv::Mat mask = cv::Mat::zeros(input.size(), CV_8UC1);
cv::fillConvexPoly(mask,myContour,255);
// create rotation mat
float angleDEG = 45;
cv::Mat transformation = cv::getRotationMatrix2D(cog,angleDEG,1);
std::cout << transformation << std::endl;
// rotated mask holds the object position after rotation
cv::Mat rotatedMask;
cv::warpAffine(mask,rotatedMask,transformation,input.size());
cv::Mat rotatedInput;
cv::warpAffine(input,rotatedInput,transformation, input.size());
cv::imshow("input",input);
cv::imshow("rotated input",rotatedInput);
cv::imshow("rotated mask",rotatedMask);
// copy rotated object to original image:
cv::Mat output = input.clone();
rotatedInput.copyTo(output, rotatedMask);
cv::imwrite("rotateObjects_beforeHolefilling.png", output);
// now there are pixel left from the old object position.
cv::Mat holePixelMask = mask & (255-rotatedMask);
// we have to fill those pixel with some kind of background...
cv::Mat legalBackground = (255-mask);
//cv::erode(legalBackground,)
// fill holes. here you could try to find some better background color by averaging in neighborhood or sth.
cv::Vec3b lastLegalPixel(0,0,0);
for(unsigned int j=0; j<mask.rows; ++j)
for(unsigned int i=0; i<mask.cols; ++i)
{
if(holePixelMask.at<unsigned char>(j,i))
{
output.at<cv::Vec3b>(j,i) = lastLegalPixel;
}
else
{
if(legalBackground.at<unsigned char>(j,i))
lastLegalPixel = input.at<cv::Vec3b>(j,i);
}
}
cv::imshow("holes before filling", holePixelMask);
cv::imshow("legal background", legalBackground);
cv::imshow("result", output);
cv::waitKey(-1);
return 0;
}
this is the output before hole filling
and this is after hole filling

Create Mat from vector<point2f>

I am extremely new to computer vision and the opencv library.
I've done some googling around to try to find how to make a new image from a vector of Point2fs and haven't found any examples that work. I've seen vector<Point> to Mat but when I use those examples I always get errors.
I'm working from this example and any help would be appreciated.
Code: I pass in occludedSquare.
resize(occludedSquare, occludedSquare, Size(0, 0), 0.5, 0.5);
Mat occludedSquare8u;
cvtColor(occludedSquare, occludedSquare8u, CV_BGR2GRAY);
//convert to a binary image. pixel values greater than 200 turn to white. otherwize black
Mat thresh;
threshold(occludedSquare8u, thresh, 170.0, 255.0, THRESH_BINARY);
GaussianBlur(thresh, thresh, Size(7, 7), 2.0, 2.0);
//Do edge detection
Mat edges;
Canny(thresh, edges, 45.0, 160.0, 3);
//Do straight line detection
vector<Vec2f> lines;
HoughLines( edges, lines, 1.5, CV_PI/180, 50, 0, 0 );
//imshow("thresholded", edges);
cout << "Detected " << lines.size() << " lines." << endl;
// compute the intersection from the lines detected...
vector<Point2f> intersections;
for( size_t i = 0; i < lines.size(); i++ )
{
for(size_t j = 0; j < lines.size(); j++)
{
Vec2f line1 = lines[i];
Vec2f line2 = lines[j];
if(acceptLinePair(line1, line2, CV_PI / 32))
{
Point2f intersection = computeIntersect(line1, line2);
intersections.push_back(intersection);
}
}
}
if(intersections.size() > 0)
{
vector<Point2f>::iterator i;
for(i = intersections.begin(); i != intersections.end(); ++i)
{
cout << "Intersection is " << i->x << ", " << i->y << endl;
circle(occludedSquare8u, *i, 1, Scalar(0, 255, 0), 3);
}
}
//Make new matrix bounded by the intersections
...
imshow("localized", localized);
Should be as simple as
std::vector<cv::Point2f> points;
cv::Mat image(points);
//or
cv::Mat image = cv::Mat(points)
The probably confusion is that a cv::Mat is an image width*height*number of channels but it also a mathematical matrix , rows*columns*other dimension.
If you make a Mat from a vector of 'n' 2D points it will create a 2 column by 'n' rows matrix. You are passing this to a function which expects an image.
If you just have a scattered set of 2D points and want to display them as an image you need to make an empty cv::Mat of large enough size (whatever your maximum x,y point is) and then draw the dots using the drawing functions http://docs.opencv.org/doc/tutorials/core/basic_geometric_drawing/basic_geometric_drawing.html
If you just want to set the pixel values at those point coordinates search SO for opencv setting pixel values, there are lots of answers
Martin's answer is right but IMO it depends on how image cv::Mat is used further along the line. I had some issues and Haofeng's comment helped me fix them. Here is my attempt to explain it in detail:
Let's say the code looks like this:
std::vector<cv::Point2f> points = {cv::Point2f(1.0, 2.0), cv::Point2f(3.0, 4.0), cv::Point2f(5.0, 6.0), cv::Point2f(7.0, 8.0), cv::Point2f(9.0, 10.0)};
cv::Mat image(points); // or cv::Mat image = cv::Mat(points)
std::cout << image << std::endl;
This will print:
[1, 2;
3, 4;
5, 6;
7, 8;
9, 10]
So, at first glance, this looks perfectly correct and as expected: for the five 2D points in the given vector, we got a cv::Mat with 5 rows and 2 columns, right? However, that's not the case here!
If further properties are inspected:
std::cout << image.rows << std::endl; // 5
std::cout << image.cols << std::endl; // 1
std::cout << image.channels() << std::endl; // 2
it can be seen that the above cv::Mat has 5 rows, 1 column, and 2 channels. Depending on the pipeline, we may not want that. Most of the time, we want a matrix with 5 rows, 2 columns, and just 1 channel.
To fix this problem, all we need to do is reshape the matrix:
cv::Mat image(points).reshape(1);
In the above code, 1 is for 1 channel. Check out OpenCV reshape() documentation for further information.
If this matrix is printed out, it will look the same as the previous one. However, that's not the whole picture (metaphorically!) The new matrix has 5 rows, 2 columns, and 1 channel.
I wish OpenCV had different ways of printing out these two similar yet different matrices (from the OpenCV data structure point of view)!

get the center of the detected circle in an image / opencv / c++

I am using this code to get the coordinate of the center of detected circles in the image.
vector<Vec3f> circles;
cv::HoughCircles( t, circles, CV_HOUGH_GRADIENT, 1, t.rows/8, 200, 100, 0, 0 );
for( size_t i = 0; i < circles.size(); i++ ){
Point center(cvRound(circles[i][0]), cvRound(circles[i][1]));
cout << "center" << center << endl;
int radius = cvRound(circles[i][2]);
// circle center
circle( t2, center, 3, 1 , -1, 8, 0 );
// circle outline
circle( t2, center, radius, 1 , 3, 8, 0 );
}
imshow( "circles", t2 );
I can detect the circles but did no get any result for cooardinate of the center points :(
thanks in advance.
after edition:
I added this line but the answer was zero.
cout << "number of circles found: " << circles.size() << endl;
images:
the first one is the main circle and the second one is after applying gaussian filter and HoughCircles function:
If I understand you correctly your code Draws the circles but the
cout << "center" << center << endl;
line does not give the correct output.
This is because cv::Point does not support direct output via <<.
Try to use:
cout << "center" << center.x << ", " << center.y << endl;
If the Problem is that you canĀ“t find any circles make sure that the min_radius and max_radius are choosen correctly. Try starting with a wide range of allowed radii and then try to choose a smaller Range until you get only the circles you want.
This Values can make a huge difference in the detection Ratio.

OpenCV's fitEllipse() sometimes returns completely wrong ellipses

My goal is to recognize all the shapes present in an image.
The idea is:
Extract contours
Fit each contour with different shapes
The correct shape should be the one with area closest to the
contour's area.
Example image:
I use fitEllipse() to find the best fit ellipse to the contours, but the result is a bit messy:
The likely-correct ellipses are filled with blue, and the bounding ellipses are yellow.
The likely-incorrect contours are filled with green, and the (wrong) bounding ellipses are cyan.
As you can see, the ellipse bounding the triangle in the first row looks pretty good for the best fit. The bounding ellipse of the triangle in the third row doesn't seem to be the best fit, but still acceptable as a criteria for rejecting an incorrect ellipse.
But I can't understand why the remaining triangles have bounding ellipse completely outside their contour.
And the worst case is the third triangle in the last row: The ellipse is completely wrong but it happens to have the area close to the contour's area, so the triangle is wrongly recognized as an ellipse.
Do I miss anything? My code:
#include <iostream>
#include <opencv/cv.h>
#include <opencv/highgui.h>
using namespace std;
using namespace cv;
void getEllipses(vector<vector<Point> >& contours, vector<RotatedRect>& ellipses) {
ellipses.clear();
Mat img(Size(800,500), CV_8UC3);
for (unsigned i = 0; i<contours.size(); i++) {
if (contours[i].size() >= 5) {
RotatedRect temp = fitEllipse(Mat(contours[i]));
if (area(temp) <= 1.1 * contourArea(contours[i])) {
//cout << area(temp) << " < 1.1* " << contourArea(contours[i]) << endl;
ellipses.push_back(temp);
drawContours(img, contours, i, Scalar(255,0,0), -1, 8);
ellipse(img, temp, Scalar(0,255,255), 2, 8);
imshow("Ellipses", img);
waitKey();
} else {
//cout << "Reject ellipse " << i << endl;
drawContours(img, contours, i, Scalar(0,255,0), -1, 8);
ellipse(img, temp, Scalar(255,255,0), 2, 8);
imshow("Ellipses", img);
waitKey();
}
}
}
}
int main() {
Mat img = imread("image.png", CV_8UC1);
threshold(img, img, 127,255,CV_THRESH_BINARY);
vector<vector<Point> > contours;
vector<Vec4i> hierarchy;
findContours(img, contours, hierarchy, CV_RETR_EXTERNAL, CV_CHAIN_APPROX_SIMPLE);
vector<RotatedRect> ellipses;
getEllipses(contours, ellipses);
return 0;
}
Keep in mind, that fitEllipse is not the computation of a boundingEllipse but a least square optimization that assumes the points to lie on an ellipse.
I can't tell you why it fails on the 3 triangles in the last row so badly but "works" on the triangle one line above, but one thing I've seen is, that all 3 triangles in the last row were fitted to a rotatedRect with angle 0. Probably the least square fitting just failed there.
But I don't know whether there is a bug in the openCV implementation, or wether the algorithm can't handle those cases. This algorithm is used: http://www.bmva.org/bmvc/1995/bmvc-95-050.pdf
My advice is, to only use fitEllipse if you are quite sure that the points really belong to an ellipse. You wont either assume to get reasonable results from fitLine if you have random data points. Other functions you might want to look at are: minAreaRect and minEnclosingCircle
if you use RotatedRect temp = minAreaRect(Mat(contours[i])); instead of fitEllipse you will get an image like this:
maybe you can even use both methods and refuse all ellipses that fail in both versions and accept all that are accepted in both versions, but investigate further in the ones that differ?!?
Changing cv::CHAIN_APPROX_SIMPLE to cv::CHAIN_APPROX_NONE
in the call to cv::findContours() gives me much more reasonable results.
It makes sense that we would get a better ellipse approximation with more points included in the contour but I am still not sure why the results are so off with the simple chain approximation. See opencv docs for explanation of the difference
It appears that when using cv::CHAIN_APPROX_SIMPLE, the relatively horizontal edges of the triangles are almost completely removed from the contour.
As to your classification of best fit, as others have pointed out, using only the area will give you the results you observe as positioning is not taken into account at all.
If you are having problems with cv::fitEllipse(), this post discuss a few methods to minimize those errors that happen when the cv::RotatedRect is draw directly without any further tests. Turns out cv::fitEllipse() is not perfect and can have issues as noted in the question.
Now, it's not entirely clear what the constraints of the project are, but another way to solve this problem is to separate these shapes based on the area of the contours:
This approach is extremely simple yet efficient on this specific case: the area of a circle varies between 1300-1699 and the area of a triangle between 1-1299.
#include "opencv2/highgui/highgui.hpp"
#include "opencv2/imgproc/imgproc.hpp"
#include <iostream>
int main()
{
cv::Mat img = cv::imread("input.png");
if (img.empty())
{
std::cout << "!!! Failed to open image" << std::endl;
return -1;
}
/* Convert to grayscale */
cv::Mat gray;
cv::cvtColor(img, gray, cv::COLOR_BGR2GRAY);
/* Convert to binary */
cv::Mat thres;
cv::threshold(gray, thres, 127, 255, cv::THRESH_BINARY);
/* Find contours */
std::vector<std::vector<cv::Point> > contours;
cv::findContours(thres, contours, cv::RETR_LIST, cv::CHAIN_APPROX_SIMPLE);
int circles = 0;
int triangles = 0;
for (size_t i = 0; i < contours.size(); i++)
{
// Draw a contour based on the size of its area:
// - Area > 0 and < 1300 means it's a triangle;
// - Area >= 1300 and < 1700 means it's a circle;
double area = cv::contourArea(contours[i]);
if (area > 0 && area < 1300)
{
std::cout << "* Triangle #" << ++triangles << " area: " << area << std::endl;
cv::drawContours(img, contours, i, cv::Scalar(0, 255, 0), -1, 8); // filled (green)
cv::drawContours(img, contours, i, cv::Scalar(0, 0, 255), 2, 8); // outline (red)
}
else if (area >= 1300 && area < 1700)
{
std::cout << "* Circle #" << ++circles << " area: " << area << std::endl;
cv::drawContours(img, contours, i, cv::Scalar(255, 0, 0), -1, 8); // filled (blue)
cv::drawContours(img, contours, i, cv::Scalar(0, 0, 255), 2, 8); // outline (red)
}
else
{
std::cout << "* Ignoring area: " << area << std::endl;
continue;
}
cv::imshow("OBJ", img);
cv::waitKey(0);
}
cv::imwrite("output.png", img);
return 0;
}
You can invoke other functions to draw more precise outline (borders) of the shapes.
It may be a better idea to get a pixel-by-pixel comparison i.e. what percentage is the overlap between the contour and the "fitted" ellipse.
Another, simpler idea is to also compare the centroids of the contour and its ellipse fit.