Related
I’m using a modified version of a gauss-newton method to refine a pose estimate using OpenCV. The unmodified code can be found here: http://people.rennes.inria.fr/Eric.Marchand/pose-estimation/tutorial-pose-gauss-newton-opencv.html
The details of this approach are outlined in the corresponding paper:
Marchand, Eric, Hideaki Uchiyama, and Fabien Spindler. "Pose
estimation for augmented reality: a hands-on survey." IEEE
transactions on visualization and computer graphics 22.12 (2016):
2633-2651.
A PDF can be found here: https://hal.inria.fr/hal-01246370/document
The part that is relevant (Pages 4 and 5) are screencapped below:
Here is what I have done. First, I’ve (hopefully) “corrected” some errors: (a) dt and dR can be passed by reference to exponential_map() (even though cv::Mat is essentially a pointer). (b) The last entry of each 2x6 Jacobian matrix, J.at<double>(i*2+1,5), was -x[i].y but should be -x[i].x. (c) I’ve also tried using a different formula for the projection. Specifically, one that includes the focal length and principal point:
xq.at<double>(i*2,0) = cx + fx * cX.at<double>(0,0) / cX.at<double>(2,0);
xq.at<double>(i*2+1,0) = cy + fy * cX.at<double>(1,0) / cX.at<double>(2,0);
Here is the relevant code I am using, in its entirety (control starts at optimizePose3()):
void exponential_map(const cv::Mat &v, cv::Mat &dt, cv::Mat &dR)
{
double vx = v.at<double>(0,0);
double vy = v.at<double>(1,0);
double vz = v.at<double>(2,0);
double vtux = v.at<double>(3,0);
double vtuy = v.at<double>(4,0);
double vtuz = v.at<double>(5,0);
cv::Mat tu = (cv::Mat_<double>(3,1) << vtux, vtuy, vtuz); // theta u
cv::Rodrigues(tu, dR);
double theta = sqrt(tu.dot(tu));
double sinc = (fabs(theta) < 1.0e-8) ? 1.0 : sin(theta) / theta;
double mcosc = (fabs(theta) < 2.5e-4) ? 0.5 : (1.-cos(theta)) / theta / theta;
double msinc = (fabs(theta) < 2.5e-4) ? (1./6.) : (1.-sin(theta)/theta) / theta / theta;
dt.at<double>(0,0) = vx*(sinc + vtux*vtux*msinc)
+ vy*(vtux*vtuy*msinc - vtuz*mcosc)
+ vz*(vtux*vtuz*msinc + vtuy*mcosc);
dt.at<double>(1,0) = vx*(vtux*vtuy*msinc + vtuz*mcosc)
+ vy*(sinc + vtuy*vtuy*msinc)
+ vz*(vtuy*vtuz*msinc - vtux*mcosc);
dt.at<double>(2,0) = vx*(vtux*vtuz*msinc - vtuy*mcosc)
+ vy*(vtuy*vtuz*msinc + vtux*mcosc)
+ vz*(sinc + vtuz*vtuz*msinc);
}
void optimizePose3(const PoseEstimation &pose,
std::vector<FeatureMatch> &feature_matches,
PoseEstimation &optimized_pose) {
//Set camera parameters
double fx = camera_matrix.at<double>(0, 0); //Focal length
double fy = camera_matrix.at<double>(1, 1);
double cx = camera_matrix.at<double>(0, 2); //Principal point
double cy = camera_matrix.at<double>(1, 2);
auto inlier_matches = getInliers(pose, feature_matches);
std::vector<cv::Point3d> wX;
std::vector<cv::Point2d> x;
const unsigned int npoints = inlier_matches.size();
cv::Mat J(2*npoints, 6, CV_64F);
double lambda = 0.25;
cv::Mat xq(npoints*2, 1, CV_64F);
cv::Mat xn(npoints*2, 1, CV_64F);
double residual=0, residual_prev;
cv::Mat Jp;
for(auto i = 0u; i < npoints; i++) {
//Model points
const cv::Point2d &M = inlier_matches[i].model_point();
wX.emplace_back(M.x, M.y, 0.0);
//Imaged points
const cv::Point2d &I = inlier_matches[i].image_point();
xn.at<double>(i*2,0) = I.x; // x
xn.at<double>(i*2+1,0) = I.y; // y
x.push_back(I);
}
//Initial estimation
cv::Mat cRw = pose.rotation_matrix;
cv::Mat ctw = pose.translation_vector;
int nIters = 0;
// Iterative Gauss-Newton minimization loop
do {
for (auto i = 0u; i < npoints; i++) {
cv::Mat cX = cRw * cv::Mat(wX[i]) + ctw; // Update cX, cY, cZ
// Update x(q)
//xq.at<double>(i*2,0) = cX.at<double>(0,0) / cX.at<double>(2,0); // x(q) = cX/cZ
//xq.at<double>(i*2+1,0) = cX.at<double>(1,0) / cX.at<double>(2,0); // y(q) = cY/cZ
xq.at<double>(i*2,0) = cx + fx * cX.at<double>(0,0) / cX.at<double>(2,0);
xq.at<double>(i*2+1,0) = cy + fy * cX.at<double>(1,0) / cX.at<double>(2,0);
// Update J using equation (11)
J.at<double>(i*2,0) = -1 / cX.at<double>(2,0); // -1/cZ
J.at<double>(i*2,1) = 0;
J.at<double>(i*2,2) = x[i].x / cX.at<double>(2,0); // x/cZ
J.at<double>(i*2,3) = x[i].x * x[i].y; // xy
J.at<double>(i*2,4) = -(1 + x[i].x * x[i].x); // -(1+x^2)
J.at<double>(i*2,5) = x[i].y; // y
J.at<double>(i*2+1,0) = 0;
J.at<double>(i*2+1,1) = -1 / cX.at<double>(2,0); // -1/cZ
J.at<double>(i*2+1,2) = x[i].y / cX.at<double>(2,0); // y/cZ
J.at<double>(i*2+1,3) = 1 + x[i].y * x[i].y; // 1+y^2
J.at<double>(i*2+1,4) = -x[i].x * x[i].y; // -xy
J.at<double>(i*2+1,5) = -x[i].x; // -x
}
cv::Mat e_q = xq - xn; // Equation (7)
cv::Mat Jp = J.inv(cv::DECOMP_SVD); // Compute pseudo inverse of the Jacobian
cv::Mat dq = -lambda * Jp * e_q; // Equation (10)
cv::Mat dctw(3, 1, CV_64F), dcRw(3, 3, CV_64F);
exponential_map(dq, dctw, dcRw);
cRw = dcRw.t() * cRw; // Update the pose
ctw = dcRw.t() * (ctw - dctw);
residual_prev = residual; // Memorize previous residual
residual = e_q.dot(e_q); // Compute the actual residual
std::cout << "residual_prev: " << residual_prev << std::endl;
std::cout << "residual: " << residual << std::endl << std::endl;
nIters++;
} while (fabs(residual - residual_prev) > 0);
//} while (nIters < 30);
optimized_pose.rotation_matrix = cRw;
optimized_pose.translation_vector = ctw;
cv::Rodrigues(optimized_pose.rotation_matrix, optimized_pose.rotation_vector);
}
Even when I use the functions as given, it does not produce the correct results. My initial pose estimate is very close to optimal, but when I try run the program, the method takes a very long time to converge - and when it does, the results are very wrong. I’m not sure what could be wrong and I’m out of ideas. I’m confident my inliers are actually inliers (they were chosen using an M-estimator). I’ve compared the results from exponential map with those from other implementations, and they seem to agree.
So, where is the error in this gauss-newton implementation for pose optimization? I’ve tried to make things as easy as possible for anyone willing to lend a hand. Let me know if there is anymore information I can provide. Any help would be greatly appreciated. Thanks.
Edit: 2019/05/13
There is now solvePnPRefineVVS function in OpenCV.
Also, you should use x and y calculated from the current estimated pose instead.
In the cited paper, they expressed the measurements x in the normalized camera frame (at z=1).
When working with real data, you have:
(u,v): 2D image coordinates (e.g. keypoints, corner locations, etc.)
K: the intrinsic parameters (obtained after calibrating the camera)
D: the distortion coefficients (obtained after calibrating the camera)
To compute the 2D image coordinates in the normalized camera frame, you can use in OpenCV the function cv::undistortPoints() (link to my answer about cv::projectPoints() and cv::undistortPoints()).
When there is no distortion, the computation (also called "reverse perspective transformation") is:
x = (u - cx) / fx
y = (v - cy) / fy
I am currently trying some simple line rotation in sdl and I have found a function that rotates a point around another point. Here it is ( changed to work with sdl ).
SDL_Point rotate_point(double cx, double cy, double angle, SDL_Point p)
{
double pi = acos(-1);
double rotation_angle = (double)angle / 180.0 * pi;
double s = sin(rotation_angle);
double c = cos(rotation_angle);
// translate point back to origin:
p.x -= cx;
p.y -= cy;
// rotate point
double xnew = p.x * c - p.y * s;
double ynew = p.x * s + p.y * c;
// translate point back:
p.x = xnew + cx;
p.y = ynew + cy;
return p;
}
Now im having a problem where the line gets shorter and shorter until it stops at the center point im rotating it around. I dont know why.
Here is the way im using the function:
double angle1=0.1, angle2=0,oldangle1=0,oldangle2=0;
SDL_Point line11 = { 10,0 }, line12 = { 110,0 }, line21 = { 110,0 }, line22 = { 210,0 };
SDL_Event event;
int mousex = 0, mousey = 0;
while (SDL_PollEvent(NULL)) {
SDL_GetMouseState(&mousex, &mousey);
if (GetAsyncKeyState('W'))
angle1 += 0.1;
SDL_SetRenderDrawColor(renderer, 0, 0, 0, 0);
SDL_RenderClear(renderer);
if (angle1 < oldangle1-0.1||angle1 > oldangle1 + 0.1) {
line12 = rotate_point(0, 0, (angle1-oldangle1), line12);
line21 = rotate_point(0, 0, (angle1 - oldangle1), line21);
oldangle1 = angle1;
}
if (angle2 < oldangle2 - 0.1 || angle2 > oldangle2 + 0.1) {
line22 = rotate_point(line21.x,line21.y, (angle2 - oldangle2), line22);
oldangle2 = angle2;
}
if (angle2 == 0)
angle2 = angle1;
printf("%f ", angle2);
SDL_SetRenderDrawColor(renderer, 0, 255, 0, 0);
SDL_RenderDrawLine(renderer, line11.x, line11.y, line12.x, line12.y);
SDL_RenderDrawLine(renderer, line21.x, line21.y, line22.x, line22.y);
//solveIK(&angle1, &angle2, false, 100, 100, mousex, mousey);
//angle1 = get_degrees(angle1);
//angle2 = get_degrees(angle2);
SDL_RenderPresent(renderer);
SDL_Delay(16);
}
I know that it is incredibly messy but ive just started with sdl and c++. i will try to fix it up after ive gotten it to work.
What ive already tried:
Different functions for rotating. They all seem to be worse in terms of the line shortening. Its most likely not a problem with the rotation function.
Rewriting the code. This is version 3 and it works the best but is also the worst looking.
Do not rerotate.
Start with the initial points. Now apply a rotation.
If you have a new rotation, start with the initial points and apply a new rotation. Don't rotate the result of the first rotation.
In general, floating point operations induce error away from the theoretical result. Rounding to integers even more error (not sure if your code does that). Cascading large numbers of floating point ops will result in garbage sometimes.
Now, adding angles will also have rounding error, but not in a noticable way: it will have rotated more/less than it should have, not stretch/shrink the line. Again here you can replace repeated addition with a fixed number of multiplications to reduce this, if you care.
I am currently working on a project of object tracking and have used c++ , opencv . I have succesfully used Farneback dense optical flow to implement segmentation methods such as k means (using the displacement in each frame) . Now i want to do the same thing with Lucas Kanade sparse method. But the output of this function is :
nextPts – output vector of 2D points (with single-precision floating-point coordinates) containing the calculated new positions of input features in the second image; when OPTFLOW_USE_INITIAL_FLOW flag is passed, the vector must have the same size as in the input.
(as stated in the official site)
My question is how i am going to get the result to a Mat flow for example. I have so far tried :
// Implement Lucas Kanade algorithm
cvCalcOpticalFlowPyrLK(frame1_1C, frame2_1C, pyramid1, pyramid2,
frame1_features, frame2_features, number_of_features,
optical_flow_window, 5, optical_flow_found_feature,
optical_flow_feature_error, optical_flow_termination_criteria,
0);
// Calculate each feature point's coordinates in every frame
CvPoint p,q;
p.x = (int) frame1_features[i].x;
p.y = (int) frame1_features[i].y;
q.x = (int) frame2_features[i].x;
q.y = (int) frame2_features[i].y;
// Creating the arrows for imshow
angle = atan2((double) p.y - q.y, (double) p.x - q.x);
hypotenuse = sqrt(square(p.y - q.y) + square(p.x - q.x));
/* Here we lengthen the arrow by a factor of three. */
q.x = (int) (p.x - 3 * hypotenuse * cos(angle));
q.y = (int) (p.y - 3 * hypotenuse * sin(angle));
cvLine(frame1, p, q, line_color, line_thickness, CV_AA, 0);
p.x = (int) (q.x + 9 * cos(angle + pi / 4));
p.y = (int) (q.y + 9 * sin(angle + pi / 4));
cvLine(frame1, p, q, line_color, line_thickness, CV_AA, 0);
p.x = (int) (q.x + 9 * cos(angle - pi / 4));
p.y = (int) (q.y + 9 * sin(angle - pi / 4));
cvLine(frame1, p, q, line_color, line_thickness, CV_AA, 0);
allocateOnDemand(&framenew, frame_size, IPL_DEPTH_8U, 3);
cvConvertImage(frame1, framenew, CV_CVTIMG_FLIP);
cvShowImage("Optical Flow", framenew);
This is the optical flow presentation. Any ideas how I should get a Mat flow similar to the result of Farneback optical flow ?
(http://docs.opencv.org/2.4/modules/video/doc/motion_analysis_and_object_tracking.html#calcopticalflowfarneback )
UPDATE : Very good answer. But now i have problems with showing the kmeans image. With farneback i used :
cv::kmeans(m, K, bestLabels,
TermCriteria( CV_TERMCRIT_EPS + CV_TERMCRIT_ITER, 10, 1.0),
3, KMEANS_PP_CENTERS, centers);
int colors[K];
for (int i = 0; i < K; i++) {
colors[i] = 255 / (i + 1);
}
namedWindow("Kmeans", WINDOW_NORMAL);
Mat clustered = Mat(flow.rows, flow.cols, CV_32F);
for (int i = 0; i < flow.cols * flow.rows; i++) {
clustered.at<float>(i / flow.cols, i % flow.cols) =
(float) (colors[bestLabels.at<int>(0, i)]);
}
clustered.convertTo(clustered, CV_8U);
imshow("Kmeans", clustered);
Any ideas ? ?
To get an image like the Farneback algorithm you must first understand what is the output.
In OpenCV docs you have:
prev(y,x) ~ next(y + flow(y,x)[1], x +flow(y,x)[0])
So, it is a matrix with the displacements between image 1 and 2. Assuming that the points that you are not calculating will be without movement 0,0; you can simulate this, you only have to put for each of the points (x,y) having the new position (x', y'):
cv::Mat LKFlowMatrix(img.rows, img.cols, CV_32FC2, cv::Scalar(0,0));
LKFlowMatrix.at<cv::Vec2f>(y,x) = cv::Vec2f(x-x', y-y') ;
Also, don't forget to filter the "not found points" with the status = 0
By The Way, your functions are not the opencv c++ version of it:
cvCalcOpticalFlowPyrLK should be cv::calcOpticalFlowFarneback in c++
cvShowImageshould be cv::imshowin c++
and so on
** UPDATE **
Since you need is an input for kmeans (I suppose that is the OpenCV version), and you want to use only the Sparse Points, then you can do something like this:
cv::Mat prevImg, nextImg;
// load your images
std::vector<cv:Point2f> initial_points, new_points;
// fill the initial points vector
std::vector<uchar> status;
std::vector<float> error;
cv::calcOpticalFlowPyrLK(prevImage, nextImage, initial_points, new_points, status, errors);
std::vector<cv::Vec2f> vectorForKMeans;
for(size_t t = 0; t < status.size(); t++){
if(status[t] != 0)
vectorForKmeans.push_back(cv::Vec2f(initial_points[i] - new_points[i]));
}
// Do kmeans to vectorForKMeans
I detected lines in an image and drew them in a separate image file in OpenCv C++ using HoughLinesP method. Following is a part of that resulting image. There are actually hundreds of small and thin lines which form a big single line.
But I want single few lines that represent all those number of lines. Closer lines should be merged together to form a single line. For example above set of lines should be represented by just 3 separate lines as below.
The expected output is as above. How to accomplish this task.
Up to now progress result from akarsakov's answer.
(separate classes of lines resulted are drawn in different colors). Note that this result is the original complete image I am working on, but not the sample section I had used in the question
If you don't know the number of lines in the image you can use the cv::partition function to split lines on equivalency group.
I suggest you the following procedure:
Split your lines using cv::partition. You need to specify a good predicate function. It really depends on lines which you extract from image, but I think it should check following conditions:
Angle between lines should be quite small (less 3 degrees, for example). Use dot product to calculate angle's cosine.
Distance between centers of segments should be less than half of maximum length of two segments.
For example, it can be implemented as follows:
bool isEqual(const Vec4i& _l1, const Vec4i& _l2)
{
Vec4i l1(_l1), l2(_l2);
float length1 = sqrtf((l1[2] - l1[0])*(l1[2] - l1[0]) + (l1[3] - l1[1])*(l1[3] - l1[1]));
float length2 = sqrtf((l2[2] - l2[0])*(l2[2] - l2[0]) + (l2[3] - l2[1])*(l2[3] - l2[1]));
float product = (l1[2] - l1[0])*(l2[2] - l2[0]) + (l1[3] - l1[1])*(l2[3] - l2[1]);
if (fabs(product / (length1 * length2)) < cos(CV_PI / 30))
return false;
float mx1 = (l1[0] + l1[2]) * 0.5f;
float mx2 = (l2[0] + l2[2]) * 0.5f;
float my1 = (l1[1] + l1[3]) * 0.5f;
float my2 = (l2[1] + l2[3]) * 0.5f;
float dist = sqrtf((mx1 - mx2)*(mx1 - mx2) + (my1 - my2)*(my1 - my2));
if (dist > std::max(length1, length2) * 0.5f)
return false;
return true;
}
Guess you have your lines in vector<Vec4i> lines;. Next, you should call cv::partition as follows:
vector<Vec4i> lines;
std::vector<int> labels;
int numberOfLines = cv::partition(lines, labels, isEqual);
You need to call cv::partition once and it will clusterize all lines. Vector labels will store for each line label of cluster to which it belongs. See documentation for cv::partition
After you get all groups of line you should merge them. I suggest calculating average angle of all lines in group and estimate "border" points. For example, if angle is zero (i.e. all lines are almost horizontal) it would be the left-most and right-most points. It remains only to draw a line between this points.
I noticed that all lines in your examples are horizontal or vertical. In such case you can calculate point which is average of all segment's centers and "border" points, and then just draw horizontal or vertical line limited by "border" points through center point.
Please note that cv::partition takes O(N^2) time, so if you process a huge number of lines it may take a lot of time.
I hope it will help. I used such approach for similar task.
First off I want to note that your original image is at a slight angle, so your expected output seems just a bit off to me. I'm assuming you are okay with lines that are not 100% vertical in your output because they are slightly off on your input.
Mat image;
Mat binary = image > 125; // Convert to binary image
// Combine similar lines
int size = 3;
Mat element = getStructuringElement( MORPH_ELLIPSE, Size( 2*size + 1, 2*size+1 ), Point( size, size ) );
morphologyEx( mask, mask, MORPH_CLOSE, element );
So far this yields this image:
These lines are not at 90 degree angles because the original image is not.
You can also choose to close the gap between the lines with:
Mat out = Mat::zeros(mask.size(), mask.type());
vector<Vec4i> lines;
HoughLinesP(mask, lines, 1, CV_PI/2, 50, 50, 75);
for( size_t i = 0; i < lines.size(); i++ )
{
Vec4i l = lines[i];
line( out, Point(l[0], l[1]), Point(l[2], l[3]), Scalar(255), 5, CV_AA);
}
If these lines are too fat, I've had success thinning them with:
size = 15;
Mat eroded;
cv::Mat erodeElement = getStructuringElement( MORPH_ELLIPSE, cv::Size( size, size ) );
erode( mask, eroded, erodeElement );
Here is a refinement build upon #akarsakov answer.
A basic issue with:
Distance between centers of segments should be less than half of
maximum length of two segments.
is that parallel long lines that are visually far might end up in same equivalence class (as demonstrated in OP's edit).
Therefore the approach that I found working reasonable for me:
Construct a window (bounding rectangle) around a line1.
line2 angle is close enough to line1's and at least one point of line2 is inside line1's bounding rectangle
Often a long linear feature in the image that is quite weak will end up recognized (HoughP, LSD) by a set of line segments with considerable gaps between them. To alleviate this, our bounding rectangle is constructed around line extended in both directions, where extension is defined by a fraction of original line width.
bool extendedBoundingRectangleLineEquivalence(const Vec4i& _l1, const Vec4i& _l2, float extensionLengthFraction, float maxAngleDiff, float boundingRectangleThickness){
Vec4i l1(_l1), l2(_l2);
// extend lines by percentage of line width
float len1 = sqrtf((l1[2] - l1[0])*(l1[2] - l1[0]) + (l1[3] - l1[1])*(l1[3] - l1[1]));
float len2 = sqrtf((l2[2] - l2[0])*(l2[2] - l2[0]) + (l2[3] - l2[1])*(l2[3] - l2[1]));
Vec4i el1 = extendedLine(l1, len1 * extensionLengthFraction);
Vec4i el2 = extendedLine(l2, len2 * extensionLengthFraction);
// reject the lines that have wide difference in angles
float a1 = atan(linearParameters(el1)[0]);
float a2 = atan(linearParameters(el2)[0]);
if(fabs(a1 - a2) > maxAngleDiff * M_PI / 180.0){
return false;
}
// calculate window around extended line
// at least one point needs to inside extended bounding rectangle of other line,
std::vector<Point2i> lineBoundingContour = boundingRectangleContour(el1, boundingRectangleThickness/2);
return
pointPolygonTest(lineBoundingContour, cv::Point(el2[0], el2[1]), false) == 1 ||
pointPolygonTest(lineBoundingContour, cv::Point(el2[2], el2[3]), false) == 1;
}
where linearParameters, extendedLine, boundingRectangleContour are following:
Vec2d linearParameters(Vec4i line){
Mat a = (Mat_<double>(2, 2) <<
line[0], 1,
line[2], 1);
Mat y = (Mat_<double>(2, 1) <<
line[1],
line[3]);
Vec2d mc; solve(a, y, mc);
return mc;
}
Vec4i extendedLine(Vec4i line, double d){
// oriented left-t-right
Vec4d _line = line[2] - line[0] < 0 ? Vec4d(line[2], line[3], line[0], line[1]) : Vec4d(line[0], line[1], line[2], line[3]);
double m = linearParameters(_line)[0];
// solution of pythagorean theorem and m = yd/xd
double xd = sqrt(d * d / (m * m + 1));
double yd = xd * m;
return Vec4d(_line[0] - xd, _line[1] - yd , _line[2] + xd, _line[3] + yd);
}
std::vector<Point2i> boundingRectangleContour(Vec4i line, float d){
// finds coordinates of perpendicular lines with length d in both line points
// https://math.stackexchange.com/a/2043065/183923
Vec2f mc = linearParameters(line);
float m = mc[0];
float factor = sqrtf(
(d * d) / (1 + (1 / (m * m)))
);
float x3, y3, x4, y4, x5, y5, x6, y6;
// special case(vertical perpendicular line) when -1/m -> -infinity
if(m == 0){
x3 = line[0]; y3 = line[1] + d;
x4 = line[0]; y4 = line[1] - d;
x5 = line[2]; y5 = line[3] + d;
x6 = line[2]; y6 = line[3] - d;
} else {
// slope of perpendicular lines
float m_per = - 1/m;
// y1 = m_per * x1 + c_per
float c_per1 = line[1] - m_per * line[0];
float c_per2 = line[3] - m_per * line[2];
// coordinates of perpendicular lines
x3 = line[0] + factor; y3 = m_per * x3 + c_per1;
x4 = line[0] - factor; y4 = m_per * x4 + c_per1;
x5 = line[2] + factor; y5 = m_per * x5 + c_per2;
x6 = line[2] - factor; y6 = m_per * x6 + c_per2;
}
return std::vector<Point2i> {
Point2i(x3, y3),
Point2i(x4, y4),
Point2i(x6, y6),
Point2i(x5, y5)
};
}
To partion, call:
std::vector<int> labels;
int equilavenceClassesCount = cv::partition(linesWithoutSmall, labels, [](const Vec4i l1, const Vec4i l2){
return extendedBoundingRectangleLineEquivalence(
l1, l2,
// line extension length - as fraction of original line width
0.2,
// maximum allowed angle difference for lines to be considered in same equivalence class
2.0,
// thickness of bounding rectangle around each line
10);
});
Now, in order to reduce each equivalence class to single line, we build a point cloud out of it and find a line fit:
// fit line to each equivalence class point cloud
std::vector<Vec4i> reducedLines = std::accumulate(pointClouds.begin(), pointClouds.end(), std::vector<Vec4i>{}, [](std::vector<Vec4i> target, const std::vector<Point2i>& _pointCloud){
std::vector<Point2i> pointCloud = _pointCloud;
//lineParams: [vx,vy, x0,y0]: (normalized vector, point on our contour)
// (x,y) = (x0,y0) + t*(vx,vy), t -> (-inf; inf)
Vec4f lineParams; fitLine(pointCloud, lineParams, CV_DIST_L2, 0, 0.01, 0.01);
// derive the bounding xs of point cloud
decltype(pointCloud)::iterator minXP, maxXP;
std::tie(minXP, maxXP) = std::minmax_element(pointCloud.begin(), pointCloud.end(), [](const Point2i& p1, const Point2i& p2){ return p1.x < p2.x; });
// derive y coords of fitted line
float m = lineParams[1] / lineParams[0];
int y1 = ((minXP->x - lineParams[2]) * m) + lineParams[3];
int y2 = ((maxXP->x - lineParams[2]) * m) + lineParams[3];
target.push_back(Vec4i(minXP->x, y1, maxXP->x, y2));
return target;
});
Demonstration:
Detected partitioned line (with small lines filtered out):
Reduced:
Demonstration code:
int main(int argc, const char* argv[]){
if(argc < 2){
std::cout << "img filepath should be present in args" << std::endl;
}
Mat image = imread(argv[1]);
Mat smallerImage; resize(image, smallerImage, cv::Size(), 0.5, 0.5, INTER_CUBIC);
Mat target = smallerImage.clone();
namedWindow("Detected Lines", WINDOW_NORMAL);
namedWindow("Reduced Lines", WINDOW_NORMAL);
Mat detectedLinesImg = Mat::zeros(target.rows, target.cols, CV_8UC3);
Mat reducedLinesImg = detectedLinesImg.clone();
// delect lines in any reasonable way
Mat grayscale; cvtColor(target, grayscale, CV_BGRA2GRAY);
Ptr<LineSegmentDetector> detector = createLineSegmentDetector(LSD_REFINE_NONE);
std::vector<Vec4i> lines; detector->detect(grayscale, lines);
// remove small lines
std::vector<Vec4i> linesWithoutSmall;
std::copy_if (lines.begin(), lines.end(), std::back_inserter(linesWithoutSmall), [](Vec4f line){
float length = sqrtf((line[2] - line[0]) * (line[2] - line[0])
+ (line[3] - line[1]) * (line[3] - line[1]));
return length > 30;
});
std::cout << "Detected: " << linesWithoutSmall.size() << std::endl;
// partition via our partitioning function
std::vector<int> labels;
int equilavenceClassesCount = cv::partition(linesWithoutSmall, labels, [](const Vec4i l1, const Vec4i l2){
return extendedBoundingRectangleLineEquivalence(
l1, l2,
// line extension length - as fraction of original line width
0.2,
// maximum allowed angle difference for lines to be considered in same equivalence class
2.0,
// thickness of bounding rectangle around each line
10);
});
std::cout << "Equivalence classes: " << equilavenceClassesCount << std::endl;
// grab a random colour for each equivalence class
RNG rng(215526);
std::vector<Scalar> colors(equilavenceClassesCount);
for (int i = 0; i < equilavenceClassesCount; i++){
colors[i] = Scalar(rng.uniform(30,255), rng.uniform(30, 255), rng.uniform(30, 255));;
}
// draw original detected lines
for (int i = 0; i < linesWithoutSmall.size(); i++){
Vec4i& detectedLine = linesWithoutSmall[i];
line(detectedLinesImg,
cv::Point(detectedLine[0], detectedLine[1]),
cv::Point(detectedLine[2], detectedLine[3]), colors[labels[i]], 2);
}
// build point clouds out of each equivalence classes
std::vector<std::vector<Point2i>> pointClouds(equilavenceClassesCount);
for (int i = 0; i < linesWithoutSmall.size(); i++){
Vec4i& detectedLine = linesWithoutSmall[i];
pointClouds[labels[i]].push_back(Point2i(detectedLine[0], detectedLine[1]));
pointClouds[labels[i]].push_back(Point2i(detectedLine[2], detectedLine[3]));
}
// fit line to each equivalence class point cloud
std::vector<Vec4i> reducedLines = std::accumulate(pointClouds.begin(), pointClouds.end(), std::vector<Vec4i>{}, [](std::vector<Vec4i> target, const std::vector<Point2i>& _pointCloud){
std::vector<Point2i> pointCloud = _pointCloud;
//lineParams: [vx,vy, x0,y0]: (normalized vector, point on our contour)
// (x,y) = (x0,y0) + t*(vx,vy), t -> (-inf; inf)
Vec4f lineParams; fitLine(pointCloud, lineParams, CV_DIST_L2, 0, 0.01, 0.01);
// derive the bounding xs of point cloud
decltype(pointCloud)::iterator minXP, maxXP;
std::tie(minXP, maxXP) = std::minmax_element(pointCloud.begin(), pointCloud.end(), [](const Point2i& p1, const Point2i& p2){ return p1.x < p2.x; });
// derive y coords of fitted line
float m = lineParams[1] / lineParams[0];
int y1 = ((minXP->x - lineParams[2]) * m) + lineParams[3];
int y2 = ((maxXP->x - lineParams[2]) * m) + lineParams[3];
target.push_back(Vec4i(minXP->x, y1, maxXP->x, y2));
return target;
});
for(Vec4i reduced: reducedLines){
line(reducedLinesImg, Point(reduced[0], reduced[1]), Point(reduced[2], reduced[3]), Scalar(255, 255, 255), 2);
}
imshow("Detected Lines", detectedLinesImg);
imshow("Reduced Lines", reducedLinesImg);
waitKey();
return 0;
}
I would recommend that you use HoughLines from OpenCV.
void HoughLines(InputArray image, OutputArray lines, double rho, double theta, int threshold, double srn=0, double stn=0 )
You can adjust with rho and theta the possible orientation and position of the lines you want to observe.
In your case, theta = 90° would be fine (only vertical and horizontal lines).
After this, you can get unique line equations with Plücker coordinates. And from there you could apply a K-mean with 3 centers that should fit approximately your 3 lines in the second image.
PS : I will see if i can test the whole process with your image
You can merge multiple close line into single line by clustering lines using rho and theta and finally taking average of rho and theta.
void contourLines(vector<cv::Vec2f> lines, const float rho_threshold, const float theta_threshold, vector< cv::Vec2f > &combinedLines)
{
vector< vector<int> > combineIndex(lines.size());
for (int i = 0; i < lines.size(); i++)
{
int index = i;
for (int j = i; j < lines.size(); j++)
{
float distanceI = lines[i][0], distanceJ = lines[j][0];
float slopeI = lines[i][1], slopeJ = lines[j][1];
float disDiff = abs(distanceI - distanceJ);
float slopeDiff = abs(slopeI - slopeJ);
if (slopeDiff < theta_max && disDiff < rho_max)
{
bool isCombined = false;
for (int w = 0; w < i; w++)
{
for (int u = 0; u < combineIndex[w].size(); u++)
{
if (combineIndex[w][u] == j)
{
isCombined = true;
break;
}
if (combineIndex[w][u] == i)
index = w;
}
if (isCombined)
break;
}
if (!isCombined)
combineIndex[index].push_back(j);
}
}
}
for (int i = 0; i < combineIndex.size(); i++)
{
if (combineIndex[i].size() == 0)
continue;
cv::Vec2f line_temp(0, 0);
for (int j = 0; j < combineIndex[i].size(); j++) {
line_temp[0] += lines[combineIndex[i][j]][0];
line_temp[1] += lines[combineIndex[i][j]][1];
}
line_temp[0] /= combineIndex[i].size();
line_temp[1] /= combineIndex[i].size();
combinedLines.push_back(line_temp);
}
}
function call
You can tune houghThreshold, rho_threshold and theta_threshold as per your application.
HoughLines(edge, lines_t, 1, CV_PI / 180, houghThreshold, 0, 0);
float rho_threshold= 15;
float theta_threshold = 3*DEGREES_TO_RADIANS;
vector< cv::Vec2f > lines;
contourCluster(lines_t, rho_max, theta_max, lines);
#C_Raj made a good point, for lines like this, i.e., most likely extracted from table/form-like images, you should make full use of the fact that many of the line segments captured by Hough transform from the same lines have very similar \rho and \theta.
After clustering these line segments based on their \rho and \theta, you can apply 2D line fitting to obtain estimate of the true lines in an image.
There is a paper describing this idea and it's making further assumptions of the lines in a page.
HTH.
I am trying to plot velocity vectors like in matlab we use "quiver" function http://www.mathworks.com/help/techdoc/ref/quiver.html
I need to port same methodology in C++ using OpenCV library.
I have heard There are a few optical flow methods, i.e. Lucas and Kanade (cvCalOpticalFlowLK) or Horn and Schunck (cvCalOpticalFlowHS) or Block Matching method (cvCalOpticalFlowBM)
but all of these functions take two images , while i need to use one image because i am working on fingerprints.
Kindly help me ...
[Edit]
Solution found
void cvQuiver(IplImage*Image,int x,int y,int u,int v,CvScalar Color,
int Size,int Thickness){
cv::Point pt1,pt2;
double Theta;
double PI = 3.1416;
if(u==0)
Theta=PI/2;
else
Theta=atan2(double(v),(double)(u));
pt1.x=x;
pt1.y=y;
pt2.x=x+u;
pt2.y=y+v;
cv::line(Image,pt1,pt2,Color,Thickness,8); //Draw Line
Size=(int)(Size*0.707);
if(Theta==PI/2 && pt1.y > pt2.y)
{
pt1.x=(int)(Size*cos(Theta)-Size*sin(Theta)+pt2.x);
pt1.y=(int)(Size*sin(Theta)+Size*cos(Theta)+pt2.y);
cv::line(Image,pt1,pt2,Color,Thickness,8); //Draw Line
pt1.x=(int)(Size*cos(Theta)+Size*sin(Theta)+pt2.x);
pt1.y=(int)(Size*sin(Theta)-Size*cos(Theta)+pt2.y);
cv::line(Image,pt1,pt2,Color,Thickness,8); //Draw Line
}
else{
pt1.x=(int)(-Size*cos(Theta)-Size*sin(Theta)+pt2.x);
pt1.y=(int)(-Size*sin(Theta)+Size*cos(Theta)+pt2.y);
cv::line(Image,pt1,pt2,Color,Thickness,8); //Draw Line
pt1.x=(int)(-Size*cos(Theta)+Size*sin(Theta)+pt2.x);
pt1.y=(int)(-Size*sin(Theta)-Size*cos(Theta)+pt2.y);
cv::line(Image,pt1,pt2,Color,Thickness,8); //Draw Line
}
}
I am kind of completing the current answer here, which fails in giving the right size of each of the arrows' tip. MATLAB does it in a way that when an arrow is nearly a dot, it doesn't have any tip, while for long arrows it shows a big tip, as the following image shows.
To get this effect, we need to normalise the "tip size" of each of the arrow over the range of arrows' length. The following code does the trick
double l_max = -10;
for (int y = 0; y < img_sz.height; y+=10) // First iteration, to compute the maximum l (longest flow)
{
for (int x = 0; x < img_sz.width; x+=10)
{
double dx = cvGetReal2D(velx, y, x); // Gets X component of the flow
double dy = cvGetReal2D(vely, y, x); // Gets Y component of the flow
CvPoint p = cvPoint(x, y);
double l = sqrt(dx*dx + dy*dy); // This function sets a basic threshold for drawing on the image
if(l>l_max) l_max = l;
}
}
for (int y = 0; y < img_sz.height; y+=10)
{
for (int x = 0; x < img_sz.width; x+=10)
{
double dx = cvGetReal2D(velx, y, x); // Gets X component of the flow
double dy = cvGetReal2D(vely, y, x); // Gets Y component of the flow
CvPoint p = cvPoint(x, y);
double l = sqrt(dx*dx + dy*dy); // This function sets a basic threshold for drawing on the image
if (l > 0)
{
double spinSize = 5.0 * l/l_max; // Factor to normalise the size of the spin depeding on the length of the arrow
CvPoint p2 = cvPoint(p.x + (int)(dx), p.y + (int)(dy));
cvLine(resultDenseOpticalFlow, p, p2, CV_RGB(0,255,0), 1, CV_AA);
double angle; // Draws the spin of the arrow
angle = atan2( (double) p.y - p2.y, (double) p.x - p2.x );
p.x = (int) (p2.x + spinSize * cos(angle + 3.1416 / 4));
p.y = (int) (p2.y + spinSize * sin(angle + 3.1416 / 4));
cvLine( resultDenseOpticalFlow, p, p2, CV_RGB(0,255,0), 1, CV_AA, 0 );
p.x = (int) (p2.x + spinSize * cos(angle - 3.1416 / 4));
p.y = (int) (p2.y + spinSize * sin(angle - 3.1416 / 4));
cvLine( resultDenseOpticalFlow, p, p2, CV_RGB(0,255,0), 1, CV_AA, 0 );
}
}
}
And this is an example of how this OpenCV code would look like
I hope this help other people Googling for the same issue.
Based on the code from Dan and the suggestion of mkuse, here is a function with the same syntax as cv::line():
static void arrowedLine(InputOutputArray img, Point pt1, Point pt2, const Scalar& color,
int thickness=1, int line_type=8, int shift=0, double tipLength=0.1)
{
const double tipSize = norm(pt1-pt2)*tipLength; // Factor to normalize the size of the tip depending on the length of the arrow
line(img, pt1, pt2, color, thickness, line_type, shift);
const double angle = atan2( (double) pt1.y - pt2.y, (double) pt1.x - pt2.x );
Point p(cvRound(pt2.x + tipSize * cos(angle + CV_PI / 4)),
cvRound(pt2.y + tipSize * sin(angle + CV_PI / 4)));
line(img, p, pt2, color, thickness, line_type, shift);
p.x = cvRound(pt2.x + tipSize * cos(angle - CV_PI / 4));
p.y = cvRound(pt2.y + tipSize * sin(angle - CV_PI / 4));
line(img, p, pt2, color, thickness, line_type, shift);
}
We will see if those maintaining the OpenCV repository will like it :-)
The cvCalOpticalFlowLK does not plot velocity vectors, it computes these velocity vectors. If you do not have these vectors, you must call this function with two images. I guess you already have these vectors, and you just want to plot them.
In this case, you can use the cv::line function, for example:
cv::line(yourImage, cv::Point(baseX, baseY), cv::Point(endX, endY));
I hope this will help you!