Related
To begin, I am a complete novice in OpenCV and am beginner/reasonable in c++ code.
But OpenCV is new to me and I try to learn by doing projects and stuff.
Now for my new project I am trying to find the centre of square in a picture.
In my case there is only 1 square in picture.
I would like to build further upon the square.cpp example of OpenCV.
For my project there are 2 things I need some help with,
1: The edge of the window is detected as a square, I do not want this. Example
2: How could I get the centre of 1 square from the squares array?
This is the code from the example "square.cpp"
// The "Square Detector" program.
// It loads several images sequentially and tries to find squares in
// each image
#include "opencv2/core.hpp"
#include "opencv2/imgproc.hpp"
#include "opencv2/imgcodecs.hpp"
#include "opencv2/highgui.hpp"
#include <iostream>
using namespace cv;
using namespace std;
static void help(const char* programName)
{
cout <<
"\nA program using pyramid scaling, Canny, contours and contour simplification\n"
"to find squares in a list of images (pic1-6.png)\n"
"Returns sequence of squares detected on the image.\n"
"Call:\n"
"./" << programName << " [file_name (optional)]\n"
"Using OpenCV version " << CV_VERSION << "\n" << endl;
}
int thresh = 50, N = 11;
const char* wndname = "Square Detection Demo";
// helper function:
// finds a cosine of angle between vectors
// from pt0->pt1 and from pt0->pt2
static double angle(Point pt1, Point pt2, Point pt0)
{
double dx1 = pt1.x - pt0.x;
double dy1 = pt1.y - pt0.y;
double dx2 = pt2.x - pt0.x;
double dy2 = pt2.y - pt0.y;
return (dx1 * dx2 + dy1 * dy2) / sqrt((dx1 * dx1 + dy1 * dy1) * (dx2 * dx2 + dy2 * dy2) + 1e-10);
}
// returns sequence of squares detected on the image.
static void findSquares(const Mat& image, vector<vector<Point> >& squares)
{
squares.clear();
Mat pyr, timg, gray0(image.size(), CV_8U), gray;
// down-scale and upscale the image to filter out the noise
pyrDown(image, pyr, Size(image.cols / 2, image.rows / 2));
pyrUp(pyr, timg, image.size());
vector<vector<Point> > contours;
// find squares in every color plane of the image
for (int c = 0; c < 3; c++)
{
int ch[] = { c, 0 };
mixChannels(&timg, 1, &gray0, 1, ch, 1);
// try several threshold levels
for (int l = 0; l < N; l++)
{
// hack: use Canny instead of zero threshold level.
// Canny helps to catch squares with gradient shading
if (l == 0)
{
// apply Canny. Take the upper threshold from slider
// and set the lower to 0 (which forces edges merging)
Canny(gray0, gray, 0, thresh, 5);
// dilate canny output to remove potential
// holes between edge segments
dilate(gray, gray, Mat(), Point(-1, -1));
}
else
{
// apply threshold if l!=0:
// tgray(x,y) = gray(x,y) < (l+1)*255/N ? 255 : 0
gray = gray0 >= (l + 1) * 255 / N;
}
// find contours and store them all as a list
findContours(gray, contours, RETR_LIST, CHAIN_APPROX_SIMPLE);
vector<Point> approx;
// test each contour
for (size_t i = 0; i < contours.size(); i++)
{
// approximate contour with accuracy proportional
// to the contour perimeter
approxPolyDP(contours[i], approx, arcLength(contours[i], true) * 0.02, true);
// square contours should have 4 vertices after approximation
// relatively large area (to filter out noisy contours)
// and be convex.
// Note: absolute value of an area is used because
// area may be positive or negative - in accordance with the
// contour orientation
if (approx.size() == 4 &&
fabs(contourArea(approx)) > 1000 &&
isContourConvex(approx))
{
double maxCosine = 0;
for (int j = 2; j < 5; j++)
{
// find the maximum cosine of the angle between joint edges
double cosine = fabs(angle(approx[j % 4], approx[j - 2], approx[j - 1]));
maxCosine = MAX(maxCosine, cosine);
}
// if cosines of all angles are small
// (all angles are ~90 degree) then write quandrange
// vertices to resultant sequence
if (maxCosine < 0.3)
squares.push_back(approx);
}
}
}
}
}
int main(int argc, char** argv)
{
static const char* names[] = { "testimg.jpg", 0 };
help(argv[0]);
if (argc > 1)
{
names[0] = argv[1];
names[1] = "0";
}
for (int i = 0; names[i] != 0; i++)
{
string filename = samples::findFile(names[i]);
Mat image = imread(filename, IMREAD_COLOR);
if (image.empty())
{
cout << "Couldn't load " << filename << endl;
continue;
}
vector<vector<Point> > squares;
findSquares(image, squares);
polylines(image, squares, true, Scalar(0, 0, 255), 3, LINE_AA);
imshow(wndname, image);
int c = waitKey();
if (c == 27)
break;
}
return 0;
}
I would like some help to start off.
How could I get some information from 1 of the squares out of the array called "squares" (I am having a difficult time understand what exactly is in the array as well; is it an array of points?)
If something is not clear please let me know and I will try to re-explain.
Thank you in advance
Firstly, you are talking about squares but you are actually detecting rectangles. I provided a shorter code to be able to better answer your questions.
I read the image, apply a Canny filter for binarization and detect all contours. After that I iterate through the contours and find the ones which can be approximated by exactly four points and are convex:
int main(int argc, char** argv)
{
// Reading the images
cv::Mat img = cv::imread("squares_image.jpg", cv::IMREAD_GRAYSCALE);
cv::Mat edge_img;
std::vector <std::vector<cv::Point>> contours;
// Convert the image into a binary image using Canny filter - threshold could be automatically determined using OTSU method
cv::Canny(img, edge_img, 30, 100);
// Find all contours in the Canny image
findContours(edge_img, contours, cv::RETR_LIST, cv::CHAIN_APPROX_SIMPLE);
// Iterate through the contours and test if contours are square
std::vector<std::vector<cv::Point>> all_rectangles;
std::vector<cv::Point> single_rectangle;
for (size_t i = 0; i < contours.size(); i++)
{
// 1. Contours should be approximateable as a polygon
approxPolyDP(contours[i], single_rectangle, arcLength(contours[i], true) * 0.01, true);
// 2. Contours should have exactly 4 vertices and be convex
if (single_rectangle.size() == 4 && cv::isContourConvex(single_rectangle))
{
// 3. Determine if the polygon is really a square/rectangle using its properties (parallelity, angles etc.)
// Not necessary for the provided image
// Push the four points into your vector of squares (could be also std::vector<cv::Rect>)
all_rectangles.push_back(single_rectangle);
}
}
for (size_t num_contour = 0; num_contour < all_rectangles.size(); ++num_contour) {
cv::drawContours(img, all_rectangles, num_contour, cv::Scalar::all(-1));
}
cv::imshow("Detected rectangles", img);
cv::waitKey(0);
return 0;
}
1: The edge of the window is detected as a square, I do not want this.
There are several options depending on your applications. You can filter the outer boundary already using the Canny thresholds, using a different contour retrieval method for finding contours in findContours or by filtering single_rectangle using the area of the found contour (e.g. cv::contourArea(single_rectangle) < 1000).
2: How could I get the centre of 1 square from the squares array?
Since the code is already detecting the four corner points you could e.g. find the intersection of the diagonals. If you know that there are only rectangles in your image you could also try to detect all centroids of the detected contours using the Hu moments.
I am having a difficult time understand what exactly is in the array as well; is it an array of points?
One contour in OpenCV is always represented as a vector of single points. If you are adding multiple contours you are using a vector of vector of points. In the example you provided squares is a vector of a vector of exactly 4 points. You could also use a vector of cv::Rect in this case.
I am doing a project in OpenCV on estimating the speed of moving vehicle using the video captured. Here the camera is stationary. I have estimated the speed of single object using centroid and Euclidean distance. Now the problem is, I am not getting how to do the same for multiple objects.
Here, I need to calculate the Euclidean distance of objects between 2 subsequent frames.
I am grateful if anyone would help.
I have created the class-
class centroids
{
public:
vector<Point2f> ce;
vector<float> area;
};
centroids c[100];
And this is the code I've written. I would be grateful if anyone helped me with the code:
findContours( fgMaskMOG2,
contours,
hierarchy,
CV_RETR_CCOMP,
CV_CHAIN_APPROX_SIMPLE );
int morph_size = 6;
Mat element = getStructuringElement( MORPH_RECT,
Size( 2*morph_size+1, 2*morph_size+1 ),
Point( morph_size, morph_size ) );
Scalar color( 255, 255, 255 ); // color of the contour in the
//Draw the contour and rectangle
for( int i = 0; i < contours.size(); i++ )
{
drawContours( fgMaskMOG2,
contours,
i,
color,
CV_FILLED,
8,
hierarchy );
}
//imshow("morpho window",dst);
vector<Moments> mu( contours.size() );
vector<Point2f> mc( contours.size() );
vector<Point2f> m ;
vector<double> time;
vector<Point2f> centroid( mc.size() );
//vector< vector<Point> >::iterator itc = contours.begin();
// iterate through each contour.
double time1[1000];
for( int i = 0; i < contours.size(); i++ )
{
// Find the area of contour
double a = contourArea( contours[i], false );
if( a > 500 )
{
mu[i] = moments( contours[i], false );
mc[i] = Point2f( (mu[i].m10 / mu[i].m00), (mu[i].m01 / mu[i].m00) );
m.push_back( mc[i] );
Point2f diff;
double euclidian = 0;
for( int f = 0; f < m.size(); f++ )
{
if( k == 1 )
{
c[f].ce.push_back( m[f] );
cout << "cen" << c[f].ce << endl;
euclidian = 0;
}
else
{
c[f+1].ce.push_back( m[f] );
cout << "cent" << c[f+1].ce << endl;
diff = c[f].ce[f] - c[f-1].ce[f-1];
euclidian = abs( sqrt( (diff.x*diff.x) + (diff.y*diff.y) ) );
cout << "euclidian" << euclidian << endl;
}
}
cout << "\n centroid" << m << endl;
circle( fgMaskMOG2,
mc[i],
5,
Scalar( 0, 0, 255 ),
1,
8,
0 );
}
}
Thanks in advance :)
You can estimate speed of a moving vehicle based on the video frames only if approximate distance between vehicle and camera is constant throughout the calculation i.e. vehicle is moving in a straight line perpendicular to the camera's vision. So, if camera is looking from side, all the vehicles will be at different distance and calculation will become highly inaccurate for multiple vehicles. Even the vehicles will overlap and their segmentation will be difficult.
There are two scenarios in which your calculation may work -
First, when the camera is capturing from top looking vertically down on vehicles. In this case, there will be a stark difference between the vehicle color and road color. You can use several ways to segment out those individual vehicles, tag them based on their features and identify those vehicles in next frame using the features. This way you'll get position of individual vehicles and then you can predict the speed based on your algorithm. These are following links which will be helpful for segmenting the vechicles -
How to define the markers for Watershed in OpenCV?
http://www.codeproject.com/Articles/751744/Image-Segmentation-using-Unsupervised-Watershed-Al
http://www.bogotobogo.com/python/OpenCV_Python/python_opencv3_Image_Watershed_Algorithm_Marker_Based_Segmentation.php
Second, when vehicles are moving in a single line behind one another. In this case, you can use a combination of color and contour based segmentation depending on the background of your vehicles. After segmentation you can again use object features to identify the position of objects in the next frame. Then run your algorithm for both cases.
If you have complete video sequence of the vehicles, you can segment out different vehicles in the first frame automatically or identify them manually, and then apply motion tracking on those identified objects. You can use Opencv's motion analysis functions and object tracking functions to do so. Thus you'll get position of all tracked vehicles in each frame. So you can easily run and test your speed calculating algorithms.
I currently have a method for detecting a card in an image and for the most part it works when the lighting is fairly consistent and the background is very calm.
Here is the code I am using to preform this operation:
Mat img = inImg.clone();
outImg = Mat(inImg.size(), CV_8UC1);
inImg.copyTo(outImg);
Mat img_fullRes = img.clone();
pyrDown(img, img);
Mat imgGray;
cvtColor(img, imgGray, CV_RGB2GRAY);
outImg_gray = imgGray.clone();
// Find Edges //
Mat detectedEdges = imgGray.clone();
bilateralFilter(imgGray, detectedEdges, 0, 185, 3, 0);
Canny( detectedEdges, detectedEdges, 20, 65, 3 );
dilate(detectedEdges, detectedEdges, Mat::ones(3,3,CV_8UC1));
Mat cdst = img.clone();
vector<Vec4i> lines;
HoughLinesP(detectedEdges, lines, 1, CV_PI/180, 60, 50, 3 );
for( size_t i = 0; i < lines.size(); i++ )
{
Vec4i l = lines[i];
// For debug
//line( cdst, cv::Point(l[0], l[1]), cv::Point(l[2], l[3]), Scalar(0,0,255), 1);
}
//cdst.copyTo(inImg);
// // Find points of intersection //
cv::Rect imgROI;
int ext = 10;
imgROI.x = ext;
imgROI.y = ext;
imgROI.width = img.size().width - ext;
imgROI.height = img.size().height - ext;
int N = lines.size();
// Creating N amount of points // N == lines.size()
cv::Point** poi = new cv::Point*[N];
for( int i = 0; i < N; i++ )
poi[i] = new cv::Point[N];
vector<cv::Point> poiList;
for( int i = 0; i < N; i++ )
{
poi[i][i] = cv::Point(-1,-1);
Vec4i line1 = lines[i];
for( int j = i + 1; j < N; j++ )
{
Vec4i line2 = lines[j];
cv::Point p = computeIntersect(line1, line2, imgROI);
if( p.x != -1 )
{
//line(cdst, p-cv::Point(2,0), p+cv::Point(2,0), Scalar(0,255,0));
//line(cdst, p-cv::Point(0,2), p+cv::Point(0,2), Scalar(0,255,0));
poiList.push_back(p);
}
poi[i][j] = p;
poi[j][i] = p;
}
}
cdst.copyTo(inImg);
if(poiList.size()==0)
{
outImg = inImg.clone();
//circle(outImg, cv::Point(100,100), 50, Scalar(255,0,0), -1);
return;
}
convexHull(poiList, poiList, false, true);
for( int i=0; i<poiList.size(); i++ )
{
cv::Point p = poiList[i];
//circle(cdst, p, 3, Scalar(255,0,0), 2);
}
//Evaluate all possible quadrilaterals
cv::Point cardCorners[4];
float metric_max = 0;
int Npoi = poiList.size();
for( int p1=0; p1<Npoi; p1++ )
{
cv::Point pts[4];
pts[0] = poiList[p1];
for( int p2=p1+1; p2<Npoi; p2++ )
{
pts[1] = poiList[p2];
if( isCloseBy(pts[1],pts[0]) )
continue;
for( int p3=p2+1; p3<Npoi; p3++ )
{
pts[2] = poiList[p3];
if( isCloseBy(pts[2],pts[1]) || isCloseBy(pts[2],pts[0]) )
continue;
for( int p4=p3+1; p4<Npoi; p4++ )
{
pts[3] = poiList[p4];
if( isCloseBy(pts[3],pts[0]) || isCloseBy(pts[3],pts[1])
|| isCloseBy(pts[3],pts[2]) )
continue;
// get the metrics
float area = getArea(pts);
cv::Point a = pts[0]-pts[1];
cv::Point b = pts[1]-pts[2];
cv::Point c = pts[2]-pts[3];
cv::Point d = pts[3]-pts[0];
float oppLenDiff = abs(a.dot(a)-c.dot(c)) + abs(b.dot(b)-d.dot(d));
float metric = area - 0.35*oppLenDiff;
if( metric > metric_max )
{
metric_max = metric;
cardCorners[0] = pts[0];
cardCorners[1] = pts[1];
cardCorners[2] = pts[2];
cardCorners[3] = pts[3];
}
}
}
}
}
// find the corners corresponding to the 4 corners of the physical card
sortPointsClockwise(cardCorners);
// Calculate Homography //
vector<Point2f> srcPts(4);
srcPts[0] = cardCorners[0]*2;
srcPts[1] = cardCorners[1]*2;
srcPts[2] = cardCorners[2]*2;
srcPts[3] = cardCorners[3]*2;
vector<Point2f> dstPts(4);
cv::Size outImgSize(1400,800);
dstPts[0] = Point2f(0,0);
dstPts[1] = Point2f(outImgSize.width-1,0);
dstPts[2] = Point2f(outImgSize.width-1,outImgSize.height-1);
dstPts[3] = Point2f(0,outImgSize.height-1);
Mat Homography = findHomography(srcPts, dstPts);
// Apply Homography
warpPerspective( img_fullRes, outImg, Homography, outImgSize, INTER_CUBIC );
outImg.copyTo(inImg);
Where computeIntersect is defined as:
cv::Point computeIntersect(cv::Vec4i a, cv::Vec4i b, cv::Rect ROI)
{
int x1 = a[0], y1 = a[1], x2 = a[2], y2 = a[3];
int x3 = b[0], y3 = b[1], x4 = b[2], y4 = b[3];
cv::Point p1 = cv::Point (x1,y1);
cv::Point p2 = cv::Point (x2,y2);
cv::Point p3 = cv::Point (x3,y3);
cv::Point p4 = cv::Point (x4,y4);
// Check to make sure all points are within the image boundrys, if not reject them.
if( !ROI.contains(p1) || !ROI.contains(p2)
|| !ROI.contains(p3) || !ROI.contains(p4) )
return cv::Point (-1,-1);
cv::Point vec1 = p1-p2;
cv::Point vec2 = p3-p4;
float vec1_norm2 = vec1.x*vec1.x + vec1.y*vec1.y;
float vec2_norm2 = vec2.x*vec2.x + vec2.y*vec2.y;
float cosTheta = (vec1.dot(vec2))/sqrt(vec1_norm2*vec2_norm2);
float den = ((float)(x1-x2) * (y3-y4)) - ((y1-y2) * (x3-x4));
if(den != 0)
{
cv::Point2f pt;
pt.x = ((x1*y2 - y1*x2) * (x3-x4) - (x1-x2) * (x3*y4 - y3*x4)) / den;
pt.y = ((x1*y2 - y1*x2) * (y3-y4) - (y1-y2) * (x3*y4 - y3*x4)) / den;
if( !ROI.contains(pt) )
return cv::Point (-1,-1);
// no-confidence metric
float d1 = MIN( dist2(p1,pt), dist2(p2,pt) )/vec1_norm2;
float d2 = MIN( dist2(p3,pt), dist2(p4,pt) )/vec2_norm2;
float no_confidence_metric = MAX(sqrt(d1),sqrt(d2));
// If end point ratios are greater than .5 reject
if( no_confidence_metric < 0.5 && cosTheta < 0.707 )
return cv::Point (int(pt.x+0.5), int(pt.y+0.5));
}
return cv::Point(-1, -1);
}
sortPointsClockWise is defined as:
void sortPointsClockwise(cv::Point a[])
{
cv::Point b[4];
cv::Point ctr = (a[0]+a[1]+a[2]+a[3]);
ctr.x /= 4;
ctr.y /= 4;
b[0] = a[0]-ctr;
b[1] = a[1]-ctr;
b[2] = a[2]-ctr;
b[3] = a[3]-ctr;
for( int i=0; i<4; i++ )
{
if( b[i].x < 0 )
{
if( b[i].y < 0 )
a[0] = b[i]+ctr;
else
a[3] = b[i]+ctr;
}
else
{
if( b[i].y < 0 )
a[1] = b[i]+ctr;
else
a[2] = b[i]+ctr;
}
}
}
getArea is defined as:
float getArea(cv::Point arr[])
{
cv::Point diag1 = arr[0]-arr[2];
cv::Point diag2 = arr[1]-arr[3];
return 0.5*(diag1.cross(diag2));
}
isCloseBy is defined as:
bool isCloseBy( cv::Point p1, cv::Point p2 )
{
int D = 10;
// Checking that X values are within 10, same for Y values.
return ( abs(p1.x-p2.x)<=D && abs(p1.y-p2.y)<=D );
}
And finally dist2:
float dist2( cv::Point p1, cv::Point p2 )
{
return float((p1.x-p2.x)*(p1.x-p2.x) + (p1.y-p2.y)*(p1.y-p2.y));
}
Here are several test images and their results:
Sorry for the very lengthy post, however I am hoping someone can suggest a way I can make my method for extracting the card from the image more robust. One that can better handle disruptive backgrounds along with inconsistent lighting.
When a card is placed on a contrasting background with good lighting my method works nearly 90% of the time. But it is clear I need a more robust approach.
Does anyone have any suggestions?
Thanks.
ATTEMPT of dhanushka's soloution
Mat gray, bw; pyrDown(inImg, inImg);
cvtColor(inImg, gray, CV_RGB2GRAY);
int morph_size = 3;
Mat element = getStructuringElement( MORPH_ELLIPSE, cv::Size( 4*morph_size + 1, 2*morph_size+1 ), cv::Point( morph_size, morph_size ) );
morphologyEx(gray, gray, 2, element);
threshold(gray, bw, 160, 255, CV_THRESH_BINARY);
vector<vector<cv::Point> > contours;
vector<Vec4i> hierarchy;
findContours( bw, contours, hierarchy, CV_RETR_TREE, CV_CHAIN_APPROX_SIMPLE, cv::Point(0, 0) );
int largest_area=0;
int largest_contour_index=0;
cv::Rect bounding_rect;
for( int i = 0; i< contours.size(); i++ )
{
double a=contourArea( contours[i],false); // Find the area of contour
if(a>largest_area){
largest_area=a;
largest_contour_index=i; //Store the index of largest contour
bounding_rect=boundingRect(contours[i]);
}
}
//Scalar color( 255,255,255);
rectangle(inImg, bounding_rect, Scalar(0,255,0),1, 8,0);
Mat biggestRect = inImg(bounding_rect);
Mat card1 = biggestRect.clone();
The art of image processing is (in my 10+ years experience) just that: an art. No single answer exists, and there is always more than one way to do it. And it will definitely fail in some cases.
In my experience of working on automatically detecting features in medical images, it takes a long time to build to reliable algorithm, but in hindsight the best result is obtained with a relative simple algorithm. However, it takes a lot of time to get to this simple algorithm.
To get to this, the general approach is always the same:
get started is to build up a large database of test-images (at least 100). This defines 'normal' images which should work. By collecting the images you already start thinking about the problem.
annotate the images to build a kind of 'ground truth'. In this case, the 'ground truth' should contain the 4 corners of the card since these are the interesting points.
create an application which runs over these images an algorithm and compares the result with the ground truth. In this case, the 'comparing with ground truth' would be to take the mean distance of the found 4 corner point with the ground truth corner points.
Output a tab-delimited file which you call .xls, and therefore can be opened (on Windows) in Excel by double clicking. Good to get an quick overview of the cases. Look at the worst cases first. Then open these cases manually to try to understand why they do not work.
Now you are ready to change the algorithm. Change something, and re-run. Compare new Excel sheet to old Excel sheet. Now you start realizing the trade-offs you have to make.
That having said, I think that you need to answer these questions during the tuning of the algorithm:
Do you allow a little folded cards? So no completely straight lines? If so, concentrate more on corners instead of lines / edges.
Do you allow gradual differences in lighting? If so, a local contrast-stretch filter might help.
Do you allow the same color for the card as the background? If so, you have to concentrate on the contents of the card instead of the border of the card.
Do you allow non-perfect lenses? If so, to which extend?
Do you allow rotated cards? If so, to which extend?
Should the background be uniform in color and/or texture?
How small should the smallest detectable card be relative to the image size? If you assume that at least 80% of the width or height should be covered, you get robustness back.
If more than one card is visible in the image, should the algorithm be robust and only pick one, or is any output ok?
If no card is visible, should it detect this case? Building in detection of this case will make it more user friendly ('no card found'), but also less robust.
These will make the requirements and assumptions on the image to acquire. Assumptions on which you can rely are very strong: they make the algorithm fast, robust and simple if you choose the right ones. Also let these requirements and assumptions be part of the testing database.
So what would I choose? Based on the three images you provided I would start with something like:
Assume the cards are filling the image from 50% to 100%.
Assume the cards are rotated at most 10 degrees or so.
Assume the corners are well visible.
Assume the aspect ratio (height divided by width) of the cards to be between 1/3 and 3.
Assume no card-like objects in the background
The algorithm then would look like:
Detect in each quadrant of the image a specific corner with a corner-filter. So in the upper left quadrant of the image the upper left corner of the card. Look for example at http://www.ee.surrey.ac.uk/CVSSP/demos/corners/results3.html , or use an OpenCV function for it like cornerHarris .
To be more robust, calculate more than one corner per quadrant.
Try to build parallelograms with one corner per each quadrant by combining points from each quadrant. Create a fitness function which gives higher score to:
having internal angles close to 90 degrees
be large
optionally, compare the corners of the card based on lighting or another feature.
This fitness function gives a lot of tuning possibilities later on.
Return the parallelogram with the highest score.
So why using corner-detection instead of a hough-transform to do line detection? In my opinion the hough-transform is (next to being slow) quite sensitive to patterns in the background (which is what you see in your first image -- it detects a stronger line in the background then of the card), and it cannot handle a little curved lines that well, unless you use a larger bin size which will worsen the detection.
Good luck!
A more general approach would definitely be something like Rutger Nijlunsing suggested in his answer. However, in your case, at least for the provided sample images, a very simple approach like morphological opening followed by thresholding, contour processing and convexhull would yield the result you want. Use a scaled down version of the images for processing so that you don't have to use a large kernel for morphological operations. Below are the images processed this way.
pyrDown(large, rgb0);
pyrDown(rgb0, rgb0);
pyrDown(rgb0, rgb0);
Mat small;
cvtColor(rgb0, small, CV_BGR2GRAY);
Mat morph;
Mat kernel = getStructuringElement(MORPH_ELLIPSE, Size(11, 11));
morphologyEx(small, morph, MORPH_OPEN, kernel);
Mat bw;
threshold(morph, bw, 0, 255.0, CV_THRESH_BINARY | CV_THRESH_OTSU);
Mat bdry;
kernel = getStructuringElement(MORPH_ELLIPSE, Size(3, 3));
erode(bw, bdry, kernel);
subtract(bw, bdry, bdry);
// do contour processing on bdry
This approach will not work in general, so I would strongly recommend something like Rutger suggested.
A while ago I asked a question about square detection and karlphillip came up with a decent result.
Now I want to take this a step further and find squares which edge aren't fully visible. Take a look at this example:
Any ideas? I'm working with karlphillips code:
void find_squares(Mat& image, vector<vector<Point> >& squares)
{
// blur will enhance edge detection
Mat blurred(image);
medianBlur(image, blurred, 9);
Mat gray0(blurred.size(), CV_8U), gray;
vector<vector<Point> > contours;
// find squares in every color plane of the image
for (int c = 0; c < 3; c++)
{
int ch[] = {c, 0};
mixChannels(&blurred, 1, &gray0, 1, ch, 1);
// try several threshold levels
const int threshold_level = 2;
for (int l = 0; l < threshold_level; l++)
{
// Use Canny instead of zero threshold level!
// Canny helps to catch squares with gradient shading
if (l == 0)
{
Canny(gray0, gray, 10, 20, 3); //
// Dilate helps to remove potential holes between edge segments
dilate(gray, gray, Mat(), Point(-1,-1));
}
else
{
gray = gray0 >= (l+1) * 255 / threshold_level;
}
// Find contours and store them in a list
findContours(gray, contours, CV_RETR_LIST, CV_CHAIN_APPROX_SIMPLE);
// Test contours
vector<Point> approx;
for (size_t i = 0; i < contours.size(); i++)
{
// approximate contour with accuracy proportional
// to the contour perimeter
approxPolyDP(Mat(contours[i]), approx, arcLength(Mat(contours[i]), true)*0.02, true);
// Note: absolute value of an area is used because
// area may be positive or negative - in accordance with the
// contour orientation
if (approx.size() == 4 &&
fabs(contourArea(Mat(approx))) > 1000 &&
isContourConvex(Mat(approx)))
{
double maxCosine = 0;
for (int j = 2; j < 5; j++)
{
double cosine = fabs(angle(approx[j%4], approx[j-2], approx[j-1]));
maxCosine = MAX(maxCosine, cosine);
}
if (maxCosine < 0.3)
squares.push_back(approx);
}
}
}
}
}
You might try using HoughLines to detect the four sides of the square. Next, locate the four resulting line intersections to detect the corners. The Hough transform is fairly robust to noise and occlusions, so it could be useful here. Also, here is an interactive demo showing how the Hough transform works (I thought it was cool at least :). Here is one of my previous answers that detects a laser cross center showing most of the same math (except it just finds a single corner).
You will probably have multiple lines on each side, but locating the intersections should help to determine the inliers vs. outliers. Once you've located candidate corners, you can also filter these candidates by area or how "square-like" the polygon is.
EDIT : All these answers with code and images were making me think my answer was a bit lacking :) So, here is an implementation of how you could do this:
#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include <iostream>
#include <vector>
using namespace cv;
using namespace std;
Point2f computeIntersect(Vec2f line1, Vec2f line2);
vector<Point2f> lineToPointPair(Vec2f line);
bool acceptLinePair(Vec2f line1, Vec2f line2, float minTheta);
int main(int argc, char* argv[])
{
Mat occludedSquare = imread("Square.jpg");
resize(occludedSquare, occludedSquare, Size(0, 0), 0.25, 0.25);
Mat occludedSquare8u;
cvtColor(occludedSquare, occludedSquare8u, CV_BGR2GRAY);
Mat thresh;
threshold(occludedSquare8u, thresh, 200.0, 255.0, THRESH_BINARY);
GaussianBlur(thresh, thresh, Size(7, 7), 2.0, 2.0);
Mat edges;
Canny(thresh, edges, 66.0, 133.0, 3);
vector<Vec2f> lines;
HoughLines( edges, lines, 1, CV_PI/180, 50, 0, 0 );
cout << "Detected " << lines.size() << " lines." << endl;
// compute the intersection from the lines detected...
vector<Point2f> intersections;
for( size_t i = 0; i < lines.size(); i++ )
{
for(size_t j = 0; j < lines.size(); j++)
{
Vec2f line1 = lines[i];
Vec2f line2 = lines[j];
if(acceptLinePair(line1, line2, CV_PI / 32))
{
Point2f intersection = computeIntersect(line1, line2);
intersections.push_back(intersection);
}
}
}
if(intersections.size() > 0)
{
vector<Point2f>::iterator i;
for(i = intersections.begin(); i != intersections.end(); ++i)
{
cout << "Intersection is " << i->x << ", " << i->y << endl;
circle(occludedSquare, *i, 1, Scalar(0, 255, 0), 3);
}
}
imshow("intersect", occludedSquare);
waitKey();
return 0;
}
bool acceptLinePair(Vec2f line1, Vec2f line2, float minTheta)
{
float theta1 = line1[1], theta2 = line2[1];
if(theta1 < minTheta)
{
theta1 += CV_PI; // dealing with 0 and 180 ambiguities...
}
if(theta2 < minTheta)
{
theta2 += CV_PI; // dealing with 0 and 180 ambiguities...
}
return abs(theta1 - theta2) > minTheta;
}
// the long nasty wikipedia line-intersection equation...bleh...
Point2f computeIntersect(Vec2f line1, Vec2f line2)
{
vector<Point2f> p1 = lineToPointPair(line1);
vector<Point2f> p2 = lineToPointPair(line2);
float denom = (p1[0].x - p1[1].x)*(p2[0].y - p2[1].y) - (p1[0].y - p1[1].y)*(p2[0].x - p2[1].x);
Point2f intersect(((p1[0].x*p1[1].y - p1[0].y*p1[1].x)*(p2[0].x - p2[1].x) -
(p1[0].x - p1[1].x)*(p2[0].x*p2[1].y - p2[0].y*p2[1].x)) / denom,
((p1[0].x*p1[1].y - p1[0].y*p1[1].x)*(p2[0].y - p2[1].y) -
(p1[0].y - p1[1].y)*(p2[0].x*p2[1].y - p2[0].y*p2[1].x)) / denom);
return intersect;
}
vector<Point2f> lineToPointPair(Vec2f line)
{
vector<Point2f> points;
float r = line[0], t = line[1];
double cos_t = cos(t), sin_t = sin(t);
double x0 = r*cos_t, y0 = r*sin_t;
double alpha = 1000;
points.push_back(Point2f(x0 + alpha*(-sin_t), y0 + alpha*cos_t));
points.push_back(Point2f(x0 - alpha*(-sin_t), y0 - alpha*cos_t));
return points;
}
NOTE : The main reason I resized the image was so I could see it on my screen, and speed-up processing.
Canny
This uses Canny edge detection to help greatly reduce the number of lines detected after thresholding.
Hough transform
Then the Hough transform is used to detect the sides of the square.
Intersections
Finally, we compute the intersections of all the line pairs.
Hope that helps!
I tried to use convex hull method which is pretty simple.
Here you find convex hull of the contour detected. It removes the convexity defects at the bottom of paper.
Below is the code (in OpenCV-Python):
import cv2
import numpy as np
img = cv2.imread('sof.jpg')
img = cv2.resize(img,(500,500))
gray = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)
ret,thresh = cv2.threshold(gray,127,255,0)
contours,hier = cv2.findContours(thresh,cv2.RETR_LIST,cv2.CHAIN_APPROX_SIMPLE)
for cnt in contours:
if cv2.contourArea(cnt)>5000: # remove small areas like noise etc
hull = cv2.convexHull(cnt) # find the convex hull of contour
hull = cv2.approxPolyDP(hull,0.1*cv2.arcLength(hull,True),True)
if len(hull)==4:
cv2.drawContours(img,[hull],0,(0,255,0),2)
cv2.imshow('img',img)
cv2.waitKey(0)
cv2.destroyAllWindows()
(Here, i haven't found square in all planes. Do it yourself if you want.)
Below is the result i got:
I hope this is what you needed.
1st: start experimenting with threshold techniques to isolate the white paper sheet from the rest of the image. This is a simple way:
Mat new_img = imread(argv[1]);
double thres = 200;
double color = 255;
threshold(new_img, new_img, thres, color, CV_THRESH_BINARY);
imwrite("thres.png", new_img);
but there are other alternatives that could provide better result. One is to investigate inRange(), and another is to detect through color by converting the image to the HSV color space.
This thread also provides an interest discussion on the subject.
2nd: after you execute one of this procedures, you could try to feed the result directly into find_squares():
An alternative to find_squares() is to implement the bounding box technique, which has the potential to provide a more accurate detection of the rectangular area (provided that you have a perfect result of threshold). I've used it here and here. It's worth noting that OpenCV has it's own bounding box tutorial.
Another approach besides find_squares(), as pointed by Abid on his answer, is to use the convexHull method. Check OpenCV's C++ tutorial on this method for code.
convert to lab space
use kmeans for 2 clusters
detect suqares one internal cluster it will solve many thing in the rgb space
I am trying to find the edges of the centered box in this image:
I have tried using a HoughLines using dRho=img_width/1000, dTheta=pi/180, and threshold=250
It works great on this image, scaled to 1/3 of the size, but on the full size image it just gets lines everywhere in every direction...
What can I do to tune this to be more accurate?
The code to achieve the result below is a slight modification of the one presented in this answer: how to detect a square:
The original program can be found inside OpenCV, it's called squares.cpp. The code below was modified to search squares only in the first color plane, but as it still detects many squares, at the end of the program I discard all of them except the first, and then call draw_squares() to show what was detected. You can change this easilly to draw all of them and see everything that was detected.
You can do all sorts of thing from now own, including setting a (ROI) region of interest to extract the area that's inside the square (ignore everything else around it).
You can see that the detected rectangle is not perfectly aligned with the lines in the image. You should perform some pre-processing (erode?) operations in the image to decrease the thickness of lines and improve the detection. But from here on it's all on you:
#include <cv.h>
#include <highgui.h>
using namespace cv;
double angle( cv::Point pt1, cv::Point pt2, cv::Point pt0 ) {
double dx1 = pt1.x - pt0.x;
double dy1 = pt1.y - pt0.y;
double dx2 = pt2.x - pt0.x;
double dy2 = pt2.y - pt0.y;
return (dx1*dx2 + dy1*dy2)/sqrt((dx1*dx1 + dy1*dy1)*(dx2*dx2 + dy2*dy2) + 1e-10);
}
void find_squares(Mat& image, vector<vector<Point> >& squares)
{
// TODO: pre-processing
// blur will enhance edge detection
Mat blurred(image);
medianBlur(image, blurred, 9);
Mat gray0(blurred.size(), CV_8U), gray;
vector<vector<Point> > contours;
// find squares in the first color plane.
for (int c = 0; c < 1; c++)
{
int ch[] = {c, 0};
mixChannels(&blurred, 1, &gray0, 1, ch, 1);
// try several threshold levels
const int threshold_level = 2;
for (int l = 0; l < threshold_level; l++)
{
// Use Canny instead of zero threshold level!
// Canny helps to catch squares with gradient shading
if (l == 0)
{
Canny(gray0, gray, 10, 20, 3); //
// Dilate helps to remove potential holes between edge segments
dilate(gray, gray, Mat(), Point(-1,-1));
}
else
{
gray = gray0 >= (l+1) * 255 / threshold_level;
}
// Find contours and store them in a list
findContours(gray, contours, CV_RETR_LIST, CV_CHAIN_APPROX_SIMPLE);
// Test contours
vector<Point> approx;
for (size_t i = 0; i < contours.size(); i++)
{
// approximate contour with accuracy proportional
// to the contour perimeter
approxPolyDP(Mat(contours[i]), approx, arcLength(Mat(contours[i]), true)*0.02, true);
// Note: absolute value of an area is used because
// area may be positive or negative - in accordance with the
// contour orientation
if (approx.size() == 4 &&
fabs(contourArea(Mat(approx))) > 1000 &&
isContourConvex(Mat(approx)))
{
double maxCosine = 0;
for (int j = 2; j < 5; j++)
{
double cosine = fabs(angle(approx[j%4], approx[j-2], approx[j-1]));
maxCosine = MAX(maxCosine, cosine);
}
if (maxCosine < 0.3)
squares.push_back(approx);
}
}
}
}
}
void draw_squares(Mat& img, vector<vector<Point> > squares)
{
for (int i = 0; i < squares.size(); i++)
{
for (int j = 0; j < squares[i].size(); j++)
{
cv::line(img, squares[i][j], squares[i][(j+1) % 4], cv::Scalar(0, 255, 0), 1, CV_AA);
}
}
}
int main(int argc, char* argv[])
{
Mat img = imread(argv[1]);
vector<vector<Point> > squares;
find_squares(img, squares);
std::cout << "* " << squares.size() << " squares were found." << std::endl;
// Ignore all the detected squares and draw just the first found
vector<vector<Point> > tmp;
if (squares.size() > 0)
{
tmp.push_back(squares[0]);
draw_squares(img, tmp);
}
//imshow("squares", img);
//cvWaitKey(0);
imwrite("out.png", img);
return 0;
}
when resizing the image, the image is normally first blurred with a filter, e.g. Gaussian, in order to get rid of high frequencies. The fact that resized one works better is likely because your original image is somehow noisy.
Try blur the image first, e.g. with cv::GaussianBlur(src, target, Size(0,0), 1.5), then it should be equivalent to resizing. (It forgot the theory, if it does not work, try 3 and 6 as well)
Try using a preprocessing pass with the erosion filter. It will give you the same effect as the downscaling - the lines will become thinner and will not disappear at the same time.
The "Blur" filter is also a good idea, as chaiy says.
This way (with blur) it will become something like http://www.ic.uff.br/~laffernandes/projects/kht/index.html (Kernel Based Hough Transform)