Getting a book image using opencv and c++ - c++

Question

I'm trying to detect the following book, using findcontours but it cannot be detected at all and I get exception because there is no convex hull.
I tried to blur, dilate, canny detection, with no success at all.
I hope to get a solution for finding a rectangular paper/book using openCV.
Please let me know if you have further questions or need resources.
#include "opencv2/imgproc.hpp"
#include "opencv2/imgcodecs.hpp"
#include "opencv2/highgui.hpp"
#include <iostream>
using namespace cv;
using namespace std;
double angle(cv::Point pt1, cv::Point pt2, cv::Point pt0) {
double dx1 = pt1.x - pt0.x;
double dy1 = pt1.y - pt0.y;
double dx2 = pt2.x - pt0.x;
double dy2 = pt2.y - pt0.y;
return (dx1*dx2 + dy1*dy2) / sqrt((dx1*dx1 + dy1*dy1)*(dx2*dx2 + dy2*dy2) + 1e-10);
}
void find_squares(Mat& image, vector<vector<Point> >& squares)
{
// blur will enhance edge detection
Mat blurred(image);
Mat dst;
medianBlur(image, dst, 9);
Mat gray0(dst.size(), CV_8U), gray;
vector<vector<Point> > contours;
// find squares in every color plane of the image
for (int c = 0; c < 3; c++)
{
int ch[] = { c, 0 };
mixChannels(&dst, 1, &gray0, 1, ch, 1);
// try several threshold levels
const int threshold_level = 2;
for (int l = 0; l < threshold_level; l++)
{
// Use Canny instead of zero threshold level!
// Canny helps to catch squares with gradient shading
if (l == 0)
{
Canny(gray0, gray, 10, 20, 3); //
// Dilate helps to remove potential holes between edge segments
dilate(gray, gray, Mat(), Point(-1, -1));
}
else
{
gray = gray0 >= (l + 1) * 255 / threshold_level;
}
// Find contours and store them in a list
findContours(gray, contours, CV_RETR_LIST, CV_CHAIN_APPROX_SIMPLE);
// Test contours
vector<Point> approx;
for (size_t i = 0; i < contours.size(); i++)
{
// approximate contour with accuracy proportional
// to the contour perimeter
approxPolyDP(Mat(contours[i]), approx, arcLength(Mat(contours[i]), true)*0.02, true);
// Note: absolute value of an area is used because
// area may be positive or negative - in accordance with the
// contour orientation
if (approx.size() == 4 &&
fabs(contourArea(Mat(approx))) > 1000 &&
isContourConvex(Mat(approx)))
{
double maxCosine = 0;
for (int j = 2; j < 5; j++)
{
double cosine = fabs(angle(approx[j % 4], approx[j - 2], approx[j - 1]));
maxCosine = MAX(maxCosine, cosine);
}
if (maxCosine < 0.3)
squares.push_back(approx);
}
}
}
}
}
cv::Mat debugSquares(std::vector<std::vector<cv::Point> > squares, cv::Mat image)
{
for (int i = 0; i< squares.size(); i++) {
// draw contour
cv::drawContours(image, squares, i, cv::Scalar(255, 0, 0), 1, 8, std::vector<cv::Vec4i>(), 0, cv::Point());
// draw bounding rect
cv::Rect rect = boundingRect(cv::Mat(squares[i]));
cv::rectangle(image, rect.tl(), rect.br(), cv::Scalar(0, 255, 0), 2, 8, 0);
// draw rotated rect
cv::RotatedRect minRect = minAreaRect(cv::Mat(squares[i]));
cv::Point2f rect_points[4];
minRect.points(rect_points);
for (int j = 0; j < 4; j++) {
cv::line(image, rect_points[j], rect_points[(j + 1) % 4], cv::Scalar(0, 0, 255), 1, 8); // blue
}
}
return image;
}
static std::vector<cv::Point> extremePoints(std::vector<cv::Point>pts)
{
int xmin = 0, ymin = 0, xmax = -1, ymax = -1, i;
Point ptxmin, ptymin, ptxmax, ptymax;
Point pt = pts[0];
ptxmin = ptymin = ptxmax = ptymax = pt;
xmin = xmax = pt.x;
ymin = ymax = pt.y;
for (size_t i = 1; i < pts.size(); i++)
{
pt = pts[i];
if (xmin > pt.x)
{
xmin = pt.x;
ptxmin = pt;
}
if (xmax < pt.x)
{
xmax = pt.x;
ptxmax = pt;
}
if (ymin > pt.y)
{
ymin = pt.y;
ptymin = pt;
}
if (ymax < pt.y)
{
ymax = pt.y;
ptymax = pt;
}
}
std::vector<cv::Point> res;
res.push_back(ptxmin);
res.push_back(ptxmax);
res.push_back(ptymin);
res.push_back(ptymax);
return res;
}
void sortCorners(std::vector<cv::Point2f>& corners)
{
std::vector<cv::Point2f> top, bot;
cv::Point2f center;
// Get mass center
for (int i = 0; i < corners.size(); i++)
center += corners[i];
center *= (1. / corners.size());
for (int i = 0; i < corners.size(); i++)
{
if (corners[i].y < center.y)
top.push_back(corners[i]);
else
bot.push_back(corners[i]);
}
corners.clear();
if (top.size() == 2 && bot.size() == 2) {
cv::Point2f tl = top[0].x > top[1].x ? top[1] : top[0];
cv::Point2f tr = top[0].x > top[1].x ? top[0] : top[1];
cv::Point2f bl = bot[0].x > bot[1].x ? bot[1] : bot[0];
cv::Point2f br = bot[0].x > bot[1].x ? bot[0] : bot[1];
corners.push_back(tl);
corners.push_back(tr);
corners.push_back(br);
corners.push_back(bl);
}
}
int main(int, char**)
{
int largest_area = 0;
int largest_contour_index = 0;
cv::Rect bounding_rect;
Mat src, edges;
src = imread("20628991_10159154614610574_1244594322_o.jpg");
cvtColor(src, edges, COLOR_BGR2GRAY);
GaussianBlur(edges, edges, Size(5, 5), 1.5, 1.5);
erode(edges, edges, Mat());// these lines may need to be optimized
dilate(edges, edges, Mat());
dilate(edges, edges, Mat());
erode(edges, edges, Mat());
Canny(edges, edges, 150, 150, 3); // canny parameters may need to be optimized
imshow("edges", edges);
vector<Point> selected;
vector<vector<Point> > contours;
findContours(edges, contours, RETR_LIST, CHAIN_APPROX_SIMPLE);
for (size_t i = 0; i < contours.size(); i++)
{
Rect minRect = boundingRect(contours[i]);
if (minRect.width > 150 & minRect.height > 150) // this line also need to be optimized
{
selected.insert(selected.end(), contours[i].begin(), contours[i].end());
}
}
convexHull(selected, selected);
RotatedRect minRect = minAreaRect(selected);
std::vector<cv::Point> corner_points = extremePoints(selected);
std::vector<cv::Point2f> corners;
corners.push_back(corner_points[0]);
corners.push_back(corner_points[1]);
corners.push_back(corner_points[2]);
corners.push_back(corner_points[3]);
sortCorners(corners);
cv::Mat quad = cv::Mat::zeros(norm(corners[1] - corners[2]), norm(corners[2] - corners[3]), CV_8UC3);
std::vector<cv::Point2f> quad_pts;
quad_pts.push_back(cv::Point2f(0, 0));
quad_pts.push_back(cv::Point2f(quad.cols, 0));
quad_pts.push_back(cv::Point2f(quad.cols, quad.rows));
quad_pts.push_back(cv::Point2f(0, quad.rows));
cv::Mat transmtx = cv::getPerspectiveTransform(corners, quad_pts);
cv::warpPerspective(src, quad, transmtx, quad.size());
resize(quad, quad, Size(), 0.25, 0.25); // you can remove this line to keep the image original size
imshow("quad", quad);
polylines(src, selected, true, Scalar(0, 0, 255), 2);
resize(src, src, Size(), 0.5, 0.5); // you can remove this line to keep the image original size
imshow("result", src);
waitKey(0);
return 0;
}

Answer 1

Strange, I did it with exactly that (blur, dilate, canny):
The code (in Python, but there's nothing but OpenCV function calls, so should be easy to follow; as one of the references I used this answer, which is in C++, it also shows how to correct the perspective and turn it into a rectangle):
import numpy as np
import cv2
img = cv2.imread('sngo1.jpg')
#resize and create a copy for future drawing
resize_coeff = 0.5
w, h, c = img.shape
img_in = cv2.resize(img, (int(resize_coeff*h), int(resize_coeff*w)))
img_out = img_in.copy()
#median and canny
img_in = cv2.medianBlur(img_in, 5)
img_in = cv2.Canny(img_in, 100, 200)
#morphological close for our edges
kernel = np.ones((17, 17), np.uint8)
img_in = cv2.morphologyEx(img_in, cv2.MORPH_CLOSE, kernel, iterations = 1)
#find contours, get max by area
img_in, contours, hierarchy = cv2.findContours(img_in, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
max_index, max_area = max(enumerate([cv2.contourArea(x) for x in contours]), key = lambda x: x[1])
max_contour = contours[max_index]
#approximage it with a quadrangle
approx = cv2.approxPolyDP(max_contour, 0.1*cv2.arcLength(max_contour, True), True)
approx = approx[:,0,:]
cv2.drawContours(img_out, [approx], 0, (255, 0, 0), 2)
cv2.imwrite("result.png", img_out)