Basic Shape recognition (openCV C++) - c++

I have a project that I need to make for classes and I chose task that is a bit out of my skills.
Target is to count result of dice rolls.
For now, I'am trying to make it work on a sample pic:
sample pic of dices
and my current code is added below:
#include "opencv2/core/core.hpp"
#include "opencv2/highgui/highgui.hpp"
#include "opencv2/imgproc/imgproc.hpp"
#include "iostream"
using namespace cv;
using namespace std;
Mat KostkaFunkcja(Mat image, Mat in, Scalar low, Scalar high);
int getMaxAreaContourId(vector <vector<cv::Point>> contours);
vector<Point> contoursConvexHull(vector<vector<Point> > contours, int index);
Mat ZnakiFunkcja(Mat image, Mat in, Scalar low, Scalar high);
int main(int argc, char** argv)
{
Mat image;
image = imread("kostki.jpg", CV_LOAD_IMAGE_COLOR);
if (!image.data)
{
cout << "Could not open or find the image" << std::endl;
return -1;
}
Mat imgHSV;
Mat workimage = image;
cvtColor(workimage, imgHSV, COLOR_BGR2HSV); //Convert the captured frame from BGR to HSV
//red dice
workimage = KostkaFunkcja(workimage, imgHSV, Scalar(146, 0, 31), Scalar(179, 255, 255));
//green dice
workimage = KostkaFunkcja(workimage, imgHSV, Scalar(25, 147, 0), Scalar(98, 255, 154));
//yellow dice
workimage = KostkaFunkcja(workimage, imgHSV, Scalar(22, 45, 161), Scalar(91, 255, 255));
//black dice
workimage = KostkaFunkcja(workimage, imgHSV, Scalar(98, 0, 0), Scalar(179, 232, 107));
//white symbols
workimage = ZnakiFunkcja(workimage, imgHSV, Scalar(58, 0, 183), Scalar(179, 145, 255));
namedWindow("Kostki_kontur", CV_WINDOW_AUTOSIZE);
imshow("Kostki_kontur", workimage);
waitKey(0);
return 0;
}
Mat KostkaFunkcja(Mat image, Mat in, Scalar low, Scalar high)
{
Mat temp;
inRange(in, low, high, temp);
erode(temp, temp, getStructuringElement(MORPH_ELLIPSE, Size(5, 5)));
dilate(temp, temp, getStructuringElement(MORPH_ELLIPSE, Size(5, 5)));
dilate(temp, temp, getStructuringElement(MORPH_ELLIPSE, Size(5, 5)));
erode(temp, temp, getStructuringElement(MORPH_ELLIPSE, Size(5, 5)));
Mat srcBlur, srcCanny;
blur(temp, srcBlur, Size(3, 3));
Canny(srcBlur, srcCanny, 0, 100, 3, true);
vector<vector<Point> > contours;
findContours(srcCanny, contours, CV_RETR_EXTERNAL, CV_CHAIN_APPROX_SIMPLE);
int largest_contour_index = getMaxAreaContourId(contours);
Mat drawing = Mat::zeros(srcCanny.size(), CV_8UC3);
for (int i = 0; i< contours.size(); i++)
{
Scalar color = Scalar(255, 255, 255);
drawContours(drawing, contours, i, color, 2);
}
vector<Point> ConvexHullPoints = contoursConvexHull(contours, largest_contour_index);
polylines(image, ConvexHullPoints, true, Scalar(0, 0, 255), 2);
return image;
}
vector<Point> contoursConvexHull(vector<vector<Point> > contours, int index)
{
vector<Point> result;
vector<Point> pts;
for (size_t j = 0; j< contours[index].size(); j++)
pts.push_back(contours[index][j]);
convexHull(pts, result);
return result;
}
int getMaxAreaContourId(vector <vector<cv::Point>> contours)
{
double maxArea = 0;
int maxAreaContourId = -1;
for (int j = 0; j < contours.size(); j++) {
double newArea = cv::contourArea(contours.at(j));
if (newArea > maxArea) {
maxArea = newArea;
maxAreaContourId = j;
}
return maxAreaContourId;
}
}
Mat ZnakiFunkcja(Mat image, Mat in, Scalar low, Scalar high)
{
Mat temp;
inRange(in, low, high, temp);
erode(temp, temp, getStructuringElement(MORPH_ELLIPSE, Size(5, 5)));
dilate(temp, temp, getStructuringElement(MORPH_ELLIPSE, Size(5, 5)));
dilate(temp, temp, getStructuringElement(MORPH_ELLIPSE, Size(5, 5)));
erode(temp, temp, getStructuringElement(MORPH_ELLIPSE, Size(5, 5)));
Mat srcBlur, srcCanny;
blur(temp, srcBlur, Size(3, 3));
Canny(srcBlur, srcCanny, 0, 100, 3, true);
vector<vector<Point> > contours;
findContours(srcCanny, contours, CV_RETR_EXTERNAL, CV_CHAIN_APPROX_SIMPLE);
Mat drawing = Mat::zeros(srcCanny.size(), CV_8UC3);
for (int i = 0; i< contours.size(); i++)
{
Scalar color = Scalar(255, 255, 255);
drawContours(drawing, contours, i, color, 2);
polylines(image, contours, true, Scalar(0, 0, 255), 2);
return image;
}
}
Yet I have no idea how to count different shapes (hearts, lightnings, shields, numbers).
I will be greatfull if anybody would give me a tip or solution of how to do the job.
1) sorry for bad english
2) we had no openCV in classes [only basic c++]
3) tryed to found anything usefull on internet, but even if I found anything, I could't understand what was happening in the code

Your project can be splited in three steps:
find the dices.
extract the shapes from the visible face of
the dices.
count the faces.
For the first step among all the possible approaches I think saliency map approaches can help.
Saliency map are a family of segmentation algorithm that aim to detect the parts in the image which are more likely to attract visual attention.
OpenCV have a saliency API that already implement several saliency algorithm and for each of them you can get an segmentation map.
It is highlikely considering the example image you gave the saliency will be focus on the dices.
From this you can so extract the dices as rois from the original image.
For the step 2) saliency algorithms may also fit... or not that depend a lot of the statistical criterions that are used by the algorithm.
However the previously extracted rois should only contain the face of the dice that does contain the shapes you want to count in step 3) so approaches based on contours detection may give quite good result.
Once you get the shapes among the way to count each shape you can use templateMatching (that is also already implement in OpenCV),a clustering approach based on the a shape sensitive metric (Hausdorff, Dice, ...), or many other.
Here is a code that can help you to deal with the two first step.
#ifndef _DEBUG
#define _DEBUG
#endif
#include <iostream>
#include <opencv2/core.hpp>
#include <opencv2/imgproc.hpp>
#include <opencv2/saliency.hpp>
#include <opencv2/highgui.hpp>
#include <list>
CV_EXPORTS_W void get_regions_of_interest(cv::InputArray _src, cv::OutputArrayOfArrays mv, cv::OutputArrayOfArrays mv2 = cv::noArray());
int main()
{
cv::Mat tmp = cv::imread("C:\Desktop\dices.jpg");
if(!tmp.empty())
{
cv::imshow("source",tmp);
cv::waitKey(-1);
}
std::vector<cv::Mat> rois;
get_regions_of_interest(tmp,rois);
std::cout << "Hello World!" << std::endl;
return 0;
}
void get_regions_of_interest(cv::InputArray _src, cv::OutputArrayOfArrays _rois, cv::OutputArrayOfArrays _contours)
{
// Check that the first argument is an image and the second a vector of images.
CV_Assert(_src.isMat() && !_src.depth() && (_src.channels() == 1 || _src.channels() == 3) && _rois.isMatVector() && (!_contours.needed() || (_contours.needed() && _contours.isMatVector()) ) );
static cv::Ptr<cv::saliency::StaticSaliencySpectralResidual> saliency;
if(!saliency)
saliency = cv::saliency::StaticSaliencySpectralResidual::create();
cv::Mat src = _src.getMat();
cv::Mat gray;
if(src.depth() == src.type())
gray = src;
else
cv::cvtColor(src,gray,cv::COLOR_BGR2GRAY);
bool is_ctr_needed = _contours.needed();
std::list<cv::Mat> final_ctrs;
// Step 1) Process the saliency in order to segment the dices.
cv::Mat saliency_map;
cv::Mat binary_map;
saliency->computeSaliency(src,saliency_map);
saliency->computeBinaryMap(saliency_map,binary_map);
saliency_map.release();
// Step 2) From the binary map get the regions of interest.
cv::Mat1i stats;
std::vector<cv::Mat> rois;
cv::Mat labels;
cv::Mat centroids;
cv::connectedComponentsWithStats(binary_map, labels, stats, centroids);
labels.release();
centroids.release();
// prepare the memory
rois.reserve(stats.rows-1);
// Sort the stats in order to remove the background.
stats = stats.colRange(0,stats.cols-1);
// Extract the rois.
for(int i=0;i<stats.rows;i++)
{
cv::Rect roi = *reinterpret_cast<cv::Rect*>(stats.ptr<int>(i));
if(static_cast<std::size_t>(roi.area()) == gray.total())
continue;
rois.push_back(gray(roi));
#ifdef _DEBUG
cv::imshow("roi_"+std::to_string(i),gray(roi));
#endif
}
// Step 3) Refine.
// Because the final number of shape cannot be determine in advance it is better to use a linked list than a vector.
// In practice except if there is a huge number of elements to work with the performance will be almost the same.
std::list<cv::Mat> shapes;
int cnt=0;
for(const cv::Mat& roi : rois)
{
cv::Mat tmp = roi.clone();
// Slightly sharpen the regions contours
cv::morphologyEx(tmp,tmp, cv::MORPH_CLOSE, cv::noArray());
// Reduce the influence of local unhomogeneous illumination.
cv::GaussianBlur(tmp,tmp,cv::Size(31,31), 5);
cv::Mat thresh;
// Binarize the image.
cv::threshold(roi,thresh,0.,255.,cv::THRESH_BINARY | cv::THRESH_OTSU);
#ifdef _DEBUG
cv::imshow("thresh"+std::to_string(cnt++),thresh);
#endif
// Find the contours of each sub region on interest
std::vector<cv::Mat> contours;
cv::findContours(thresh, contours, cv::RETR_TREE, cv::CHAIN_APPROX_SIMPLE);
cv::Mat dc;
cv::merge(std::vector<cv::Mat>(3,thresh),dc);
// cv::drawContours(dc, contours,-1,cv::Scalar(0.,0.,255),2);
// cv::imshow("ctrs"+std::to_string(cnt),dc);
// Extract the sub-regions
if(is_ctr_needed)
{
for(const cv::Mat& ctrs: contours)
{
cv::Rect croi = cv::boundingRect(ctrs);
// If the sub region is to big or to small it is depreate
if(static_cast<std::size_t>(croi.area()) == roi.total() || croi.area()<50)
continue;
final_ctrs.push_back(ctrs);
shapes.push_back(roi(croi));
#ifdef _DEBUG
cv::rectangle(dc,croi,cv::Scalar(0.,0.,255.));
cv::imshow("sub_roi_"+std::to_string(cnt++),roi(croi));
#endif
}
}
else
{
for(const cv::Mat& ctrs: contours)
{
cv::Rect croi = cv::boundingRect(ctrs);
// If the sub region is to big or to small it is depreate
if(static_cast<std::size_t>(croi.area()) == roi.total() || croi.area()<50)
continue;
shapes.push_back(roi(croi));
#ifdef _DEBUG
cv::rectangle(dc,croi,cv::Scalar(0.,0.,255.));
cv::imshow("sub_roi_"+std::to_string(cnt++),roi(croi));
#endif
}
}
}
#ifdef _DEBUG
cv::waitKey(-1);
#endif
// Final Step: set the output
_rois.create(shapes.size(),1,CV_8U);
_rois.assign(std::vector<cv::Mat>(shapes.begin(),shapes.end()));
if(is_ctr_needed)
{
_contours.create(final_ctrs.size(),1,CV_32SC2);
_contours.assign(std::vector<cv::Mat>(final_ctrs.begin(), final_ctrs.end()));
}
}

Related

C++ OpenCV - Find biggest object in an webcam stream and sort it by size

My goal is to find the biggest contour of a captured webcam frame, then after it's found, find its size and determine either to be rejected or accepted.
Just to explain the objetive of this project, i am currently working for a Hygiene product's Manufacturer. There we have, in total, 6 workers that are responsible for sorting the defective soap bars out of the production line. So in order to gain this workforce for other activities, i am trying to write an algorithm to "replace" their eyes.
I've tried several methods along the way (findcontours, SimpleBlobDetection, Canny, Object tracking), but the problem that i've been facing is that i can't seem to find a way to effectively find the biggest object in a webcam image, find its size and then determine to either discard or accept it.
Below follows my newest code to find the biggest contour in an webcam stream:
#include <iostream>
#include "opencv2/highgui/highgui.hpp"
#include "opencv/cv.h"
#include "opencv2\imgproc\imgproc.hpp"
using namespace cv;
using namespace std;
int main(int argc, const char** argv)
{
Mat src;
Mat imgGrayScale;
Mat imgCanny;
Mat imgBlurred;
/// Load source image
VideoCapture capWebcam(0);
if (capWebcam.isOpened() == false)
{
cout << "Não foi possível abrir webcam!" << endl;
return(0);
}
while (capWebcam.isOpened())
{
bool blnframe = capWebcam.read(src);
if (!blnframe || src.empty())
{
cout << "Erro! Frame não lido!\n";
break;
}
int largest_area = 0;
int largest_contour_index = 0;
Rect bounding_rect;
Mat thr(src.rows, src.cols, CV_8UC1);
Mat dst(src.rows, src.cols, CV_8UC1, Scalar::all(0));
cvtColor(src, imgGrayScale, CV_BGR2GRAY); //Convert to gray
GaussianBlur(imgGrayScale, imgBlurred, Size(5, 5), 1.8);
Canny(imgBlurred, imgCanny, 45, 90); //Threshold the gray
vector<vector<Point>> contours; // Vector for storing contour
vector<Vec4i> hierarchy;
findContours(imgCanny, contours, hierarchy, CV_RETR_CCOMP, CV_CHAIN_APPROX_SIMPLE); // Find the contours in the image
for (int i = 0; i < contours.size(); i++) // iterate through each contour.
{
double a = contourArea(contours[i], false); // Find the area of contour
if (a > largest_area)
{
largest_area = a;
largest_contour_index = i; //Store the index of largest contour
bounding_rect = boundingRect(contours[i]); // Find the bounding rectangle for biggest contour
}
}
Scalar color(255, 255, 255);
drawContours(dst, contours, largest_contour_index, color, CV_FILLED, 8, hierarchy); // Draw the largest contour using previously stored index.
rectangle(src, bounding_rect, Scalar(0, 255, 0), 1, 8, 0);
imshow("src", src);
imshow("largest Contour", dst);
waitKey(27);
}
return(0);
}
And here are the results windows that the program generates and the image of the object that i want to detect and sort.
Thank you all in advance for any clues on how to achieve my goal.

OpenCV--how to get better hand contour from low quality gray image?

I need to get contour from hand image, usually I process image with 4 steps:
get raw RGB gray image from 3 channels to 1 channel:
cvtColor(sourceGrayImage, sourceGrayImage, COLOR_BGR2GRAY);
use Gaussian blur to filter gray image:
GaussianBlur(sourceGrayImage, sourceGrayImage, Size(3,3), 0);
binary gray image, I split image by height, normally I split image to 6 images by its height, then each one I do threshold process:
// we split source picture to binaryImageSectionCount(here it's 8) pieces by its height,
// then we for every piece, we do threshold,
// and at last we combine them agin to binaryImage
const binaryImageSectionCount = 8;
void GetBinaryImage(Mat &grayImage, Mat &binaryImage)
{
// get every partial gray image's height
int partImageHeight = grayImage.rows / binaryImageSectionCount;
for (int i = 0; i < binaryImageSectionCount; i++)
{
Mat partialGrayImage;
Mat partialBinaryImage;
Rect partialRect;
if (i != binaryImageSectionCount - 1)
{
// if it's not last piece, Rect's height should be partImageHeight
partialRect = Rect(0, i * partImageHeight, grayImage.cols, partImageHeight);
}
else
{
// if it's last piece, Rect's height should be (grayImage.rows - i * partImageHeight)
partialRect = Rect(0, i * partImageHeight, grayImage.cols, grayImage.rows - i * partImageHeight);
}
Mat partialResource = grayImage(partialRect);
partialResource.copyTo(partialGrayImage);
threshold( partialGrayImage, partialBinaryImage, 0, 255, THRESH_OTSU);
// combin partial binary image to one piece
partialBinaryImage.copyTo(binaryImage(partialRect));
///*stringstream resultStrm;
//resultStrm << "partial_" << (i + 1);
//string string = resultStrm.str();
//imshow(string, partialBinaryImage);
//waitKey(0);*/
}
imshow("result binary image.", binaryImage);
waitKey(0);
return;
}
use findcontour to get biggest area contour:
vector<vector<Point> > contours;
findContours(binaryImage, contours, CV_RETR_EXTERNAL, CV_CHAIN_APPROX_SIMPLE);
normally it works well,
But for some low quality gray image, it doesn't work,like below:
the complete code is here:
#include <opencv2/imgproc/imgproc.hpp>
#include<opencv2/opencv.hpp>
#include <opencv2/highgui/highgui.hpp>
using namespace std;
using namespace cv;
// we split source picture to binaryImageSectionCount(here it's 8) pieces by its height,
// then we for every piece, we do threshold,
// and at last we combine them agin to binaryImage
const binaryImageSectionCount = 8;
void GetBinaryImage(Mat &grayImage, Mat &binaryImage)
{
// get every partial gray image's height
int partImageHeight = grayImage.rows / binaryImageSectionCount;
for (int i = 0; i < binaryImageSectionCount; i++)
{
Mat partialGrayImage;
Mat partialBinaryImage;
Rect partialRect;
if (i != binaryImageSectionCount - 1)
{
// if it's not last piece, Rect's height should be partImageHeight
partialRect = Rect(0, i * partImageHeight, grayImage.cols, partImageHeight);
}
else
{
// if it's last piece, Rect's height should be (grayImage.rows - i * partImageHeight)
partialRect = Rect(0, i * partImageHeight, grayImage.cols, grayImage.rows - i * partImageHeight);
}
Mat partialResource = grayImage(partialRect);
partialResource.copyTo(partialGrayImage);
threshold( partialGrayImage, partialBinaryImage, 0, 255, THRESH_OTSU);
// combin partial binary image to one piece
partialBinaryImage.copyTo(binaryImage(partialRect));
///*stringstream resultStrm;
//resultStrm << "partial_" << (i + 1);
//string string = resultStrm.str();
//imshow(string, partialBinaryImage);
//waitKey(0);*/
}
imshow("result binary image.", binaryImage);
waitKey(0);
return;
}
int main(int argc, _TCHAR* argv[])
{
// get image path
string imgPath("C:\\Users\\Alfred\\Desktop\\gray.bmp");
// read image
Mat src = imread(imgPath);
imshow("Source", src);
//medianBlur(src, src, 7);
cvtColor(src, src, COLOR_BGR2GRAY);
imshow("gray", src);
// do filter
GaussianBlur(src, src, Size(3,3), 0);
// binary image
Mat threshold_output(src.rows, src.cols, CV_8UC1, Scalar(0, 0, 0));
GetBinaryImage(src, threshold_output);
imshow("binaryImage", threshold_output);
// get biggest contour
vector<vector<Point> > contours;
findContours(threshold_output,contours, CV_RETR_EXTERNAL, CV_CHAIN_APPROX_SIMPLE);
int biggestContourIndex = 0;
int maxContourArea = -1000;
for (int i = 0; i < contours.size(); i++)
{
if (contourArea(contours[i]) > maxContourArea)
{
maxContourArea = contourArea(contours[i]);
biggestContourIndex = i;
}
}
// show biggest contour
Mat biggestContour(threshold_output.rows, threshold_output.cols, CV_8UC1, Scalar(0, 0, 0));
drawContours(biggestContour, contours, biggestContourIndex, cv::Scalar(255,255,255), 2, 8, vector<Vec4i>(), 0, Point());
imshow("maxContour", biggestContour);
waitKey(0);
}
could anybody please help me to get a better hand contour result?
thanks!!!
I have the code snippet in python, you can follow the same approach in C:
img = cv2.imread(x, 1)
cv2.imshow("img",img)
imgray = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)
cv2.imshow("gray",imgray)
#Code for histogram equalization
equ = cv2.equalizeHist(imgray)
cv2.imshow('equ', equ)
#Code for contrast limited adaptive histogram equalization
#clahe = cv2.createCLAHE(clipLimit=3.0, tileGridSize=(8,8))
#cl2 = clahe.apply(imgray)
#cv2.imshow('clahe2', cl2)
This is the result I obtained:
If you're image is horribly bad you could try the code that I commented involving contrast limited adaptive histogram equalization.

How to count white object on Binary Image?

I'm trying to count object from image. I use logs photo, and I use some steps to get a binary image.
This is my code:
#include <opencv2/core/core.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <iostream>
#include <features2d.hpp>
using namespace cv;
using namespace std;
int main(int argc, char *argv[])
{
//load image
Mat img = imread("kayu.jpg", CV_LOAD_IMAGE_COLOR);
if(img.empty())
return -1;
//namedWindow( "kayu", CV_WINDOW_AUTOSIZE );
imshow("kayu", img);
//convert to b/w
Mat bw;
cvtColor(img, bw, CV_BGR2GRAY);
imshow("bw1", bw);
threshold(bw, bw, 40, 255, CV_THRESH_BINARY);
imshow("bw", bw);
//distance transform & normalisasi
Mat dist;
distanceTransform(bw, dist, CV_DIST_L2, 3);
normalize(dist, dist, 0, 2., NORM_MINMAX);
imshow("dist", dist);
//threshold to draw line
threshold(dist, dist, .5, 1., CV_THRESH_BINARY);
imshow("dist2", dist);
//dist = bw;
//dilasi
Mat dilation, erotion, element;
int dilation_type = MORPH_ELLIPSE;
int dilation_size = 17;
element = getStructuringElement(dilation_type, Size(2*dilation_size + 1, 2*dilation_size+1), Point(dilation_size, dilation_size ));
erode(dist, erotion, element);
int erotionCount = 0;
for(int i=0; i<erotionCount; i++){
erode(erotion, erotion, element);
}
imshow("erotion", erotion);
dilate(erotion, dilation, element);
imshow("dilation", dilation);
waitKey(0);
return 0;
}
As you can see, I use Erosion and Dilation to get better circular object of log. My problem is, I'm stuck at counting the object. I tried SimpleBlobDetector but I got nothing, because when I try to convert the result of "dilation" step to CV_8U, the white object disappear. I got error too when I use findContours(). It say something about channel of image. I can't show the error here, because that's too many step and I already delete it from my code.
Btw, at the end, i got 1 channel of image.
Can i just use it to counting, or am i have to convert it and what is the best method to do it?
Two simple steps:
Find contours for the binarized image.
Get the count of the contours.
Code:
int count_trees(const cv::Mat& bin_image){
cv::Mat img;
if(bin_image.channels()>1){
cv::cvtColor(bin_image,img,cv::COLOR_BGR2GRAY);
}
else{
img=bin_image.clone();;
}
if(img.type()!=CV_8UC1){
img*=255.f; //This could be stupid, but I do not have an environment to try it
img.convertTo(img,CV_8UC1);
}
std::vector<std::vector<cv::Point>> contours
std::vector<Vec4i> hierarchy;
cv::findContours( img, contours, hierarchy, CV_RETR_EXTERNAL, CV_CHAIN_APPROX_SIMPLE);
return contours.size();
}
I have the same problem, here's an idea I'm about to implement.
1) Represent your image as an array of integers; 0 = black, 1 = white.
2) set N = 2;
3) Scan your image, pixel-by-pixel. Whenever you find a white pixel, activate a flood-fill algorithm, starting at the pixel just found; paint the region with the value of N++;
4) Iterate 3 until you reach the last pixel. (N-2) is the number of regions found.
This method depends on the shape of the objects; mine are more chaotic than yours (wish me luck..). I'll make use of a recursive flood-fill recipe found somewhere (maybe Rosetta Code).
This solution also makes it easy to compute the size of each region.
try to apply that on the your deleted img
// count
for (int i = 0; i< contours.size(); i = hierarchy[i][0]) // iteration sur chaque contour .
{
Rect r = boundingRect(contours[i]);
if (hierarchy[i][2]<0) {
rectangle(canny_output, Point(r.x, r.y), Point(r.x + r.width, r.y + r.height), Scalar(20, 50, 255), 3, 8, 0);
count++;
}
}
cout << "Numeber of contour = " << count << endl;
imshow("src", src);
imshow("contour", dst);
waitKey(0);

Removing lines from image

I am a beginner in OpenCV, I need to remove the horizontal and vertical lines in the image so that only the text remains ( The lines were causing trouble when extracting text in ocr ). I am trying to extract text from the Nutrient Fact Table. Can anyone help me?
This was an interesting question, so I gave it a shot. Below I will show you how to extract and remove horizontal and vertical lines. You could extrapolate from it. Also, for sake of saving time, I did not preprocess your image to crop out the background as one should, which is an avenue for improvement.
The result:
The code (edit: added vertical lines):
#include <iostream>
#include <opencv2/opencv.hpp>
using namespace std;
using namespace cv;
int main(int, char** argv)
{
// Load the image
Mat src = imread(argv[1]);
// Check if image is loaded fine
if(!src.data)
cerr << "Problem loading image!!!" << endl;
Mat gray;
if (src.channels() == 3)
{
cvtColor(src, gray, CV_BGR2GRAY);
}
else
{
gray = src;
}
//inverse binary img
Mat bw;
//this will hold the result, image to be passed to OCR
Mat fin;
//I find OTSU binarization best for text.
//Would perform better if background had been cropped out
threshold(gray, bw, 0, 255, THRESH_BINARY_INV | THRESH_OTSU);
threshold(gray, fin, 0, 255, THRESH_BINARY | THRESH_OTSU);
imshow("binary", bw);
Mat dst;
Canny( fin, dst, 50, 200, 3 );
Mat str = getStructuringElement(MORPH_RECT, Size(3,3));
dilate(dst, dst, str, Point(-1, -1), 3);
imshow("dilated_canny", dst);
//bitwise_and w/ canny image helps w/ background noise
bitwise_and(bw, dst, dst);
imshow("and", dst);
Mat horizontal = dst.clone();
Mat vertical = dst.clone();
fin = ~dst;
//Image that will be horizontal lines
Mat horizontal = bw.clone();
//Selected this value arbitrarily
int horizontalsize = horizontal.cols / 30;
Mat horizontalStructure = getStructuringElement(MORPH_RECT, Size(horizontalsize,1));
erode(horizontal, horizontal, horizontalStructure, Point(-1, -1));
dilate(horizontal, horizontal, horizontalStructure, Point(-1, -1), 1);
imshow("horizontal_lines", horizontal);
//Need to find horizontal contours, so as to not damage letters
vector<Vec4i> hierarchy;
vector<vector<Point> >contours;
findContours(horizontal, contours, hierarchy, CV_RETR_TREE, CV_CHAIN_APPROX_NONE);
for (const auto& c : contours)
{
Rect r = boundingRect(c);
float percentage_height = (float)r.height / (float)src.rows;
float percentage_width = (float)r.width / (float)src.cols;
//These exclude contours that probably are not dividing lines
if (percentage_height > 0.05)
continue;
if (percentage_width < 0.50)
continue;
//fills in line with white rectange
rectangle(fin, r, Scalar(255,255,255), CV_FILLED);
}
int verticalsize = vertical.rows / 30;
Mat verticalStructure = getStructuringElement(MORPH_RECT, Size(1,verticalsize));
erode(vertical, vertical, verticalStructure, Point(-1, -1));
dilate(vertical, vertical, verticalStructure, Point(-1, -1), 1);
imshow("verticalal", vertical);
findContours(vertical, contours, hierarchy, CV_RETR_TREE, CV_CHAIN_APPROX_NONE);
for (const auto& c : contours)
{
Rect r = boundingRect(c);
float percentage_height = (float)r.height / (float)src.rows;
float percentage_width = (float)r.width / (float)src.cols;
//These exclude contours that probably are not dividing lines
if (percentage_width > 0.05)
continue;
if (percentage_height < 0.50)
continue;
//fills in line with white rectange
rectangle(fin, r, Scalar(255,255,255), CV_FILLED);
}
imshow("Result", fin);
waitKey(0);
return 0;
}
The limitations of this approach are that the lines need to be straight. Due to the curve in the bottom line, it cuts slightly into "E" in "Energy". Perhaps with a hough line detection like suggested (I've never used it), a similar but more robust approach could be devised. Also, filling in the lines with rectangles probably is not the best approach.

Derivatives in OpenCV

I'm writing a program using opencv that does text detection and extraction.
Im using the Sobel derivative in order to do edge detection and have gotten the following result:
But I wish to get the following result:
(I appologize for the blurry image.)
The problem I'm having is the "blank areas" inside the edges "confuse" the algorithem I'm using so when the algorithem detects the "blank part" seperating between two lines from the lines themselves it gets confused and start running into the letter themselves instead of keepeing between two lines. This error, I believe would be solves by achieving the second result.
Anyone knows what changes i need to make? in the soble derivative? maybe use a different derivative?
Code:
Mat ProfileSeamTextLineExtractor::computeDerivative(){
Mat img = _image;
Mat gradiant_mat;
int scale = 2;
int delta = 0;
int ddepth = CV_16S;
GaussianBlur(img, img, Size(3, 3), 0, 0, BORDER_DEFAULT);
Mat grad_x, grad_y;
Mat abs_grad_x, abs_grad_y;
Sobel(img, grad_x, ddepth, 1, 0, 3, scale, delta, BORDER_DEFAULT);
convertScaleAbs(grad_x, abs_grad_x);
Sobel(img, grad_y, ddepth, 0, 1, 3, scale, delta, BORDER_DEFAULT);
convertScaleAbs(grad_y, abs_grad_y);
/// Total Gradient (approximate)
addWeighted(abs_grad_x, 0.5, abs_grad_y, 0.5, 0, gradiant_mat);
return gradiant_mat;
}
Regards,
Try using the second sobel derivative, add, normalize (this may do the same as addWeighted), and then thresholding optimally. I had results similar to yours with different threshold values.
Here's an example:
cv::Mat result;
cvtColor(image, gray, CV_BGR2GRAY);
cv::medianBlur(gray, gray, 3);
cv::Mat sobel_x, sobel_y, result;
cv::Sobel(gray, sobel_x, CV_32FC1, 2, 0, 5);
cv::Sobel(gray, sobel_y, CV_32FC1, 0, 2, 5);
cv::Mat sum = sobel_x + sobel_y;
cv::normalize(sum, result, 0, 255, CV_MINMAX, CV_8UC1);
//Determine optimal threshold value using THRESH_OTSU.
// This didn't give me optimal results, but was a good starting point.
cv::Mat temp, final;
double threshold = cv::threshold(result, temp, 0, 255, CV_THRESH_BINARY+CV_THRESH_OTSU);
cv::threshold(result, final, threshold*.9, 255, CV_THRESH_BINARY);
I was able to clearly extract both light text on a dark background, and dark text on a light background.
If you need the final image to consistently be white background with black text, you can do this:
cv::Scalar avgPixelIntensity = cv::mean( final );
if(avgPixelIntensity[0] < 127.0)
cv::bitwise_not(final, final);
I tried a lot of different text extraction methods and couldn't find any that worked across the board, but this seems to. This took a lot of trial and error to figure out, so I hope this helps.
I don't really understand what your final aim is. Do you eventually want a nice filled in version of the text so you can recognise the characters? I can give that a shot if that's what you are looking for.
This is what I did while trying to remove inner holes:
For this one I didn't bother:
It fails at the edges where the text is cut off.
Obviously, I had to work with the image that had already gone through some processing. I might be able to give you more help if I had the original and produce a better output. You might not even need to use derivatives at all if the background is clean enough.
Here is the code:
#include <opencv2/imgproc/imgproc.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <iostream>
using namespace cv;
using namespace std;
void printInnerContours (int contourPos, Mat &filled, vector<vector<Point2i > > &contours, vector<Vec4i> &hierarchy, int area);
int main() {
int areaThresh;
vector<vector<Point2i > > contours;
vector<Vec4i> hierarchy;
Mat text = imread ("../wHWHA.jpg", 0); //write greyscale
threshold (text, text, 50, 255, THRESH_BINARY);
imwrite ("../text1.jpg", text);
areaThresh = (0.01 * text.rows * text.cols) / 100;
findContours (text, contours, hierarchy, CV_RETR_TREE, CV_CHAIN_APPROX_NONE);
Mat filled = Mat::zeros(text.rows, text.cols, CV_8U);
cout << contours.size() << endl;
for (int i = 0; i < contours.size(); i++) {
int area = contourArea(contours[i]);
if (area > areaThresh) {
if ((hierarchy[i][2] != -1) && (hierarchy[i][3] == -1)) {
drawContours (filled, contours, i, 255, -1);
if (hierarchy[i][2] != -1) {
printInnerContours (hierarchy[i][2], filled, contours, hierarchy, area);
}
}
}
}
imwrite("../output.jpg", filled);
return 0;
}
void printInnerContours (int contourPos, Mat &filled, vector<vector<Point2i > > &contours, vector<Vec4i> &hierarchy, int area) {
int areaFrac = 5;
if (((contourArea (contours[contourPos]) * 100) / area) < areaFrac) {
//drawContours (filled, contours, contourPos, 0, -1);
}
if (hierarchy[contourPos][2] != -1) {
printInnerContours (hierarchy[contourPos][2], filled, contours, hierarchy, area);
}
if (hierarchy[contourPos][0] != -1) {
printInnerContours (hierarchy[contourPos][0], filled, contours, hierarchy, area);
}
}