How to split image in OpenCV based on lines - c++

Question

I am trying to do text segmentation. The attachment below is the results of it.
I manage to form lines to divide the image. However, I am stuck in splitting the image according to the lines that I'd found.
As labeled (red text) in the attached picture, I would like to split the image into 5 different images and I do not know where should I start. All the method I found only work for straight lines.
Header
Code - Source:
#include <opencv2/core/core.hpp>
#include <opencv2/highgui.hpp>
#include <opencv2/opencv.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include <iostream>
#include <ctype.h>
#include <fstream>
#define _USE_MATH_DEFINES
#include <math.h>
#define JC_VORONOI_IMPLEMENTATION
#include "jc_voronoi.h"
typedef struct compPoint
{
cv::Point pointer;
int siteNum, size ;
};
int maximumSize;
float average=0;
std::vector<compPoint> generatePoint(cv::Mat image);
void generateVoronoi(std::vector<cv::Point> points, int width, int height);
static inline jcv_point remap(const jcv_point* pt, const jcv_point* min, const jcv_point* max, const jcv_point* scale);
static void draw_line(int x0, int y0, int x1, int y1, unsigned char* image, int width, int height, int nchannels, unsigned char* color);
static void plot(int x, int y, unsigned char* image, int width, int height, int nchannels, unsigned char* color);
float areaDifference(int s1,int s2);
float areaDifference(int s1, int s2)
{
if (s1 > s2)
{
return s1 / s2;
}
else
{
return s2 / s1;
}
}
std::vector<compPoint> generatePoint(cv::Mat image)
{
cv::Mat grayscale, binary;
cv::cvtColor(image, grayscale, cv::COLOR_BGR2GRAY);
cv::threshold(grayscale, binary, 190, 255, 1);
std::vector<std::vector<cv::Point> > contours;
std::vector<cv::Vec4i> hierarchy;
cv::findContours(binary, contours, hierarchy, cv::RETR_TREE, cv::CHAIN_APPROX_NONE, cv::Point(0, 0));
std::vector<compPoint> extractedPoint;
cv::Mat drawing = cv::Mat::zeros(binary.size(), CV_8UC3);
cv::Scalar color = cv::Scalar(255, 255, 255);
maximumSize = cv::contourArea(contours[0]);
int skip = 0;
for (int i = 0; i < contours.size(); i++)
{
int jumpPoint = contours[i].size() / (contours[i].size() * 0.12);
bool isInner = false;
cv::Vec4i currentHierarchy = hierarchy[i];
if (contours[i].size() <= 20) //Remove small component
continue;
for (int g = 0; g < contours[i].size(); g = g + jumpPoint) //Sample point from connected component
{
compPoint temp;
temp.pointer = contours[i].at(g);
line(drawing, contours[i].at(g), contours[i].at(g), color, 1, 8, 0);
if (currentHierarchy.val[3] != -1)
{
int currentIndex = currentHierarchy.val[3];
while (hierarchy[currentIndex].val[3] != -1)
{
currentIndex = hierarchy[currentIndex].val[3];
}
temp.siteNum = currentIndex;
temp.size = cv::contourArea(contours[currentIndex]);
isInner = true;
}
else
{
temp.siteNum = i;
temp.size = cv::contourArea(contours[i]);
if (cv::contourArea(contours[i])>maximumSize)
{
maximumSize = cv::contourArea(contours[i]);
}
}
extractedPoint.push_back(temp);
}
if (isInner == false)
{
average = average + cv::contourArea(contours[i]);
skip++;
}
}
average = average/skip;
return extractedPoint;
}
static inline jcv_point remap(const jcv_point* pt, const jcv_point* min, const jcv_point* max, const jcv_point* scale)
{
jcv_point p;
p.x = (pt->x - min->x) / (max->x - min->x) * scale->x;
p.y = (pt->y - min->y) / (max->y - min->y) * scale->y;
return p;
}
static void plot(int x, int y, unsigned char* image, int width, int height, int nchannels, unsigned char* color)
{
if (x < 0 || y < 0 || x >(width - 1) || y >(height - 1))
return;
int index = y * width * nchannels + x * nchannels;
for (int i = 0; i < nchannels; ++i)
{
image[index + i] = color[i];
}
}
static void draw_line(int x0, int y0, int x1, int y1, unsigned char* image, int width, int height, int nchannels, unsigned char* color)
{
int dx = abs(x1 - x0), sx = x0<x1 ? 1 : -1;
int dy = -abs(y1 - y0), sy = y0<y1 ? 1 : -1;
int err = dx + dy, e2; // error value e_xy
for (;;)
{ // loop
plot(x0, y0, image, width, height, nchannels, color);
if (x0 == x1 && y0 == y1) break;
e2 = 2 * err;
if (e2 >= dy) { err += dy; x0 += sx; } // e_xy+e_x > 0
if (e2 <= dx) { err += dx; y0 += sy; } // e_xy+e_y < 0
}
}
void generateVoronoi(std::vector<compPoint> points, int width, int height)
{
int size = points.size();
jcv_point* voronoiPoint = (jcv_point*)malloc(sizeof(jcv_point) * (size_t)size);
for (int i = 0; i < size; i++)
{
voronoiPoint[i].x = (float)points[i].pointer.x;
voronoiPoint[i].y = (float)points[i].pointer.y;
voronoiPoint[i].site = points[i].siteNum;
voronoiPoint[i].totalPoint = points[i].size;
}
jcv_rect* rect = 0;
size_t imagesize = (size_t)(width*height * 3);
unsigned char* image = (unsigned char*)malloc(imagesize);
unsigned char* image2 = (unsigned char*)malloc(imagesize);
memset(image, 0, imagesize);
unsigned char color_pt[] = { 255, 255, 255 };
unsigned char color_line[] = { 220, 220, 220 };
jcv_diagram diagram;
jcv_point dimensions;
dimensions.x = (jcv_real)width;
dimensions.y = (jcv_real)height;
memset(&diagram, 0, sizeof(jcv_diagram));
jcv_diagram_generate(size, (const jcv_point*)voronoiPoint, rect, &diagram);
//Edge
const jcv_edge* edge = jcv_diagram_get_edges(&diagram);
std::vector<filtered_edge> filteredEdge;
float min_x = 0.0, min_y = 0.0;
while (edge) //Remove edge from the same connected component
{
jcv_point p0 = edge->pos[0];
jcv_point p1 = edge->pos[1];
if (edge->sites[0]->p.site != edge->sites[1]->p.site)
{
filteredEdge.push_back(jcv_save_edge(edge));
min_x = min_x + abs(edge->sites[0]->p.x - edge->sites[1]->p.x);
min_y = min_y + abs(edge->sites[0]->p.y - edge->sites[1]->p.y);
}
edge = edge->next;
}
min_x = min_x / filteredEdge.size();
min_y = min_y / filteredEdge.size();
std::vector<filtered_edge> selectedEdge;
for (int i = 0; i < filteredEdge.size(); i++)
{
jcv_point p0 = remap(&filteredEdge.at(i).pos[0], &diagram.min, &diagram.max, &dimensions);
jcv_point p1 = remap(&filteredEdge.at(i).pos[1], &diagram.min, &diagram.max, &dimensions);
float site_x = abs(filteredEdge.at(i).sites[0]->p.x - filteredEdge.at(i).sites[1]->p.x);
float site_y = abs(filteredEdge.at(i).sites[0]->p.y - filteredEdge.at(i).sites[1]->p.y);
float x_difference = abs(filteredEdge.at(i).pos[0].x- filteredEdge.at(i).pos[1].x);
float y_difference = abs(filteredEdge.at(i).pos[0].y - filteredEdge.at(i).pos[1].y);
float areaDiff = areaDifference(filteredEdge.at(i).sites[0]->p.totalPoint, filteredEdge.at(i).sites[1]->p.totalPoint);
if (p0.x - p1.x == 0 && p0.y - p1.y == 0.0) //Remove short edges
continue;
if (areaDiff > 20) //Keep edge between small(text) and big(image) component
{
float difference = abs(filteredEdge.at(i).sites[0]->p.totalPoint - filteredEdge.at(i).sites[1]->p.totalPoint);
if (difference > average*4 )
{
unsigned char color_line2[] = { 0, 220, 220 };
selectedEdge.push_back(filteredEdge.at(i));
draw_line((int)p0.x, (int)p0.y, (int)p1.x, (int)p1.y, image, width, height, 3, color_line2);
continue;
}
}
if (x_difference > y_difference) //Remove edge between close component
{
if (site_y > min_y*1.6)
{
unsigned char color_line2[] = { 220, 0, 220 };
selectedEdge.push_back(filteredEdge.at(i));
draw_line((int)p0.x, (int)p0.y, (int)p1.x, (int)p1.y, image, width, height, 3, color_line2);
}
}
else
{
if (site_x > min_x*2.5)
{
unsigned char color_line2[] = { 220, 220, 0 };
selectedEdge.push_back(filteredEdge.at(i));
draw_line((int)p0.x, (int)p0.y, (int)p1.x, (int)p1.y, image, width, height, 3, color_line2);
}
}
}
jcv_diagram_free(&diagram);
for (int i = 0; i < size; ++i)
{
jcv_point p = remap(&voronoiPoint[i], &diagram.min, &diagram.max, &dimensions);
plot((int)p.x, (int)p.y, image, width, height, 3, color_pt);
}
free(voronoiPoint);
cv::Mat segmentedImg = cv::Mat(height, width, CV_8UC3, image);
cv::imshow("Testing", segmentedImg);
cv::waitKey(0);
free(image);
}
int main()
{
cv::Mat image, skewCorrected;
image = cv::imread("C:\\figure5.PNG");
if (!image.data)
{
std::cout << "Error" << std::endl;
system("PAUSE");
return 0;
}
std::vector<compPoint> points = generatePoint(image);
int width = image.size().width, height = image.size().height;
generateVoronoi(points, width, height);
cv::waitKey(0);
}
Input image:

Answer 1

I don't understand many things in your code so I just appended some lines to do what you want.
1 - Create a Mat of zeros to draw the lines (CV_8U)
Mat dst = cv::Mat(height, width, CV_8U, cvScalar(0.));
2 - Draw the lines (using your points)
line( dst, Point((int)p0.x, (int)p0.y), Point((int)p1.x, (int)p1.y), Scalar( 255, 255, 255 ), 1, 8);
3 - Close the "holes" between the lines (CLOSE morphology operation)
int morph_size = 20; // adjust this values to your image
Mat element = getStructuringElement( MORPH_RECT, Size( 2*morph_size + 1, 2*morph_size+1 ), Point( morph_size, morph_size ) );
// Apply the CLOSE morphology operation
morphologyEx( dst, closed, MORPH_CLOSE, element );
4 - Flood fill to a mask (= "painting" the splitted areas)
// iterate through the points
for (int i = 0; i < closed.rows; i++ ) {
for (int j = 0; j < closed.cols; j++) {
// if point is not "painted" yet
if (closed.at<uchar>(i, j) == 0) {
// copy Mat before Flood fill
Mat previous_closed = closed.clone();
// Flood fill that seed point ("paint" that area)
floodFill(closed, Point(j, i), 255);
// Get mask with the "painted" area
Mat mask = closed - previous_closed;
/// Copy from segmentedImg using the mask
Mat outputMat;
segmentedImg.copyTo(outputMat, mask);
cv::imshow("Closed lines", closed);
imshow("Splitted Area", outputMat);
waitKey(0);
break;
}
}
}
Area 1:
Area 2:
Area 3:
... And so on, for the 5 areas, that loop basically keeps on painting the "black areas" in white and creating mats given the difference before and after each flood fill.
Full code (your code + this lines):
#include <opencv2/core/core.hpp>
#include <opencv2/highgui.hpp>
#include <opencv2/opencv.hpp>
using namespace cv;
#define JC_VORONOI_IMPLEMENTATION
#include "jc_voronoi.h"
typedef struct compPoint
{
cv::Point pointer;
int siteNum, size ;
};
int maximumSize;
float average=0;
std::vector<compPoint> generatePoint(cv::Mat image);
void generateVoronoi(std::vector<cv::Point> points, int width, int height);
static inline jcv_point remap(const jcv_point* pt, const jcv_point* min, const jcv_point* max, const jcv_point* scale);
static void draw_line(int x0, int y0, int x1, int y1, unsigned char* image, int width, int height, int nchannels, unsigned char* color);
static void plot(int x, int y, unsigned char* image, int width, int height, int nchannels, unsigned char* color);
float areaDifference(int s1,int s2);
float areaDifference(int s1, int s2)
{
if (s1 > s2)
{
return s1 / s2;
}
else
{
return s2 / s1;
}
}
std::vector<compPoint> generatePoint(cv::Mat image)
{
cv::Mat grayscale, binary;
cv::cvtColor(image, grayscale, cv::COLOR_BGR2GRAY);
cv::threshold(grayscale, binary, 190, 255, 1);
std::vector<std::vector<cv::Point> > contours;
std::vector<cv::Vec4i> hierarchy;
cv::findContours(binary, contours, hierarchy, cv::RETR_TREE, cv::CHAIN_APPROX_NONE, cv::Point(0, 0));
std::vector<compPoint> extractedPoint;
cv::Mat drawing = cv::Mat::zeros(binary.size(), CV_8UC3);
cv::Scalar color = cv::Scalar(255, 255, 255);
maximumSize = cv::contourArea(contours[0]);
int skip = 0;
for (int i = 0; i < contours.size(); i++)
{
int jumpPoint = contours[i].size() / (contours[i].size() * 0.12);
bool isInner = false;
cv::Vec4i currentHierarchy = hierarchy[i];
if (contours[i].size() <= 20) //Remove small component
continue;
for (int g = 0; g < contours[i].size(); g = g + jumpPoint) //Sample point from connected component
{
compPoint temp;
temp.pointer = contours[i].at(g);
line(drawing, contours[i].at(g), contours[i].at(g), color, 1, 8, 0);
if (currentHierarchy.val[3] != -1)
{
int currentIndex = currentHierarchy.val[3];
while (hierarchy[currentIndex].val[3] != -1)
{
currentIndex = hierarchy[currentIndex].val[3];
}
temp.siteNum = currentIndex;
temp.size = cv::contourArea(contours[currentIndex]);
isInner = true;
}
else
{
temp.siteNum = i;
temp.size = cv::contourArea(contours[i]);
if (cv::contourArea(contours[i])>maximumSize)
{
maximumSize = cv::contourArea(contours[i]);
}
}
extractedPoint.push_back(temp);
}
if (isInner == false)
{
average = average + cv::contourArea(contours[i]);
skip++;
}
}
average = average/skip;
return extractedPoint;
}
static inline jcv_point remap(const jcv_point* pt, const jcv_point* min, const jcv_point* max, const jcv_point* scale)
{
jcv_point p;
p.x = (pt->x - min->x) / (max->x - min->x) * scale->x;
p.y = (pt->y - min->y) / (max->y - min->y) * scale->y;
return p;
}
static void plot(int x, int y, unsigned char* image, int width, int height, int nchannels, unsigned char* color)
{
if (x < 0 || y < 0 || x >(width - 1) || y >(height - 1))
return;
int index = y * width * nchannels + x * nchannels;
for (int i = 0; i < nchannels; ++i)
{
image[index + i] = color[i];
}
}
static void draw_line(int x0, int y0, int x1, int y1, unsigned char* image, int width, int height, int nchannels, unsigned char* color)
{
int dx = abs(x1 - x0), sx = x0<x1 ? 1 : -1;
int dy = -abs(y1 - y0), sy = y0<y1 ? 1 : -1;
int err = dx + dy, e2; // error value e_xy
for (;;)
{ // loop
plot(x0, y0, image, width, height, nchannels, color);
if (x0 == x1 && y0 == y1) break;
e2 = 2 * err;
if (e2 >= dy) { err += dy; x0 += sx; } // e_xy+e_x > 0
if (e2 <= dx) { err += dx; y0 += sy; } // e_xy+e_y < 0
}
}
void generateVoronoi(std::vector<compPoint> points, int width, int height)
{
/// 1 - Create Mat of zeros to draw the lines
Mat dst = cv::Mat(height,width, CV_8U, cvScalar(0.));
int size = points.size();
jcv_point* voronoiPoint = (jcv_point*)malloc(sizeof(jcv_point) * (size_t)size);
for (int i = 0; i < size; i++)
{
voronoiPoint[i].x = (float)points[i].pointer.x;
voronoiPoint[i].y = (float)points[i].pointer.y;
voronoiPoint[i].site = points[i].siteNum;
voronoiPoint[i].totalPoint = points[i].size;
}
jcv_rect* rect = 0;
size_t imagesize = (size_t)(width*height * 3);
unsigned char* image = (unsigned char*)malloc(imagesize);
memset(image, 0, imagesize);
unsigned char color_pt[] = { 255, 255, 255 };
jcv_diagram diagram;
jcv_point dimensions;
dimensions.x = (jcv_real)width;
dimensions.y = (jcv_real)height;
memset(&diagram, 0, sizeof(jcv_diagram));
jcv_diagram_generate(size, (const jcv_point*)voronoiPoint, rect, &diagram);
//Edge
const jcv_edge* edge = jcv_diagram_get_edges(&diagram);
std::vector<filtered_edge> filteredEdge;
float min_x = 0.0, min_y = 0.0;
while (edge) //Remove edge from the same connected component
{
jcv_point p0 = edge->pos[0];
jcv_point p1 = edge->pos[1];
if (edge->sites[0]->p.site != edge->sites[1]->p.site)
{
filteredEdge.push_back(jcv_save_edge(edge));
min_x = min_x + abs(edge->sites[0]->p.x - edge->sites[1]->p.x);
min_y = min_y + abs(edge->sites[0]->p.y - edge->sites[1]->p.y);
}
edge = edge->next;
}
min_x = min_x / filteredEdge.size();
min_y = min_y / filteredEdge.size();
std::vector<filtered_edge> selectedEdge;
for (int i = 0; i < filteredEdge.size(); i++)
{
jcv_point p0 = remap(&filteredEdge.at(i).pos[0], &diagram.min, &diagram.max, &dimensions);
jcv_point p1 = remap(&filteredEdge.at(i).pos[1], &diagram.min, &diagram.max, &dimensions);
float site_x = abs(filteredEdge.at(i).sites[0]->p.x - filteredEdge.at(i).sites[1]->p.x);
float site_y = abs(filteredEdge.at(i).sites[0]->p.y - filteredEdge.at(i).sites[1]->p.y);
float x_difference = abs(filteredEdge.at(i).pos[0].x- filteredEdge.at(i).pos[1].x);
float y_difference = abs(filteredEdge.at(i).pos[0].y - filteredEdge.at(i).pos[1].y);
float areaDiff = areaDifference(filteredEdge.at(i).sites[0]->p.totalPoint, filteredEdge.at(i).sites[1]->p.totalPoint);
if (p0.x - p1.x == 0 && p0.y - p1.y == 0.0) //Remove short edges
continue;
/// 2 - Draw lines
if (areaDiff > 20) //Keep edge between small(text) and big(image) component
{
float difference = abs(filteredEdge.at(i).sites[0]->p.totalPoint - filteredEdge.at(i).sites[1]->p.totalPoint);
if (difference > average*4 )
{
unsigned char color_line2[] = { 0, 220, 220 };
selectedEdge.push_back(filteredEdge.at(i));
draw_line((int)p0.x, (int)p0.y, (int)p1.x, (int)p1.y, image, width, height, 3, color_line2);
line( dst, Point((int)p0.x, (int)p0.y), Point((int)p1.x, (int)p1.y), Scalar( 255, 255, 255 ), 1, 8);
continue;
}
}
if (x_difference > y_difference) //Remove edge between close component
{
if (site_y > min_y*1.6)
{
unsigned char color_line2[] = { 220, 0, 220 };
selectedEdge.push_back(filteredEdge.at(i));
draw_line((int)p0.x, (int)p0.y, (int)p1.x, (int)p1.y, image, width, height, 3, color_line2);
line( dst, Point((int)p0.x, (int)p0.y), Point((int)p1.x, (int)p1.y), Scalar( 255, 255, 255 ), 1, 8);
}
}
else
{
if (site_x > min_x*2.5)
{
unsigned char color_line2[] = { 220, 220, 0 };
selectedEdge.push_back(filteredEdge.at(i));
draw_line((int)p0.x, (int)p0.y, (int)p1.x, (int)p1.y, image, width, height, 3, color_line2);
line( dst, Point((int)p0.x, (int)p0.y), Point((int)p1.x, (int)p1.y), Scalar( 255, 255, 255 ), 1, 8);
}
}
}
jcv_diagram_free(&diagram);
for (int i = 0; i < size; ++i)
{
jcv_point p = remap(&voronoiPoint[i], &diagram.min, &diagram.max, &dimensions);
plot((int)p.x, (int)p.y, image, width, height, 3, color_pt);
}
free(voronoiPoint);
cv::Mat segmentedImg = cv::Mat(height, width, CV_8UC3, image);
cv::imshow("Testing", segmentedImg);
cv::imshow("Lines", dst);
/// New code:
Mat closed = dst.clone();
/// 3 - Close the "holes" between the lines
int morph_size = 20; // adjust this values to your image
Mat element = getStructuringElement( MORPH_RECT, Size( 2*morph_size + 1, 2*morph_size+1 ), Point( morph_size, morph_size ) );
// Apply the CLOSE morphology operation
morphologyEx( dst, closed, MORPH_CLOSE, element );
imshow("Closed lines", closed);
waitKey(0);
/// 4 - Flood fill to a mask
// iterate through the points
for (int i = 0; i < closed.rows; i++ ) {
for (int j = 0; j < closed.cols; j++) {
// if point is not "painted" yet
if (closed.at<uchar>(i, j) == 0) {
// copy Mat before Flood fill
Mat previous_closed = closed.clone();
// Flood fill that seed point ("paint" that area)
floodFill(closed, Point(j, i), 255);
// Get mask with the "painted" area
Mat mask = closed - previous_closed;
/// 5 - Copy from segmentedImg using the mask
Mat outputMat;
segmentedImg.copyTo(outputMat, mask);
cv::imshow("Closed lines", closed);
imshow("Splitted Area", outputMat);
waitKey(0);
break;
}
}
}
free(image);
}
int main()
{
cv::Mat image, skewCorrected;
image = cv::imread("/home/tribta/Downloads/HI2IT.png");
if (!image.data)
{
std::cout << "Error" << std::endl;
system("PAUSE");
return 0;
}
std::vector<compPoint> points = generatePoint(image);
int width = image.size().width, height = image.size().height;
generateVoronoi(points, width, height);
cv::waitKey(0);
}