Combining 2 images with transparent mask in opencv - c++

What I'm basically trying to do is blur an image, and combine it back with the orignal, so that only certain areas in the original image are blurred (the face should be blurred).
My general idea was to mask the parts in the original Iwant to have blurred, then blur the original as a copy and "merge" them together again.
To a certain extend this also worked.
My images:
(1) Original
(2) Original with parts that should be blurred
(3) Blurred
My C++ code that creates these images:
int main(void) {
cv::Mat srcImage = cv::imread(path);
srcImage.convertTo(srcImage, CV_32FC3, 1.0/255.0);
Mat _mask;
Mat img_gray;
cv::Scalar white = cv::Scalar(255, 255, 255);
cv::Scalar black = cv::Scalar(0, 0, 0);
cv::cvtColor(srcImage, img_gray, cv::COLOR_BGR2GRAY);
img_gray.convertTo(_mask, CV_32FC1);
// face
cv::circle(_mask, cv::Point(430, 350), 200, black, -1, 8, 0);
// eyes
cv::circle(_mask, cv::Point(502, 260), 27, white, -1, 8, 0);
cv::circle(_mask, cv::Point(390, 260), 27, white, -1, 8, 0);
// mouth
cv::ellipse(_mask, cv::Point(440, 390), cv::Point(60, 25), 0, 0, 360, white, -1, 8, 0);
cv::threshold(1.0-_mask, _mask, 0.9, 1.0, cv::THRESH_BINARY_INV);
cv::GaussianBlur(_mask,_mask,Size(21,21),11.0);
cv::Mat res;
cv::Mat bg = Mat(srcImage.size(), CV_32FC3);
bg = cv::Scalar(1.0, 1.0 ,1.0);
vector<Mat> ch_img(3);
vector<Mat> ch_bg(3);
cv::split(srcImage, ch_img);
cv::split(bg, ch_bg);
ch_img[0] = ch_img[0].mul(_mask) + ch_bg[0].mul(1.0 - _mask);
ch_img[1] = ch_img[1].mul(_mask) + ch_bg[1].mul(1.0 - _mask);
ch_img[2] = ch_img[2].mul(_mask) + ch_bg[2].mul(1.0 - _mask);
cv::merge(ch_img, res);
cv::merge(ch_bg, bg);
// original but with white mask
res.convertTo(res, CV_8UC3, 255.0);
imwrite("original_with_mask.jpg", res);
// blur original image
cv::Mat blurredImage;
bilateralFilter(srcImage, blurredImage, 10, 20, 5);
GaussianBlur(srcImage, blurredImage, Size(19, 19), 0, 0);
blurredImage.convertTo(blurredImage, CV_8UC3, 255.0);
imwrite("blurred.jpg", blurredImage);
cv::Mat maskedImage;
maskedImage = Mat(srcImage.size(), CV_32FC3);
// now combine blurred image and original using mask
// this fails
cv::bitwise_and(blurredImage, _mask, maskedImage);
cv::imwrite("masked.jpg", maskedImage);
}
My problem is that cv::bitwise_and(blurredImage, _mask, maskedImage); fails with
OpenCV Error: Sizes of input arguments do not match (The operation is neither 'array op array' (where arrays have the same size and type), nor 'array op scalar', nor 'scalar op array') in binary_op
Probably because _mask is a single channel image and blurredImage and maskedImage are 3-channel images.
How can I combine the images I got so that the currently white areas in image (2) are blurred using a transparent mask with "soft" edges?

Instead of float conversion you can just use the linearcombination of byte channel values. See
int main(int argc, char* argv[])
{
cv::Mat srcImage = cv::imread("C:/StackOverflow/Input/transparentMaskInput.jpg");
// blur whole image
cv::Mat blurredImage;
//cv::bilateralFilter(srcImage, blurredImage, 10, 20, 5); // use EITHER bilateral OR Gaússian filter
cv::GaussianBlur(srcImage, blurredImage, cv::Size(19, 19), 0, 0);
// create mask
cv::Scalar white = cv::Scalar(255, 255, 255);
cv::Scalar black = cv::Scalar(0, 0, 0);
cv::Mat mask = cv::Mat::zeros(srcImage.size(), CV_8UC1);
// face
cv::circle(mask, cv::Point(430, 350), 200, black, -1, 8, 0);
// eyes
cv::circle(mask, cv::Point(502, 260), 27, white, -1, 8, 0);
cv::circle(mask, cv::Point(390, 260), 27, white, -1, 8, 0);
// mouth
cv::ellipse(mask, cv::Point(440, 390), cv::Point(60, 25), 0, 0, 360, white, -1, 8, 0);
cv::GaussianBlur(mask, mask, cv::Size(21, 21), 11.0);
// byte inversion:
cv::Mat invertedMask = 255 - mask; // instead of inversion you could just draw the "face" black on a white background!
cv::Mat outputImage = cv::Mat(srcImage.size(), srcImage.type());
// for each pixel, merge blurred and original image regarding the blur-mask
for (int y = 0; y < outputImage.rows; ++y)
for (int x = 0; x < outputImage.cols; ++x)
{
cv::Vec3b pixelOrig = srcImage.at<cv::Vec3b>(y, x);
cv::Vec3b pixelBlur = blurredImage.at<cv::Vec3b>(y, x);
float blurVal = invertedMask.at<unsigned char>(y, x)/255.0f; // value between 0 and 1: zero means 100% orig image, one means 100% blurry image
cv::Vec3b pixelOut = blurVal * pixelBlur + (1.0f - blurVal)* pixelOrig;
outputImage.at<cv::Vec3b>(y, x) = pixelOut;
}
cv::imshow("input", srcImage);
cv::imshow("blurred", blurredImage);
cv::imshow("mask", mask);
cv::imshow("inverted mask", invertedMask);
cv::imshow("output", outputImage);
return 0;
}
using this input image:
computing this blurred and mask:
resulting in this output, by computing (mask/255) * blur + (1-mask/255)*blur (linear combination):

I define a function to do alphaBlend for two images of CV_8UC3 with a mask of CV_8UC1 in OpenCV:
//! 2018.01.16 13:54:39 CST
//! 2018.01.16 14:43:26 CST
void alphaBlend(Mat& img1, Mat&img2, Mat& mask, Mat& blended){
// Blend img1 and img2 (of CV_8UC3) with mask (CV_8UC1)
assert(img1.size() == img2.size() && img1.size() == mask.size());
blended = cv::Mat(img1.size(), img1.type());
for (int y = 0; y < blended.rows; ++y){
for (int x = 0; x < blended.cols; ++x){
float alpha = mask.at<unsigned char>(y, x)/255.0f;
blended.at<cv::Vec3b>(y,x) = alpha*img1.at<cv::Vec3b>(y,x) + (1-alpha)*img2.at<cv::Vec3b>(y,x);
}
}
}
Then, it's easy to do alpha bend on the images, just call alphaBlend(...). Here is an example:
#include <opencv2/opencv.hpp>
using namespace cv;
//! 2018.01.16 13:54:39 CST
//! 2018.01.16 14:43:26 CST
void alphaBlend(Mat& img1, Mat&img2, Mat& mask, Mat& blended){
// Blend img1 and img2 (of CV_8UC3) with mask (CV_8UC1)
assert(img1.size() == img2.size() && img1.size() == mask.size());
blended = cv::Mat(img1.size(), img1.type());
for (int y = 0; y < blended.rows; ++y){
for (int x = 0; x < blended.cols; ++x){
float alpha = mask.at<unsigned char>(y, x)/255.0f;
blended.at<cv::Vec3b>(y,x) = alpha*img1.at<cv::Vec3b>(y,x) + (1-alpha)*img2.at<cv::Vec3b>(y,x);
}
}
}
Mat createMask(Size sz){
// create mask
cv::Mat mask = cv::Mat::zeros(sz, CV_8UC1);
// white and black
cv::Scalar white = cv::Scalar(255, 255, 255);
cv::Scalar black = cv::Scalar(0, 0, 0);
// face
cv::circle(mask, cv::Point(430, 350), 200, black, -1, 8, 0);
// eyes
cv::circle(mask, cv::Point(502, 260), 27, white, -1, 8, 0);
cv::circle(mask, cv::Point(390, 260), 27, white, -1, 8, 0);
// mouth
cv::ellipse(mask, cv::Point(440, 390), cv::Point(60, 25), 0, 0, 360, white, -1, 8, 0);
// Blur
cv::GaussianBlur(mask, mask, cv::Size(21, 21), 11.0);
return mask;
}
int main(){
cv::Mat img = cv::imread("img04.jpg");
// blur whole image
cv::Mat blured;
//cv::bilateralFilter(img, blured, 10, 20, 5); // use EITHER bilateral OR Gaússian filter
cv::GaussianBlur(img, blured, cv::Size(19, 19), 0, 0);
// Create the mask
Mat mask = createMask(img.size());
Mat mask_inv = 255 - mask;
// Alpha blend
Mat blended1, blended2;
alphaBlend(img, blured, mask, blended1);
alphaBlend(img, blured, mask_inv, blended2);
// Display
cv::imshow("source", img);
cv::imshow("blured", blured);
cv::imshow("mask", mask);
cv::imshow("mask_inv", mask_inv);
cv::imshow("blended1", blended1);
cv::imshow("blended2", blended2);
cv::waitKey();
return 0;
}
Source:
Blured:
Mask1:
AlphaBlend 1:
Mask 2:
AlphaBlend 2:
Some useful links:
Alpha Blending in OpenCV C++ : Combining 2 images with transparent mask in opencv
Alpha Blending in OpenCV Python:
Gradient mask blending in opencv python

Probably because _mask is a single channel image and blurredImage and
maskedImage are 3-channel images.
Put this before calling the cv::bitwise_and:
P.S if you do not want to alter your mask becuase you want to use it in another place just do it in a temporary variable:
cv::Mat _mask_temp;
cv::cvtColor(_mask,_mask_temp,cv::COLOR_GRAY2BGR);
cv::bitwise_and(blurredImage, _mask_temp, maskedImage);
_mask_temp.release(); // just in case you do not want it anymore to be in your memory(optional)
EDIT (another problem):
The mask is 32F while the image is 8U. So, you need this:
cv::cvtColor(_mask,_mask,cv::COLOR_GRAY2BGR);
_mask.convertTo(_mask, CV_8UC3);

Related

Having trouble with replacing color masks

I am selecting the color from first frame using mouse-handler.
I am trying to replace the selected color with background frame.
This is working fine for red color but this is not working for any other color like green, blue, etc. I am using following graph for color selection:
Click here!
#include "opencv2/opencv.hpp"
#include "opencv2/imgproc/imgproc.hpp"
#include "opencv2/highgui/highgui.hpp"
#include <iostream>
using namespace std;
using namespace cv;
// structure to be used in mouseHandler function
struct userdata {
Mat im;
vector<Point2f> points;
};
void mouseHandler(int event, int x, int y, int flags, void* data_ptr)
{
if (event == EVENT_LBUTTONDOWN) {
userdata* data = ((userdata*)data_ptr);
circle(data->im, Point(x, y), 3, Scalar(0, 0, 255), 5, LINE_AA);
imshow("Image", data->im);
if (data->points.size() < 1) {
data->points.push_back(Point2f(x, y));
}
}
}
int main(int argc, char** argv) {
// Take video frame from camera to select color of material
VideoCapture capt(0);
Mat frames;
capt >> frames;
Mat hsvimg;
// Converting image from BGR to HSV
cvtColor(frames, hsvimg, COLOR_BGR2HSV);
// Set data for mouse event
Mat im_temp = frames.clone();
userdata data;
data.im = im_temp;
cout << "Select the point on image for the color you want to create cloak and than press 'Enter'" << endl;
// Show image and wait for a click.
imshow("Image", im_temp);
// Set the callback function for any mouse event
setMouseCallback("Image", mouseHandler, &data);
waitKey(0);
//defining the HSV values of the point selected
Vec3b HSV_Color = hsvimg.at<Vec3b>(data.points[0]);
int hue = HSV_Color.val[0];
int saturation = HSV_Color.val[1];
int value = HSV_Color.val[2];
// Create a VideoCapture object and open the input file for demonstrating the cloak working
// If the input is the web camera, pass 0 instead of the video file name
// In first frame only background should be there, i.e., no person present
VideoCapture cap(0);
// Check if camera opened successfully
if (!cap.isOpened()) {
cout << "Error opening video stream or file" << endl;
return -1;
}
Mat background;
for (int i = 0; i < 30; i++)
{
cap >> background;
}
//Laterally invert the image / flip the image.
flip(background, background, 1);
while (1)
{
Mat frame;
// Capture frame-by-frame
cap >> frame;
// Laterally invert the image / flip the image
flip(frame, frame, 1);
//Converting image from BGR to HSV color space.
Mat hsv;
cvtColor(frame, hsv, COLOR_BGR2HSV);
Mat mask1, mask2;
// Creating masks to detect the upper and lower red color.
// Otherwise mask1 and mask2 are same for other colors
// Making different conditions according to hue values
// Take help from this: https://stackoverflow.com/questions/10948589/choosing-the-correct-upper-and-lower-hsv-boundaries-for-color-detection-withcv
if (saturation > 100) {
if (hue <= 10 || hue>165) {
inRange(hsv, Scalar(0, 120, 20), Scalar(10, 255, 255), mask1);
inRange(hsv, Scalar(170, 120, 20), Scalar(180, 255, 255), mask2);
}
else if (10<hue<=25) {
inRange(hsv, Scalar(10, 120, 20), Scalar(25, 255, 255), mask1);
inRange(hsv, Scalar(10, 120, 20), Scalar(25, 255, 255), mask2);
}
else if (25 < hue <= 38) {
inRange(hsv, Scalar(25, 120, 20), Scalar(35, 255, 255), mask1);
inRange(hsv, Scalar(25, 120, 20), Scalar(35, 255, 255), mask2);
}
else if (38 < hue <= 71) {
inRange(hsv, Scalar(38, 100, 20), Scalar(71, 255, 255), mask1);
inRange(hsv, Scalar(38, 100, 20), Scalar(71, 255, 255), mask2);
}
else if (71 < hue <= 100) {
inRange(hsv, Scalar(71, 120, 20), Scalar(95, 255, 255), mask1);
inRange(hsv, Scalar(71, 120, 20), Scalar(95, 255, 255), mask2);
}
else if (100 < hue <= 140) {
inRange(hsv, Scalar(100, 150, 20), Scalar(130, 255, 255), mask1);
inRange(hsv, Scalar(100, 150, 20), Scalar(130, 255, 255), mask2);
}
else if (140 < hue <= 165) {
inRange(hsv, Scalar(140, 120, 20), Scalar(170, 255, 255), mask1);
inRange(hsv, Scalar(140, 120, 20), Scalar(170, 255, 255), mask2);
}
}
else {
cout << "Use colored material." << endl;
break;
}
// Generating the final mask
mask1 = mask1 + mask2;
Mat kernel = Mat::ones(3, 3, CV_32F);
morphologyEx(mask1, mask1, cv::MORPH_OPEN, kernel);
morphologyEx(mask1, mask1, cv::MORPH_DILATE, kernel);
// creating an inverted mask to segment out the cloth from the frame
bitwise_not(mask1, mask2);
Mat res1, res2, final_output;
// Segmenting the cloth out of the frame using bitwise and with the inverted mask
bitwise_and(frame, frame, res1, mask2);
// creating image showing static background frame pixels only for the masked region
bitwise_and(background, background, res2, mask1);
// Generating the final augmented output.
addWeighted(res1, 1, res2, 1, 0, final_output);
imshow("magic", final_output);
waitKey(1);
// Press ESC on keyboard to exit
char c = (char)waitKey(25);
if (c == 27)
break;
// Also relese all the mat created in the code to avoid memory leakage.
frame.release(), hsv.release(), mask1.release(), mask2.release(), res1.release(), res2.release(), final_output.release();
}
// When everything done, release the video capture object
cap.release();
// Closes all the frames
cv::destroyAllWindows();
return 0;
}
It should do the same for all major colors as it is doing for red.
Have you checked the output of these two points when you are selecting another color?
Vec3b HSV_Color = hsvimg.at<Vec3b>(data.points[0]);
int hue = HSV_Color.val[0];
int saturation = HSV_Color.val[1];
int value = HSV_Color.val[2];
and this one
cvtColor(frame, hsv, COLOR_BGR2HSV);

How to find rectangle in black region opencv (C++)

I'm newbie OpenCV. I am using opencv for find position attach a stamp in image. The stamp can't overlap with other object in image.
Example binary image with white region is object on image, black region is none object
Example image
Result : Find rect in black region (in white circle) can attach stamp
Result imagge
Please help me find rect in black region same blue rect.
Thank you!
If you know about rectangles count then it can use kmeans for clusterization points.
First get only blue points and binarization they:
cv::Mat img = cv::imread("NQdmi.png", cv::IMREAD_COLOR);
std::vector<cv::Mat> chans;
cv::split(img, chans);
cv::Mat diff;
cv::absdiff(chans[2], chans[1], diff);
cv::threshold(diff, diff, 1, 255, cv::THRESH_BINARY);
cv::imshow("diff", diff);
Only blue points:
Clusterization points and find rotated rectangles:
std::vector<cv::Point2f> points;
for (int y = 0; y < diff.rows; ++y)
{
for (int x = 0; x < diff.cols; ++x)
{
if (diff.at<uchar>(y, x))
{
points.emplace_back(x, y);
}
}
}
cv::Mat pointsKmeans(points.size(), 1, CV_32FC2, &points[0]);
cv::Mat labels;
int clusterCount = 2;
cv::Mat centers;
cv::kmeans(pointsKmeans, clusterCount, labels,
cv::TermCriteria(cv::TermCriteria::EPS+cv::TermCriteria::COUNT, 100, 1.0),
3, cv::KMEANS_PP_CENTERS, centers);
std::vector<cv::Point2f> points1;
std::vector<cv::Point2f> points2;
cv::Mat draw = img.clone();
for (size_t i = 0; i < points.size(); ++i)
{
int clusterIdx = labels.at<int>(i);
if (clusterIdx > 0)
{
cv::circle(draw, points[i], 2, cv::Scalar(255, 0, 0), cv::FILLED, cv::LINE_AA);
points1.push_back(points[i]);
}
else
{
cv::circle(draw, points[i], 2, cv::Scalar(0, 0, 255), cv::FILLED, cv::LINE_AA);
points2.push_back(points[i]);
}
}
auto DrawRRect = [draw](const std::vector<cv::Point2f>& pp)
{
cv::RotatedRect rr = cv::minAreaRect(pp);
cv::Point2f corners[4];
rr.points(corners);
cv::line(draw, corners[0], corners[1], cv::Scalar(0, 255, 0), 2);
cv::line(draw, corners[1], corners[2], cv::Scalar(0, 255, 0), 2);
cv::line(draw, corners[2], corners[3], cv::Scalar(0, 255, 0), 2);
cv::line(draw, corners[3], corners[0], cv::Scalar(0, 255, 0), 2);
};
DrawRRect(points1);
DrawRRect(points2);
cv::imshow("draw", draw);
Result:

Draw rotated bounding box on image without fixed angle

Im new to programming and opencv and i try to detect a hdd using color segmentation. My code so far loads an image, creates 3 masks with different colors and draws an upright bounding box around the non zero points:
int main( int argc, char** argv )
{
//Load the image
Mat img = imread(argv[1], 1);
if (img.empty()){
cout << "No image found..." << endl;
return -1;
}
//Extracting colors - BGR
Mat silver, white, black;
//Silver
inRange(img, Scalar(180, 180, 180), Scalar(200, 200, 200), silver);
//White
inRange(img, Scalar(240, 240, 240), Scalar(255, 255, 255), white);
//Black
inRange(img, Scalar(0, 0, 0), Scalar(30, 30, 30), black);
// logical OR mask
Mat1b mask = silver | white | black;
// Find non zero pixels
vector<Point> pts;
findNonZero(mask, pts);
cout << "Non-Zero Locations = " << pts << endl << endl; // get non zero coordinates
// Compute bounding box
Rect box = boundingRect(pts);
// Show bounding box
rectangle(img, box, Scalar(0, 0, 255), 3);
namedWindow("box", CV_WINDOW_NORMAL);
imshow("box", img);
imshow("mask", mask);
waitKey(0);
destroyAllWindows;
return 0;}
Now I want to draw the smallest bounding box, so I tried to use
cv::RotatedRect box2 = cv::minAreaRect(pts);
instead. But it doesnt compile when I try to visualize that by replacing
Rect box = boundingRect(pts);
with
RotatedRect box2 = minAreaRect(pts);
Error Output:
error: no matching function for call to ‘rectangle(cv::Mat&, cv::RotatedRect&, cv::Scalar, int)’
rectangle(img, box2, Scalar(0, 0, 255), 3);
As per the cv::Rectangle Opencv Docs, the function has only 2 variants:
void rectangle(Mat& img, Point pt1, Point pt2, const Scalar& color, int thickness=1, int lineType=8, int shift=0)
void rectangle(Mat& img, Rect rec, const Scalar& color, int thickness=1, int lineType=8, int shift=0 )
So it is clear that it only accepts either cv::Rect or cv::Point. Hence there is no provision to directly input the cv::RotatedRect, due to which you are getting the above mentioned error.
To fix this issue, you can extract the 4 points of cv::RotatedRect using:
cv::Point2f points[4];
rotatedRect.points(points);
And then use cv::line() to draw the edges in pairs as:
cv::RotatedRect rotatedRect = cv::RotatedRect(cv::Point(70, 70), cv::Size(90, 90), 30);
cv::Mat canvas = cv::Mat(200, 200, CV_8UC3, cv::Scalar(255, 255, 255));
cv::Point2f points[4];
rotatedRect.points(points);
cv::line(canvas, points[0], points[1], cv::Scalar(0, 255, 0), 3);
cv::line(canvas, points[1], points[2], cv::Scalar(0, 255, 0), 3);
cv::line(canvas, points[2], points[3], cv::Scalar(0, 255, 0), 3);
cv::line(canvas, points[3], points[0], cv::Scalar(0, 255, 0), 3);

OpenCv overlay two Mat (drawings not images) with transparency

Hi what I have is a couple of Mat that I want to overlay (in a custom order). The Mat holdes some opencv polygons (which means a lot transparency). This Mat I need to overlay/merge. But not with the classical alpha blending more like with a 100% opacity but with transparency.
This is a simple sample code of what I want to merge.
Mat m1, m2;
m1.create(Point{ 100,100 }, CV_8UC4);
m2.create(Point{ 100,100 }, CV_8UC4);
cv::polylines(m1, std::vector<Point>{ Point{ 2,20 },Point{ 20,40 } }, true, Scalar(6, 6, 255));
cv::polylines(m2, std::vector<Point>{Point{ 100,100 }, Point{ 0,0 } }, true, Scalar(192, 112, 0));
Please note, that I cannot draw the polygons directly in one Mat due to various reasons.
I thought that maybe m1.copyTo(m2); will work, but its overwriting everything (incl. the black background)
Any idea how to get it merged/overlayed without the background? May I construct the mat's wrong?
I suspect you've had a problem looking for black in those images, as they were not initialized (it became apparent in debug mode). If we start with a zeroed out matrix, and draw using a 4-channel colour, so that the lines are visible, we get inputs such as this:
Input 1:
Input 2:
Now, we can use inRange to find all pixels set to (0,0,0,0). Since we want a mask of all non-black pixels, we just invert it by subtracting from 255. (i.e. mask = 255 - mask)
Mask:
Finally, use the mask as the second parameter of copyTo.
Result:
Code:
#include <opencv2/opencv.hpp>
int main()
{
cv::Mat m1(100, 100, CV_8UC4, cv::Scalar(0, 0, 0, 0));
cv::Mat m2(100, 100, CV_8UC4, cv::Scalar(0, 0, 0, 0));
cv::polylines(m1
, std::vector<cv::Point>{cv::Point{2, 20}, cv::Point{20, 40}}
, true, cv::Scalar(6, 6, 255, 255));
cv::polylines(m2
, std::vector<cv::Point>{cv::Point{100, 100}, cv::Point{0, 0}}
, true, cv::Scalar(192, 112, 0, 255));
cv::Mat mask;
cv::inRange(m2, cv::Scalar(0, 0, 0, 0), cv::Scalar(0, 0, 0, 0), mask);
mask = 255 - mask; // invert the mask
cv::Mat result(m1.clone());
m2.copyTo(result, mask);
cv::imwrite("transp_in_1.png", m1);
cv::imwrite("transp_in_2.png", m2);
cv::imwrite("transp_mask.png", mask);
cv::imwrite("transp_res.png", result);
return 0;
}
Instead of inverting the mask, you can invert the direction in which you copy. (i.e. overwrite everything black in m2 with stuff from m1)
cv::Mat mask;
cv::inRange(m2, cv::Scalar(0, 0, 0, 0), cv::Scalar(0, 0, 0, 0), mask);
cv::Mat result(m2.clone());
m1.copyTo(result, mask);

OpenCV copy bounded text area to new image

I am new to OpenCV and I am using this code to bound the text area in image. After that I am filtering contours and putting the bounded rectangle to a vector<Rect> to copy these to new image.
Mat large = img1;
Mat rgb;
// downsample and use it for processing
pyrUp(large, rgb);
Mat small;
cvtColor(rgb, small, CV_BGR2GRAY);
// morphological gradient
Mat grad;
Mat morphKernel = getStructuringElement(MORPH_ELLIPSE, Size(2, 2));
morphologyEx(small, grad, MORPH_GRADIENT, morphKernel);
// binarize
Mat bw;
threshold(grad, bw, 0.0, 255.0, THRESH_BINARY | THRESH_OTSU);
// connect horizontally oriented regions
Mat connected;
//morphKernel = getStructuringElement(MORPH_RECT, Size(7, 1));
//morphologyEx(bw, connected, MORPH_CLOSE, morphKernel);
// find contours
connected = bw;
Mat mask = Mat::zeros(bw.size(), CV_8UC1);
Mat mask2;
Mat mask3;
vector<vector<Point>> contours;
vector<Vec4i> hierarchy;
findContours(connected, contours, hierarchy, CV_RETR_CCOMP, CV_CHAIN_APPROX_SIMPLE, Point(0, 0));
/*drawContours(mask2, contours, -1, Scalar(255), CV_FILLED);
Mat Crop(img1.rows, img1.cols, CV_8UC3);
Crop.setTo(Scalar(0, 255, 0));
img1.copyTo(Crop, mask2);
normalize(mask2.clone(), mask2, 0.0, 255.0, CV_MINMAX, CV_8UC1);
*/
vector<Rect> rect1;
int i = 0;
//filter contours
for (int idx = 0; idx >= 0; idx = hierarchy[idx][0])
{
Rect rect = boundingRect(contours[idx]);
Mat maskROI(mask, rect);
maskROI = Scalar(0, 0, 0);
// fill the contour
drawContours(mask, contours, idx, Scalar(255, 255, 255), CV_FILLED);
// ratio of non-zero pixels in the filled region
double r = (double)countNonZero(maskROI) / (rect.width*rect.height);
if (r > .45 /* assume at least 45% of the area is filled if it contains text */
&&
(rect.height > 10 && rect.width > 10 && rect.height<150 && rect.width<150) /* constraints on region size */
/* these two conditions alone are not very robust. better to use something
like the number of significant peaks in a horizontal projection as a third condition */
)
{
//making rectangles on bounded area
rectangle(rgb, rect, Scalar(0, 255, 0), 2);
//pushing bounding rectangles in vector for new mask
rect1.push_back(rect);
}
}
Input output I am getting after bounded text ares is:
After that I am using this code to copy the bounded area only to new mask
//copying bounded rectangles area from small to new mask2
for (int i = 0; i < rect1.size(); i++){
mask2 = rgb(rect1[i]);
}
but by using this I only get this last bounded text area:
How can I get or update the mask2 rows or cols to get all the mapping of bounded text areas from rgb to mask2.
That's because mask2 will be equal to the last rgb(rect1[i]) called.
You can easily solve this in two ways (using copyTo):
Create a mask (black initialized, same size as input image), where you draw (white) rectangles. Then you copy the original image to a black initialized image of the same size, using the obtained mask.
Copy each sub-image directly to a black initialized image.
Starting from this image, where the red rectangles will be your detected rectangles:
With first approach you'll get a mask like:
and, for both approaches, the final result will be:
Code for first approach:
#include <opencv2/opencv.hpp>
#include <vector>
using namespace std;
using namespace cv;
int main()
{
// Your image
Mat3b img = imread("path_to_image");
// Your rectangles
vector<Rect> rects{Rect(100, 100, 100, 200), Rect(300, 200, 200, 100), Rect(500, 400, 80, 130)};
// Mask for rectangles (black initializeds)
Mat1b mask(img.rows, img.cols, uchar(0));
Mat3b dbgRects = img.clone();
for (int i = 0; i < rects.size(); ++i)
{
// Draw white rectangles on mask
rectangle(mask, rects[i], Scalar(255), CV_FILLED);
// Show rectangles
rectangle(dbgRects, rects[i], Scalar(0, 0, 255), 2);
}
// Black initizlied result
Mat3b result(img.rows, img.cols, Vec3b(0,0,0));
img.copyTo(result, mask);
imshow("Rectangles", dbgRects);
imshow("Result", result);
waitKey();
return 0;
}
Code for second approach:
#include <opencv2/opencv.hpp>
#include <vector>
using namespace std;
using namespace cv;
int main()
{
// Your image
Mat3b img = imread("path_to_image");
// Your rectangles
vector<Rect> rects{Rect(100, 100, 100, 200), Rect(300, 200, 200, 100), Rect(500, 400, 80, 130)};
// Black initizlied result
Mat3b result(img.rows, img.cols, Vec3b(0, 0, 0));
Mat3b dbgRects = img.clone();
for (int i = 0; i < rects.size(); ++i)
{
img(rects[i]).copyTo(result(rects[i]));
// Show rectangles
rectangle(dbgRects, rects[i], Scalar(0, 0, 255), 2);
}
imshow("Rectangles", dbgRects);
imshow("Result", result);
waitKey();
return 0;
}