Image shearing OpenCV CPP - c++

I'm trying to use cv::warpAffine to perform image transformations, but I have an issue.
Note: I already saw questions/46998895 and it works well too, but with no interpolation and I wanted to use affine matrixes.
So I want to shear the image and the result goes in a bigger image to have the entire sheared image like this:
Right now, my idea was to perform the shear on inverted image if the values are negatives rather than calculate the new image size with negative values etc as seen on paperspace blog.
When we use a negative shear, the direction of shear is right to left, while x2 is not further in the negative direction than x1. One way to solve this could be to get the other set of corners (that would satisfy the constraint, can you prove it?). Apply the shear transformation and then change to the other set of corners because of the notation we follow.
Well, we could do that, but there's a better method. Here's how how perform a negative shear with shearing factor -alpha.
Flip the image and boxes horizontally.
Apply the positive shear transformation with shearing factor alpha
Flip the image and boxes horizontally again.
The left ones are my results, and the right ones are the expected results :
So as you can see it works well when x and y are both positive values or when one is 0 and the other is negative, its ok. When one is positive and the other is negative, or when they are both negative, it provides always the same result for my shear function, see the 4 last lines in the pic above. Again, the left forms are my results, the right ones are the mickaShear() results (and obviously the correct ones).
My question is: what do I have to do in my shear_matrix to perform the correct transformation with negative values? Is the cv::flip() method described by paperSpace the right way to achieve this transformation?
Here you can find the reproductible code with the comparison between my function and the function in this answer. You will need OpenCV. I work on OpenCV 4.5.1
#include <opencv2/core/mat.hpp>
#include <opencv2/highgui.hpp>
#include <opencv2/features2d.hpp>
#include <opencv2/xfeatures2d.hpp>
#include <opencv2/imgproc.hpp>
#include <opencv2/core.hpp>
#include <iostream>
cv::Mat myShear(const cv::Mat & src, float sx, float sy) {
cv::Mat tmp;
cv::Mat dst;
std::vector<cv::Point2f> extremePoints;
extremePoints.emplace_back(cv::Point2f(0, 0));
extremePoints.emplace_back(cv::Point2f((float)src.cols, 0));
extremePoints.emplace_back(cv::Point2f((float)src.cols, (float)src.rows));
extremePoints.emplace_back(cv::Point2f(0, (float)src.rows));
for(auto & pt : extremePoints){
pt = cv::Point2f(pt.x + pt.y * abs(sx), pt.y + pt.x * abs(sy));
}
cv::Rect offsets = cv::boundingRect(extremePoints);
cv::Size new_size = offsets.size();
float mat_values[] = {1.0f, abs(sx), 0.0f, abs(sy), 1.0f, 0.0f};
cv::Mat shear_matrix = cv::Mat(2, 3, CV_32F, mat_values);
dst = cv::Mat::zeros(new_size, src.type());
/*
*cv::flip(img, tmp, INT_FLIP_CODE) where INT_FLIP_CODE can be:
* 0 (Vertically)
* 1 (Horizontally)
* -1 (Both)
*/
if(sx < 0.0f and sy < 0.0f) {
cv::flip(img, tmp, -1);
cv::warpAffine(tmp, dst, shear_matrix, new_size, cv::INTER_LINEAR);
cv::flip(dst, dst, -1);
} else if(sx < 0.0f) {
cv::flip(img, tmp, 1);
cv::warpAffine(tmp, dst, shear_matrix, new_size, cv::INTER_LINEAR);
cv::flip(dst, dst, 1);
} else if(sy < 0.0f) {
cv::flip(img, tmp, 0);
cv::warpAffine(tmp, dst, shear_matrix, new_size, cv::INTER_LINEAR);
cv::flip(dst, dst, 0);
} else {
tmp = src.clone();
cv::warpAffine(tmp, dst, shear_matrix, new_size, cv::INTER_LINEAR);
}
return dst;
}
cv::Mat mickaShear(const cv::Mat & input, float Bx, float By)
{
if (Bx*By == 1)
{
std::cerr << "error == 1" << std::endl;
}
if (input.type() != CV_8UC3) return {};
std::vector<cv::Point2f> extremePoints;
extremePoints.emplace_back(0, 0);
extremePoints.emplace_back(input.cols, 0);
extremePoints.emplace_back(input.cols, input.rows);
extremePoints.emplace_back(0, input.rows);
for (auto & pt : extremePoints)
{
pt = cv::Point2f(pt.x + pt.y*Bx, pt.y + pt.x*By);
}
cv::Rect offsets = cv::boundingRect(extremePoints);
cv::Point2f offset = -offsets.tl();
cv::Size resultSize = offsets.size();
cv::Mat shearedImage = cv::Mat::zeros(resultSize, input.type());
for (int j = 0; j < shearedImage.rows; ++j){
for (int i = 0; i < shearedImage.cols; ++i){
cv::Point2f pp((float)i, (float)j);
pp = pp - offset; // go back to original coordinate system
cv::Point2f p;
p.x = int((-pp.y*Bx + pp.x) / (1 - By*Bx));
p.y = int(pp.y - p.x*By);
if ((p.x >= 0 && p.x < (float)input.cols) && (p.y >= 0 && p.y < (float)input.rows)){
shearedImage.at<cv::Vec3b>(j, i) = input.at<cv::Vec3b>(p);
}
}
}
return shearedImage;
}
int main(int argc, char *argv[]){
float x = -0.2f; //CHANGE SIGN TO TEST
float y = 0.5f; //CHANGE SIGN TO TEST
cv::Mat im = cv::imread("MODIFY BY JPG FILE PATH");
cv::Mat result = im.clone();
cv::Mat output = cv::Mat();
for(auto & aug: augments){
output = mickaShear(im, x, y);
result = myShear(im);
}
cv::imshow("result", result);
cv::imshow("output", output);
auto k = cv::waitKey(0);
if(k == (int)'q'){
cv::destroyAllWindows();
break;
}
cv::destroyAllWindows();
return 0;
}

Related

How to rotate a point cloud given 3 points?

I have a 3D depth camera placed above three moving belt lanes and I'm trying to rotate the depth image (or the point cloud) so that the three lanes match the camera's angle. I'm not experienced at all with point clouds, but after some research I've tried to do the following:
Acquire an XYZ cartesian image from the sensor which I turn into a point cloud vector.
Define three points on the point cloud, one on each of the three lanes.
Fit a plane through them by finding the plane coefficients.
Finding the cross product between the plane and the z_normal, and then finding the angle of
rotation.
Use the Eigen library to transform the PCL cloud and turn it back into an openCV Mat.
For whatever reason, I always end up with a bad image with max-int values on one side and zeros on the other. I'm not certain anymore if there's something wrong with the code or if the method above is incorrect to start with.
My code so far:
// helper functions
pcl::PointCloud<pcl::PointXYZ>::Ptr MatToPcl(cv::Mat xyzMat);
cv::Mat PclToMat(pcl::PointCloud<pcl::PointXYZ>::Ptr point_cloud_ptr);
void colorize(cv::Mat& src, cv::Mat& dst);
void clip(cv::Mat& m, const uint16_t lowerBound, const uint16_t upperBound);
while(1)
{
// camera framegrabber object to capture an image
fg->SWTrigger();
if (!fg->WaitForFrame(im.get(), 2000))
{
throw;
}
// openCV Mat declerations
cv::Mat zDepth, zDepthColor;
cv::Mat xyz = im->XYZImage();
vector<cv::Mat> channels(3);
cv::split(xyz, channels);
zDepth = channels[0];
cv::imwrite("xyzMat.png", xyz);
cv::imwrite("depthImage.png", zDepth);
clip(zDepth, 1250, 1400);
colorise(zDepth, zDepthColor);
cv::imwrite("depthColored.png", zDepthColor);
// specify a 3D point on each lane
cv::Point3i p1, p2, p3;
p1.x = w / 4;
p1.y = 24;
p1.z = zDepth.at<uint16_t>(p1.x, p1.y);
p2.x = w / 2;
p2.y = 70;
p2.z = zDepth.at<uint16_t>(p2.x, p2.y);
p3.x = int(w * 0.75);
p3.y = 114;
p3.z = zDepth.at<uint16_t>(p3.x, p3.y);
auto cross = (p2 - p1).cross(p3 - p1);
// transform Mats to point clouds
pcl::PointCloud<pcl::PointXYZ>::Ptr floor_plane, xyzCentered;
floor_plane = MatToPcl(zDepth);
Eigen::Matrix<float, 1, 3> floor_plane_normal_vector, xy_plane_normal_vector, rotation_vector;
floor_plane_normal_vector[0] = cross.x;
floor_plane_normal_vector[1] = cross.y;
floor_plane_normal_vector[2] = cross.z;
// specify the z normal from the xy-plane
xy_plane_normal_vector[0] = 0.0;
xy_plane_normal_vector[1] = 0.0;
xy_plane_normal_vector[2] = 1.0;
// cross product and normalize vector
rotation_vector = xy_plane_normal_vector.cross(floor_plane_normal_vector);
rotation_vector.normalized();
// angle of rotation
float theta = -atan2(rotation_vector.norm(), xy_plane_normal_vector.dot(floor_plane_normal_vector));
// transform plane according to angle
Eigen::Affine3f transform_2 = Eigen::Affine3f::Identity();
transform_2.translation() << 0, 0, 30;
transform_2.rotate(Eigen::AngleAxisf(theta, rotation_vector));
pcl::transformPointCloud(*floor_plane, *xyzCentered, transform_2);
// Pointcloud to Mat again
cv::Mat xyzRot = PclToMat(xyzCentered);
// clipLow and clipHigh values obtained from trackbars
clip(xyzRot, clipLow, clipHigh);
cv::Mat xyzRotColor;
colorize(xyzRot, xyzRotColor)
cv::imshow("result", xyzRotColor);
cv::waitKey(1);
}
pcl::PointCloud<pcl::PointXYZ>::Ptr MatToPcl(cv::Mat xyzMat)
{
/*
* Function: Get from a Mat to pcl pointcloud datatype
* In: cv::Mat
* Out: pcl::PointCloud
*/
//char pr=100, pg=100, pb=100;
pcl::PointCloud<pcl::PointXYZ>::Ptr point_cloud_ptr(new pcl::PointCloud<pcl::PointXYZ>);;
vector<cv::Mat> channels(3);
cv::split(xyzMat, channels);
for (int i = 0; i < ifmXYZ.rows; i++)
{
for (int j = 0; j < ifmXYZ.cols; j++)
{
pcl::PointXYZ point;
point.x = channels[0].at<short>(i,j);
point.y = channels[1].at<short>(i, j);
point.z = channels[2].at<short>(i, j);
// when color needs to be added:
//uint32_t rgb = (static_cast<uint32_t>(pr) << 16 | static_cast<uint32_t>(pg) << 8 | static_cast<uint32_t>(pb));
//point.rgb = *reinterpret_cast<float*>(&rgb);
point_cloud_ptr->points.push_back(point);
}
}
point_cloud_ptr->width = (int)point_cloud_ptr->points.size();
/*point_cloud_ptr->height = 1;*/
return point_cloud_ptr;
}
// convert PCL to cv::Mat, taking only the depth values at z.
cv::Mat PclToMat(pcl::PointCloud<pcl::PointXYZ>::Ptr point_cloud_ptr)
{
cv::Mat depth_image;
if (!depth_image.empty())
depth_image.release();
depth_image.create(132, 176, CV_32F);
int count = 0;
for (int i = 0; i < 132; i++)
{
for (int j = 0; j < 176; j++)
{
depth_image.at<float>(i, j) = point_cloud_ptr->points.at(count++).z;
}
}
depth_image.convertTo(depth_image, CV_16UC1);
return depth_image;
}
/*
* For display purposes with cv::imshow, will convert a 16bit depth image to 8bit 3 channel colored image
* thanks to fmw42 for the function at https://stackoverflow.com/a/67678634/13184944
*/
void colorize(cv::Mat& src, cv::Mat& dst)
{
// stretch the image by rescaling intensity within the output 8-bit range
double oldMin;
double oldMax;
cv::Point minLoc;
cv::Point maxLoc;
cv::minMaxLoc(src, &oldMin, &oldMax, &minLoc, &maxLoc);
double oldRange = oldMax - oldMin;
double newMin = 0.0;
double newMax = 255.0;
double newRange = newMax - newMin;
//cout << oldMin << ' ' << oldMax << ' ' << oldRange << '\n';
// clip the values of the image to the required range
clip(src, oldMin, oldMax);
//TODO: Look at difference between OpenCV normalization and skimage
normalize(src, dst, 0, 255, NORM_MINMAX, CV_8UC1);
//img = (img - cv::Scalar(oldMin)) / (cv::Scalar(oldRange));
//img = (img * cv::Scalar(newRange)) + cv::Scalar(newMin);
cv::Mat channels[3] = { dst, dst, dst };
cv::merge(channels, 3, dst);
cv::Mat C(1, 6, CV_8UC(3));
cv::Vec3b color1 = { 0, 0, 255 };
cv::Vec3b color2 = { 0, 165, 255 };
cv::Vec3b color3 = { 0, 255, 255 };
cv::Vec3b color4 = { 255, 255, 0 };
cv::Vec3b color5 = { 255, 0, 0 };
cv::Vec3b color6 = { 128, 64, 64 };
C.at<cv::Vec3b>(0, 0) = color1;
C.at<cv::Vec3b>(0, 1) = color2;
C.at<cv::Vec3b>(0, 2) = color3;
C.at<cv::Vec3b>(0, 3) = color4;
C.at<cv::Vec3b>(0, 4) = color5;
C.at<cv::Vec3b>(0, 5) = color6;
cv::Mat lut;
cv::resize(C, lut, cv::Size(256, 1), 0.0, 0.0, cv::INTER_LINEAR);
//cout << lut.size << '\n';
cv::LUT(dst, lut, dst);
return;
}
void clip(cv::Mat& m, const uint16_t lowerBound, const uint16_t upperBound)
{
m.setTo(lowerBound, m < lowerBound);
m.setTo(upperBound, m > upperBound);
return;
}
Apologies if this is really basic or something is obviously wrong but I feel stuck here. I also tried segmentation with ransac but the it never aligns the plane in the way I wanted.
Thanks!
Edit: Updated the code to include additional steps and functions. Only the camera initialization is skipped.
The clip and colorize functions aid in displaying the 16bit depth image. My end goal here is to be able to use trackbars with clip(zImg, low, high) where the three lanes will always be vertically aligns (as in, change color at the same rate) as I change the clip values.
download link with image files: link
Colorized depth image:

Unsharp mask implementation with OpenCV

I want to apply unsharp mask like Adobe Photoshop,
I know this answer, but it's not as sharp as Photoshop.
Photoshop has 3 parameters in Smart Sharpen dialog: Amount, Radius, Reduce Noise; I want to implement all of them.
This is the code I wrote, according to various sources in SO.
But the result is good in some stages ("blurred", "unsharpMask", "highContrast"), but in the last stage ("retval") the result is not good.
Where am I wrong, what should I improve?
Is it possible to improve the following algorithm in terms of performance?
#include "opencv2/opencv.hpp"
#include "fstream"
#include "iostream"
#include <chrono>
using namespace std;
using namespace cv;
// from https://docs.opencv.org/3.4/d3/dc1/tutorial_basic_linear_transform.html
void increaseContrast(Mat img, Mat* dst, int amountPercent)
{
*dst = img.clone();
double alpha = amountPercent / 100.0;
*dst *= alpha;
}
// from https://stackoverflow.com/a/596243/7206675
float luminanceAsPercent(Vec3b color)
{
return (0.2126 * color[2]) + (0.7152 * color[1]) + (0.0722 * color[0]);
}
// from https://stackoverflow.com/a/2938365/7206675
Mat usm(Mat original, int radius, int amountPercent, int threshold)
{
// copy original for our return value
Mat retval = original.clone();
// create the blurred copy
Mat blurred;
cv::GaussianBlur(original, blurred, cv::Size(0, 0), radius);
cv::imshow("blurred", blurred);
waitKey();
// subtract blurred from original, pixel-by-pixel to make unsharp mask
Mat unsharpMask;
cv::subtract(original, blurred, unsharpMask);
cv::imshow("unsharpMask", unsharpMask);
waitKey();
Mat highContrast;
increaseContrast(original, &highContrast, amountPercent);
cv::imshow("highContrast", highContrast);
waitKey();
// assuming row-major ordering
for (int row = 0; row < original.rows; row++)
{
for (int col = 0; col < original.cols; col++)
{
Vec3b origColor = original.at<Vec3b>(row, col);
Vec3b contrastColor = highContrast.at<Vec3b>(row, col);
Vec3b difference = contrastColor - origColor;
float percent = luminanceAsPercent(unsharpMask.at<Vec3b>(row, col));
Vec3b delta = difference * percent;
if (*(uchar*)&delta > threshold) {
retval.at<Vec3b>(row, col) += delta;
//retval.at<Vec3b>(row, col) = contrastColor;
}
}
}
return retval;
}
int main(int argc, char* argv[])
{
if (argc < 2) exit(1);
Mat mat = imread(argv[1]);
mat = usm(mat, 4, 110, 66);
imshow("usm", mat);
waitKey();
//imwrite("USM.png", mat);
}
Original Image:
Blurred stage - Seemingly good:
UnsharpMask stage - Seemingly good:
HighContrast stage - Seemingly good:
Result stage of my code - Looks bad!
Result From Photoshop - Excellent!
First of all, judging by the artefacts that Photoshop left on the borders of the petals, I'd say that it applies the mask by using a weighted sum between the original image and the mask, as in the answer you tried first.
I modified your code to implement this scheme and I tried to tweak the parameters to get as close as the Photoshop result, but I couldn't without creating a lot of noise. I wouldn't try to guess what Photoshop is exactly doing (I would definitely like to know), however I discovered that it is fairly reproducible by applying some filter on the mask to reduce the noise. The algorithm scheme would be:
blurred = blur(image, Radius)
mask = image - blurred
mask = some_filter(mask)
sharpened = (mask < Threshold) ? image : image - Amount * mask
I implemented this and tried using basic filters (median blur, mean filter, etc) on the mask and this is the kind of result I can get:
which is a bit noisier than the Photoshop image but, in my opinion, close enough to what you wanted.
On another note, it will of course depend on the usage you have for your filter, but I think that the settings you used in Photoshop are too strong (you have big overshoots near petals borders). This is sufficient to have a nice image at the naked eye, with limited overshoot:
Finally, here is the code I used to generate the two images above:
#include <opencv2/opencv.hpp>
#include <iostream>
using namespace std;
using namespace cv;
Mat usm(Mat original, float radius, float amount, float threshold)
{
// work using floating point images to avoid overflows
cv::Mat input;
original.convertTo(input, CV_32FC3);
// copy original for our return value
Mat retbuf = input.clone();
// create the blurred copy
Mat blurred;
cv::GaussianBlur(input, blurred, cv::Size(0, 0), radius);
// subtract blurred from original, pixel-by-pixel to make unsharp mask
Mat unsharpMask;
cv::subtract(input, blurred, unsharpMask);
// --- filter on the mask ---
//cv::medianBlur(unsharpMask, unsharpMask, 3);
cv::blur(unsharpMask, unsharpMask, {3,3});
// --- end filter ---
// apply mask to image
for (int row = 0; row < original.rows; row++)
{
for (int col = 0; col < original.cols; col++)
{
Vec3f origColor = input.at<Vec3f>(row, col);
Vec3f difference = unsharpMask.at<Vec3f>(row, col);
if(cv::norm(difference) >= threshold) {
retbuf.at<Vec3f>(row, col) = origColor + amount * difference;
}
}
}
// convert back to unsigned char
cv::Mat ret;
retbuf.convertTo(ret, CV_8UC3);
return ret;
}
int main(int argc, char* argv[])
{
if (argc < 3) exit(1);
Mat original = imread(argv[1]);
Mat expected = imread(argv[2]);
// closer to Photoshop
Mat current = usm(original, 0.8, 12., 1.);
// better settings (in my opinion)
//Mat current = usm(original, 2., 1., 3.);
cv::imwrite("current.png", current);
// comparison plot
cv::Rect crop(127, 505, 163, 120);
cv::Mat crops[3];
cv::resize(original(crop), crops[0], {0,0}, 4, 4, cv::INTER_NEAREST);
cv::resize(expected(crop), crops[1], {0,0}, 4, 4, cv::INTER_NEAREST);
cv::resize( current(crop), crops[2], {0,0}, 4, 4, cv::INTER_NEAREST);
char const* texts[] = {"original", "photoshop", "current"};
cv::Mat plot = cv::Mat::zeros(120 * 4, 163 * 4 * 3, CV_8UC3);
for(int i = 0; i < 3; ++i) {
cv::Rect region(163 * 4 * i, 0, 163 * 4, 120 * 4);
crops[i].copyTo(plot(region));
cv::putText(plot, texts[i], region.tl() + cv::Point{5,40},
cv::FONT_HERSHEY_SIMPLEX, 1.5, CV_RGB(255, 0, 0), 2.0);
}
cv::imwrite("plot.png", plot);
}
Here's my attempt at 'smart' unsharp masking. Result isn't very good, but I'm posting anyway. Wikipedia article on unsharp masking has details about smart sharpening.
Several things I did differently:
Convert BGR to Lab color space and apply the enhancements to the brightness channel
Use an edge map to apply enhancement to the edge regions
Original:
Enhanced: sigma=2 amount=3 low=0.3 high=.8 w=2
Edge map: low=0.3 high=.8 w=2
#include "opencv2/core.hpp"
#include "opencv2/imgproc.hpp"
#include "opencv2/highgui.hpp"
#include <cstring>
cv::Mat not_so_smart_sharpen(
const cv::Mat& bgr,
double sigma,
double amount,
double canny_low_threshold_weight,
double canny_high_threshold_weight,
int edge_weight)
{
cv::Mat enhanced_bgr, lab, enhanced_lab, channel[3], blurred, difference, bw, kernel, edges;
// convert to Lab
cv::cvtColor(bgr, lab, cv::ColorConversionCodes::COLOR_BGR2Lab);
// perform the enhancement on the brightness component
cv::split(lab, channel);
cv::Mat& brightness = channel[0];
// smoothing for unsharp masking
cv::GaussianBlur(brightness, blurred, cv::Size(0, 0), sigma);
difference = brightness - blurred;
// calculate an edge map. I'll use Otsu threshold as the basis
double thresh = cv::threshold(brightness, bw, 0, 255, cv::ThresholdTypes::THRESH_BINARY | cv::ThresholdTypes::THRESH_OTSU);
cv::Canny(brightness, edges, thresh * canny_low_threshold_weight, thresh * canny_high_threshold_weight);
// control edge thickness. use edge_weight=0 to use Canny edges unaltered
cv::dilate(edges, edges, kernel, cv::Point(-1, -1), edge_weight);
// unsharp masking on the edges
cv::add(brightness, difference * amount, brightness, edges);
// use the enhanced brightness channel
cv::merge(channel, 3, enhanced_lab);
// convert to BGR
cv::cvtColor(enhanced_lab, enhanced_bgr, cv::ColorConversionCodes::COLOR_Lab2BGR);
// cv::imshow("edges", edges);
// cv::imshow("difference", difference * amount);
// cv::imshow("original", bgr);
// cv::imshow("enhanced", enhanced_bgr);
// cv::waitKey(0);
return enhanced_bgr;
}
int main(int argc, char *argv[])
{
double sigma = std::stod(argv[1]);
double amount = std::stod(argv[2]);
double low = std::stod(argv[3]);
double high = std::stod(argv[4]);
int w = std::stoi(argv[5]);
cv::Mat bgr = cv::imread("flower.jpg");
cv::Mat enhanced = not_so_smart_sharpen(bgr, sigma, amount, low, high, w);
cv::imshow("original", bgr);
cv::imshow("enhanced", enhanced);
cv::waitKey(0);
return 0;
}

Plotting Velocity Vectors in a Binary Image Using OpenCV

I have a binary (Black and White) image over which I want to plot velocity vectors of certain velocity. In MATLAB, we can use quiver to plot these vectors. I am looking for a solution to this problem in OpenCV using C++. I would be grateful if someone can share a solution to this. Although one of the solutions is provided in using the static image on SO (OpenCV How to Plot velocity vectors as arrows in using single static image), its not clear of how to implement it on a binary image. I would be grateful if someone can guide me.
Look forward to some suggestion towards implementation.
This is my solution to your problem: in this example I start with an rgb image coming from the webcam, then i convert it to grayscale and then to binary after applying a threshold.
The next step, when you have a binary image, is to convert it to RGB again (or BGR as the OpenCV convention) and draw whatever you want on it. The code for the arrow is a copypasta of what you have linked.
Hope it helps
cv::VideoCapture cam(n_source);
cam >> frame;
cv::Mat grey_image;
cv::Mat binary_image; // Your binary image
cv::cvtColor(frame, grey_image, CV_RGB2GRAY);
cv::threshold(grey_image, binary_image, 100, 255, 0);
// Convert the binary to RGB
cv::Mat dst_rgb;
cv::cvtColor(binary_image, dst_rgb, CV_GRAY2BGR);
// Draw the arrow on the RGB image
int x = 200;
int y = 200;
int u = 100;
int v = 100;
cv::Point pt1,pt2;
double Theta;
double PI = 3.1416;
cv::Scalar Color(255,0,0);
int size = 5;
int Thickness = 5;
if(u==0)
Theta=PI/2;
else
Theta=atan2(double(v),(double)(u));
pt1.x=x;
pt1.y=y;
pt2.x=x+u;
pt2.y=y+v;
cv::line(dst_rgb,pt1,pt2,Color,Thickness,8); //Draw Line
size=(int)(size*0.707);
if(Theta==PI/2 && pt1.y > pt2.y)
{
pt1.x=(int)(size*cos(Theta)-size*sin(Theta)+pt2.x);
pt1.y=(int)(size*sin(Theta)+size*cos(Theta)+pt2.y);
cv::line(dst_rgb,pt1,pt2,Color,Thickness,8); //Draw Line
pt1.x=(int)(size*cos(Theta)+size*sin(Theta)+pt2.x);
pt1.y=(int)(size*sin(Theta)-size*cos(Theta)+pt2.y);
cv::line(dst_rgb,pt1,pt2,Color,Thickness,8); //Draw Line
}
else{
pt1.x=(int)(-size*cos(Theta)-size*sin(Theta)+pt2.x);
pt1.y=(int)(-size*sin(Theta)+size*cos(Theta)+pt2.y);
cv::line(dst_rgb,pt1,pt2,Color,Thickness,8); //Draw Line
pt1.x=(int)(-size*cos(Theta)+size*sin(Theta)+pt2.x);
pt1.y=(int)(-size*sin(Theta)-size*cos(Theta)+pt2.y);
cv::line(dst_rgb,pt1,pt2,Color,Thickness,8); //Draw Line
}
// Plot
cv::namedWindow("test rgb");
cv::imshow("test rgb", dst_rgb);
cv::waitKey(0);
An example:
After some discussion I have come across cv::arrowedLine in OpenCV whose one of the usage is as follows:
#define _CRT_SECURE_NO_WARNINGS
#include <iostream>
#include <string>
#include <opencv2/opencv.hpp>
#include <opencv2/highgui.hpp>
using namespace std;
using namespace cv;
int main(int argc, char* argv[])
{
auto width = 320;
auto height = 320;
auto img = cv::Mat(cv::Size(width, height), CV_8UC3); // create background image
auto center = cv::Point(width / 2, height / 2); // center point
int lineType = 8;
int thickness = 1;
double tipLength = 0.1;
img.setTo(255); // clear image - set to black (0) or white (255)
for (int angle = 0; angle < 360; angle += 15)
{
auto angleRad = angle*CV_PI / 180.0; // convert angle to radians
auto length = 150;
auto direction = cv::Point(length * cos(angleRad), length * sin(angleRad)); // calculate direction
tipLength = .01 + 0.4 * (angle % 180) / 360;
cv::arrowedLine(img, center + direction*0.5, center + direction, CV_RGB(255, angle, 0), thickness, lineType, 0, tipLength); // draw arrow!
++thickness;
if (0 == angle % 45)
thickness = 0;
if (180 <= angle)
lineType = CV_AA;
}
imshow("Arrowed Image", img); // show image
waitKey();
return EXIT_SUCCESS;
}

Creating rectangle within a blob using OpenCV

Input Image:
Output Image:
I have several colored blobs in an image and I'm trying to create rectangles (or squares--which seems to be much easier) inside the largest blob of each color. I've found the answer to how to create a rectangle that bounds a single largest blob, but am unsure as to how to find a square that simply fits inside a blob. It doesn't have to be the largest, it just has to be larger than a certain area otherwise I just won't include it. I've also seen some work done on polygons, but nothing for amorphous shapes.
For a single blob, the problem can be formulated as: find the largest rectangle containing only zeros in a matrix.
To find the largest axes-oriented rectangle inside a blob, you can refer to the function findMinRect in my other answer. The code is a porting in C++ of the original in Python from here.
Then the second problem is to find all blobs with the same color. This is a little tricky because your image is jpeg, and compression creates a lot artificial colors near to borders. So I created a png image (shown below), just to show that the algorithm works. It's up to you to provide an image without compression artifacts.
Then you just need to create a mask for each color, find connected components for each blob in this mask, and compute the minimum rectangle for each blob.
Initial image:
Here I show the rects found for each blob, divided by color. You can then take only the rectangles you need, either the maximum rectangle for each color, or the rectangle for the largest blob for each color.
Result:
Here the code:
#include <opencv2/opencv.hpp>
#include <algorithm>
#include <set>
using namespace std;
using namespace cv;
// https://stackoverflow.com/a/30418912/5008845
Rect findMinRect(const Mat1b& src)
{
Mat1f W(src.rows, src.cols, float(0));
Mat1f H(src.rows, src.cols, float(0));
Rect maxRect(0, 0, 0, 0);
float maxArea = 0.f;
for (int r = 0; r < src.rows; ++r)
{
for (int c = 0; c < src.cols; ++c)
{
if (src(r, c) == 0)
{
H(r, c) = 1.f + ((r>0) ? H(r - 1, c) : 0);
W(r, c) = 1.f + ((c>0) ? W(r, c - 1) : 0);
}
float minw = W(r, c);
for (int h = 0; h < H(r, c); ++h)
{
minw = min(minw, W(r - h, c));
float area = (h + 1) * minw;
if (area > maxArea)
{
maxArea = area;
maxRect = Rect(Point(c - minw + 1, r - h), Point(c + 1, r + 1));
}
}
}
}
return maxRect;
}
struct lessVec3b
{
bool operator()(const Vec3b& lhs, const Vec3b& rhs) {
return (lhs[0] != rhs[0]) ? (lhs[0] < rhs[0]) : ((lhs[1] != rhs[1]) ? (lhs[1] < rhs[1]) : (lhs[2] < rhs[2]));
}
};
int main()
{
// Load image
Mat3b img = imread("path_to_image");
// Find unique colors
set<Vec3b, lessVec3b> s(img.begin(), img.end());
// Divide planes of original image
vector<Mat1b> planes;
split(img, planes);
for (auto color : s)
{
// Create a mask with only pixels of the given color
Mat1b mask(img.rows, img.cols, uchar(255));
for (int i = 0; i < 3; ++i)
{
mask &= (planes[i] == color[i]);
}
// Find blobs
vector<vector<Point>> contours;
findContours(mask, contours, RETR_EXTERNAL, CHAIN_APPROX_SIMPLE);
for (int i = 0; i < contours.size(); ++i)
{
// Create a mask for each single blob
Mat1b maskSingleContour(img.rows, img.cols, uchar(0));
drawContours(maskSingleContour, contours, i, Scalar(255), CV_FILLED);
// Find minimum rect for each blob
Rect box = findMinRect(~maskSingleContour);
// Draw rect
Scalar rectColor(color[1], color[2], color[0]);
rectangle(img, box, rectColor, 2);
}
}
imshow("Result", img);
waitKey();
return 0;
}
You can use this code to locate the largest square or rectangle inscribed inside arbitrary shape. Although it's MATLAB instead of C++/OpenCV, you can easily change its source code to fit to your needs.
For locating the largest rectangle inscribed inside convex polygons, check out here (with code).

Calculating skew of text OpenCV

I am trying to calculate the skew of text in an image so I can correct it for the best OCR results.
Currently this is the function I am using:
double compute_skew(Mat &img)
{
// Binarize
cv::threshold(img, img, 225, 255, cv::THRESH_BINARY);
// Invert colors
cv::bitwise_not(img, img);
cv::Mat element = cv::getStructuringElement(cv::MORPH_RECT, cv::Size(5, 3));
cv::erode(img, img, element);
std::vector<cv::Point> points;
cv::Mat_<uchar>::iterator it = img.begin<uchar>();
cv::Mat_<uchar>::iterator end = img.end<uchar>();
for (; it != end; ++it)
if (*it)
points.push_back(it.pos());
cv::RotatedRect box = cv::minAreaRect(cv::Mat(points));
double angle = box.angle;
if (angle < -45.)
angle += 90.;
cv::Point2f vertices[4];
box.points(vertices);
for(int i = 0; i < 4; ++i)
cv::line(img, vertices[i], vertices[(i + 1) % 4], cv::Scalar(255, 0, 0), 1, CV_AA);
return angle;
}
When I look at then angle in debug I get 0.000000
However when I give it this image I get proper results of a skew of about 16 degrees:
How can I properly detect the skew in the first image?
there are a few other ways to get the skew degree, 1) by hough transform 2) by horizontal projection profile. rotate the image in different angle bins and calculate horizontal projection. the angle with the greatest horizontal histogram value is the deskewed angle.
i have provided below implementation of 1). i believe this to be superior to the boxing method you are using because it requires that you completely clean the image of any noise,which just isnt possible in most of the time.
you should know that the method doesnt work well if there's too much noise. you can reduce noise in different ways depending on what type of "line" you want to treat as the most dominant in the image. i have provided two methods for this. be sure to play with parameters and threshold etc.
results (all run using preprocess2, all run using same parameter set)
code
#include <opencv2/opencv.hpp>
using namespace cv;
using namespace std;
void hough_transform(Mat& im,Mat& orig,double* skew)
{
double max_r=sqrt(pow(.5*im.cols,2)+pow(.5*im.rows,2));
int angleBins = 180;
Mat acc = Mat::zeros(Size(2*max_r,angleBins),CV_32SC1);
int cenx = im.cols/2;
int ceny = im.rows/2;
for(int x=1;x<im.cols-1;x++)
{
for(int y=1;y<im.rows-1;y++)
{
if(im.at<uchar>(y,x)==255)
{
for(int t=0;t<angleBins;t++)
{
double r =(x-cenx)*cos((double)t/angleBins*CV_PI)+(y-ceny)*sin((double)t /angleBins*CV_PI);
r+=max_r;
acc.at<int>(t,int(r))++;
}
}
}
}
Mat thresh;
normalize(acc,acc,255,0,NORM_MINMAX);
convertScaleAbs(acc,acc);
/*debug
Mat cmap;
applyColorMap(acc,cmap,COLORMAP_JET);
imshow("cmap",cmap);
imshow("acc",acc);*/
Point maxLoc;
minMaxLoc(acc,0,0,0,&maxLoc);
double theta = (double)maxLoc.y/angleBins*CV_PI;
double rho = maxLoc.x-max_r;
if(abs(sin(theta))<0.000001)//check vertical
{
//when vertical, line equation becomes
//x = rho
double m = -cos(theta)/sin(theta);
Point2d p1 = Point2d(rho+im.cols/2,0);
Point2d p2 = Point2d(rho+im.cols/2,im.rows);
line(orig,p1,p2,Scalar(0,0,255),1);
*skew=90;
cout<<"skew angle "<<" 90"<<endl;
}else
{
//convert normal form back to slope intercept form
//y = mx + b
double m = -cos(theta)/sin(theta);
double b = rho/sin(theta)+im.rows/2.-m*im.cols/2.;
Point2d p1 = Point2d(0,b);
Point2d p2 = Point2d(im.cols,im.cols*m+b);
line(orig,p1,p2,Scalar(0,0,255),1);
double skewangle;
skewangle= p1.x-p2.x>0? (atan2(p1.y-p2.y,p1.x-p2.x)*180./CV_PI):(atan2(p2.y-p1.y,p2. x-p1.x)*180./CV_PI);
*skew=skewangle;
cout<<"skew angle "<<skewangle<<endl;
}
imshow("orig",orig);
}
Mat preprocess1(Mat& im)
{
Mat ret = Mat::zeros(im.size(),CV_32SC1);
for(int x=1;x<im.cols-1;x++)
{
for(int y=1;y<im.rows-1;y++)
{
int gy = (im.at<uchar>(y-1,x+1)-im.at<uchar>(y-1,x-1))
+2*(im.at<uchar>(y,x+1)-im.at<uchar>(y,x-1))
+(im.at<uchar>(y+1,x+1)-im.at<uchar>(y+1,x-1));
int gx = (im.at<uchar>(y+1,x-1) -im.at<uchar>(y-1,x-1))
+2*(im.at<uchar>(y+1,x)-im.at<uchar>(y-1,x))
+(im.at<uchar>(y+1,x+1)-im.at<uchar>(y-1,x+1));
int g2 = (gy*gy + gx*gx);
ret.at<int>(y,x)=g2;
}
}
normalize(ret,ret,255,0,NORM_MINMAX);
ret.convertTo(ret,CV_8UC1);
threshold(ret,ret,50,255,THRESH_BINARY);
return ret;
}
Mat preprocess2(Mat& im)
{
// 1) assume white on black and does local thresholding
// 2) only allow voting top is white and buttom is black(buttom text line)
Mat thresh;
//thresh=255-im;
thresh=im.clone();
adaptiveThreshold(thresh,thresh,255,CV_ADAPTIVE_THRESH_GAUSSIAN_C,THRESH_BINARY,15,-2);
Mat ret = Mat::zeros(im.size(),CV_8UC1);
for(int x=1;x<thresh.cols-1;x++)
{
for(int y=1;y<thresh.rows-1;y++)
{
bool toprowblack = thresh.at<uchar>(y-1,x)==0 || thresh.at<uchar>(y-1,x-1)==0 || thresh.at<uchar>(y-1,x+1)==0;
bool belowrowblack = thresh.at<uchar>(y+1,x)==0 || thresh.at<uchar>(y+1, x-1)==0 || thresh.at<uchar>(y+1,x+1)==0;
uchar pix=thresh.at<uchar>(y,x);
if((!toprowblack && pix==255 && belowrowblack))
{
ret.at<uchar>(y,x) = 255;
}
}
}
return ret;
}
Mat rot(Mat& im,double thetaRad)
{
cv::Mat rotated;
double rskew = thetaRad* CV_PI/180;
double nw = abs(sin(thetaRad))*im.rows+abs(cos(thetaRad))*im.cols;
double nh = abs(cos(thetaRad))*im.rows+abs(sin(thetaRad))*im.cols;
cv::Mat rot_mat = cv::getRotationMatrix2D(Point2d(nw*.5,nh*.5), thetaRad*180/CV_PI, 1);
Mat pos = Mat::zeros(Size(1,3),CV_64FC1);
pos.at<double>(0)=(nw-im.cols)*.5;
pos.at<double>(1)=(nh-im.rows)*.5;
Mat res = rot_mat*pos;
rot_mat.at<double>(0,2) += res.at<double>(0);
rot_mat.at<double>(1,2) += res.at<double>(1);
cv::warpAffine(im, rotated, rot_mat,Size(nw,nh), cv::INTER_LANCZOS4);
return rotated;
}
int main(int argc, char** argv)
{
string src="C:/data/skew.png";
Mat im= imread(src);
Mat gray;
cvtColor(im,gray,CV_BGR2GRAY);
Mat preprocessed = preprocess2(gray);
imshow("preprocessed2",preprocessed);
double skew;
hough_transform(preprocessed,im,&skew);
Mat rotated = rot(im,skew* CV_PI/180);
imshow("corrected",rotated);
waitKey(0);
return 0;
}
the approach you posted has its own "ideal binarization" assumption. the threshold value directly affects the process. utilize otsu threshold, or think about DFT for a generic solution.
otsu trial:
int main()
{
Mat input = imread("your text");
cvtColor(input, input, CV_BGR2GRAY);
Mat img;
cv::threshold(input, img, 100, 255, cv::THRESH_OTSU);
cv::bitwise_not(img, img);
imshow("img ", img);
waitKey(0);
vector<Point> points;
findNonZero(img, points);
cv::RotatedRect box = cv::minAreaRect(points);
double angle = box.angle;
if (angle < -45.)
angle += 90.;
cv::Point2f vertices[4];
box.points(vertices);
for(int i = 0; i < 4; ++i)
cv::line(img, vertices[i], vertices[(i + 1) % 4], cv::Scalar(255, 0, 0));
imshow("img ", img);
waitKey(0);
return 0;
}