Debug assertion fail when using grayscale - c++

I am trying to make a hand recognition system but when i used grayscale for cvtColor, i get debug assertion fail but when i use HSV the code works fine. Can you resolve this ? I am a newbie in opencv.
#include "stdafx.h"
#include "opencv2/imgproc/imgproc.hpp"
#include "opencv2/highgui/highgui.hpp"
#include "opencv2/objdetect.hpp"
#include < opencv2\opencv.hpp>
#include < stdio.h>
#include <iostream>
using namespace std;
using namespace cv;
int thresh = 100;
int findBiggestContour(vector<vector<Point> > contours){
int indexOfBiggestContour = -1;
int sizeOfBiggestContour = 0;
for (int i = 0; i < contours.size(); i++){
if (contours[i].size() > sizeOfBiggestContour){
sizeOfBiggestContour = contours[i].size();
indexOfBiggestContour = i;
}
}
return indexOfBiggestContour;
}
void shifcontour(vector<Point>& contour, int x, int y)
{
for (size_t i = 0; i<contour.size(); i++)
{
contour[i].x += x;
contour[i].y += y;
}
}
int main()
{
cout << "beginning";
VideoCapture cap("pathaka.MP4");
if (!cap.isOpened()) // check if we succeeded
return -1;
Ptr<BackgroundSubtractor> pMOG2 = createBackgroundSubtractorMOG2();
for (;;)
{
Mat original, img;
cap >> img;
imshow("Source", img);
Mat hsv;
cvtColor(img, hsv, CV_BGR2GRAY);
Mat bw;
inRange(hsv, Scalar(0, 30, 80), Scalar(20, 150, 255), bw);
GaussianBlur(bw, bw, Size(7, 7), 1.5, 1.5);
Canny(bw, bw, 0, 30, 3);
vector<vector<Point> > contours;
vector<vector<Point> > convex_hull;
vector<Vec4i> hierarchy;
int erosion_type = MORPH_ELLIPSE;
int erosion_size = 0;
Mat element = getStructuringElement(erosion_type,
Size(2 * erosion_size + 1, 2 * erosion_size + 1),
Point(erosion_size, erosion_size));
dilate(bw, bw, element);
findContours(bw, contours, hierarchy, CV_RETR_TREE, CV_CHAIN_APPROX_SIMPLE, Point(0, 0));
int s = findBiggestContour(contours);
Mat drawing = Mat::zeros(img.size(), CV_8UC1);
dilate(drawing, drawing, getStructuringElement(MORPH_ELLIPSE, Size(5, 5)));
dilate(drawing, drawing, getStructuringElement(MORPH_ELLIPSE, Size(5, 5)));
std::vector<cv::Point> cnt;
cnt = contours[s];
Moments M;
M = cv::moments(cnt);
cv::Point result;
result = cv::Point(M.m10 / M.m00, M.m01 / M.m00);
Point center(drawing.cols / 2, drawing.rows / 2);
cv::circle(drawing, center, 3, Scalar(255, 255, 255), -1, 8, 0);
int x;
if (result.x > center.x)
{
x = result.x - center.x;
x = -x;
}
else
{
x = result.x - center.x;
}
int y;
if (result.y < center.y)
{
y = center.y - result.y;
}
else
{
y = center.y - result.y;
}
cout << "x:" << x << endl;
cout << "y: " << y << endl;
shifcontour(contours[s], x, y);
drawContours(drawing, contours, s, Scalar(255), -1, 8, hierarchy, 0, Point());
imshow("Hsv", drawing);
if (waitKey(30) >= 0) break;
}
return 0;
}

I think the problem is:
findContours(bw, contours, hierarchy, CV_RETR_TREE, CV_CHAIN_APPROX_SIMPLE, Point(0, 0));
contours now may have something inside, but it could be empty, right? Then, you do this:
int s = findBiggestContour(contours);
If contours.size() == 0, then s == -1, correct?
But after that, you do this:
std::vector<cv::Point> cnt;
cnt = contours[s];
If contours is empty, contours[-1] throws vector subscript out of range.
You should check if (s != -1) before using contours[s], ok?
Perhaps you should process the contours only if there is any, like this:
findContours(bw, contours, hierarchy, CV_RETR_TREE, CV_CHAIN_APPROX_SIMPLE, Point(0, 0));
if (contours.size() > 0) {
int s = findBiggestContour(contours);
Mat drawing = Mat::zeros(img.size(), CV_8UC1);
// these dilates are useless, because drawing is an empty image!
dilate(drawing, drawing, getStructuringElement(MORPH_ELLIPSE, Size(5, 5)));
dilate(drawing, drawing, getStructuringElement(MORPH_ELLIPSE, Size(5, 5)));
std::vector<cv::Point> cnt = contours[s];
Moments M = cv::moments(cnt);
Point result = cv::Point(M.m10 / M.m00, M.m01 / M.m00);
Point center(drawing.cols / 2, drawing.rows / 2);
circle(drawing, center, 3, Scalar(255, 255, 255), -1, 8, 0);
int x;
if (result.x > center.x) {
x = result.x - center.x;
x = -x;
} else {
x = result.x - center.x;
}
// is this correct? y has the same value in both cases...
int y;
if (result.y < center.y) y = center.y - result.y;
else y = center.y - result.y;
cout << "x:" << x << endl;
cout << "y: " << y << endl;
shifcontour(contours[s], x, y);
drawContours(drawing, contours, s, Scalar(255), -1, 8, hierarchy, 0, Point());
imshow("Hsv", drawing);
}

Related

Can't run c++ project with webcam, but it's actually work with image

I create c++ project named "Document Scanner". With images, he works perfectly but with my webcam, he gives me an error "vector subscript out of range".
Code for images:
#include <opencv2/imgcodecs.hpp>
#include <opencv2/highgui.hpp>
#include <opencv2/imgproc.hpp>
#include <iostream>
using namespace cv;
using namespace std;
//////////////// Project 2 – Document Scanner ////////////
Mat imgOriginal, imgGray, imgCanny, imgThre, imgBlur, imgDil, imgErode, imgCrop, imgWarp;
vector<Point> initialPoints, docPoints;
float w = 420, h = 596; //dimensions of a4 paper multiplied by 2
Mat preProcessing(Mat img) {
cvtColor(img, imgGray, COLOR_BGR2GRAY);
GaussianBlur(imgGray, imgBlur, Size(3, 3), 3, 0);
Canny(imgBlur, imgCanny, 25, 75);
Mat kernel = getStructuringElement(MORPH_RECT, Size(3, 3));
dilate(imgCanny, imgDil, kernel);
//erode(imgDil, imgErode, kernel);
return imgDil;
}
vector<Point> getContours(Mat image) {
vector<vector<Point>> contours;
vector<Vec4i> hierarchy;
findContours(imgDil, contours, hierarchy, RETR_EXTERNAL, CHAIN_APPROX_SIMPLE);
//drawContours(img, contours, -1, Scalar(255, 0, 255), 2);
vector<vector<Point>> conPoly(contours.size());
vector<Rect> boundRect(contours.size());
vector<Point> biggest;
int maxArea = 0;
for (int i = 0; i < contours.size(); i++)
{
int area = contourArea(contours[i]);
cout << area << endl;
string objectType;
if (area > 1000) {
float peri = arcLength(contours[i], true);
approxPolyDP(contours[i], conPoly[i], 0.02 * peri, true);
if (area > maxArea && conPoly[i].size() == 4) {
//drawContours(imgOriginal, conPoly, i, Scalar(255, 0, 255), 5);
biggest = { conPoly[i][0],conPoly[i][1], conPoly[i][2], conPoly[i][3] };
maxArea = area;
}
//drawContours(imgOriginal, conPoly, i, Scalar(255, 0, 255), 2);
//rectangle(imgOriginal, boundRect[i].tl(), boundRect[i].br(), Scalar(0, 255, 0), 5);
}
}
return biggest;
}
void drawPoints(vector<Point> points, Scalar color) {
for (int i = 0; i < points.size(); i++) {
circle(imgOriginal, points[i], 10, color, FILLED);
putText(imgOriginal, to_string(i), points[i], FONT_HERSHEY_PLAIN, 4, color, 4);
}
}
vector<Point> reorder(vector<Point> points) {
vector<Point> newPoints;
vector<int> sumPoints, subPoints;
for (int i = 0; i < 4; i++) {
sumPoints.push_back(points[i].x + points[i].y);
subPoints.push_back(points[i].x - points[i].y);
}
newPoints.push_back(points[min_element(sumPoints.begin(), sumPoints.end()) - sumPoints.begin()]); //0
newPoints.push_back(points[max_element(subPoints.begin(), subPoints.end()) - subPoints.begin()]); //1
newPoints.push_back(points[min_element(subPoints.begin(), subPoints.end()) - subPoints.begin()]); //2
newPoints.push_back(points[max_element(sumPoints.begin(), sumPoints.end()) - sumPoints.begin()]); //3
return newPoints;
}
Mat getWarp(Mat img, vector<Point> points, float w, float h) {
Point2f src[4] = { points[0], points[1], points[2], points[3] };
Point2f dst[4] = { {0.0f,0.0f}, {w,0.0f}, {0.0f,h}, {w, h} };
Mat matrix = getPerspectiveTransform(src, dst);
warpPerspective(img, imgWarp, matrix, Point(w, h));
return imgWarp;
}
void main() {
string path = "Resources/paper.jpg";
imgOriginal = imread(path);
//resize(imgOriginal, imgOriginal, Size(), 0.5, 0.4);
// Preprocessing
imgThre = preProcessing(imgOriginal);
// Get Contours - Biggest
initialPoints = getContours(imgThre);
///drawPoints(initialPoints, Scalar(0,0,255));
docPoints = reorder(initialPoints);
//drawPoints(docPoints, Scalar(0, 255, 0));
// Warp
imgWarp = getWarp(imgOriginal, docPoints, w, h);
//Crop
int cropVal = 5;
Rect roi(cropVal, cropVal, w - (2 * cropVal), h - (2 * cropVal));
imgCrop = imgWarp(roi);
imshow("Image", imgOriginal);
imshow("Image Dilation", imgThre);
imshow("Image Warp", imgWarp);
imshow("Image Crop", imgCrop);
waitKey(0);
}
Code for webcam:
#include <opencv2/imgcodecs.hpp>
#include <opencv2/highgui.hpp>
#include <opencv2/imgproc.hpp>
#include <iostream>
using namespace cv;
using namespace std;
//////////////// Project 2 – Document Scanner ////////////
Mat img, imgGray, imgCanny, imgThre, imgBlur, imgDil, imgErode, imgCrop, imgWarp;
vector<Point> initialPoints, docPoints;
float w = 420, h = 596; //dimensions of a4 paper multiplied by 2
Mat preProcessing(Mat img) {
cvtColor(img, imgGray, COLOR_BGR2GRAY);
GaussianBlur(imgGray, imgBlur, Size(3, 3), 3, 0);
Canny(imgBlur, imgCanny, 25, 75);
Mat kernel = getStructuringElement(MORPH_RECT, Size(3, 3));
dilate(imgCanny, imgDil, kernel);
//erode(imgDil, imgErode, kernel);
return imgDil;
}
vector<Point> getContours(Mat image) {
vector<vector<Point>> contours;
vector<Vec4i> hierarchy;
findContours(image, contours, hierarchy, RETR_EXTERNAL, CHAIN_APPROX_SIMPLE);
//drawContours(img, contours, -1, Scalar(255, 0, 255), 2);
vector<vector<Point>> conPoly(contours.size());
vector<Rect> boundRect(contours.size());
vector<Point> biggest;
int maxArea = 0;
for (int i = 0; i < contours.size(); i++)
{
int area = contourArea(contours[i]);
cout << area << endl;
if (area > 1000) {
float peri = arcLength(contours[i], true);
approxPolyDP(contours[i], conPoly[i], 0.02 * peri, true);
if (area > maxArea && conPoly[i].size() == 4) {
//drawContours(imgOriginal, conPoly, i, Scalar(255, 0, 255), 5);
biggest = { conPoly[i][0],conPoly[i][1], conPoly[i][2], conPoly[i][3] };
maxArea = area;
}
//drawContours(imgOriginal, conPoly, i, Scalar(255, 0, 255), 2);
//rectangle(imgOriginal, boundRect[i].tl(), boundRect[i].br(), Scalar(0, 255, 0), 5);
}
}
return biggest;
}
void drawPoints(vector<Point> points, Scalar color) {
for (int i = 0; i < points.size(); i++) {
circle(img, points[i], 10, color, FILLED);
putText(img, to_string(i), points[i], FONT_HERSHEY_PLAIN, 4, color, 4);
}
}
vector<Point> reorder(vector<Point> points) {
vector<Point> newPoints;
vector<int> sumPoints, subPoints;
for (int i = 0; i < 4; i++) {
sumPoints.push_back(points[i].x + points[i].y);
subPoints.push_back(points[i].x - points[i].y);
}
newPoints.push_back(points[min_element(sumPoints.begin(), sumPoints.end()) - sumPoints.begin()]); //0
newPoints.push_back(points[max_element(subPoints.begin(), subPoints.end()) - subPoints.begin()]); //1
newPoints.push_back(points[min_element(subPoints.begin(), subPoints.end()) - subPoints.begin()]); //2
newPoints.push_back(points[max_element(sumPoints.begin(), sumPoints.end()) - sumPoints.begin()]); //3
return newPoints;
}
Mat getWarp(Mat img, vector<Point> points, float w, float h) {
Point2f src[4] = { points[0], points[1], points[2], points[3] };
Point2f dst[4] = { {0.0f,0.0f}, {w,0.0f}, {0.0f,h}, {w, h} };
Mat matrix = getPerspectiveTransform(src, dst);
warpPerspective(img, imgWarp, matrix, Point(w, h));
return imgWarp;
}
void main() {
VideoCapture cap(0);
//string path = "Resources/paper.jpg";
//imgOriginal = imread(path);
//resize(imgOriginal, imgOriginal, Size(), 0.5, 0.4);
while (true) {
cap.read(img);
// Preprocessing
imgThre = preProcessing(img);
// Get Contours - Biggest
initialPoints = getContours(imgThre);
///drawPoints(initialPoints, Scalar(0,0,255));
docPoints = reorder(initialPoints);
//drawPoints(docPoints, Scalar(0, 255, 0));
// Warp
imgWarp = getWarp(img, docPoints, w, h);
//Crop
int cropVal = 5;
Rect roi(cropVal, cropVal, w - (2 * cropVal), h - (2 * cropVal));
imgCrop = imgWarp(roi);
//imshow("Image", imgOriginal);
//imshow("Image Dilation", imgThre);
//imshow("Image Warp", imgWarp);
imshow("Image Crop", imgCrop);
waitKey(1);
}
}
After debugging my program I find an error in function "reorder", when I push_back(). How can I resolve that problem?

Debug assertion failure, "__acrt_first_block == header" when using openCV for live webcam? [duplicate]

This question already has answers here:
why opencv imshow() create a new window has the same name as namedWindow() does in Debug Mode?
(1 answer)
Debug Assertion Failed! Expression: __acrt_first_block == header
(6 answers)
Closed 4 years ago.
I'm trying to do finger recognition using opencv, which has been working correctly when simply processing one image from the capture, however after adding the while loop to have it go from single image capture to live processing, there seems to be a heap error showing. I'm currently running it on Visual Studio 2017 w/ OpenCV 3.41.
The error from the Microsoft Visual C++ Runtime Library is
Debug Assertion Failed!
File: minkernel\crts\ucrt\src\appcrt\heap\debug_heap.cpp
Line: 996
Expression: __acrt_first_block == header
The code I'm using is:
#include "stdafx.h"
#include "opencv2/imgcodecs.hpp"
#include "opencv2/highgui.hpp"
#include "opencv2/imgproc.hpp"
#include <iostream>
using namespace cv;
using namespace std;
int myMax(int a, int b, int c);
int myMin(int a, int b, int c);
void mySkinDetect(Mat& src, Mat& dst);
Mat src; Mat src_gray;
int thresh = 100;
int max_thresh = 255;
RNG rng(12345);
void thresh_callback(int, void*);
int main(){
VideoCapture cap(0);
while (1) {
cap >> src;
Mat frameDest;
frameDest = Mat::zeros(src.rows, src.cols, CV_8UC1);
mySkinDetect(src, frameDest);
int erosion_size = 1;
Mat element = getStructuringElement(MORPH_RECT,
Size(2 * erosion_size + 1, 2 * erosion_size + 1),
Point(erosion_size, erosion_size));
erode(frameDest, frameDest, element);
erode(frameDest, frameDest, element);
namedWindow("Skin", WINDOW_AUTOSIZE);
imshow("Skin", frameDest);
blur(frameDest, src, Size(3, 3));
createTrackbar(" Threshold:", "Source", &thresh, max_thresh, thresh_callback);
thresh_callback(0, 0);
if (waitKey(30) == 27) { break; }
}
return(0);
}
void thresh_callback(int, void*)
{
Mat threshold_output;
vector<vector<Point> > contours;
vector<Vec4i> hierarchy;
threshold(src, threshold_output, thresh, 255, THRESH_BINARY);
findContours(threshold_output, contours, hierarchy, RETR_TREE, CHAIN_APPROX_SIMPLE, Point(0, 0));
vector<vector<Point>>hull(contours.size());
vector<vector<int> > hullsI(contours.size());
vector<vector<Vec4i>>defects(contours.size());
int index = 0;
int area = 0;
for (int i = 0; i < contours.size(); i++)
{
double a = contourArea(contours[i]);
if (a>area)
{
area = a;
index = i;
}
}
for (int i = 0; i < contours.size(); i++)
{
convexHull(contours[i], hull[i], false);
convexHull(contours[i], hullsI[i], false);
if (hullsI[i].size() > 3)
{
convexityDefects(contours[i], hullsI[i], defects[i]);
}
}
Mat drawing = Mat::zeros(threshold_output.size(), CV_8UC3);
for (size_t i = 0; i< contours.size(); i++)
{
Scalar color = Scalar(rng.uniform(0, 255), rng.uniform(0, 255), rng.uniform(0, 255));
drawContours(drawing, contours, (int)i, color, 1, 8, vector<Vec4i>(), 0, Point());
drawContours(drawing, hull, (int)i, color, 1, 8, vector<Vec4i>(), 0, Point());
}
int fingers = 0;
if (area>50)
{
for (int j = 0; j<defects[index].size(); ++j)
{
const Vec4i& def = defects[index][j];
float depth = def[3] / 256;
if (depth > 5) // filter defects by depth
{
int start = def[0];
Point ptStart(contours[index][start]);
int end = def[1];
Point ptEnd(contours[index][end]);
int min = def[2];
Point ptFar(contours[index][min]);
line(drawing, ptStart, ptEnd, Scalar(0, 255, 0), 1);
line(drawing, ptStart, ptFar, Scalar(0, 255, 0), 1);
line(drawing, ptEnd, ptFar, Scalar(0, 255, 0), 1);
circle(drawing, ptFar, 4, Scalar(0, 255, 0), 2);
fingers += 1;
}
}
}
std::string s = std::to_string(fingers-1);
namedWindow("Hull demo", WINDOW_AUTOSIZE);
putText(drawing, "Number Fingers = "+s, Point(drawing.cols/1.5, drawing.rows / 10), FONT_HERSHEY_PLAIN, 1.2f, Scalar(200, 0, 0), 2);
imshow("Hull demo", drawing);
}
int myMax(int a, int b, int c) {
int m = a;
(void)((m < b) && (m = b));
(void)((m < c) && (m = c));
return m;
}
//Function that returns the minimum of 3 integers
int myMin(int a, int b, int c) {
int m = a;
(void)((m > b) && (m = b));
(void)((m > c) && (m = c));
return m;
}
//Function that detects whether a pixel belongs to the skin based on RGB values
void mySkinDetect(Mat& src, Mat& dst) {
//Surveys of skin color modeling and detection techniques:
//Vezhnevets, Vladimir, Vassili Sazonov, and Alla Andreeva. "A survey on pixel-based skin color detection techniques." Proc. Graphicon. Vol. 3. 2003.
//Kakumanu, Praveen, Sokratis Makrogiannis, and Nikolaos Bourbakis. "A survey of skin-color modeling and detection methods." Pattern recognition 40.3 (2007): 1106-1122.
for (int i = 0; i < src.rows; i++) {
for (int j = 0; j < src.cols; j++) {
//For each pixel, compute the average intensity of the 3 color channels
Vec3b intensity = src.at<Vec3b>(i, j); //Vec3b is a vector of 3 uchar (unsigned character)
int B = intensity[0]; int G = intensity[1]; int R = intensity[2];
if ((R > 95 && G > 40 && B > 20) && (myMax(R, G, B) - myMin(R, G, B) > 15) && (abs(R - G) > 15) && (R > G) && (R > B)) {
dst.at<uchar>(i, j) = 255;
}
}
}
}

smoothing a contour for hand using OpenCV in C++

I have this code but my contours for hand are not smooth. How can I fix it?
void Hand::analyzeHand(cv::Mat xyzMap)
{
cv::Mat normalizedDepthMap;
cv::Mat channel[3];
cv::split(xyzMap, channel);
cv::normalize(channel[2], normalizedDepthMap, 0, 255, cv::NORM_MINMAX, CV_8UC1);
// Resize input
cv::Mat input;
cv::pyrUp(normalizedDepthMap, input, cv::Size(normalizedDepthMap.cols * 2, normalizedDepthMap.rows * 2));
cv::pyrUp(input, input, cv::Size(input.cols * 2, input.rows * 2));
cv::Mat threshold_output;
std::vector< std::vector<cv::Point> > contours;
std::vector<cv::Vec4i> hierarchy;
// Find contours
cv::threshold(input, threshold_output, 100, 255, cv::THRESH_BINARY);
cv::findContours(threshold_output, contours, hierarchy, CV_RETR_TREE, CV_CHAIN_APPROX_SIMPLE, cv::Point(0, 0));
// Find contour polygon
std::vector< std::vector< cv::Point> > contours_poly(contours.size());
for (auto i = 0; i < contours.size(); i++)
{
cv::approxPolyDP(cv::Mat(contours[i]), contours_poly[i], 3, true);
}
// Find largest contour
auto contour = Hand::findComplexContour(contours);
// Find approximated convex hull
std::vector<cv::Point> hull;
std::vector<cv::Point> completeHull;
std::vector<int> indexHull;
if (contour.size() > 1)
{
cv::convexHull(contour, completeHull, false, true);
cv::convexHull(contour, indexHull, false, false);
hull = Hand::clusterConvexHull(completeHull, Hand::CLUSTER_THRESHOLD);
}
// Find convexityDefects
std::vector<cv::Vec4i> defects;
if (indexHull.size() > 3)
{
cv::convexityDefects(contour, indexHull, defects);
}
// Find max and min distances
double minVal, maxVal;
cv::Point minLoc, maxLoc;
cv::minMaxLoc(channel[2], &minVal, &maxVal, &minLoc, &maxLoc);
// Find center of contour
auto center = Hand::findCenter(contour);
centroid_xyz = xyzMap.at<cv::Vec3f>(center.y / 4, center.x / 4);
centroid_ij = cv::Point2i(center.x, center.y); // SCALING
// Generate visual
cv::Mat img = cv::Mat::zeros(input.rows, input.cols, CV_8UC3);
auto color = cv::Scalar(0, 255, 0);
// Draw contours
cv::circle(img, center, 5, cv::Scalar(255, 0, 0), 2);
for (auto i = 0; i < contours.size(); i++)
{
cv::drawContours(img, contours_poly, i, color, 1, 8, std::vector<cv::Vec4i>(), 0, cv::Point());
}
// Draw hull
cv::Point index;
cv::Point index_right;
cv::Point index_left;
double farthest = 0;
if (hull.size() > 1)
{
for (auto i = 0; i < hull.size(); i++)
{
auto p1 = hull[i];
auto p2 = hull[(i + 1) % hull.size()];
//cv::line(img, p1, p2, cv::Scalar(255, 0, 0), 1);
if (p1.y < centroid_ij.y && Util::euclideanDistance2D(p1, centroid_ij) > farthest)
{
farthest = Util::euclideanDistance2D(p1, centroid_ij);
index = p1;
index_right = hull[(i + 1) % hull.size()];
index_left = hull[(i - 1) % hull.size()];
}
}
}
// Draw defects (filter)
std::vector<cv::Point> endpoints;
std::vector<cv::Point> fingerDefects;
cv::Point lastStart;
auto found = -1;
for (auto i = 0; i < defects.size(); i++)
{
auto defect = defects[i];
auto start = contour[defect[0]];
auto end = contour[defect[1]];
auto farPt = contour[defect[2]];
// Depth from edge of contour
// std::cout << "Depth: " << depth << "\tThreshold: " << cv::norm(maxLoc - center) << "\t";
// Defect conditions: depth is sufficient, inside contour, y value is above center
auto depth = defect[3];
// maxLoc largest depth
// first condition replace with meters distance from the edge
// second test if inside the hull (no change)
// above the center (no change)
if (cv::norm(maxLoc - center) * 15 < depth && cv::pointPolygonTest(hull, farPt, false) > 0 && farPt.y < center.y)
{
auto pt1 = xyzMap.at<cv::Vec3f>(farPt.y / 4, farPt.x / 4);
if (Util::euclidianDistance3D(pt1, centroid_xyz) > 0.05)
{
endpoints.push_back(start);
endpoints.push_back(end);
fingerDefects.push_back(farPt);
}
}
}
// Cluster fingertip locations
endpoints = Hand::clusterConvexHull(endpoints, Hand::CLUSTER_THRESHOLD);
for (auto i = 0; i < endpoints.size(); i++)
{
auto endpoint = endpoints[i];
cv::Point closestDefect;
auto minDefectDistance = 1 << 29;
for (auto j = 0; j < fingerDefects.size(); j++)
{
if (cv::norm(endpoint - fingerDefects[j]) < minDefectDistance)
{
minDefectDistance = cv::norm(endpoint - fingerDefects[j]);
closestDefect = fingerDefects[j];
}
}
auto endPoint_xyz = Util::averageAroundPoint(xyzMap, cv::Point2i(endpoint.x / 4, endpoint.y / 4), 10);
auto closestDefect_xyz = Util::averageAroundPoint(xyzMap, cv::Point2i(closestDefect.x / 4, closestDefect.y / 4), 10);
auto finger_length = Util::euclidianDistance3D(endPoint_xyz, closestDefect_xyz);
if (finger_length < 0.08 && finger_length > 0.025 && endpoint.y < closestDefect.y)
{
fingers_xyz.push_back(endPoint_xyz);
fingers_ij.push_back(cv::Point2i(endpoint.x, endpoint.y)); // SCALING
defects_xyz.push_back(Util::averageAroundPoint(xyzMap, cv::Point2i(closestDefect.x / 4, closestDefect.y / 4), 5));
defects_ij.push_back(cv::Point2i(closestDefect.x, closestDefect.y)); // SCALING
}
}
if (static_cast<float>(cv::countNonZero(channel[2])) / (xyzMap.rows*xyzMap.cols) > 0.3)
{
return;
}
// If there is one or less visible fingers
if (fingers_xyz.size() <= 1)
{
fingers_xyz.clear();
fingers_ij.clear();
auto indexFinger = Util::averageAroundPoint(xyzMap, cv::Point2i(index.x / 4, index.y / 4), 10);
fingers_xyz.push_back(indexFinger);
fingers_ij.push_back(cv::Point2i(index.x, index.y)); // SCALING
auto angle = Util::TriangleAngleCalculation(index_left.x, index_left.y, index.x, index.y, index_right.x, index_right.y);
if (defects_ij.size() != 0)
{
for (auto i = 0; i < fingers_xyz.size(); i++)
{
cv::circle(img, fingers_ij[i], 5, cv::Scalar(0, 0, 255), 3);
cv::line(img, defects_ij[i], fingers_ij[i], cv::Scalar(255, 0, 255), 2);
cv::circle(img, defects_ij[i], 5, cv::Scalar(0, 255, 255), 2);
cv::line(img, defects_ij[i], centroid_ij, cv::Scalar(255, 0, 255), 2);
}
}
else if (angle > ANGLE_THRESHHOLD)
{
cv::circle(img, fingers_ij[0], 5, cv::Scalar(0, 0, 255), 3);
cv::line(img, fingers_ij[0], centroid_ij, cv::Scalar(255, 0, 255), 2);
}
}
else {
for (auto i = 0; i < fingers_xyz.size(); i++)
{
cv::circle(img, fingers_ij[i], 5, cv::Scalar(0, 0, 255), 3);
cv::line(img, defects_ij[i], fingers_ij[i], cv::Scalar(255, 0, 255), 2);
cv::circle(img, defects_ij[i], 3, cv::Scalar(0, 255, 255), 2);
cv::line(img, defects_ij[i], centroid_ij, cv::Scalar(255, 0, 255), 2);
}
}
if (camera_name == "sr300")
{
cv::Mat img_dst;
cv::resize(img, img_dst, cv::Size(640, 489), 0, 0, cv::INTER_AREA);
cv::namedWindow("Contours", CV_WINDOW_AUTOSIZE);
cv::imshow("Contours", img_dst);
}
else
{
cv::namedWindow("Contours", CV_WINDOW_AUTOSIZE);
cv::imshow("Contours", img);
}
}
This is what I get:
additionally, I read in a book this is a good idea to use the following formula for epsilon but doesn't work well at all
cv::approxPolyDP(cv::Mat(contours[i]), contours_poly[i], 0.01*cv::arcLength(cv::Mat(contours[i]), true), true);

opencv binarize images with text

I need to binarize images with text.. It works very well but in some cases the output is empty (white image)
code
/*
* Compile
* # g++ txtbin.cpp -o txtbin `pkg-config opencv --cflags --libs`
*
* Run
* # ./txtbin input.jpg output.png
*/
#include "string"
#include "fstream"
#include "/usr/include/opencv2/opencv.hpp"
#include "/usr/include/boost/tuple/tuple.hpp"
using namespace std;
using namespace cv;
using namespace boost;
void CalcBlockMeanVariance(Mat& Img, Mat& Res, float blockSide=21, float contrast=0.01){
/*
* blockSide: set greater for larger fonts in image
* contrast: set smaller for lower contrast image
*/
Mat I;
Img.convertTo(I, CV_32FC1);
Res = Mat::zeros(Img.rows / blockSide, Img.cols / blockSide, CV_32FC1);
Mat inpaintmask;
Mat patch;
Mat smallImg;
Scalar m, s;
for(int i = 0; i < Img.rows - blockSide; i += blockSide){
for(int j = 0; j < Img.cols - blockSide; j += blockSide){
patch = I(Range(i, i + blockSide + 1), Range(j, j + blockSide + 1));
meanStdDev(patch, m, s);
if(s[0] > contrast){
Res.at<float>(i / blockSide, j / blockSide) = m[0];
}
else{
Res.at<float>(i / blockSide, j / blockSide) = 0;
}
}
}
resize(I, smallImg, Res.size());
threshold(Res, inpaintmask, 0.02, 1.0, THRESH_BINARY);
Mat inpainted;
smallImg.convertTo(smallImg, CV_8UC1, 255);
inpaintmask.convertTo(inpaintmask, CV_8UC1);
inpaint(smallImg, inpaintmask, inpainted, 5, INPAINT_TELEA);
resize(inpainted, Res, Img.size());
Res.convertTo(Res, CV_32FC1, 1.0 / 255.0);
}
tuple<int, int, int, int> detect_text_box(string input, Mat& res, bool draw_contours=false){
Mat large = imread(input);
bool test_output = false;
int
top = large.rows,
bottom = 0,
left = large.cols,
right = 0;
int
rect_bottom,
rect_right;
Mat rgb;
// downsample and use it for processing
pyrDown(large, rgb);
Mat small;
cvtColor(rgb, small, CV_BGR2GRAY);
// morphological gradient
Mat grad;
Mat morphKernel = getStructuringElement(MORPH_ELLIPSE, Size(3, 3));
morphologyEx(small, grad, MORPH_GRADIENT, morphKernel);
// binarize
Mat bw;
threshold(grad, bw, 0.0, 255.0, THRESH_BINARY | THRESH_OTSU);
// connect horizontally oriented regions
Mat connected;
morphKernel = getStructuringElement(MORPH_RECT, Size(9, 1));
morphologyEx(bw, connected, MORPH_CLOSE, morphKernel);
// find contours
Mat mask = Mat::zeros(bw.size(), CV_8UC1);
vector<vector<Point> > contours;
vector<Vec4i> hierarchy;
findContours(connected, contours, hierarchy, CV_RETR_CCOMP, CV_CHAIN_APPROX_SIMPLE, Point(0, 0));
// filter contours
for(int idx = 0; idx >= 0; idx = hierarchy[idx][0]){
Rect rect = boundingRect(contours[idx]);
Mat maskROI(mask, rect);
maskROI = Scalar(0, 0, 0);
// fill the contour
drawContours(mask, contours, idx, Scalar(255, 255, 255), CV_FILLED);
// ratio of non-zero pixels in the filled region
double r = (double)countNonZero(maskROI) / (rect.width * rect.height);
// assume at least 45% of the area is filled if it contains text
if (r > 0.45 &&
(rect.height > 8 && rect.width > 8) // constraints on region size
// these two conditions alone are not very robust. better to use something
//like the number of significant peaks in a horizontal projection as a third condition
){
if(draw_contours){
rectangle(res, Rect(rect.x * 2, rect.y * 2, rect.width * 2, rect.height * 2), Scalar(0, 255, 0), 2);
}
if(test_output){
rectangle(rgb, rect, Scalar(0, 255, 0), 2);
}
if(rect.y < top){
top = rect.y;
}
rect_bottom = rect.y + rect.height;
if(rect_bottom > bottom){
bottom = rect_bottom;
}
if(rect.x < left){
left = rect.x;
}
rect_right = rect.x + rect.width;
if(rect_right > right){
right = rect_right;
}
}
}
if(draw_contours){
rectangle(res, Point(left * 2, top * 2), Point(right * 2, bottom * 2), Scalar(0, 0, 255), 2);
}
if(test_output){
rectangle(rgb, Point(left, top), Point(right, bottom), Scalar(0, 0, 255), 2);
imwrite(string("test_text_contours.jpg"), rgb);
}
return make_tuple(left * 2, top * 2, (right - left) * 2, (bottom - top) * 2);
}
int main(int argc, char* argv[]){
string input;
string output = "output.png";
int
width = 0,
height = 0;
bool
crop = false,
draw = false;
float margin = 0;
// Return error if arguments are missing
if(argc < 3){
cerr << "\nUsage: txtbin input [options] output\n\n"
"Options:\n"
"\t-w <number> -- set max width (keeps aspect ratio)\n"
"\t-h <number> -- set max height (keeps aspect ratio)\n"
"\t-c -- crop text content contour\n"
"\t-m <number> -- add margins (number in %)\n"
"\t-d -- draw text content contours (debugging)\n" << endl;
return 1;
}
// Parse arguments
for(int i = 1; i < argc; i++){
if(i == 1){
input = string(argv[i]);
// Return error if input file is invalid
ifstream stream(input.c_str());
if(!stream.good()){
cerr << "Error: Input file is invalid!" << endl;
return 1;
}
}
else if(string(argv[i]) == "-w"){
width = atoi(argv[++i]);
}
else if(string(argv[i]) == "-h"){
height = atoi(argv[++i]);
}
else if(string(argv[i]) == "-c"){
crop = true;
}
else if(string(argv[i]) == "-m"){
margin = atoi(argv[++i]);
}
else if(string(argv[i]) == "-d"){
draw = true;
}
else if(i == argc - 1){
output = string(argv[i]);
}
}
Mat Img = imread(input, CV_LOAD_IMAGE_GRAYSCALE);
Mat res;
Img.convertTo(Img, CV_32FC1, 1.0 / 255.0);
CalcBlockMeanVariance(Img, res);
res = 1.0 - res;
res = Img + res;
threshold(res, res, 0.85, 1, THRESH_BINARY);
int
txt_x,
txt_y,
txt_width,
txt_height;
if(crop || draw){
tie(txt_x, txt_y, txt_width, txt_height) = detect_text_box(input, res, draw);
}
if(crop){
//res = res(Rect(txt_x, txt_y, txt_width, txt_height)).clone();
res = res(Rect(txt_x, txt_y, txt_width, txt_height));
}
if(margin){
int border = res.cols * margin / 100;
copyMakeBorder(res, res, border, border, border, border, BORDER_CONSTANT, Scalar(255, 255, 255));
}
float
width_input = res.cols,
height_input = res.rows;
bool resized = false;
// Downscale image
if(width > 0 && width_input > width){
float scale = width_input / width;
width_input /= scale;
height_input /= scale;
resized = true;
}
if(height > 0 && height_input > height){
float scale = height_input / height;
width_input /= scale;
height_input /= scale;
resized = true;
}
if(resized){
resize(res, res, Size(round(width_input), round(height_input)));
}
imwrite(output, res * 255);
return 0;
}
Ok :)
Set blockSide smaller (7 for instance) it will give you result image as shown below. It depends on font size, smaller fonts need smaller block size, else text will be filtered out and you get empty image.
#include <iostream>
#include <vector>
#include <stdio.h>
#include <stdarg.h>
#include "/usr/include/opencv2/opencv.hpp"
#include "fstream"
#include "iostream"
using namespace std;
using namespace cv;
void CalcBlockMeanVariance(Mat& Img,Mat& Res,float blockSide=9) // blockSide - the parameter (set greater for larger font on image)
{
Mat I;
Img.convertTo(I,CV_32FC1);
Res=Mat::zeros(Img.rows/blockSide,Img.cols/blockSide,CV_32FC1);
Mat inpaintmask;
Mat patch;
Mat smallImg;
Scalar m,s;
for(int i=0;i<Img.rows-blockSide;i+=blockSide)
{
for (int j=0;j<Img.cols-blockSide;j+=blockSide)
{
patch=I(Range(i,i+blockSide+1),Range(j,j+blockSide+1));
cv::meanStdDev(patch,m,s);
if(s[0]>0.01) // Thresholding parameter (set smaller for lower contrast image)
{
Res.at<float>(i/blockSide,j/blockSide)=m[0];
}else
{
Res.at<float>(i/blockSide,j/blockSide)=0;
}
}
}
cv::resize(I,smallImg,Res.size());
cv::threshold(Res,inpaintmask,0.02,1.0,cv::THRESH_BINARY);
Mat inpainted;
smallImg.convertTo(smallImg,CV_8UC1,255);
inpaintmask.convertTo(inpaintmask,CV_8UC1);
inpaint(smallImg, inpaintmask, inpainted, 5, INPAINT_TELEA);
cv::resize(inpainted,Res,Img.size());
Res.convertTo(Res,CV_32FC1,1.0/255.0);
}
int main( int argc, char** argv )
{
namedWindow("Img");
namedWindow("Edges");
//Mat Img=imread("D:\\ImagesForTest\\BookPage.JPG",0);
Mat Img=imread("test2.jpg",0);
Mat res;
Img.convertTo(Img,CV_32FC1,1.0/255.0);
CalcBlockMeanVariance(Img,res);
res=1.0-res;
res=Img+res;
imshow("Img",Img);
cv::threshold(res,res,0.85,1,cv::THRESH_BINARY);
cv::resize(res,res,cv::Size(res.cols/2,res.rows/2));
imwrite("result.jpg",res*255);
imshow("Edges",res);
waitKey(0);
return 0;
}

blob tracking with kalman filter in opencv

Well, I am trying to create a small example of blob tracking using the kalman filter. I am using openCV in order to accomplish this task, however it does not seem to work as it supposed to, since when I am hiding the object which tracking the output with, the kalman filter does not try to estimate where the object should be. I am attaching the code below, I hope that someone can give a hint on what I am doing wrong.
Thanks in advance.... :-)
#include <iostream>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/core/core.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include <opencv2/video/tracking.hpp>
using namespace std;
using namespace cv;
#define drawCross( img, center, color, d )\
line(img, Point(center.x - d, center.y - d), Point(center.x + d, center.y + d), color, 2, CV_AA, 0);\
line(img, Point(center.x + d, center.y - d), Point(center.x - d, center.y + d), color, 2, CV_AA, 0 )\
int main()
{
Mat frame, thresh_frame;
vector<Mat> channels;
VideoCapture capture;
vector<Vec4i> hierarchy;
vector<vector<Point> > contours;
capture.open("capture.avi");
if(!capture.isOpened())
cerr << "Problem opening video source" << endl;
KalmanFilter KF(4, 2, 0);
Mat_<float> state(4, 1);
Mat_<float> processNoise(4, 1, CV_32F);
Mat_<float> measurement(2,1); measurement.setTo(Scalar(0));
KF.statePre.at<float>(0) = 0;
KF.statePre.at<float>(1) = 0;
KF.statePre.at<float>(2) = 0;
KF.statePre.at<float>(3) = 0;
KF.transitionMatrix = *(Mat_<float>(4, 4) << 1,0,1,0, 0,1,0,1, 0,0,1,0, 0,0,0,1); // Including velocity
KF.processNoiseCov = *(cv::Mat_<float>(4,4) << 0.2,0,0.2,0, 0,0.2,0,0.2, 0,0,0.3,0, 0,0,0,0.3);
setIdentity(KF.measurementMatrix);
setIdentity(KF.processNoiseCov, Scalar::all(1e-4));
setIdentity(KF.measurementNoiseCov, Scalar::all(1e-1));
setIdentity(KF.errorCovPost, Scalar::all(.1));
while((char)waitKey(1) != 'q' && capture.grab())
{
capture.retrieve(frame);
split(frame, channels);
add(channels[0], channels[1], channels[1]);
subtract(channels[2], channels[1], channels[2]);
threshold(channels[2], thresh_frame, 50, 255, CV_THRESH_BINARY);
medianBlur(thresh_frame, thresh_frame, 5);
findContours(thresh_frame, contours, hierarchy, CV_RETR_EXTERNAL, CV_CHAIN_APPROX_SIMPLE, Point(0, 0));
Mat drawing = Mat::zeros(thresh_frame.size(), CV_8UC1);
for(size_t i = 0; i < contours.size(); i++)
{
// cout << contourArea(contours[i]) << endl;
if(contourArea(contours[i]) > 500)
drawContours(drawing, contours, i, Scalar::all(255), CV_FILLED, 8, vector<Vec4i>(), 0, Point());
}
thresh_frame = drawing;
findContours(thresh_frame, contours, hierarchy, CV_RETR_EXTERNAL, CV_CHAIN_APPROX_SIMPLE, Point(0, 0));
drawing = Mat::zeros(thresh_frame.size(), CV_8UC1);
for(size_t i = 0; i < contours.size(); i++)
{
// cout << contourArea(contours[i]) << endl;
if(contourArea(contours[i]) > 500)
drawContours(drawing, contours, i, Scalar::all(255), CV_FILLED, 8, vector<Vec4i>(), 0, Point());
}
thresh_frame = drawing;
// Get the moments
vector<Moments> mu(contours.size() );
for( size_t i = 0; i < contours.size(); i++ )
{ mu[i] = moments( contours[i], false ); }
// Get the mass centers:
vector<Point2f> mc( contours.size() );
for( size_t i = 0; i < contours.size(); i++ )
{ mc[i] = Point2f( mu[i].m10/mu[i].m00 , mu[i].m01/mu[i].m00 ); }
Mat prediction = KF.predict();
Point predictPt(prediction.at<float>(0),prediction.at<float>(1));
for(size_t i = 0; i < mc.size(); i++)
{
drawCross(frame, mc[i], Scalar(255, 0, 0), 5);
measurement(0) = mc[i].x;
measurement(1) = mc[i].y;
}
Point measPt(measurement(0),measurement(1));
Mat estimated = KF.correct(measurement);
Point statePt(estimated.at<float>(0),estimated.at<float>(1));
drawCross(frame, statePt, Scalar(255, 255, 255), 5);
vector<vector<Point> > contours_poly( contours.size() );
vector<Rect> boundRect( contours.size() );
for( size_t i = 0; i < contours.size(); i++ )
{ approxPolyDP( Mat(contours[i]), contours_poly[i], 3, true );
boundRect[i] = boundingRect( Mat(contours_poly[i]) );
}
for( size_t i = 0; i < contours.size(); i++ )
{
rectangle( frame, boundRect[i].tl(), boundRect[i].br(), Scalar(0, 255, 0), 2, 8, 0 );
}
imshow("Video", frame);
imshow("Red", channels[2]);
imshow("Binary", thresh_frame);
}
return 0;
}