My code opens image with road signs, detects them, rescale to specified size and then puts them into matrix.
vector<vector<Point> > contours;
vector<Vec4i> hierarchy; findContours(maski, contours, hierarchy, CV_RETR_EXTERNAL, CV_CHAIN_APPROX_SIMPLE);
Mat output1= Mat::zeros(cropImg.rows,cropImg.cols, CV_8UC3);
for(int i = 0; i < contours.size(); i++)
{
drawContours(output1 , contours, i, Scalar(0,0,255), 1, 8, hierarchy );
imshow("kontury z findContours", output1);
}
vector<Rect> boundRect( contours.size() );
Mat drawing1 = Mat::zeros(cropImg.size(), CV_8UC3 );
Mat image_roi = Mat::zeros(Size(1000,1000), CV_8UC3 );
Mat przeskalowane1;
for( int i = 0; i < contours.size(); i++ )
{
double obwod = arcLength(Mat(contours[i]), true);
if(obwod>150)
{
boundRect[i] = boundingRect(Mat(contours[i]));
cout<<"Obwod: "<<obwod<<" Wymiar: "<<boundRect[i].width<<"x"<<boundRect[i].height<<endl;
if(boundRect[i].height > 50 && boundRect[i].width > 50)
{
drawContours( drawing1, contours, i, Scalar(3, 200, 2), CV_FILLED, 8, hierarchy, 0, Point() );
imshow("kontury brane pod uwage przed skalowaniem", drawing1);
Rect mask(boundRect[i].x, boundRect[i].y, boundRect[i].width, boundRect[i].height);
//cout << "#" << i << " rectangle x:" << mask.x << " y:" << mask.y << " " << mask.width << "x" << mask.height << endl;
Mat image_roi = drawing1(mask);
double wys = boundRect[i].height;
double szer = boundRect[i].width;
double skala1 = wys/128;
double y = wys/skala1;
double x = szer/skala1;
resize(image_roi, image_roi, Size(x,y));
przeskalowane1.push_back(image_roi);
} // ERROR in this line
}
}
if(przeskalowane1.cols > 0)
{
cout<<"Przeskalowane: "<<przeskalowane1.cols<<"x"<<przeskalowane1.rows<<endl;
imshow("Przeskalowane", przeskalowane1);
cvMoveWindow("Przeskalowane", 1128, 0);
cvtColor(przeskalowane1, przeskalowane1, CV_BGR2GRAY);
}
It all works properly when there is only one road sign found or signs found in the image have very similar dimensions to specified. If sizes of found signs are different, then I get following error:
Error: Sizes of input arguments do not match <> in unknown function, file......\modules\core\src\matrix.cpp, line 598"
It is very important for me to have these signs in matrix.
The help page for the cv::Mat::push_back method says:
The methods add one or more elements to the bottom of the matrix. They
emulate the corresponding method of the STL vector class. When elem is
Mat , its type and the number of columns must be the same as in the
container matrix.
So, in order to add multiple images to przeskalowane1, you need to rescale them to the same width (not height).
Mat image_roi = drawing1(mask);
double wys = boundRect[i].height;
double szer = boundRect[i].width;
double x = 128;
double skala1 = szer/x;
double y = wys/skala1;
resize(image_roi, image_roi, Size(x, y));
Related
I am currently playing around idea of calculating an of average HSV for points in a contour. I did some research and came across the split function which allows for a mat of an image to be broken into it's channels, However the contour datatype is a vector of points. Here is an example of code.
findcontours(detected_edges,contours,CV_RETR_LIST,CV_CHAIN_APPROX_SIMPLE);
vector<vector<Point>> ContourHsvChannels(3);
split(contours,ContourHsvChannels);
Basically the goal is to split each point of a contour into its HSV channels so I can perform operations on them. Any guidance would be appreciated.
You can simply draw the contours onto a blank image the same size as your original image to create a mask, and then use that to mask your image (in HSV or whatever colorspace you want). The mean() function takes in a mask parameter so that you only get the mean of the values highlighted by the mask.
If you also want the standard deviation you can use the meanStdDev() function, it also accepts a mask.
Here's an example in Python:
import cv2
import numpy as np
# read image, ensure binary
img = cv2.imread('fg.png', 0)
img[img>0] = 255
# find contours in the image
contours = cv2.findContours(img, cv2.RETR_LIST, cv2.CHAIN_APPROX_NONE)[1]
# create an array of blank images to draw contours on
n_contours = len(contours)
contour_imgs = [np.zeros_like(img) for i in range(n_contours)]
# draw each contour on a new image
for i in range(n_contours):
cv2.drawContours(contour_imgs[i], contours, i, 255)
# color image of where the HSV values are coming from
color_img = cv2.imread('image.png')
hsv = cv2.cvtColor(color_img, cv2.COLOR_BGR2HSV)
# find the means and standard deviations of the HSV values for each contour
means = []
stddevs = []
for cnt_img in contour_imgs:
mean, stddev = cv2.meanStdDev(hsv, mask=cnt_img)
means.append(mean)
stddevs.append(stddev)
print('First mean:')
print(means[0])
print('First stddev:')
print(stddevs[0])
First mean:
[[ 146.3908046 ]
[ 51.2183908 ]
[ 202.95402299]]
First stddev:
[[ 7.92835204]
[ 11.78682811]
[ 9.61549043]]
There's three values; one for each channel.
The other option is to just look up all the values; a contour is an array of points, so you can index the image with those points for each contour in your contour array and store them in individual arrays, and then find the meanStdDev() or mean() over those (and not bother with the mask). For e.g. (again in Python, sorry about that):
# color image of where the HSV values are coming from
color_img = cv2.imread('image.png')
hsv = cv2.cvtColor(color_img, cv2.COLOR_BGR2HSV)
# read image, ensure binary
img = cv2.imread('fg.png', 0)
img[img>0] = 255
# find contours in the image
contours = cv2.findContours(img, cv2.RETR_LIST, cv2.CHAIN_APPROX_NONE)[1]
means = []
stddevs = []
for contour in contours:
contour_colors = []
n_points = len(contour)
for point in contour:
x, y = point[0]
contour_colors.append(hsv[y, x])
contour_colors = np.array(contour_colors).reshape(1, n_points, 3)
mean, stddev = cv2.meanStdDev(contour_colors)
means.append(mean)
stddevs.append(stddev)
print('First mean:')
print(means[0])
print('First stddev:')
print(stddevs[0])
First mean:
[[ 146.3908046 ]
[ 51.2183908 ]
[ 202.95402299]]
First stddev:
[[ 7.92835204]
[ 11.78682811]
[ 9.61549043]]
So this gives the same values. In Python I just simply created blank lists for the means and standard deviations and appended to them. In C++ you can create a std::vector<cv::Vec3b> (assuming uint8 image, otherwise Vec3f or whatever is appropriate) for each. Then inside the loop I create another blank list to hold the colors for each contour; again this would be a std::vector<cv::Vec3b>, and then run the meanStdDev() on that vector in each loop, and append the value to the means and standard deviations vectors. You don't have to append, you can easily grab the number of contours and the number of points in each contour and preallocate for speed, and then just index into those vectors instead of appending.
In Python there's virtually no speed difference between either method. Of course there's better memory efficiency in the second example; instead of storing a whole blank Mat we just store a few of the values. However the backend OpenCV methods work really quickly for masking operations, so you'll have to test the speed difference yourself in C++ and see which way is better. As the number of contours increases I imagine the benefits of the second method increases. If you do time both approaches, please let us know your results!
Here is the solution written in c++
#include <opencv2\opencv.hpp>
#include <iostream>
#include <vector>
#include <cmath>
using namespace cv;
using namespace std;
int main(int argc, char** argv) {
// Mat Declarations
// Mat img = imread("white.jpg");
// Mat src = imread("Rainbro.png");
Mat src = imread("multi.jpg");
// Mat src = imread("DarkRed.png");
Mat Hist;
Mat HSV;
Mat Edges;
Mat Grey;
vector<vector<Vec3b>> hueMEAN;
vector<vector<Point>> contours;
// Variables
int edgeThreshold = 1;
int const max_lowThreshold = 100;
int ratio = 3;
int kernel_size = 3;
int lowThreshold = 0;
// Windows
namedWindow("img", WINDOW_NORMAL);
namedWindow("HSV", WINDOW_AUTOSIZE);
namedWindow("Edges", WINDOW_AUTOSIZE);
namedWindow("contours", WINDOW_AUTOSIZE);
// Color Transforms
cvtColor(src, HSV, CV_BGR2HSV);
cvtColor(src, Grey, CV_BGR2GRAY);
// Perform Hist Equalization to help equalize Red hues so they stand out for
// better Edge Detection
equalizeHist(Grey, Grey);
// Image Transforms
blur(Grey, Edges, Size(3, 3));
Canny(Edges, Edges, max_lowThreshold, lowThreshold * ratio, kernel_size);
findContours(Edges, contours, CV_RETR_LIST, CV_CHAIN_APPROX_NONE);
//Rainbro MAT
//Mat drawing = Mat::zeros(432, 700, CV_8UC1);
//Multi MAT
Mat drawing = Mat::zeros(630, 1200, CV_8UC1);
//Red variation Mat
//Mat drawing = Mat::zeros(600, 900, CV_8UC1);
vector <vector<Point>> ContourPoints;
/* This code for loops through all contours and assigns the value of the y coordinate as a parameter
for the row pointer in the HSV mat. The value vec3b pointer pointing to the pixel in the mat is accessed
and stored for any Hue value that is between 0-10 and 165-179 as Red only contours.*/
for (int i = 0; i < contours.size(); i++) {
vector<Vec3b> vf;
vector<Point> points;
bool isContourRed = false;
for (int j = 0; j < contours[i].size(); j++) {
//Row Y-Coordinate of Mat from Y-Coordinate of Contour
int MatRow = int(contours[i][j].y);
//Row X-Coordinate of Mat from X-Coordinate of Contour
int MatCol = int(contours[i][j].x);
Vec3b *HsvRow = HSV.ptr <Vec3b>(MatRow);
int h = int(HsvRow[int(MatCol)][0]);
int s = int(HsvRow[int(MatCol)][1]);
int v = int(HsvRow[int(MatCol)][2]);
cout << "Coordinate: ";
cout << contours[i][j].x;
cout << ",";
cout << contours[i][j].y << endl;
cout << "Hue: " << h << endl;
// Get contours that are only in the red spectrum Hue 0-10, 165-179
if ((h <= 10 || h >= 165 && h <= 180) && ((s > 0) && (v > 0))) {
cout << "Coordinate: ";
cout << contours[i][j].x;
cout << ",";
cout << contours[i][j].y << endl;
cout << "Hue: " << h << endl;
vf.push_back(Vec3b(h, s, v));
points.push_back(contours[i][j]);
isContourRed = true;
}
}
if (isContourRed == true) {
hueMEAN.push_back(vf);
ContourPoints.push_back(points);
}
}
drawContours(drawing, ContourPoints, -1, Scalar(255, 255, 255), 2, 8);
// Calculate Mean and STD for each Contour
cout << "contour Means & STD of Vec3b:" << endl;
for (int i = 0; i < hueMEAN.size(); i++) {
Scalar meanTemp = mean(hueMEAN.at(i));
Scalar sdTemp;
cout << i << ": " << endl;
cout << meanTemp << endl;
cout << " " << endl;
meanStdDev(hueMEAN.at(i), meanTemp, sdTemp);
cout << sdTemp << endl;
cout << " " << endl;
}
cout << "Actual Contours: " << contours.size() << endl;
cout << "# Contours: " << hueMEAN.size() << endl;
imshow("img", src);
imshow("HSV", HSV);
imshow("Edges", Edges);
imshow("contours", drawing);
waitKey(0);
return 0;
}
I have implemented Neural network using OpenCV ANN Library. I am newbie in this field and I learn everything about it online (Mostly StackOverflow).
I am using this ANN for detection of number plate. I did segmentation part using OpenCV image processing library and it is working good. It performs character segmentation and gives it to the NN part of the project. NN is going to recognize the number plate.
I have sample images of 20x30, therefore I have 600 neurons in input layer. As there are 36 possibilities (0-9,A-Z) I have 36 output neurons. I kept 100 neurons in hidden layer. The predict function of OpenCV is giving me the same output for every segmented image. That output is also showing some large negative(< -1). I have used cv::ml::ANN_MLP::SIGMOID_SYM as an activation function.
Please don't mind as there is lot of code wrongly commented (I am doing trial and error).
I need to find out what is the output of predict function. Thank you for your help.
#include <opencv2/opencv.hpp>
int inputLayerSize = 1;
int outputLayerSize = 1;
int numSamples = 2;
Mat layers = Mat(3, 1, CV_32S);
layers.row(0) =Scalar(600) ;
layers.row(1) = Scalar(20);
layers.row(2) = Scalar(36);
vector<int> layerSizes = { 600,100,36 };
Ptr<ml::ANN_MLP> nnPtr = ml::ANN_MLP::create();
vector <int> n;
//nnPtr->setLayerSizes(3);
nnPtr->setLayerSizes(layers);
nnPtr->setTrainMethod(ml::ANN_MLP::BACKPROP);
nnPtr->setTermCriteria(TermCriteria(cv::TermCriteria::COUNT | cv::TermCriteria::EPS, 1000, 0.00001f));
nnPtr->setActivationFunction(cv::ml::ANN_MLP::SIGMOID_SYM, 1, 1);
nnPtr->setBackpropWeightScale(0.5f);
nnPtr->setBackpropMomentumScale(0.5f);
/*CvANN_MLP_TrainParams params = CvANN_MLP_TrainParams(
// terminate the training after either 1000
// iterations or a very small change in the
// network wieghts below the specified value
cvTermCriteria(CV_TERMCRIT_ITER + CV_TERMCRIT_EPS, 1000, 0.000001),
// use backpropogation for training
CvANN_MLP_TrainParams::BACKPROP,
// co-efficents for backpropogation training
// (refer to manual)
0.1,
0.1);*/
/* Mat samples(Size(inputLayerSize, numSamples), CV_32F);
samples.at<float>(Point(0, 0)) = 0.1f;
samples.at<float>(Point(0, 1)) = 0.2f;
Mat responses(Size(outputLayerSize, numSamples), CV_32F);
responses.at<float>(Point(0, 0)) = 0.2f;
responses.at<float>(Point(0, 1)) = 0.4f;
*/
//reading chaos image
// we will read the classification numbers into this variable as though it is a vector
// close the traning images file
/*vector<int> layerInfo;
layerInfo=nnPtr->get;
for (int i = 0; i < layerInfo.size(); i++) {
cout << "size of 0" <<layerInfo[i] << endl;
}*/
cv::imshow("chaos", matTrainingImagesAsFlattenedFloats);
// cout <<abc << endl;
matTrainingImagesAsFlattenedFloats.convertTo(matTrainingImagesAsFlattenedFloats, CV_32F);
//matClassificationInts.reshape(1, 496);
matClassificationInts.convertTo(matClassificationInts, CV_32F);
matSamples.convertTo(matSamples, CV_32F);
std::cout << matClassificationInts.rows << " " << matClassificationInts.cols << " ";
std::cout << matTrainingImagesAsFlattenedFloats.rows << " " << matTrainingImagesAsFlattenedFloats.cols << " ";
std::cout << matSamples.rows << " " << matSamples.cols;
imshow("Samples", matSamples);
imshow("chaos", matTrainingImagesAsFlattenedFloats);
Ptr<ml::TrainData> trainData = ml::TrainData::create(matTrainingImagesAsFlattenedFloats, ml::SampleTypes::ROW_SAMPLE, matSamples);
nnPtr->train(trainData);
bool m = nnPtr->isTrained();
if (m)
std::cout << "training complete\n\n";
// cv::Mat matCurrentChar = Mat(cv::Size(matTrainingImagesAsFlattenedFloats.cols, matTrainingImagesAsFlattenedFloats.rows), CV_32F);
// cout << "samples:\n" << samples << endl;
//cout << "\nresponses:\n" << responses << endl;
/* if (!nnPtr->train(trainData))
return 1;*/
/* cout << "\nweights[0]:\n" << nnPtr->getWeights(0) << endl;
cout << "\nweights[1]:\n" << nnPtr->getWeights(1) << endl;
cout << "\nweights[2]:\n" << nnPtr->getWeights(2) << endl;
cout << "\nweights[3]:\n" << nnPtr->getWeights(3) << endl;*/
//predicting
std::vector <cv::String> filename;
cv::String folder = "./plate/";
cv::glob(folder, filename);
if (filename.empty()) { // if unable to open image
std::cout << "error: image not read from file\n\n"; // show error message on command line
return(0); // and exit program
}
String strFinalString;
for (int i = 0; i < filename.size(); i++) {
cv::Mat matTestingNumbers = cv::imread(filename[i]);
cv::Mat matGrayscale; //
cv::Mat matBlurred; // declare more image variables
cv::Mat matThresh; //
cv::Mat matThreshCopy;
cv::Mat matCanny;
//
cv::cvtColor(matTestingNumbers, matGrayscale, CV_BGR2GRAY); // convert to grayscale
matThresh = cv::Mat(cv::Size(matGrayscale.cols, matGrayscale.rows), CV_8UC1);
for (int i = 0; i < matGrayscale.cols; i++) {
for (int j = 0; j < matGrayscale.rows; j++) {
if (matGrayscale.at<uchar>(j, i) <= 130) {
matThresh.at<uchar>(j, i) = 255;
}
else {
matThresh.at<uchar>(j, i) = 0;
}
}
}
// blur
cv::GaussianBlur(matThresh, // input image
matBlurred, // output image
cv::Size(5, 5), // smoothing window width and height in pixels
0); // sigma value, determines how much the image will be blurred, zero makes function choose the sigma value
// filter image from grayscale to black and white
/* cv::adaptiveThreshold(matBlurred, // input image
matThresh, // output image
255, // make pixels that pass the threshold full white
cv::ADAPTIVE_THRESH_GAUSSIAN_C, // use gaussian rather than mean, seems to give better results
cv::THRESH_BINARY_INV, // invert so foreground will be white, background will be black
11, // size of a pixel neighborhood used to calculate threshold value
2); */ // constant subtracted from the mean or weighted mean
// cv::imshow("thresh" + std::to_string(i), matThresh);
matThreshCopy = matThresh.clone();
std::vector<std::vector<cv::Point> > ptContours; // declare a vector for the contours
std::vector<cv::Vec4i> v4iHierarchy;// make a copy of the thresh image, this in necessary b/c findContours modifies the image
cv::Canny(matBlurred, matCanny, 20, 40, 3);
/*std::vector<std::vector<cv::Point> > ptContours; // declare a vector for the contours
std::vector<cv::Vec4i> v4iHierarchy; // declare a vector for the hierarchy (we won't use this in this program but this may be helpful for reference)
cv::findContours(matThreshCopy, // input image, make sure to use a copy since the function will modify this image in the course of finding contours
ptContours, // output contours
v4iHierarchy, // output hierarchy
cv::RETR_EXTERNAL, // retrieve the outermost contours only
cv::CHAIN_APPROX_SIMPLE); // compress horizontal, vertical, and diagonal segments and leave only their end points
/*std::vector<std::vector<cv::Point> > contours_poly(ptContours.size());
std::vector<cv::Rect> boundRect(ptContours.size());
for (int i = 0; i < ptContours.size(); i++)
{
approxPolyDP(cv::Mat(ptContours[i]), contours_poly[i], 3, true);
boundRect[i] = cv::boundingRect(cv::Mat(contours_poly[i]));
}*/
/*for (int i = 0; i < ptContours.size(); i++) { // for each contour
ContourWithData contourWithData; // instantiate a contour with data object
contourWithData.ptContour = ptContours[i]; // assign contour to contour with data
contourWithData.boundingRect = cv::boundingRect(contourWithData.ptContour); // get the bounding rect
contourWithData.fltArea = cv::contourArea(contourWithData.ptContour); // calculate the contour area
allContoursWithData.push_back(contourWithData); // add contour with data object to list of all contours with data
}
for (int i = 0; i < allContoursWithData.size(); i++) { // for all contours
if (allContoursWithData[i].checkIfContourIsValid()) { // check if valid
validContoursWithData.push_back(allContoursWithData[i]); // if so, append to valid contour list
}
}
//sort contours from left to right
std::sort(validContoursWithData.begin(), validContoursWithData.end(), ContourWithData::sortByBoundingRectXPosition);
// std::string strFinalString; // declare final string, this will have the final number sequence by the end of the program
*/
/*for (int i = 0; i < validContoursWithData.size(); i++) { // for each contour
// draw a green rect around the current char
cv::rectangle(matTestingNumbers, // draw rectangle on original image
validContoursWithData[i].boundingRect, // rect to draw
cv::Scalar(0, 255, 0), // green
2); // thickness
cv::Mat matROI = matThresh(validContoursWithData[i].boundingRect); // get ROI image of bounding rect
cv::Mat matROIResized;
cv::resize(matROI, matROIResized, cv::Size(RESIZED_IMAGE_WIDTH, RESIZED_IMAGE_HEIGHT)); // resize image, this will be more consistent for recognition and storage
*/
cv::Mat matROIFloat;
cv::resize(matThresh, matThresh, cv::Size(RESIZED_IMAGE_WIDTH, RESIZED_IMAGE_HEIGHT));
matThresh.convertTo(matROIFloat, CV_32FC1, 1.0 / 255.0); // convert Mat to float, necessary for call to find_nearest
cv::Mat matROIFlattenedFloat = matROIFloat.reshape(1, 1);
cv::Point maxLoc = { 0,0 };
cv::Point minLoc;
cv::Mat output = cv::Mat(cv::Size(36, 1), CV_32F);
vector<float>output2;
// cv::Mat output2 = cv::Mat(cv::Size(36, 1), CV_32F);
nnPtr->predict(matROIFlattenedFloat, output2);
// float max = output.at<float>(0, 0);
int fo = 0;
float m = output2[0];
imshow("predicted input", matROIFlattenedFloat);
// float b = output.at<float>(0, 0);
// cout <<"\n output0,0:"<<b<<endl;
// minMaxLoc(output, 0, 0, &minLoc, &maxLoc, Mat());
// cout << "\noutput:\n" << maxLoc.x << endl;
for (int j = 1; j < 36; j++) {
float value =output2[j];
if (value > m) {
m = value;
fo = j;
}
}
float * p = 0;
p = &m;
cout << "j value in output " << fo << " Max value " << p << endl;
//imshow("output image" + to_string(i), output);
// cout << "\noutput:\n" << minLoc.x << endl;
//float fltCurrentChar = (float)maxLoc.x;
output.release();
m = 0;
fo = 0;
}
// strFinalString = strFinalString + char(int(fltCurrentChar)); // append current char to full string
// cv::imshow("Predict output", output);
/*cv::Point maxLoc = {0,0};
Mat output=Mat (cv::Size(matSamples.cols,matSamples.rows),CV_32F);
nnPtr->predict(matTrainingImagesAsFlattenedFloats, output);
minMaxLoc(output, 0, 0, 0, &maxLoc, 0);
cout << "\noutput:\n" << maxLoc.x << endl;*/
// getchar();
/*for (int i = 0; i < 10;i++) {
for (int j = 0; j < 36; j++) {
if (matCurrentChar.at<float>(i, j) >= 0.6) {
cout << " "<<j<<" ";
}
}
}*/
waitKey(0);
return(0);
}
void gen() {
std::string dir, filepath;
int num, imgArea, minArea;
int pos = 0;
bool f = true;
struct stat filestat;
cv::Mat imgTrainingNumbers;
cv::Mat imgGrayscale;
cv::Mat imgBlurred;
cv::Mat imgThresh;
cv::Mat imgThreshCopy;
cv::Mat matROIResized=cv::Mat (cv::Size(RESIZED_IMAGE_WIDTH,RESIZED_IMAGE_HEIGHT),CV_8UC1);
cv::Mat matROI;
std::vector <cv::String> filename;
std::vector<std::vector<cv::Point> > ptContours;
std::vector<cv::Vec4i> v4iHierarchy;
int count = 0, contoursCount = 0;
matSamples = cv::Mat(cv::Size(36, 496), CV_32FC1);
matTrainingImagesAsFlattenedFloats = cv::Mat(cv::Size(600, 496), CV_32FC1);
for (int j = 0; j <= 35; j++) {
int tmp = j;
cv::String folder = "./Training Data/" + std::to_string(tmp);
cv::glob(folder, filename);
for (int k = 0; k < filename.size(); k++) {
count++;
// If the file is a directory (or is in some way invalid) we'll skip it
// if (stat(filepath.c_str(), &filestat)) continue;
//if (S_ISDIR(filestat.st_mode)) continue;
imgTrainingNumbers = cv::imread(filename[k]);
imgArea = imgTrainingNumbers.cols*imgTrainingNumbers.rows;
// read in training numbers image
minArea = imgArea * 50 / 100;
if (imgTrainingNumbers.empty()) {
std::cout << "error: image not read from file\n\n";
//return(0);
}
cv::cvtColor(imgTrainingNumbers, imgGrayscale, CV_BGR2GRAY);
//cv::equalizeHist(imgGrayscale, imgGrayscale);
imgThresh = cv::Mat(cv::Size(imgGrayscale.cols, imgGrayscale.rows), CV_8UC1);
/*cv::adaptiveThreshold(imgGrayscale,
imgThresh,
255,
cv::ADAPTIVE_THRESH_GAUSSIAN_C,
cv::THRESH_BINARY_INV,
3,
0);
*/
for (int i = 0; i < imgGrayscale.cols; i++) {
for (int j = 0; j < imgGrayscale.rows; j++) {
if (imgGrayscale.at<uchar>(j, i) <= 130) {
imgThresh.at<uchar>(j, i) = 255;
}
else {
imgThresh.at<uchar>(j, i) = 0;
}
}
}
// cv::imshow("imgThresh"+std::to_string(count), imgThresh);
imgThreshCopy = imgThresh.clone();
cv::GaussianBlur(imgThreshCopy,
imgBlurred,
cv::Size(5, 5),
0);
cv::Mat imgCanny;
// cv::Canny(imgBlurred,imgCanny,20,40,3);
cv::findContours(imgBlurred,
ptContours,
v4iHierarchy,
cv::RETR_EXTERNAL,
cv::CHAIN_APPROX_SIMPLE);
for (int i = 0; i < ptContours.size(); i++) {
if (cv::contourArea(ptContours[i]) > MIN_CONTOUR_AREA) {
contoursCount++;
cv::Rect boundingRect = cv::boundingRect(ptContours[i]);
cv::rectangle(imgTrainingNumbers, boundingRect, cv::Scalar(0, 0, 255), 2); // draw red rectangle around each contour as we ask user for input
matROI = imgThreshCopy(boundingRect); // get ROI image of bounding rect
std::string path = "./" + std::to_string(contoursCount) + ".JPG";
cv::imwrite(path, matROI);
// cv::imshow("matROI" + std::to_string(count), matROI);
cv::resize(matROI, matROIResized, cv::Size(RESIZED_IMAGE_WIDTH, RESIZED_IMAGE_HEIGHT)); // resize image, this will be more consistent for recognition and storage
std::cout << filename[k] << " " << contoursCount << "\n";
//cv::imshow("matROI", matROI);
//cv::imshow("matROIResized"+std::to_string(count), matROIResized);
// cv::imshow("imgTrainingNumbers" + std::to_string(contoursCount), imgTrainingNumbers);
int intChar;
if (j<10)
intChar = j + 48;
else {
intChar = j + 55;
}
/*if (intChar == 27) { // if esc key was pressed
return(0); // exit program
}*/
// if (std::find(intValidChars.begin(), intValidChars.end(), intChar) != intValidChars.end()) { // else if the char is in the list of chars we are looking for . . .
// append classification char to integer list of chars
cv::Mat matImageFloat;
matROIResized.convertTo(matImageFloat,CV_32FC1);// now add the training image (some conversion is necessary first) . . .
//matROIResized.convertTo(matImageFloat, CV_32FC1); // convert Mat to float
cv::Mat matImageFlattenedFloat = matImageFloat.reshape(1, 1);
//matTrainingImagesAsFlattenedFloats.push_back(matImageFlattenedFloat);// flatten
try {
//matTrainingImagesAsFlattenedFloats.push_back(matImageFlattenedFloat);
std::cout << matTrainingImagesAsFlattenedFloats.rows << " " << matTrainingImagesAsFlattenedFloats.cols;
//unsigned char* re;
int ii = 0; // Current column in training_mat
for (int i = 0; i<matImageFloat.rows; i++) {
for (int j = 0; j < matImageFloat.cols; j++) {
matTrainingImagesAsFlattenedFloats.at<float>(contoursCount-1, ii++) = matImageFloat.at<float>(i,j);
}
}
}
catch (std::exception &exc) {
f = false;
exc.what();
}
if (f) {
matClassificationInts.push_back((float)intChar);
matSamples.at<float>(contoursCount-1, j) = 1.0;
}
f = true;
// add to Mat as though it was a vector, this is necessary due to the
// data types that KNearest.train accepts
} // end if
//} // end if
} // end for
}//end i
}//end j
}
Output of predict function
Unfortunately, I don't have the necessary time to really review the code, but I can say off the top that to train a model that performs well for prediction with 36 classes, you will need several things:
A large number of good quality images. Ideally, you'd want thousands of images for each class. Of course, you can see somewhat decent results with less than that, but if you only have a few images per class, it's never going to be able to generalize adequately.
You need a model that is large and sophisticated enough to provide the necessary expressiveness to solve the problem. For a problem like this, a plain old multi-layer perceptron with one hidden layer with 100 units may not be enough. This is actually a problem that would benefit from using a Convolutional Neural Net (CNN) with a couple layers just to extract useful features first. But assuming you don't want to go down that path, you may at least want to tweak the size of your hidden layer.
To even get to a point where the training process converges, you will probably need to experiment and crucially, you need an effective way to test the accuracy of the ANN after each experiment. Ideally, you want to observe the loss as the training is proceeding, but I'm not sure whether that's possible using OpenCV's ML functionality. At a minimum, you should fully expect to have to play around with the various so-called "hyper-parameters" and run many experiments before you have a reasonable model.
Anyway, the most important thing is to make sure you have a solid mechanism for validating the accuracy of the model after training. If you aren't already doing so, set aside some images as a separate test set, and after each experiment, use the trained ANN to predict each test image to see the accuracy.
One final general note: what you're trying to do is complex. You will save yourself a huge number of headaches if you take the time early and often to refactor your code. No matter how many experiments you run, if there's some defect causing (for example) your training data to be fundamentally different in some way than your test data, you will never see good results.
Good luck!
EDIT: I should also point out that seeing the same result for every input image is a classic sign that training failed. Unfortunately, there are many reasons why that might happen and it will be very difficult for anyone to isolate that for you without some cleaner code and access to your image data.
I have solved the issue of not getting the output of predict. The issue was created because of the input Mat image to train (ie. matTrainingImagesAsFlattenedFloats) was having values 255.0 for a white pixel. This happened because I haven't use convertTo() properly. You need to use convertTo(OutputImage name, CV_32FC1, 1.0 / 255.0); like this which will convert all the pixel values with 255.0 to 1.0 and after that I am getting the correct output.
Thank you for all the help.
This is too broad to be in one question. Sorry for the bad news. I tried this over and over and couldn't find a solution. I recommend that you implement a simple AND, OR or XOR first just to make sure that the learning part is working and that you are getting better results the more passes you do. Also I suggest to try the Tangent Hyperbolic as a Transfer Function instead of Sigmoid. And Good luck!
Here is some of my own posts that might help you:
Exact results as yours: HERE
Some codes: HERE
I don't want to say that, but several professors I met said Backpropagation just doesn't work and they had (and me have) to implement my own method of teaching the network.
I'm trying to plot the centroid of a specific blob detected using contour techniques. I don't wish to loop through all the blob detected in an image - I only want to plot the centroid of one (i.e. contour[2]). Ideally I'd like to accomplish this using the most efficient / fastest method.
Here's my code:
#include "opencv2/highgui/highgui.hpp"
#include "opencv2/opencv.hpp"
#include <iostream>
#define _USE_MATH_DEFINES
#include <math.h>
using namespace cv;
using namespace std;
int main(int argc, const char** argv)
{
cv::Mat src = cv::imread("frame-1.jpg");
if (src.empty())
return -1;
cv::Mat gray;
cv::cvtColor(~src, gray, CV_BGR2GRAY);
cv::threshold(gray, gray, 160, 255, cv::THRESH_BINARY);
// Find all contours
std::vector<std::vector<cv::Point> > contours;
cv::findContours(gray.clone(), contours, CV_RETR_EXTERNAL, CV_CHAIN_APPROX_NONE);
// Fill holes in each contour
cv::drawContours(gray, contours, -1, CV_RGB(255, 255, 255), -1);
cout << contours.size();
double avg_x(0), avg_y(0); // average of contour points
for (int j = 0; j < contours[2].size(); ++j)
{
avg_x += contours[2][j].x;
avg_y += contours[2][j].y;
}
avg_x /= contours[2].size();
avg_y /= contours[2].size();
cout << avg_x << " " << avg_y << endl;
cv::circle(gray, {avg_x, avg_y}, 5, CV_RGB(5, 100, 100), 5);
namedWindow("MyWindow", CV_WINDOW_AUTOSIZE);
imshow("MyWindow", gray);
waitKey(0);
destroyWindow("MyWindow");
return 0;
}
However, plotting the circle using the coordinates (avg_x, avg_y) results in a 'no instance of constructor "cv::Point_<Tp>::Point[with_Tp=int]" matches the argument list - argument types are: (double, double)' error.
Use min enclosing circle
float radius ;
Point2f center ;
minEnclosingCircle ( contours[i] , center , radius ) ;
cv::circle(gray, center, 5, CV_RGB(5, 100, 100), 5);
I have the following code. I read from the image a block of pixels and I would like to get the value from every block (array 16*16).
However, I get this error:
OpenCV Error: Assertion failed (dims <= 2 && data && (unsigned)i0 < (unsigned)size.p[0] && (unsigned)(i1*DataType<_Tp>::channels) < (unsigned)(size.p[1]*channels()) && ((((sizeof(size_t)<<28)|0x8442211)
((DataType<_Tp>::depth) & ((1 << 3) - 1))*4) & 15) == elemSize1()) in unknown function, file C:\opencv231\build\include\opencv2/core/mat.hpp, line 537
What should I change so that I can run my code ?
enum Color {White, Black};
Color checkBlock(Mat& img, int& i, int& j, double& T)
{
unsigned int Sum=0;
for(int k=0;k<16;k++)
for(int l=0;l<16;l++)
Sum += img.at<unsigned char>(i+k,j+l);
double Average = Sum/256;
std::cout << Average << std::endl;
return (Average > T) ? (White) : (Black);
}
void main()
{
Mat img = imread("Frame.jpg",0);
namedWindow( "Display window", CV_NORMAL );// Create a window for display.
if(!img.data)
std::cout << "error";
// STEPS TO CONVERT TO BINARY IMAGE
// LOAD THE IMAGE
cv::Mat imageMat = cv::imread("Frame.jpg", CV_LOAD_IMAGE_COLOR);
cv::Mat grayscaleMat (imageMat.size(), CV_8U);
//Convert BGR to Gray
cv::cvtColor(imageMat, grayscaleMat, CV_BGR2GRAY );
//Binary image
cv::Mat binaryMat(grayscaleMat.size(), grayscaleMat.type());
//Apply thresholding
cv::threshold(grayscaleMat, binaryMat, 100, 255, cv::THRESH_BINARY);
//Show the results
// cv::namedWindow("Output",CV_NORMAL);
//cv::imshow("Output", binaryMat);
// cv::waitKey(0);
double minVal, maxVal;
minMaxLoc(img,&minVal,&maxVal,NULL,NULL);
double Threshold = 0.5 * (minVal + maxVal);
int i=4,j=4;
Size s = img.size();
Color old_c, new_c;
// define the position wher i will begin to read the first row from the image
for (j=16*55;j<=s.height;j=j+16)
for(i=0;i<=s.width;i=i+16)
{
Point x(i,j);
Point y(i+16,j+16);
//std::cout << x << " " << y << std::endl;
rectangle(img, x, y, Scalar(255,0,0),3);
Color c = checkBlock(img,i,j,Threshold);
}
In this line you are using i to index a row:
Sum += img.at<unsigned char>(i+k,j+l);
But here, where i comes from, it is clearly the index of a col.
for(i=0;i<=s.width;i=i+16)
So that first line should be:
Sum += img.at<unsigned char>(j+l, i+k);
Just to be clear, the arguments to at are (row, col), the paramters for Point are (x,y), which is a bit of a trap.
Also
for (j=16*55;j<=s.height;j=j+16)
for(i=0;i<=s.width;i=i+16)
...
Point y(i+16,j+16);
should be
for (j = 16 * 55; j < s.height - 15 ; j = j + 16)
for(i = 0; i < s.width - 15; i = i + 16)
...
Point y(i + 15, j + 15);
The imread() must include the full path of your file, and be sure to use double dashes, e.g.
cv::Mat imageMat = cv::imread(C:\\Folder\\Frame.jpg", CV_LOAD_IMAGE_COLOR);
It is not necessary to specify the dimension or type of grayscaleMat or binaryMat, as the opencv functions will prepare them for you.
In your cvtColor function, it should be CV_RGB2GRAY not CV_BGR2GRAY.
In you threshold function, it should not be cv::THRESH_BINARY but rather CV_THRESH_BINARY
Hope this helps.
I'm using the program squares.c available in the samples of OpenCV libraries. It works well with every image, but I really can't figure it out why it doesn't recognize the square drawn in that image
http://desmond.imageshack.us/Himg12/scaled.php?server=12&filename=26725680.jpg&res=medium
After CANNY:
After DILATE:
The RESULT image (in red)
http://img267.imageshack.us/img267/8016/resultuq.jpg
As you can see, the square is NOT detected.
After the detection I need to extract the area contained in the square...How is it possible without a ROI?
The source code below presents a small variation of the Square Detector program. It's not perfect, but it illustrates one way to approach your problem.
You can diff this code to the original and check all the changes that were made, but the main ones are:
Decrease the number of threshold levels to 2.
In the beginning of findSquares(), dilate the image to detect the thin white square, and then blur the entire image so the algorithm doesn't detect the sea and the sky as individual squares.
Once compiled, run the application with the following syntax: ./app <image>
// The "Square Detector" program.
// It loads several images sequentially and tries to find squares in
// each image
#include "highgui.h"
#include "cv.h"
#include <iostream>
#include <math.h>
#include <string.h>
using namespace cv;
using namespace std;
void help()
{
cout <<
"\nA program using pyramid scaling, Canny, contours, contour simpification and\n"
"memory storage (it's got it all folks) to find\n"
"squares in a list of images pic1-6.png\n"
"Returns sequence of squares detected on the image.\n"
"the sequence is stored in the specified memory storage\n"
"Call:\n"
"./squares\n"
"Using OpenCV version %s\n" << CV_VERSION << "\n" << endl;
}
int thresh = 50, N = 2; // karlphillip: decreased N to 2, was 11.
const char* wndname = "Square Detection Demo";
// helper function:
// finds a cosine of angle between vectors
// from pt0->pt1 and from pt0->pt2
double angle( Point pt1, Point pt2, Point pt0 )
{
double dx1 = pt1.x - pt0.x;
double dy1 = pt1.y - pt0.y;
double dx2 = pt2.x - pt0.x;
double dy2 = pt2.y - pt0.y;
return (dx1*dx2 + dy1*dy2)/sqrt((dx1*dx1 + dy1*dy1)*(dx2*dx2 + dy2*dy2) + 1e-10);
}
// returns sequence of squares detected on the image.
// the sequence is stored in the specified memory storage
void findSquares( const Mat& image, vector<vector<Point> >& squares )
{
squares.clear();
Mat pyr, timg, gray0(image.size(), CV_8U), gray;
// karlphillip: dilate the image so this technique can detect the white square,
Mat out(image);
dilate(out, out, Mat(), Point(-1,-1));
// then blur it so that the ocean/sea become one big segment to avoid detecting them as 2 big squares.
medianBlur(out, out, 7);
// down-scale and upscale the image to filter out the noise
pyrDown(out, pyr, Size(out.cols/2, out.rows/2));
pyrUp(pyr, timg, out.size());
vector<vector<Point> > contours;
// find squares in every color plane of the image
for( int c = 0; c < 3; c++ )
{
int ch[] = {c, 0};
mixChannels(&timg, 1, &gray0, 1, ch, 1);
// try several threshold levels
for( int l = 0; l < N; l++ )
{
// hack: use Canny instead of zero threshold level.
// Canny helps to catch squares with gradient shading
if( l == 0 )
{
// apply Canny. Take the upper threshold from slider
// and set the lower to 0 (which forces edges merging)
Canny(gray0, gray, 0, thresh, 5);
// dilate canny output to remove potential
// holes between edge segments
dilate(gray, gray, Mat(), Point(-1,-1));
}
else
{
// apply threshold if l!=0:
// tgray(x,y) = gray(x,y) < (l+1)*255/N ? 255 : 0
gray = gray0 >= (l+1)*255/N;
}
// find contours and store them all as a list
findContours(gray, contours, CV_RETR_LIST, CV_CHAIN_APPROX_SIMPLE);
vector<Point> approx;
// test each contour
for( size_t i = 0; i < contours.size(); i++ )
{
// approximate contour with accuracy proportional
// to the contour perimeter
approxPolyDP(Mat(contours[i]), approx, arcLength(Mat(contours[i]), true)*0.02, true);
// square contours should have 4 vertices after approximation
// relatively large area (to filter out noisy contours)
// and be convex.
// Note: absolute value of an area is used because
// area may be positive or negative - in accordance with the
// contour orientation
if( approx.size() == 4 &&
fabs(contourArea(Mat(approx))) > 1000 &&
isContourConvex(Mat(approx)) )
{
double maxCosine = 0;
for( int j = 2; j < 5; j++ )
{
// find the maximum cosine of the angle between joint edges
double cosine = fabs(angle(approx[j%4], approx[j-2], approx[j-1]));
maxCosine = MAX(maxCosine, cosine);
}
// if cosines of all angles are small
// (all angles are ~90 degree) then write quandrange
// vertices to resultant sequence
if( maxCosine < 0.3 )
squares.push_back(approx);
}
}
}
}
}
// the function draws all the squares in the image
void drawSquares( Mat& image, const vector<vector<Point> >& squares )
{
for( size_t i = 0; i < squares.size(); i++ )
{
const Point* p = &squares[i][0];
int n = (int)squares[i].size();
polylines(image, &p, &n, 1, true, Scalar(0,255,0), 3, CV_AA);
}
imshow(wndname, image);
}
int main(int argc, char** argv)
{
if (argc < 2)
{
cout << "Usage: ./program <file>" << endl;
return -1;
}
// static const char* names[] = { "pic1.png", "pic2.png", "pic3.png",
// "pic4.png", "pic5.png", "pic6.png", 0 };
static const char* names[] = { argv[1], 0 };
help();
namedWindow( wndname, 1 );
vector<vector<Point> > squares;
for( int i = 0; names[i] != 0; i++ )
{
Mat image = imread(names[i], 1);
if( image.empty() )
{
cout << "Couldn't load " << names[i] << endl;
continue;
}
findSquares(image, squares);
drawSquares(image, squares);
imwrite("out.jpg", image);
int c = waitKey();
if( (char)c == 27 )
break;
}
return 0;
}
Outputs:
I would suggest that your square in this image is too thin. The first step in squares.c is to scale the image down and back up to reduce noise before passing to the Canny edge detector.
The scaling convolves with a 5x5 kernel, so in your case this could result in losing any gradient in such a thin edge.
Try making your square's edges at least 5 pixels if you are going to overlay them on a continuous background.