Object tracking delay color tracking OpenCV - c++

I am trying to detect colored balls like ps3 move controller balls from 2 mt distance.I have 10 camera in same room hanging from the ceiling.Room is dark and balls have led inside.I have 4-5 balls.(red,blue,green,yellow,pink). I want track their position with opencv.Whats the right mehtod for doing this in opencv ? Can u give link , example for this ?
I use this code but i have delay problem.When i comment // my trackFilteredObject line there is no lag.But when using this code i have lot latency.I cant understand why happening because my normal cpu usage ~%15 ram usage 6.3GB/15GB (%40) when run this code cpu usage ~20-23 ram usage 6.4GB . I think its not about cpu-ram performance.What am i doing wrong ?
Video: https://www.youtube.com/watch?v=_BKtJpPrkO4 (You can see lag in first 10 sec.After 10 sen i comment tracking codes.)
Note:Kamerasayisi mean cameracount My Track Function:
void trackFilteredObject(Object theObject,Mat threshold,Mat HSV, Mat &cameraFeed){
//max number of objects to be detected in frame
const int FRAME_WIDTH = 5120;
const int FRAME_HEIGHT = 480;
const int MAX_NUM_OBJECTS=50;
//minimum and maximum object area
const int MIN_OBJECT_AREA = 10*10;
const int MAX_OBJECT_AREA = FRAME_HEIGHT*FRAME_WIDTH/1.5;
vector <Object> objects;
Mat temp;
threshold.copyTo(temp);
//these two vectors needed for output of findContours
vector< vector<Point> > contours;
vector<Vec4i> hierarchy;
//find contours of filtered image using openCV findContours function
findContours(temp,contours,hierarchy,CV_RETR_CCOMP,CV_CHAIN_APPROX_SIMPLE );
//use moments method to find our filtered object
double refArea = 0;
bool objectFound = false;
if (hierarchy.size() > 0) {
int numObjects = hierarchy.size();
//if number of objects greater than MAX_NUM_OBJECTS we have a noisy filter
if(numObjects<MAX_NUM_OBJECTS){
for (int index = 0; index >= 0; index = hierarchy[index][0]) {
Moments moment = moments((cv::Mat)contours[index]);
double area = moment.m00;
//if the area is less than 20 px by 20px then it is probably just noise
//if the area is the same as the 3/2 of the image size, probably just a bad filter
//we only want the object with the largest area so we safe a reference area each
//iteration and compare it to the area in the next iteration.
if(area>MIN_OBJECT_AREA){
Object object;
object.setXPos(moment.m10/area);
object.setYPos(moment.m01/area);
object.setType(theObject.getType());
object.setColor(theObject.getColor());
objects.push_back(object);
objectFound = true;
}else objectFound = false;
}
//let user know you found an object
if(objectFound ==true){
//draw object location on screen
drawObject(objects,cameraFeed,temp,contours,hierarchy);}
}else putText(cameraFeed,"TOO MUCH NOISE! ADJUST FILTER",Point(0,50),1,2,Scalar(0,0,255),2);
}
}
};
Main Code:
void Run()
{
int w, h;
_fps = 30;
IplImage *pCapImage[kameraSayisi];
IplImage *pDisplayImage;
PBYTE pCapBuffer = NULL;
// Create camera instance
for(int i = 0; i < kameraSayisi; i++)
{
_cam[i] = CLEyeCreateCamera(_cameraGUID[i], _mode, _resolution, _fps);
if(_cam[i] == NULL) return;
// Get camera frame dimensions
CLEyeCameraGetFrameDimensions(_cam[i], w, h);
// Create the OpenCV images
pCapImage[i] = cvCreateImage(cvSize(w, h), IPL_DEPTH_8U, 1);
// Set some camera parameters
CLEyeSetCameraParameter(_cam[i], CLEYE_GAIN, 0);
CLEyeSetCameraParameter(_cam[i], CLEYE_EXPOSURE, 511);
// Start capturing
CLEyeCameraStart(_cam[i]);
}
pDisplayImage = cvCreateImage(cvSize(w*kameraSayisi / 2, h * kameraSayisi/4 ), IPL_DEPTH_8U ,1);
if(_cam == NULL) return;
int iLastX = -1;
int iLastY = -1;
//Capture a temporary image from the camera
//program
bool trackObjects = true;
bool useMorphOps = true;
Mat HSV;
//Create a black image with the size as the camera output
Mat imgLines;
// imgLines = Mat::zeros( cvarrToMat(image).size(), CV_8UC3 );;
Mat threshold;
//x and y values for the location of the object
int x=0, y=0;
bool calibrationMode = false;
if(calibrationMode){
//create slider bars for HSV filtering
createTrackbars();
}
// image capturing loop
while(_running)
{
PBYTE pCapBuffer;
// Capture camera images
for(int i = 0; i < kameraSayisi; i++)
{
cvGetImageRawData(pCapImage[i], &pCapBuffer);
CLEyeCameraGetFrame(_cam[i], pCapBuffer, (i==0)?2000:0);
}
// Display stereo image
for(int i = 0; i < kameraSayisi; i++)
{
cvSetImageROI(pDisplayImage, cvRect(w * (i%4) ,i/4 * h, w, h));
cvCopy(pCapImage[i], pDisplayImage);
}
cvResetImageROI(pDisplayImage);
Mat imgOriginal;
Mat imgConverted = cvarrToMat(pDisplayImage);
if(calibrationMode==true)
{
//need to find the appropriate color range values
// calibrationMode must be false
//if in calibration mode, we track objects based on the HSV slider values.
//cvtColor(imgOriginal,imgOriginal,CV_BayerRG2RGB);
cvtColor(imgConverted,imgOriginal,CV_BayerGB2BGR);
cvtColor(imgOriginal,HSV,CV_BGR2HSV);
inRange(HSV,Scalar(H_MIN,S_MIN,V_MIN),Scalar(H_MAX,S_MAX,V_MAX),threshold);
morphOps(threshold);
imshow(_windowName + 'T',threshold);
//the folowing for canny edge detec
/// Create a matrix of the same type and size as src (for dst)
dst.create( imgOriginal.size(), src.type() );
/// Convert the image to grayscale
cvtColor( imgOriginal, src_gray, CV_BGR2GRAY );
/// Create a window
namedWindow( window_name, CV_WINDOW_AUTOSIZE );
/// Create a Trackbar for user to enter threshold
// createTrackbar( "Min Threshold:", window_name, &lowThreshold, max_lowThreshold, CannyThreshold );
/// Show the image
Object a = Object(H_MIN,S_MIN,V_MIN,H_MAX,S_MAX,V_MAX);
trackFilteredObject(a,threshold,HSV,imgOriginal);
}
else{
//we can use their member functions/information
Object blue("blue"), yellow("yellow"), red("red"), orange("orange"),white("white");
cvtColor(imgConverted,imgOriginal,CV_BayerGB2BGR);
//first find blue objects
cvtColor(imgOriginal,HSV,CV_RGB2HSV);
inRange(HSV,blue.getHSVmin(),blue.getHSVmax(),threshold);
morphOps(threshold);
//then yellows
inRange(HSV,yellow.getHSVmin(),yellow.getHSVmax(),threshold);
//then reds
inRange(HSV,red.getHSVmin(),red.getHSVmax(),threshold);
//then white
inRange(HSV,white.getHSVmin(),white.getHSVmax(),threshold);
//then orange
inRange(HSV,orange.getHSVmin(),orange.getHSVmax(),threshold);
trackFilteredObject(yellow,threshold,HSV,imgOriginal);
trackFilteredObject(white,threshold,HSV,imgOriginal);
trackFilteredObject(red,threshold,HSV,imgOriginal);
trackFilteredObject(blue,threshold,HSV,imgOriginal);
trackFilteredObject(orange,threshold,HSV,imgOriginal);
}
//delay 10ms so that screen can refresh.
//image will not appear without this waitKey() command
if (cvWaitKey(30) == 27) //wait for 'esc' key press for 30ms. If 'esc' key is pressed, break loop
{
cout << "esc key is pressed by user" << endl;
break;
}
// cvShowImage(_windowName, image);
imshow(_windowName,imgOriginal);
}
for(int i = 0; i < kameraSayisi; i++)
{
// Stop camera capture
CLEyeCameraStop(_cam[i]);
// Destroy camera object
CLEyeDestroyCamera(_cam[i]);
// Destroy the allocated OpenCV image
cvReleaseImage(&pCapImage[i]);
_cam[i] = NULL;
}
}

Related

Changing displacement over a sequence of frames

guys.
I've written a code to describe motion on interest points over a .avi video file.
Here is the code:
#include "opencv2/video/tracking.hpp"
#include<opencv2/core/core.hpp>
#include<opencv2/highgui/highgui.hpp>
#include<opencv2/imgproc/imgproc.hpp>
#include<iostream>
using namespace cv;
using namespace std;
int main() {
VideoCapture capture("video.avi");
if (!capture.isOpened()) {
cout << "ERROR OPENING VIDEO\n\n";
return(0);
}
double rate = capture.get(CV_CAP_PROP_FPS);
unsigned int numberFrames = (unsigned int) capture.get(CV_CAP_PROP_FRAME_COUNT);
int width = (unsigned int) capture.get(CV_CAP_PROP_FRAME_WIDTH);
int height = (unsigned int) capture.get(CV_CAP_PROP_FRAME_HEIGHT);
unsigned int codec = (unsigned int) capture.get(CV_CAP_PROP_FOURCC);
Mat currentGray;
Mat previousGray;
vector< Point2f > points[2];
vector< Point2f > initial;
vector< Point2f > features;
vector< uchar > status;
vector< float > error;
int maxCorners = 500; // maximum number of features to detect
double qualityLevel = 0.01; // quality level for feature detection
double minDistance = 10; // min distance between two points
Mat frame, output;
VideoWriter createdVideo("output.avi", codec, rate, Size(width,height), 1);
for (unsigned frameCounter = 0; frameCounter < numberFrames; frameCounter++) {
capture >> frame;
if (frame.empty())
break;
imshow("Video", frame);
cvtColor(frame, currentGray, CV_BGR2GRAY);
frame.copyTo(output);
if (points[0].size() <= 10){
goodFeaturesToTrack(currentGray, // the image
features, // the output detected features
maxCorners, // the maximum number of features
qualityLevel, // quality level
minDistance); // min distance between two features
// add the detected features to
// the currently tracked features
points[0].insert(points[0].end(),
features.begin(), features.end());
initial.insert(initial.end(),
features.begin(), features.end());
}
if (previousGray.empty())
currentGray.copyTo(previousGray);
calcOpticalFlowPyrLK(previousGray, currentGray, // 2 consecutive images
points[0], // input point positions in first image
points[1], // output point positions in the 2nd image
status, // tracking success
error); // tracking error
int k = 0;
for (int i = 0; i < points[1].size(); i++) {
// do we keep this point?
if (status[i] && // if point has moved
(abs(points[0][i].x - points[1][i].x) +
(abs(points[0][i].y - points[1][i].y)) > 2))
initial[k] = initial[i];
points[1][k++] = points[1][i];
}
points[1].resize(k);
initial.resize(k);
for (int i = 0; i < points[1].size(); i++) {
// draw line and circle
line(output,
initial[i], // initial position
points[1][i],// new position
Scalar(0, 255, 0), 2);
circle(output,
points[1][i],
2,
Scalar(0, 0, 255), -1);
}
std::swap(points[1], points[0]);
cv::swap(previousGray, currentGray);
createdVideo.write(output);
}
waitKey(0);
return(0);
}
My code tracks displacement of points frame by frame and keeps the first location of them until the end of video.
However, I would like not the keep the location's points of the first frame, but change them over time, i.e. changing the first point location with the second point location so on and then huge lines will not appear but only the displacement between two points in two frames.
Is there any possibility of doing this?
Since you only want the position of points in two frames, just use two vectors; one holding the keypoints from the last frame, and one holding keypoints from the previous frame. At the end of each iteration, just set the previous points to the current points. Something like this pseudocode:
// first frame
// detect keypoints
prev_frame_points = keypoints
// rest of the frames
for frame in frames:
// detect keypoints
curr_frame_points = keypoints
line(..., prev_frame_points, curr_frame_points, ...)
prev_frame_points = curr_frame_points

Real time object tracking in birds-eye in OpenCV

This program is my 'Real time Color Tracking in Birds-eye System'.
#include <sstream>
#include <string>
#include <iostream>
#include <vector>
#include <stdio.h>
#include <stdlib.h>
#include "stdafx.h"
#include "Fruit.h"
#include "opencv2\\opencv.hpp"
#pragma comment(lib,"C:\\opencv\\build\\x86\\vc10\\lib\\opencv_core249d.lib")
#pragma comment(lib,"C:\\opencv\\build\\x86\\vc10\\lib\\opencv_imgproc249d.lib")
#pragma comment(lib,"C:\\opencv\\build\\x86\\vc10\\lib\\opencv_highgui249d.lib")
#pragma comment(lib,"C:\\opencv\\build\\x86\\vc10\\lib\\opencv_objdetect249d.lib")
#pragma comment(lib,"C:\\opencv\\build\\x86\\vc10\\lib\\opencv_ml249d.lib")
#pragma comment(lib,"C:\\opencv\\build\\x86\\vc10\\lib\\opencv_video249d.lib")
#pragma comment(lib,"C:\\opencv\\build\\x86\\vc10\\lib\\opencv_calib3d249d.lib")
using namespace cv;
//initial min and max HSV filter values.
//these will be changed using trackbars
int H_MIN = 0;
int H_MAX = 256;
int S_MIN = 0;
int S_MAX = 256;
int V_MIN = 0;
int V_MAX = 256;
//default capture width and height
const int FRAME_WIDTH = 640;
const int FRAME_HEIGHT = 480;
//max number of objects to be detected in frame
const int MAX_NUM_OBJECTS=50;
//minimum and maximum object area
const int MIN_OBJECT_AREA = 7*7;
const int MAX_OBJECT_AREA = FRAME_HEIGHT*FRAME_WIDTH/1.5;
//names that will appear at the top of each window
const string windowName = "Original Image";
const string windowName1 = "HSV Image";
const string windowName2 = "Thresholded Image";
const string windowName3 = "After Morphological Operations";
const string trackbarWindowName = "Trackbars";
void on_trackbar( int, void* )
{
//This function gets called whenever a
// trackbar position is changed
}
string intToString(int number){
std::stringstream ss;
ss << number;
return ss.str();
}
void createTrackbars(){
//create window for trackbars
namedWindow(trackbarWindowName,0);
//create memory to store trackbar name on window
char TrackbarName[50];
sprintf( TrackbarName, "H_MIN", H_MIN);
sprintf( TrackbarName, "H_MAX", H_MAX);
sprintf( TrackbarName, "S_MIN", S_MIN);
sprintf( TrackbarName, "S_MAX", S_MAX);
sprintf( TrackbarName, "V_MIN", V_MIN);
sprintf( TrackbarName, "V_MAX", V_MAX);
createTrackbar( "H_MIN", trackbarWindowName, &H_MIN, H_MAX, on_trackbar );
createTrackbar( "H_MAX", trackbarWindowName, &H_MAX, H_MAX, on_trackbar );
createTrackbar( "S_MIN", trackbarWindowName, &S_MIN, S_MAX, on_trackbar );
createTrackbar( "S_MAX", trackbarWindowName, &S_MAX, S_MAX, on_trackbar );
createTrackbar( "V_MIN", trackbarWindowName, &V_MIN, V_MAX, on_trackbar );
createTrackbar( "V_MAX", trackbarWindowName, &V_MAX, V_MAX, on_trackbar );
}
void drawObject(vector<Fruit> theFruits,Mat &frame){
for(int i =0; i<theFruits.size(); i++){
cv::circle(frame,cv::Point(theFruits.at(i).getXPos(),theFruits.at(i).getYPos()),10,cv::Scalar(0,0,255));
cv::putText(frame,intToString(theFruits.at(i).getXPos())+ " , " + intToString(theFruits.at(i).getYPos()),cv::Point(theFruits.at(i).getXPos(),theFruits.at(i).getYPos()+20),1,1,Scalar(0,255,0));
}
}
void morphOps(Mat &thresh){
//create structuring element that will be used to "dilate" and "erode" image.
//the element chosen here is a 3px by 3px rectangle
Mat erodeElement = getStructuringElement( MORPH_RECT,Size(3,3));
//dilate with larger element so make sure object is nicely visible
Mat dilateElement = getStructuringElement( MORPH_RECT,Size(8,8));
erode(thresh,thresh,erodeElement);
erode(thresh,thresh,erodeElement);
dilate(thresh,thresh,dilateElement);
dilate(thresh,thresh,dilateElement);
}
void trackFilteredObject(Mat threshold,Mat HSV, Mat &cameraFeed_BE){
vector <Fruit> apples;
Mat temp;
threshold.copyTo(temp);
//these two vectors needed for output of findContours
vector< vector<Point> > contours;
vector<Vec4i> hierarchy;
//find contours of filtered image using openCV findContours function
findContours(temp,contours,hierarchy,CV_RETR_CCOMP,CV_CHAIN_APPROX_SIMPLE );
//use moments method to find our filtered object
double refArea = 0;
bool objectFound = false;
if (hierarchy.size() > 0) {
int numObjects = hierarchy.size();
//if number of objects greater than MAX_NUM_OBJECTS we have a noisy filter
if(numObjects<MAX_NUM_OBJECTS){
for (int index = 0; index >= 0; index = hierarchy[index][0]) {
Moments moment = moments((cv::Mat)contours[index]);
double area = moment.m00;
if(area>MIN_OBJECT_AREA){
Fruit apple;
apple.setXPos(moment.m10/area);
apple.setYPos(moment.m01/area);
apples.push_back(apple);
objectFound = true;
}else objectFound = false;
}
//let user know you found an object
if(objectFound ==true){
//draw object location on screen
drawObject(apples,cameraFeed_BE);
}
}else putText(cameraFeed_BE,"TOO MUCH NOISE! ADJUST FILTER",Point(0,50),1,2,Scalar(0,0,255),2);
}
}
int main(int argc, char* argv[])
{
//if we would like to calibrate our filter values, set to true.
bool calibrationMode = true;
//Matrix to store each frame of the webcam feed
Mat cameraFeed;
Mat threshold;
Mat HSV;
if(calibrationMode){
//create slider bars for HSV filtering
createTrackbars();
}
//video capture object to acquire webcam feed
VideoCapture capture;
//open capture object at location zero (default location for webcam)
capture.open(0);
//set height and width of capture frame
capture.set(CV_CAP_PROP_FRAME_WIDTH,FRAME_WIDTH);
capture.set(CV_CAP_PROP_FRAME_HEIGHT,FRAME_HEIGHT);
if(!capture.isOpened()) //confirm camera opened
{
return -1;
}
//start an infinite loop where webcam feed is copied to cameraFeed matrix
//all of our operations will be performed within this loop
while(1){
do{
capture >> cameraFeed;
}while(cameraFeed.empty());
int key = 0;
IplImage* Frame ;
IplImage* birds_image;
CvMat *H;
float Z;
cvNamedWindow("Birds eye");
Frame =new IplImage(cameraFeed);
birds_image = cvCloneImage(Frame);
// EXAMPLE OF LOADING THESE MATRICES BACK IN:
CvMat *intrinsic = (CvMat*)cvLoad("Intrinsics.xml");
CvMat *distortion = (CvMat*)cvLoad("Distortion.xml");
// Build the undistort map which we will use for all subsequent frames.
IplImage* mapx = cvCreateImage( cvGetSize(Frame), IPL_DEPTH_32F, 1 );
IplImage* mapy = cvCreateImage( cvGetSize(Frame), IPL_DEPTH_32F, 1 );
cvInitUndistortMap(
intrinsic,
distortion,
mapx,
mapy
);
IplImage *t = cvCloneImage(Frame);
//cvShowImage( "Calibration", image ); // Show raw image
cvRemap( t, Frame, mapx, mapy ); // Undistort image
cvReleaseImage(&t);
H = (CvMat*)cvLoad("H.xml");
cvWarpPerspective(
Frame,
birds_image,
H,
CV_INTER_LINEAR | CV_WARP_INVERSE_MAP | CV_WARP_FILL_OUTLIERS
);
cvShowImage("Birds eye", birds_image);
Mat cameraFeed_BE = cvarrToMat(birds_image);
//store image to matrix
capture.read(cameraFeed_BE);
//convert frame from BGR to HSV colorspace
cvtColor(cameraFeed_BE,HSV,COLOR_BGR2HSV);
if(calibrationMode==true){
//if in calibration mode, we track objects based on the HSV slider values.
cvtColor(cameraFeed_BE,HSV,COLOR_BGR2HSV);
inRange(HSV,Scalar(H_MIN,S_MIN,V_MIN),Scalar(H_MAX,S_MAX,V_MAX),threshold);
morphOps(threshold);
imshow(windowName2,threshold);
trackFilteredObject(threshold,HSV,cameraFeed_BE);
}
//show frames
imshow(windowName2,threshold);
imshow(windowName,cameraFeed_BE);
imshow(windowName1,HSV);
//delay 30ms so that screen can refresh.
//image will not appear without this waitKey() command
if(cv::waitKey(30) >= 0) break;
}
return 0;
}
I think Original Image, HSV Image and Thresholded Image must
be birds image but actually these aren't. I don't know why.
And I run the program for a minute, this Error appears.
enter image description here
I think this error appears when there are some memory leak. SO I added this code.
cvReleaseImage(&birds_image)
but this code didn't work well.
Could you give me your ideas regarding 1. and 2.?
As you noted, the error message is related to an memory error. You have a memory allocation line
Frame =new IplImage(cameraFeed);
inside your infinite loop. This array (a RGB image from the webcam!!) is never released, and the loop writes over the pointer, losing the information and making it impossible to destroy that memory allocation.
Also, might be a good idea to move reading .xml files out of the infinite loop
CvMat *intrinsic = (CvMat*)cvLoad("Intrinsics.xml");
CvMat *distortion = (CvMat*)cvLoad("Distortion.xml");

How to detect squares in video with OpenCV?

So I combined squares.cpp with cvBoundingRect.cpp code to detect squares in video. I therefore, had to convert from IplImage to Mat type so that findSquares and drawSquares methods could run (By using cvarrToMat function). But unfortunately, after successful compilation I get this error when running:
OpenCV Error: Assertion failed (j < nsrcs && src[j].depth() == depth) in mixChannels, file /Users/Desktop/opencv-3.0.0-rc1/modules/core/src/convert.cpp, line 1205
libc++abi.dylib: terminating with uncaught exception of type cv::Exception: /Users/Desktop/opencv-3.0.0-rc1/modules/core/src/convert.cpp:1205: error: (-215) j < nsrcs && src[j].depth() == depth in function mixChannels
Abort trap: 6
Here's the code:
#include "opencv2/core/core.hpp"
#include "opencv2/imgproc/imgproc.hpp"
#include "opencv2/imgcodecs.hpp"
#include "opencv2/highgui/highgui.hpp"
#include <iostream>
#include <math.h>
#include <string.h>
using namespace cv;
using namespace std;
int thresh = 50, N = 11;
const char* wndname = "Square Detection Demo";
// finds a cosine of angle between vectors
// from pt0->pt1 and from pt0->pt2
static double angle( Point pt1, Point pt2, Point pt0 )
{
double dx1 = pt1.x - pt0.x;
double dy1 = pt1.y - pt0.y;
double dx2 = pt2.x - pt0.x;
double dy2 = pt2.y - pt0.y;
return (dx1*dx2 + dy1*dy2)/sqrt((dx1*dx1 + dy1*dy1)*(dx2*dx2 + dy2*dy2) + 1e-10);
}
// returns sequence of squares detected on the image.
// the sequence is stored in the specified memory storage
static void findSquares( const Mat& image, vector<vector<Point> >& squares )
{
squares.clear();
Mat pyr, timg, gray0(image.size(), CV_8U), gray;
// down-scale and upscale the image to filter out the noise
pyrDown(image, pyr, Size(image.cols/2, image.rows/2));
pyrUp(pyr, timg, image.size());
vector<vector<Point> > contours;
// find squares in every color plane of the image
for( int c = 0; c < 3; c++ )
{
int ch[] = {c, 0};
mixChannels(&timg, 1, &gray0, 1, ch, 1);
// try several threshold levels
for( int l = 0; l < N; l++ )
{
// hack: use Canny instead of zero threshold level.
// Canny helps to catch squares with gradient shading
if( l == 0 )
{
// apply Canny. Take the upper threshold from slider
// and set the lower to 0 (which forces edges merging)
Canny(gray0, gray, 0, thresh, 5);
// dilate canny output to remove potential
// holes between edge segments
dilate(gray, gray, Mat(), Point(-1,-1));
}
else
{
// apply threshold if l!=0:
// tgray(x,y) = gray(x,y) < (l+1)*255/N ? 255 : 0
gray = gray0 >= (l+1)*255/N;
}
// find contours and store them all as a list
findContours(gray, contours, RETR_LIST, CHAIN_APPROX_SIMPLE);
vector<Point> approx;
// test each contour
for( size_t i = 0; i < contours.size(); i++ )
{
// approximate contour with accuracy proportional
// to the contour perimeter
approxPolyDP(Mat(contours[i]), approx, arcLength(Mat(contours[i]), true)*0.02, true);
// square contours should have 4 vertices after approximation
// relatively large area (to filter out noisy contours)
// and be convex.
// Note: absolute value of an area is used because
// area may be positive or negative - in accordance with the
// contour orientation
if( approx.size() == 4 &&
fabs(contourArea(Mat(approx))) > 1000 &&
isContourConvex(Mat(approx)) )
{
double maxCosine = 0;
for( int j = 2; j < 5; j++ )
{
// find the maximum cosine of the angle between joint edges
double cosine = fabs(angle(approx[j%4], approx[j-2], approx[j-1]));
maxCosine = MAX(maxCosine, cosine);
}
// if cosines of all angles are small
// (all angles are ~90 degree) then write quandrange
// vertices to resultant sequence
if( maxCosine < 0.3 )
squares.push_back(approx);
}
}
}
}
}
// the function draws all the squares in the image
static void drawSquares( Mat& image, const vector<vector<Point> >& squares )
{
for( size_t i = 0; i < squares.size(); i++ )
{
const Point* p = &squares[i][0];
int n = (int)squares[i].size();
polylines(image, &p, &n, 1, true, Scalar(255,0,0), 3, LINE_AA);
}
imshow(wndname, image);
}
CvRect rect;
CvSeq* contours = 0;
CvMemStorage* storage = NULL;
CvCapture *cam;
IplImage *currentFrame, *currentFrame_grey, *differenceImg, *oldFrame_grey;
bool first = true;
int main(int argc, char* argv[])
{
//Create a new movie capture object.
cam = cvCaptureFromCAM(0);
//create storage for contours
storage = cvCreateMemStorage(0);
//capture current frame from webcam
currentFrame = cvQueryFrame(cam);
//Size of the image.
CvSize imgSize;
imgSize.width = currentFrame->width;
imgSize.height = currentFrame->height;
//Images to use in the program.
currentFrame_grey = cvCreateImage( imgSize, IPL_DEPTH_8U, 1);
namedWindow( wndname, 1 );
vector<vector<Point> > squares;
while(1)
{
currentFrame = cvQueryFrame( cam );
if( !currentFrame ) break;
//Convert the image to grayscale.
cvCvtColor(currentFrame,currentFrame_grey,CV_RGB2GRAY);
if(first) //Capturing Background for the first time
{
differenceImg = cvCloneImage(currentFrame_grey);
oldFrame_grey = cvCloneImage(currentFrame_grey);
cvConvertScale(currentFrame_grey, oldFrame_grey, 1.0, 0.0);
first = false;
continue;
}
//Minus the current frame from the moving average.
cvAbsDiff(oldFrame_grey,currentFrame_grey,differenceImg);
//bluring the differnece image
cvSmooth(differenceImg, differenceImg, CV_BLUR);
//apply threshold to discard small unwanted movements
cvThreshold(differenceImg, differenceImg, 25, 255, CV_THRESH_BINARY);
//find contours
cv::Mat diffImg = cv::cvarrToMat(differenceImg);
cv::Mat currFrame = cv::cvarrToMat(currentFrame);
findSquares(diffImg, squares);
//draw bounding box around each contour
drawSquares(currFrame, squares);
//display colour image with bounding box
cvShowImage("Output Image", currentFrame);
//display threshold image
cvShowImage("Difference image", differenceImg);
//New Background
cvConvertScale(currentFrame_grey, oldFrame_grey, 1.0, 0.0);
//clear memory and contours
cvClearMemStorage( storage );
contours = 0;
//press Esc to exit
char c = cvWaitKey(33);
if( c == 27 ) break;
}
// Destroy the image & movies objects
cvReleaseImage(&oldFrame_grey);
cvReleaseImage(&differenceImg);
cvReleaseImage(&currentFrame);
cvReleaseImage(&currentFrame_grey);
return 0;
}
As the error message says, your problem is in cv::mixChannels(). See documentation.
Or you could simply do something like
cv::Mat channels[3];
cv::split(multiChannelImage, channels);
and then access each channel using
cv::Mat currChannel = channels[channelNumber]

how to make the program wait until I move the chessboard

I want to calibrate my stereo camera so, I am trying to capture 10 images for a chessboard from the right and the left camera .
what the code should do:
capture 10 images of a chessboard from the right and the left camera.The program saves the two images if both contain a clear corner points. The program should wait 20 frames for me to change the position of the chessboard.
what is happening :
it capture the 10 good images and saves them but, it does not wait for me to change the position of the chessboard
this is my code:
int captureImages_stereoCal()
{
CvCapture* captureL = cvCreateCameraCapture(1);
assert(captureL);
waitKey(10000);
CvCapture* captureR = cvCreateCameraCapture(2);
assert(captureR);
waitKey(10000);
/*Mat imageL ;
Mat imageR*/ ;
int nx=8 , nh=5;
int frame=0;
int s =1;
int ss;
CvPoint2D32f* cornersL = new CvPoint2D32f[nx*nh];
int corner_countL;
CvPoint2D32f* cornersR = new CvPoint2D32f[nx*nh];
int corner_countR;
IplImage *imageL=cvQueryFrame(captureL);
IplImage *gray_imageL=cvCreateImage(cvGetSize(imageL),8,1);
IplImage *CimageL=cvCreateImage(cvGetSize(imageL),32,3);
IplImage *imageR=cvQueryFrame(captureR);
IplImage *gray_imageR=cvCreateImage(cvGetSize(imageR),8,1);
IplImage *CimageR=cvCreateImage(cvGetSize(imageL),32,3);
const int board_dt=20;
while(s<=12)
{
if(frame++ % board_dt == 0)
{
string Result;
ostringstream convert;
ss=s-2;
convert << ss;
Result = convert.str();
//n=(char)s;
//waitKey(1000);
//Left -----------------------------------------------------------------------------------------------------------
string nameL="L.jpg";
//Find chessboard corners:
int foundL = cvFindChessboardCorners(imageL, Size(nx,nh), cornersL, &corner_countL,CV_CALIB_CB_ADAPTIVE_THRESH | CV_CALIB_CB_FILTER_QUADS);
//Get Subpixel accuracy on those corners
cvCvtColor(imageL, gray_imageL, CV_BGR2GRAY);
cvFindCornerSubPix(gray_imageL, cornersL, corner_countL,cvSize(11,11),cvSize(-1,-1), cvTermCriteria(CV_TERMCRIT_EPS+CV_TERMCRIT_ITER, 30, 0.1 ));
//Draw it
Mat MimageL(imageL);
CimageL=imageL;
cvDrawChessboardCorners(CimageL, Size(nx,nh), cornersL,corner_countL, foundL);
//Right -----------------------------------------------------------------------------------------------
string nameR="R.jpg";
//Find chessboard corners:
int foundR = cvFindChessboardCorners(imageR, Size(nx,nh), cornersR, &corner_countR,CV_CALIB_CB_ADAPTIVE_THRESH | CV_CALIB_CB_FILTER_QUADS);
//Get Subpixel accuracy on those corners
cvCvtColor(imageR, gray_imageR, CV_BGR2GRAY);
cvFindCornerSubPix(gray_imageR, cornersR, corner_countR,cvSize(11,11),cvSize(-1,-1), cvTermCriteria(CV_TERMCRIT_EPS+CV_TERMCRIT_ITER, 30, 0.1 ));
//Draw it
Mat MimageR(imageR);
CimageR=imageR;
cvDrawChessboardCorners(CimageR, Size(nx,nh), cornersR,corner_countR, foundR);
cvShowImage( "CalibrationL", CimageL );
cvShowImage( "CalibrationR", CimageR );
if(s>2)
{
if((corner_countL==(nx*nh)) && (corner_countR==(nx*nh)) )
{
nameL.insert(1,Result);
imwrite(nameL,MimageL);
nameR.insert(1,Result);
imwrite(nameR,MimageR);
s++;
}
}
int c = cvWaitKey(15);
if(c == 'p')
{
c = 0;
while(c != 'p' && c != 27)
{
c = cvWaitKey(250);
}
}
if(c == 27)
return 0;
imageL = cvQueryFrame(captureL);
imageR = cvQueryFrame(captureR);
if(s<3)
s++;
}// frame++ end
} // while end
return 0;
}
Also , after it draws the corners it saves the image with the corners drawn on it. I want to save the images with out any changes.
If I understand the problem correctly, it's because you're using a post-increment on your if-statement conditional:
if(frame++ % board_dt == 0)
You initialize frame to be 0 above, so on the 1st call, the if-statement is effectively testing (0 % 20) == 0, which is true.
You can change the if-statement to be a pre-increment, e.g.
if(++frame % board_dt == 0) to get the behavior you are looking for

OpenCV splitting camera feed into grid and determining colour

I've written a piece of code to take my camera feed, split it into a grid (like a chess board) and evaluate each square for colour.
The code i currently have looks like this:
using namespace std;
using namespace cv;
//Standard Dilate and erode functions to improve white/black areas in Binary Image
// Pointer &thresh used so it affects threshImg so it can be used in tracking.
void morphOps(Mat &thresh){
//Increases size of black to remove unwanted white specks outside of object
Mat erodeElement = getStructuringElement( MORPH_RECT,Size(3,3));
//Increases white-area size to remove holes in object
Mat dilateElement = getStructuringElement( MORPH_RECT,Size(8,8));
erode(thresh,thresh,erodeElement);
erode(thresh,thresh,erodeElement);
dilate(thresh,thresh,dilateElement);
dilate(thresh,thresh,dilateElement);
}
//Tracking for the Filtered Object
void trackFilteredObject(int noteNum, string colourtype, Mat &thresh ,Mat HSVImage, Mat &cam){
vector<Brick> Bricks;
Mat temp;
thresh.copyTo(temp);
threshold(temp, thresh, 120, 255, 3); //3 = Threshold to Zero
int whitePixs = countNonZero(thresh);
int cols = thresh.cols;
int rows = thresh.rows;
int imgSize = (rows*cols)/0.75;
if(whitePixs > imgSize){
Brick Brick;
Brick.setColour(colourtype);
Brick.setnoteNum(noteNum);
Bricks.push_back(Brick);
}
int main(int argc, char* argv[])
{
/// Create a window
namedWindow("window", CV_WINDOW_AUTOSIZE );
while(1){
//initialtes camera, sets capture resolution
VideoCapture capture;
capture.open(1);
capture.set(CV_CAP_PROP_FPS, 30);
capture.set(CV_CAP_PROP_FRAME_WIDTH,640);
capture.set(CV_CAP_PROP_FRAME_HEIGHT,480);
Mat cam;
// Saves camera image to Matrix "cam"
capture.read(cam);
//Sets Widths and Heights based on camera resolution (cam.cols/cam.rows retrieves this)
int Width = cam.cols;
int gridWidth = Width/16;
int Height = cam.rows;
int gridHeight = Height/16;
//Splits image into 256 squares going left to right through rows and descending vertically. (16 squares per row for 4/4 pattern)
Mat BigImage;
Mat HSVImage;
// Converts cam to HSV pallete
cvtColor(cam, HSVImage, COLOR_BGR2HSV);
Size smallSize(gridWidth,gridHeight);
std::vector<Mat> smallImages;
for (int y = 0; y < HSVImage.rows; y += smallSize.height)
{
for (int x = 0; x < HSVImage.cols; x += smallSize.width)
{
cv::Rect rect = cv::Rect(x,y, smallSize.width, smallSize.height);
//Saves the matrix to vector
smallImages.push_back(cv::Mat(HSVImage, rect));
}
}
for (int i = 0; i < smallImages.size(); i++){
Mat HSV;
smallImages.at(i).copyTo(HSV);
int noteNum = i;
Mat threshImg;
inRange(HSV,Scalar(0,0,0),Scalar(255,255,255),threshImg);
morphOps(threshImg); //erodes image
string colour = "Red";
trackFilteredObject(noteNum,colour,threshImg,HSV,cam);
inRange(HSV,Scalar(0,0,0),Scalar(255,255,255),threshImg);
morphOps(threshImg); // threshold = mat after erosion/dilation
colour = "yellow";
trackFilteredObject(noteNum,colour,threshImg,HSV,cam);
inRange(HSV,Scalar(0,0,0),Scalar(255,255,255),threshImg);
morphOps(threshImg);
colour = "Black";
trackFilteredObject(noteNum,colour,threshImg,HSV,cam);
inRange(HSV,Scalar(0,0,0),Scalar(255,255,255),threshImg);
morphOps(threshImg); // threshold = mat after erosion/dilation
colour = "White";
trackFilteredObject(noteNum,colour,threshImg,HSV,cam);
inRange(HSV,Scalar(0,0,0),Scalar(255,255,255),threshImg);
morphOps(threshImg); // threshold = mat after erosion/dilation
colour = "Green";
trackFilteredObject(noteNum,colour,threshImg,HSV,cam);
}
imshow("window", cam);
}
return 0;
}
At the moment the code takes quite a long time to execute a full loop (about 1.5 seconds) but i ideally need it to run as close to real time as possible for a music application.
Could anyone suggest why it takes so long to execute? Is there a better way to evaluate the colour of each square?
My class is as follows:
//Brick.h
#include <string>
using namespace std;
class Brick{
public:
Brick(void);
~Brick(void);
string getColour();
void setColour(string whatColour);
int getnoteNum();
void setnoteNum(int whatnoteNum);
private:
int noteNum;
string colour;
};
///
Brick.cpp
#include <stdio.h>
#include <Brick.h>
Brick::Brick(void){
}
Brick::~Brick(void){
}
// get/set Colour
////////////////////////////////
string Brick::getColour(){
return Brick::colour;
}
void Brick::setColour(string whatColour){
Brick::colour = whatColour;
}
// get/set Note Number
////////////////////////////////
int Brick::getnoteNum(){
return Brick::noteNum;
}
void Brick::setnoteNum(int whatnoteNum){
Brick::noteNum = whatnoteNum;
}
I will be so grateful to anyone who replies!
Thank you.
Try hard to not use erode and dilate. These operations are extremely time intensive. I'm quite confident that they are the bottleneck in your program.
There are some measures you can take:
Downscaling(or downsampling) the image. Ideally, you want the downscaled image's pixel to be of the same order of magnitude of a grid square's size.
Remove dilate and erode.
Off-topic: Bugfix. Fix the inRange() parameters used. Consult the HSV color space diagram and normalize to your space. e.g. extracting "green pixels" would correspond to inRange(HSV,Scalar(80f*255/360,0.3*255,0.3*255),Scalar(160f*255/360,255,255),threshImg);