OpenCV: process every frame - c++

Question

I want to write a cross-platform application using OpenCV for video capture. In all the examples, i've found frames from the camera are processed using the grab function and waiting for a while. And i want to process every frame in a sequence. I want to define my own callback function, which will be executed every time, when a new frame is ready to be processed (like in directshow for Windows, when you defining and putting into the graph your own filter for such purposes).
So the question is: how can i do this?

Answer 1

According to the code below, all callbacks would have to follow this definition:
IplImage* custom_callback(IplImage* frame);
This signature means the callback is going to be executed on each frame retrieved by the system. On my example, make_it_gray() allocates a new image to save the result of the grayscale conversion and returns it. This means you must free this frame later on your code. I added comments on the code about it.
Note that if your callback demands a lot of processing, the system might skip a few frames from the camera. Consider the suggestions Paul R and diverscuba23 did.
#include <stdio.h>
#include "cv.h"
#include "highgui.h"
typedef IplImage* (*callback_prototype)(IplImage*);
/*
* make_it_gray: our custom callback to convert a colored frame to its grayscale version.
* Remember that you must deallocate the returned IplImage* yourself after calling this function.
*/
IplImage* make_it_gray(IplImage* frame)
{
// Allocate space for a new image
IplImage* gray_frame = 0;
gray_frame = cvCreateImage(cvSize(frame->width, frame->height), frame->depth, 1);
if (!gray_frame)
{
fprintf(stderr, "!!! cvCreateImage failed!\n" );
return NULL;
}
cvCvtColor(frame, gray_frame, CV_RGB2GRAY);
return gray_frame;
}
/*
* process_video: retrieves frames from camera and executes a callback to do individual frame processing.
* Keep in mind that if your callback takes too much time to execute, you might loose a few frames from
* the camera.
*/
void process_video(callback_prototype custom_cb)
{
// Initialize camera
CvCapture *capture = 0;
capture = cvCaptureFromCAM(-1);
if (!capture)
{
fprintf(stderr, "!!! Cannot open initialize webcam!\n" );
return;
}
// Create a window for the video
cvNamedWindow("result", CV_WINDOW_AUTOSIZE);
IplImage* frame = 0;
char key = 0;
while (key != 27) // ESC
{
frame = cvQueryFrame(capture);
if(!frame)
{
fprintf( stderr, "!!! cvQueryFrame failed!\n" );
break;
}
// Execute callback on each frame
IplImage* processed_frame = (*custom_cb)(frame);
// Display processed frame
cvShowImage("result", processed_frame);
// Release resources
cvReleaseImage(&processed_frame);
// Exit when user press ESC
key = cvWaitKey(10);
}
// Free memory
cvDestroyWindow("result");
cvReleaseCapture(&capture);
}
int main( int argc, char **argv )
{
process_video(make_it_gray);
return 0;
}
EDIT:
I changed the code above so it prints the current framerate and performs a manual grayscale conversion. They are small tweaks on the code and I did it for education purposes so one knows how to perform operations at pixel level.
#include <stdio.h>
#include <time.h>
#include "cv.h"
#include "highgui.h"
typedef IplImage* (*callback_prototype)(IplImage*);
/*
* make_it_gray: our custom callback to convert a colored frame to its grayscale version.
* Remember that you must deallocate the returned IplImage* yourself after calling this function.
*/
IplImage* make_it_gray(IplImage* frame)
{
// New IplImage* to store the processed image
IplImage* gray_frame = 0;
// Manual grayscale conversion: ugly, but shows how to access each channel of the pixels individually
gray_frame = cvCreateImage(cvSize(frame->width, frame->height), frame->depth, frame->nChannels);
if (!gray_frame)
{
fprintf(stderr, "!!! cvCreateImage failed!\n" );
return NULL;
}
for (int i = 0; i < frame->width * frame->height * frame->nChannels; i += frame->nChannels)
{
gray_frame->imageData[i] = (frame->imageData[i] + frame->imageData[i+1] + frame->imageData[i+2])/3; //B
gray_frame->imageData[i+1] = (frame->imageData[i] + frame->imageData[i+1] + frame->imageData[i+2])/3; //G
gray_frame->imageData[i+2] = (frame->imageData[i] + frame->imageData[i+1] + frame->imageData[i+2])/3; //R
}
return gray_frame;
}
/*
* process_video: retrieves frames from camera and executes a callback to do individual frame processing.
* Keep in mind that if your callback takes too much time to execute, you might loose a few frames from
* the camera.
*/
void process_video(callback_prototype custom_cb)
{
// Initialize camera
CvCapture *capture = 0;
capture = cvCaptureFromCAM(-1);
if (!capture)
{
fprintf(stderr, "!!! Cannot open initialize webcam!\n" );
return;
}
// Create a window for the video
cvNamedWindow("result", CV_WINDOW_AUTOSIZE);
double elapsed = 0;
int last_time = 0;
int num_frames = 0;
IplImage* frame = 0;
char key = 0;
while (key != 27) // ESC
{
frame = cvQueryFrame(capture);
if(!frame)
{
fprintf( stderr, "!!! cvQueryFrame failed!\n" );
break;
}
// Calculating framerate
num_frames++;
elapsed = clock() - last_time;
int fps = 0;
if (elapsed > 1)
{
fps = floor(num_frames / (float)(1 + (float)elapsed / (float)CLOCKS_PER_SEC));
num_frames = 0;
last_time = clock() + 1 * CLOCKS_PER_SEC;
printf("FPS: %d\n", fps);
}
// Execute callback on each frame
IplImage* processed_frame = (*custom_cb)(frame);
// Display processed frame
cvShowImage("result", processed_frame);
// Release resources
cvReleaseImage(&processed_frame);
// Exit when user press ESC
key = cvWaitKey(10);
}
// Free memory
cvDestroyWindow("result");
cvReleaseCapture(&capture);
}
int main( int argc, char **argv )
{
process_video(make_it_gray);
return 0;
}

Answer 2

Quick thoughts would be to have 2 threads, the first thread is responsible for grabbing the frames and notifiy the second thread when they are available (places them in a processing queue), the second thread does all your processing in an event loop type manner.
See boost::thread and boost::signals2 as those two together should provide most of the framework (except for the queue) for what I described above.