Drawing with MouseCallback in OpenCV - c++

Using OpenCV 2.4.3, I am trying to draw a circle on top of an image centered on the mouse (x,y) position when the user is moving the mouse, and just that circle should be there once the mouse stops moving (only the original image with just one circle drawn should be shown in that moment). I thought it was going to be easy, however, I´ve been researching and trying for a couple of hours and can´t make it work the way I described.
Im attaching my code underneath. If anyone could help out I´d be really grateful.
void my_mouse_callback( int event, int x, int y, int flags, void* param );
bool moving_mouse = false;
int main()
{
const char* name = "Circle Example";
IplImage* image_circle = cvLoadImage( "../data/lena.png" );
IplImage* image = cvLoadImage( "../data/lena.png" );
namedWindow(name, CV_WINDOW_AUTOSIZE );
// Set up the callback
cvSetMouseCallback( name, my_mouse_callback, (void*) image_circle);
//Main Loop
while(cvWaitKey(15) != 27){
//If mouse is moving draw circle on top of image
if(moving_mouse){
cvShowImage(name, image_circle);
moving_mouse = false;
}
//If mouse stops moving draw original image and reset image_cicle
else{
cvShowImage(name, image);
image_circle = cvCloneImage(image);
}
}
cvReleaseImage(&image_circle);
cvReleaseImage(&image);
cvDestroyWindow(name);
return 0;
}
// Mouse callback
void my_mouse_callback( int event, int x, int y, int flags, void* param ){
switch( event ){
case CV_EVENT_MOUSEMOVE:
//Drawing a Circle
cvCircle(param,cvPoint(x,y),25,CV_RGB(0,255,0),1);
moving_mouse = true;
break;
}
}

I show you one approach to do this. I explain it later and tell you a difficulty you will have:
static int mouse_x = -1;
static int mouse_y = -1;
void my_mouse_callback( int event, int x, int y, int flags, void* param )
{
if(event == CV_EVENT_MOUSEMOVE)
{
mouse_x = x;
mouse_y = y;
}
}
int main()
{
IplImage* image;
IplImage* image_circle = NULL;
... // load image, create window, initiate callback, etc
int x = -1;
int y = -1;
while(cvWaitKey(15) != 27)
{
if(x != mouse_x || y != mouse_y)
{
x = mouse_x;
y = mouse_y;
cvReleaseImage(&image_circle);
image_circle = cvCloneImage(image);
cvCircle(image_circle,cvPoint(x,y),25,CV_RGB(0,255,0),1);
cvShowImage(name, image_circle);
}
}
... // destroy image
}
Explanation
Here, in the mouse event, you just store the coordinates of the mouse pointer. When an event takes place, the main program will check if the mouse moved to redraw the full image again. Since you want to erase the previous circle, you must copy the original image first to draw the new circle later. You could make this smarter by just copying the part of the original image that were the previous circle was, instead of the entire image.
Problem
A problem when doing something like this in OpenCV is that you can't detect when the mouse goes out of your window, so that you will always have a circle drawn in your image. I don't think you can solve this just with OpenCV, since I don't think there is a kind of MOUSE_OUT event. You would need to look for some Qt callback or system function.

Related

OpenCV stop rendering while mouse moving

I have a simple window that contains simple black image with small solid circle inside it. I have wrote a simple code to be able to drag and drop this circle. I could do it correctly. Inside the mouse_event function:
void on_mouse_event(int event_type, int x, int y, int flags, void*){
if (event_type == cv::EVENT_RBUTTONDOWN){
//Catch the circle
}
else if (event_type == cv::EVENT_MOUSEMOVE){
//Release the point
}
else if (event_type == cv::EVENT_MOUSEMOVE){
//Change circle position according to curser moving
//re draw the circle again
//show the new image
}
}
The main function:
while (true){
//show image code (simple cv::imshow);
if (cv::waitKey(1) == 27){
break;
}
}
The problem is that if I drag the circle and start to move fast, the image will not change till I stop. However, if I go slowly it will change according to the move. What is the reason of this problem?
P.S I am not in doubt of slow hardware at all. I am working on workstation and I am mentoring the processor utilization and just one of its 8 core reach around 50% and the memory is almost free.
I am using Windows 10 if it helps.
you could test the following code.(adapted from opencv_annotation.cpp)
#include <opencv2/highgui.hpp>
#include <opencv2/imgproc.hpp>
using namespace std;
using namespace cv;
// Function prototypes
void on_mouse(int, int, int, int, void*);
// Public parameters
Mat image(600, 800, CV_8UC3, Scalar(220, 220, 220));
Mat current_view;
int circle_center_x = image.cols / 2, circle_center_y = image.rows / 2, radius = 40;
bool dragging = false;
const string window_name = "OpenCV Mouse Event Demo";
void on_mouse(int event, int x, int y, int, void *)
{
// Action when left button is clicked
if (event == EVENT_LBUTTONDOWN & (abs(circle_center_x - x) < 20) & (abs(circle_center_y - y) < 20))
{
dragging = true;
}
if (event == EVENT_LBUTTONUP)
{
dragging = false;
}
// Action when mouse is moving
if ((event == EVENT_MOUSEMOVE) && dragging)
{
image.copyTo(current_view);
circle_center_x = x;
circle_center_y = y;
circle(current_view, Point(circle_center_x, circle_center_y), radius, Scalar(255, 0, 0), 5);
imshow(window_name, current_view);
}
}
int main(int argc, const char** argv)
{
// Init window interface and couple mouse actions
namedWindow(window_name, WINDOW_AUTOSIZE);
setMouseCallback(window_name, on_mouse);
image.copyTo(current_view);
circle(current_view, Point(circle_center_x, circle_center_y), radius, Scalar(255, 0, 0), 5);
imshow(window_name, current_view);
int key_pressed = 0;
do
{
// Keys for processing
// Based on the universal ASCII code of the keystroke: http://www.asciitable.com/
// <SPACE> = 32 add circle to current image
// <ESC> = 27 exit program
key_pressed = 0xFF & waitKey(0);
if (key_pressed==32)
{
// draw a circle on the image
circle(image, Point(circle_center_x, circle_center_y), radius, Scalar(0, 0, 255), -1);
image.copyTo(current_view);
circle(current_view, Point(circle_center_x, circle_center_y), radius, Scalar(255, 0, 0), 5);
imshow(window_name, current_view);
}
}
// Continue as long as the <ESC> key has not been pressed
while (key_pressed != 27);
// Close down the window
destroyWindow(window_name);
return 0;
}

SDL C Program freezes on sdl_blitsurface

I'm having an issue with a program I'm working on. Occasionally, it will just freeze. No errors or anything.
The game is a multiplayer game where you fly a ship around. Pictures of other players and powerups move in and out of view depending on your location. For the most part, it works great, but under certain circumstances, it locks up.
I've tracked it down to when it BLITs one surface onto another. (SDL_BlitSurface).
If I comment out the single line of code where it blits (SDL_BlitSurface), and replace the graphic with a simple circle, it'll never freeze under any circumstances. But, comment out the circle and replace it with blitting the graphic again, and it'll randomly freeze. The frustrating part is, sometimes it will, sometimes it won't. Sometimes the graphic will sit on screen for a few moments and then freeze, sometimes it'll freeze the moment it shows up. Sometimes, it won't freeze at all. I simply cannot track it down to anything in particular.
I have ample amount of code that checks for NULL surfaces and it doesn't seem to stop it.
I also have it set up to output information about all the graphics to a file (such as width, height, location in memory, x, y, etc) and nothing seems out of the ordinary.
My main questions are, what about surfaces can cause SDL_BlitSurface to freeze? And what other checks can I add for surfaces to make sure it doesn't try to blit bad surfaces?
The code is too long to list, but here is how it works:
class Player
{
Player();
int x;
int y;
int xvel;
int yvel;
SDL_Surface *DrawScreen;
SDL_Surface *ShipPic;
void check_player_dist();
void check_powerup_dist();
void update();
};
class PowerUp
{
int x;
int y;
int type;
SDL_Surface *Powerup_Pic;
};
Player::Player()
{
Apply_Surface(0, 0, PlayerShipPics, ShipPic);
}
Player::Update(Player p[], PowerUp pu[])
{
x += xvel;
y += yvel;
for (int i = 0; i < Num_Players; i++)
{
if (check_on_screen(p[i].x, p[i].y) == true)
{
Apply_Surface(x - p[i].x, y - p[i].y, p[i].ShipPic, DrawScreen);
}
}
for (int i = 0; i < Num_PowerUps; i++)
{
if (check_on_screen(pu[i].x, pu[i].y) == true)
{
Apply_Surface(x - pu[i].x, y - pu[i].y, pu[i].Pic, DrawScreen);
}
}
}
int main()
{
SDL_Surface *Screen;
Player players[4];
PowerUp powerups[200];
Num_Players = 4;
Num_PowerUps = 200;
while (quit == false)
{
for (int i = 0; i < Num_Players; i++)
{
players[i].update(players, powerups);
switch (i)
{
case 0: ScreenX = 0; ScreenY = 0; break;
case 1: ScreenX = ScreenWid / 2; ScreenY = 0; break;
case 2: ScreenX = 0; ScreenY = ScreenHigh / 2; break;
case 3: ScreenX = ScreenWid / 2; ScreenY = ScreenHigh / 2; break;
}
Apply_Surface (ScreenX, ScreenY, players[i].DrawScreen, Screen);
}
if (SDL_Flip(Screen) == -1)
{
return -1;
}
}
}
void Apply_Surface (int x, int y, SDL_Surface* Source, SDL_Surface* Destination, SDL_Rect* Clip)
{
SDL_Rect Offset;
Offset.x = x;
Offset.y = y;
if ((Source != NULL) && (Destination != NULL))
{
SDL_BlitSurface (Source, Clip, Destination, &Offset );
}
}
I've noticed it generally freezes when two or more players are near each other and it tries to draw the same power-up on both of their screens. But again...not always!
Well, I figured out what it was.
I was using the SDL_GFX library along with my game. Many of the images were created using rotozoomSurface(), which is a function of SDL_GFX.
Turns out there's a bug in it where, under certain circumstances that I don't know, it'll create a bad surface that will work "most" of the time, but under the right conditions, will crash. Such as, being placed at a particular x & y coordinate on the screen. (Don't know for sure). The rotated/zoomed images would work about 95% of the time, so it was very difficult to pin point what the issue was.
The work around was, when the image was created, just SDL_BlitSurface() it onto another surface under controlled conditions, such as putting it at coordinates (0, 0). Then, delete the rotated and zoomed surface, and just use the new "safe" surface.
Works great after that.
Hopefully this will help anyone who's using SDL_GFX and cannot figure out why their program is crashing.
Example:
Before:
SDL_Surface *original = SDL_CreateRGBSurface(SDL_SWSURFACE, Ship_Width, Ship_Height, Screen_BPP, 0, 0, 0, 0);
Apply_Surface(0, 0, ShipsPic, original, &bounds);
SDL_Surface *finished = rotozoomSurface(original, pic_angle, zoom, SMOOTHING_ON);
SDL_FreeSurface(original);
return finished;
After (fixed):
SDL_Surface *original = SDL_CreateRGBSurface(SDL_SWSURFACE, Ship_Width, Ship_Height, Screen_BPP, 0, 0, 0, 0);
Apply_Surface(0, 0, ShipsPic, original, &bounds);
SDL_Surface *temp = rotozoomSurface(original, pic_angle, zoom, SMOOTHING_ON);
SDL_Surface *finished = SDL_CreateRGBSurface(SDL_SWSURFACE, temp->w, temp->h, Screen_BPP, 0, 0, 0, 0);
Apply_Surface(0, 0, temp, finished);
SDL_FreeSurface(temp);
SDL_FreeSurface(original);
return finished;
And for what it's worth, the Apply_Surface() function:
void Apply_Surface (int x, int y, SDL_Surface* Source, SDL_Surface* Destination, SDL_Rect* Clip)
{
SDL_Rect Offset;
Offset.x = x;
Offset.y = y;
if ((Source != NULL) && (Destination != NULL))
{
SDL_BlitSurface (Source, Clip, Destination, &Offset );
}
}
There's not really enough information to figure out what exactly is going on. Computers don't like to do things "sometimes," they either do them or not, so it leads me to believe that maybe there's some variable that's doing something it shouldn't.
Just in case, what does your Apply_Surface() function look like? I assume that's where you're doing your actual blitting, and if that's where you're having your problems, that would be useful for those of us trying to figure out your dilemma.

Picking contours using the mouse?

My project takes a bitmap snapshot of the client area of a given window, and converts this data into an IplImage instance. Then after a grayscale conversion, threshing etc, bounding boxes are drawn around any contours which exceed a given minimum size (area volume).
The result is then shown within a cvNamedWindow
All I need to do now is allow the user to click within these rects to effectively "choose" this contour, so that the application can then extract the subrect as a new image and save it to disk.
How can this be achieved using OpenCV in C++?
If you store your bounding boxes you can check in a for loop in a mouse event handler if a box is clicked and which box is clicked. The code for creating a mouse event:
cvNamedWindow("MyWindow", CV_WINDOW_NORMAL);
cvSetMouseCallback("MyWindow", mouseEvent, 0);
imshow("MyWindow", image);
void mouseEvent(int evt, int x, int y, int flags, void *param) {
if (evt == CV_EVENT_LBUTTONDOWN) {
printf("%d %d\n", x, y);
}
}
You can probably find out yourself how to check if these coordinates are within a bounding box. I would also recommend to use the C++ API, as stated in the comments, as it is much easier once you get the hang of it.
In case anyone else was interested on how to actually extract the chosen image,
here's how I was able to do it:
inline
IplImage* getSubImage(IplImage *image, CvRect region)
{
cvSetImageROI(image, region);
IplImage *imgRet = cvCreateImage( cvSize(region.width, region.height), image->depth, image->nChannels );
cvCopy(image, imgRet);
cvResetImageROI(image);
return imgRet;
}
inline
bool pointInRect(const int x, const int y, const CvRect& r)
{
return (
(x > r.x) && (x < (r.x + r.width)) &&
(y > r.y) && (y < (r.y + r.height))
);
}
void onMouseEvent(int evt, int x, int y, int flags, void *param)
{
if (evt == CV_EVENT_LBUTTONDOWN) {
// boundingBoxes is declared as a vector of CvRects and
// filled in the main loop using cvBoundingRect(contour)
for(UINT i = 0; i < boundingBoxes.size(); i++)
{
CvRect rect = boundingBoxes[i].rect;
if( pointInRect(x, y, rect) )
{
IplImage* img = getSubImage(imgSource, rect);
// Do whatever you want with the sub-image here
cvNamedWindow("Selection");
cvShowImage("Selection", img);
cvReleaseImage(&img);
break;
}
}
}
}

Displaying a moving object in sdl

I have a problem with my SDL program. My goal is to make a dot move along a line. I have all the coordinates saved in a data file. So I just wanted to read them from the file and display the dot at the right position.
The dot class (which is named linefollower) looks like this.
class Linefollower
{
private:
int x, y;
char orientation;
public:
//Initializes the variables
Linefollower();
void set(int m_x, int m_y, char m_orietnation);
void show();
char get_orientation();
};
Linefollower::Linefollower()
{
x = 0;
y = 0;
orientation = 'E';
}
void Linefollower::set(int m_x, int m_y, char m_orientation)
{
x = m_x;
y = m_y;
orientation = m_orientation;
}
void Linefollower::show()
{
//Show the linefollower
apply_surface(x, y, linefollower, screen );
}
char Linefollower::get_orientation()
{
return orientation;
}
The apply_surface function.
void apply_surface( int x, int y, SDL_Surface * source, SDL_Surface* destination)
{
//Temporary rectangle to hold the offsets
SDL_Rect offset;
//Get the offsets
offset.x = x;
offset.y = y;
//Blit the surface
SDL_BlitSurface( source, NULL, destination, &offset);
}
The loop which ought to display the animation looks like this.
//While the user hasn't quit
while( quit == false )
{
//Apply the surface to the screen
apply_surface( 0, 0, image, screen );
fin.read((char*) &my_linefollower, sizeof my_linefollower);
if(my_linefollower.get_orientation() == 'Q')
break;
my_linefollower.show();
//Upadate the screen
if( SDL_Flip( screen ) == -1 )
{
return 1;
}
SDL_Delay(200);
}
Now I was expecting, that I get a moving dot on the screen, but the only thing which I got is the background (image) for a few seconds untill the if(my_linefollower.get_orientation() == 'Q')
break; was true. What do I do wrong?
PS: I guess it is worth noticing that I am a beginner in SDL and I took most of the code from a tutorial. Learning it exactly would be a waste of time for me, since it is unlikely that I am going to use it again any time soon.
First, you should change your offset in apply_surface to something like this:
SDL_Rect offset = { x, y, 0, 0 };
SDL_Rect doesn't have a constructor to set your members to 0 by default, so you get uninitialized memory for your width and height.
Also, you should check what linefollower contains, if it's a valid SDL_Surface. Removing your file reading code and manually controlling a Linefollower will allow you to easily find where the error comes from.
Use a debugger to validate your x and y coordinates.
Other than that, you code should work, although your window will be unresponsive because you're not pumping events through SDL_PollEvent.

Draw on webcam using OpenCV

I want to draw/paint on a webcam screen using OpenCV. Since I'm reading from a cam, the frames are constantly changing, so I'm trying to figure out a way to keep or save the drawing on the current frame and use it for the next frame. The code below allows you to draw on the screen but when it gets the next frame, the drawing is gone and it starts over.
Could someone please help me ... Thanks.
CvCapture *input;
input = cvCaptureFromCAM( 0 );
cvSetMouseCallback("Demo",&on_mouse, 0);
for(;;)
{
frame = cvQueryFrame(input);
if(!image)
{
image = cvCreateImage( cvSize(frame->width, frame->height), IPL_DEPTH_8U, 3);
screenBuffer = cvCreateImage( cvSize(frame->width, frame->height), IPL_DEPTH_8U, 3);
}
cvCopy(frame, image, 0);
if(drawing) //drawing is a global variable
{
cvCircle(image, cvPoint(last_x,last_y), 10,CV_RGB(red,green,blue), -1, CV_AA, 0);
cvCopy(image, screenBuffer, 0);
}
cvShowImage( "Demo", screenBuffer );
}
void on_mouse( int event, int x, int y, int flags, void* param )
{
last_x = x;
last_y = y;
if(event==CV_EVENT_LBUTTONDOWN)
{
drawing = 1;
}
}
Draw into a separate image and then cvAdd() that to the video image immediately before dispalying it
I will not go into all the details why your approach is bad, but keep in mind that creating 2 extra frames for drawing is a little bit too much.
It's important that you realize that all this kinky stuff is being done on the same thread used to capture new frames. This means what exactly? It means that the extra code you are adding inside the loop will slow the process of capturing and displaying new frames. In other words, you are sabotaging yourself by lowering the framerate of your application. If you don't care, it's ok. If you do, my tip for you is that you stack the captured frames into a buffer and have another thread read, process and display them.
Ok, so you REALLY want to draw over the window that's displaying the captured frames. Well, the obvious thing you can't do (and you discovered this yourself) is that the drawing cannot be made on the captured frame because the frame it's replaced with new data on every loop. So what do you do? You create a 2nd frame to do the drawing. Let's call it the drawing_frame.
The only thing that will be on the drawing_frame are the circles that will appear when the mouse moves over the window, when the LBUTTON of the mouse is clicked (a 2nd click switches between ON/OFF).
After the drawing of the circle occurs, the drawing_frame is overlayed on top on the frame captured by the camera. This process is a little expensive on the CPU, and since we are doing it in the main thread of the application, it will also decrease the framerate.
I strongly suggest that everyone interested in adding/merging/overlaying transparent frames with OpenCV take a look at Transparent image overlays in OpenCV.
By the way, I'm using cvCaptureFromCAM(-1) becouse I'm on Linux. You probably should change that to whatever works for you. According to your post it's cvCaptureFromCAM(0).
#include <stdio.h>
#include <cv.h>
#include <highgui.h>
int drawing = 0;
int last_x = 0;
int last_y = 0;
void on_mouse(int event, int x, int y, int flags, void* param)
{
last_x = x;
last_y = y;
if (event == CV_EVENT_LBUTTONDOWN)
{
// switches between On and Off
if (drawing)
drawing = 0;
else
drawing = 1;
}
}
int main()
{
CvCapture* capture = NULL;
if ((capture = cvCaptureFromCAM(-1)) == NULL)
{
fprintf(stderr, "ERROR: capture is NULL \n");
return -1;
}
cvNamedWindow("mywindow", CV_WINDOW_AUTOSIZE);
cvQueryFrame(capture); // Sometimes needed to get correct data
cvSetMouseCallback("mywindow",&on_mouse, 0);
IplImage* frame = NULL;
IplImage* drawing_frame = NULL;
while (1)
{
if ((frame = cvQueryFrame(capture)) == NULL)
{
fprintf( stderr, "ERROR: cvQueryFrame failed\n");
break;
}
if (frame == NULL)
{
fprintf( stderr, "WARNING: cvQueryFrame returned NULL, sleeping..\n");
usleep(100000);
continue;
}
if (!drawing_frame) // This frame is created only once
{
drawing_frame = cvCreateImage(cvSize(frame->width, frame->height), frame->depth, frame->nChannels);
cvZero(drawing_frame);
}
if (drawing)
{
cvCircle(drawing_frame, cvPoint(last_x,last_y), 10,CV_RGB(0, 255, 0), -1, CV_AA, 0);
// For overlaying (copying transparent images) in OpenCV
// http://www.aishack.in/2010/07/transparent-image-overlays-in-opencv/
for (int x = 0; x < frame->width; x++)
{
for (int y = 0; y < frame->height; y++)
{
CvScalar source = cvGet2D(frame, y, x);
CvScalar over = cvGet2D(drawing_frame, y, x);
CvScalar merged;
CvScalar S = { 1,1,1,1 };
CvScalar D = { 1,1,1,1 };
for(int i = 0; i < 4; i++)
merged.val[i] = (S.val[i] * source.val[i] + D.val[i] * over.val[i]);
cvSet2D(frame, y, x, merged);
}
}
}
cvShowImage("mywindow", frame);
int key = cvWaitKey(10);
if (key == 113) // q was pressed on the keyboard
break;
}
cvReleaseImage(&frame);
cvReleaseImage(&drawing_frame);
cvReleaseCapture(&capture);
cvDestroyWindow("mywindow");
return 0;
}
You usually will have problems of adding images (they will eventually saturate), so I guess thats why you start over. I see you have color images... if you use more powerful stuff like OpenGL for your drawing you could use the overlay for your drawings. Otherwise check this out:
http://aishack.in/tutorials/transparent-image-overlays-in-opencv/