How to automate Drag&Drop using SendInput - c++

I want to test a flash application running on browser. I need to drag pieces around in the application. I want to do this by simulating mouse movements using windows api.
Here is my navie solution to simulate a mouse drag:
void MoveMouse(LPINPUT input, int x, int y) {
input->type = INPUT_MOUSE;
input->mi.dx = x*(65536/GetSystemMetrics(SM_CXSCREEN));
input->mi.dy = y*(65536/GetSystemMetrics(SM_CYSCREEN));
input->mi.dwFlags = MOUSEEVENTF_ABSOLUTE | MOUSEEVENTF_MOVE;
}
void LeftDownMouse(LPINPUT input, int x, int y) {
input->type = INPUT_MOUSE;
input->mi.dx = x*(65536/GetSystemMetrics(SM_CXSCREEN));
input->mi.dy = y*(65536/GetSystemMetrics(SM_CYSCREEN));
input->mi.dwFlags = MOUSEEVENTF_LEFTDOWN;
}
void LeftUpMouse(LPINPUT input, int x, int y) {
input->type = INPUT_MOUSE;
input->mi.dx = x*(65536/GetSystemMetrics(SM_CXSCREEN));
input->mi.dy = y*(65536/GetSystemMetrics(SM_CYSCREEN));
input->mi.dwFlags = MOUSEEVENTF_LEFTUP;
}
void DragMouse(LPINPUT inputs, int startX, int startY, int dropX, int dropY) {
MoveMouse(&inputs[0], startX, startY);
LeftDownMouse(&inputs[1], startX, startY);
LeftUpMouse(&inputs[2], startX, startY);
LeftDownMouse(&inputs[3], startX, startY);
MoveMouse(&inputs[4], dropX, dropY);
LeftUpMouse(&inputs[5], dropX, dropY);
}
int _tmain(int argc, _TCHAR* argv[])
{
INPUT inputs [6] = {};
// Assume there is a maximized window, mouse drags the window from 40, 10 to 100,100 coordinates.
DragMouse(inputs, 40, 10, 100, 100);
SendInput(6, inputs, sizeof(INPUT));
Sleep(1000);
return 0;
}
This code mostly doesn't work, sometimes works, it has undefined behaviour. What could be the problem? How can i solve this?

I had a similar problem with drag and drop, although my results were consistent in that the mouse was moving but not the item being dragged.
Your MoveMouse function is missing the MOUSEEVENTF_LEFTDOWN flag. Try input->mi.dwFlags = MOUSEEVENTF_ABSOLUTE | MOUSEEVENTF_MOVE | MOUSEEVENTF_LEFTDOWN;
Without the flag setting you are just asking the mouse to move without the left mouse button being down.
Edit
I just checked my implementation and I don't set coordinates when pressing or releasing the mouse buttons. Only when moving.
As for variance in results, you may stop these by zeroing out the INPUT before setting fields.

Related

How to change a mouse cursor image for all windows?

I am trying to change a mouse cursor for the entire system (for each window on a display) with the help of XCursor and XInput2.
In the code below, the cursor changes. However, only for browser window =>
When I focus on other windows, the cursor is still default.
As I understand, XTranslateCoordinates() function has to return the window which is under the cursor and then I just DefineCursor for this window. It also does not work if I replace XTranslateCoordinates by XGetInputFocus(d, &w, &revert_to).
Probably I do not understand the way cursors are going to be changed on windows. Would be extremely glad for helping!
int main(int argc, char **argv)
{
Display *display;
Window root_window;
display = XOpenDisplay(0);
root_window = XRootWindow(display, 0);
Cursor cursorDefault;
Cursor CursorClickR;
Cursor CursorClickL;
CreateCursor("1.png", cursorDefault, display);
unsigned char mask_bytes[(XI_LASTEVENT + 7) / 8] = {0}; /* must be zeroed! */
XISetMask(mask_bytes, XI_RawMotion);
XIEventMask evmasks[1];
evmasks[0].deviceid = XIAllMasterDevices;
evmasks[0].mask_len = sizeof(mask_bytes);
evmasks[0].mask = mask_bytes;
XISelectEvents(display, root_window, evmasks, 1);
XEvent xevent;
while (1)
{
XNextEvent(display, &xevent);
XGetEventData(display, &xevent.xcookie);
Window root_return, child_return;
int root_x_return, root_y_return;
int win_x_return, win_y_return;
unsigned int mask_return;
XQueryPointer(display, root_window, &root_return, &child_return,
&root_x_return, &root_y_return,
&win_x_return, &win_y_return,
&mask_return);
int local_x, local_y;
XTranslateCoordinates(display, root_window, child_return,
root_x_return, root_y_return,
&local_x, &local_y, &child_return);
if (child_return)
XDefineCursor(display, child_return, cursorDefault);
}
XCloseDisplay(display);
return 0;
}

FLTK: Clearing a graph when drawing another

I wrote a simple FLTK program to draw a circle when clicking on the "Draw Circle" button and to draw a line when clicking on the "Draw Line" button. I supposed to have only one graph. But I got two graphs in the panel. I want only one showing and the other disappearing. The following is the code:
#include <FL/Fl.H>
#include <FL/Fl_Button.H>
#include <FL/Fl_Double_Window.H>
#include <FL/fl_draw.H>
#include <FL/Fl_Box.H>
using namespace std;
int flag = 0;
class Drawing : public Fl_Box {
void draw() {
fl_color(255, 0, 0);
int x, y, x1, y1;
if (flag == 1) {
double radius = 100;
x = (int)(w() / 2);
y = (int)(h() / 2);
fl_circle(x, y, radius);
}
else if (flag == -1) {
x = (int)(w() / 4);
y = (int)(h() / 4);
x1 = (int)(w() *3/ 4);
y1 = (int)(h() *3/ 4);
fl_line(x, y, x1, y1);
}
}
public:
Drawing(int X, int Y, int W, int H) : Fl_Box(X, Y, W, H) {}
};
Drawing* d;
void circle_cb(Fl_Widget*, void*) {
flag = 1;
fl_overlay_clear();
d->redraw();
} // end sumbit_cb
void line_cb(Fl_Widget*, void*) {
flag = -1;
fl_overlay_clear();
d->redraw();
} // end clear_cb
int main(int argc, char** argv) {
Fl_Window* window = new Fl_Window(600, 550); // create a window, originally(400,400)
Drawing dr(0, 0, 600, 600);
d = &dr;
Fl_Button *b, *c;
b = new Fl_Button(150, 80, 100, 25, "&Draw Circle");
b->callback(circle_cb);
c = new Fl_Button(350, 80, 100, 25, "&Draw Line");
c->callback(line_cb);
window->end(); //show the window
window->show(argc, argv);
return Fl::run();
}
I have used fl_overlay_clear() to clear graph. However it is not working. Any help will be appreciated.
There are several issues that need to be fixed in your program, but first of all using the draw() method as you did is basically correct. However, using fl_overlay_clear(); is useless, you can remove it.
My solution: your widget doesn't have a solid background (boxtype), i.e. your draw method draws over the background over and over again w/o clearing it. There are several ways to solve this, but if you want to learn what happens, try this first: add window->resizable(window); before window->show(argc, argv);, run the program again and resize the window. You'll notice that the previous drawing disappears and only one drawing stays. That's because the background is cleared when you resize the widget.
Next step: add a solid boxtype:
d = &dr;
d->box(FL_DOWN_BOX);
and add Fl_Box::draw(); right at the beginning of your draw() method.
If you do that you may notice that your button(s) disappear when you click one of them - because your buttons are inside the area of your Drawing. The last thing(s) I fixed was to correct the coordinates of buttons and to enlarge the window (it was too small anyway to cover the entire Drawing). Here's my complete result:
#include <FL/Fl.H>
#include <FL/Fl_Button.H>
#include <FL/Fl_Double_Window.H>
#include <FL/fl_draw.H>
#include <FL/Fl_Box.H>
using namespace std;
int flag = 0;
class Drawing : public Fl_Box {
void draw() {
Fl_Box::draw();
fl_color(255, 0, 0);
int x, y, x1, y1;
if (flag == 1) {
double radius = 100;
x = (int)(w() / 2);
y = (int)(h() / 2);
fl_circle(x, y, radius);
} else if (flag == -1) {
x = (int)(w() / 4);
y = (int)(h() / 4);
x1 = (int)(w() * 3 / 4);
y1 = (int)(h() * 3 / 4);
fl_line(x, y, x1, y1);
}
}
public:
Drawing(int X, int Y, int W, int H)
: Fl_Box(X, Y, W, H) {}
};
Drawing *d;
void circle_cb(Fl_Widget *, void *) {
flag = 1;
// fl_overlay_clear(); // not useful
d->redraw();
} // end sumbit_cb
void line_cb(Fl_Widget *, void *) {
flag = -1;
// fl_overlay_clear(); // not useful
d->redraw();
} // end clear_cb
int main(int argc, char **argv) {
Fl_Window *window = new Fl_Window(600, 660); // create a window, originally(400,400)
Drawing dr(0, 60, 600, 600); // FIXED
d = &dr;
d->box(FL_DOWN_BOX); // ADDED
Fl_Button *b, *c;
b = new Fl_Button(150, 20, 100, 25, "&Draw Circle"); // FIXED
b->callback(circle_cb);
c = new Fl_Button(350, 20, 100, 25, "&Draw Line"); // FIXED
c->callback(line_cb);
window->end(); // show the window
window->resizable(window); // ADDED
window->show(argc, argv);
return Fl::run();
}
I believe this does what you want.
PS: the official FLTK support forum can be found on our website https://www.fltk.org/ and the direct link to the user forum (Google Groups) is https://groups.google.com/g/fltkgeneral
Just a quick addition to what Albrecht put so perfectly: FLTK drawing coordinates are relative to the window, not relative to the widget. You probably want to offset your drawing by the x() and y() coordinates of your widget.
In your handle() methods line_cb() , circle_cb() should call window()->make_current() and then fl_overlay_rect() after FL_DRAG events, and should call fl_overlay_clear() after a FL_RELEASE event. Refer for more details

How to automate mouse actions

I need to automate some mouse actions.
I need to do
mousemove1, lbuttondown1, wait1, mousemove1, lbuttonup1, wait1,
mousemove2, lbuttondown2, wait2, mousemove2, lbuttonup2, wait2,
...
The actions have to work regarding screen coordinates. The window which have to accept an event is the top window at this point.
There is a file with data.
For example
500 450 1000 500 300 2000
600 450 1000 600 300 5000
What did I try to do
#include <fstream>
#include <vector>
#include <windows.h>
struct A
{
POINT point1;
unsigned sleep1;
POINT point2;
unsigned sleep2;
A() { point1.x = point1.y = sleep1 = point2.x = point2.y = sleep2 = 0; }
};
void f(const A &a)
{
mouse_event(MOUSEEVENTF_LEFTDOWN, a.point1.x, a.point1.y, 0, 0);
mouse_event(MOUSEEVENTF_MOVE, a.point1.x, a.point1.y, 0, 0);
Sleep(a.sleep1);
mouse_event(MOUSEEVENTF_LEFTUP, a.point2.x, a.point2.y, 0, 0);
mouse_event(MOUSEEVENTF_MOVE, a.point2.x, a.point2.y, 0, 0);
Sleep(a.sleep2);
}
int main()
{
std::vector<A> as;
std::ifstream fin("params.txt");
if (fin) {
A a;
while (fin.good()) {
fin >> a.point1.x;
fin >> a.point1.y;
fin >> a.sleep1;
fin >> a.point2.x;
fin >> a.point2.y;
fin >> a.sleep2;
if (fin.eof()) {
break;
}
as.push_back(a);
}
}
for (;;) {
for (const A &a : as) {
f(a);
}
}
}
Something is happening but I can not understand what is and where is a mistake.
A problem with your code is that you are using mouse_event with screen coordinates rather than normalized absolute coordinates. Normalized absolute coordinates always range between (0,0) in the top-left corner to (65535,65535) in the bottom-right corner, no matter what the desktop size happens to be.
The MouseTo function in the example below accepts screen coordinates as inputs, then uses the dekstop window's size to convert to normalized absolute coordinates. This example uses SendInput, which supersedes mouse_event, but they both use the same coordinates. I'm not sure if mouse_event can take the MOUSEEVENTF_VIRTUALDESK flag, but this is for supporting multi-monitor desktops.
If you wish build this example, start with a new Win32 Console application.
#include <Windows.h>
#include <cmath>
void MouseTo(int x, int y) {
RECT desktop_rect;
GetClientRect(GetDesktopWindow(), &desktop_rect);
INPUT input = {0};
input.type = INPUT_MOUSE;
input.mi.dwFlags =
MOUSEEVENTF_ABSOLUTE | MOUSEEVENTF_VIRTUALDESK | MOUSEEVENTF_MOVE;
input.mi.dx = x * 65536 / desktop_rect.right;
input.mi.dy = y * 65536 / desktop_rect.bottom;
SendInput(1, &input, sizeof(input));
}
void MouseLButton(bool tf_down_up) {
INPUT input = {0};
input.type = INPUT_MOUSE;
input.mi.dwFlags = tf_down_up ? MOUSEEVENTF_LEFTDOWN : MOUSEEVENTF_LEFTUP;
SendInput(1, &input, sizeof(input));
}
void MouseLButtonDown() { MouseLButton(true); }
void MouseLButtonUp() { MouseLButton(false); }
void AnimatedDrag(const POINT& from, const POINT& to) {
static const double iteration_dist = 20;
static const DWORD iteration_delay_ms = 1;
const double dx = to.x - from.x;
const double dy = to.y - from.y;
const double dist = sqrt(dx*dx + dy*dy);
const int count = static_cast<int>(dist / iteration_dist);
MouseTo(from.x, from.y);
MouseLButtonDown();
for(int i=1; i<count; ++i) {
const int x = from.x + static_cast<int>(dx * i / count);
const int y = from.y + static_cast<int>(dy * i / count);
MouseTo(x, y);
Sleep(iteration_delay_ms);
}
MouseTo(to.x, to.y);
MouseLButtonUp();
}
int main() {
// minimize console window
ShowWindow(GetConsoleWindow(), SW_SHOWMINNOACTIVE);
Sleep(500);
// Drag whatever is at the window coordinates in "from" to "to"
const POINT from = {300, 100};
const POINT to = {900, 600};
AnimatedDrag(from, to);
}
Christopher's answer should suffice, but might be a little intimidating to anyone not well-versed with C++, and just trying to hack together a click-utility. This should be easy enough to hack away at for most newbies.
Pardon the use of macros; I'm using them to make the intent of the code a little more English-friendly.
It should right-click on your primary display (unless you changed the X coordinate line as-commented) then shift a few pixels over and left-click to close the Right-click menu prompt, if one was created. You can see what else is available on MSDN for your own custom requirements.
I kept click / unclick / move as seperate actions, so things like drag & drop should be fairly intuitive to perform when starting with all the right ingredients.
#include <Windows.h>
// Uses absolute coords where the primary display starts at 0,0
// That works well with enumerated monitors structures and their reported coords.
#define QUEUE_MV_MOUSE ip.mi.dwFlags = MOUSEEVENTF_ABSOLUTE | MOUSEEVENTF_MOVE;
#define QUEUE_RC_START_MOUSE ip.mi.dwFlags = MOUSEEVENTF_RIGHTDOWN;
#define QUEUE_RC_END_MOUSE ip.mi.dwFlags = MOUSEEVENTF_RIGHTUP;
#define QUEUE_LC_START_MOUSE ip.mi.dwFlags = MOUSEEVENTF_LEFTDOWN;
#define QUEUE_LC_END_MOUSE ip.mi.dwFlags = MOUSEEVENTF_LEFTUP;
#define SEND_IT SendInput(1, &ip, sizeof(ip));
#define VIRTUAL_X_MODIFIER (65536 / GetSystemMetrics(SM_CXSCREEN));
#define VIRTUAL_Y_MODIFIER (65536 / GetSystemMetrics(SM_CYSCREEN));
int main()
{
INPUT ip;
ip.type = INPUT_MOUSE;
ip.mi.mouseData = 0;
// Change 500 to -500 for a left-hand extended display.
ip.mi.dx = 500 * VIRTUAL_X_MODIFIER;
ip.mi.dy = 1000 * VIRTUAL_Y_MODIFIER;
// Un-comment this Sleep timer if you're debugging in an IDE and need a quick pause.
// Sleep(500);
QUEUE_MV_MOUSE;
SEND_IT;
// Various users advise brief Sleep pauses between queued mouse and keyboard events.
// 500 milliseconds is probably overkill for your automation requirements.
Sleep(500);
QUEUE_RC_START_MOUSE;
SEND_IT;
Sleep(500);
QUEUE_RC_END_MOUSE;
SEND_IT;
Sleep(500);
ip.mi.dx -= 10 * VIRTUAL_X_MODIFIER;
ip.mi.dx -= 10 * VIRTUAL_Y_MODIFIER;
QUEUE_MV_MOUSE;
SEND_IT;
Sleep(500);
QUEUE_LC_START_MOUSE;
SEND_IT;
Sleep(500);
QUEUE_LC_END_MOUSE;
SEND_IT;
return 0;
}

How to create a full screen window on the current monitor with GLFW

Creating a window with GLFW3 is done using glfwCreateWindow:
GLFWwindow* glfwCreateWindow ( int width,
int height,
const char *title,
GLFWmonitor *monitor,
GLFWwindow *share
)
If the monitor parameter is not NULL, the window is created in full screen mode on the given monitor. One can receive the primary monitor by calling glfwGetPrimaryMonitor, or chose one of the results of glfwGetMonitors. But how can I create a full screen window on the current monitor, i.e. the monitor the window is currently running in windowed mode? There seems to be no way to receive the currently used monitor. There is glfwGetWindowMonitor, but it only returns the monitor in full screen mode, NULL in windowed mode.
You can find the current monitor with glfwGetWindowPos/glfwGetWindowSize.
This function returns the monitor that contains the greater window area.
static int mini(int x, int y)
{
return x < y ? x : y;
}
static int maxi(int x, int y)
{
return x > y ? x : y;
}
GLFWmonitor* get_current_monitor(GLFWwindow *window)
{
int nmonitors, i;
int wx, wy, ww, wh;
int mx, my, mw, mh;
int overlap, bestoverlap;
GLFWmonitor *bestmonitor;
GLFWmonitor **monitors;
const GLFWvidmode *mode;
bestoverlap = 0;
bestmonitor = NULL;
glfwGetWindowPos(window, &wx, &wy);
glfwGetWindowSize(window, &ww, &wh);
monitors = glfwGetMonitors(&nmonitors);
for (i = 0; i < nmonitors; i++) {
mode = glfwGetVideoMode(monitors[i]);
glfwGetMonitorPos(monitors[i], &mx, &my);
mw = mode->width;
mh = mode->height;
overlap =
maxi(0, mini(wx + ww, mx + mw) - maxi(wx, mx)) *
maxi(0, mini(wy + wh, my + mh) - maxi(wy, my));
if (bestoverlap < overlap) {
bestoverlap = overlap;
bestmonitor = monitors[i];
}
}
return bestmonitor;
}
After discussion on IRC it seems that it is not possible to retrieve the currently active monitor (as in the monitor the window is currently drawn on) with GLFW. Therefore it is not possible to create a full screen window on the current monitor.
EDIT: Even though there is no GLFW functionality to directly achieve this, the answer of Shmo provides an elegant solution.
Here is Shmo's answer, ported over to LWJGL:
/** Determines the current monitor that the specified window is being displayed on.
* If the monitor could not be determined, the primary monitor will be returned.
*
* #param window The window to query
* #return The current monitor on which the window is being displayed, or the primary monitor if one could not be determined
* #author Shmo<br>
* Ported to LWJGL by Brian_Entei */
#NativeType("GLFWmonitor *")
public static final long glfwGetCurrentMonitor(long window) {
int[] wx = {0}, wy = {0}, ww = {0}, wh = {0};
int[] mx = {0}, my = {0}, mw = {0}, mh = {0};
int overlap, bestoverlap;
long bestmonitor;
PointerBuffer monitors;
GLFWVidMode mode;
bestoverlap = 0;
bestmonitor = glfwGetPrimaryMonitor();// (You could set this back to NULL, but I'd rather be guaranteed to get a valid monitor);
glfwGetWindowPos(window, wx, wy);
glfwGetWindowSize(window, ww, wh);
monitors = glfwGetMonitors();
while(monitors.hasRemaining()) {
long monitor = monitors.get();
mode = glfwGetVideoMode(monitor);
glfwGetMonitorPos(monitor, mx, my);
mw[0] = mode.width();
mh[0] = mode.height();
overlap =
Math.max(0, Math.min(wx[0] + ww[0], mx[0] + mw[0]) - Math.max(wx[0], mx[0])) *
Math.max(0, Math.min(wy[0] + wh[0], my[0] + mh[0]) - Math.max(wy[0], my[0]));
if (bestoverlap < overlap) {
bestoverlap = overlap;
bestmonitor = monitor;
}
}
return bestmonitor;
}

How to move mouse with c++

I want to move the mouse cursor with a c++ script. I am using Visual C++ 2010 Express in a Windows 7 inside Parallels and I created a console application.
I know SetCursorPos method but it is just not working (it does nothing).
I managed to simulate clicks with SendInput but it does not actually move the mouse.
This is my code:
#include <Windows.h>
#include <Tlhelp32.h>
#include <stdio.h>
#include <string>
#include <iostream>
#include <fstream>
#include <sstream>
#include <time.h>
void mouseLeftClick(const int x, const int y);
// window
HWND hWindow;
int main()
{
// find window
hWindow = FindWindow(NULL, "Calculadora");
if (NULL == hWindow) {
OutputDebugStringA("Couldn't find application.");
}else{
if (!SetForegroundWindow(hWindow)) {
OutputDebugStringA("Couldn't set application to foreground.");
}else{
// click on 1
mouseLeftClick(20 265));
Sleep(500);
// click on 2
mouseLeftClick(60, 265);
Sleep(500);
}
}
return 0;
}
void mouseLeftClick(const int x, const int y)
{
// get the window position
RECT rect;
GetWindowRect(hWindow, &rect);
// calculate scale factor
const double XSCALEFACTOR = 65535 / (GetSystemMetrics(SM_CXSCREEN) - 1);
const double YSCALEFACTOR = 65535 / (GetSystemMetrics(SM_CYSCREEN) - 1);
// get current position
POINT cursorPos;
GetCursorPos(&cursorPos);
double cx = cursorPos.x * XSCALEFACTOR;
double cy = cursorPos.y * YSCALEFACTOR;
// calculate target position relative to application
double nx = (x + rect.left) * XSCALEFACTOR;
double ny = (y + rect.top) * YSCALEFACTOR;
INPUT Input={0};
Input.type = INPUT_MOUSE;
Input.mi.dx = (LONG)nx;
Input.mi.dy = (LONG)ny;
// set move cursor directly and left click
Input.mi.dwFlags = MOUSEEVENTF_MOVE | MOUSEEVENTF_ABSOLUTE | MOUSEEVENTF_LEFTDOWN | MOUSEEVENTF_LEFTUP;
SendInput(1,&Input,sizeof(INPUT));
}
This happens in Parallels because of SmartMouse is On or on Auto.
In order for a program in Parallels VM to control the mouse with SetCursorPos you need to hide cursor first. You can do that with ShowCursor(0); before you do any mouse movement e.g. SetCursorPos.
Now you will be able to control the mouse while SmartMouse is set to Auto or Off.
I found the problem. It turns out Parallels has a feature called Smart Mouse that allows you move freely between OSX and Windows. Once I deactivated it, mouse moved as expected.