OPENCV desktop capture Part II - c++

Continuing in previous post from here OPENCV Destop Capture and this hwnd2mat function, a method to capture desktop as source in opencv is create a bitmap from the screen image with this hwnd2mat function. done.
this function is already create a bitmap from the screen image.but it gives weird effect like a trailing inside the video, i just want a normal video source more likely like this one . i've already try to find what cause it, but i don't know anymore.
#include "opencv2/imgproc.hpp"
#include "opencv2/highgui.hpp"
#include <Windows.h>
#include <iostream>
using namespace std;
using namespace cv;
Mat hwnd2mat(HWND hwnd)
{
HDC hwindowDC, hwindowCompatibleDC;
int height, width, srcheight, srcwidth;
HBITMAP hbwindow;
Mat src;
BITMAPINFOHEADER bi;
hwindowDC = GetDC(hwnd);
hwindowCompatibleDC = CreateCompatibleDC(hwindowDC);
SetStretchBltMode(hwindowCompatibleDC, COLORONCOLOR);
RECT windowsize; // get the height and width of the screen
GetClientRect(hwnd, &windowsize);
srcheight = windowsize.bottom;
srcwidth = windowsize.right;
height = windowsize.bottom / 1; //change this to whatever size you want to resize to
width = windowsize.right / 1;
src.create(height, width, CV_8UC4);
// create a bitmap
hbwindow = CreateCompatibleBitmap(hwindowDC, width, height);
bi.biSize = sizeof(BITMAPINFOHEADER); //http://msdn.microsoft.com/en-us/library/windows/window/dd183402%28v=vs.85%29.aspx
bi.biWidth = width;
bi.biHeight = -height; //this is the line that makes it draw upside down or not
bi.biPlanes = 1;
bi.biBitCount = 32;
bi.biCompression = BI_RGB;
bi.biSizeImage = 0;
bi.biXPelsPerMeter = 0;
bi.biYPelsPerMeter = 0;
bi.biClrUsed = 0;
bi.biClrImportant = 0;
// use the previously created device context with the bitmap
SelectObject(hwindowCompatibleDC, hbwindow);
// copy from the window device context to the bitmap device context
StretchBlt(hwindowCompatibleDC, 0, 0, width, height, hwindowDC, 0, 0, srcwidth, srcheight, SRCCOPY); //change SRCCOPY to NOTSRCCOPY for wacky colors !
GetDIBits(hwindowCompatibleDC, hbwindow, 0, height, src.data, (BITMAPINFO *)&bi, DIB_RGB_COLORS); //copy from hwindowCompatibleDC to hbwindow
// avoid memory leak
DeleteObject(hbwindow);
DeleteDC(hwindowCompatibleDC);
ReleaseDC(hwnd, hwindowDC);
return src;
}
int main(int argc, char **argv)
{
HWND hwndDesktop = GetDesktopWindow();
namedWindow("output", CV_WINDOW_NORMAL);
int key = 0;
while (key != 30 )
{
Mat src = hwnd2mat(hwndDesktop);
// you can do some image processing here
imshow("output", src);
key = waitKey(30); // you can change wait time
}
ReleaseCapture();
destroyAllWindows();
return 0;
}

Using GetDesktopWindow() will make infinite-mirror-effect, instead that, using what window you want capture, and me, i'm using FindWindowEx(), and use Spy++ to find what window you want to capture, that should works. but of course it will go to the next bug :).

Related

opencv + cuda how to record video from the screen at 64 fps?

Good afternoon.
At the moment I cannot find information on how to record video from the screen using opencv + cuda at 64 fps. I have some code that uses cpu but not gpu, so it's about 25 fps. Please help me find information.
#include <iostream>
#include <opencv2/opencv.hpp>
#include <opencv2/cudaarithm.hpp>
#include <opencv2/video.hpp>
#include "opencv2/imgproc.hpp"
#include "opencv2/highgui.hpp"
#include <Windows.h>
using namespace std;
using namespace cv;
using namespace cv::cuda;
Mat hwnd2mat(HWND hwnd)
{
HDC hwindowDC, hwindowCompatibleDC;
int height, width, srcheight, srcwidth;
HBITMAP hbwindow;
Mat src;
BITMAPINFOHEADER bi;
hwindowDC = GetDC(hwnd);
hwindowCompatibleDC = CreateCompatibleDC(hwindowDC);
SetStretchBltMode(hwindowCompatibleDC, COLORONCOLOR);
RECT windowsize; // get the height and width of the screen
GetClientRect(hwnd, &windowsize);
srcheight = windowsize.bottom;
srcwidth = windowsize.right;
height = windowsize.bottom / 1; //change this to whatever size you want to resize to
width = windowsize.right / 1;
src.create(height, width, CV_8UC4);
// create a bitmap
hbwindow = CreateCompatibleBitmap(hwindowDC, width, height);
bi.biSize = sizeof(BITMAPINFOHEADER); //http://msdn.microsoft.com/en-us/library/windows/window/dd183402%28v=vs.85%29.aspx
bi.biWidth = width;
bi.biHeight = -height; //this is the line that makes it draw upside down or not
bi.biPlanes = 1;
bi.biBitCount = 32;
bi.biCompression = BI_RGB;
bi.biSizeImage = 0;
bi.biXPelsPerMeter = 0;
bi.biYPelsPerMeter = 0;
bi.biClrUsed = 0;
bi.biClrImportant = 0;
// use the previously created device context with the bitmap
SelectObject(hwindowCompatibleDC, hbwindow);
// copy from the window device context to the bitmap device context
StretchBlt(hwindowCompatibleDC, 0, 0, width, height, hwindowDC, 0, 0, srcwidth, srcheight, SRCCOPY); //change SRCCOPY to NOTSRCCOPY for wacky colors !
GetDIBits(hwindowCompatibleDC, hbwindow, 0, height, src.data, (BITMAPINFO*)&bi, DIB_RGB_COLORS); //copy from hwindowCompatibleDC to hbwindow
// avoid memory leak
DeleteObject(hbwindow);
DeleteDC(hwindowCompatibleDC);
ReleaseDC(hwnd, hwindowDC);
return src;
}
int main(int argc, char** argv)
{
HWND hwndDesktop = GetDesktopWindow();
//namedWindow("output", WINDOW_NORMAL);
int key = 0;
while (1)
{
Mat src = hwnd2mat(hwndDesktop);
// you can do some image processing here
imshow("output", src);
key = waitKey(1);
}
}
how to record video from desktop at 64 fps?
I tried to replace MAT with UMAT, which uses mostly gpu, but unfortunately it didn't work because of this function:
GetDIBits(hwindowCompatibleDC, hbwindow, 0, height, src.data, (BITMAPINFO*)&bi, DIB_RGB_COLORS);
Because the UMAT class has no variable data.
Edit:
Good afternoon, I updated the hwnd2mat function for gpu, but it gives me a black screen for some reason. Help me please.
UMat hwnd2mat(HWND hwnd)
{
HDC hwindowDC, hwindowCompatibleDC;
int height, width, srcheight, srcwidth;
HBITMAP hbwindow;
UMat src;
BITMAPINFOHEADER bi;
hwindowDC = GetDC(hwnd);
hwindowCompatibleDC = CreateCompatibleDC(hwindowDC);
SetStretchBltMode(hwindowCompatibleDC, COLORONCOLOR);
RECT windowsize; // get the height and width of the screen
GetClientRect(hwnd, &windowsize);
srcheight = windowsize.bottom;
srcwidth = windowsize.right;
height = windowsize.bottom / 1; //change this to whatever size you want to resize to
width = windowsize.right / 1;
src.create(height, width, CV_8UC4);
// create a bitmap
hbwindow = CreateCompatibleBitmap(hwindowDC, width, height);
bi.biSize = sizeof(BITMAPINFOHEADER); //http://msdn.microsoft.com/en-us/library/windows/window/dd183402%28v=vs.85%29.aspx
bi.biWidth = width;
bi.biHeight = -height; //this is the line that makes it draw upside down or not
bi.biPlanes = 1;
bi.biBitCount = 32;
bi.biCompression = BI_RGB;
bi.biSizeImage = 0;
bi.biXPelsPerMeter = 0;
bi.biYPelsPerMeter = 0;
bi.biClrUsed = 0;
bi.biClrImportant = 0;
// use the previously created device context with the bitmap
SelectObject(hwindowCompatibleDC, hbwindow);
// copy from the window device context to the bitmap device context
StretchBlt(hwindowCompatibleDC, 0, 0, width, height, hwindowDC, 0, 0, srcwidth, srcheight, SRCCOPY); //change SRCCOPY to NOTSRCCOPY for wacky colors !
GetDIBits(hwindowCompatibleDC, hbwindow, 0, height, src.u->data, (BITMAPINFO*)&bi, DIB_RGB_COLORS); //copy from hwindowCompatibleDC to hbwindow
// avoid memory leak
DeleteObject(hbwindow);
DeleteDC(hwindowCompatibleDC);
ReleaseDC(hwnd, hwindowDC);
return src;
}
CUDA helps to accelerate processing functions, but cannot help in accelerating capture of desktop. In this case, bottleneck is the blit API you are using to capture the screen (GetDIBits). Since you seem to be having a Nvidia GPU, you should look at OS/driver level alternatives like mentioned here (https://developer.nvidia.com/capture-sdk).

Acquiring frames from Chrome (simple game) and displaying them with OpenCV

I've searched for ways to display a real-time simple game in a small window, and got to understand that OpenCV can handle it for me. But to accomplish that I need to acquire the frames in each iteration, just before showing it in the desired window.
This is the current solution that I am using:
HDC hwindowDC;
HDC hwindowCompatibleDC; // Memory for given DC
HBITMAP hbwindow;
int height, width;
// handles information about DIB (device-independent bitmap)
// https://learn.microsoft.com/en-us/windows/desktop/gdi/device-independent-bitmaps
BITMAPINFOHEADER bi;
// retrieces a handle to DC for the client area for entire screen
hwindowDC = GetDC(NULL);
// creates memory DC for the entire screen
hwindowCompatibleDC = CreateCompatibleDC(hwindowDC);
// enables strecn mode which has low costs - because of the grayscale game
SetStretchBltMode(hwindowCompatibleDC, COLORONCOLOR);
// size of the game window - according to the inspect of the game within chrome
height = 150;
width = 600;
// creates matrix with four color channels and values in the range 0 to 255
// - https://stackoverflow.com/questions/39543580/what-is-the-purpose-of-cvtype-cv-8uc4
frame.create(height, width, CV_8UC4);
// create a bitmap for the window
hbwindow = CreateCompatibleBitmap(hwindowDC, width, height);
bi.biSize = sizeof(BITMAPINFOHEADER);
bi.biWidth = width;
bi.biHeight = -height; //the minus sign makes it draw upside down or not
bi.biPlanes = 1;
bi.biBitCount = 32;
bi.biCompression = BI_RGB;
bi.biSizeImage = 0;
bi.biXPelsPerMeter = 0;
bi.biYPelsPerMeter = 0;
bi.biClrUsed = 0;
bi.biClrImportant = 0;
// attach the bitmap with the memory DC for our window
SelectObject(hwindowCompatibleDC, hbwindow);
// copy from the window device context to the bitmap device context
StretchBlt(hwindowCompatibleDC, 0, 0, width, height, hwindowDC, 650, 130, 620, 150, SRCCOPY); //change SRCCOPY to NOTSRCCOPY for wacky colors !
// Gets the "bits" from the bitmap and copies them into src.data
GetDIBits(hwindowCompatibleDC, hbwindow, 0, height, frame.data, (BITMAPINFO *)&bi, DIB_RGB_COLORS); //copy from hwindowCompatibleDC to hbwindow
// avoid memory leak // avoid memory leak
DeleteObject(hbwindow);
DeleteDC(hwindowCompatibleDC); // for any DC that was created by calling the CreateDC function
ReleaseDC(hwnd, hwindowDC); // releasing the DC for use of other applications (MUST)
img2 = frame;
//img1 = frame;
return frame;
After trying to figure out how to make the game running smoother, I've tried to implement screen recorder with the DirectX API, but without any success. This code results in a black screen.
IDirect3DSurface9* pSurface;
d3ddev->CreateOffscreenPlainSurface(100, 100,
D3DFMT_A8R8G8B8, D3DPOOL_SCRATCH, &pSurface, NULL);
d3ddev->GetFrontBufferData(NULL , pSurface);
/*D3DLOCKED_RECT lockedRect;
ZeroMemory(&lockedRect, sizeof(D3DLOCKED_RECT));
pSurface->LockRect(&lockedRect, 0, D3DLOCK_READONLY);
*/
LPD3DXBUFFER buffer;
D3DXSaveSurfaceToFileInMemory(&buffer, D3DXIFF_BMP, pSurface, NULL, NULL);
DWORD imSize = buffer->GetBufferSize();
void* imgBuffer = buffer->GetBufferPointer();
//**** OpenCV
Mat D3DSurface(100, 100, CV_8UC4, imgBuffer);
imshow("D3DSurface", D3DSurface);
pSurface->Release();
return D3DSurface;
Can anyone can help me figure out how to accomplish this task in a better way?
I just think that there is something that I'm not quite understanding in this process, mirroring a live moving game to a desired window with minimal lags.
My purpose is to make a bot for an online game.

opencv desktop capture returns only part of the screen on windows in large fonts mode

I use this standard sample to grab screen with opencv. It works correctly if my font size in control panel is 100%:
Mat hwnd2mat(HWND hwnd) {
HDC hwindowDC, hwindowCompatibleDC;
int height, width, srcheight, srcwidth;
HBITMAP hbwindow;
Mat src;
BITMAPINFOHEADER bi;
hwindowDC = GetDC(hwnd);
hwindowCompatibleDC = CreateCompatibleDC(hwindowDC);
SetStretchBltMode(hwindowCompatibleDC, COLORONCOLOR);
RECT windowsize; // get the height and width of the screen
GetClientRect(hwnd, &windowsize);
srcheight = windowsize.bottom;
srcwidth = windowsize.right;
height = windowsize.bottom / 1; //change this to whatever size you want to resize to
width = windowsize.right / 1;
src.create(height, width, CV_8UC4);
// create a bitmap
hbwindow = CreateCompatibleBitmap(hwindowDC, width, height);
bi.biSize = sizeof(BITMAPINFOHEADER); //http://msdn.microsoft.com/en-us/library/windows/window/dd183402%28v=vs.85%29.aspx
bi.biWidth = width;
bi.biHeight = -height; //this is the line that makes it draw upside down or not
bi.biPlanes = 1;
bi.biBitCount = 32;
bi.biCompression = BI_RGB;
bi.biSizeImage = 0;
bi.biXPelsPerMeter = 0;
bi.biYPelsPerMeter = 0;
bi.biClrUsed = 0;
bi.biClrImportant = 0;
SelectObject(hwindowCompatibleDC, hbwindow);
StretchBlt(hwindowCompatibleDC, 0, 0, width, height, hwindowDC, 0, 0, srcwidth, srcheight, SRCCOPY); //change SRCCOPY to NOTSRCCOPY for wacky colors !
GetDIBits(hwindowCompatibleDC, hbwindow, 0, height, src.data, (BITMAPINFO *)&bi, DIB_RGB_COLORS); //copy from hwindowCompatibleDC to hbwindow
DeleteObject(hbwindow);
DeleteDC(hwindowCompatibleDC);
ReleaseDC(hwnd, hwindowDC);
return src;
}
int main() {
HWND hwndDesktop = GetDesktopWindow();
Mat src = hwnd2mat(hwndDesktop);
imwrite("output.bmp", src);
return 0;
}
But if I enable large fonts (on Windows 10 at least), then it gives me only upper left corner of the screen:
how can I fix that?
I think you need to mark application as HighDpi-aware by creating corresponding manifest entry. Otherwise your application will deal with DPI virtualization.
Another solution might be calling SetProcessDPIAware() in the beginning of your program. Helped me.

OpenCV Read Mat from Memory returns null

I create a MAT from a given hWnd
cv::Mat hwnd2mat(HWND hwnd) {
HDC hwindowDC, hwindowCompatibleDC;
int height, width, srcheight, srcwidth;
HBITMAP hbwindow;
cv::Mat src;
BITMAPINFOHEADER bi;
hwindowDC = GetDC(hwnd);
hwindowCompatibleDC = CreateCompatibleDC(hwindowDC);
SetStretchBltMode(hwindowCompatibleDC, COLORONCOLOR);
RECT windowsize; // get the height and width of the screen
GetClientRect(hwnd, &windowsize);
srcheight = windowsize.bottom;
srcwidth = windowsize.right;
height = windowsize.bottom;
width = windowsize.right;
src.create(height, width, CV_8UC4);
// create a bitmap
hbwindow = CreateCompatibleBitmap(hwindowDC, width, height);
bi.biSize = sizeof(BITMAPINFOHEADER); //http://msdn.microsoft.com/en-us/library/windows/window/dd183402%28v=vs.85%29.aspx
bi.biWidth = width;
bi.biHeight = -height; //this is the line that makes it draw upside down or not
bi.biPlanes = 1;
bi.biBitCount = 32;
bi.biCompression = BI_RGB;
bi.biSizeImage = 0;
bi.biXPelsPerMeter = 0;
bi.biYPelsPerMeter = 0;
bi.biClrUsed = 0;
bi.biClrImportant = 0;
// use the previously created device context with the bitmap
SelectObject(hwindowCompatibleDC, hbwindow);
// copy from the window device context to the bitmap device context
StretchBlt(hwindowCompatibleDC, 0, 0, width, height, hwindowDC, 0, 0, srcwidth, srcheight, SRCCOPY); //change SRCCOPY to NOTSRCCOPY for wacky colors !
GetDIBits(hwindowCompatibleDC, hbwindow, 0, height, src.data, (BITMAPINFO *)&bi, DIB_RGB_COLORS); //copy from hwindowCompatibleDC to hbwindow
// avoid memory leak
DeleteObject(hbwindow); DeleteDC(hwindowCompatibleDC); ReleaseDC(hwnd, hwindowDC);
return src;
}
And I try to load the images to run the template matching algorithm.
However, when I try to imgdecode I get img = NULL. If I save the file to a png and read it afterwards, it's working.
I even tried to use the different flags to read the image.
cv::Mat resMat = hwnd2mat(blueStackshWnd);
cv::imwrite("foo.png", resMat);
img = cv::imdecode(resMat, cv::IMREAD_GRAYSCALE);
//img = cv::imread("fuck.png", cv::IMREAD_COLOR);
templ = cv::imread("images\\giant.png", cv::IMREAD_GRAYSCALE);
Here is a screenshot while debugging
EDIT:
When assigning the ret value of hwnd2mat to img I get the following error
error: (-215) (depth == 0 || depth == 5) && type == _templ.type() && _img.dims() <= 2 in function cv... ...} ...} const cv::Exception &

C++ Window capture output isn't the same size as said window

I have a short program in C++ to capture the pixels from this BlueStacks emulator window, and then I'll be manipulating the pixels using OpenCV and then sending mouse input (win32api) based on some decisions here and there. I'm working in Visual Studio Enterprise 2017.
Only problem is, the function seems to be capturing a smaller region of pixels compared to the full window.
Here's an example image. Original window is on the left, mirrored output is on the right.
How can I fix this? I've already enabled High DPI awareness in my project settings, not sure what other steps to take other than adding random magic numbers into the function.
I've done this program already in Python but I'd like to redo this in C++ for the performance boost.
Here's a video (warning: noise). And another one.
Here's my current code:
#include "stdafx.h"
#include "opencv2/imgproc.hpp"
#include "opencv2/highgui.hpp"
#include <Windows.h>
#include <iostream>
using namespace std;
using namespace cv;
Mat hwnd2mat(HWND hwnd);
int main(int argc, char **argv)
{
printf("Hello, world!\n");
Sleep(1000);
HWND hwndDesktop;
hwndDesktop = GetForegroundWindow();
// namedWindow("output", WINDOW_NORMAL);
int key = 0;
Mat src;
while (key != 27)
{
src = hwnd2mat(hwndDesktop);
// you can do some image processing here
imshow("output", src);
key = waitKey(1); // you can change wait time
}
}
Mat hwnd2mat(HWND hwnd)
{
HDC hwindowDC, hwindowCompatibleDC;
int height, width, srcheight, srcwidth;
HBITMAP hbwindow;
Mat src;
BITMAPINFOHEADER bi;
hwindowDC = GetDC(hwnd);
hwindowCompatibleDC = CreateCompatibleDC(hwindowDC);
SetStretchBltMode(hwindowCompatibleDC, COLORONCOLOR);
RECT windowsize; // get the height and width of the screen
GetClientRect(hwnd, &windowsize);
srcheight = windowsize.bottom;
srcwidth = windowsize.right;
height = windowsize.bottom / 1; //change this to whatever size you want to resize to
width = windowsize.right / 1;
src.create(height, width, CV_8UC4);
// create a bitmap
hbwindow = CreateCompatibleBitmap(hwindowDC, width, height);
bi.biSize = sizeof(BITMAPINFOHEADER); //http://msdn.microsoft.com/en-us/library/windows/window/dd183402%28v=vs.85%29.aspx
bi.biWidth = width;
bi.biHeight = -height; //this is the line that makes it draw upside down or not
bi.biPlanes = 1;
bi.biBitCount = 32;
bi.biCompression = BI_RGB;
bi.biSizeImage = 0;
bi.biXPelsPerMeter = 0;
bi.biYPelsPerMeter = 0;
bi.biClrUsed = 0;
bi.biClrImportant = 0;
// use the previously created device context with the bitmap
SelectObject(hwindowCompatibleDC, hbwindow);
// copy from the window device context to the bitmap device context
StretchBlt(hwindowCompatibleDC, 0, 0, width, height, hwindowDC, 0, 0, srcwidth, srcheight, SRCCOPY); //change SRCCOPY to NOTSRCCOPY for wacky colors !
GetDIBits(hwindowCompatibleDC, hbwindow, 0, height, src.data, (BITMAPINFO *)&bi, DIB_RGB_COLORS); //copy from hwindowCompatibleDC to hbwindow
// avoid memory leak
DeleteObject(hbwindow);
DeleteDC(hwindowCompatibleDC);
ReleaseDC(hwnd, hwindowDC);
return src;
}
The stretching is due to DPI scaling. Your own program is not DPI aware, the other program seems to be DPI aware. The easiest way to make your program DPI aware is by calling SetProcessDPIAware(); at the start of the program.
Aside note, HBITMAP handle should not be selected in a device context when calling GetDIBits. You can rewrite your code as
SetProcessDPIAware();
...
Mat hwnd2mat(HWND hwnd)
{
RECT rc;
GetClientRect(hwnd, &rc);
int width = rc.right;
int height = rc.bottom;
Mat src;
src.create(height, width, CV_8UC4);
HDC hdc = GetDC(hwnd);
HDC memdc = CreateCompatibleDC(hdc);
HBITMAP hbitmap = CreateCompatibleBitmap(hdc, width, height);
HBITMAP oldbmp = (HBITMAP)SelectObject(memdc, hbitmap);
BitBlt(memdc, 0, 0, width, height, hdc, 0, 0, SRCCOPY);
SelectObject(memdc, oldbmp);
BITMAPINFOHEADER bi = { sizeof(BITMAPINFOHEADER), width, -height, 1, 32, BI_RGB };
GetDIBits(hdc, hbitmap, 0, height, src.data, (BITMAPINFO*)&bi, DIB_RGB_COLORS);
DeleteObject(hbitmap);
DeleteDC(memdc);
ReleaseDC(hwnd, hdc);
return src;
}
This code will only work for native Win32 programs. For other programs like Chrome, WPF, Qt apps, this will show blank screen. You would need to take screen shot of desktop window instead.
I think you want GetActiveWindow() instead of GetForegroundWindow()
See this SO answer