Working with bitmaps is very new to me so I've been really struggling with the online tutorials and strategies that I've read through. Basically my goal is to scan the screen for a particular RGB value. I believe the steps to do this is to capture the screen in a hBitmap and then produce an array of RGB values from it that I can scan through.
I originally started with GetPixel but that is very slow. The solution was to use GetDIBits which produces the array of RGB values. The problem is that it returns weird and possibly random RGB values instead.
I'm using the following code which I found from another tutorial:
/* Globals */
int ScreenX = GetDeviceCaps(GetDC(0), HORZRES);
int ScreenY = GetDeviceCaps(GetDC(0), VERTRES);
BYTE* ScreenData = new BYTE[3*ScreenX*ScreenY];
void ScreenCap() {
HDC hdc = GetDC(GetDesktopWindow());
HDC hdcMem = CreateCompatibleDC (hdc);
HBITMAP hBitmap = CreateCompatibleBitmap(hdc, ScreenX, ScreenY);
BITMAPINFOHEADER bmi = {0};
bmi.biSize = sizeof(BITMAPINFOHEADER);
bmi.biPlanes = 1;
bmi.biBitCount = 24;
bmi.biWidth = ScreenX;
bmi.biHeight = -ScreenY;
bmi.biCompression = BI_RGB;
bmi.biSizeImage = ScreenX * ScreenY;
SelectObject(hdcMem, hBitmap);
BitBlt(hdcMem, 0, 0, ScreenX, ScreenY, hdc, 0, 0, SRCCOPY);
GetDIBits(hdc, hBitmap, 0, ScreenY, ScreenData, (BITMAPINFO*)&bmi, DIB_RGB_COLORS);
DeleteDC(hdcMem);
ReleaseDC(NULL, hdc);
}
inline int PosR(int x, int y) {
return ScreenData[3*((y*ScreenX)+x)+2];
}
inline int PosG(int x, int y) {
return ScreenData[3*((y*ScreenX)+x)+1];
}
inline int PosB(int x, int y) {
return ScreenData[3*((y*ScreenX)+x)];
}
I test this with the following code. I hit Shift to call ScreenCap and then I move my cursor to the desired location and hit Space to see what the RGB value is at that location. Am I completely nuts?
int main() {
while ( true ) {
if (GetAsyncKeyState(VK_SPACE)){
// Print out current cursor position
GetCursorPos(&p);
printf("X:%d Y:%d \n",p.x,p.y);
// Print out RGB value at that position
int r = PosR(p.x, p.y);
int g = PosG(p.x, p.y);
int b = PosB(p.x, p.y);
printf("r:%d g:%d b:%d \n",r,g,b);
} else if (GetAsyncKeyState(VK_ESCAPE)){
printf("Quit\n");
break;
} else if (GetAsyncKeyState(VK_SHIFT)){
ScreenCap();
printf("Captured\n");
}
}
system("PAUSE");
return 0;
}
The issue is that your screen is actually 32bits deep not 24. The code below will give you the result you need:
/* Globals */
int ScreenX = 0;
int ScreenY = 0;
BYTE* ScreenData = 0;
void ScreenCap()
{
HDC hScreen = GetDC(NULL);
ScreenX = GetDeviceCaps(hScreen, HORZRES);
ScreenY = GetDeviceCaps(hScreen, VERTRES);
HDC hdcMem = CreateCompatibleDC(hScreen);
HBITMAP hBitmap = CreateCompatibleBitmap(hScreen, ScreenX, ScreenY);
HGDIOBJ hOld = SelectObject(hdcMem, hBitmap);
BitBlt(hdcMem, 0, 0, ScreenX, ScreenY, hScreen, 0, 0, SRCCOPY);
SelectObject(hdcMem, hOld);
BITMAPINFOHEADER bmi = {0};
bmi.biSize = sizeof(BITMAPINFOHEADER);
bmi.biPlanes = 1;
bmi.biBitCount = 32;
bmi.biWidth = ScreenX;
bmi.biHeight = -ScreenY;
bmi.biCompression = BI_RGB;
bmi.biSizeImage = 0;// 3 * ScreenX * ScreenY;
if(ScreenData)
free(ScreenData);
ScreenData = (BYTE*)malloc(4 * ScreenX * ScreenY);
GetDIBits(hdcMem, hBitmap, 0, ScreenY, ScreenData, (BITMAPINFO*)&bmi, DIB_RGB_COLORS);
ReleaseDC(GetDesktopWindow(),hScreen);
DeleteDC(hdcMem);
DeleteObject(hBitmap);
}
inline int PosB(int x, int y)
{
return ScreenData[4*((y*ScreenX)+x)];
}
inline int PosG(int x, int y)
{
return ScreenData[4*((y*ScreenX)+x)+1];
}
inline int PosR(int x, int y)
{
return ScreenData[4*((y*ScreenX)+x)+2];
}
bool ButtonPress(int Key)
{
bool button_pressed = false;
while(GetAsyncKeyState(Key))
button_pressed = true;
return button_pressed;
}
int main()
{
while (true)
{
if (ButtonPress(VK_SPACE))
{
// Print out current cursor position
POINT p;
GetCursorPos(&p);
printf("X:%d Y:%d \n",p.x,p.y);
// Print out RGB value at that position
std::cout << "Bitmap: r: " << PosR(p.x, p.y) << " g: " << PosG(p.x, p.y) << " b: " << PosB(p.x, p.y) << "\n";
} else if (ButtonPress(VK_ESCAPE))
{
printf("Quit\n");
break;
} else if (ButtonPress(VK_SHIFT))
{
ScreenCap();
printf("Captured\n");
}
}
system("PAUSE");
return 0;
}
Your image size is specified in pixels, it should be specified in bytes
**bmi.biSizeImage = ScreenX * ScreenY;**
**bmi.biBitCount = 24;**
bmi.biWidth = ScreenX;
bmi.biHeight = -ScreenY;
**bmi.biCompression = BI_RGB;**
biSizeImage its defined units are bytes and you are specifying RGB 3 bytes per pixel.
http://msdn.microsoft.com/en-us/library/windows/desktop/dd183376(v=vs.85).aspx
biSizeImage
The size, in bytes, of the image. This may be set to zero for BI_RGB bitmaps.
Related
I'm trying to somehow get screenshot from uint8 array of RGB, but for example this code doesn't work:
#include <torch/torch.h>
#include <iostream>
#include <Windows.h>
#include <gdiplus.h>
#include <gdipluspixelformats.h> // PixelFormat24bppRGB
#include <vector>
#include <cstdlib>
#pragma comment(lib, "gdiplus.lib")
int main()
{
Gdiplus::GdiplusStartupInput input;
ULONG_PTR token;
Gdiplus::GdiplusStartup(&token, &input, NULL);
const int mWidth = 1920;
const int mHeight = 1080;
std::vector<uint8_t> pixels;
Gdiplus::BitmapData bmpData;
DWORD start = GetTickCount64();
HDC hdcScreen = GetDC(NULL);
BITMAPINFO bmpInfo;
bmpInfo.bmiHeader.biSize = sizeof(BITMAPINFOHEADER);
bmpInfo.bmiHeader.biWidth = mWidth;
bmpInfo.bmiHeader.biHeight = mHeight;
bmpInfo.bmiHeader.biPlanes = 1;
bmpInfo.bmiHeader.biBitCount = 24;
bmpInfo.bmiHeader.biCompression = BI_RGB;
HDC scComDC = CreateCompatibleDC(hdcScreen);
BitBlt(
scComDC,
0,0,
mWidth,
mHeight,
hdcScreen,
0,0,
SRCCOPY
);
LPDWORD lpPixel;
HBITMAP display = CreateDIBSection(scComDC, &bmpInfo, DIB_RGB_COLORS, (void**)&lpPixel, NULL, 0);
ReleaseDC(NULL,hdcScreen);
DeleteDC(scComDC);
LOGPALETTE lpPalette;
lpPalette.palVersion = 0x0300;
lpPalette.palNumEntries = 1;
lpPalette.palPalEntry[0].peRed =
lpPalette.palPalEntry[0].peGreen =
lpPalette.palPalEntry[0].peBlue =
lpPalette.palPalEntry[0].peFlags = NULL;
HPALETTE hPalette = CreatePalette(&lpPalette);
auto image = Gdiplus::Bitmap::FromHBITMAP(display, hPalette);
int bWidth = image->GetWidth();
int bHeight = image->GetHeight();
std::cout << image << std::endl;
std::cout << bWidth << std::endl;
std::cout << bHeight << std::endl;
auto stride = 3 * bWidth;
pixels.resize(stride * bHeight);
Gdiplus::Rect rect(0, 0, bWidth, bHeight);
image->LockBits(&rect, Gdiplus::ImageLockModeRead, PixelFormat24bppRGB, &bmpData);
for (int y = 0; y < bHeight; ++y) {
memcpy(pixels.data() + y * stride, (byte*)bmpData.Scan0 + y * bmpData.Stride, stride);
}
image->UnlockBits(&bmpData);
DWORD end = GetTickCount64();
std::cout << end - start << "ms\n";
while (true){
int inint;
std::cin >> inint;
if (inint == -1) {
break;
}
std::cout << ">> " << +pixels[inint*3]
<< "," << +pixels[inint * 3 + 1]
<< "," << +pixels[inint * 3 + 2]
<< "\n";
}
Gdiplus::GdiplusShutdown(token);
//system("PAUSE");
return 1;
}
I cannot successfully convert pixels to vector by running this program. It returns 0,0,0 indicating that all pixels are black.
I think I'm right around memcpy, since I was able to array successfully with
auto image = Gdiplus::Bitmap::FromFile(L"C:\sample.jpg");
but when I use Bitmap::FromHBITMAP(display, hPalette);, it doesn't work at all.
get screenshot from uint8 array of RGB
I am trying to create a bot to recognize text in an inactive window. To do this, I capture each frame using Bitmap. And I want Tesseract to scan the text (if there is one) on this frame and display it on the screen.
#include <iostream>
#include <opencv2/imgcodecs.hpp>
#include <opencv2/highgui.hpp>
#include <opencv2/imgproc.hpp>
#include <Windows.h>
#include <string>
#include <tesseract/capi.h>
#include <tesseract/baseapi.h>
#include <leptonica/allheaders.h>
using namespace cv;
Mat getMat(HWND hwnd) {
HDC deviceContext = GetDC(hwnd);
HDC memoryDeviceContext = CreateCompatibleDC(deviceContext);
RECT windowRect;
GetClientRect(hwnd, &windowRect);
int height = 500; /// windowRect.bottom windowRect.right
int width = 500;
HBITMAP bitmap = CreateCompatibleBitmap(deviceContext, width, height);
SelectObject(memoryDeviceContext, bitmap);
// Copy data into bitmap
BitBlt(memoryDeviceContext, 0, 0, width, height, deviceContext, 750, 359, SRCCOPY);
// Spesify format by using bitmapinfoheader
BITMAPINFOHEADER bi;
bi.biSize = sizeof(BITMAPINFOHEADER);
bi.biWidth = width;
bi.biHeight = -height;
bi.biPlanes = 1;
bi.biBitCount = 32;
bi.biCompression = BI_RGB;
bi.biSizeImage = 0; // No compression
bi.biXPelsPerMeter = 1;
bi.biYPelsPerMeter = 2;
bi.biClrUsed = 3;
bi.biClrImportant = 4;
Mat mat = Mat(height, width, CV_8UC4); // 8 bit unsigned ints 4 channels -> RGBA
// Transform data and store into mat.data
GetDIBits(memoryDeviceContext, bitmap, 0, height, mat.data, (BITMAPINFO*)&bi, DIB_RGB_COLORS);
// Clean up
DeleteObject(bitmap);
DeleteDC(memoryDeviceContext);
ReleaseDC(hwnd, deviceContext);
return mat;
}
int main() {
LPCWSTR windowTitle = L"Discord";
HWND hwnd = FindWindow(NULL, windowTitle);
/// String outText, imPath = argv[1];
/// tesseract::TessBaseAPI* temp = new tesseract::TessBaseAPI();
std::cout << "Start" << "\n";
while (true) {
Mat temp = getMat(hwnd);
cv::imshow("output", temp);
cv::waitKey(2);
}
std::cout << "Done\n";
return 1;
}
My attempts were unsuccessful:
while (true) {
Mat temp = getMat(hwnd);
cv::imshow("output", temp);
cv::waitKey(2);
String outText, imPath = argv[1];
tesseract::TessBaseAPI* temp = new tesseract::TessBaseAPI();
cv::Mat img = cv::imread((const unsigned char*)temp);
ocr->Init(NULL, "eng", tesseract::OEM_LSTM_ONLY);
ocr->SetPageSegMode(tesseract::PSM_AUTO);
ocr->SetImage(img);
outText = String(ocr->GetUTF8Text());
}
Another attempt
while (true) {
Mat temp = getMat(hwnd);
cv::imshow("output", temp);
cv::waitKey(2);
tesseract::TessBaseAPI* ocr = new tesseract::TessBaseAPI();
char* outText;
ocr->SetPageSegMode(tesseract::PSM_AUTO);
ocr->Init(NULL, "eng", tesseract::OEM_LSTM_ONLY);
ocr->SetImage(temp, 0, 0, 0, 0);
outText = ocr->GetUTF8Text();
ocr->End();
delete ocr;
return outText;
}
I just want every frame from Bitmap to be scanned for the presence of text. And this text was displayed. I will be glad of any help. Thanks
Just google for it. e.g. blogpost OpenCV and tesseract or example in tesseract doc.
The program window wont update when I try to draw a rectangle. It isn't the program not responding because I can still draw the background, but the rectangle wont update and I don't know what to change. There are no errors popping up, and the only warning is:
Warning C28251 Inconsistent annotation for 'WinMain': this instance has no annotations.
This program is in two .cpp files, the first one doesn't create a window (make sure to set that in properties), and the second one does create a window.
Note: the first .cpp file is called render.cpp and is included in the second file.
Here is the code for debugging:
#include <Windows.h>
struct Render_State
{
int width;
int hight;
void* memory;
BITMAPINFO bitmap_info;
};
Render_State render_state;
void render_backround(HWND hwnd, int colour)
{
if (WM_PAINT)
{
PAINTSTRUCT ps;
HDC hdc = BeginPaint(hwnd, &ps);
unsigned int* pixel = (unsigned int*)render_state.memory;
for (int y = 0; y < render_state.hight; y += 1)
{
for (int x = 0; x < render_state.width; x += 1)
{
*pixel++ = colour;
}
}
// render
StretchDIBits(hdc, 0, 0, render_state.width, render_state.hight, 0, 0,
render_state.width,
render_state.hight,
render_state.memory, &render_state.bitmap_info, DIB_RGB_COLORS, SRCCOPY); {}
EndPaint(hwnd, &ps);
}
}
void clear_screen(HWND hwnd, int colour)
{
if (WM_PAINT)
{
PAINTSTRUCT ps;
HDC hdc = BeginPaint(hwnd, &ps);
unsigned int* pixel = (unsigned int*)render_state.memory;
for (int y = 0; y < render_state.hight; y += 1)
{
for (int x = 0; x < render_state.width; x += 1)
{
*pixel++ = colour;
}
}
// render
StretchDIBits(hdc, 0, 0, render_state.width, render_state.hight, 0, 0, render_state.width,
render_state.hight,
render_state.memory, &render_state.bitmap_info, DIB_RGB_COLORS, SRCCOPY); {}
EndPaint(hwnd, &ps);
}
}
void draw_rect(HWND hwnd, int X, int Y, int X2, int Y2, int colour)
{
if (WM_PAINT)
{
PAINTSTRUCT ps;
HDC hdc = BeginPaint(hwnd, &ps);
for (int y = Y; y < Y2; y++)
{
// unsigned int* pixel = (unsigned int*)render_state.memory;
size_t pixel = size_t(render_state.memory) + static_cast<size_t>(X) + static_cast<size_t> (y) * static_cast<size_t> (render_state.width);
for (int x = X; x < X2; x++)
{
pixel += 0xf5500;
}
}
// render
StretchDIBits(hdc, X, Y, X2, Y2, X, Y, render_state.width, render_state.hight,
render_state.memory, &render_state.bitmap_info, DIB_RGB_COLORS, SRCCOPY); {}
EndPaint(hwnd, &ps);
}
}
#include <Windows.h>
bool running = true;
#include "renderer.cpp"
LRESULT CALLBACK windows_callback(HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam)
{
LRESULT result = 0;
switch (uMsg)
{
case WM_CLOSE:
case WM_DESTROY:
{
PostQuitMessage(0);
}
break;
case WM_SIZE:
{
RECT rect;
GetClientRect(hwnd, &rect);
render_state.width = rect.right - rect.left;
render_state.hight = rect.bottom - rect.top;
int size = render_state.width * render_state.hight * sizeof(unsigned int);
if (render_state.memory) VirtualFree(render_state.memory, 0, MEM_RELEASE);
render_state.memory = VirtualAlloc(0, size, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE);
render_state.bitmap_info.bmiHeader.biSize = sizeof(render_state.bitmap_info.bmiHeader);
render_state.bitmap_info.bmiHeader.biWidth = render_state.width;
render_state.bitmap_info.bmiHeader.biHeight = render_state.hight;
render_state.bitmap_info.bmiHeader.biPlanes = 1;
render_state.bitmap_info.bmiHeader.biBitCount = 32;
render_state.bitmap_info.bmiHeader.biCompression = BI_RGB;
//render_backround(hwnd, 0xf2000);
//clear_screen(hwnd, 0xff5500);
draw_rect(hwnd, 3, 5, 50, 50, 0xff5500);
}
break;
default:
{
result = DefWindowProc(hwnd, uMsg, wParam, lParam);
}
}
return result;
}
int WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, LPSTR lpCmdLine, int nShowCmd)
{
//compile window
CHAR clsName[] = "test";
WNDCLASSA window_class = {};
window_class.style = CS_HREDRAW | CS_VREDRAW;
window_class.lpszClassName = clsName;
window_class.lpfnWndProc = windows_callback;
//register clases
ATOM atom = RegisterClassA(&window_class);
if (0 == atom)
{
DWORD err = GetLastError();
return 1;
}
// create window
HWND window = CreateWindow(clsName, "game", WS_OVERLAPPEDWINDOW | WS_VISIBLE, CW_USEDEFAULT,
CW_USEDEFAULT, 720, 360, 0, 0, hInstance, 0);
if (NULL == window)
{
DWORD err = GetLastError();
return 1;
}
MSG message;
HDC hdc = GetDC(window);
// Main message loop:
while (GetMessage(&message, nullptr, 0, 0))
{
TranslateMessage(&message);
DispatchMessage(&message);
}
//simulate
//render
StretchDIBits(hdc, 0, 0, render_state.width, render_state.hight, 0, 0, render_state.width,
render_state.hight,
render_state.memory, &render_state.bitmap_info, DIB_RGB_COLORS, SRCCOPY); {}
}
In the draw_rect function, pixel is an unsigned int type.
for (int x = X; x < X2; ++x)
{
pixel += 0xf5500;
}
pixel += 0xf5500 will only increase the size of pixel, because pixel is not an address.
This is the reason why square cannot be drawn =>render_state.memory has no color data.
You need to declare pixel as a pointer variable. As you declared in the render_backround function.
size_t* pixel = (size_t*)render_state.memory;
for (int y = 0; y < Y2; y += 1)
{
for (int x = 0; x < X2; x += 1)
{
*pixel++ = colour;
}
}
The whole draw_rect code,
void draw_rect(HWND hwnd, int X, int Y, int X2, int Y2, int colour)
{
if (WM_PAINT)
{
PAINTSTRUCT ps;
HDC hdc = BeginPaint(hwnd, &ps);
size_t* pixel = (size_t*)render_state.memory;
for (int y = 0; y < Y2; y += 1)
{
for (int x = 0; x < X2; x += 1)
{
*pixel++ = colour;
}
}
render_state.bitmap_info.bmiHeader.biWidth = X2;
render_state.bitmap_info.bmiHeader.biHeight = Y2;
StretchDIBits(hdc, X, Y, X2, Y2, X, Y, X2, Y2,
render_state.memory, &render_state.bitmap_info, DIB_RGB_COLORS, SRCCOPY);
EndPaint(hwnd, &ps);
}
}
Note: Responsible only for your current code examples. As stated in the comment section, you can re-read the drawing of win32 window, especially the WindowProc callback function.
As stated in comments, the problem is that you are not handling the WM_PAINT message in your windows_callback() at all. You need to do something more like this instead:
renderer.h
#ifndef renderer_H
#define renderer_H
#include <Windows.h>
struct Render_State
{
int width;
int hight;
void* memory;
BITMAPINFO bitmap_info;
};
void render_background(HDC hdc, Render_State &state, int colour);
void clear_screen(HDC hdc, Render_State &state, int colour);
void draw_rect(HDC hdc, Render_State &state, int X, int Y, int X2, int Y2, int colour);
#endif
renderer.cpp
#include "renderer.h"
void render_background(HDC hdc, Render_State &state, int colour)
{
unsigned int* pixel = static_cast<unsigned int*>(state.memory);
for (int y = 0; y < state.hight; ++y)
{
for (int x = 0; x < state.width; ++x)
{
*pixel++ = colour;
}
}
// render
StretchDIBits(hdc,
0, 0, state.width, state.hight,
0, 0, state.width, state.hight,
state.memory, &state.bitmap_info,
DIB_RGB_COLORS, SRCCOPY);
}
void clear_screen(HDC hdc, Render_State &state, int colour)
{
render_background(hdc, state, colour);
}
void draw_rect(HDC hdc, Render_State &state, int X, int Y, int X2, int Y2, int colour)
{
for (int y = Y; y < Y2; ++y)
{
// unsigned int* pixel = static_cast<unsigned int*>(state.memory);
size_t pixel = size_t(state.memory) + static_cast<size_t>(X) + static_cast<size_t>(y) * static_cast<size_t>(state.width);
for (int x = X; x < X2; ++x)
{
pixel += 0xf5500;
}
}
// render
StretchDIBits(hdc,
X, Y, X2, Y2,
X, Y, state.width, state.hight,
state.memory, &state.bitmap_info,
DIB_RGB_COLORS, SRCCOPY);
}
#include <Windows.h>
#include "renderer.h"
bool running = true;
Render_State render_state = {};
LRESULT CALLBACK windows_callback(HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam)
{
switch (uMsg)
{
case WM_CLOSE:
DestroyWindow(hwnd);
return 0;
case WM_DESTROY:
PostQuitMessage(0);
return 0;
case WM_SIZE:
{
RECT rect;
GetClientRect(hwnd, &rect);
render_state.width = rect.right - rect.left;
render_state.hight = rect.bottom - rect.top;
int size = render_state.width * render_state.hight * sizeof(unsigned int);
if (render_state.memory) VirtualFree(render_state.memory, 0, MEM_RELEASE);
render_state.memory = VirtualAlloc(0, size, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE);
render_state.bitmap_info.bmiHeader.biSize = sizeof(render_state.bitmap_info.bmiHeader);
render_state.bitmap_info.bmiHeader.biWidth = render_state.width;
render_state. bitmap_info.bmiHeader.biHeight = render_state.hight;
render_state. bitmap_info.bmiHeader.biPlanes = 1;
render_state. bitmap_info.bmiHeader.biBitCount = 32;
render_state .bitmap_info.bmiHeader.biCompression = BI_RGB;
InvalidateRect(hwnd, NULL, TRUE);
return 0;
}
case WM_PAINT:
{
PAINTSTRUCT ps;
HDC hdc = BeginPaint(hwnd, &ps);
// render_background(hdc, render_state, white);
// clear_screen(hdc, render_state, 0xff5500);
draw_rect(hdc, render_state, 30, 50, 500, 500, 0xff5500);
EndPaint(hwnd, &ps);
return 0;
}
}
return DefWindowProc(hwnd, uMsg, wParam, lParam);
}
int WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, LPSTR lpCmdLine, int nShowCmd)
{
//compile window
CHAR clsName[] = "test";
WNDCLASSA window_class = {};
window_class.style = CS_HREDRAW | CS_VREDRAW;
window_class.lpszClassName = clsName;
window_class.lpfnWndProc = windows_callback;
//register class
if (!RegisterClassA(&window_class))
{
DWORD err = GetLastError();
return 1;
}
// create window
HWND window = CreateWindow(clsName, "game", WS_OVERLAPPEDWINDOW | WS_VISIBLE, CW_USEDEFAULT,
CW_USEDEFAULT, 720, 360, 0, 0, hInstance, 0);
if (!window)
{
DWORD err = GetLastError();
return 1;
}
// Main message loop:
MSG message;
while (GetMessage(&message, nullptr, 0, 0))
{
TranslateMessage(&message);
DispatchMessage(&message);
}
return 0;
}
I am working on screen capture object recognition system.
My code:
#include "opencv2/imgproc.hpp"
#include "opencv2/highgui.hpp"
#include "opencv2/dnn.hpp"
#include <opencv2/core/utils/trace.hpp>
#include <Windows.h>
#include <iostream>
using namespace std;
using namespace cv;
using namespace cv::dnn;
Mat hwnd2mat(HWND hwnd) {
HDC hwindowDC,hwindowCompatibleDC;
int height,width,srcheight,srcwidth;
HBITMAP hbwindow;
Mat src;
BITMAPINFOHEADER bi;
hwindowDC=GetDC(hwnd);
hwindowCompatibleDC=CreateCompatibleDC(hwindowDC);
SetStretchBltMode(hwindowCompatibleDC,COLORONCOLOR);
RECT windowsize; // get the height and width of the screen
GetClientRect(hwnd, &windowsize);
srcheight = windowsize.bottom;
srcwidth = windowsize.right;
height = windowsize.bottom/1; //change this to whatever size you want to resize to
width = windowsize.right/1;
src.create(height,width,CV_8UC4);
// create a bitmap
hbwindow = CreateCompatibleBitmap( hwindowDC, width, height);
bi.biSize = sizeof(BITMAPINFOHEADER); //http://msdn.microsoft.com/en-us/library/windows/window/dd183402%28v=vs.85%29.aspx
bi.biWidth = width;
bi.biHeight = -height; //this is the line that makes it draw upside down or not
bi.biPlanes = 1;
bi.biBitCount = 32;
bi.biCompression = BI_RGB;
bi.biSizeImage = 0;
bi.biXPelsPerMeter = 0;
bi.biYPelsPerMeter = 0;
bi.biClrUsed = 0;
bi.biClrImportant = 0;
// use the previously created device context with the bitmap
SelectObject(hwindowCompatibleDC, hbwindow);
// copy from the window device context to the bitmap device context
StretchBlt( hwindowCompatibleDC, 0,0, width, height, hwindowDC, 0, 0,srcwidth,srcheight, SRCCOPY); //change SRCCOPY to NOTSRCCOPY for wacky colors !
GetDIBits(hwindowCompatibleDC,hbwindow,0,height,src.data,(BITMAPINFO *)&bi,DIB_RGB_COLORS); //copy from hwindowCompatibleDC to hbwindow
// avoid memory leak
DeleteObject (hbwindow);
DeleteDC(hwindowCompatibleDC);
ReleaseDC(hwnd, hwindowDC);
return src;
}
string CLASSES[] = {"background", "aeroplane", "bicycle", "bird", "boat",
"bottle", "bus", "car", "cat", "chair", "cow", "diningtable",
"dog", "horse", "motorbike", "person", "pottedplant", "sheep",
"sofa", "train", "tvmonitor"};
float confidenceThreshold = 0.2;
int main(int argc, char **argv) {
CV_TRACE_FUNCTION();
String modelTxt = "resources/Caffe/MobileNetSSD_deploy.prototxt";
String modelBin = "resources/Caffe/MobileNetSSD_deploy.caffemodel";
Net net = readNetFromCaffe(modelTxt, modelBin);
if (net.empty()) {
std::cerr << "Can't load network by using the following files: " << std::endl;
std::cerr << "prototxt: " << modelTxt << std::endl;
std::cerr << "caffemodel: " << modelBin << std::endl;
exit(-1);
}
HWND hwndDesktop = GetDesktopWindow();
// namedWindow("output", WINDOW_NORMAL); // Zeby mozna bylo zmieniac rozmiar okna "real-time"
int key = 0;
// while( key != 27 ) {
Mat frame = hwnd2mat(hwndDesktop);
// resize(frame, frame, Size(800, 450));
// Mat frame = imread("resources/auto.png");
Mat img2;
resize(frame, img2, Size(300,300));
Mat inputBlob = blobFromImage(img2, 0.007843, Size(300,300), Scalar(127.5, 127.5, 127.5), false);
net.setInput(inputBlob, "data");
Mat detection = net.forward("detection_out");
Mat detectionMat(detection.size[2], detection.size[3], CV_32F, detection.ptr<float>());
ostringstream ss;
for (int i = 0; i < detectionMat.rows; i++) {
float confidence = detectionMat.at<float>(i, 2);
if (confidence > confidenceThreshold) {
int idx = static_cast<int>(detectionMat.at<float>(i, 1));
int xLeftBottom = static_cast<int>(detectionMat.at<float>(i, 3) * frame.cols);
int yLeftBottom = static_cast<int>(detectionMat.at<float>(i, 4) * frame.rows);
int xRightTop = static_cast<int>(detectionMat.at<float>(i, 5) * frame.cols);
int yRightTop = static_cast<int>(detectionMat.at<float>(i, 6) * frame.rows);
Rect object((int)xLeftBottom, (int)yLeftBottom,
(int)(xRightTop - xLeftBottom),
(int)(yRightTop - yLeftBottom));
rectangle(frame, object, Scalar(0, 255, 0), 2);
cout << CLASSES[idx] << ": " << confidence << endl;
ss.str("");
ss << confidence;
String conf(ss.str());
String label = CLASSES[idx] + ": " + conf;
int baseLine = 0;
Size labelSize = getTextSize(label, FONT_HERSHEY_SIMPLEX, 0.5, 1, &baseLine);
putText(frame, label, Point(xLeftBottom, yLeftBottom-10),
FONT_HERSHEY_SIMPLEX, 0.5, Scalar(0,255,0));
}
}
imshow("output", frame);
// key = waitKey(60);
waitKey();
// }
}
When i use exteranl image everything works fine but when i use function called hwnd2mat it throws me exception:
Error: Assertion failed (inputs[0]->size[1] % blobs[0].size[1] == 0) in forward
I don't know what does it mean. I tried to resize image but it doesnt work.
I am using OpenCV-3.4.1 and MobileNetSSD model.
Thanks in advance.
Ok, i finally did it!
Code for taking screenshots:
Mat getScreenshot() {
HWND hwnd = GetDesktopWindow();
HDC hwindowDC,hwindowCompatibleDC;
int height,width,srcheight,srcwidth;
HBITMAP hbwindow;
Mat src;
BITMAPINFOHEADER bi;
hwindowDC=GetDC(hwnd);
hwindowCompatibleDC=CreateCompatibleDC(hwindowDC);
SetStretchBltMode(hwindowCompatibleDC,COLORONCOLOR);
RECT windowsize; // get the height and width of the screen
GetClientRect(hwnd, &windowsize);
srcheight = windowsize.bottom;
srcwidth = windowsize.right;
height = windowsize.bottom/1; //change this to whatever size you want to resize to
width = windowsize.right/1;
src.create(height,width,CV_8UC3); // This was problem
// create a bitmap
int iBits = GetDeviceCaps(hwindowDC, BITSPIXEL) * GetDeviceCaps(hwindowDC, PLANES);
WORD wBitCount;
if (iBits <= 1)
wBitCount = 1;
else if (iBits <= 4)
wBitCount = 4;
else if (iBits <= 8)
wBitCount = 8;
else
wBitCount = 24;
hbwindow = CreateCompatibleBitmap( hwindowDC, width, height);
bi.biSize = sizeof(BITMAPINFOHEADER); //http://msdn.microsoft.com/en-us/library/windows/window/dd183402%28v=vs.85%29.aspx
bi.biWidth = width;
bi.biHeight = -height; //this is the line that makes it draw upside down or not
bi.biPlanes = 1;
bi.biBitCount = wBitCount;
bi.biCompression = BI_RGB;
bi.biSizeImage = 0;
bi.biXPelsPerMeter = 0;
bi.biYPelsPerMeter = 0;
bi.biClrUsed = 256;
bi.biClrImportant = 0;
// use the previously created device context with the bitmap
SelectObject(hwindowCompatibleDC, hbwindow);
// copy from the window device context to the bitmap device context
StretchBlt( hwindowCompatibleDC, 0,0, width, height, hwindowDC, 0, 0,srcwidth,srcheight, SRCCOPY); //change SRCCOPY to NOTSRCCOPY for wacky colors !
GetDIBits(hwindowCompatibleDC,hbwindow,0,height,src.data,(BITMAPINFO *)&bi,DIB_RGB_COLORS); //copy from hwindowCompatibleDC to hbwindow
// avoid memory leak
DeleteObject (hbwindow);
DeleteDC(hwindowCompatibleDC);
ReleaseDC(hwnd, hwindowDC);
return src;
}
first I write some UINT as color in UINT** framebuffer,then create a BITMAPINFO by CreateDIBSection, but after run the program the window are black instead of some color I set,what's wrong?
PAINTSTRUCT ps;
HDC hdc;
static int s_widthClient, s_heightClient;
static BITMAPINFO s_bitmapInfo;
static HDC s_hdcBackbuffer;
static HBITMAP s_hBitmap;
static HBITMAP s_hOldBitmap;
static void* s_pData;
switch (message)
{
case WM_CREATE:
{
RECT rc;
GetClientRect(hWnd, &rc);
s_widthClient = rc.right - rc.left;
s_heightClient = rc.bottom - rc.top;
Tiny3DDevice pDevice(s_widthClient, s_heightClient, s_pData);
pDevice.Test();
BITMAPINFOHEADER bmphdr = { 0 };
bmphdr.biSize = sizeof(BITMAPINFOHEADER);
bmphdr.biWidth = s_widthClient;
bmphdr.biHeight = -s_heightClient;
bmphdr.biPlanes = 1;
bmphdr.biBitCount = 32;
bmphdr.biSizeImage = s_heightClient * s_widthClient * 4;
s_hdcBackbuffer = CreateCompatibleDC(nullptr);
HDC hdc = GetDC(hWnd);
//s_hBitmap = CreateCompatibleBitmap(hdc, s_widthClient, s_heightClient);
s_hBitmap = CreateDIBSection(nullptr, (PBITMAPINFO)&bmphdr, DIB_RGB_COLORS,
reinterpret_cast<void**>(&pDevice.m_pFramebuffer), nullptr, 0);
s_hOldBitmap = (HBITMAP)SelectObject(s_hdcBackbuffer, s_hBitmap);
ReleaseDC(hWnd, hdc);
}
break;
case WM_PAINT:
{
hdc = BeginPaint(hWnd, &ps);
//BitBlt(s_hdcBackbuffer, 0, 0, s_widthClient, s_heightClient, nullptr, 0, 0, WHITENESS);
////draw text
//SetTextColor(s_hdcBackbuffer, RGB(0, 0, 0));
//SetBkMode(s_hdcBackbuffer, TRANSPARENT);
//TextOut(s_hdcBackbuffer, 0, 5, text.c_str(), text.size());
BitBlt(ps.hdc, 0, 0, s_widthClient, s_heightClient, s_hdcBackbuffer, 0, 0, SRCCOPY);
EndPaint(hWnd, &ps);
}
break;
and the Tiny3DDevice:
class Tiny3DDevice
{
public:
Tiny3DDevice(int width, int height, void *fb);
~Tiny3DDevice();
public:
void Test();
public:
int m_width;
int m_height;
UINT** m_pFramebuffer;
};
Tiny3DDevice::Tiny3DDevice(int width, int height, void *fb)
{
m_width = width;
m_height = height;
m_pFramebuffer = new UINT*[width];
for (int i = 0; i < width; ++i)
{
m_pFramebuffer[i] = new UINT[height];
}
}
void Tiny3DDevice::Test()
{
ZCFLOAT3 color(0.5f, 0.5f, 0.5f);
for (int i = 0; i < m_width; ++i)
for (int j = 0; j < m_height; ++j)
{
//m_pFramebuffer[i][j] = MathUtil::ColorToUINT(color);
m_pFramebuffer[i][j] = 0x3fbcefff;
}
}
what is wrong ? How should I write data in m_framebuffer? Any idea ?.
you need code like this
PVOID pv;
if (s_hBitmap = CreateDIBSection(nullptr, (PBITMAPINFO)&bmphdr, DIB_RGB_COLORS, &pv, 0, 0))
{
RtlFillMemoryUlong((PULONG)pv, bmphdr.biSizeImage, 0x3fbcefff);
}
you not need allocate pv(pDevice.m_pFramebuffer in your code), because it allocated in CreateDIBSection. you just need fill it. your code of Tiny3DDevice completely wrong and senseless.
use static vars for s_hdcBackbuffer, etc - nightmare
BITMAPINFOHEADER bmphdr = { 0 };
nobody try use this :) ?
BITMAPINFOHEADER bmphdr = { };