OpenCV c++ hwnd2mat(screenshot) -> blobFromImage : Error - c++

Question

I am working on screen capture object recognition system.
My code:
#include "opencv2/imgproc.hpp"
#include "opencv2/highgui.hpp"
#include "opencv2/dnn.hpp"
#include <opencv2/core/utils/trace.hpp>
#include <Windows.h>
#include <iostream>
using namespace std;
using namespace cv;
using namespace cv::dnn;
Mat hwnd2mat(HWND hwnd) {
HDC hwindowDC,hwindowCompatibleDC;
int height,width,srcheight,srcwidth;
HBITMAP hbwindow;
Mat src;
BITMAPINFOHEADER bi;
hwindowDC=GetDC(hwnd);
hwindowCompatibleDC=CreateCompatibleDC(hwindowDC);
SetStretchBltMode(hwindowCompatibleDC,COLORONCOLOR);
RECT windowsize; // get the height and width of the screen
GetClientRect(hwnd, &windowsize);
srcheight = windowsize.bottom;
srcwidth = windowsize.right;
height = windowsize.bottom/1; //change this to whatever size you want to resize to
width = windowsize.right/1;
src.create(height,width,CV_8UC4);
// create a bitmap
hbwindow = CreateCompatibleBitmap( hwindowDC, width, height);
bi.biSize = sizeof(BITMAPINFOHEADER); //http://msdn.microsoft.com/en-us/library/windows/window/dd183402%28v=vs.85%29.aspx
bi.biWidth = width;
bi.biHeight = -height; //this is the line that makes it draw upside down or not
bi.biPlanes = 1;
bi.biBitCount = 32;
bi.biCompression = BI_RGB;
bi.biSizeImage = 0;
bi.biXPelsPerMeter = 0;
bi.biYPelsPerMeter = 0;
bi.biClrUsed = 0;
bi.biClrImportant = 0;
// use the previously created device context with the bitmap
SelectObject(hwindowCompatibleDC, hbwindow);
// copy from the window device context to the bitmap device context
StretchBlt( hwindowCompatibleDC, 0,0, width, height, hwindowDC, 0, 0,srcwidth,srcheight, SRCCOPY); //change SRCCOPY to NOTSRCCOPY for wacky colors !
GetDIBits(hwindowCompatibleDC,hbwindow,0,height,src.data,(BITMAPINFO *)&bi,DIB_RGB_COLORS); //copy from hwindowCompatibleDC to hbwindow
// avoid memory leak
DeleteObject (hbwindow);
DeleteDC(hwindowCompatibleDC);
ReleaseDC(hwnd, hwindowDC);
return src;
}
string CLASSES[] = {"background", "aeroplane", "bicycle", "bird", "boat",
"bottle", "bus", "car", "cat", "chair", "cow", "diningtable",
"dog", "horse", "motorbike", "person", "pottedplant", "sheep",
"sofa", "train", "tvmonitor"};
float confidenceThreshold = 0.2;
int main(int argc, char **argv) {
CV_TRACE_FUNCTION();
String modelTxt = "resources/Caffe/MobileNetSSD_deploy.prototxt";
String modelBin = "resources/Caffe/MobileNetSSD_deploy.caffemodel";
Net net = readNetFromCaffe(modelTxt, modelBin);
if (net.empty()) {
std::cerr << "Can't load network by using the following files: " << std::endl;
std::cerr << "prototxt: " << modelTxt << std::endl;
std::cerr << "caffemodel: " << modelBin << std::endl;
exit(-1);
}
HWND hwndDesktop = GetDesktopWindow();
// namedWindow("output", WINDOW_NORMAL); // Zeby mozna bylo zmieniac rozmiar okna "real-time"
int key = 0;
// while( key != 27 ) {
Mat frame = hwnd2mat(hwndDesktop);
// resize(frame, frame, Size(800, 450));
// Mat frame = imread("resources/auto.png");
Mat img2;
resize(frame, img2, Size(300,300));
Mat inputBlob = blobFromImage(img2, 0.007843, Size(300,300), Scalar(127.5, 127.5, 127.5), false);
net.setInput(inputBlob, "data");
Mat detection = net.forward("detection_out");
Mat detectionMat(detection.size[2], detection.size[3], CV_32F, detection.ptr<float>());
ostringstream ss;
for (int i = 0; i < detectionMat.rows; i++) {
float confidence = detectionMat.at<float>(i, 2);
if (confidence > confidenceThreshold) {
int idx = static_cast<int>(detectionMat.at<float>(i, 1));
int xLeftBottom = static_cast<int>(detectionMat.at<float>(i, 3) * frame.cols);
int yLeftBottom = static_cast<int>(detectionMat.at<float>(i, 4) * frame.rows);
int xRightTop = static_cast<int>(detectionMat.at<float>(i, 5) * frame.cols);
int yRightTop = static_cast<int>(detectionMat.at<float>(i, 6) * frame.rows);
Rect object((int)xLeftBottom, (int)yLeftBottom,
(int)(xRightTop - xLeftBottom),
(int)(yRightTop - yLeftBottom));
rectangle(frame, object, Scalar(0, 255, 0), 2);
cout << CLASSES[idx] << ": " << confidence << endl;
ss.str("");
ss << confidence;
String conf(ss.str());
String label = CLASSES[idx] + ": " + conf;
int baseLine = 0;
Size labelSize = getTextSize(label, FONT_HERSHEY_SIMPLEX, 0.5, 1, &baseLine);
putText(frame, label, Point(xLeftBottom, yLeftBottom-10),
FONT_HERSHEY_SIMPLEX, 0.5, Scalar(0,255,0));
}
}
imshow("output", frame);
// key = waitKey(60);
waitKey();
// }
}
When i use exteranl image everything works fine but when i use function called hwnd2mat it throws me exception:
Error: Assertion failed (inputs[0]->size[1] % blobs[0].size[1] == 0) in forward
I don't know what does it mean. I tried to resize image but it doesnt work.
I am using OpenCV-3.4.1 and MobileNetSSD model.
Thanks in advance.

Answer 1

Ok, i finally did it!
Code for taking screenshots:
Mat getScreenshot() {
HWND hwnd = GetDesktopWindow();
HDC hwindowDC,hwindowCompatibleDC;
int height,width,srcheight,srcwidth;
HBITMAP hbwindow;
Mat src;
BITMAPINFOHEADER bi;
hwindowDC=GetDC(hwnd);
hwindowCompatibleDC=CreateCompatibleDC(hwindowDC);
SetStretchBltMode(hwindowCompatibleDC,COLORONCOLOR);
RECT windowsize; // get the height and width of the screen
GetClientRect(hwnd, &windowsize);
srcheight = windowsize.bottom;
srcwidth = windowsize.right;
height = windowsize.bottom/1; //change this to whatever size you want to resize to
width = windowsize.right/1;
src.create(height,width,CV_8UC3); // This was problem
// create a bitmap
int iBits = GetDeviceCaps(hwindowDC, BITSPIXEL) * GetDeviceCaps(hwindowDC, PLANES);
WORD wBitCount;
if (iBits <= 1)
wBitCount = 1;
else if (iBits <= 4)
wBitCount = 4;
else if (iBits <= 8)
wBitCount = 8;
else
wBitCount = 24;
hbwindow = CreateCompatibleBitmap( hwindowDC, width, height);
bi.biSize = sizeof(BITMAPINFOHEADER); //http://msdn.microsoft.com/en-us/library/windows/window/dd183402%28v=vs.85%29.aspx
bi.biWidth = width;
bi.biHeight = -height; //this is the line that makes it draw upside down or not
bi.biPlanes = 1;
bi.biBitCount = wBitCount;
bi.biCompression = BI_RGB;
bi.biSizeImage = 0;
bi.biXPelsPerMeter = 0;
bi.biYPelsPerMeter = 0;
bi.biClrUsed = 256;
bi.biClrImportant = 0;
// use the previously created device context with the bitmap
SelectObject(hwindowCompatibleDC, hbwindow);
// copy from the window device context to the bitmap device context
StretchBlt( hwindowCompatibleDC, 0,0, width, height, hwindowDC, 0, 0,srcwidth,srcheight, SRCCOPY); //change SRCCOPY to NOTSRCCOPY for wacky colors !
GetDIBits(hwindowCompatibleDC,hbwindow,0,height,src.data,(BITMAPINFO *)&bi,DIB_RGB_COLORS); //copy from hwindowCompatibleDC to hbwindow
// avoid memory leak
DeleteObject (hbwindow);
DeleteDC(hwindowCompatibleDC);
ReleaseDC(hwnd, hwindowDC);
return src;
}