Template matching from a screenshot of a window - c++

Question

What I've done
I have a small template image which is meant to be used to find coordinates of matching subimages within a larger screenshot image. The screenshot itself is captured into a memory DC with the help of BitBlt, then converted into a cv::Mat via GetDIBits, like so:
HDC windowDc = GetWindowDC(hwndTarget);
HDC memDc = CreateCompatibleDC(windowDc);
// ...
HBITMAP hbmp = CreateCompatibleBitmap(windowDc, width, height);
SelectObject(memDc, hbmp);
BITMAPINFOHEADER bi =
{
sizeof(BITMAPINFOHEADER), // biSize
width, // biWidth
-height, // biHeight
1, // biPlanes
32, // biBitCount
BI_RGB, // biCompression
0, // biSizeImage
0, // biXPelsPerMeter
0, // biYPelsPerMeter
0, // biClrUser
0 // biClrImportant
};
// ...
BitBlt(memDc, 0, 0, width, height, windowDc, offsetX, offsetY, SRCCOPY);
matScreen.create(height, width, CV_8UC4);
GetDIBits(memDc, hbmp, 0, (UINT)height, matScreen.data, (BITMAPINFO*)&bi, DIB_RGB_COLORS);
This appears to work fine, and a quick imshow("Source", matScreen) displays the image correctly.
However...
Since this screenshot image has been created as a 32-bit RGB memory bitmap, and placed inside a cv::Mat using the CV_8UC4 flag, it fails several assertions in OpenCV (as well as outputs whacky results when utilizing several OpenCV methods). Most notably, matchTemplate always fails on the following line:
CV_Assert( (depth == CV_8U || depth == CV_32F) && type == _templ.type() && _img.dims() <= 2 );
I've tried matching up the depth flags of the screenshot and the template/result Mats, and the best case scenario is that I am able to display all 3 images, but the template matching doesn't work because I'm assuming the source (screenshot) image is displayed in color whilst the others are in grayscale. Other conversions either fail, or display strange results and fail the template matching (or simply raise an error)... And changing the screenshot image's Mat to use any other depth flag ends up displaying the image incorrectly.
Question
What can I do to utilize OpenCV's template matching with a screenshot taken in C++? Is there some sort of manual conversion I should be doing to the screenshot image, or something?
Code:
Screenshot code taken from: github.com/acdx/opencv-screen-capture
My main code: codeshare.io/vLio1

Answer 1

I have included my code for finding a Template image from your Desktop image. Hope this solves your problem!
#include <fstream>
#include <memory>
#include <string>
#include <iostream>
#include <strstream>
#include <functional>
#include <Windows.h>
#include <iostream>
#include <string>
using namespace std;
using namespace cv;
Mat hwnd2mat(HWND hwnd){
HDC hwindowDC,hwindowCompatibleDC;
int height,width,srcheight,srcwidth;
HBITMAP hbwindow;
Mat src;
BITMAPINFOHEADER bi;
hwindowDC=GetDC(hwnd);
hwindowCompatibleDC=CreateCompatibleDC(hwindowDC);
SetStretchBltMode(hwindowCompatibleDC,COLORONCOLOR);
RECT windowsize; // get the height and width of the screen
GetClientRect(hwnd, &windowsize);
srcheight = windowsize.bottom;
srcwidth = windowsize.right;
height = windowsize.bottom; //change this to whatever size you want to resize to
width = windowsize.right;
src.create(height,width,CV_8UC4);
// create a bitmap
hbwindow = CreateCompatibleBitmap( hwindowDC, width, height);
bi.biSize = sizeof(BITMAPINFOHEADER); //http://msdn.microsoft.com/en-us/library/windows/window/dd183402%28v=vs.85%29.aspx
bi.biWidth = width;
bi.biHeight = -height; //this is the line that makes it draw upside down or not
bi.biPlanes = 1;
bi.biBitCount = 32;
bi.biCompression = BI_RGB;
bi.biSizeImage = 0;
bi.biXPelsPerMeter = 0;
bi.biYPelsPerMeter = 0;
bi.biClrUsed = 0;
bi.biClrImportant = 0;
// use the previously created device context with the bitmap
SelectObject(hwindowCompatibleDC, hbwindow);
// copy from the window device context to the bitmap device context
StretchBlt( hwindowCompatibleDC, 0,0, width, height, hwindowDC, 0, 0,srcwidth,srcheight, SRCCOPY); //change SRCCOPY to NOTSRCCOPY for wacky colors !
GetDIBits(hwindowCompatibleDC,hbwindow,0,height,src.data,(BITMAPINFO *)&bi,DIB_RGB_COLORS); //copy from hwindowCompatibleDC to hbwindow
// avoid memory leak
DeleteObject (hbwindow); DeleteDC(hwindowCompatibleDC); ReleaseDC(hwnd, hwindowDC);
return src;
}
bool NMultipleTemplateMatching(Mat mInput,Mat mTemplate,float Threshold,float Closeness,vector<Point2f> &List_Matches)
{
Mat mResult;
Size szTemplate= mTemplate.size();
Size szTemplateCloseRadius((szTemplate.width/2)* Closeness,(szTemplate.height/2)* Closeness);
matchTemplate(mInput, mTemplate, mResult, TM_CCOEFF_NORMED);
threshold(mResult, mResult, Threshold, 1.0, THRESH_TOZERO);
while (true)
{
double minval, maxval ;
Point minloc, maxloc;
minMaxLoc(mResult, &minval, &maxval, &minloc, &maxloc);
if (maxval >= Threshold)
{
List_Matches.push_back(maxloc);
rectangle(mResult,Point2f(maxloc.x-szTemplateCloseRadius.width,maxloc.y-szTemplateCloseRadius.height),Point2f(maxloc.x+szTemplateCloseRadius.width,maxloc.y+szTemplateCloseRadius.height),Scalar(0),-1);
}
else
break;
}
//imshow("reference", mDebug_Bgr);
return true;
}
int main(int argc, char** argv)
{
Mat mTemplate_Bgr,mTemplate_Gray;
mTemplate_Bgr= imread("Template.png",1);
imshow("mTemplate_Bgr",mTemplate_Bgr);
HWND hDesktopWnd;
hDesktopWnd=GetDesktopWindow();
Mat mScreenShot= hwnd2mat(hDesktopWnd);
Mat mSource_Gray,mResult_Bgr= mScreenShot.clone();
float Threshold= 0.9;
float Closeness= 0.9;
vector<Point2f> List_Matches;
cvtColor(mScreenShot,mSource_Gray,COLOR_BGR2GRAY);
cvtColor(mTemplate_Bgr,mTemplate_Gray,COLOR_BGR2GRAY);
namedWindow("Screen Shot",WINDOW_AUTOSIZE);
imshow("Screen Shot",mSource_Gray);
NMultipleTemplateMatching(mSource_Gray,mTemplate_Gray,Threshold,Closeness,List_Matches);
for (int i = 0; i < List_Matches.size(); i++)
{
rectangle(mResult_Bgr,List_Matches[i],Point(List_Matches[i].x + mTemplate_Bgr.cols, List_Matches[i].y + mTemplate_Bgr.rows),Scalar(0,255,0), 2);
}
imshow("Final Results",mResult_Bgr);
waitKey(0);
return 0;
}

Answer 2

To expand on Balaji's very useful answer, make sure that all the Mats being passed to the function are of the same type with the .type() function. You can change src.create(height,width,CV_8UC4); to the same type as template (or vice versa) and the error should be gone. Here's another version of the solution.