How to combine OpenCV + Tesseract? - c++

Question

I am trying to create a bot to recognize text in an inactive window. To do this, I capture each frame using Bitmap. And I want Tesseract to scan the text (if there is one) on this frame and display it on the screen.
#include <iostream>
#include <opencv2/imgcodecs.hpp>
#include <opencv2/highgui.hpp>
#include <opencv2/imgproc.hpp>
#include <Windows.h>
#include <string>
#include <tesseract/capi.h>
#include <tesseract/baseapi.h>
#include <leptonica/allheaders.h>
using namespace cv;
Mat getMat(HWND hwnd) {
HDC deviceContext = GetDC(hwnd);
HDC memoryDeviceContext = CreateCompatibleDC(deviceContext);
RECT windowRect;
GetClientRect(hwnd, &windowRect);
int height = 500; /// windowRect.bottom windowRect.right
int width = 500;
HBITMAP bitmap = CreateCompatibleBitmap(deviceContext, width, height);
SelectObject(memoryDeviceContext, bitmap);
// Copy data into bitmap
BitBlt(memoryDeviceContext, 0, 0, width, height, deviceContext, 750, 359, SRCCOPY);
// Spesify format by using bitmapinfoheader
BITMAPINFOHEADER bi;
bi.biSize = sizeof(BITMAPINFOHEADER);
bi.biWidth = width;
bi.biHeight = -height;
bi.biPlanes = 1;
bi.biBitCount = 32;
bi.biCompression = BI_RGB;
bi.biSizeImage = 0; // No compression
bi.biXPelsPerMeter = 1;
bi.biYPelsPerMeter = 2;
bi.biClrUsed = 3;
bi.biClrImportant = 4;
Mat mat = Mat(height, width, CV_8UC4); // 8 bit unsigned ints 4 channels -> RGBA
// Transform data and store into mat.data
GetDIBits(memoryDeviceContext, bitmap, 0, height, mat.data, (BITMAPINFO*)&bi, DIB_RGB_COLORS);
// Clean up
DeleteObject(bitmap);
DeleteDC(memoryDeviceContext);
ReleaseDC(hwnd, deviceContext);
return mat;
}
int main() {
LPCWSTR windowTitle = L"Discord";
HWND hwnd = FindWindow(NULL, windowTitle);
/// String outText, imPath = argv[1];
/// tesseract::TessBaseAPI* temp = new tesseract::TessBaseAPI();
std::cout << "Start" << "\n";
while (true) {
Mat temp = getMat(hwnd);
cv::imshow("output", temp);
cv::waitKey(2);
}
std::cout << "Done\n";
return 1;
}
My attempts were unsuccessful:
while (true) {
Mat temp = getMat(hwnd);
cv::imshow("output", temp);
cv::waitKey(2);
String outText, imPath = argv[1];
tesseract::TessBaseAPI* temp = new tesseract::TessBaseAPI();
cv::Mat img = cv::imread((const unsigned char*)temp);
ocr->Init(NULL, "eng", tesseract::OEM_LSTM_ONLY);
ocr->SetPageSegMode(tesseract::PSM_AUTO);
ocr->SetImage(img);
outText = String(ocr->GetUTF8Text());
}
Another attempt
while (true) {
Mat temp = getMat(hwnd);
cv::imshow("output", temp);
cv::waitKey(2);
tesseract::TessBaseAPI* ocr = new tesseract::TessBaseAPI();
char* outText;
ocr->SetPageSegMode(tesseract::PSM_AUTO);
ocr->Init(NULL, "eng", tesseract::OEM_LSTM_ONLY);
ocr->SetImage(temp, 0, 0, 0, 0);
outText = ocr->GetUTF8Text();
ocr->End();
delete ocr;
return outText;
}
I just want every frame from Bitmap to be scanned for the presence of text. And this text was displayed. I will be glad of any help. Thanks

Answer 1

Just google for it. e.g. blogpost OpenCV and tesseract or example in tesseract doc.