imshow() function freezes - c++

I am developing some video analytic algorithms using openCV. However, after I process a frame and want to display it on a window, it hangs at imshow() function. I have searched for this issue online but still cannot find the problem! Here is the code where I am using multithread and openCV:
void CFeatureExtraction::extract(){
boost::thread OpFlowThread, BGSThread;
while (m_nRun){
ImgPtr frame_ptr;
m_pQueue->wait_and_pop(frame_ptr);
Mat frame1, frame2;
(*frame_ptr).copyTo(frame1);
(*frame_ptr).copyTo(frame2);
OpFlowThread = boost::thread(&COpFlow::op_flow, m_pAlgo,frame1);
BGSThread = boost::thread(&CBGSub::bgsub, m_pBGS, frame2);
OpFlowThread.join();
BGSThread.join();
}
}
And inside op_flow and bgsub function, I am using imshow() and cvWaitKey() together! But it keeps hanging the whole program.
If my question is still not clear for you, pls feel free to ask me for more detail.
Here is the detail code of calling imshow():
CFarnebackAlgo::CFarnebackAlgo() : COpFlow()
{
namedWindow("Optical Flow");
}
CFarnebackAlgo::~CFarnebackAlgo()
{
destroyWindow("Optical Flow");
}
int CFarnebackAlgo::op_flow(Mat frame)
{
Mat flow;
cvtColor(frame, gray, COLOR_BGR2GRAY);
if (prevgray.data)
{
calcOpticalFlowFarneback(prevgray, gray, flow, 0.5, 3, 15, 3, 5, 1.2, 0);
for (int y = 0; y < frame.rows; y += 8)
for (int x = 0; x < frame.cols; x += 8){
const Point2f& fxy = flow.at<Point2f>(y, x);
line(frame, Point(x, y), Point(cvRound(x + fxy.x), cvRound(y + fxy.y)), Scalar(0, 255, 0));
circle(frame, Point(x, y), 2, Scalar(0, 255, 0), -1);
}
if (frame.data == NULL){
cout << "No Img!" << endl;
exit(0);
}
imshow("Optical Flow", frame);
waitKey(50);
}
std::swap(prevgray, gray);
return 0;
}
If I put the namedWindow() in the op_flow(), it is working. But if I put it in the constructor, it's not working and freezes. Anyone knows why?

Related

corrupted size vs. prev_size in C++ [closed]

Closed. This question needs debugging details. It is not currently accepting answers.
Edit the question to include desired behavior, a specific problem or error, and the shortest code necessary to reproduce the problem. This will help others answer the question.
Closed 11 months ago.
Locked. There are disputes about this question’s content being resolved at this time. It is not currently accepting new answers or interactions.
I am new to c++, I started with it today and I got an error that I can't solve myself.
When I run my code it sometimes works fine, and I sometimes get an error after a few seconds.
The errors that I have gotten are:
"corrupted size vs. prev_size Aborted (core dumped)"
"segmentation fault (core dumped)"
This is my code:
#include <opencv2/opencv.hpp>
#include <iostream>
#include <random>
using namespace cv;
using namespace std;
using chrono::duration;
using chrono::duration_cast;
using chrono::high_resolution_clock;
using chrono::milliseconds;
class Product
{
public:
Point location = Point(-80, 0);
Scalar color;
Product(int given_y, Scalar given_color)
{
location.y = given_y;
color = given_color;
cout << "Product created" << endl;
}
Mat update(double speed, Mat img, double dt)
{
location.x += speed* dt;
circle(img, location, 25, Scalar(255, 255, 255), FILLED, LINE_8);
circle(img, location, 20, color, FILLED, LINE_8);
putText(img, "12", location + Point(-17, +7), FONT_HERSHEY_DUPLEX, .8, Scalar(255, 255, 255), 1, LINE_AA);
return (img);
}
};
int main(int argc, char **argv)
{
// Constant variables
const int kScreenWidth = 1920;
const int kScreenHeight = 1080;
const int kUpdateFpsMilliseconds = 600;
const String kWindowName = "Moving dots";
const Mat kBlackImg(kScreenHeight, kScreenWidth, CV_8UC3, Scalar(0, 0, 0));
// Variables
Scalar red = Scalar(255, 0, 0);
Scalar green = Scalar(0, 255, 0);
Scalar blue = Scalar(0, 0, 255);
Scalar cyan = Scalar(0, 255, 255);
Scalar magenta = Scalar(255, 0, 255);
Scalar yellow = Scalar(255, 255, 0);
Scalar black = Scalar(0, 0, 0);
Scalar white = Scalar(255, 255, 255);
vector<Product> products;
Mat img;
double speed = .2;
double fps;
int frame_counter = 0;
int loop_dt;
int time_since_fps_update = 0;
int counter = 0;
bool remove_product = false;
bool full_screen = false;
// Random number generators
random_device rd_seed;
mt19937 rng(rd_seed());
uniform_int_distribution<mt19937::result_type> rd_screen_height(50, kScreenHeight - 50);
uniform_int_distribution<mt19937::result_type> rd_new_product(0, 100);
namedWindow(kWindowName, WINDOW_NORMAL);
setWindowProperty(kWindowName, WND_PROP_FULLSCREEN, WINDOW_FULLSCREEN);
auto loop_start = high_resolution_clock::now();
while (true)
{
// Black image
Mat img = kBlackImg.clone();
// Add new products
if (rd_new_product(rng) < .1 * loop_dt)
{
products.push_back(Product(rd_screen_height(rng), red));
}
// Update products
if (products.size() > 0)
{
for (int i = 0; i <= products.size(); i++)
{
img = products[i].update(speed, img, loop_dt);
}
}
// Show text and border
stringstream temp;
temp << "FPS: " << fps << " Products: " << products.size();
putText(img, temp.str(), Point(18, 38), FONT_HERSHEY_DUPLEX, .8, white, 1, LINE_AA);
rectangle(img, Point(6, 6), Point(kScreenWidth - 6, kScreenHeight - 6), white, 12);
rectangle(img, Point(6, 6), Point(kScreenWidth - 6, 60 - 6), white, 12);
rectangle(img, Point(6, 6), Point(kScreenWidth - 6, kScreenHeight - 6), cyan, 3);
rectangle(img, Point(6, 6), Point(kScreenWidth - 6, 60 - 6), cyan, 3);
// Update screen
cvtColor(img, img, COLOR_RGB2BGR);
imshow(kWindowName, img);
// Exit script
if (waitKey(1) >= 0)
{
break;
};
// Handle FPS counter
frame_counter++;
loop_dt = duration_cast<milliseconds>(high_resolution_clock::now() - loop_start).count();
loop_start = high_resolution_clock::now();
time_since_fps_update += loop_dt;
if (time_since_fps_update > kUpdateFpsMilliseconds)
{
fps = (double)frame_counter / (double)time_since_fps_update * 1000;
time_since_fps_update -= kUpdateFpsMilliseconds;
frame_counter = 0;
}
}
destroyWindow(kWindowName);
return 0;
}
I think that it might have something to do with the color scalars or with the products vector.
Any help is appreciated!
The main culprit is likely:
for (int i = 0; i <= products.size(); i++)
{
img = products[i].update(speed, img, loop_dt);
}
That should be < products.size(). Vectors use indices from 0 to size()-1, like pretty much all other std containers.

OpenCV C++ detectMultiScale crashes UE4 - Access violation writing location

I am trying to create an OpenCV plugin for Unreal Engine 4 to detect faces, but when the code reaches the detectMultiScale function I get this error:
Exception thrown at 0x00007FFD4F34FF1B (UE4Editor-Core.dll) in
UE4Editor-Win64-DebugGame.exe: 0xC0000005: Access violation writing
location 0x0000000000000010.
I am using OpenCV version 4.1.0 with VS2017. I also tried with version 3.4.7 of OpenCV, but having the same issue.
This is my function that gets the webcam input and renders it in UE4:
In the constructor:
if (!face_cascade.load("C:\\models\\haarcascade_frontalface_alt.xml")) { printf("--(!)Error loading\n"); }
And the function:
void AWebcamReader::UpdateTexture()
{
std::vector<cv::Rect> faces;
cv::Mat frame_gray;
//cvtColor(frame, frame_gray, cv::COLOR_BGR2GRAY);
if (isStreamOpen && frame.data)
{
cv::cvtColor(frame, frame_gray, cv::COLOR_BGR2GRAY);
cv::equalizeHist(frame_gray, frame_gray);
face_cascade.detectMultiScale(frame_gray, faces, 1.1, 2, 0 | cv::CASCADE_SCALE_IMAGE, cv::Size(30, 30));
for (size_t i = 0; i < faces.size(); i++)
{
cv::Point center(faces[i].x + faces[i].width*0.5, faces[i].y + faces[i].height*0.5);
ellipse(frame, center, cv::Size(faces[i].width*0.5, faces[i].height*0.5), 0, 0, 360, cv::Scalar(255, 0, 255), 4, 8, 0);
cv::Mat faceROI = frame_gray(faces[i]);
}
// Copy Mat data to Data array
for (int y = 0; y < VideoSize.Y; y++)
{
for (int x = 0; x < VideoSize.X; x++)
{
int i = x + (y * VideoSize.X);
Data[i].B = frame.data[i * 3 + 0];
Data[i].G = frame.data[i * 3 + 1];
Data[i].R = frame.data[i * 3 + 2];
}
}
// Update texture 2D
UpdateTextureRegions(VideoTexture, (int32)0, (uint32)1, VideoUpdateTextureRegion, (uint32)(4 * VideoSize.X), (uint32)4, (uint8*)Data.GetData(), false);
}
}
If I comment out the detectMultiScale function, the plugin runs without a problem and I see the webcam input rendered correctly.
The Exception gets called on this line in VT2.cpp (VT2 is the project name)
IMPLEMENT_PRIMARY_GAME_MODULE( FDefaultGameModuleImpl, VT2, "VT2" );
Any suggestions would be greatly appreciated!

Dlib not detecting face in kurento opencv filter

I have created an opencv filter that can detect if a person blinks for Kurento the WebRTC framework. My code works in a standalone opencv app. However, once I converted to the opencv filter for Kurento it started playing up. When the module/filter was compiled without optimisation flags it would briefly detect the face and draw contours around the eyes. However, after compiling the module/filter with optimisation flags, performance improved, but no face was being detected. Here's the code I have in the filter:
void BlinkDetectorOpenCVImpl::process(cv::Mat &mat) {
std::vector <dlib::rectangle> faces;
// Just resize input image if you want
resize(mat, mat, Size(800, 450));
cv_image <rgb_alpha_pixel> cimg(mat);
dlib::array2d<unsigned char> img_gray;
dlib::assign_image(img_gray, cimg);
faces = detector(img_gray);
std::cout << "XXXXXXXXXXXXXXXXXXXXX FACES: " << faces.size() << std::endl;
std::vector <full_object_detection> shapes;
for (unsigned long i = 0; i < faces.size(); ++i) {
full_object_detection shape = pose_model(cimg, faces[i]);
std::vector <Point> left_eye_points = get_points_for_eye(shape, LEFT_EYE_START, LEFT_EYE_END);
std::vector <Point> right_eye_points = get_points_for_eye(shape, RIGHT_EYE_START, RIGHT_EYE_END);
double left_eye_ear = get_eye_aspect_ratio(left_eye_points);
double right_eye_ear = get_eye_aspect_ratio(right_eye_points);
double ear = (left_eye_ear + right_eye_ear) / 2.0;
// Draw left eye
std::vector <std::vector<Point>> contours;
contours.push_back(left_eye_points);
std::vector <std::vector<Point>> hull(1);
convexHull(contours[0], hull[0]);
drawContours(mat, hull, -1, Scalar(0, 255, 0));
// Draw right eye
contours[0] = right_eye_points;
convexHull(contours[0], hull[0]);
drawContours(mat, hull, -1, Scalar(0, 255, 0));
if (ear < EYE_AR_THRESH) {
counter++;
} else {
if (counter >= EYE_AR_CONSEC_FRAMES) {
total++;
/* std::string sJson = "{\"blink\": \"blink\"}";
try
{
onResult event(getSharedFromThis(), onResult::getName(), sJson);
signalonResult(event);
}
catch (std::bad_weak_ptr &e)
{
}*/
}
counter = 0;
}
cv::putText(mat, (boost::format{"Blinks: %d"} % total).str(), cv::Point(10, 30),
cv::FONT_HERSHEY_SIMPLEX,
0.7, Scalar(0, 0, 255), 2);
cv::putText(mat, (boost::format{"EAR: %.2f"} % ear).str(), cv::Point(300, 30),
cv::FONT_HERSHEY_SIMPLEX,
0.7, Scalar(0, 0, 255), 2);
}
}
} /* blinkdetector */
I was able to fix my own problem. I found that instead of resizing the image to an arbitrary resolution you should resize it by half the width and half the height of the actual image resolution. Resizing an image to a lower size makes Dlib face detection fast. So here's what I did to solve the issue:
Mat tmpMat = mat.clone();
resize(tmpMat, tmpMat, Size(tmpMat.size().width / 2, tmpMat.size().height / 2));
I had to clone the image sent by Kurento to my method because for some odd reason the original Mat doesn't show the contours when turned into a Dlib image with cv_image.

Dynamically changing video in OpenCV

I have 3 video source connected to my PC and I want to show them on one screen.
I initially started to put the video sources next to each other and that works fine, but I want to be able to enable/disable each video source at run time.
So I am want to use keyboard keys (r(right) and l(left)) to change what cameras are being shown at the moment.
I want to move the declaration of the following 3 variables to the outside of the while loop so I can access them in the if-cases and change them.
cv::Mat3b combinedFrame(camRightSize.height, camMiddleSize.width + camRightSize.width);
cv::Mat3b leftSideOfScreen(combinedFrame, Rect(0, 0, camMiddleSize.width, camMiddleSize.height));
cameraMiddleFrameMirroredResize.copyTo(leftSideOfScreen);
cv::Mat3b rightSideOfScreen(combinedFrame, Rect(camMiddleSize.width, 0, camRightSize.width, camRightSize.height));
Below is my whole code:
int main(int argc, char **argv) {
int combinedScreenWidth = 1440;
int combinedScreenHeight = 540;
int rearCameraBiggerByThis = 200;
int combinedScreenWidthHalv = combinedScreenWidth / 2;
bool showRight = true;
bool showLeft = false;
//initialize and allocate memory to load the video stream from camera
cv::VideoCapture cameraRight(0); // RIGHT
cv::VideoCapture cameraMiddle(3); // REAR
cv::VideoCapture cameraLeft(3); // LEFT
if (!cameraRight.isOpened()) return 1;
if (!cameraMiddle.isOpened()) return 1;
if (!cameraLeft.isOpened()) return 1;
cv::Mat3b cameraRightFrame;
cv::Mat3b cameraMiddleFrame;
cv::Mat3b cameraLeftFrame;
cv::Mat3b cameraRightFrameMirrored;
cv::Mat3b cameraMiddleFrameMirrored;
cv::Mat3b cameraLeftFrameMirrored;
Size camRightSize;
Size camMiddleSize;
Size camLeftSize;
cv::Mat3b cameraRightFrameMirroredResize;
cv::Mat3b cameraMiddleFrameMirroredResize;
cv::Mat3b cameraLeftFrameMirroredResize;
while (true) {
// Grab and retrieve each frames of the video sequentially
cameraRight >> cameraRightFrame;
cameraMiddle >> cameraMiddleFrame;
cameraLeft >> cameraLeftFrame;
// Mirror
cv::flip(cameraRightFrame, cameraRightFrameMirrored, 1);
cv::flip(cameraMiddleFrame, cameraMiddleFrameMirrored, 1);
cv::flip(cameraLeftFrame, cameraMiddleFrameMirrored, 1);
// Resize
camRightSize = cameraRightFrame.size();
camMiddleSize = cameraMiddleFrame.size();
camLeftSize = cameraLeftFrame.size();
resize(cameraMiddleFrameMirrored, cameraMiddleFrameMirroredResize, Size(combinedScreenWidthHalv + rearCameraBiggerByThis, combinedScreenHeight));
resize(cameraRightFrameMirrored, cameraRightFrameMirroredResize, Size(combinedScreenWidthHalv - rearCameraBiggerByThis, combinedScreenHeight));
// Compilation
camRightSize = cameraRightFrameMirroredResize.size();
camMiddleSize = cameraMiddleFrameMirroredResize.size();
camLeftSize = cameraLeftFrameMirroredResize.size();
if (showRight && showLeft) { // LEFT + REAR + RIGHT
} else if (showRight) { // REAR + RIGHT
} else if (showLeft) { // LEFT + REAR
} else { // REAR
}
cv::Mat3b combinedFrame(camRightSize.height, camMiddleSize.width + camRightSize.width);
cv::Mat3b leftSideOfScreen(combinedFrame, Rect(0, 0, camMiddleSize.width, camMiddleSize.height));
cameraMiddleFrameMirroredResize.copyTo(leftSideOfScreen);
cv::Mat3b rightSideOfScreen(combinedFrame, Rect(camMiddleSize.width, 0, camRightSize.width, camRightSize.height));
cameraRightFrameMirroredResize.copyTo(rightSideOfScreen);
// declare windows
cv:namedWindow("Combined", CV_WINDOW_NORMAL);
cv::setWindowProperty("Combined", CV_WND_PROP_FULLSCREEN, CV_WINDOW_FULLSCREEN);
cv::putText(combinedFrame, "REAR", cv::Point(500, 50), cv::FONT_HERSHEY_PLAIN, 2.0, cv::Scalar(255, 255, 255), 2 );
cv::putText(combinedFrame, "RIGHT", cv::Point(950, 50), cv::FONT_HERSHEY_PLAIN, 2.0, cv::Scalar(255, 255, 255), 2 );
cv::imshow("Combined", combinedFrame); // 1440 x 540 Screen size
//cv::imshow("Right Cam", cameraRightFrame);
//cv::imshow("Middle Cam", cameraMiddleFrame);
//cv::imshow("Left Cam", cameraLeftFrame);
//wait for 40 milliseconds
int c = cvWaitKey(1);
//exit the loop if user press "Esc" key (ASCII value of "Esc" is 27)
if (27 == char(c)) {
break;
}
else if (114 == char(c)) {
showRight = !showRight;
}
else if (108 == char(c)) {
showLeft = !showLeft;
}
}
return 0;
}
According to your description, I think what you're wanting to write is:
combinedFrame(Rect(0, 0, camMiddleSize.width, camMiddleSize.height)).copyTo(leftSideOfScreen);
That is:
create a cv::Mat from another one (call to cv::Mat::copyTo(cv::Mat&))
the "source matrix" is obtained by extracting a rectangle from combinedFrame : combinedFrame(Rect(0, 0, camMiddleSize.width, camMiddleSize.height))

OpencCV: calcOpticalFlowSF function

I try to use the calcOpticalFlowSF() function, but when I launch it, the programm doesn't repond, here the part of the code that use it:
frame1 = cv::imread("frame10.png");
frame2 = cv::imread("frame11.png");
if (frame1.empty()) {
cout<<"could not read image oldori"<<endl;
return;
}
if (frame2.empty()) {
cout<<"could not read image ori"<<endl;
return;
}
if (frame1.rows != frame2.rows && frame1.cols != frame2.cols) {
cout<<"images should be of equal sizes "endl;
return;
}
if (frame1.type() != 16 || frame2.type() != 16) {
cout<<"images should be of equal type CV_8UC3")endl;
return;
}
cv::Mat flow;
cv::calcOpticalFlowSF(frame1, frame2, flow, 2, 2, 4);
// calcOpticalFlowSF(frame1, frame1, // doesn't work too.
// flow,
// 3, 2, 4, 4.1, 25.5, 18, 55.0, 25.5, 0.35, 18, 55.0, 25.5, 10);
I know that the error come from the function calcOpticalFlowSF, because if I comment it, the programm works. I use the same pictures as they use in the SimpleFlow demo. If you look here:How to get cv::calcOpticalFlowSF to work? it seems that he got no problem with the function itself...
Do you have an idea why it doesn't work?
thanks,
best regards.
make sure the image is 3-channel, is better to see the docs here
#if 1
#define _CRT_SECURE_NO_WARNINGS
#include <cstdio>
#include <cstdlib>
#include <vector>
#include <stdlib.h>
#include <opencv2/opencv.hpp>
#include "opencv2/optflow.hpp"
#include "opencv2/highgui.hpp"
using namespace std;
using namespace cv;
using namespace cv::optflow;//calcOpticalFlowSF 's namespace
const size_t choice = 2 ;
// choice
// 1 2 3
// calcOpticalFlowFarneback calcOpticalFlowSF calcOpticalFlowPyrLK
void drawOptFlowMap(const Mat& flow, Mat& cflowmap, int step, const Scalar& color) {
// cflowmap is the pre frame with the line of Optical Flow
// flow is a xxx array, store float point
// store the delta of x,y
// step is every step pixel
for (int y = 0; y < cflowmap.rows; y += step)
for (int x = 0; x < cflowmap.cols; x += step)
{
const Point2f& fxy = flow.at< Point2f>(y, x);
line(cflowmap, Point(x, y), Point(cvRound(x + fxy.x), cvRound(y + fxy.y)),
color);
circle(cflowmap, Point(cvRound(x + fxy.x), cvRound(y + fxy.y)), 1, color, -1);
}
}
void drawOptFlowMap(Mat& cflowmap, int step, const Scalar& color, vector<Point2f> &retPts) {
// same as above, retPts is the next frame point
auto it = retPts.begin();
for (int y = 0; y < cflowmap.rows; y += step)
for (int x = 0; x < cflowmap.cols; x += step)
{
line(cflowmap, Point(x, y), *it, color);
circle(cflowmap, *it, 1, color, -1);
it++;
}
}
int main(int argc, char *argv[])
{
Mat flow;//flow = aft - pre
Mat pre = imread("1hf.png", IMREAD_COLOR);
Mat aft = imread("2hf.png", IMREAD_COLOR);// CV_LOAD_IMAGE_GRAYSCALE gray ; IMREAD_COLOR color
if (pre.empty() || aft.empty()){
printf("Unable to load the image");
return 1;
}
Mat cflow = pre; Mat cflow2 = aft;// store the 3-channel mat of frame, cflow is for show color with line
cvtColor(pre, pre, CV_BGR2GRAY);
cvtColor(aft, aft, CV_BGR2GRAY);
//below parameter of calcOpticalFlowPyrLK
vector<Point2f> prePts;
size_t step = 10;
vector<Point2f> nextPts(pre.rows * pre.cols);
vector<uchar> status;
vector<float> err;
TermCriteria termcrit(CV_TERMCRIT_ITER | CV_TERMCRIT_EPS, 20, 0.03);
switch (choice)
{
case 1:// calcOpticalFlowFarneback
calcOpticalFlowFarneback(pre, aft, flow, 0.5, 3, 15, 3, 5, 1.2, 0); // info in the flow; note that input mat should in 1-channel
drawOptFlowMap(flow, cflow, 10, CV_RGB(0, 255, 0)); break;
case 2:// calcOpticalFlowSF
calcOpticalFlowSF(cflow, cflow2,
flow,
3, 2, 4, 4.1, 25.5, 18, 55.0, 25.5, 0.35, 18, 55.0, 25.5, 10);// info in the flow; note that input mat should in 3-channel
drawOptFlowMap(flow, cflow, 10, CV_RGB(0, 255, 0)); break;
case 3:// calcOpticalFlowPyrLK
for (int y = 0; y < pre.rows; y += step)
for (int x = 0; x < pre.cols; x += step)
{
prePts.push_back(Point(x, y));
}
// above get a point vector in step
calcOpticalFlowPyrLK(pre, aft, prePts, nextPts, status, err, Size(31, 31), 3, termcrit, 0, 0.001);// info in the flow; note that input mat should in 1-channel
drawOptFlowMap(cflow, step, CV_RGB(0, 255, 0), nextPts);
break;
default:
break;
}
imshow("pre", pre);
imshow("after", aft);
//cflow is the pre with OpticalFlow line
imshow("pre with OpticalFlow line", cflow);
waitKey(0);
return 0;
}
#endif