Related
I need to find the yaw, roll and pitch angles from a webcam stream of a face. I used opencv to get face landmarks and detect the face. Can someone help or guide me on how to find and print those angles?
I am using Visual Studio 2022 (community).
Ptr<Facemark> facemark; //mark detection
CascadeClassifier faceDetector; //face detection
void process(Mat img, Mat imgcol) {
vector<Rect> faces;
faceDetector.detectMultiScale(img, faces);
Mat imFace;
if (faces.size() != 0) {
for (size_t i = 0; i < faces.size(); i++)
{
cv::rectangle(imgcol, faces[i], Scalar(255, 0, 0));
imFace = imgcol(faces[i]);
resize(imFace, imFace, Size(imFace.cols * 5, imFace.rows * 5));
faces[i] = Rect(faces[i].x = 0, faces[i].y = 0, faces[i].width * 5,
(faces[i].height) * 5);
}
vector< vector<Point2f> > shapes;
if (facemark->fit(imFace, faces, shapes))
{
for (unsigned long i = 0; i < faces.size(); i++) {
for (unsigned long k = 0; k < shapes[i].size(); k++) {
cv::circle(imFace, shapes[i][k], 5, cv::Scalar(0, 0, 255),FILLED);
}
}
}
namedWindow("Detected_shape");
imshow("Detected_shape", imFace);
waitKey(5);
}
else {
cout << "Faces not detected." << endl;
}
}
int main()
{
facemark = FacemarkLBF::create();
facemark->loadModel("lbfmodel.yml");
faceDetector.load("D:/opencv/build/install/etc/haarcascades/haarcascade_frontalface_alt2.xml");
cout << "Loaded model" << endl;
VideoCapture cap(1); //capture image
int initialized = 0;
for (;;)
{
if (!cap.isOpened()) {
cout << "Video Capture Fail" << endl;
break;
}
else {
Mat img; //image containers
Mat imgbw;
cap >> img; //image from webcam
resize(img, img, Size(460, 460), 0, 0, INTER_LINEAR_EXACT);
cvtColor(img, imgbw, COLOR_BGR2GRAY);
process(imgbw, img);
namedWindow("Live", WINDOW_AUTOSIZE);
imshow("Live", img);
waitKey(5);
}
}
}
The code includes what I could do so far. It detects a face and landmarks. How do I find the angles and print them from this?
I use below code for object detection.
int main(int argc, char* argv[]){
VideoCapture cap(0);
if (!cap.isOpened()){
cout << "Cannot open the video cam" << endl;
return -1;}
int totalFrameNumber = cap.get(CV_CAP_PROP_FRAME_COUNT);
Mat frame;
namedWindow("MyVideo", CV_WINDOW_AUTOSIZE);
while (1)
{
bool bSuccess = cap.read(frame); // read a new frame from video
if (!bSuccess)
{
cout << "Cannot read a frame from video stream" << endl;
break;
}
Mat frame2;
Rect rectangle2(420,280, 40, 40);
rectangle(frame, rectangle2, Scalar(255, 255, 255));
Mat cornerstrength;
cornerHarris(frame, cornerstrength, 3, 3, 0.1);
//threshold the corner strength
Mat harriscorners;
double th = 0.00001;
threshold(cornerstrength, harriscorners, th, 255, THRESH_BINARY);
morphologyEx(harriscorners, harriscorners, MORPH_CLOSE, Mat(), Point(-1, -1), 6);
//local maxima detection
Mat dilated, localMax;
dilate(cornerstrength, dilated, Mat());
compare(cornerstrength, dilated, localMax, CMP_EQ);
threshold(cornerstrength, harriscorners, th, 255, THRESH_BINARY);
harriscorners.convertTo(harriscorners, CV_8U);
bitwise_and(harriscorners, localMax, harriscorners);
harriscorners.convertTo(harriscorners, CV_32F);
Mat S(0, 2, CV_32SC1);
//drawing a circle around corners
for (int j = 0;j < harriscorners.rows;j++)
for (int i = 0;i < harriscorners.cols;i++)
{
if (harriscorners.at<float>(j, i)> 0)
{ circle(frame, Point(i, j), 5, Scalar(255), 2, 8);
Mat pt(1, 2, CV_32SC1);
pt.at<int>(1, 0) = i;
pt.at<int>(0, 1) = j;
// Add the point to S
S.push_back(pt);
for (int x = 430; x < 460; x++)
for (int y = 285; y < 315; y++)
if ((pt.at<int>(1, 0) = i) == x && (pt.at<int>(0, 1) = j) == y))
{
Rect rectangle2(430, 285, 30,30);}}}
imshow("MyVideo", frame);
if (waitKey(30) == 27)
{
cout << "esc key is pressed by user" << endl;
break;
}
}
return 0;
}
I want When the condition if ((pt.at(1, 0) = i) == x && (pt.at(0, 1) = j) == y)) is established then the size of rectangle2 from
rectangle2(420,280,40,40) Change to rectangle2(430,285,30,30).I can do this change but when I use my code Both rectangle(previous rectangle2 and new rectangle2)are displayed in picture.but I want to display onle new rectangle2.do you have any idea to solve my problem? thanks a lot..
As I said in my comments, you need to create a clone and keep frame almost a constant.
So, in this part:
Rect rectangle2(420,280, 40, 40);
rectangle(frame, rectangle2, Scalar(255, 255, 255));
Mat cornerstrength;
cornerHarris(frame, cornerstrength, 3, 3, 0.1);
Write something more like this
cv::Mat frameCopy = frame.clone();
Rect rectangle2(420,280, 40, 40);
rectangle(frameCopy , rectangle2, Scalar(255, 255, 255));
Mat cornerstrength;
cornerHarris(frameCopy , cornerstrength, 3, 3, 0.1);
Then, in this part, I am not sure what it is intended to do
for (int x = 430; x < 460; x++)
for (int y = 285; y < 315; y++)
if ((pt.at<int>(1, 0) = i) == x && (pt.at<int>(0, 1) = j) == y))
{
Rect rectangle2(430, 285, 30,30);
}
}
}
imshow("MyVideo", frame);
But probably you want to show the new image with a new rectangle, so You can do something like before again:
cv::Mat anotherCopy= frame.clone();
Rect rectangleInLoop(430,280,30,30);
rectangle(anotherCopy, rectangleInLoop, Scalar(255, 255, 255));
imshow("MyVideo", anotherCopy);
I make cognizing number on ptz camera using Opencv.
But my program has big problem.
Error in `./main.out': double free or corruption (!prev):
I think it is leaking memory. So I try to 'clear()' for desallocating memory. But it does not work.
How to fix it?
#include "opencv2/opencv.hpp"
#include <iostream>
#include <unistd.h>
#include <time.h>
using namespace cv;
using namespace std;
Mat3b canvas;
string buttonText("Click me!");
Mat frame1;
Mat frame2;
Mat frame3;
Rect rect, temp_rect;
double ratio, delta_x, delta_y, gradient;
int count, friend_count = 0, refinery_count = 0;
void testfunc()
{
vector<Vec4i> hierarchy;
vector<vector<Point> > contours;
findContours(frame3, contours, hierarchy, CV_RETR_TREE, CV_CHAIN_APPROX_SIMPLE, Point());
vector<vector<Point> > contours_poly(contours.size());
vector<Rect> boundRect(contours.size());
vector<Rect> boundRect2(contours.size());
for (int i = 0; i < contours.size(); i++) {
approxPolyDP(Mat(contours[i]), contours_poly[i], 1, true);
boundRect[i] = boundingRect(Mat(contours_poly[i]));
}
Mat drawing = Mat::zeros(frame3.size(), CV_8UC3);
for (int i = 0; i < contours.size(); i++)
{
ratio = (double)boundRect[i].height / boundRect[i].width;
if ((ratio <= 2.5) && (ratio >= 0.5) && (boundRect[i].area() <= 700) && (boundRect[i].area() >= 100))
{
drawContours(drawing, contours, i, Scalar(0, 255, 255), 1, 8, hierarchy, 0, Point());
rectangle(drawing, boundRect[i].tl(), boundRect[i].br(), Scalar(255, 0, 0), 1, 8, 0);
boundRect2[refinery_count] = boundRect[i];
refinery_count++;
}
}
boundRect2.resize(refinery_count);
imshow("camera4", drawing);
contours_poly.clear();
boundRect.clear();
boundRect2.clear();
contours.clear();
hierarchy.clear();
return;
}
int main(int argc, char** argv)
{
VideoCapture cap1(0);
if (!cap1.isOpened())
{
printf("ERROR. \n");
return -1;
}
// ************* This is notebook camera **********
// ************* This is PTZ camera **********
/*VideoCapture cap2;
string vStreamArs = "rtsp://root:pass#192.168.10.235/ufirststream";
Mat video;
if (!cap2.open(vStreamArs))
{
cout << "[-] ERROR CODE 0 : Not connect camera!!" << endl;
return -1;
}
else
{
cout << "[+] Camera is connected!!" << endl;
}*/
// ************* This is PTZ camera **********
// ************* This is notebook camera **********
int select, plate_width;
while (1)
{
clock_t start = clock();
while (1)
{
if (!cap1.read(frame1))
{
cout << "[-] ERROR CODE 2 : No camera" << endl;
break;
}
imshow("output", frame1);
if ((clock() - start) / CLOCKS_PER_SEC > 10)
{
cout << "[+]Find" << endl;
break;
}
waitKey(1);
}
cvtColor(frame1, frame2, CV_BGR2GRAY);
Canny(frame2, frame3, 100, 300, 3);
imshow("camera3", frame3);
testfunc();
}
getchar();
return 0;
}
Try to use release() method over Mat images after using them. This will force to release memory. Please try and tell me.
EDIT: I found the cause of the problem, the fisheye::undistortImage() function was not working correctly, I replaced it with estimateNewCameraMatrixForUndistortRectify(), initUndistortRectifyMap(), and remap() as in the original calibrate camera example. Not perfect yet but going in the right direction. Output image: http://imgur.com/a/Xm5vq
Mat output;
Mat newK;
Mat view, map1, map2;
Size newSize(1200, 1200);
Mat rview(newSize, frame.type());
//resize(rview, rview, newSize);
fisheye::estimateNewCameraMatrixForUndistortRectify(K, D, frame.size(), Matx33d::eye(), newK, 1);
fisheye::initUndistortRectifyMap(K, D, Matx33d::eye(), newK, frame.size(), CV_16SC2, map1, map2);
//fisheye::undistortImage(frame, output, K, D, identity);
remap(frame, rview, map1, map2, INTER_LINEAR);
imshow("Image View", rview);
imshow(window_name, frame);
if (waitKey(50) == 27) {
break;
}
Original post:
I'm trying to calibrate and undistort an image coming from an 180 degree fisheye USB camera. Most of this code is from existing examples that claim to be functional.
The code runs fine until fisheye::undistortImage where the output image is very distorted and centered around the top left corner of the window.
Screen shot of the "undistorted" chess board and calibration matrix outputs -
http://imgur.com/a/RTIoT
What am I missing here?
int main(int argc, char** argv) {
VideoCapture camera;
camera.open(1);
if (!camera.isOpened()) {
cout << "Failed to open camera." << std::endl;
return -1;
}
double fWidth = camera.get(CAP_PROP_FRAME_WIDTH);
double fHeight = camera.get(CAP_PROP_FRAME_HEIGHT);
cout << fWidth << std::endl;
cout << fHeight << std::endl;
/*
640 320
480 240
*/
const char* window_name = "output";
namedWindow(window_name, WINDOW_NORMAL);
Mat frame;
Size boardSize;
boardSize.width = 9;
boardSize.height = 6;
int remaining_frames = 30;
Mat K;// = Mat(3, 3, CV_64F, vK);
Mat D;
Mat identity = Mat::eye(3, 3, CV_64F);
vector<vector<Point2f> > img_points;
vector<vector<Point3f> > obj_points(1);
int sq_sz = 25;
for (int i = 0; i < boardSize.height; i++) {
for (int j = 0; j < boardSize.width; j++) {
obj_points[0].push_back(Point3f(float(j * sq_sz), float(i * sq_sz), 0));
}
}
obj_points.resize(remaining_frames, obj_points[0]);
bool found = false;
clock_t prevTimestamp = 0;
int delay = 500;
while (1) {
frame = nextFrame(camera);
bool blinkOutput = false;
if (remaining_frames > 0) {
vector<Point2f> corners;
int chessBoardFlags = CALIB_CB_ADAPTIVE_THRESH + CALIB_CB_NORMALIZE_IMAGE;
found = findChessboardCorners(frame, boardSize, corners, chessBoardFlags);
if (found) {
drawChessboardCorners(frame, boardSize, corners, found);
if (clock() - prevTimestamp > delay*1e-3*CLOCKS_PER_SEC) {
Mat viewGray;
cvtColor(frame, viewGray, COLOR_BGR2GRAY);
cornerSubPix(viewGray, corners, Size(11, 11), Size(-1, -1), TermCriteria(TermCriteria::EPS + TermCriteria::COUNT, 30, 0.1));
img_points.push_back(corners);
remaining_frames--;
cout << remaining_frames << " frames to calibration." << endl;
blinkOutput = true;
prevTimestamp = clock();
}
if (remaining_frames == 0) {
cout << "Computing distortion" << endl;
int flags = 0;
flags |= cv::fisheye::CALIB_RECOMPUTE_EXTRINSIC;
flags |= cv::fisheye::CALIB_CHECK_COND;
flags |= cv::fisheye::CALIB_FIX_SKEW;
fisheye::calibrate(obj_points, img_points, frame.size(), K, D, noArray(), noArray(), flags);
cout << "Finished computing distortion" << endl;
cout << K << endl;
cout << D << endl;
}
}
if (blinkOutput) { bitwise_not(frame, frame); }
cv::imshow(window_name, frame);
if (waitKey(50) == 27) {
break;
}
}
else {
Mat output;
fisheye::undistortImage(frame, output, K, D, identity);
cv::imshow(window_name, output);
if (waitKey(50) == 27) {
break;
}
}
}
return 0;
}
I'm having Heap corruption exception while compiling this code which I found on the internet. I'm a beginner and I don't know how to solve the problem.
the exception occurred in the following line:
matcher.knnMatch(desObject, des_image, matches, 2);
Could someone tell me what should I replace this line of code with?
The complete source code:
cv::initModule_nonfree();
// turn performance analysis functions on if testing = true
bool testing = false;
double t; // timing variable
// load training image
Mat object = imread("boo2.png", CV_LOAD_IMAGE_GRAYSCALE);
if (!object.data) {
cout << "Can't open image";
return -1;
}
namedWindow("Good Matches", CV_WINDOW_AUTOSIZE);
// SURF Detector, and descriptor parameters
int minHess = 3000;
vector<KeyPoint> kpObject, kpImage;
Mat desObject, desImage;
// Performance measures calculations for report
if (testing) {
cout << object.rows << " " << object.cols << endl;
// calculate integral image
Mat iObject;
integral(object, iObject);
imshow("Good Matches", iObject);
imwrite("boo2.png", iObject);
cvWaitKey(0);
// calculate number of interest points, computation time as f(minHess)
int minHessVector[] = {100, 500, 1000, 1500, 2000, 2500, 3000,
3500, 4000, 4500, 5000, 5500, 6000, 6500,
7000, 7500, 8000, 8500, 9000, 9500, 10000};
int minH;
std::ofstream file;
file.open("C:/School/Image Processing/TimingC.csv", std::ofstream::out);
for (int i = 0; i < 20; i++) {
minH = minHessVector[i];
t = (double)getTickCount();
SurfFeatureDetector detector(minH);
detector.detect(object, kpObject);
t = ((double)getTickCount() - t) / getTickFrequency();
file << minHess << "," << kpObject.size() << "," << t << ",";
cout << t << " " << kpObject.size() << " " << desObject.size() << endl;
t = (double)getTickCount();
SurfDescriptorExtractor extractor;
extractor.compute(object, kpObject, desObject);
t = ((double)getTickCount() - t) / getTickFrequency();
file << t << endl;
}
file.close();
// Display keypoints on training image
Mat interestPointObject = object;
for (unsigned int i = 0; i < kpObject.size(); i++) {
if (kpObject[i].octave) {
circle(interestPointObject, kpObject[i].pt, kpObject[i].size, 0);
string octaveS;
switch (kpObject[i].octave) {
case 0:
octaveS = "0";
break;
case 1:
octaveS = '1';
break;
case 2:
octaveS = '2';
break;
default:
break;
}
putText(interestPointObject, octaveS, kpObject[i].pt,
FONT_HERSHEY_COMPLEX_SMALL, 1, cvScalar(0, 0, 250), 1, CV_AA);
}
}
imshow("Good Matches", interestPointObject);
imwrite("C:/School/Image Processing/bookIP2.jpg", interestPointObject);
cvWaitKey(0);
}
// SURF Detector, and descriptor parameters, match object
// initialization
minHess = 2000;
SurfFeatureDetector detector(minHess);
detector.detect(object, kpObject);
SurfDescriptorExtractor extractor;
extractor.compute(object, kpObject, desObject);
FlannBasedMatcher matcher;
// Initialize video and display window
VideoCapture cap(1); // camera 1 is webcam
if (!cap.isOpened()) return -1;
// Object corner points for plotting box
vector<Point2f> obj_corners(4);
obj_corners[0] = cvPoint(0, 0);
obj_corners[1] = cvPoint(object.cols, 0);
obj_corners[2] = cvPoint(object.cols, object.rows);
obj_corners[3] = cvPoint(0, object.rows);
// video loop
char escapeKey = 'k';
double frameCount = 0;
float thresholdMatchingNN = 0.7;
unsigned int thresholdGoodMatches = 4;
unsigned int thresholdGoodMatchesV[] = {4, 5, 6, 7, 8, 9, 10};
for (int j = 0; j < 7; j++) {
thresholdGoodMatches = thresholdGoodMatchesV[j];
// thresholdGoodMatches=8;
cout << thresholdGoodMatches << endl;
if (true) {
t = (double)getTickCount();
}
while (escapeKey != 'q') {
frameCount++;
Mat frame;
Mat image;
cap >> frame;
cvtColor(frame, image, CV_RGB2GRAY);
Mat des_image, img_matches, H;
vector<KeyPoint> kp_image;
vector<vector<DMatch>> matches;
vector<DMatch> good_matches;
vector<Point2f> obj;
vector<Point2f> scene;
vector<Point2f> scene_corners(4);
detector.detect(image, kp_image);
extractor.compute(image, kp_image, des_image);
matcher.knnMatch(desObject, des_image, matches, 2);
for (int i = 0; i < min(des_image.rows - 1, (int)matches.size()); i++)
// THIS LOOP IS SENSITIVE TO SEGFAULTS
{
if ((matches[i][0].distance <
thresholdMatchingNN * (matches[i][1].distance)) &&
((int)matches[i].size() <= 2 && (int)matches[i].size() > 0)) {
good_matches.push_back(matches[i][0]);
}
}
// if (good_matches.size()<1)
// good_matches.resize(0,cv::DMatch);
// Draw only "good" matches
drawMatches(object, kpObject, image, kp_image, good_matches, img_matches,
Scalar::all(-1), Scalar::all(-1), vector<char>(),
DrawMatchesFlags::NOT_DRAW_SINGLE_POINTS);
if (good_matches.size() >= thresholdGoodMatches) {
// Display that the object is found
putText(img_matches, "Object Found", cvPoint(10, 50),
FONT_HERSHEY_COMPLEX_SMALL, 2, cvScalar(0, 0, 250), 1, CV_AA);
for (unsigned int i = 0; i < good_matches.size(); i++) {
// Get the keypoints from the good matches
obj.push_back(kpObject[good_matches[i].queryIdx].pt);
scene.push_back(kp_image[good_matches[i].trainIdx].pt);
}
H = findHomography(obj, scene, CV_RANSAC);
perspectiveTransform(obj_corners, scene_corners, H);
// Draw lines between the corners (the mapped object in the scene
// image )
line(img_matches, scene_corners[0] + Point2f(object.cols, 0),
scene_corners[1] + Point2f(object.cols, 0), Scalar(0, 255, 0), 4);
line(img_matches, scene_corners[1] + Point2f(object.cols, 0),
scene_corners[2] + Point2f(object.cols, 0), Scalar(0, 255, 0), 4);
line(img_matches, scene_corners[2] + Point2f(object.cols, 0),
scene_corners[3] + Point2f(object.cols, 0), Scalar(0, 255, 0), 4);
line(img_matches, scene_corners[3] + Point2f(object.cols, 0),
scene_corners[0] + Point2f(object.cols, 0), Scalar(0, 255, 0), 4);
} else {
putText(img_matches, "", cvPoint(10, 50), FONT_HERSHEY_COMPLEX_SMALL, 3,
cvScalar(0, 0, 250), 1, CV_AA);
}
// Show detected matches
imshow("Good Matches", img_matches);
escapeKey = cvWaitKey(10);
// imwrite("C:/School/Image Processing/bookIP3.jpg", img_matches);
if (frameCount > 10) escapeKey = 'q';
}
// average frames per second
if (true) {
t = ((double)getTickCount() - t) / getTickFrequency();
cout << t << " " << frameCount / t << endl;
cvWaitKey(0);
}
frameCount = 0;
escapeKey = 'a';
}
// Release camera and exit
cap.release();