Sharpening video images using openCV - c++

I would like to sharpen my image using OpenCV and I have looked at an example online which is performing the sharpening on a grayscale image and I tried it and it is working perfectly. However, I am now trying to do the same but with RGB colors so I am performing the same functionality on the three channel separately, but it is not giving me any result the image is exactly as the original image.
#include "Sharpening.h"
using namespace std;
Sharpening::Sharpening() {
}
Sharpening::~Sharpening() {
}
IplImage* Sharpening::laplace(IplImage* channel) {
CvSize size = cvSize(channel->width, channel->height);
IplImage* temp = cvCreateImage(size, IPL_DEPTH_8U, 1);
IplImage* lapl = cvCreateImage(size, IPL_DEPTH_8U, 1);
int width = size.width;
int height = size.height;
cvConvertScale(channel, temp, 1.0);
CvMat* ker = cvCreateMat(3, 3, CV_32FC1);
cvSet(ker, cvScalarAll(-1.0));
cvSet2D(ker, 1, 1, cvScalarAll(15.0));
cout << "this is been executed";
cvFilter2D(temp, lapl, ker);
cvReleaseMat(&ker);
double maxv = 0.0;
float maxFloat = 1.79769e+308;
double minv = maxFloat;
cvMinMaxLoc(lapl, &minv, &maxv);
for (int i = 0; i < width * height; i++) {
double lap_val = cvGet1D(lapl, i).val[0];
int v = (int) ((255.0 * lap_val / maxv) + 0.5); // this calculation does nothing particularly
cvSet1D(temp, i, cvScalarAll(v));
}
maxv = 0.0;
cvMinMaxLoc(channel, &minv, &maxv);
for (int i = 0; i < width * height; i++) {
double val = cvGet1D(channel, i).val[0];
int v = (int) ((255.0 * val / maxv) + 0.5);
cvSet1D(channel, i, cvScalarAll(v));
}
cvReleaseImage(&temp);
cvReleaseImage(&lapl);
cvReleaseMat(&ker);
return channel;
} // end of function
int Sharpening::calculateLoop(int number) {
int value = 2;
for (int i = 0; i < 10; i++) {
number = number * value;
cout << number << endl;
}
return number;
}
//======================================================================================
int Sharpening::SharpenColored(Sharpening sharp) {
int key = 0;
CvCapture *capture = 0;
IplImage* frame = 0;
cvNamedWindow("deblur", CV_WINDOW_AUTOSIZE);
cvNamedWindow("deblur2", CV_WINDOW_AUTOSIZE);
cvNamedWindow("origional", CV_WINDOW_AUTOSIZE);
// initialize camera
capture = cvCaptureFromCAM(0); //capture from a camera
//capture = cvCaptureFromAVI("jabelH2.avi");
//frame = cvQueryFrame(capture);
if (!cvGrabFrame(capture)) { // capture a frame
printf("Could not grab a frame\n\7");
exit(0);
}
frame = cvQueryFrame(capture);
CvSize imageSize1 = cvSize(frame->width, frame->height);
IplImage* R = cvCreateImage(imageSize1, IPL_DEPTH_8U, 1);
IplImage* G = cvCreateImage(imageSize1, IPL_DEPTH_8U, 1);
IplImage* B = cvCreateImage(imageSize1, IPL_DEPTH_8U, 1);
IplImage* R2 = cvCreateImage(imageSize1, IPL_DEPTH_8U, 1);
IplImage* G2 = cvCreateImage(imageSize1, IPL_DEPTH_8U, 1);
IplImage* B2 = cvCreateImage(imageSize1, IPL_DEPTH_8U, 1);
IplImage* source = cvCreateImage(imageSize1, IPL_DEPTH_8U, 3);
IplImage* result = cvCreateImage(imageSize1, IPL_DEPTH_8U, 3);
IplImage* result2 = cvCreateImage(imageSize1, IPL_DEPTH_8U, 3);
QFuture<IplImage*> future1;
QFuture<IplImage*> future2;
QFuture<IplImage*> future3;
while (key != 'q') {
// get a frame
frame = cvQueryFrame(capture);
// always check
if (!frame)
break;
source = frame;
cvSplit(frame, B, G, R, NULL);
future1 = QtConcurrent::run(sharp, &Sharpening::laplace, R);
future2 = QtConcurrent::run(sharp, &Sharpening::laplace, G);
future3 = QtConcurrent::run(sharp, &Sharpening::laplace, B);
R2 = future1.result();
G2 = future2.result();
B2 = future3.result();
cvMerge(B2, G2, R2, NULL, result);
cvAdd(source, result, result2, NULL);
cvShowImage("origional", source);
cvShowImage("deblur", R2);
cvShowImage("deblur2", G2);
key = cvWaitKey(1);
} //end of while
cvDestroyWindow("deblur");
cvDestroyWindow("deblur2");
cvDestroyWindow("origional");
cvReleaseImage(&R);
cvReleaseImage(&source);
cvReleaseImage(&R2);
cvReleaseImage(&G);
cvReleaseImage(&G2);
cvReleaseImage(&B);
cvReleaseImage(&B2);
cvReleaseImage(&result);
cvReleaseImage(&result2);
cvReleaseCapture(&capture);
delete future1;
delete future2;
delete future3;
return 0;
} //end of function
//======================================================================================
int main(int argc, char *argv[]) {
Sharpening sh;
sh.SharpenColored(sh);
}

I am now trying to do the same but with RGB colors so I am performing
the same functionality
Check your assumptions ! I don't think you are doing the same. I don't know openCv
very well, but your not doing anything with your temp image, so there is no reason for the channel image to change ! You should set the result of the temp image in the original image, something like this perhaps :
for (int i = 0; i < width * height; i++) {
double lap_val = cvGet1D(lapl, i).val[0]; // get modified image data
int v = (int) ((255.0 * lap_val / maxv) + 0.5); // scale to 0 255
cvSet1D(channel, i, cvScalarAll(v)); // store in original image
}
Or you can take the original code, and comment it to explain what each cvImage contains, and see what you have missed when reusing it in your app.

Related

OpenCV compiling issues

With compiling this project I have several issues:
Due to "file not found" I had to change header
from:
#include <cv.h>
#include <highgui.h>
to:
#include <opencv.hpp>
#include <highgui.hpp>
that solved Include issues but compiling give several more faults:
What I'm doing wrong?
Below whole code:
#include <stdio.h>
#include <unistd.h>
#include <opencv.hpp>
#include <highgui.hpp>
int main(int argc, char **argv)
{
// get command line parameters
if(argc < 6)
{
printf("Usage: %s <video> <vga window x> <vga window y> <vga width> <vga height>\n", argv[0]);
printf("Usage: %s <camera id> <vga window x> <vga window y> <vga width> <vga height>\n", argv[0]);
exit(0);
}
const char *filename = argv[1];
int vgaX = atoi(argv[2]);
int vgaY = atoi(argv[3]);
int vgaWidth = atoi(argv[4]);
int vgaHeight = atoi(argv[5]);
int outputPixelCount = vgaWidth * vgaHeight;
// opencv initializations
CvCapture* cap = cvCaptureFromFile(filename);
int isVideoFile = cap != 0;
if (!cap)
cap = cvCaptureFromCAM(atoi(filename));
if (!cap)
{
printf("Could not open file/camera!\n");
exit(1);
}
IplImage* frame = cvQueryFrame(cap); // get first frame for size
if (!frame)
{
printf("The Video is empty!\n");
cvReleaseCapture(&cap);
exit(1);
}
IplImage* edges = cvCreateImage(cvGetSize(frame), IPL_DEPTH_8U, 1);
IplImage* lines = cvCreateImage(cvGetSize(frame), IPL_DEPTH_8U, 3);
IplImage* out = cvCreateImage(cvSize(vgaWidth, vgaHeight), IPL_DEPTH_8U, 3);
int outStep = out->widthStep;
int outChannels = out->nChannels;
unsigned char *outData = (unsigned char*)out->imageData;
// position windows // TODO: make debug output windows optional?
cvNamedWindow("frame", CV_WINDOW_AUTOSIZE);
cvMoveWindow("frame", 0, 32);
cvNamedWindow("edges", CV_WINDOW_AUTOSIZE);
cvMoveWindow("edges", frame->width, 32);
cvNamedWindow("lines", CV_WINDOW_AUTOSIZE);
cvMoveWindow("lines", 2 * frame->width, 32);
cvNamedWindow("out", CV_WINDOW_AUTOSIZE);
cvMoveWindow("out", vgaX, vgaY);
while (42)
{
frame = cvQueryFrame(cap);
if (!frame || (cvWaitKey(1) & 0xff) == 'q')
break;
cvShowImage("frame", frame);
// edge detection
cvCanny(frame, edges, 128.0, 130.0, 3); // TODO: tweakable parameters?
cvShowImage("edges", edges);
// get contours
CvMemStorage *storage = cvCreateMemStorage(0);
CvSeq *contours;
int contourCount = cvFindContours(
edges, storage, &contours, sizeof(CvContour),
CV_RETR_LIST, CV_CHAIN_APPROX_SIMPLE, cvPoint(0, 0));
cvZero(lines);
cvDrawContours(lines, contours, cvScalar(32, 255, 32, 255), cvScalarAll(0), 100, 1, 8, cvPoint(0, 0));
cvShowImage("lines", lines);
// calculate total length over all contours
float contourLengthSum = 0.0f;
for(CvSeq *c = contours; c; c = c->h_next)
{
for(int i = 0; i < c->total - 1; i++)
{
CvPoint *p0 = CV_GET_SEQ_ELEM(CvPoint, c, i);
CvPoint *p1 = CV_GET_SEQ_ELEM(CvPoint, c, i + 1);
int dx = p1->x - p0->x;
int dy = p1->y - p0->y;
contourLengthSum += sqrtf(dx * dx + dy * dy);
}
}
float factor = (float)outputPixelCount / contourLengthSum;
// write output image
int cx = 0, cy = 0;
float xScale = 255.0f / frame->width;
float yScale = 255.0f / frame->height;
unsigned char *dp = &outData[1];
for(CvSeq *c = contours; c; c = c->h_next)
{
CvPoint *p0 = CV_GET_SEQ_ELEM(CvPoint, c, 0);
for(int i = 1; i < c->total; i++)
{
CvPoint *p1 = CV_GET_SEQ_ELEM(CvPoint, c, i);
float x0x1 = p1->x - p0->x;
float y0y1 = p1->y - p0->y;
int n = (int)(sqrtf(x0x1 * x0x1 + y0y1 * y0y1) * factor);
float x = (float)p0->x * xScale;
float y = 255.0f - (float)p0->y * yScale;
float dt = 1.0f / (float)(n - 1);
float dx = dt * x0x1 * xScale;
float dy = dt * -y0y1 * yScale;
for (int j = 0; j < n; j++)
{
dp[0] = (unsigned char)x;
dp[1] = (unsigned char)y;
x += dx;
y += dy;
dp += outChannels;
if (++cx == vgaWidth)
{
cx = 0;
dp = &outData[++cy * outStep + 1];
if (cy == vgaHeight)
goto full;
}
}
p0 = p1;
}
}
// fill last few pixels with last pixel value, if there are any left
for (; cy < vgaHeight; cy++)
{
for (;cx < vgaWidth; cx++)
{
outData[cy * outStep + cx * outChannels + 1] = 0;
outData[cy * outStep + cx * outChannels + 2] = 0;
}
cx = 0;
}
full:
cvReleaseMemStorage(&storage);
cvShowImage("out", out);
//if (isVideoFile)
// usleep(8000); // TODO: proper synchronization
}
cvReleaseImage(&out);
cvReleaseImage(&lines);
cvReleaseImage(&edges);
cvReleaseCapture(&cap);
return 0;
}
Looks like you are using old OpenCV API. They are replaced with new methods. Your code does not work with recent OpenCV 4.3.0. cvCaptureFromFile, cvCaptureFromCAM and some others exists in 4.0.0-rc of OpenCV docs.
https://docs.opencv.org/4.0.0-rc/dd/d01/group__videoio__c.html
After 4.0.0-rc of OpenCV the documentation points to videoio.
https://docs.opencv.org/master/dd/de7/group__videoio.html
Further Reference:
https://answers.opencv.org/question/55344/undeclared-indentifier-opencv-cvcapturefromcam-and-cvqueryframe/
Also if you have include problems in new OpenCV version try,
#include <opencv2/opencv.hpp>
#include <opencv2/highgui.hpp>

Compute coordinates from source images after stitching

I use an algorithm of panorama stitching from opencv, in order to stitch 2 or 3 images into one new result image.
I have coordinates of points in each source image. I need to calculate what are the new coordinates for these points in the result image.
I describe below the algorithm. My code is similar to a sample "stitching_detailed" from opencv (branch 3.4). A result_mask of type Mat is produced, maybe it is the solution? But I don't know how to use it. I found a related question here but not on stitching.
Any idea?
Here is the algorithm (for detailed code: stitching_detailed.cpp):
Find features for each image:
Ptr<FeaturesFinder> finder = makePtr<SurfFeaturesFinder>()
vector<ImageFeatures> features(num_images);
for (int i = 0; i < num_images; ++i)
{
(*finder)(images[i], features[i]);
}
Make pairwise_matches:
vector<MatchesInfo> pairwise_matches;
Ptr<FeaturesMatcher> matcher = makePtr<BestOf2NearestMatcher>(false, match_conf);
(*matcher)(features, pairwise_matches);
Reorder the images:
vector<int> indices = leaveBiggestComponent(features, pairwise_matches, conf_thresh);
# here some code to reorder 'images'
Estimate an homography in cameras:
vector<CameraParams> cameras;
Ptr<Estimator> estimator = makePtr<HomographyBasedEstimator>();
(*estimator)(features, pairwise_matches, cameras);
Convert to CV_32F:
for (size_t i = 0; i < cameras.size(); ++i)
{
Mat R;
cameras[i].R.convertTo(R, CV_32F);
cameras[i].R = R;
}
Execute a BundleAdjuster:
Ptr<detail::BundleAdjusterBase> adjuster = makePtr<detail::BundleAdjusterRay>();
adjuster->setConfThresh(conf_thresh);
adjuster->setRefinementMask(refine_mask);
(*adjuster)(features, pairwise_matches, cameras);
Compute a value for warped_image_scale:
for (int i = 0; i < cameras.size(); ++i)
focals.push_back(cameras[i].focal);
float warped_image_scale = static_cast<float>(focals[focals.size() / 2 - 1] + focals[focals.size() / 2]) * 0.5f;
Do wave correction:
vector<Mat> rmats;
for (size_t i = 0; i < cameras.size(); ++i)
rmats.push_back(cameras[i].R.clone());
waveCorrect(rmats, wave_correct);
for (size_t i = 0; i < cameras.size(); ++i)
cameras[i].R = rmats[i];
Create a warper:
Ptr<WarperCreator> warper_creator = makePtr<cv::SphericalWarper>();
Ptr<RotationWarper> warper = warper_creator->create(static_cast<float>(warped_image_scale * seam_work_aspect));
Create a blender and feed it:
Ptr<Blender> blender;
for (size_t i = 0; i < cameras.size(); ++i)
{
full_img = input_imgs[img_idx];
if (!is_compose_scale_set)
{
is_compose_scale_set = true;
compose_scale = /* … */
}
if (abs(compose_scale - 1) > 1e-1)
resize(full_img, img, Size(), compose_scale, compose_scale, INTER_LINEAR_EXACT);
else
img = full_img;
// Warp the current image
warper->warp(img, K, cameras[img_idx].R, INTER_LINEAR, BORDER_REFLECT, img_warped);
// Warp the current image mask
mask.create(img_size, CV_8U);
mask.setTo(Scalar::all(255));
warper->warp(mask, K, cameras[img_idx].R, INTER_NEAREST, BORDER_CONSTANT, mask_warped);
// Compensate exposure
compensator->apply(img_idx, corners[img_idx], img_warped, mask_warped);
dilate(masks_warped[img_idx], dilated_mask, Mat());
resize(dilated_mask, seam_mask, mask_warped.size(), 0, 0, INTER_LINEAR_EXACT);
mask_warped = seam_mask & mask_warped;
if (!blender)
{
blender = Blender::createDefault(blend_type, try_gpu);
Size dst_sz = resultRoi(corners, sizes).size();
float blend_width = sqrt(static_cast<float>(dst_sz.area())) * blend_strength / 100.f;
MultiBandBlender *mb = dynamic_cast<MultiBandBlender *>(blender.get());
mb->setNumBands(static_cast<int>(ceil(log(blend_width) / log(2.)) - 1.));
blender->prepare(corners, sizes);
}
// Blend the current image
blender->feed(img_warped_s, mask_warped, corners[i]);
}
Then, use the blender:
Mat result, result_mask;
blender->blend(result, result_mask);
// The result image is in 'result'
When I was a school boy, I foundopencv/samples/cpp/stitching_detailed.cpp in OpenCV samples folder. At that time, my programming skills were very poor. I can't understand it even though I racked my brains. This question attracts my attention, and arouses my memory. After a whole night of hard work and debugging, I finally get it.
Basic steps:
Given the three images: blue.png, green.png, and red.png
We can get the stitching result(result.png) using the stitching_detailed.cpp.
.
blender->blend(result, result_mask);
imwrite("result.png", result);
imwrite("result_mask.png", result_mask);
I choose the centers from the three images, and calculate the corresponding coordinates (warped) on the stitching image, and draw in solid as follow:
Warping images (auxiliary)...
Compensating exposure...
Blending ...
Warp each center point, and draw solid circle.
[408, 204] => [532, 224]
[408, 204] => [359, 301]
[408, 204] => [727, 320]
Check `result.png`, `result_mask.png` and `result2.png`!
Done!
This is the function calcWarpedPoint I wrote to calculate the warped point on the stitching image:
cv::Point2f calcWarpedPoint(
const cv::Point2f& pt,
InputArray K, // Camera K parameter
InputArray R, // Camera R parameter
Ptr<RotationWarper> warper, // The Rotation Warper
const std::vector<cv::Point> &corners,
const std::vector<cv::Size> &sizes)
{
// Calculate the wrapped point using camera parameter.
cv::Point2f dst = warper->warpPoint(pt, K, R);
// Calculate the stitching image roi using corners and sizes.
// the corners and sizes have already been calculated.
cv::Point2f tl = cv::detail::resultRoi(corners, sizes).tl();
// Finally adjust the wrapped point to the stitching image.
return cv::Point2f(dst.x - tl.x, dst.y - tl.y);
}
This is example code snippet:
std::cout << "\nWarp each center point, and draw solid circle.\n";
std::vector<cv::Scalar> colors = { {255,0,0}, {0, 255, 0}, {0, 0, 255} };
for (int idx = 0; idx < img_names.size(); ++idx) {
img = cv::imread(img_names[idx]);
Mat K;
cameras[idx].K().convertTo(K, CV_32F);
Mat R = cameras[idx].R;
cv::Point2f cpt = cv::Point2f(img.cols / 2, img.rows / 2);
cv::Point pt = calcWarpedPoint(cpt, K, R, warper, corners, sizes);
cv::circle(result, pt, 5, colors[idx], -1, cv::LINE_AA);
std::cout << cpt << " => " << pt << std::endl;
}
std::cout << "\nCheck `result.png`, `result_mask.png` and `result2.png`!\n";
imwrite("result2.png", result);
The full code:
/*
* Author : Kinght-金(https://stackoverflow.com/users/3547485/)
* Created : 2019/03/01 23:00 (CST)
* Finished : 2019/03/01 07:50 (CST)
*
* Modified on opencv401/samples/cpp/stitching_detailed.cpp
* From https://github.com/opencv/opencv/blob/4.0.1/samples/cpp/stitching_detailed.cpp
*
*
* Description: A simple opencv(4.0.1) image stitching code for Stack Overflow answers.
* For https://stackoverflow.com/questions/54904718/compute-coordinates-from-source-images-after-stitching/54953792#comment96681412_54953792
*
*/
#include <iostream>
#include <fstream>
#include <string>
#include "opencv2/opencv_modules.hpp"
#include <opencv2/core/utility.hpp>
#include "opencv2/imgcodecs.hpp"
#include "opencv2/highgui.hpp"
#include "opencv2/stitching/detail/autocalib.hpp"
#include "opencv2/stitching/detail/blenders.hpp"
#include "opencv2/stitching/detail/camera.hpp"
#include "opencv2/stitching/detail/exposure_compensate.hpp"
#include "opencv2/stitching/detail/matchers.hpp"
#include "opencv2/stitching/detail/motion_estimators.hpp"
#include "opencv2/stitching/detail/seam_finders.hpp"
#include "opencv2/stitching/detail/warpers.hpp"
#include "opencv2/stitching/warpers.hpp"
using namespace std;
using namespace cv;
using namespace cv::detail;
//! img_names are the input image (full) paths
// You can download from using the links from the answer.
//! Blue: https://i.stack.imgur.com/Yz3U1.png
//! Green: https://i.stack.imgur.com/AbUTH.png
//! Red: https://i.stack.imgur.com/9wcGc.png
vector<String> img_names = {"D:/stitching/blue.png", "D:/stitching/green.png", "D:/stitching/red.png"};
//! The function to calculate the warped point on the stitching image.
cv::Point2f calcWarpedPoint(
const cv::Point2f& pt,
InputArray K, // Camera K parameter
InputArray R, // Camera R parameter
Ptr<RotationWarper> warper, // The Rotation Warper
const std::vector<cv::Point> &corners,
const std::vector<cv::Size> &sizes)
{
// Calculate the wrapped point
cv::Point2f dst = warper->warpPoint(pt, K, R);
// Calculate the stitching image roi using corners and sizes,
// the corners and sizes have already been calculated.
cv::Point2f tl = cv::detail::resultRoi(corners, sizes).tl();
// Finally adjust the wrapped point
return cv::Point2f(dst.x - tl.x, dst.y - tl.y);
}
int main(int argc, char* argv[])
{
double work_megapix = 0.6;
double seam_megapix = 0.1;
double compose_megapix = -1;
float conf_thresh = 1.f;
float match_conf = 0.3f;
float blend_strength = 5;
// Check if have enough images
int num_images = static_cast<int>(img_names.size());
if (num_images < 2)
{
std::cout << "Need more images\n";
return -1;
}
double work_scale = 1, seam_scale = 1, compose_scale = 1;
bool is_work_scale_set = false, is_seam_scale_set = false, is_compose_scale_set = false;
//(1) 创建特征查找器
Ptr<Feature2D> finder = ORB::create();
// (2) 读取图像,适当缩放,并计算图像的特征描述
Mat full_img, img;
vector<ImageFeatures> features(num_images);
vector<Mat> images(num_images);
vector<Size> full_img_sizes(num_images);
double seam_work_aspect = 1;
for (int i = 0; i < num_images; ++i)
{
full_img = imread(img_names[i]);
full_img_sizes[i] = full_img.size();
if (full_img.empty())
{
cout << "Can't open image " << img_names[i] << std::endl;
return -1;
}
if (!is_work_scale_set)
{
work_scale = min(1.0, sqrt(work_megapix * 1e6 / full_img.size().area()));
is_work_scale_set = true;
}
resize(full_img, img, Size(), work_scale, work_scale, INTER_LINEAR_EXACT);
if (!is_seam_scale_set)
{
seam_scale = min(1.0, sqrt(seam_megapix * 1e6 / full_img.size().area()));
seam_work_aspect = seam_scale / work_scale;
is_seam_scale_set = true;
}
computeImageFeatures(finder, img, features[i]);
features[i].img_idx = i;
std::cout << "Features in image #" << i + 1 << ": " << features[i].keypoints.size() << std::endl;
resize(full_img, img, Size(), seam_scale, seam_scale, INTER_LINEAR_EXACT);
images[i] = img.clone();
}
full_img.release();
img.release();
// (3) 创建图像特征匹配器,计算匹配信息
vector<MatchesInfo> pairwise_matches;
Ptr<FeaturesMatcher> matcher = makePtr<BestOf2NearestMatcher>(false, match_conf);
(*matcher)(features, pairwise_matches);
matcher->collectGarbage();
//! (4) 剔除外点,保留最确信的大成分
// Leave only images we are sure are from the same panorama
vector<int> indices = leaveBiggestComponent(features, pairwise_matches, conf_thresh);
vector<Mat> img_subset;
vector<String> img_names_subset;
vector<Size> full_img_sizes_subset;
for (size_t i = 0; i < indices.size(); ++i)
{
img_names_subset.push_back(img_names[indices[i]]);
img_subset.push_back(images[indices[i]]);
full_img_sizes_subset.push_back(full_img_sizes[indices[i]]);
}
images = img_subset;
img_names = img_names_subset;
full_img_sizes = full_img_sizes_subset;
// Check if we still have enough images
num_images = static_cast<int>(img_names.size());
if (num_images < 2)
{
std::cout << "Need more images\n";
return -1;
}
//!(5) 估计 homography
Ptr<Estimator> estimator = makePtr<HomographyBasedEstimator>();
vector<CameraParams> cameras;
if (!(*estimator)(features, pairwise_matches, cameras))
{
cout << "Homography estimation failed.\n";
return -1;
}
for (size_t i = 0; i < cameras.size(); ++i)
{
Mat R;
cameras[i].R.convertTo(R, CV_32F);
cameras[i].R = R;
std::cout << "\nInitial camera intrinsics #" << indices[i] + 1 << ":\nK:\n" << cameras[i].K() << "\nR:\n" << cameras[i].R << std::endl;
}
//(6) 创建约束调整器
Ptr<detail::BundleAdjusterBase> adjuster = makePtr<detail::BundleAdjusterRay>();
adjuster->setConfThresh(conf_thresh);
Mat_<uchar> refine_mask = Mat::zeros(3, 3, CV_8U);
refine_mask(0, 0) = 1;
refine_mask(0, 1) = 1;
refine_mask(0, 2) = 1;
refine_mask(1, 1) = 1;
refine_mask(1, 2) = 1;
adjuster->setRefinementMask(refine_mask);
if (!(*adjuster)(features, pairwise_matches, cameras))
{
cout << "Camera parameters adjusting failed.\n";
return -1;
}
// Find median focal length
vector<double> focals;
for (size_t i = 0; i < cameras.size(); ++i)
{
focals.push_back(cameras[i].focal);
}
sort(focals.begin(), focals.end());
float warped_image_scale;
if (focals.size() % 2 == 1)
warped_image_scale = static_cast<float>(focals[focals.size() / 2]);
else
warped_image_scale = static_cast<float>(focals[focals.size() / 2 - 1] + focals[focals.size() / 2]) * 0.5f;
std::cout << "\nWarping images (auxiliary)... \n";
vector<Point> corners(num_images);
vector<UMat> masks_warped(num_images);
vector<UMat> images_warped(num_images);
vector<Size> sizes(num_images);
vector<UMat> masks(num_images);
// Preapre images masks
for (int i = 0; i < num_images; ++i)
{
masks[i].create(images[i].size(), CV_8U);
masks[i].setTo(Scalar::all(255));
}
// Warp images and their masks
Ptr<WarperCreator> warper_creator = makePtr<cv::CylindricalWarper>();
if (!warper_creator)
{
cout << "Can't create the warper \n";
return 1;
}
//! Create RotationWarper
Ptr<RotationWarper> warper = warper_creator->create(static_cast<float>(warped_image_scale * seam_work_aspect));
//! Calculate warped corners/sizes/mask
for (int i = 0; i < num_images; ++i)
{
Mat_<float> K;
cameras[i].K().convertTo(K, CV_32F);
float swa = (float)seam_work_aspect;
K(0, 0) *= swa; K(0, 2) *= swa;
K(1, 1) *= swa; K(1, 2) *= swa;
corners[i] = warper->warp(images[i], K, cameras[i].R, INTER_LINEAR, BORDER_REFLECT, images_warped[i]);
sizes[i] = images_warped[i].size();
warper->warp(masks[i], K, cameras[i].R, INTER_NEAREST, BORDER_CONSTANT, masks_warped[i]);
}
vector<UMat> images_warped_f(num_images);
for (int i = 0; i < num_images; ++i)
images_warped[i].convertTo(images_warped_f[i], CV_32F);
std::cout << "Compensating exposure... \n";
//! 计算曝光度,调整图像曝光,减少亮度差异
Ptr<ExposureCompensator> compensator = ExposureCompensator::createDefault(ExposureCompensator::GAIN_BLOCKS);
if (dynamic_cast<BlocksCompensator*>(compensator.get()))
{
BlocksCompensator* bcompensator = dynamic_cast<BlocksCompensator*>(compensator.get());
bcompensator->setNrFeeds(1);
bcompensator->setNrGainsFilteringIterations(2);
bcompensator->setBlockSize(32, 32);
}
compensator->feed(corners, images_warped, masks_warped);
Ptr<SeamFinder> seam_finder = makePtr<detail::GraphCutSeamFinder>(GraphCutSeamFinderBase::COST_COLOR);
seam_finder->find(images_warped_f, corners, masks_warped);
// Release unused memory
images.clear();
images_warped.clear();
images_warped_f.clear();
masks.clear();
Mat img_warped, img_warped_s;
Mat dilated_mask, seam_mask, mask, mask_warped;
Ptr<Blender> blender;
double compose_work_aspect = 1;
for (int img_idx = 0; img_idx < num_images; ++img_idx)
{
// Read image and resize it if necessary
full_img = imread(img_names[img_idx]);
if (!is_compose_scale_set)
{
is_compose_scale_set = true;
compose_work_aspect = compose_scale / work_scale;
// Update warped image scale
warped_image_scale *= static_cast<float>(compose_work_aspect);
warper = warper_creator->create(warped_image_scale);
// Update corners and sizes
for (int i = 0; i < num_images; ++i)
{
cameras[i].focal *= compose_work_aspect;
cameras[i].ppx *= compose_work_aspect;
cameras[i].ppy *= compose_work_aspect;
Size sz = full_img_sizes[i];
if (std::abs(compose_scale - 1) > 1e-1)
{
sz.width = cvRound(full_img_sizes[i].width * compose_scale);
sz.height = cvRound(full_img_sizes[i].height * compose_scale);
}
Mat K;
cameras[i].K().convertTo(K, CV_32F);
Rect roi = warper->warpRoi(sz, K, cameras[i].R);
corners[i] = roi.tl();
sizes[i] = roi.size();
}
}
if (abs(compose_scale - 1) > 1e-1)
resize(full_img, img, Size(), compose_scale, compose_scale, INTER_LINEAR_EXACT);
else
img = full_img;
full_img.release();
Size img_size = img.size();
Mat K, R;
cameras[img_idx].K().convertTo(K, CV_32F);
R = cameras[img_idx].R;
// Warp the current image : img => img_warped
warper->warp(img, K, cameras[img_idx].R, INTER_LINEAR, BORDER_REFLECT, img_warped);
// Warp the current image mask
mask.create(img_size, CV_8U);
mask.setTo(Scalar::all(255));
warper->warp(mask, K, cameras[img_idx].R, INTER_NEAREST, BORDER_CONSTANT, mask_warped);
compensator->apply(img_idx, corners[img_idx], img_warped, mask_warped);
img_warped.convertTo(img_warped_s, CV_16S);
img_warped.release();
img.release();
mask.release();
dilate(masks_warped[img_idx], dilated_mask, Mat());
resize(dilated_mask, seam_mask, mask_warped.size(), 0, 0, INTER_LINEAR_EXACT);
mask_warped = seam_mask & mask_warped;
if (!blender)
{
blender = Blender::createDefault(Blender::MULTI_BAND, false);
Size dst_sz = resultRoi(corners, sizes).size();
float blend_width = sqrt(static_cast<float>(dst_sz.area())) * blend_strength / 100.f;
if (blend_width < 1.f){
blender = Blender::createDefault(Blender::NO, false);
}
else
{
MultiBandBlender* mb = dynamic_cast<MultiBandBlender*>(blender.get());
mb->setNumBands(static_cast<int>(ceil(log(blend_width) / log(2.)) - 1.));
}
blender->prepare(corners, sizes);
}
blender->feed(img_warped_s, mask_warped, corners[img_idx]);
}
/* ===========================================================================*/
// Blend image
std::cout << "\nBlending ...\n";
Mat result, result_mask;
blender->blend(result, result_mask);
imwrite("result.png", result);
imwrite("result_mask.png", result_mask);
std::cout << "\nWarp each center point, and draw solid circle.\n";
std::vector<cv::Scalar> colors = { {255,0,0}, {0, 255, 0}, {0, 0, 255} };
for (int idx = 0; idx < img_names.size(); ++idx) {
img = cv::imread(img_names[idx]);
Mat K;
cameras[idx].K().convertTo(K, CV_32F);
Mat R = cameras[idx].R;
cv::Point2f cpt = cv::Point2f(img.cols / 2, img.rows / 2);
cv::Point pt = calcWarpedPoint(cpt, K, R, warper, corners, sizes);
cv::circle(result, pt, 5, colors[idx], -1, cv::LINE_AA);
std::cout << cpt << " => " << pt << std::endl;
}
std::cout << "\nCheck `result.png`, `result_mask.png` and `result2.png`!\n";
imwrite("result2.png", result);
std::cout << "\nDone!\n";
/* ===========================================================================*/
return 0;
}
Some links maybe useful:
stitching_detailed.cpp : https://github.com/opencv/opencv/blob/4.0.1/samples/cpp/stitching_detailed.cpp
waper->warp(), warpPoint(), warpRoi() https://github.com/opencv/opencv/blob/master/modules/stitching/src/warpers.cpp#L153
resultRoi() https://github.com/opencv/opencv/blob/master/modules/stitching/src/util.cpp#L116
Other links maybe interesting:
Converting opencv remap code from c++ to python
Split text lines in scanned document
How do I use the relationships between Flann matches to determine a sensible homography?

(OpenCV Stitching) How to get better performance using OpenCV Stitcher class?

I have some problems when using Stitcher class.
First, I use ORB Feature Finder because it's faster than SURF.
but it's still slow.
Second, Stitcher class accuracy is too low.
Third, How can I get more performance by using Stitcher class?
Additional, How can I catch directions between two images?
This is my code.
Thank you.
#include "opencv2/core/core.hpp"
#include "opencv2/highgui/highgui.hpp"
#include "opencv2/stitching.hpp"
#include "opencv2/features2d.hpp"
using namespace cv;
using namespace std;
void overlayImage(const cv::Mat &background, const cv::Mat &foreground, cv::Mat &output, cv::Point2i location);
int main(int argc, char* argv[])
{
Mat first;
Mat second;
Mat m_first;
Mat m_second;
vector<Mat> images;
// vector<Mat> re_images;
Mat panorama;
Mat result;
unsigned long t;
t = getTickCount();
first = imread(argv[1], CV_LOAD_IMAGE_COLOR);
second = imread(argv[2], CV_LOAD_IMAGE_COLOR);
//Mat m_first = Mat::zeros( first.size(), first.type() );
//Mat m_second = Mat::zeros( second.size(), second.type() );
/*
for( int y = 0; y < first.rows; y++ ) {
for( int x = 0; x < first.cols; x++ ) {
for( int c = 0; c < 3; c++ ) {
m_first.at<Vec3b>(y,x)[c] = saturate_cast<uchar>( 1.2*( first.at<Vec3b>(y,x)[c] ) + 20 );
}
}
}
for( int y = 0; y < second.rows; y++ ){
for( int x = 0; x < second.cols; x++ ) {
for( int c = 0; c < 3; c++ ) {
m_second.at<Vec3b>(y,x)[c] =
saturate_cast<uchar>( 1.2*( second.at<Vec3b>(y,x)[c] ) + 20 );
}
}
}
*/
//imwrite("first.png", m_first);
//imwrite("second.png", m_second);
resize(first, m_first, Size(640, 480));
resize(second, m_second, Size(640, 480));
images.push_back(m_first);
images.push_back(m_second);
Stitcher stitcher = Stitcher::createDefault(false);
//Stitcher::Status status = stitcher.stitch(imgs, pano);
//stitcher.setWarper(new PlaneWarper());
stitcher.setWarper(new SphericalWarper());
// stitcher.setWarper(new CylindricalWarper());
stitcher.setFeaturesFinder(new detail::OrbFeaturesFinder(Size(3,1),1500));
// stitcher.setRegistrationResol(0.6);
// stitcher.setSeamEstimationResol(0.1);
// stitcher.setCompositingResol(0.5);
//stitcher.setPanoConfidenceThresh(1);
stitcher.setWaveCorrection(true);
stitcher.setWaveCorrectKind(detail::WAVE_CORRECT_HORIZ);
stitcher.setFeaturesMatcher(new detail::BestOf2NearestMatcher(false,0.3));
stitcher.setBundleAdjuster(new detail::BundleAdjusterRay());
stitcher.setBlender(new detail::MultiBandBlender());
stitcher.stitch(images, panorama);
printf("%.2lf sec \n", (getTickCount() - t) / getTickFrequency() );
Rect rect(panorama.cols / 2 - 320, panorama.rows / 2 - 240, 640, 480);
Mat subimage = panorama(rect);
Mat car = imread("car.png");
overlayImage(subimage, car, result, cv::Point(320 - (car.cols / 2), 240 - (car.rows / 2 )));
imshow("panorama", result);
// resize(panorama, result, Size(640, 480));
imwrite("result.jpg", result);
waitKey(0);
return 0;
}
void overlayImage(const cv::Mat &background, const cv::Mat &foreground, cv::Mat &output, cv::Point2i location)
{
background.copyTo(output);
// start at the row indicated by location, or at row 0 if location.y is negative.
for(int y = std::max(location.y , 0); y < background.rows; ++y)
{
int fY = y - location.y; // because of the translation
// we are done of we have processed all rows of the foreground image.
if(fY >= foreground.rows)
break;
// start at the column indicated by location,
// or at column 0 if location.x is negative.
for(int x = std::max(location.x, 0); x < background.cols; ++x)
{
int fX = x - location.x; // because of the translation.
// we are done with this row if the column is outside of the foreground image.
if(fX >= foreground.cols)
break;
// determine the opacity of the foregrond pixel, using its fourth (alpha) channel.
double opacity =
((double)foreground.data[fY * foreground.step + fX * foreground.channels() + 3])
/ 255.;
// and now combine the background and foreground pixel, using the opacity,
// but only if opacity > 0.
for(int c = 0; opacity > 0 && c < output.channels(); ++c)
{
unsigned char foregroundPx =
foreground.data[fY * foreground.step + fX * foreground.channels() + c];
unsigned char backgroundPx =
background.data[y * background.step + x * background.channels() + c];
output.data[y*output.step + output.channels()*x + c] =
backgroundPx * (1.-opacity) + foregroundPx * opacity;
}
}
}
}
FAST feature detector is faster than SURF and ORB.
Moreover, finding 1500 features in a 640*480 picture takes too much time. 300 features is ok. So you can use this code instead:
detail::OrbFeaturesFinder(Size(3,1),300));
Stitcher Class is so slow. I suggest you try to implement stitcher class yourself. Try using feature detectors, descriptors, then matching and after that find homography then making mask and then warping.
I don't understand your third question, "How can I catch directions between two images?". What do you mean exactly?

Different results with cvDFT and DFT in OpenCV 2.4.8

I'm having problems with the DFT function in OpenCV 2.4.8 for c++.
I used an image of a 10 phases sinus curve to compare the old cvDFT() with the newer c++ function DFT() (one dimensional DFT row-wise).
The old version gives me logical results: very high peak at pixel 0 and 10, the rest being almost 0.
The new version gives me strange results with peaks all over the spectrum.
Here is my code:
#include "stdafx.h"
#include <opencv2\core\core_c.h>
#include <opencv2\core\core.hpp>
#include <opencv2\imgproc\imgproc_c.h>
#include <opencv2\imgproc\imgproc.hpp>
#include <opencv2\highgui\highgui_c.h>
#include <opencv2\highgui\highgui.hpp>
#include <opencv2\legacy\compat.hpp>
using namespace cv;
void OldMakeDFT(Mat original, double* result)
{
const int width = original.cols;
const int height = 1;
IplImage* fftBlock = cvCreateImage(cvSize(width, height), IPL_DEPTH_8U, 1);
IplImage* imgReal = cvCreateImage(cvSize(width, height), IPL_DEPTH_32F, 1);
IplImage* imgImag = cvCreateImage(cvSize(width, height), IPL_DEPTH_32F, 1);
IplImage* imgDFT = cvCreateImage(cvSize(width, height), IPL_DEPTH_32F, 2);
Rect roi(0, 0, width, 1);
Mat image_roi = original(roi);
fftBlock->imageData = (char*)image_roi.data;
//cvSaveImage("C:/fftBlock1.png", fftBlock);
cvConvert(fftBlock, imgReal);
cvMerge(imgReal, imgImag, NULL, NULL, imgDFT);
cvDFT(imgDFT, imgDFT, (CV_DXT_FORWARD | CV_DXT_ROWS));
cvSplit(imgDFT, imgReal, imgImag, NULL, NULL);
double re,imag;
for (int i = 0; i < width; i++)
{
re = ((float*)imgReal->imageData)[i];
imag = ((float*)imgImag->imageData)[i];
result[i] = re * re + imag * imag;
}
cvReleaseImage(&imgReal);
cvReleaseImage(&imgImag);
cvReleaseImage(&imgDFT);
cvReleaseImage(&fftBlock);
}
void MakeDFT(Mat original, double* result)
{
const int width = original.cols;
const int height = 1;
Mat fftBlock(1,width, CV_8UC1);
Rect roi(0, 0, width, height);
Mat image_roi = original(roi);
image_roi.copyTo(fftBlock);
//imwrite("C:/fftBlock2.png", fftBlock);
Mat planes[] = {Mat_<float>(fftBlock), Mat::zeros(fftBlock.size(), CV_32F)};
Mat complexI;
merge(planes, 2, complexI);
dft(complexI, complexI, DFT_ROWS); //also tried with DFT_COMPLEX_OUTPUT | DFT_ROWS
split(complexI, planes);
double re, imag;
for (int i = 0; i < width; i++)
{
re = (float)planes[0].data[i];
imag = (float)planes[1].data[i];
result[i] = re * re + imag * imag;
}
}
bool SinusFFTTest()
{
const int size = 1024;
Mat sinTest(size,size,CV_8UC1, Scalar(0));
const int n_sin_curves = 10;
double deg_step = (double)n_sin_curves*360/size;
for (int j = 0; j < size; j++)
{
for (int i = 0; i <size; i++)
{
sinTest.data[j*size+i] = 127.5 * sin(i*deg_step*CV_PI/180) + 127.5;
}
}
double* result1 = new double[size];
double* result2 = new double[size];
OldMakeDFT(sinTest,result1);
MakeDFT(sinTest,result2);
bool identical = true;
for (int i = 0; i < size; i++)
{
if (abs(result1[i] - result2[i]) > 1000)
{
identical = false;
break;
}
}
delete[] result1;
delete[] result2;
return identical;
}
int _tmain(int argc, _TCHAR* argv[])
{
if (SinusFFTTest())
{
printf("identical");
}
else
{
printf("different");
}
getchar();
return 0;
}
Could someone explain the difference?
imgReal - is not filled with zeroes by default.
The bug in in the MakeDFT() function:
re = (float)planes[0].data[i];
imag = (float)planes[1].data[i];
data[i]'s type is uchar, and its conversion to float is not right.
The fix:
re = planes[0].at<float>(0,i);
imag = planes[1].at<float>(0,i);
After this change, the old and the new DFT versions gives the same results. Or, you can use cv::magnitude() instead of calculating the sum of squares of re and imag:
Mat magn;
magnitude(planes[0], planes[1], magn);
for (int i = 0; i < width; i++)
result[i] = pow(magn.at<float>(0,i),2);
This gives also the same result as the old cvDFT.

How to resize the box to fix size when using boundingRect?

In this project I wanna use contour numbers (cnum) as the index of the objects so I can use them to the next process (I'm working on false human detection using blob tracking). I wanna to resize boundingRect inside DetectMultiScale, how to do that?
and here we go my code:
int main(int argc, char *argv[])
{
cv::Mat frame;
cv::Mat blurred;
cv::Mat fg;
cv::Mat bgmodel;
cv::Mat threshfg;
cv::Mat blob;
int pixblob = 0;
int tot_bgr = 0;
int tot_ex_bgr = 0;
int green0 = 0;
int green1 = 0;
int green2 = 0;
int green3 = 0;
cv::namedWindow("Frame");
cv::namedWindow("Background Model");
cv::namedWindow("Blob");
cv::VideoCapture cap("campus.avi");
cv::BackgroundSubtractorMOG2 bgs;
bgs.nmixtures = 3;
bgs.history = 500;
bgs.bShadowDetection = true;
bgs.nShadowDetection = 0;
bgs.fTau = 0.25;
std::vector<std::vector<cv::Point>> contours;
cv::CascadeClassifier human;
assert(human.load("hogcascade_pedestrians.xml"));
for(;;){
cap >> frame;
cv::GaussianBlur(frame,blurred,cv::Size(3,3),0,0,cv::BORDER_DEFAULT);
bgs.operator()(blurred,fg);
bgs.getBackgroundImage(bgmodel);
cv::erode(fg,fg,cv::Mat(),cv::Point(-1,-1),1);
cv::dilate(fg,fg,cv::Mat(),cv::Point(-1,-1),3);
cv::threshold(fg,threshfg,70.0f,255,CV_THRESH_BINARY);
cv::findContours(threshfg,contours,CV_RETR_EXTERNAL,CV_CHAIN_APPROX_SIMPLE);
cv::cvtColor(threshfg,blob,CV_GRAY2RGB);
cv::drawContours(blob,contours,-1,cv::Scalar(255,255,255),CV_FILLED,8);
int cmin = 20;
int cmax = 1000;
bool FOD1 = true;
bool FOD2 = true;
std::vector<cv::Rect> rects;
for(int cnum = 0; cnum < contours.size(); cnum++){
if(contours[cnum].size() > cmin && contours[cnum].size() < cmax){
human.detectMultiScale(frame(cv::boundingRect(contours[cnum])),rects); //I wanna resize the box here!
if(rects.size() > 0){
cv::Rect r0 = cv::boundingRect(contours[cnum]);
cv::rectangle(frame,
r0,
cv::Scalar(255, 0, 0));
cv::putText(frame,
"HUMAN",
cv::Point(r0.x + r0.width / 2, r0.y + r0.height / 2),
cv::FONT_HERSHEY_SIMPLEX,
0.5,
cv::Scalar(0,0,255),
2,
8);
}
}
}
cv::imshow("Frame",frame);
cv::imshow("Background Model",bgmodel);
cv::imshow("Blob",blob);
if(cv::waitKey(30) >= 0) break;
}
return 0;
}
I'll appreciate any help here. Thanks :)