Distance between same point in two different images not consistent - OpenCV/C++ - c++

Question

So I have two Thermal images (Potato quality I know but it is what I have to work with), the first two images in this album. I am using some code from a tutorial that is super common but have edited a lot of it.
http://imgur.com/a/Zch7C
So what I am doing in my code is
1. Detecting KeyPoints
2. Describe the KeyPoints
3. Match the KeyPoints
4. Keep only good points
5. Gather both Query and Train points
6. Find Homography
7. Warp one of the images
8. Repeat the above steps for the warped image and the other original image
Now my question is: Should the change in the (x,y) distance between two of the same points on the two different images be the same for every set of points?
The whole frame is moving in the same direction so no matter what matching points we look at the change should be the same should it not?
What I am finding is that the points all different in the distance, some are 5 pixels different and some are 700 pixels, the only thing I can think is happening is that the match is not actually good and it is comparing two points that are no where near the same point in the separate frames.
I need to know what the offset is so that I can overlay one frame on top of the other then average out the pixel values that are overlapping and build a new images from the composite/average of the two originals.
My code I am using is below:
#include <stdio.h>
#include <iostream>
#include "opencv2/core/core.hpp"
#include "opencv2/features2d/features2d.hpp"
#include "opencv2/highgui/highgui.hpp"
#include "opencv2/nonfree/nonfree.hpp"
#include "opencv2/calib3d/calib3d.hpp"
#include "opencv2/imgproc/imgproc.hpp"
#include "stitch.cpp"
#include "opencv2\stitching\stitcher.hpp"
#include "opencv2\nonfree\features2d.hpp"
using namespace cv;
void readme();
Mat describe(Mat img, vector<KeyPoint> key);
vector<KeyPoint> detect(Mat img);
vector<DMatch> match(Mat descriptionOne, Mat descriptionTwo);
/** #function main */
int main(int argc, char** argv)
{
VideoCapture cap("vid.mp4");
vector<Mat> Vimg;
cout << "Grabbing Images" << endl;
for (int i = 0; i < 2; i++)
{
cout << "Grabbing Frame" << i << endl;
Mat temp;
cap.read(temp);
Vimg.push_back(temp);
imwrite("image" + to_string(i) + ".jpg", temp);
for (int j = 0; j < 80; j++)
cap.grab();
}
//Mat cimg1 = Vimg[0];
//Mat cimg2 = Vimg[1];
Mat cimg1 = imread("cap1.png");
Mat cimg2 = imread("cap2.png");
cout << "Starting Stitching" << endl;
//Converting the original images to grayscale
Mat img1, img2;
cvtColor(cimg1, img1, CV_BGR2GRAY);
cvtColor(cimg2, img2, CV_BGR2GRAY);
//Detecting Keypoints for original two images
vector<KeyPoint> keypointOne = detect(img1), keypointTwo = detect(img2);
Mat mkeypointOne, mkeypointTwo;
drawKeypoints(cimg1, keypointOne, mkeypointOne, Scalar(0, 0, 255), DrawMatchesFlags::DRAW_RICH_KEYPOINTS);
drawKeypoints(cimg2, keypointTwo, mkeypointTwo, Scalar(0, 0, 255), DrawMatchesFlags::DRAW_RICH_KEYPOINTS);
imwrite("keypointOne.jpg", mkeypointOne);
imwrite("keypointTwo.jpg", mkeypointTwo);
//Computing descriptors
Mat descriptionOne = describe(img1, keypointOne), descriptionTwo = describe(img2, keypointTwo);
//Matching descriptors
vector<DMatch> matches = match(descriptionOne, descriptionTwo);
double max = 0;
double min = 100;
//Calculation of max and min distances
for (int i = 0; i < matches.size(); i++)
{
double dist = matches[i].distance;
if (dist < min) min = dist;
if (dist > max) max = dist;
}
vector<DMatch> goodMatches;
//Keep only good matches
for (int i = 0; i < matches.size(); i++)
{
if (matches[i].distance < 2*min)
goodMatches.push_back(matches[i]);
}
//Localize
vector<Point2f> obj;
vector<Point2f> scene;
for (int i = 0; i < goodMatches.size(); i++)
{
obj.push_back(keypointOne[goodMatches[i].queryIdx].pt);
scene.push_back(keypointTwo[goodMatches[i].trainIdx].pt);
}
/*
for (int k = 0; k < obj.size(); k++)
{
cout << "Point data for Match #" << k << endl;
cout << "\tImage 1 Point: " << obj[k] << endl;
cout << "\tImage 2 Point: " << scene[k] << endl;
}*/
Mat H = findHomography(obj, scene, CV_RANSAC);
//Warping the image to fit on first image
Mat cwarpImage, warpImage;
//TODO: figure out the right size for this image that is created
warpPerspective(cimg2, cwarpImage, H, Size(img2.cols + img1.cols, img2.rows + img1.rows));
/*
Mat result;
Mat half(warpImage, Rect(0, 0, img2.cols, img2.rows));
cimg2.copyTo(half);
*/
imwrite("warp.jpg", warpImage);
//Processing Image
cvtColor(cwarpImage, warpImage, CV_BGR2GRAY);
vector<KeyPoint> keypointWarp = detect(warpImage);
Mat descriptionWarp = describe(warpImage, keypointWarp);
vector<DMatch> warpMatches = match(descriptionOne, descriptionWarp);
Mat mkeypointWarp;
drawKeypoints(cwarpImage, keypointWarp, mkeypointWarp, Scalar(0, 0, 255), DrawMatchesFlags::DRAW_RICH_KEYPOINTS);
imwrite("keypointWarp.jpg", mkeypointWarp);
Mat match;
drawMatches(cimg1, keypointOne, warpImage, keypointWarp, warpMatches, match, Scalar(0, 0, 255), Scalar(255, 0, 0), vector<char>(), DrawMatchesFlags::DRAW_RICH_KEYPOINTS);
//imshow("match", match);
imwrite("matches.jpg", match);
//Localize
vector<Point2f> obj2;
vector<Point2f> scene2;
for (int i = 0; i < warpMatches.size(); i++)
{
obj2.push_back(keypointOne[warpMatches[i].queryIdx].pt);
scene2.push_back(keypointWarp[warpMatches[i].trainIdx].pt);
}
for (int k = 0; k < obj.size(); k++)
{
cout << "Point data for Match #" << k << endl;
cout << "\tImage 1 Point: " << obj2[k] << endl;
cout << "\tImage 2 Point: " << scene2[k] << endl;
}
vector<unsigned char> inliersMask;
Mat H2 = findHomography(obj, scene, CV_RANSAC, 3, inliersMask);
vector<DMatch> inliers;
for (size_t i = 0; i < inliersMask.size(); i++)
{
if (inliersMask[i])
inliers.push_back(warpMatches[i]);
}
warpMatches.swap(inliers);
Mat match2;
drawMatches(cimg1, keypointOne, warpImage, keypointWarp, warpMatches, match2, Scalar(0, 0, 255), Scalar(255, 0, 0), vector<char>(), DrawMatchesFlags::DRAW_RICH_KEYPOINTS);
imwrite("homorgraphyOutlierMatch.jpg", match2);
cout << "Writing Warp Image" << endl;
imwrite("warpimage.jpg", warpImage);
cout << H << endl;
waitKey(0);
}
Mat describe(Mat img, vector<KeyPoint> key)
{
Mat temp;
SurfDescriptorExtractor extractor;
extractor.compute(img, key, temp);
return temp;
}
vector<KeyPoint> detect(Mat img)
{
vector<KeyPoint> temp;
SurfFeatureDetector detector(400);
detector.detect(img, temp);
return temp;
}
vector<DMatch> match(Mat descriptionOne, Mat descriptionTwo)
{
vector<DMatch> temp;
BFMatcher matcher(NORM_L2, true);
matcher.match(descriptionOne, descriptionTwo, temp);
return temp;
}
EDIT:
I set Cross Check to true in the BFMatcher and implemented Homography outlier detection from Mastering_OpenCV. Here are the two new results. I was not sure if I was supposed to implement both cross check and KnnMatch so I only did cross check.
http://imgur.com/a/1P7Xt
As you can see they are a lot better but there are still some there that should not be there. I ran it with both full color and thermal images.
New code is above as well.

Answer 1

While the change in distance between point correspondences won't be the same for all points in the general case, you wouldn't expect to have deltas of the order of 700 pixels with an image size of 1300ish.
by inspection of the images you've posted, it's clear that you have point correspondences which are not correct (simply, you have lots of crossed lines in your matches between images)
This suggests that your step 4 isn't doing a great job. You might want to try setting the second parameter of your Brute Force matcher to true to enable cross-check test:
BFMatcher matcher(NORM_L2, true);
You might also want to consider the ratio test for outlier removal as described here How to apply Ratio Test in order to remove outliers in a multiple object detection matcher?