How to use the OpenCV CUDA Fourier Transform - c++

Instead of OpenCV's normal dft, I'd like to use cuda::dft. As a start I tried performing a forward and inverse transform, but the result doesn't look anything like the input. Here's a minimal example using an OpenCV example image:
// Load 8bit test image (https://raw.githubusercontent.com/opencv/opencv/master/samples/data/basketball1.png)
Mat testImg;
testImg = imread("basketball1.png", CV_LOAD_IMAGE_GRAYSCALE);
// Convert input to complex float image
Mat_<float> imgReal;
testImg.convertTo(imgReal, CV_32F, 1.0/255.0);
Mat imgImag = Mat(imgReal.rows, imgReal.cols, CV_32F, float(0));
vector<Mat> channels;
channels.push_back(imgReal);
channels.push_back(imgImag);
Mat imgComplex;
merge(channels,imgComplex);
imshow("Img real", imgReal);
waitKey(0);
//Perform a Fourier transform
cuda::GpuMat imgGpu, fftGpu;
imgGpu.upload(imgComplex);
cuda::dft(imgGpu, fftGpu, imgGpu.size());
//Performs an inverse Fourier transform
cuda::GpuMat propGpu, convFftGpu;
cuda::dft(fftGpu, propGpu, imgGpu.size(), DFT_REAL_OUTPUT | DFT_SCALE);
Mat output(propGpu);
output.convertTo(output, CV_8U, 255, 0);
imshow("Output", output);
waitKey(0);
I played with the flags but output never looks anything like input. Using the above code I get as output:
While it should look like this:

I found the answer here. Apparently, when starting with a complex input image, it's not possible to use the flag DFT_REAL_OUTPUT.
Either you do the forward transform with a one channel float input and then you get the same as an output from the inverse transform, or you start with a two channel complex input image and get that type as output. The upside to using a complex input image is that the forward transform delivers the full sized complex field to work with, e.g. perform a convolution (see linked answer for details).
I'll add that in order to obtain an 8bit image from the inverse transform, compute the magnitude yourself like so:
Mat output(propGpu);
Mat planes[2];
split(output,planes);
Mat mag;
magnitude(planes[0],planes[1],mag);
mag.convertTo(mag, CV_8U, 255, 0);

To go into Fourier domain using OpenCV Cuda FFT and back into the spatial domain, you can simply follow the below example (to learn more, you can refer to cufft documentation, on which OpenCV Cuda FFT source code is based).
Mat test_im;
test_im = imread("frame.png", IMREAD_GRAYSCALE);
// Convert input input to real value type (CV_64F for double precision)
Mat im_real;
test_im.convertTo(im_real, CV_32F, 1.0/255.0);
imshow("Input Image", im_real);
waitKey(0);
// Perform The Fourier Transform
cuda::GpuMat in_im_gpu, fft_im;
in_im_gpu.upload(im_real);
cuda::dft(in_im_gpu, fft_im, in_im_gpu.size(), 0);
// Performs an inverse Fourier transform
cuda::GpuMat ifft_im_gpu;
//! int odd_size = imgGpu.size().width % 2;
//! cv::Size dest_size((fftGpu.size().width-1)*2 + (odd_size ? 1 : 0), fftGpu.size().height);
cv::Size dest_size = in_im_gpu.size();
int flag = (DFT_SCALE + DFT_REAL_OUTPUT) | DFT_INVERSE;
cuda::dft(fft_im, ifft_im_gpu, dest_size, flag);
Mat ifft_im(ifft_im_gpu);
ifft_im.convertTo(ifft_im, CV_8U, 255, 0);
imshow("Inverse FFT", ifft_im);
waitKey(0);

Related

OpenCV Inverse Fourier Transform Distorting image

I am attempting to convert a greyscale image to and from the frequency domain using the Fourier transform in OpenCV. However, the resulting image in very distorted even though I made no changes to the image while in frequency domain. Could anyone help me with this? I've found several other questions explaining this like the links below and I have followed them exactly, but the result always ends up like this.
Inverse fourier transformation in OpenCV
https://coderedirect.com/questions/165340/inverse-fourier-transformation-in-opencv
//Make grayscale image
cvtColor(src, gray_in, COLOR_BGR2GRAY);
gray_in.convertTo(gray_in, CV_32FC1);
//Create complex output variable
//From https://docs.opencv.org/4.x/d8/d01/tutorial_discrete_fourier_transform.html
Mat planes[] = { Mat_<float>(gray_in), Mat::zeros(gray_in.size(), CV_32F) };
Mat complexI;
merge(planes, 2, complexI);
//Transform
dft(gray_in, complexI, DFT_COMPLEX_OUTPUT);
//Compute inverse transform
dft(complexI, tgt, DFT_SCALE | DFT_INVERSE | DFT_REAL_OUTPUT);
//Save file
tgt.convertTo(tgt, CV_32FC2);
imwrite(outfile, tgt);
//Display image
namedWindow(windowName);
imshow(windowName, tgt);
waitKey(0);
destroyWindow(windowName);

OpenCV, DFT function don't use in the image with IMREAD_COLOR

While reading the image with IMREAD_COLOR, 'dft' function throws the error:
DFT function works just fine when reading an image with IMREAD_GRAYSCALE. But I want to read the image with IMREAD_COLOR.
main function
const char* filename = "face.jpg";
Mat I = imread(filename, IMREAD_COLOR);
if(I.empty()) return 0;
Mat padded;
I.convertTo(padded, CV_32F);
Mat fft;
Mat planes[2];
dft(padded, fft, DFT_SCALE|DFT_COMPLEX_OUTPUT);
Mat fftBlur = fft.clone();
fftBlur *= 0.5;
split(fftBlur, planes);
Mat ph, mag;
mag.zeros(planes[0].rows, planes[0].cols, CV_32F);
ph.zeros(planes[0].rows, planes[0].cols, CV_32F);
cartToPolar(planes[0], planes[1], mag, ph);
merge(planes, 2, fftBlur);
//inverse
Mat invfft;
dft(fftBlur, invfft, DFT_INVERSE|DFT_REAL_OUTPUT);
Mat result;
invfft.convertTo(result, CV_8U);
Mat image;
cvtColor(result, image, COLOR_GRAY2RGB);
imshow("Output", result);
imshow("Image", image);
waitKey();
The message you receive is an assertion it tells you DFT function only takes single precision floating point image with one or two channels (CV_32FC1, CV_32FC2, the letter C at the end of the flag mean channel) or double precision floating point images with one or two channels (CV_64FC1, CV_64FC2).
The two channel case is actually the representation of complex image in OpenCV data storage.
If you want you can split you image to std::vector<cv::Mat> where each element does represent one channel, using cv::split apply the DFT on each channels do the processing you want on it and recreate an multichannel image thanks to cv::merge.
From Learning OpenCV (about dft function):
The input array must be of floating-point type and may be single- or double-channel. In the single-channel case, the entries are assumed to be real numbers, and the output will be packed in a special space-saving format called complex conjugate symmetrical.
The same question is mentioned here in terms of matlab image processing.
You can check out cv::split function if you want to separate channels of your initial image.

OpenCV keep background transparent during warpAffine

I create a Bird-View-Image with the warpPerspective()-function like this:
warpPerspective(frame, result, H, result.size(), CV_WARP_INVERSE_MAP, BORDER_TRANSPARENT);
The result looks very good and also the border is transparent:
Bird-View-Image
Now I want to put this image on top of another image "out". I try doing this with the function warpAffine like this:
warpAffine(result, out, M, out.size(), CV_INTER_LINEAR, BORDER_TRANSPARENT);
I also converted "out" to a four channel image with alpha channel according to a question which was already asked on stackoverflow:
Convert Image
This is the code: cvtColor(out, out, CV_BGR2BGRA);
I expected to see the chessboard but not the gray background. But in fact, my result looks like this:
Result Image
What am I doing wrong? Do I forget something to do? Is there another way to solve my problem? Any help is appreciated :)
Thanks!
Best regards
DamBedEi
I hope there is a better way, but here it is something you could do:
Do warpaffine normally (without the transparency thing)
Find the contour that encloses the image warped
Use this contour for creating a mask (white values inside the image warped, blacks in the borders)
Use this mask for copy the image warped into the other image
Sample code:
// load images
cv::Mat image2 = cv::imread("lena.png");
cv::Mat image = cv::imread("IKnowOpencv.jpg");
cv::resize(image, image, image2.size());
// perform warp perspective
std::vector<cv::Point2f> prev;
prev.push_back(cv::Point2f(-30,-60));
prev.push_back(cv::Point2f(image.cols+50,-50));
prev.push_back(cv::Point2f(image.cols+100,image.rows+50));
prev.push_back(cv::Point2f(-50,image.rows+50 ));
std::vector<cv::Point2f> post;
post.push_back(cv::Point2f(0,0));
post.push_back(cv::Point2f(image.cols-1,0));
post.push_back(cv::Point2f(image.cols-1,image.rows-1));
post.push_back(cv::Point2f(0,image.rows-1));
cv::Mat homography = cv::findHomography(prev, post);
cv::Mat imageWarped;
cv::warpPerspective(image, imageWarped, homography, image.size());
// find external contour and create mask
std::vector<std::vector<cv::Point> > contours;
cv::Mat imageWarpedCloned = imageWarped.clone(); // clone the image because findContours will modify it
cv::cvtColor(imageWarpedCloned, imageWarpedCloned, CV_BGR2GRAY); //only if the image is BGR
cv::findContours (imageWarpedCloned, contours, CV_RETR_EXTERNAL, CV_CHAIN_APPROX_NONE);
// create mask
cv::Mat mask = cv::Mat::zeros(image.size(), CV_8U);
cv::drawContours(mask, contours, 0, cv::Scalar(255), -1);
// copy warped image into image2 using the mask
cv::erode(mask, mask, cv::Mat()); // for avoid artefacts
imageWarped.copyTo(image2, mask); // copy the image using the mask
//show images
cv::imshow("imageWarpedCloned", imageWarpedCloned);
cv::imshow("warped", imageWarped);
cv::imshow("image2", image2);
cv::waitKey();
One of the easiest ways to approach this (not necessarily the most efficient) is to warp the image twice, but set the OpenCV constant boundary value to different values each time (i.e. zero the first time and 255 the second time). These constant values should be chosen towards the minimum and maximum values in the image.
Then it is easy to find a binary mask where the two warp values are close to equal.
More importantly, you can also create a transparency effect through simple algebra like the following:
new_image = np.float32((warp_const_255 - warp_const_0) *
preferred_bkg_img) / 255.0 + np.float32(warp_const_0)
The main reason I prefer this method is that openCV seems to interpolate smoothly down (or up) to the constant value at the image edges. A fully binary mask will pick up these dark or light fringe areas as artifacts. The above method acts more like true transparency and blends properly with the preferred background.
Here's a small test program that warps with transparent "border", then copies the warped image to a solid background.
int main()
{
cv::Mat input = cv::imread("../inputData/Lenna.png");
cv::Mat transparentInput, transparentWarped;
cv::cvtColor(input, transparentInput, CV_BGR2BGRA);
//transparentInput = input.clone();
// create sample transformation mat
cv::Mat M = cv::Mat::eye(2,3, CV_64FC1);
// as a sample, just scale down and translate a little:
M.at<double>(0,0) = 0.3;
M.at<double>(0,2) = 100;
M.at<double>(1,1) = 0.3;
M.at<double>(1,2) = 100;
// warp to same size with transparent border:
cv::warpAffine(transparentInput, transparentWarped, M, transparentInput.size(), CV_INTER_LINEAR, cv::BORDER_TRANSPARENT);
// NOW: merge image with background, here I use the original image as background:
cv::Mat background = input;
// create output buffer with same size as input
cv::Mat outputImage = input.clone();
for(int j=0; j<transparentWarped.rows; ++j)
for(int i=0; i<transparentWarped.cols; ++i)
{
cv::Scalar pixWarped = transparentWarped.at<cv::Vec4b>(j,i);
cv::Scalar pixBackground = background.at<cv::Vec3b>(j,i);
float transparency = pixWarped[3] / 255.0f; // pixel value: 0 (0.0f) = fully transparent, 255 (1.0f) = fully solid
outputImage.at<cv::Vec3b>(j,i)[0] = transparency * pixWarped[0] + (1.0f-transparency)*pixBackground[0];
outputImage.at<cv::Vec3b>(j,i)[1] = transparency * pixWarped[1] + (1.0f-transparency)*pixBackground[1];
outputImage.at<cv::Vec3b>(j,i)[2] = transparency * pixWarped[2] + (1.0f-transparency)*pixBackground[2];
}
cv::imshow("warped", outputImage);
cv::imshow("input", input);
cv::imwrite("../outputData/TransparentWarped.png", outputImage);
cv::waitKey(0);
return 0;
}
I use this as input:
and get this output:
which looks like ALPHA channel isn't set to ZERO by warpAffine but to something like 205...
But in general this is the way I would do it (unoptimized)

How to get cv::calcOpticalFlowSF to work?

I am useing the 2.4.4 version of OpenCV. - i know its a beta
but there is an example about cv::calcOpticalFlowSF the method in the example folder called: simpleflow_demo.cpp. But when i copy this demo and use it with my input images, it starts processing and after some seconds it came back a crash report.
The documentation about the method is a little bit strange, saying the output files are a x- and yflow instead of the cv::Mat& flow which the method actually wants.
Any ideas how to fix the problem to get the function working?
Try this simple demo that worked for me, then modify for your needs (display help from here):
Mat frame1 = imread("/home/radford/Desktop/1.png");
Mat frame2 = imread("/home/radford/Desktop/2.png");
namedWindow("flow");
Mat flow;
calcOpticalFlowSF(frame1, frame2, flow, 3, 2, 4);
Mat xy[2];
split(flow, xy);
//calculate angle and magnitude
Mat magnitude, angle;
cartToPolar(xy[0], xy[1], magnitude, angle, true);
//translate magnitude to range [0;1]
double mag_max;
minMaxLoc(magnitude, 0, &mag_max);
magnitude.convertTo(magnitude, -1, 1.0/mag_max);
//build hsv image
Mat _hsv[3], hsv;
_hsv[0] = angle;
_hsv[1] = Mat::ones(angle.size(), CV_32F);
_hsv[2] = magnitude;
merge(_hsv, 3, hsv);
//convert to BGR and show
Mat bgr;//CV_32FC3 matrix
cvtColor(hsv, bgr, COLOR_HSV2BGR);
imshow("flow", bgr);
waitKey(0);
In the example opencv/samples/cpp/simpleflow_demo.cpp there is a code block
if (frame1.type() != 16 || frame2.type() != 16) {
printf(APP_NAME "Images should be of equal type CV_8UC3\n");
exit(1);
}
So, grey images should be converted to CV_8UC3. For example using cvtColor(grey, grey3, CV_GRAY2RGB);

Deblurring algorithm in the Fourier domain

I'm trying to implement a deconvolution algorithm in Fourier transformed domain. I've a working implementation by myself in Matlab, and I want to translate it to C++ using OpenCV library.
Basically what I do is to extract the gradients from an input image, I do some stuff in transformed domain and then I come back to space domain.
The problematic part to me is to perform this (element wise) division:
DFT(im) = ( conj( DFT(f) ) * DFT(image) + L2 * conj( DFT( gradKernel-x ) ) * DFT(mux) )+ ... ) / ( norm( DFT(f) )^2 + L2 * norm(gradKernel-x)^2 + ... )
f is a gaussian kernel that is defined in the code.
DFT( gradKernel-x ) is the FFT of the gradient kernel in x direction, i.e. DFT([1,-1]) zero-padded to the size of the blurred image. mux is an auxiliary variable to perform the deconvolution.
I decided to perform the division by magnitude and phase separately in transformed domain before performing an inverse DFT.
I don't know where is the error in my code, maybe in the division, maybe in the forward/inverse transform of my variables, in the gaussian kernel...
If somebody can help me, I'd very grateful.
Here is the critical part of the code (note that I simplified it before posting, so don't expect a good deblurring result if you try it, basically what I expect from this is a visually pleasant output image):
imH00=imread("Cameraman256.png",0);
if(!imH00.data)
{
std::cout<< "Could not open or find the image" << std::endl ;
return -1;
}
imH00.convertTo(imH00,CV_32F,1.0/255.0,0.0);
// Gaussian Kernel
Mat ker1D=getGaussianKernel(ksize,sigma,CV_32F);
fkernel.create(imH00.size(),CV_32F);
// zero-padding
fkernel.setTo(Scalar::all(0));
temp=ker1D*ker1D.t();
temp.copyTo(fkernel(Rect(0,0,temp.rows,temp.cols)));
// Fourier transform
Mat planes[] = {Mat_<float>(fkernel), Mat::zeros(fkernel.size(), CV_32F)};
Mat ffkernel;
merge(planes, 2, ffkernel);
dft(ffkernel, ffkernel,DFT_COMPLEX_OUTPUT);
// Gradient filter in frequency domain, trying to do something similar to psf2otf([1;-1],size(imH00)); in Matlab.
dx=Mat::zeros(imH00.size(),CV_32F);
dx.at<float>(0,0)=1;
dx.at<float>(0,1)=-1;
Mat dxplanes[] = {Mat_<float>(dx), Mat::zeros(dx.size(), CV_32F)};
Mat fdx;
merge(dxplanes, 2, fdx);
dft(fdx, fdx,DFT_COMPLEX_OUTPUT);
dy=Mat::zeros(imH00.size(),CV_32F);
dy.at<float>(0,0)=1;
dy.at<float>(1,0)=-1;
Mat dyplanes[] = {Mat_<float>(dy), Mat::zeros(dy.size(), CV_32F)};
Mat fdy;
merge(dyplanes, 2, fdy);
dft(fdy, fdy,DFT_COMPLEX_OUTPUT);
// Denominators
Mat den1;
Mat den2;
Mat den21;
Mat den22;
// ||fdx||^2 and ||fdy||^2
mulSpectrums(fdx,fdx,den21,DFT_COMPLEX_OUTPUT,true);
mulSpectrums(fdy,fdy,den22,DFT_COMPLEX_OUTPUT,true);
add(den21,den22,den2);
mulSpectrums(ffkernel,ffkernel,den1,0,true);
imHk=imH00.clone();
mux=Mat::zeros(imH00.size(),CV_32F);
muy=Mat::zeros(imH00.size(),CV_32F);
// FFT imH00
Mat fHktplanes[] = {Mat_<float>(imH00), Mat::zeros(imH00.size(), CV_32F)};
Mat fHkt;
merge(fHktplanes, 2, fHkt);
dft(fHkt, fHkt,DFT_COMPLEX_OUTPUT);
std::cout<<"starting deconvolution"<<std::endl;
for (int j=0; j<4; j++)
{
// Deconvolution
// Gradients
Mat ddx(1,2,CV_32F,Scalar(0));
ddx.at<float>(0,0)=1;
ddx.at<float>(0,1)=-1;
filter2D(imHk,dHx,CV_32F,ddx);
Mat ddy(2,1,CV_32F,Scalar(0));
ddy.at<float>(0,0)=1;
ddy.at<float>(1,0)=-1;
filter2D(imHk,dHy,CV_32F,ddy);
mux=Scalar((float)-0.5*L1/L2);
add(mux,dHx,mux);
muy=Scalar((float)-0.5*L1/L2);
add(muy,dHy,muy);
// FFT mux, muy
Mat fmuxplanes[] = {Mat_<float>(mux), Mat::zeros(mux.size(), CV_32F)};
Mat fmux;
merge(fmuxplanes, 2, fmux);
dft(fmux, fmux,DFT_COMPLEX_OUTPUT);
Mat fmuyplanes[] = {Mat_<float>(muy), Mat::zeros(muy.size(), CV_32F)};
Mat fmuy;
merge(fmuyplanes, 2, fmuy);
dft(fmuy, fmuy,DFT_COMPLEX_OUTPUT);
Mat num1,num2,num3,num,den;
// Spectrums multiplication, complex conjugate
mulSpectrums(fHkt,ffkernel,num1,DFT_COMPLEX_OUTPUT,true);
mulSpectrums(fmux,fdx,num2,DFT_COMPLEX_OUTPUT,true);
mulSpectrums(fmuy,fdy,num3,DFT_COMPLEX_OUTPUT,true);
add(num2,num3,num2);
add(num1,L2*num2,num);
add(den1,L2*den2,den);
// Division in polar coordinates
Mat auxnum[2];
split(num,auxnum);
Mat auxden[2];
split(den,auxden);
Mat numMag,numPhase;
magnitude(auxnum[0],auxnum[1],numMag);
phase(auxnum[0],auxnum[1],numPhase);
Mat denMag,denPhase;
magnitude(auxden[0],auxden[1],denMag);
phase(auxden[0],auxden[1],denPhase);
Mat division[2];
divide(numMag,denMag,division[0]);
division[1]=numPhase-denPhase;
polarToCart(division[0],division[1],division[0],division[1]);
Mat fHk;
merge(division,2,fHk);
Mat imHkaux;
Mat planesfHk[2];
dft(fHk, imHkaux, DFT_INVERSE+DFT_SCALE);
split(imHkaux,planesfHk);
imHk=planesfHk[0]; // imHk is the Real part
}
imHk.convertTo(imHk,CV_8U,255);
imshow( "Deblurred image", imHk);
Thank you
The problem was in the Fourier transform of the filters. We need to shift filters kernels before transforming. This is the same that psf2otf function does in Matlab. If someone is interested, this simple code should perform the DFT of a Gaussian kernel which is not influenced by centering (as psf2otf):
float sigma=1.0;
short int ksize=13; // always odd
Mat ker1D=getGaussianKernel(ksize,sigma,CV_32F); //1D gaussian kernel
fkernel.create(myImage.size(),CV_32F); //
// zero-padding
fkernel.setTo(Scalar::all(0));
temp=ker1D*ker1D.t(); // 2D gaussian kernel, (Gaussian filter is separable)
int r=(ksize-1)/2; //radius
// shifting
temp(Rect(r,r,r+1,r+1)).copyTo(fkernel(Rect(0,0,r+1,r+1))); // q1
temp(Rect(r,0,r+1,r)).copyTo(fkernel(Rect(0,fkernel.cols-r,r+1,r))); // q2
temp(Rect(0,r,r,r+1)).copyTo(fkernel(Rect(fkernel.rows-r,0,r,r+1))); // q3
temp(Rect(0,0,r,r)).copyTo(fkernel(Rect(fkernel.rows-r,fkernel.cols-r,r,r))); // q4
// DFT
Mat planes[] = {Mat_<float>(fkernel), Mat::zeros(fkernel.size(), CV_32F)};
Mat ffkernel;
merge(planes, 2, ffkernel);
dft(ffkernel, ffkernel,DFT_COMPLEX_OUTPUT);