cuda calculations give nan after so many loops - c++

Question

This is my first time working with cuda. I am running some calculations involving cufft and two simple kernels on an NxNxN mesh (N=128). It seems to work fine until some time between 4040 and 4050 loops, the values of my mesh points become nan. On a smaller mesh, it can complete more loops before failing. This makes me think there is a memory leak somewhere. I tried running cuda-memcheck but it returned no errors. Can you spot any problems that could be causing this? I have reduced the code to a minimum but it is still long, my apologies. Thank you for your help.
#define _USE_MATH_DEFINES
#include <iostream>
#include <math.h>
#include <cstdlib>
#include <cuda_runtime.h>
#include <cufft.h>
#include <cufftXt.h>
using namespace std;
__global__ void Cube (cufftComplex *data, cufftComplex *data3, int n) {
int i = blockIdx.x * blockDim.x + threadIdx.x;
if (i<n){
data3[i].x = pow(data[i].x, 3);
data3[i].y = 0;
}
__syncthreads();
}
__global__ void Spectral (cufftComplex *data, cufftComplex *data3, float *w, float *v, int n) {
int i = blockIdx.x * blockDim.x + threadIdx.x;
if (i<n){
data[i].x = (w[i] * data[i].x + data3[i].x * v[i]) / n;
data[i].y = 0;
}
__syncthreads();
}
float ran();
int main (int argc, char **argv) {
float QQ, C;
float tmax = 5000;
int N = 128;
int n = N*N*N;
float dn = M_PI/8;
float dt = .075;
float psi0 = -0.175;
float r = -0.1;
tmax *= dt;
//setup cuda complex arrays
int mem_size = sizeof(cufftComplex)*n;
int float_mem_size = sizeof(float)*n;
cufftComplex *h_data = (cufftComplex*)malloc(mem_size);
cufftComplex *d_data;
cudaMalloc((void**)&d_data, mem_size);
cufftComplex *h_data3 = (cufftComplex*)malloc(mem_size);
cufftComplex *d_data3;
cudaMalloc((void**)&d_data3, mem_size);
float * h_w = (float*)malloc(float_mem_size);
float *d_w;
cudaMalloc(&d_w, float_mem_size);
float * h_v = (float*)malloc(float_mem_size);
float *d_v;
cudaMalloc(&d_v, float_mem_size);
for (int i=0; i<n; i++){
h_data[i].x = psi0 + r * ran();
h_data[i].y = 0;
}
int nx, ny, nz;
float B = -4 * M_PI * M_PI / ( pow((N*dn),2));
for (int i=0; i<n; i++){
nx = (i % N);
ny = (i / N) % N;
nz = i / (N * N);
if (nx > (N / 2)) {
nx = (N - nx);
}
if (ny > (N / 2)) {
ny = (N - ny);
}
if (nz > (N / 2)) {
nz = (N - nz);
}
QQ = B * (pow(nx, 2.0) + pow(ny, 2.0) + pow(nz, 2.0));
C = -r - 2.0 * QQ - pow(QQ, 2.0);
h_w[i] = exp(QQ * (1.0 - C) * dt);
h_v[i] = (h_w[i] - 1.0) / (1.0 - C);
}
cudaMemcpy(d_w, h_w, float_mem_size, cudaMemcpyHostToDevice);
cudaMemcpy(d_v, h_v, float_mem_size, cudaMemcpyHostToDevice);
cudaMemcpy(d_data, h_data, mem_size, cudaMemcpyHostToDevice);
cufftHandle plan;
cufftPlan3d(&plan, N, N, N, CUFFT_C2C);
int maxThreads=(N>1024)?1024:N;
int threadsPerBlock = maxThreads;
int numBlocks = n/maxThreads;
for (float t = 0; t < tmax; t += dt) {
Cube <<<numBlocks, threadsPerBlock>>> (d_data, d_data3, n);
cudaDeviceSynchronize();
cufftExecC2C(plan, d_data3, d_data3, CUFFT_FORWARD);
cudaDeviceSynchronize();
cufftExecC2C(plan, d_data, d_data, CUFFT_FORWARD);
cudaDeviceSynchronize();
Spectral <<<numBlocks, threadsPerBlock>>> (d_data, d_data3, d_w, d_v, n);
cudaDeviceSynchronize();
cufftExecC2C(plan, d_data, d_data, CUFFT_INVERSE);
cudaDeviceSynchronize();
}
//check output (should be a number)
cudaMemcpy(h_data, d_data, mem_size, cudaMemcpyDeviceToHost);
cout <<h_data[0].x <<endl;
//clean up
cufftDestroy(plan);
cudaFree(d_data);
cudaFree(d_data3);
cudaFree(d_w);
cudaFree(d_v);
free(h_w);
free(h_v);
free(h_data);
free(h_data3);
return 0;
}
float ran(){ //random in range [-1,1]
float u= float (rand())/(RAND_MAX);
//return round(u);
return 2*u-1;
}

Answer 1

Here is my instrumentation of your code so far. When I enabled the device assert in my_assert, it indicated that d_data3 input at the nan5 point was failing (i.e. it was nan). That indicated that the cufftExecC2C call on d_data3 immmediately prior was producing nan data. If you have invalid inputs, I believe an FFT can produce out-of-range results.
The code is instrumented to allow you to dump the data and look at it. You will have to modify dump_data to display whatever it is you wish to see.
When I run the code below, it eventually prints out:
4850.14
4851.14
4852.14
4853.14
4854.14
4855.14
4856.14
4857.14
4858.14
4859.14
4860.14
d_data3 output nan check failed
$
So the nan first occurs on iteration 4860, and the d_data3 input check did not fail, so the nan occurs in d_data3 as a result of the FFT operation in loop iteration 4860. You'll need to study the input and output data to see if you can determine why. There may be some modification to the d_data3 data in the Cube kernel that is causing this. For example, since you are repetitively cubing the data, doesn't it seem reasonable at some point that it would exceed float range?
Here's my instrumented code:
#include <iostream>
#include <math.h>
#include <cstdlib>
#include <cuda_runtime.h>
#include <cufft.h>
#include <cufftXt.h>
#include <assert.h>
#include <stdio.h>
using namespace std;
__host__ __device__ void my_assert(bool cond){
//assert(cond);
}
__global__ void Cube (cufftComplex *data, cufftComplex *data3, int n) {
int i = blockIdx.x * blockDim.x + threadIdx.x;
if (i<n){
float temp = data[i].x;
if (isnan(temp)) {printf("nan1: %d\n", i); my_assert(0);}
data3[i].x = pow(data[i].x, 3);
if (isnan(data3[i].x)) {printf("nan2: %d %f\n", i, data[i].x); my_assert(0);}
data3[i].y = 0;
}
__syncthreads();
}
__global__ void Spectral (cufftComplex *data, cufftComplex *data3, float *w, float *v, int n) {
int i = blockIdx.x * blockDim.x + threadIdx.x;
if (i<n){
float temp1 = w[i];
if (isnan(temp1)) {printf("nan3: %d\n", i); my_assert(0);}
float temp2 = data[i].x;
if (isnan(temp2)) {printf("nan4: %d\n", i); my_assert(0);}
float temp3 = data3[i].x;
if (isnan(temp3)) {printf("nan5: %d\n", i); my_assert(0);}
float temp4 = v[i];
if (isnan(temp4)) {printf("nan6: %d\n", i); my_assert(0);}
data[i].x = (w[i] * data[i].x + data3[i].x * v[i]) / n;
if (isnan(data[i].x)) {printf("nan7: %d, %f, %f, %f, %f, %d\n",i, temp1, temp2, temp3, temp4, n); my_assert(0);}
data[i].y = 0;
}
__syncthreads();
}
__global__ void nan_kernel(cufftComplex *d, int len, bool *res){
int idx=threadIdx.x+blockDim.x*blockIdx.x;
if (idx < len)
if (isnan(d[idx].x) || isnan(d[idx].y)) *res = true;
}
bool *d_nan;
bool checknan(cufftComplex *d, int len){
bool h_nan = false;
cudaMemcpy(d_nan, &h_nan, sizeof(bool), cudaMemcpyHostToDevice);
nan_kernel<<<(len/1024)+1, 1024>>>(d, len, d_nan);
cudaMemcpy(&h_nan, d_nan, sizeof(bool), cudaMemcpyDeviceToHost);
return h_nan;
}
void dump_data(cufftComplex *d1, cufftComplex *d2, int len)
{
// add code here to spit out the data however you would like to see it
// perhaps to a file
std::cout << "input: output: " << std::endl;
for (int i = 0; i < len; i++)
std::cout << d1[i].x << "," << d1[i].y << " " << d2[i].x << "," << d2[i].y << std::endl;
};
float ran();
int main (int argc, char **argv) {
float QQ, C;
float tmax = 5000;
int N = 128;
int n = N*N*N;
float dn = M_PI/8;
float dt = .075;
float psi0 = -0.175;
float r = -0.1;
tmax *= dt;
//setup cuda complex arrays
int mem_size = sizeof(cufftComplex)*n;
int float_mem_size = sizeof(float)*n;
cufftComplex *h_data = (cufftComplex*)malloc(mem_size);
cufftComplex *d_data;
cudaMalloc((void**)&d_data, mem_size);
cufftComplex *h_data3 = (cufftComplex*)malloc(mem_size);
cufftComplex *d_data3;
cudaMalloc((void**)&d_data3, mem_size);
float * h_w = (float*)malloc(float_mem_size);
float *d_w;
cudaMalloc(&d_w, float_mem_size);
float * h_v = (float*)malloc(float_mem_size);
float *d_v;
cudaMalloc(&d_v, float_mem_size);
for (int i=0; i<n; i++){
h_data[i].x = psi0 + r * ran();
h_data[i].y = 0;
}
int nx, ny, nz;
float B = -4 * M_PI * M_PI / ( pow((N*dn),2));
for (int i=0; i<n; i++){
nx = (i % N);
ny = (i / N) % N;
nz = i / (N * N);
if (nx > (N / 2)) {
nx = (N - nx);
}
if (ny > (N / 2)) {
ny = (N - ny);
}
if (nz > (N / 2)) {
nz = (N - nz);
}
QQ = B * (pow(nx, 2.0) + pow(ny, 2.0) + pow(nz, 2.0));
C = -r - 2.0 * QQ - pow(QQ, 2.0);
h_w[i] = exp(QQ * (1.0 - C) * dt);
h_v[i] = (h_w[i] - 1.0) / (1.0 - C);
}
cudaMemcpy(d_w, h_w, float_mem_size, cudaMemcpyHostToDevice);
cudaMemcpy(d_v, h_v, float_mem_size, cudaMemcpyHostToDevice);
cudaMemcpy(d_data, h_data, mem_size, cudaMemcpyHostToDevice);
cufftHandle plan;
cufftPlan3d(&plan, N, N, N, CUFFT_C2C);
int maxThreads=(N>1024)?1024:N;
int threadsPerBlock = maxThreads;
int numBlocks = n/maxThreads;
cufftResult res;
cudaMalloc(&d_nan, sizeof(bool));
cufftComplex *i3, *o3;
i3 = (cufftComplex *)malloc(mem_size);
o3 = (cufftComplex *)malloc(mem_size);
std::cout << "start loop" << std::endl;
for (float t = 0; t < tmax; t += dt) {
std::cout << t/dt << std::endl;
Cube <<<numBlocks, threadsPerBlock>>> (d_data, d_data3, n);
cudaDeviceSynchronize();
cudaMemcpy(i3, d_data3, mem_size, cudaMemcpyDeviceToHost);
if (checknan(d_data3, n)) {std::cout << "d_data3 input nan check failed" << std::endl; return -1;}
res = cufftExecC2C(plan, d_data3, d_data3, CUFFT_FORWARD);
if (res != CUFFT_SUCCESS) {std::cout << "cufft1 error: " << (int)res << " , " << t/dt << std::endl; return 1;}
cudaDeviceSynchronize();
if (checknan(d_data3, n)) {std::cout << "d_data3 output nan check failed" << std::endl; cudaMemcpy(o3, d_data3, mem_size, cudaMemcpyDeviceToHost); dump_data(i3, o3, n); return -1;}
res = cufftExecC2C(plan, d_data, d_data, CUFFT_FORWARD);
if (res != CUFFT_SUCCESS) {std::cout << "cufft2 error: " << (int)res << " , " << t/dt << std::endl; return 1;}
cudaDeviceSynchronize();
Spectral <<<numBlocks, threadsPerBlock>>> (d_data, d_data3, d_w, d_v, n);
cudaDeviceSynchronize();
res = cufftExecC2C(plan, d_data, d_data, CUFFT_INVERSE);
if (res != CUFFT_SUCCESS) {std::cout << "cufft3 error: " << (int)res << " , " << t/dt << std::endl; return 1;}
cudaDeviceSynchronize();
}
//check output (should be a number)
cudaMemcpy(h_data, d_data, mem_size, cudaMemcpyDeviceToHost);
cout <<h_data[0].x <<endl;
cudaError_t cres = cudaGetLastError();
if (cres != cudaSuccess) std::cout << "cuda error: " << cudaGetErrorString(cres) << std::endl;
//clean up
cufftDestroy(plan);
cudaFree(d_data);
cudaFree(d_data3);
cudaFree(d_w);
cudaFree(d_v);
free(h_w);
free(h_v);
free(h_data);
free(h_data3);
return 0;
}
float ran(){ //random in range [-1,1]
float u= float (rand())/(RAND_MAX);
//return round(u);
return 2*u-1;
}
EDIT:
After some addition of printout code to dump_data (see modification above) I see this:
...
4859.14
4860.14
d_data3 output nan check failed
input: output:
3.37127e+19,0 nan,nan
3.21072e+19,0 nan,nan
2.76453e+19,0 nan,nan
2.13248e+19,0 nan,nan
1.44669e+19,0 nan,nan
8.37214e+18,0 nan,nan
3.93645e+18,0 nan,nan
1.35501e+18,0 nan,nan
2.55741e+17,0 nan,nan
5.96468e+15,0 nan,nan
-1.36656e+16,0 nan,nan
-2.33688e+17,0 nan,nan
-8.37407e+17,0 nan,nan
-1.79915e+18,0 nan,nan
-2.96302e+18,0 nan,nan
-4.11485e+18,0 nan,nan
-5.03876e+18,0 nan,nan
-5.57617e+18,0 nan,nan
-5.65307e+18,0 nan,nan
-5.28957e+18,0 nan,nan
-4.5872e+18,0 nan,nan
-3.68309e+18,0 nan,nan
...
I'm not an FFT expert, but it might be the case that if you do an FFT on a large array filled with large values, using float precision, that overflow may occur. If you only need to get to 5000 iterations and you're failing at 4860, you might get there if you change all your datatypes to double from float, but I'm not sure about the numerical sense of what you are doing here.
Finally, note that both cufft and fftw perform un-normalized transforms. This may be playing a role in the seeming growth of magnitudes in your data set. As I stated already, I'm not familiar with the arithmetic or algorithm you are trying to implement here.

Answer 2

Is it possible that you have a float underflow happening around iteration 4040? Taking the cube of your data3 would lead me to check out that possibility. It is pretty easy to spiral into an underflow on a float32 if your not careful. You could throw a check in there to limit your value to some minimum epsilon to prevent this.