Summary
I'm porting a simple raytracing application based on the Scratchapixel version to a bunch of GPU libraries. I sucessfully ported it to CUDA using the runtime API and the driver API, but It throws a Segmentation fault (core dumped) when I try to use the PTX compiled at runtime with NVRTC.
If I uncomment the #include <math.h> directive at the beginning of the kernel file (see below), it still works using NVCC (the generated PTX is exactly the same) but fails at compilation using NVRTC.
I want to know how can I make NVRTC behave just like NVCC (is it even possible?), or at least to understand the reason behind this issues.
Detailed description
File kernel.cu (Kernel source):
//#include <math.h>
#define MAX_RAY_DEPTH 5
template<typename T>
class Vec3
{
public:
T x, y, z;
__device__ Vec3() : x(T(0)), y(T(0)), z(T(0)) {}
__device__ Vec3(T xx) : x(xx), y(xx), z(xx) {}
__device__ Vec3(T xx, T yy, T zz) : x(xx), y(yy), z(zz) {}
__device__ Vec3& normalize()
{
T nor2 = length2();
if (nor2 > 0) {
T invNor = 1 / sqrt(nor2);
x *= invNor, y *= invNor, z *= invNor;
}
return *this;
}
__device__ Vec3<T> operator * (const T &f) const { return Vec3<T>(x * f, y * f, z * f); }
__device__ Vec3<T> operator * (const Vec3<T> &v) const { return Vec3<T>(x * v.x, y * v.y, z * v.z); }
__device__ T dot(const Vec3<T> &v) const { return x * v.x + y * v.y + z * v.z; }
__device__ Vec3<T> operator - (const Vec3<T> &v) const { return Vec3<T>(x - v.x, y - v.y, z - v.z); }
__device__ Vec3<T> operator + (const Vec3<T> &v) const { return Vec3<T>(x + v.x, y + v.y, z + v.z); }
__device__ Vec3<T>& operator += (const Vec3<T> &v) { x += v.x, y += v.y, z += v.z; return *this; }
__device__ Vec3<T>& operator *= (const Vec3<T> &v) { x *= v.x, y *= v.y, z *= v.z; return *this; }
__device__ Vec3<T> operator - () const { return Vec3<T>(-x, -y, -z); }
__device__ T length2() const { return x * x + y * y + z * z; }
__device__ T length() const { return sqrt(length2()); }
};
typedef Vec3<float> Vec3f;
typedef Vec3<bool> Vec3b;
class Sphere
{
public:
const char* id;
Vec3f center; /// position of the sphere
float radius, radius2; /// sphere radius and radius^2
Vec3f surfaceColor, emissionColor; /// surface color and emission (light)
float transparency, reflection; /// surface transparency and reflectivity
int animation_frame;
Vec3b animation_position_rand;
Vec3f animation_position;
Sphere(
const char* id,
const Vec3f &c,
const float &r,
const Vec3f &sc,
const float &refl = 0,
const float &transp = 0,
const Vec3f &ec = 0) :
id(id), center(c), radius(r), radius2(r * r), surfaceColor(sc),
emissionColor(ec), transparency(transp), reflection(refl)
{
animation_frame = 0;
}
//[comment]
// Compute a ray-sphere intersection using the geometric solution
//[/comment]
__device__ bool intersect(const Vec3f &rayorig, const Vec3f &raydir, float &t0, float &t1) const
{
Vec3f l = center - rayorig;
float tca = l.dot(raydir);
if (tca < 0) return false;
float d2 = l.dot(l) - tca * tca;
if (d2 > radius2) return false;
float thc = sqrt(radius2 - d2);
t0 = tca - thc;
t1 = tca + thc;
return true;
}
};
__device__ float mix(const float &a, const float &b, const float &mixval)
{
return b * mixval + a * (1 - mixval);
}
__device__ Vec3f trace(
const Vec3f &rayorig,
const Vec3f &raydir,
const Sphere *spheres,
const unsigned int spheres_size,
const int &depth)
{
float tnear = INFINITY;
const Sphere* sphere = NULL;
// find intersection of this ray with the sphere in the scene
for (unsigned i = 0; i < spheres_size; ++i) {
float t0 = INFINITY, t1 = INFINITY;
if (spheres[i].intersect(rayorig, raydir, t0, t1)) {
if (t0 < 0) t0 = t1;
if (t0 < tnear) {
tnear = t0;
sphere = &spheres[i];
}
}
}
// if there's no intersection return black or background color
if (!sphere) return Vec3f(2);
Vec3f surfaceColor = 0; // color of the ray/surfaceof the object intersected by the ray
Vec3f phit = rayorig + raydir * tnear; // point of intersection
Vec3f nhit = phit - sphere->center; // normal at the intersection point
nhit.normalize(); // normalize normal direction
// If the normal and the view direction are not opposite to each other
// reverse the normal direction. That also means we are inside the sphere so set
// the inside bool to true. Finally reverse the sign of IdotN which we want
// positive.
float bias = 1e-4; // add some bias to the point from which we will be tracing
bool inside = false;
if (raydir.dot(nhit) > 0) nhit = -nhit, inside = true;
if ((sphere->transparency > 0 || sphere->reflection > 0) && depth < MAX_RAY_DEPTH) {
float facingratio = -raydir.dot(nhit);
// change the mix value to tweak the effect
float fresneleffect = mix(pow(1 - facingratio, 3), 1, 0.1);
// compute reflection direction (not need to normalize because all vectors
// are already normalized)
Vec3f refldir = raydir - nhit * 2 * raydir.dot(nhit);
refldir.normalize();
Vec3f reflection = trace(phit + nhit * bias, refldir, spheres, spheres_size, depth + 1);
Vec3f refraction = 0;
// if the sphere is also transparent compute refraction ray (transmission)
if (sphere->transparency) {
float ior = 1.1, eta = (inside) ? ior : 1 / ior; // are we inside or outside the surface?
float cosi = -nhit.dot(raydir);
float k = 1 - eta * eta * (1 - cosi * cosi);
Vec3f refrdir = raydir * eta + nhit * (eta * cosi - sqrt(k));
refrdir.normalize();
refraction = trace(phit - nhit * bias, refrdir, spheres, spheres_size, depth + 1);
}
// the result is a mix of reflection and refraction (if the sphere is transparent)
surfaceColor = (
reflection * fresneleffect +
refraction * (1 - fresneleffect) * sphere->transparency) * sphere->surfaceColor;
}
else {
// it's a diffuse object, no need to raytrace any further
for (unsigned i = 0; i < spheres_size; ++i) {
if (spheres[i].emissionColor.x > 0) {
// this is a light
Vec3f transmission = 1;
Vec3f lightDirection = spheres[i].center - phit;
lightDirection.normalize();
for (unsigned j = 0; j < spheres_size; ++j) {
if (i != j) {
float t0, t1;
if (spheres[j].intersect(phit + nhit * bias, lightDirection, t0, t1)) {
transmission = 0;
break;
}
}
}
surfaceColor += sphere->surfaceColor * transmission *
max(float(0), nhit.dot(lightDirection)) * spheres[i].emissionColor;
}
}
}
return surfaceColor + sphere->emissionColor;
}
extern "C" __global__
void raytrace_kernel(unsigned int width, unsigned int height, Vec3f *image, Sphere *spheres, unsigned int spheres_size, float invWidth, float invHeight, float aspectratio, float angle) {
int x = blockIdx.x * blockDim.x + threadIdx.x;
int y = blockIdx.y * blockDim.y + threadIdx.y;
if (y < height && x < width) {
float xx = (2 * ((x + 0.5) * invWidth) - 1) * angle * aspectratio;
float yy = (1 - 2 * ((y + 0.5) * invHeight)) * angle;
Vec3f raydir(xx, yy, -1);
raydir.normalize();
image[y*width+x] = trace(Vec3f(0), raydir, spheres, spheres_size, 0);
}
}
I can successfully compile it with: nvcc --ptx kernel.cu -o kernel.ptx (full PTX here) and use that PTX in the driver API with cuModuleLoadDataEx using the following snippet. It works as expected.
It works fine even if I uncomment the #include <math.h> line (actually, the PTX generated is exactly the same).
CudaSafeCall( cuInit(0) );
CUdevice device;
CudaSafeCall( cuDeviceGet(&device, 0) );
CUcontext context;
CudaSafeCall( cuCtxCreate(&context, 0, device) );
unsigned int error_buffer_size = 1024;
std::vector<CUjit_option> options;
std::vector<void*> values;
char* error_log = new char[error_buffer_size];
options.push_back(CU_JIT_ERROR_LOG_BUFFER); //Pointer to a buffer in which to print any log messages that reflect errors
values.push_back(error_log);
options.push_back(CU_JIT_ERROR_LOG_BUFFER_SIZE_BYTES); //Log buffer size in bytes. Log messages will be capped at this size (including null terminator)
values.push_back(&error_buffer_size);
options.push_back(CU_JIT_TARGET_FROM_CUCONTEXT); //Determines the target based on the current attached context (default)
values.push_back(0); //No option value required for CU_JIT_TARGET_FROM_CUCONTEXT
CUmodule module;
CUresult status = cuModuleLoadDataEx(&module, ptxSource, options.size(), options.data(), values.data());
if (error_log && error_log[0]) { //https://stackoverflow.com/a/7970669/3136474
std::cout << "Compiler error: " << error_log << std::endl;
}
CudaSafeCall( status );
However, whenever I try to compile this exact kernel using NVRTC (full PTX here), it compiles successfully but gives me a Segmentation fault (core dumped) on the call to cuModuleLoadDataEx (when trying to use the resulting PTX).
If I uncomment the #include <math.h> line, it fails at the nvrtcCompileProgram call with the following output:
nvrtcSafeBuild() failed at cuda_raytracer_nvrtc_api.cpp:221 : NVRTC_ERROR_COMPILATION
Build log:
/usr/include/bits/mathcalls.h(177): error: linkage specification is incompatible with previous "isinf"
__nv_nvrtc_builtin_header.h(126689): here
/usr/include/bits/mathcalls.h(211): error: linkage specification is incompatible with previous "isnan"
__nv_nvrtc_builtin_header.h(126686): here
2 errors detected in the compilation of "kernel.cu".
The code I'm using to compile it with NVRTC is:
nvrtcProgram prog;
NvrtcSafeCall( nvrtcCreateProgram(&prog, kernelSource, "kernel.cu", 0, NULL, NULL) );
// https://docs.nvidia.com/cuda/nvrtc/index.html#group__options
std::vector<const char*> compilationOpts;
compilationOpts.push_back("--device-as-default-execution-space");
// NvrtcSafeBuild is a macro which automatically prints nvrtcGetProgramLog if the compilation fails
NvrtcSafeBuild( nvrtcCompileProgram(prog, compilationOpts.size(), compilationOpts.data()), prog );
size_t ptxSize;
NvrtcSafeCall( nvrtcGetPTXSize(prog, &ptxSize) );
char* ptxSource = new char[ptxSize];
NvrtcSafeCall( nvrtcGetPTX(prog, ptxSource) );
NvrtcSafeCall( nvrtcDestroyProgram(&prog) );
Then I simply load the ptxSource using the previous snippet (note: that code block is the same used for both the driver API version and the NVRTC version).
Additional things that I've noticed/tried so far
The PTX generated by the NVCC and the one generated by NVRTC are quite different, but I'm unable to understand them to identify possible problems.
Tried to specify the specific GPU architecture (in my case, CC 6.1) to the compiler, no difference.
Tried to disable any compiler optimizations (options --ftz=false --prec-sqrt=true --prec-div=true --fmad=false in nvrtcCompileProgram). PTX file got bigger, but still Segfaulting.
Tried to add --std=c++11 or --std=c++14 to the NVRTC compiler options. With any of them NVRTC generates an almost empty (4 lines) PTX but issue no warning nor error until I try to use it.
Environment
SO: Ubuntu 18.04.4 LTS 64-bit
nvcc --version: Cuda compilation tools, release 10.1, V10.1.168. Built on Wed_Apr_24_19:10:27_PDT_2019
gcc --version: gcc (Ubuntu 7.5.0-3ubuntu1~18.04) 7.5.0
Hardware: Intel I7-7700HQ, GeForce GTX 1050 Ti
Edit on OP+1 day
I forgot to add my environment. See previous section.
Also can you compile the nvrtc output with ptxas? – #talonmies' comment
The nvcc-generated PTX compiles with a warning:
$ ptxas -o /tmp/temp_ptxas_output.o kernel.ptx
ptxas warning : Stack size for entry function 'raytrace_kernel' cannot be statically determined
Which is due to the recursive kernel function (more on that).
It can be safely ignored.
The nvrtc-generated PTX does not compile and issues the error:
$ ptxas -o /tmp/temp_ptxas_output.o nvrtc_kernel.ptx
ptxas fatal : Unresolved extern function '_Z5powiffi'
Based on this question I added __device__ to Sphere class constructor and removed --device-as-default-execution-space compiler option.
It generates a slightly different PTX now, but still presents the same error.
Compiling with the #include <math.h> now generates a lot of "A function without execution space annotations is considered a host function, and host functions are not allowed in JIT mode." warnings besides the previous errors.
If I try to use the accepted solution of the question it throws me a bunch of syntax errors and does not compile. NVCC still works flawlessly.
Just found the culprit by the ancient comment-and-test method: the error goes away if I remove the pow call used to calculate the fresnel effect inside the trace method.
For now, I've just replaced pow(var, 3) for var*var*var.
I created a MVCE and filled a bug report to NVIDIA: https://developer.nvidia.com/nvidia_bug/2917596.
Which Liam Zhang answered and pointed me the problem:
The issue in your code is that there is an incorrect option value being passed to cuModuleLoadDataEx. In lines:
options.push_back(CU_JIT_ERROR_LOG_BUFFER_SIZE_BYTES); //Log buffer size in bytes. Log messages will be capped at this size (including null terminator)
values.push_back(&error_buffer_size);
the buffer size option is provided, but instead of passing a value with the size, a pointer to that value is passed. Since this pointer is then read as a number, the driver assumed a much larger buffer size than 1024.
During the NVRTC compilation a "Unresolved extern function" error occurred, because the pow function signature, as you can find in the documentation is:
__device__ double pow ( double x, double y )
When the driver tried to zero the buffer when putting the error message in it, the segfault happened.
Without the call to pow, there was no compilation error, so the error buffer was not used and there was no segfault.
To ensure the device code is correct, the values used to call pow function as well as the output pointer should be a double number, or a float equivalent function, powf, could be used.
If I change the call to values.push_back((void*)error_buffer_size); it reports the same error as ptxas compilation of the generated PTX:
Compiler error: ptxas fatal : Unresolved extern function '_Z5powiffi'
cudaSafeCall() failed at file.cpp:74 : CUDA_ERROR_INVALID_PTX - a PTX JIT compilation failed
I'm using SSE instruction on my program to increase it performances but it often crashes on a call to _mm_shuffle_ps.
I know that most probably it is due to alignment that needs to be at 16 byte, but I can't really get around this issue.
This is the code I use (my program is compiled at 32bit with VisualStudio 2017):
#define SHUFFLEMASK(A0,A1,B2,B3) ( (A0) | ((A1)<<2) | ((B2)<<4) | ((B3)<<6) )
inline __m128 RotateVector(const __m128& quaternion, const __m128& vector)
{
const uint32 shuffleMask = SHUFFLEMASK(3, 3, 3, 3);
// THE NEXT LINE IS THE ONE CRASHING
const __m128 qw = _mm_shuffle_ps(quaternion, quaternion, shuffleMask);
// The rest isn't useful since it crashes before even getting there
...
}
inline __m128 MakeVectorRegister(float X, float Y, float Z, float W)
{
return _mm_setr_ps(X, Y, Z, W);
}
class Vertex
{
public:
union
{
float vec[3];
struct
{
float x, y, z;
};
};
// Rest of class (only methods, no other attributes)
...
};
__declspec(align(16)) class X
{
...
__m128 _scale;
__m128 _rotation;
...
Vertex TransformVector(const Vertex& vector) const
{
float __declspec(align(16)) vectorData[3];
memcpy(vectorData, &vector.x, sizeof(float) * 3);
// The next line was originally this: const __m128 inputVectorW0 = MakeVectorRegister(((const float*)(&vector.x))[0], ((const float*)(&vector.x))[1], ((const float*)(&vector.x))[2], 0.0f)
const __m128 inputVectorW0 = MakeVectorRegister(((const float*)(vectorData))[0], ((const float*)(vectorData))[1], ((const float*)(vectorData))[2], 0.0f)
const __m128 scaledVec = _mm_mul_ps(_scale, inputVectorW0);
const __m128 rotatedVec = RotateVector(_rotation, scaledVec);
// The rest isn't useful since it crashes before
...
}
}
// Example of usage
int main(...)
{
Vertex v;
X x;
// This crashes calling _mm_shuffle_ps inside RotateVector
Vertex result = x.TransformVector(v);
}
I'm trying to interpolate a 2D angle and it works 99.9% of the time. For some reason I'm getting -nan(ind) for some values, like:
lastAngle = -0.0613451
currentAngle = -0.061421
alpha = 0.218813
This is the code:
inline float slerpRotation(const float& angle1, const float& angle2, const float& alpha)
{
auto v1 = b2Vec2{std::cos(angle1), std::sin(angle1)};
auto v2 = b2Vec2{std::cos(angle2), std::sin(angle2)};
auto v = this->slerp(v1, v2, alpha);
return std::atan2(v.y, v.x);
}
inline b2Vec2 slerp(const b2Vec2& v1, const b2Vec2& v2, const float& alpha)
{
auto cosAngle = v1.x * v2.x + v1.y * v2.y;
auto angle = std::acos(cosAngle);
auto angleAlpha = angle * alpha;
auto v3 = (v2 - (cosAngle * v1)).Normalize();
auto x = v1.x * std::cos(angleAlpha) + v3 * std::sin(angleAlpha);
auto y = v1.y * std::cos(angleAlpha) + v3 * std::sin(angleAlpha);
return b2Vec2{x, y};
}
All this examples results in inf num:
slerpRotation(-0.0613451f, -0.061421f, 0.218813f);
slerpRotation(-1.63139f, -1.63139f, 0.723703f);
slerpRotation(-0.0614404f, -0.0614034f, 0.199831f);
slerpRotation(0.0194162f, 0.0194164f, 0.259074f);
I've tried to solve this problem for a while now without knowing what causes this problem, do you guys happened to know how to solve this?
In the end you are computing
angle1+alpha*(angle2-angle1)
or if you want to exclude some fringe cases,
angle1+alpha*reduce2pi(angle2-angle1)
where
reduce2pi(phi) = fmod( 3*pi + fmod(phi, 2*pi), 2*pi)-pi
Note that these formulas are completely singularity free, as there is no division. It is not necessary to switch forth and back between angles and their point on the unit circle.
In code, that would be
inline float slerpRotation(const float& angle1, const float& angle2, const float& alpha)
{
auto angleDiff = angle2-angle1;
angleDiff = std::fmod(angleDiff, 2*std::M_PI);
angleDiff = std::fmod(angleDiff + 3*std::M_PI, 2*std::M_PI)-std::M_PI;
return angle1+alpha*angleDiff;
}
(12/13/2016) combining several comments: If you insist on using exactly this interface structure, then you can get a singularity free method as follows:
inline b2Vec2 slerp(const b2Vec2& v1, const b2Vec2& v2, const float& alpha)
{
auto angle = std::atan2(v1.x*v2.y - v1.y*v2.x, v1.x*v2.x + v1.y*v2.y);
auto angleAlpha = angle * alpha;
auto v3=b2Vec2{-v1.y, v1.x}; // rotation by 90°
return std::cos(angleAlpha)*v1 + std::sin(angleAlpha)*v3;
}
Ok, i'm hoping someone can help me out here. I'm trying to multiply a Vector3 by a Matrix4 technically...
My FMatrix
struct FMatrix
{
struct FPlane XPlane;
struct FPlane YPlane;
struct FPlane ZPlane;
struct FPlane WPlane;
};
So Each FPlane has X,Y,Z,W
Making them FMatrix.XPlane.X FMatrix.XPlane.Y etc etc
Here's the code I have so far and I understand that my Vector3 technically needs transformed to a Vector4 using 1 or 0 for my W.
FVector2D get2dPoint(FVector worldloc)
{
FMatrix mat = mHUD->ViewProjectionMatrix;
FVector res;
float x = worldloc.X;
float y = worldloc.Y;
float z = worldloc.Z;
res.X = (mat.XPlane.X*x) + (mat.YPlane.X*y) + (mat.ZPlane.X*z) + (mat.WPlane.X * 1);
res.Y = (mat.XPlane.Y*x) + (mat.YPlane.Y*y) + (mat.ZPlane.Y*z) + (mat.WPlane.Y * 1);
res.Z = (mat.XPlane.Z*x) + (mat.YPlane.Z*y) + (mat.ZPlane.Z*z) + (mat.WPlane.Z * 1);
float winX = round(((res.X + 1.0f) / 2.0f) * mHUD->uiResolution.X);
float winY = round(((1.0f - res.Y) / 2.0f) * mHUD->uiResolution.Y);
return{ winX, winY };
}
As I'm sure most of you can see I'm trying to make a world to screen function. I am using boost library and I saw something about ublas and matrix's but I definitely got lost in that, but if that helps you to help me I'm cool with it. So either an example with normal c++ or using boost functions is fine with me.
I'm using a data structure not written by myself that returns a realVec. This is the declaration of realVec (in typedef.h):
typedef TNT::Vector<PCReal> realVec;
For the definition of TNT Vector please see: http://calicam.sourceforge.net/doxygen/tnt__vector_8h_source.html
Definition of PCReal:
typedef double PCReal;
I need to convert this realVec into the following vec3:
struct vec3 {
GLfloat x;
GLfloat y;
GLfloat z;
//
// --- Constructors and Destructors ---
//
vec3( GLfloat s = GLfloat(0.0) ) :
x(s), y(s), z(s) {}
vec3( GLfloat _x, GLfloat _y, GLfloat _z ) :
x(_x), y(_y), z(_z) {}
vec3( const vec3& v ) { x = v.x; y = v.y; z = v.z; }
vec3( const vec2& v, const float f ) { x = v.x; y = v.y; z = f; }
...
I'm very new to C++ so my confusion probably lies in using TNT::Vector's iterator and converting values returned by it. I'm thinking something like the below, so tell me if it makes sense. It seems to compile (no 'make' errors):
realVec normal = this->meshPoints.getNormalForVertex(i);
PCReal* iter = normal.begin();
vec3(*iter++, *iter++, *iter++);
I need this because I'm doing gl programming, and it's convenient for my shaders to take vec3's as input.
What you have might work, but you could improve it with some safety checks, and you don't really have to use iterators at all. It appears realVec provides operator[] like most other vector classes.
realVec normal = this->meshPoints.getNormalForVertex(i);
if (normal.size() >= 3)
{
vec3 x(normal[0], normal[1], normal[2]);
}