opengl get window space to world space cursor position - c++

Question

i am trying to get the mouse cursor in world space pos from window space(-1,1 window width and height) using the viewprojectionmatrix.
This is how i calculate my projection matrix:
static mat4 Perspective4x4(float FOV, float AspectRatio, float FarC, float NearC)
{
// Positive x is right
// Positive y is up
// Positive z is forward into the screen
float Cotangent = 1.0f / tanf((FOV)*0.5f);
float Depth = NearC - FarC;
float A = (-FarC - NearC) / Depth;
float B = 2.0f * FarC * NearC / Depth;
return
{
Cotangent/AspectRatio, 0.0f, 0.0f, 0.0f,
0.0f, Cotangent, 0.0f, 0.0f,
0.0f, 0.0f, -A, -B,
0.0f, 0.0f, 1.0f, 0.0f,
};
}
...
float WidthOverHeight = ...;
mat4 ProjectionMatrix = Perspective4x4(DegreesToRadians(90.0f), WidthOverHeight, 50.0f, 0.1f);
This is how i calculate my view matrix:
static mat4 Translate4x4(vec3 V)
{
return
{
1.0f, 0.0f, 0.0f, V.X,
0.0f, 1.0f, 0.0f, V.Y,
0.0f, 0.0f, 1.0f, V.Z,
0.0f, 0.0f, 0.0f, 1.0f
};
}
...
mat4 ViewMatrix = Translate4x4(-CameraPosition);
This is how i calculate the cursor position in worldspace:
mat4 ProjectionMatrix = ...;
mat4 ViewMatrix = ...;
vec2 CursorP = GetBilateralCursorPos(Input); // Values between -1 and 1
v4_f32 WorldSpacePNear = Inverse(ProjectionMatrix) * V4F32(CursorP, -1.0f, 1.0f);
WorldSpacePNear /= WorldSpacePNear.W;
WorldSpacePNear = Inverse(ViewMatrix) * WorldSpacePNear;
v4_f32 WorldSpacePFar = Inverse(ProjectionMatrix) * V4F32(CursorP, 1.0f, 1.0f);
WorldSpacePFar /= WorldSpacePFar.W;
WorldSpacePFar = Inverse(ViewMatrix) * WorldSpacePFar;
WorldSpacePFar.Z *= -1.0f;
WorldSpacePNear.Z *= -1.0f;
EDIT: I also tried dividing by W at the end but it doesnt work properly either.
This is how i send the matrices to OpenGL (legacy):
// I send them transposed because my matrices are row-major
mat4 ProjectionMatrix = Transpose(...);
mat4 ViewMatrix = Transpose(...);
glMatrixMode(GL_PROJECTION);
glLoadMatrixf(ProjectionMatrix.E);
glMatrixMode(GL_MODELVIEW);
glLoadMatrixf(ViewMatrix.E);
The resulting positions do not seem to take in account the view projection because a change in the camera position will offset them making them not accurate.
NOTES:
vec4, vec2, mat4 are custom float-based math types. (they are what you would expect)
I know legacy OpenGL is deprecated, in fact im going to switch to modern opengl very soon, i just want to get this working.

Answer 1

i fixed the issue calculating the cursor pos like this:
mat4 ProjectionMatrix = ...;
mat4 ViewMatrix = ...;
// Z Distance from camera pos
float WorldDistanceFromCameraZ = 1.0f;
vec2 CursorP = GetBilateralCursorPos(Input); // Values between -1 and 1
vec4 ProbeZ = V4F32(World->Camera.P - WorldDistanceFromCameraZ*World->Camera.P.Z, 1.0f);
ProbeZ = (ProjectionMatrix*ViewMatrix) * ProbeZ;
vec4 ClipP = V4F32(CursorP.X*ProbeZ.W, CursorP.Y*ProbeZ.W, ProbeZ.Z, ProbeZ.W);
vec4 WorldP = Inverse(ProjectionMatrix*ViewMatrix) * ClipP;