Weird eigen matrix multiplication behavior when using row-major matrices - c++

Question

The following is working just fine for me:
using Mat4 = Eigen::Matrix4f;
using Vec3 = Eigen::Vector3f;
class Camera {
public:
Vec3 pos = { 0.0f, 0.0f, 0.0f };
Vec3 forward = { 0.0f, 0.0f, 1.0f };
Vec3 up = { 0.0f, 1.0f, 0.0f };
float vfov = degrees(45.0f);
float near_plane = 0.1f;
float far_plane = 100.0f;
Mat4 cameraToScreen(float aspect_ratio) const {
Mat4 result;
float tanHalfFovy = tan(vfov *0.5f);
float x_scale = 1.0f / (aspect_ratio * tanHalfFovy);
float y_scale = 1.0f / (tanHalfFovy);
result <<
x_scale, 0, 0, 0,
0, y_scale, 0, 0,
0, 0, -(far_plane + near_plane) / (far_plane - near_plane), -1.0f,
0, 0, -2.0f * near_plane * far_plane / (far_plane - near_plane), 0;
return result;
}
Mat4 worldToCamera() const {
Mat4 result;
Vec3 p = pos;
Vec3 f = forward.normalized();
Vec3 r = f.cross(up).normalized();
Vec3 u = r.cross(f).normalized();
result <<
r.x(), u.x(), -f.x(), 0.0f,
r.y(), u.y(), -f.y(), 0.0f,
r.z(), u.z(), -f.z(), 0.0f,
-r.dot(p), -u.dot(p), f.dot(p), 1.0f;
return result;
}
Mat4 worldToScreen(float aspect_ratio) const {
return cameraToScreen(aspect_ratio) * worldToCamera();
}
};
However, since this is going to OpenGL, and I don't like dealing with transpositions, I though I could simply switch to row-major matrices:
using Mat4 = Eigen::Matrix<float, 4, 4, Eigen::RowMajor>;
Unfortunately, everything broke. Interestingly enough, inverting the final matrix multiplication (which is wrong) fixes the issue:
Mat4 worldToScreen(float aspect_ratio) const {
return worldToCamera() * cameraToScreen(aspect_ratio);
}
It's almost as if eigen performed the matrix multiplication with the assumption that the storage was column-major, regardless of my template tag, but that can't be right.
I tested both matrices individually, and they are performing as I want in row-major mode when used in isolation, it's only the product of both matrices that's wrong.
What am I not understanding with the way Eigen handles row-major matrices?
Edit:
Here's effectively how I send the data to OpenGL. My code has a lot of additional boilerplate, but it boils down to this:
struct CameraData {
Mat4 world_to_screen ;
};
void draw(Camera const& cam) {
CameraData cam_data;
// I remove the transposed() call when using Eigen::RowMajor
cam_data.world_to_screen = cam.worldToScreen(16.0f/9.0f).transposed();
// Camera data has a permanent ubo binding point shared by all programs.
glBindBuffer(GL_UNIFORM_BUFFER, cam_data_ubo);
glBufferSubData(GL_UNIFORM_BUFFER, 0, sizeof(cam_data), &cam_data);
//.. perform drawing
}
and the glsl:
#version 420
in vec3 vertex;
layout (std140) uniform CameraData {
mat4 world_to_screen;
};
void main() {
gl_Position = world_to_screen * vec4(vertex, 1.0);
}