Recreating scene from depth encoded image in Vulkan vs OpenGL - c++

Question

I'm in the process of translating a piece of OpenGL code to Vulkan. The code recreates a rendered scene from an image (on a hemisphere projection) with depth information encoded. Note that I also load the model view matrix used for the projection to recreate the scene. The translation has been pretty straightforward but I'm running into issues due to the new Vulkan coordinate system.
The original OpenGL shader with comments follows:
#version 430
layout (triangles) in;
layout (triangle_strip, max_vertices = 3) out;
in vec2 posGeom[];
out vec2 texCoord;
uniform mat4 view;
uniform mat4 projection;
uniform float threshold;
uniform vec3 quantization;
uniform mat4 inverseStaticView;
uniform sampler2D rgbdTexture;
//get the image space for each pixel of our hemisphere image
vec3 getSphereRay(const vec2 coord) {
//get length of ray from camera to point on image plane
float len = 1 - coord.x * coord.x - coord.y * coord.y;
if (len > 0)
return vec3(coord, -sqrt(len));//scale to unit length vector as viewing ray
else
return vec3(0);
}
vec4 getPosition(const in vec2 inCoord, const in float depth) {
vec2 coord = inCoord;
//reverse the stretching from sphere to quad (based on y-coordinate)
float percent = sqrt(1.0 - coord.y * coord.y);
coord.x = coord.x * percent;
//scale ray with corresponding depth
vec3 normal = getSphereRay(coord) * depth;
//move from image space to world space by inverse view matrix
return inverseStaticView * vec4(normal, 1);
}
bool hasZeroDepth = false;
//get the real depth from quantized and packed depth by inverting the gamma correction and inverting min, max
float getDepth(int idx) {
float depth = texture(rgbdTexture, posGeom[idx] * 0.5 + 0.5).w;
if(depth == 0)
hasZeroDepth = true;
float minDepth = quantization.x;
float maxDepth = quantization.y;
float gamma = quantization.z;
depth = pow(depth, gamma);
depth = depth * (maxDepth - minDepth) + minDepth;
return depth;
}
//emit the position and texcoord
void emitPosition(int idx, float depth) {
texCoord = posGeom[idx] * 0.5 + 0.5;
gl_Position = projection * view * getPosition(posGeom[idx], depth);
EmitVertex();
}
void main() {
float d0 = getDepth(0);
float d1 = getDepth(1);
float d2 = getDepth(2);
//do not emit tris with zero (invalid) depth
if(!hasZeroDepth) {
float minDepth = min(d0, min(d1, d2));
float maxDepth = max(d0, max(d1, d2));
float minDist = maxDepth - minDepth;
float avgDepth = (d0 + d1 + d2) / 3;
float thres = threshold;
//look at tri stretching factor
if(minDist / avgDepth < thres) {
//emit original tri
emitPosition(0, d0);
emitPosition(1, d1);
emitPosition(2, d2);
} else {
//emit tri with maxDepth to only show background
emitPosition(0, maxDepth);
emitPosition(1, maxDepth);
emitPosition(2, maxDepth);
}
}
}
In the Vulkan shader, I account for the Vulkan coordinate system by inverting the y value. I also must normalize the world values for reasons that are unclear to me (otherwise what's rendered is completely nonsense). The shader code follows:
#version 450
layout (triangles) in;
layout (triangle_strip, max_vertices = 3) out;
layout(binding = 0) uniform UniformBufferObject {
mat4 modelView;
mat4 inverseStaticModelView;
float quantization;
} ubo;
layout(binding = 1) uniform sampler2D texSampler;
layout(location = 0) in vec2 posGeom[];
layout(location = 0) out vec2 texCoord;
bool hasZeroDepth = false;
float minDepth = 0;
float maxDepth = 1.0;
vec3 unproject(vec2 win) {
float scale = 1 - win.y * win.y;
// Invert y to account for Vulkan coordinate system.
float y = win.y * -1;
// Scale x to account for hemisphere projection.
float x = win.x * scale;
float z = -sqrt(1 - x * x - y * y);
if(z < 0){
vec4 outVals = ubo.inverseStaticModelView * vec4(x, y, z, 1.0);
return vec3(outVals[0], outVals[1], outVals[2]) / outVals.w;
}else
return vec3(0);
}
vec3 reconstructWorldPosition(vec2 ndc, float depth) {
vec3 pos = unproject(ndc);
return depth * normalize(pos);
}
float getDepth(int idx) {
float depth = texture(texSampler, posGeom[idx] * 0.5 + 0.5).w;
if(depth == 0)
hasZeroDepth = true;
depth = pow(depth, ubo.quantization);
return depth;
}
void emitPosition(int idx, float depth) {
vec2 pos = posGeom[idx].xy;
texCoord = pos * 0.5 + 0.5;
vec3 positionFromDepth = reconstructWorldPosition(pos, depth);
gl_Position = ubo.modelView * vec4(positionFromDepth,1);
EmitVertex();
}
void main() {
float d0 = getDepth(0);
float d1 = getDepth(1);
float d2 = getDepth(2);
if(!hasZeroDepth) {
float minDepth = min(d0, min(d1, d2));
float maxDepth = max(d0, max(d1, d2));
float minDist = maxDepth - minDepth;
float avgDepth = (d0 + d1 + d2) / 3.0;
float thres = 0.1;
if(minDist / avgDepth < thres ) {
emitPosition(0, d0);
emitPosition(1, d1);
emitPosition(2, d2);
} else {
emitPosition(0, maxDepth);
emitPosition(1, maxDepth);
emitPosition(2, maxDepth);
}
}
}
Images of the output of the two programs are contained in this album: http://imgur.com/a/KUl57
The Vulkan output appears to almost be correct except for some odd artifacts in the lower left hand of the scene. My suspicion is that the scaling to the x coordinate to account for the hemisphere projection is causing the issue. I've played around with the scaling and other parts of the shader but I can't seem to get it right. Am I overlooking something else that is different between Vulkan and OpenGL, especially with regards to the coordinate system?