GLSL Tessellation Displacement Mapping - opengl

Question

in my recent project I am working with hardware side tessellation. The pipeline I want to implement should take a low poly mesh, tessellate it and apply a displacement map.
The Tessellation works fine and just as I expected it to look like. However, when I apply the displacement map in the tessellation evaluation shader I get an output which is somewhat random.
This is the output without displacement (I used the heightmap as a texture to verify whether my texCoords are accurate)
This is what I get when I enable my displacement (using the same texture for both coloring and displacement):
The shader code is as follows:
//VERTEX SHADER
#version 430
layout(location = 0) in vec4 vertex;
layout(location = 1) in vec4 normal;
layout(location = 2) in vec2 texCoord;
out vec3 vPosition;
out vec3 vNormal;
out vec2 vTexCoord;
void main() {
vPosition = vertex.xyz;
vNormal = normal.xyz;
vTexCoord = texCoord;
}
//TESS CONTROL
#version 430
layout(vertices = 3) out;
in vec3 vPosition[];
in vec3 vNormal[];
in vec2 vTexCoord[];
out vec3 tcPosition[];
out vec3 tcNormal[];
out vec2 tcTexCoord[];
uniform float innerTessLevel;
uniform float outerTessLevel;
void main(){
float inTess = innerTessLevel;
float outTess = outerTessLevel;
tcPosition[gl_InvocationID] = vPosition[gl_InvocationID];
tcNormal[gl_InvocationID] = vNormal[gl_InvocationID];
tcTexCoord[gl_InvocationID] = vTexCoord[gl_InvocationID];
if(gl_InvocationID == 0) {
gl_TessLevelInner[0] = inTess;
gl_TessLevelInner[1] = inTess;
gl_TessLevelOuter[0] = outTess;
gl_TessLevelOuter[1] = outTess;
gl_TessLevelOuter[2] = outTess;
gl_TessLevelOuter[3] = outTess;
}
}
//TESS EVAL
#version 430
layout(triangles, equal_spacing, ccw) in;
in vec3 tcPosition[];
in vec3 tcNormal[];
in vec2 tcTexCoord[];
out vec3 tePosition;
out vec2 teTexCoord;
uniform mat4 ModelViewProjection;
uniform mat4 ModelView;
uniform sampler2D texHeight;
void main(){
vec3 p0 = gl_TessCoord.x * tcPosition[0];
vec3 p1 = gl_TessCoord.y * tcPosition[1];
vec3 p2 = gl_TessCoord.z * tcPosition[2];
vec3 pos = p0 + p1 + p2;
vec3 n0 = gl_TessCoord.x * tcNormal[0];
vec3 n1 = gl_TessCoord.y * tcNormal[1];
vec3 n2 = gl_TessCoord.z * tcNormal[2];
vec3 normal = normalize(n0 + n1 + n2);
vec2 tc0 = gl_TessCoord.x * tcTexCoord[0];
vec2 tc1 = gl_TessCoord.y * tcTexCoord[1];
vec2 tc2 = gl_TessCoord.z * tcTexCoord[2];
teTexCoord = tc0 + tc1 + tc2;
float height = texture(texHeight, teTexCoord).x;
pos += normal * (height * 0.2f);
gl_Position = ModelViewProjection * vec4(pos, 1);
tePosition = vec3(ModelView * vec4(pos,1.0)).xyz;
}
//GEOMETRY
#version 430
layout(triangles) in;
layout(triangle_strip, max_vertices = 3) out;
uniform mat4 ModelView;
in vec3 tePosition[3];
in vec3 tePatchDistance[3];
in vec2 teTexCoord[3];
out vec3 gFacetNormal;
out vec2 gTexCoord;
void main() {
vec3 A = tePosition[2] - tePosition[0];
vec3 B = tePosition[1] - tePosition[0];
vec4 N = vec4( normalize(cross(A, B)) , 0.0);
gFacetNormal = N.xyz;
gTexCoord = teTexCoord[0];
gl_Position = gl_in[0].gl_Position; EmitVertex();
gTexCoord = teTexCoord[1];
gl_Position = gl_in[1].gl_Position; EmitVertex();
gTexCoord = teTexCoord[2];
gl_Position = gl_in[2].gl_Position; EmitVertex();
EndPrimitive();
}
//FRAGMENT
#version 430
layout(location = 0) out vec4 fragColor;
in vec3 gFacetNormal;
in vec2 gTexCoord;
uniform float lit;
uniform vec3 light;
uniform sampler2D texHeight;
void main() {
#ifndef ORANGE_PURPLE
vec3 color = gl_FrontFacing ? vec3(1.0,0.0,0.0) : vec3(0.0,0.0,1.0);
#else
vec3 color = gl_FrontFacing ? vec3(1.0,0.6,0.0) : vec3(0.6,0.0,1.0);
#endif
if (lit > 0.5) {
color = texture(texHeight, gTexCoord).xyz;
vec3 N = normalize(gFacetNormal);
vec3 L = light;
float df = abs(dot(N,L));
color = df * color;
fragColor = vec4(color,1.0);
}
else {
fragColor = vec4(color,1.0);
}
}
It would be nice if someone could help me on that one.

Answer 1

Thanks to #AndonM.Coleman I solved the matter
In fact: the output gFacetNormal is only defined for your first vertex in the geometry shader. Outputs have to be set after every EmitVertex (...) as-per the GLSL specification, or they will be undefined. Many implementations re-use the last value set, but you cannot rely on that behavior if you want this to work portably. You need to set gFacetNormal once before every EmitVertex. void EmitVertex () - "Emits the current values of output variables to the current output primitive. On return from this call, the values of output variables are undefined."
Stupid of me not to notice that, but here is the working code:
//VERTEX
#version 430
layout(location = 0) in vec4 vertex;
layout(location = 1) in vec4 normal;
layout(location = 2) in vec2 texCoord;
out vec3 vPosition;
out vec3 vNormal;
out vec2 vTexCoord;
void main() {
vPosition = vertex.xyz;
vNormal = normal.xyz;
vTexCoord = texCoord;
}
//TESSELLATION CONTROL
#version 430
layout(vertices = 3) out;
in vec3 vPosition[];
in vec3 vNormal[];
in vec2 vTexCoord[];
out vec3 tcPosition[];
out vec3 tcNormal[];
out vec2 tcTexCoord[];
uniform float innerTessLevel;
uniform float outerTessLevel;
void main(){
float inTess = innerTessLevel;
float outTess = outerTessLevel;
tcPosition[gl_InvocationID] = vPosition[gl_InvocationID];
tcNormal[gl_InvocationID] = vNormal[gl_InvocationID];
tcTexCoord[gl_InvocationID] = vTexCoord[gl_InvocationID];
if(gl_InvocationID == 0) {
gl_TessLevelInner[0] = inTess;
gl_TessLevelInner[1] = inTess;
gl_TessLevelOuter[0] = outTess;
gl_TessLevelOuter[1] = outTess;
gl_TessLevelOuter[2] = outTess;
gl_TessLevelOuter[3] = outTess;
}
}
//TESSELLATION EVALUATION
#version 430
layout(triangles, equal_spacing, ccw) in;
in vec3 tcPosition[];
in vec3 tcNormal[];
in vec2 tcTexCoord[];
out vec3 tePosition;
out vec2 teTexCoord;
out vec3 teNormal;
uniform mat4 ModelViewProjection;
uniform mat4 ModelView;
uniform sampler2D texHeight;
void main(){
vec3 p0 = gl_TessCoord.x * tcPosition[0];
vec3 p1 = gl_TessCoord.y * tcPosition[1];
vec3 p2 = gl_TessCoord.z * tcPosition[2];
vec3 pos = p0 + p1 + p2;
vec3 n0 = gl_TessCoord.x * tcNormal[0];
vec3 n1 = gl_TessCoord.y * tcNormal[1];
vec3 n2 = gl_TessCoord.z * tcNormal[2];
vec3 normal = normalize(n0 + n1 + n2);
vec2 tc0 = gl_TessCoord.x * tcTexCoord[0];
vec2 tc1 = gl_TessCoord.y * tcTexCoord[1];
vec2 tc2 = gl_TessCoord.z * tcTexCoord[2];
teTexCoord = tc0 + tc1 + tc2;
float height = texture(texHeight, teTexCoord).x;
pos += normal * (height * 0.5f);
gl_Position = ModelViewProjection * vec4(pos, 1);
teNormal = vec3(ModelView * vec4(normal,0.0)).xyz;
tePosition = vec3(ModelView * vec4(pos,1.0)).xyz;
}
//GEOMETRY
#version 430
layout(triangles) in;
layout(triangle_strip, max_vertices = 3) out;
uniform mat4 ModelView;
in vec3 tePosition[3];
in vec2 teTexCoord[3];
in vec3 teNormal[3];
out vec3 gFacetNormal;
out vec2 gTexCoord;
void main() {
gFacetNormal = teNormal[0];
gTexCoord = teTexCoord[0];
gl_Position = gl_in[0].gl_Position; EmitVertex();
gFacetNormal = teNormal[1];
gTexCoord = teTexCoord[1];
gl_Position = gl_in[1].gl_Position; EmitVertex();
gFacetNormal = teNormal[2];
gTexCoord = teTexCoord[2];
gl_Position = gl_in[2].gl_Position; EmitVertex();
EndPrimitive();
}
//FRAGMENT
#version 430
layout(location = 0) out vec4 fragColor;
in vec3 gFacetNormal;
in vec2 gTexCoord;
uniform float lit;
uniform vec3 light;
uniform sampler2D texHeight;
void main() {
#ifndef ORANGE_PURPLE
vec3 color = gl_FrontFacing ? vec3(1.0,0.0,0.0) : vec3(0.0,0.0,1.0);
#else
vec3 color = gl_FrontFacing ? vec3(1.0,0.6,0.0) : vec3(0.6,0.0,1.0);
#endif
if (lit > 0.5) {
color = texture(texHeight, gTexCoord).xyz;
vec3 N = normalize(gFacetNormal);
vec3 L = light;
float df = abs(dot(N,L));
color = df * color;
fragColor = vec4(color,1.0);
}
else {
fragColor = vec4(color,1.0);
}
}