Problem with diffuse lighting with OpenGL - c++

Question

My diffuse lighting doesn't seem to be working properly.
Fragment Shader:
#version 330 core
out vec4 gl_FragColor;
in vec4 vertexColor;
in vec2 texelCoord;
in vec3 Normal;
struct DirectionalLight
{
vec3 color;
float ambientIntensity;
vec3 direction;
float diffuseIntensity;
};
uniform sampler2D textureSampler;
uniform DirectionalLight directionalLight;
void main()
{
vec4 ambientColor = vec4(directionalLight.color, 1.0f) * directionalLight.ambientIntensity;
float diffuseFactor = max(dot(normalize(Normal), normalize(directionalLight.direction)), 0.0f);
vec4 diffuseColor = vec4(directionalLight.color, 1.0f) * directionalLight.diffuseIntensity * diffuseFactor;
gl_FragColor = texture(textureSampler, texelCoord) * (ambientColor + diffuseColor);
}
Vertex Shader:
#version 330 core
layout (location = 0) in vec3 position;
layout (location = 1) in vec2 texCoord;
layout (location = 2) in vec3 normal;
out vec4 vertexColor;
out vec2 texelCoord;
out vec3 Normal;
uniform mat4 transformation;
uniform mat4 projection;
uniform mat4 view;
void main()
{
gl_Position = projection * view * transformation * vec4( position, 1.0f );
vertexColor = vec4(clamp(position, 0.0f, 1.0f), 1.0f);
texelCoord = texCoord;
Normal = mat3(transpose(inverse(transformation))) * normal;
}
How I create Meshes:
void CalcAverageNormals( unsigned int* indices , unsigned int indicesCount , float* vertices , unsigned int verticesCount , unsigned int vertexLength , unsigned int normalOffset )
{
for ( int i = 0; i < indicesCount; i += 3 )
{
unsigned int v1 = indices[i] * vertexLength;
unsigned int v2 = indices[ i + 1 ] * vertexLength;
unsigned int v3 = indices[ i + 2 ] * vertexLength;
glm::vec3 line1( vertices[ v2 ] - vertices[ v1 ] , vertices[ v2 + 1 ] - vertices[ v1 + 1 ] , vertices[ v2 + 2 ] - vertices[ v1 + 2 ] );
glm::vec3 line2( vertices[ v3 ] - vertices[ v1 ] , vertices[ v3 + 1 ] - vertices[ v1 + 1 ] , vertices[ v3 + 2 ] - vertices[ v1 + 2 ] );
glm::vec3 normal = glm::normalize( glm::cross( line1 , line2 ) );
v1 += normalOffset;
v2 += normalOffset;
v3 += normalOffset;
vertices[ v1 ] += normal.x; vertices[ v1 + 1 ] += normal.y; vertices[ v1 + 2 ] += normal.z;
vertices[ v2 ] += normal.x; vertices[ v2 + 1 ] += normal.y; vertices[ v2 + 2 ] += normal.z;
vertices[ v3 ] += normal.x; vertices[ v3 + 1 ] += normal.y; vertices[ v3 + 2 ] += normal.z;
}
for ( int j = 0; j < verticesCount / vertexLength; j++ )
{
unsigned int offset = j * vertexLength + normalOffset;
glm::vec3 normalVertex( vertices[ offset ] , vertices[ offset + 1 ] , vertices[ offset + 2 ] );
normalVertex = glm::normalize( normalVertex );
vertices[ offset ] = normalVertex.x;
vertices[ offset + 1 ] = normalVertex.y;
vertices[ offset + 2 ] = normalVertex.z;
}
}
void CreateTriangle() {
float vertices[] {
-0.5f,-0.5f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, // Left
0.5f,-0.5f, 0.0f, 1.0f, 0.0f, 0.0f, 0.0f, 0.0f, // Right
0.0f, 0.5f, 0.0f, 0.5f, 1.0f, 0.0f, 0.0f, 0.0f, // Top
0.0f,-0.5f, 0.5f, 0.5f, 0.0f, 0.0f, 0.0f, 0.0f // Back Z
};
unsigned int indices[]{
0, 1, 2, // Front
3, 2, 1, // Right
3, 2, 0, // Left
3, 0, 1 // Bottom
};
CalcAverageNormals( indices , 12 , vertices , 32 , 8 , 5 );
for ( int i = 0; i < 1; i++ )
{
Mesh* obj = new Mesh();
obj->CreateMesh( vertices , 32 , indices , 12 );
meshlist.push_back( obj );
}
}
CreateMesh()
void Mesh::CreateMesh( float* vertices , unsigned int numVertices , unsigned int* indices , unsigned int numIndices )
{
uIndices = numIndices;
glGenVertexArrays( 1 , &vao );
glBindVertexArray( vao );
/*Create Buffers*/
glGenBuffers( 1 , &ibo );
glBindBuffer( GL_ELEMENT_ARRAY_BUFFER , ibo );
glBufferData( GL_ELEMENT_ARRAY_BUFFER , numIndices * sizeof(unsigned) , indices , GL_STATIC_DRAW );
glGenBuffers( 1 , &vbo );
glBindBuffer( GL_ARRAY_BUFFER , vbo );
glBufferData( GL_ARRAY_BUFFER , numVertices * sizeof(float) , vertices , GL_STATIC_DRAW );
glVertexAttribPointer( 0 , 3 , GL_FLOAT , GL_FALSE , sizeof( vertices[ 0 ] ) * 8 , 0 );
glEnableVertexAttribArray( 0 );
glVertexAttribPointer( 1 , 2 , GL_FLOAT , GL_FALSE , sizeof( vertices[ 0 ] ) * 8 , ( void* )( sizeof( vertices[ 0 ] ) * 3 ) );
glEnableVertexAttribArray( 1 );
glVertexAttribPointer( 2 , 3 , GL_FLOAT , GL_FALSE , sizeof( vertices[ 0 ] ) * 8 , ( void* )( sizeof( vertices[ 0 ] ) * 5 ) );
glEnableVertexAttribArray( 2 );
/*Unbind Objects*/
glBindBuffer( GL_ARRAY_BUFFER , 0 );
glBindVertexArray( 0 );
glBindBuffer( GL_ELEMENT_ARRAY_BUFFER , 0 );
}
I only get some sort of lighting if I rotate the mesh:
Normal ( No lighting):
I been trying to figure this out for a days but I am not sure what I did wrong. If you could help me that would be great.

Answer 1

It turns out it was my order of indices having problem with triangle winding. I fixed this by drawing the indices in counter clockwise order because of how my diffuse factor is calculated.
unsigned int indices[]{
0, 1, 2, // Front
3, 1, 2, // Right
3, 0, 2, // Left
3, 1, 0 // Bottom
};
The transpose and inverse calculation is to correct non-uniform scale matrices.

Answer 2

this:
Normal = mat3(transpose(inverse(transformation))) * normal;
looks suspicious the normal itself should be transformed into the same coordinate system as the light direction vector is in. As normal is a vector the w should be zero so I would expect either
Normal = view * transformation * vec4( normal, 0.0f );
if the light direction is in camera coordinates or:
Normal = transformation * vec4( normal, 0.0f );
if the light direction is in global world coordinates (which is more likely the case)...
Also setting shader output variable like gl_Position should be the last line of your shader or you risk GL implementation optimize out the rest of code after it on some implementations.
btw. IIRC mat3(transpose(inverse(transformation))) is the same as mat3(transformation) in case the transformation but as Rabbid76 pointed out it has its purpose...
One last thing I can think of is wrong normal direction (opposite or inconsistent) in such case I would try exchange:
max(dot(normalize(Normal), normalize(directionalLight.direction)), 0.0f);
with:
abs(dot(normalize(Normal), normalize(directionalLight.direction));
If it helps you have to check the normals or just negate the dot result...
For more info see:
OpenGL - vertex normals in OBJ ... however there I use w=1 as the matrix used has no offset so it does not matter
Understanding 4x4 homogenous transform matrices