Spotlight with shadows becomes square-like - c++

Question

I've been following a two part tutorial on shadow mapping from OGLDev (part 1, part 2) for a couple of days now, and I've nearly finished implementing it.
My problem is that when I enable shadow maps the illuminated region which shadows appear in appears as a "strip" and I'm unsure if this is intended in the tutorials or not.
Perhaps images will help explain my problem.
However shadows are functional:
My vertex shader for spotlights:
#version 330
in vec2 textureCoordinate;
in vec4 vertex;
uniform mat4 mMatrix;
uniform mat4 pMatrix;
uniform mat4 vMatrix;
uniform mat4 lightV;
uniform mat4 lightP;
uniform vec3 normal;
uniform vec3 eye;
out vec2 TexCoord0;
out vec4 LightSpacePos;
out vec3 Normal0;
out vec3 WorldPos0;
out vec3 EyePos;
void main()
{
EyePos = eye;
TexCoord0 = textureCoordinate;
WorldPos0 = (mMatrix * vertex).xyz;
Normal0 = (mMatrix * vec4(normal, 0.0)).xyz;
mat4 lightMVP = lightP * inverse(lightV) * mMatrix;
LightSpacePos = lightMVP * vertex;
mat4 worldMVP = pMatrix * vMatrix * mMatrix;
gl_Position = worldMVP * vertex;
}
and the fragment shader:
#version 330
struct BaseLight
{
vec4 color;
float intensity;
};
struct Attenuation
{
float constant;
float linear;
float exponent;
};
struct PointLight
{
BaseLight base;
Attenuation atten;
vec3 position;
float range;
};
struct SpotLight
{
struct PointLight base;
vec3 direction;
float cutoff;
};
in vec2 TexCoord0;
in vec3 WorldPos0;
in vec3 EyePos;
in vec4 LightSpacePos;
out vec4 fragColor;
uniform sampler2D sampler;
uniform sampler2D shadowMap;
in vec3 Normal0;
uniform float specularIntensity;
uniform float specularPower;
uniform SpotLight spotLight;
float CalcShadowFactor(vec4 LightSpacePos)
{
vec3 ProjCoords = LightSpacePos.xyz / LightSpacePos.w;
vec2 UVCoords;
UVCoords.x = 0.5 * ProjCoords.x + 0.5;
UVCoords.y = 0.5 * ProjCoords.y + 0.5;
float z = 0.5 * ProjCoords.z + 0.5;
float Depth = texture(shadowMap, UVCoords).x;
if (Depth < z + 0.00001)
return 0.5;
else
return 1.0;
}
vec4 CalcLightInternal(BaseLight Light, vec3 LightDirection, vec3 Normal, float ShadowFactor)
{
vec4 AmbientColor = Light.color * Light.intensity;
float DiffuseFactor = dot(Normal, -LightDirection);
vec4 DiffuseColor = vec4(0, 0, 0, 0);
vec4 SpecularColor = vec4(0, 0, 0, 0);
if (DiffuseFactor > 0) {
DiffuseColor = Light.color * Light.intensity * DiffuseFactor;
vec3 VertexToEye = normalize(EyePos - WorldPos0);
vec3 LightReflect = normalize(reflect(LightDirection, Normal));
float SpecularFactor = dot(VertexToEye, LightReflect);
SpecularFactor = pow(SpecularFactor, specularPower);
if (SpecularFactor > 0) {
SpecularColor = Light.color * specularIntensity * SpecularFactor;
}
}
return (AmbientColor + ShadowFactor * (DiffuseColor + SpecularColor));
}
vec4 CalcPointLight(PointLight l, vec3 Normal, vec4 LightSpacePos)
{
vec3 LightDirection = WorldPos0 - l.position;
float Distance = length(LightDirection);
LightDirection = normalize(LightDirection);
float ShadowFactor = CalcShadowFactor(LightSpacePos);
vec4 Color = CalcLightInternal(l.base, LightDirection, Normal, ShadowFactor);
float Attenuation = l.atten.constant +
l.atten.linear * Distance +
l.atten.exponent * Distance * Distance;
return Color / Attenuation;
}
vec4 CalcSpotLight(SpotLight l, vec3 Normal, vec4 LightSpacePos)
{
vec3 LightToPixel = normalize(WorldPos0 - l.base.position);
float SpotFactor = dot(LightToPixel, l.direction);
if (SpotFactor > l.cutoff) {
vec4 Color = CalcPointLight(l.base, Normal, LightSpacePos);
return Color * (1.0 - (1.0 - SpotFactor) * 1.0/(1.0 - l.cutoff));
}
else {
return vec4(0,0,0,0);
}
}
void main(void)
{
fragColor = texture2D(sampler, TexCoord0.xy) * CalcSpotLight(spotLight, normalize(Normal0), LightSpacePos);
}

Answer 1

This is intended. Since your shadowmap covers a pyramid-like region in space, your spotlight's cone can be occluded by it. This is happening because where you render something that is outside of the shadow camera's view, it will be considered unlitten. Therefore the shadow camera's view pyramid will be visible.
src
To fix this you have 2 options:
1: Make the shadowmap camera's fov bigger, so that the shadow camera's pyramid becomes wider than your spotlight's cone
2: Change the way you calculate shadows on out-of-bounds area. Currently when you sample the shadowmap, and if it's outside of the shadow texture, you apply shadow there. If you change this so that if something that is out of the shadow texture you consider it litten, this problem will disappear.
Edit:
I recommend the second option. A solution for that could be:
Insert the following line into FragmentShader::CalcShadowFactor(), just before the float Depth = ... part
if (UVCoords.x < 0.0 || UVCoords.x > 1.0 || UVCoords.y < 0.0 || UVCoords.y > 1.0) return 1.0;
Note:
There is no universal way to say which is better. In a terrain environment involving sunlight, you may want to consider out-of-bounds regions as litten. However when you use a flashlight for example, that is in the user's hand, you have to consider out-of-bounds regions as unlitten.