Shader for a spotlight - opengl

Question

How can I get this shader to have a smooth edge on the spot light instead of a hard one? In addition, the shader has to cope with a variable value of GL_SPOT_CUTOFF. Note that not all the lights are spot lights -- GL_LIGHT0 is a point light.
varying vec3 N;
varying vec3 v;
#define MAX_LIGHTS 2
void main (void)
{
vec4 finalColour;
float spotEffect;
for (int i=0; i<MAX_LIGHTS; i++)
{
vec3 L = normalize(gl_LightSource[i].position.xyz - v);
vec3 E = normalize(-v);
vec3 R = normalize(-reflect(L,N));
spotEffect = dot(normalize(gl_LightSource[i].spotDirection),
normalize(-L));
if (spotEffect > gl_LightSource[i].spotCosCutoff) {
vec4 Iamb = gl_FrontLightProduct[i].ambient;
vec4 Idiff = gl_FrontLightProduct[i].diffuse * max(dot(N,L), 0.0);
Idiff = clamp(Idiff, 0.0, 1.0);
vec4 Ispec = gl_FrontLightProduct[i].specular
* pow(max(dot(R,E),0.0),0.3*gl_FrontMaterial.shininess);
Ispec = clamp(Ispec, 0.0, 1.0);
finalColour += Iamb + Idiff + Ispec;
}
}
gl_FragColor = gl_FrontLightModelProduct.sceneColor + finalColour;
}
The scene looks like this:

Answer 1

This shader from http://www.ozone3d.net/tutorials/glsl_lighting_phong_p3.php produces the soft edges to the spotlight you are after.
[Pixel_Shader]
varying vec3 normal, lightDir, eyeVec;
const float cos_outer_cone_angle = 0.8; // 36 degrees
void main (void)
{
vec4 final_color =
(gl_FrontLightModelProduct.sceneColor * gl_FrontMaterial.ambient) +
(gl_LightSource[0].ambient * gl_FrontMaterial.ambient);
vec3 L = normalize(lightDir);
vec3 D = normalize(gl_LightSource[0].spotDirection);
float cos_cur_angle = dot(-L, D);
float cos_inner_cone_angle = gl_LightSource[0].spotCosCutoff;
float cos_inner_minus_outer_angle =
cos_inner_cone_angle - cos_outer_cone_angle;
//****************************************************
// Don't need dynamic branching at all, precompute
// falloff(i will call it spot)
float spot = 0.0;
spot = clamp((cos_cur_angle - cos_outer_cone_angle) /
cos_inner_minus_outer_angle, 0.0, 1.0);
//****************************************************
vec3 N = normalize(normal);
float lambertTerm = max( dot(N,L), 0.0);
if(lambertTerm > 0.0)
{
final_color += gl_LightSource[0].diffuse *
gl_FrontMaterial.diffuse *
lambertTerm * spot;
vec3 E = normalize(eyeVec);
vec3 R = reflect(-L, N);
float specular = pow( max(dot(R, E), 0.0),
gl_FrontMaterial.shininess );
final_color += gl_LightSource[0].specular *
gl_FrontMaterial.specular *
specular * spot;
}
gl_FragColor = final_color;