GLSL ES Check if fragment is at texture's boundary - glsl

Question

I'm trying to make an outline shader for 2d sprites, basically it takes a sprite and checks for a color, if the fragment has that color it is considered an outline, it then checks the texels around it and if none of them are transparent its alpha is set to 0.
Basically what I need to do is make the shader ignore the borders of the texture, since if the fragment is on a border it will always be considered to be an outline.
I send the custom_FragCoord variable containing the absolute uv coordinates from the vertex shader to the fragment shader, and then I say, for instance, if "custom_FragCoord.x > 1. do outline check", to make everything drawn on the first column be considered an outline.
The problem is when the sprite has a border with nothing drawn on it, then the shader doesn't seem to start drawing at the border of the sprite, so for instance if a sprite has nothing on its left border, then it will start drawing at custom_FragCoord.x = 1., not 0., so it will not automatically consider it an outline and instead will check the adjacent texels, and when it checks the left texel it won't find a transparent texel because it tried to check the left texel from the texture's boundary.
If someone could please shed some light on what could be done that would be an immense help.
Here's the code if the link doesn't work:
//////////////////////// Vertex shader ////////////////////////
attribute vec3 in_Position; // (x,y,z)
//attribute vec3 in_Normal; // (x,y,z) unused in this shader.
attribute vec4 in_Colour; // (r,g,b,a)
attribute vec2 in_TextureCoord; // (u,v)
varying vec2 v_vTexcoord;
varying vec4 v_vColour;
varying vec2 custom_FragCoord;
void main()
{
vec4 object_space_pos = vec4( in_Position.x, in_Position.y, in_Position.z, 1.0);
gl_Position = gm_Matrices[MATRIX_WORLD_VIEW_PROJECTION] * object_space_pos;
v_vColour = in_Colour;
v_vTexcoord = in_TextureCoord;
//Send absolute fragment coordinate to fragment shader, maybe there's a different coordinate that should be sent instead since checks using this one only work when the sprite's texture touches all borders of the sprite size
custom_FragCoord = (gm_Matrices[MATRIX_WORLD] * object_space_pos).xy;
}
//////////////////////// Fragment shader ////////////////////////
///Outlines shader
varying vec2 v_vTexcoord;
varying vec4 v_vColour;
uniform vec3 sl_v3_ColorTo; //What color should the outline be
uniform vec2 sl_v2_PixelSize; //Distance to next fragment's x/y, for size of step calculation
uniform vec2 sl_v2_SpriteSize; //Size of current drawn sprite (Not used currently, but could be relevant idk)
varying vec2 custom_FragCoord; //Absolute fragment coordinate
void main()
{
vec3 v3_colorToTest = vec3(1.,1.,1.); //White outline color, for testing
vec3 v3_outLineColor = vec3(0.149, 0.149, 0.149); //Color of outline to look for, if fragment is not this color just ignore
//Check difference between fragment color and acceptable outline color
vec3 v3_colDiff = vec3 ( texture2D(gm_BaseTexture, v_vTexcoord).r - v3_outLineColor.r,
texture2D(gm_BaseTexture, v_vTexcoord).g - v3_outLineColor.g,
texture2D(gm_BaseTexture, v_vTexcoord).b - v3_outLineColor.b);
//How much does the fragment's color differ from the outline color it seeks
float f_colDiff = (v3_colDiff.x+v3_colDiff.y+v3_colDiff.z)/3.;
//If fragment color varies by more than 0.001 set alpha to 0, otherwise set it to 8
float alpha = 8.*floor(texture2D(gm_BaseTexture, v_vTexcoord).a + 0.001 -abs(f_colDiff));
//Bunch of conditionals, just to test, I'll take them off once stuff works
/*Here lies the problem: If the sprite is, for instance, 32x32, but only the bottom-half of it has stuff to draw, the "custom_FragCoord.y > 1" check will be useless,
since it will start drawing at custom_FragCoord.y = 15, not custom_FragCoord.y = 0*/
if (custom_FragCoord.x > 1. && custom_FragCoord.y > 1. && custom_FragCoord.x < sl_v2_SpriteSize.x-1. && custom_FragCoord.y < sl_v2_SpriteSize.y-1.)
{
//Check all around for transparency, if none is found it is not an outline
for (float i = 0.; i <= 315.; i+= 45.)
{
alpha -= ceil(texture2D(gm_BaseTexture, v_vTexcoord +vec2(sign(cos(i))*sl_v2_PixelSize.x,sign(sin(i))*sl_v2_PixelSize.y)).a);
}
}
//Paint result, with a white color to test out
vec4 col = vec4(v3_colorToTest, alpha);
gl_FragColor = col;
}

Answer 1

Figured it out, had to manually pass the sprite's texture UV borders to the shader, through sprite_get_uvs().
Here's the shader if anyone is interested:
//////////////////////// Vertex shader ////////////////////////
attribute vec3 in_Position; // (x,y,z)
//attribute vec3 in_Normal; // (x,y,z) unused in this shader.
attribute vec4 in_Colour; // (r,g,b,a)
attribute vec2 in_TextureCoord; // (u,v)
varying vec2 v_vTexcoord;
varying vec4 v_vColour;
void main()
{
vec4 object_space_pos = vec4( in_Position.x, in_Position.y, in_Position.z, 1.0);
gl_Position = gm_Matrices[MATRIX_WORLD_VIEW_PROJECTION] * object_space_pos;
v_vColour = in_Colour;
v_vTexcoord = in_TextureCoord;
}
//////////////////////// Fragment shader ////////////////////////
//Outlines shader
varying vec2 v_vTexcoord;
varying vec4 v_vColour;
uniform vec3 sl_v3_ColorTo; //What color should the outline be
uniform vec2 sl_v2_PixelSize; //Size of display, for size of step calculation
uniform vec4 sl_v2_TextureUV; //Texture's UV coordinates
void main()
{
vec3 v3_colorToTest = vec3(1.,1.,1.);
vec3 v3_outLineColor = vec3(0.149, 0.149, 0.149);
vec3 v3_colDiff = vec3 ( texture2D(gm_BaseTexture, v_vTexcoord).r - v3_outLineColor.r,
texture2D(gm_BaseTexture, v_vTexcoord).g - v3_outLineColor.g,
texture2D(gm_BaseTexture, v_vTexcoord).b - v3_outLineColor.b);
float f_colDiff = (v3_colDiff.x+v3_colDiff.y+v3_colDiff.z)/3.;
float alpha = 8.*floor(texture2D(gm_BaseTexture, v_vTexcoord).a + 0.001 -abs(f_colDiff));
vec4 v3_borderCheck = vec4 ( v_vTexcoord.x - sl_v2_TextureUV.x,
v_vTexcoord.y - sl_v2_TextureUV.y,
sl_v2_TextureUV.z - v_vTexcoord.x,
sl_v2_TextureUV.w - v_vTexcoord.y);
//Checks the borders, if on border is always outline
alpha += floor(1.-v3_borderCheck.x +sl_v2_PixelSize.x);
alpha += floor(1.-v3_borderCheck.y +sl_v2_PixelSize.y);
alpha += floor(1.-v3_borderCheck.z +sl_v2_PixelSize.x);
alpha += floor(1.-v3_borderCheck.w +sl_v2_PixelSize.x);
//Check neighbors
alpha -= ceil(texture2D(gm_BaseTexture, v_vTexcoord + vec2(sl_v2_PixelSize.x, 0.)).a);
alpha -= ceil(texture2D(gm_BaseTexture, v_vTexcoord + vec2(-sl_v2_PixelSize.x, 0.)).a);
alpha -= ceil(texture2D(gm_BaseTexture, v_vTexcoord + vec2(0., sl_v2_PixelSize.y)).a);
alpha -= ceil(texture2D(gm_BaseTexture, v_vTexcoord + vec2(0., -sl_v2_PixelSize.y)).a);
//Check diagonal neighbors
alpha -= ceil(texture2D(gm_BaseTexture, v_vTexcoord + vec2(sl_v2_PixelSize.x, sl_v2_PixelSize.y)).a);
alpha -= ceil(texture2D(gm_BaseTexture, v_vTexcoord + vec2(-sl_v2_PixelSize.x, sl_v2_PixelSize.y)).a);
alpha -= ceil(texture2D(gm_BaseTexture, v_vTexcoord + vec2(sl_v2_PixelSize.x, -sl_v2_PixelSize.y)).a);
alpha -= ceil(texture2D(gm_BaseTexture, v_vTexcoord + vec2(-sl_v2_PixelSize.x, -sl_v2_PixelSize.y)).a);
vec4 col = vec4(v3_colorToTest, alpha); //alpha * sl_f_OutlineAlpha here later, sl_OutlineAlpha being a variable changeable in object (not dependent on object's image_alpha, set it to object_alpha inside object when appropriate)
gl_FragColor = col;
}
It works in a very specific way so I don't know if it will be useful for anyone else.
I'm sure there are places that could be optimized, if someone has suggestions please tell me.