Correctly use stencil_texturing in OpenGL - opengl

Question

I am trying to implement the stencil_texturing extension of OpenGL as a proof of concept. My video card supports up to GL 4.3 so stencil_texturing is available to me. If more clarification is necessary here is the spec provided: http://www.opengl.org/registry/specs/ARB/stencil_texturing.txt.
So the goal of my test is to render my color buffer to a texture in frame 0, then the depth buffer in frame 1 and finally the stencil buffer in frame 2. The easy part is done and I have my color and depth buffer textures rendered fine. My issue lies with the stencil buffer and I believe the issue is coming from either my lack of understanding stencil buffers (which could very well be the case) or is my misuse of stencil_texturing. I tried to find some info online but there is very little available.
To give you an idea of what I am rendering here are my current frame captures:
Color buffer, Depth buffer, Stencil buffer
So my vision for the stencil buffer is to just stencil out the middle triangle, so everything in the middle triangle has a value of 1 and every part of the texture has a value of 0. I am not sure how this will come up when rendering but I imagine the areas with a stencil value of 1 will be different than those with 0.
Here is my code below. It is just a test class that I throw into a framework I made for them. I believe the only thing not definied is GLERR() which basically calls glGetError() to make sure everything is correct.
typedef struct
{
GLuint program;
GLuint vshader;
GLuint fshader;
} StencilTexturingState;
class TestStencilTexturing : public TestInfo
{
public:
TestStencilTexturing(TestConfig& config, int argc, char** argv)
:width(config.windowWidth), height(config.windowHeight)
{
state = (StencilTexturingState*) malloc(sizeof(StencilTexturingState));
}
~TestStencilTexturing()
{
destroyTestStencilTexturing();
}
void loadFBOShaders()
{
const char* vshader = "assets/stencil_texturing/fbo_vert.vs";
const char* fshader = "assets/stencil_texturing/fbo_frag.fs";
state->vshader = LoadShader(vshader, GL_VERTEX_SHADER);
GLERR();
state->fshader = LoadShader(fshader, GL_FRAGMENT_SHADER);
GLERR();
state->program = Link(state->vshader, state->fshader, 1, "inPosition");
GLERR();
glUseProgram(state->program);
}
void loadTextureShaders()
{
const char* vshader = "assets/stencil_texturing/tex_vert.vs";
const char* fshader = "assets/stencil_texturing/tex_frag.fs";
state->vshader = LoadShader(vshader, GL_VERTEX_SHADER);
GLERR();
state->fshader = LoadShader(fshader, GL_FRAGMENT_SHADER);
GLERR();
state->program = Link(state->vshader, state->fshader, 1, "inPosition");
GLERR();
glUseProgram(state->program);
}
void destroyTestStencilTexturing()
{
glUseProgram(0);
glDeleteShader(state->vshader);
glDeleteShader(state->fshader);
glDeleteProgram(state->program);
free(state);
}
void RenderToTexture(GLuint renderedTexture, int frame)
{
GLint posId, colId;
GLuint fboId, depth_stencil_rb;
const float vertexFBOPositions[] =
{
-0.7f, -0.7f, 0.5f, 1.0f,
0.7f, -0.7f, 0.5f, 1.0f,
0.6f, 0.7f, 0.5f, 1.0f,
};
const float vertexFBOColors[] =
{
1.0f, 1.0f, 0.0f, 1.0f,
1.0f, 1.0f, 1.0f, 1.0f,
1.0f, 0.0f, 0.0f, 1.0f,
};
// Load shaders for the FBO
loadFBOShaders();
// Setup the FBO
glGenFramebuffers(1, &fboId);
glBindFramebuffer(GL_FRAMEBUFFER, fboId);
glViewport(0, 0, width, height);
// Set up renderbuffer for depth_stencil formats.
glGenRenderbuffers(1, &depth_stencil_rb);
glBindRenderbuffer(GL_RENDERBUFFER, depth_stencil_rb);
glRenderbufferStorage(GL_RENDERBUFFER, GL_DEPTH24_STENCIL8, width, height);
glFramebufferRenderbuffer(GL_FRAMEBUFFER, GL_DEPTH_STENCIL_ATTACHMENT,
GL_RENDERBUFFER, depth_stencil_rb);
// Depending on the frame bind the 2D texture differently.
// Frame 0 - Color, Frame 1 - Depth, Frame 2 - Stencil
glBindTexture(GL_TEXTURE_2D, renderedTexture);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
// Create our RGBA texture to render our color buffer into.
if (frame == 0)
{
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, width, height, 0, GL_RGBA, GL_UNSIGNED_BYTE, NULL);
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, renderedTexture, 0);
}
// Create our Depth24_Stencil8 texture to render our depth buffer into.
if (frame == 1)
{
glEnable(GL_DEPTH_TEST);
glTexImage2D(GL_TEXTURE_2D, 0, GL_DEPTH24_STENCIL8, width, height, 0, GL_DEPTH_STENCIL, GL_UNSIGNED_INT_24_8, NULL);
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_DEPTH_STENCIL_ATTACHMENT, GL_TEXTURE_2D, renderedTexture, 0);
}
// Create our Depth24_Stencil8 texture and change depth_stencil_texture mode
// to render our stencil buffer into.
if (frame == 2)
{
glEnable(GL_DEPTH_TEST | GL_STENCIL_TEST);
glTexImage2D(GL_TEXTURE_2D, 0, GL_DEPTH24_STENCIL8, width, height, 0, GL_DEPTH_STENCIL, GL_UNSIGNED_INT_24_8, NULL);
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_DEPTH_STENCIL_ATTACHMENT, GL_TEXTURE_2D, renderedTexture, 0);
glTexParameteri(GL_TEXTURE_2D, GL_DEPTH_STENCIL_TEXTURE_MODE, GL_STENCIL_INDEX);
}
GLERR();
GLenum status = glCheckFramebufferStatus(GL_FRAMEBUFFER);
if (status != GL_FRAMEBUFFER_COMPLETE)
{
printf("There is an error with the Framebuffer, fix it!\n");
}
GLERR();
// Give the values of the position and color of our triangle to the shaders.
posId = glGetAttribLocation(state->program, "position");
colId = glGetAttribLocation(state->program, "color");
GLERR();
glVertexAttribPointer(posId, 4, GL_FLOAT, 0, 0, vertexFBOPositions);
glEnableVertexAttribArray(posId);
glVertexAttribPointer(colId, 4, GL_FLOAT, 0, 0, vertexFBOColors);
glEnableVertexAttribArray(colId);
// Clear the depth buffer back to 1.0f to draw our RGB stripes far back.
glClearDepth(1.0f);
glClear(GL_DEPTH_BUFFER_BIT);
if (frame == 2)
{
glStencilFunc(GL_NEVER, 1, 0xFF); // never pass stencil test
glStencilOp(GL_REPLACE, GL_KEEP, GL_KEEP); // replace stencil buffer values to ref=1
glStencilMask(0xFF); // stencil buffer free to write
glClear(GL_STENCIL_BUFFER_BIT); // first clear stencil buffer by writing default stencil value (0) to all of stencil buffer.
glDrawArrays(GL_TRIANGLES, 0, 3); // at stencil shape pixel locations in stencil buffer replace stencil buffer values to ref = 1
// no more modifying of stencil buffer on stencil and depth pass.
glStencilMask(0x00);
// can also be achieved by glStencilOp(GL_KEEP, GL_KEEP, GL_KEEP);
// stencil test: only pass stencil test at stencilValue == 1 (Assuming depth test would pass.) and write actual content to depth and color buffer only at stencil shape locations.
glStencilFunc(GL_EQUAL, 1, 0xFF);
}
// Use the Scissors to clear the FBO with a RGB stripped pattern.
glEnable(GL_SCISSOR_TEST);
glScissor(width * 0/3, 0, width * 1/3, height);
glClearColor(0.54321f, 0.0f, 0.0f, 0.54321f); // Red
glClear(GL_COLOR_BUFFER_BIT);
glScissor(width * 1/3, 0, width * 2/3, height);
glClearColor(0.0f, 0.65432f, 0.0f, 0.65432f); // Green
glClear(GL_COLOR_BUFFER_BIT);
glScissor(width * 2/3, 0, width * 3/3, height);
glClearColor(0.0f, 0.0f, 0.98765f, 0.98765f); // Blue
glClear(GL_COLOR_BUFFER_BIT);
glDisable(GL_SCISSOR_TEST);
GLERR();
glDrawArrays(GL_TRIANGLES, 0, 3);
glDisable(GL_DEPTH_TEST);
GLERR();
// Remove FBO and shaders and return to original viewport.
glUseProgram(0);
glBindFramebuffer(GL_FRAMEBUFFER, 0);
glDeleteShader(state->vshader);
glDeleteShader(state->fshader);
glDeleteProgram(state->program);
glDeleteFramebuffers(1, &fboId);
glViewport(0, 0, width, height);
GLERR();
}
void drawFrameTestStencilTexturing(int frame)
{
GLint posLoc, texLoc;
GLuint renderedTexture;
const GLubyte indxBuf[] = {0, 1, 2, 1, 3, 2};
const float positions[] =
{
-0.8f, -0.8f,
-0.8f, 0.8f,
0.8f, -0.8f,
0.8f, 0.8f,
};
const float texCoords[] =
{
0.0f, 0.0f,
0.0f, 1.0f,
1.0f, 0.0f,
1.0f, 1.0f
};
// Allocate and initialize the texture that will be rendered to, and then
// textured onto a quad on the default framebuffer.
glGenTextures(1, &renderedTexture);
// Render to the texture using FBO.
RenderToTexture(renderedTexture, frame);
// Create and load shaders to draw the texture.
loadTextureShaders();
// Draw texture to the window.
glClearColor(0.25f, 0.25f, 0.25f, 1.0f);
glClear(GL_COLOR_BUFFER_BIT);
posLoc = glGetAttribLocation(state->program, "position");
texLoc = glGetAttribLocation(state->program, "a_texCoords");
glVertexAttribPointer(posLoc, 2, GL_FLOAT, 0, 0, positions);
glEnableVertexAttribArray(posLoc);
glVertexAttribPointer(texLoc, 2, GL_FLOAT, 0, 0, texCoords);
glEnableVertexAttribArray(texLoc);
// Draw our generated texture onto a quad.
glDrawElements(GL_TRIANGLES, 6, GL_UNSIGNED_BYTE, indxBuf);
glFlush();
glDeleteTextures(1, &renderedTexture);
GLERR();
}
void renderTest(int frame)
{
drawFrameTestStencilTexturing(frame);
}
private:
StencilTexturingState* state;
const int height, width;
};
RUN_TEST(StencilTexturing, "stencil_texturing", 2);

Answer 1

The line
glEnable(GL_DEPTH_TEST | GL_STENCIL_TEST);
is not going to work, the GL enable enums are NOT bit values, but just enums, so you might enable something else, or just get some GL_INVALID_ENUM error, but you don't enable the stencil test here.