Why stencil test does not discard fragments? - c++

Question

I want to run stencil test to outline cubes. It is from learnopengl https://learnopengl.com/Advanced-OpenGL/Stencil-testing.
A cubes that do outlining should have discarded fragments that are on a textured cubes. It does not happen, the outlining cubes cover the whole area.
static const char* cubeVtx = R"(
#version 460 core
layout (location = 0) in vec3 a_position;
layout (location = 1) in vec2 a_uv;
out vec2 v_uv;
uniform mat4 model;
uniform mat4 view;
uniform mat4 projection;
void main()
{
v_uv = a_uv;
gl_Position = projection * view * model * vec4(a_position, 1.0f);
}
)";
static const char* cubeFrag = R"(
#version 460 core
layout (location = 0) out vec4 f_color;
in vec2 v_uv;
uniform sampler2D u_texture1;
void main()
{
f_color = texture(u_texture1, v_uv);
}
)";
static const char* singleColorVtx = R"(
#version 460 core
layout (location = 0) in vec3 a_position;
uniform mat4 model;
uniform mat4 view;
uniform mat4 projection;
void main()
{
gl_Position = projection * view * model * vec4(a_position, 1.0f);
}
)";
static const char* singleColorFrag = R"(
#version 460 core
layout (location = 0) out vec4 f_color;
void main()
{
f_color = vec4(1.0f, 0.0f, 0.0f, 1.0f);
}
)";
void Cube::InitAPI()
{
shader = Shader::Create();
ShaderProgram program[2] = {
shader->ShaderFromSource(cubeVtx, ShaderType::Vertex),
shader->ShaderFromSource(cubeFrag, ShaderType::Fragment)
};
shader->End(program, 2);
singleColorShader = Shader::Create();
ShaderProgram singleColorProgram[2] = {
singleColorShader->ShaderFromSource(singleColorVtx, ShaderType::Vertex),
singleColorShader->ShaderFromSource(singleColorFrag, ShaderType::Fragment)
};
singleColorShader->End(singleColorProgram, 2);
glCreateVertexArrays(1, &vao);
glBindVertexArray(vao);
glCreateBuffers(1, &vbo);
glBindBuffer(GL_ARRAY_BUFFER, vbo);
glNamedBufferData(vbo, sizeof(cubeVertices), cubeVertices, GL_STATIC_DRAW);
glEnableVertexAttribArray(0);
glEnableVertexAttribArray(1);
glVertexAttribPointer(0, 3, GL_FLOAT, false, 5 * sizeof(float), 0);
glVertexAttribPointer(1, 2, GL_FLOAT, false, 5 * sizeof(float), (void*)(3 * sizeof(float)));
glCreateVertexArrays(1, &planeVao);
glBindVertexArray(planeVao);
glCreateBuffers(1, &planeVbo);
glBindBuffer(GL_ARRAY_BUFFER, planeVbo);
glNamedBufferData(planeVbo, sizeof(planeVertices), planeVertices, GL_STATIC_DRAW);
glEnableVertexAttribArray(0);
glEnableVertexAttribArray(1);
glVertexAttribPointer(0, 3, GL_FLOAT, false, 5 * sizeof(float), 0);
glVertexAttribPointer(1, 2, GL_FLOAT, false, 5 * sizeof(float), (void*)(3 * sizeof(float)));
texture = Texture::LoadPNG("Media/container2");
planeTexture = Texture::LoadPNG("Media/matrix");
glEnable(GL_DEPTH_TEST);
glDepthFunc(GL_LESS);
glEnable(GL_STENCIL_TEST);
glStencilFunc(GL_NOTEQUAL, 1, 0xFF);
glStencilOp(GL_KEEP, GL_KEEP, GL_REPLACE);
shader->UseProgram();
shader->Uniform1i("u_texture1", 0);
}
void Cube::Render()
{
singleColorShader->UseProgram();
glm::mat4 model = glm::mat4(1.0f);
glm::mat4 view = camera.GetViewMatrix();
glm::mat4 projection = glm::perspective(glm::radians(camera.Zoom), (float)1280.0f / (float)720.0f, 0.1f, 100.0f);
singleColorShader->UniformMatrix4fv("view", view);
singleColorShader->UniformMatrix4fv("projection", projection);
shader->UseProgram();
shader->UniformMatrix4fv("view", view);
shader->UniformMatrix4fv("projection", projection);
glStencilMask(0x00);
glBindVertexArray(planeVao);
planeTexture->Active();
planeTexture->Bind();
glDrawArrays(GL_TRIANGLES, 0, 6);
glStencilFunc(GL_ALWAYS, 1, 0xFF);
glStencilMask(0xFF);
glBindVertexArray(vao);
model = glm::translate(model, glm::vec3(-1.0f, 0.0f, -1.0f));
shader->UniformMatrix4fv("model", model);
texture->Active();
texture->Bind();
glDrawArrays(GL_TRIANGLES, 0, 36);
model = glm::mat4(1.0f);
model = glm::translate(model, glm::vec3(2.0f, 0.0f, 0.0f));
shader->UniformMatrix4fv("model", model);
glDrawArrays(GL_TRIANGLES, 0, 36);
glStencilFunc(GL_NOTEQUAL, 1, 0xFF);
glStencilMask(0x00);
glDisable(GL_DEPTH_TEST);
singleColorShader->UseProgram();
float scale = 1.1;
glBindVertexArray(vao);
model = glm::mat4(1.0f);
model = glm::translate(model, glm::vec3(-1.0f, 0.0f, -1.0f));
model = glm::scale(model, glm::vec3(scale, scale, scale));
shader->UniformMatrix4fv("model", model);
glDrawArrays(GL_TRIANGLES, 0, 36);
model = glm::mat4(1.0f);
model = glm::translate(model, glm::vec3(2.0f, 0.0f, 0.0f));
model = glm::scale(model, glm::vec3(scale, scale, scale));
shader->UniformMatrix4fv("model", model);
glDrawArrays(GL_TRIANGLES, 0, 36);
glBindVertexArray(0);
glStencilMask(0xFF);
glEnable(GL_DEPTH_TEST);
}
Test::Test()
{
//tried many patterns and stencil values
SDL_GL_SetAttribute(SDL_GL_CONTEXT_PROFILE_MASK, SDL_GL_CONTEXT_PROFILE_CORE);
SDL_GL_SetAttribute(SDL_GL_CONTEXT_MAJOR_VERSION, 4);
SDL_GL_SetAttribute(SDL_GL_CONTEXT_MINOR_VERSION, 6);
SDL_GL_SetAttribute(SDL_GL_DOUBLEBUFFER, 1);
SDL_GL_SetAttribute(SDL_GL_STENCIL_SIZE, 8);
window = SDL_CreateWindow("window", SDL_WINDOWPOS_CENTERED, SDL_WINDOWPOS_CENTERED, 1280, 720, SDL_WINDOW_OPENGL);
context = SDL_GL_CreateContext(window);
SDL_GL_MakeCurrent(window, context);
glewExperimental = true;
glewInit();
cube.InitAPI();
}
void Test::MainLoop()
{
while (!quit) {
while (SDL_PollEvent(&event)) {
if (event.type == SDL_QUIT) {
quit = true;
}
}
glClearColor(0.5f, 0.5f, 0.5f, 1.0f);
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT | GL_STENCIL_BUFFER_BIT);
cube.Render();
SDL_GL_SwapWindow(window);
}
}
I still quite do not get the depth test thing and when I started thinking that I am getting stencil test, it all messed up, because it does not work.
Yes, I do clear color, depth and stencil buffer, somewhere else, but before Render() method.
What I got:
What I expect is a red outline, not the whole cube
Could you please explain like I were an idiot why it does not discard fragments that are 1

Answer 1

Call SDL_Init, before calling any other SDL-function
SDL_Init(SDL_INIT_EVERYTHING);
See the documentation of SDL_Init():
Use this function to initialize the SDL library. This must be called before using most other SDL functions.
To make the stencil buffer work, the SDL_GL_STENCIL_SIZE attribute has to be set. By default this attribute is 0. In general a stencil buffer has 8 bits. e.g.:
SDL_GL_SetAttribute(SDL_GL_STENCIL_SIZE, 8);
SDL_GL_SetAttribute has to be called before SDL_CreateWindow else it won't have any effect.
See the documentation of SDL_GL_SetAttribute:
Use this function to set an OpenGL window attribute before window creation.

Answer 2

The original code calls SDL_Init after SDL_GL_SetAttribute()
The code above, does not even call SDL_Init. Initializing SDL fixes everything.