glReadPixels returning incorrect values - c++

Question

I am trying to implement picking via following an opengl tutorial, I have a mesh with 6000 vertices and I wish to pick particular ones; I have chosen to do by redering uniquely coloured boxes at each vertex point, read the pixel on my mouse click at that point and that should return to me the ID of the closest vertex. The background is rendered as white so if I miss I get nothing.
However I have a problem, it only works most of the time; there are certain areas that if I click there I get white returned even though its clearly a vertex, and when rendering the colorized scene clearly has a redish box at the point I clicked.
Then there are white areas near the mesh, to the bottom left and at some random point away from it, returns me a hit.
I do not understand at all why this is happening, it should work.
void Display() {
Controls->setVector(indexed_vertices);
if (Controls->getPicking()) {
// Clear the screen in white
glClearColor(1.0f, 1.0f, 1.0f, 1.0f);
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
for ( int i=0; i< indexed_vertices.size(); i++) {
// use shaders
glUseProgram(pickingProgramID);
// Get a handle for our "MVP" uniform
GLuint PickingMatrixID = glGetUniformLocation(pickingProgramID, "MVP");
glm::mat4 RotationMatrix = glm::toMat4(orientations);
glm::mat4 btTranslationMatrix = glm::translate(glm::mat4(1.0f), indexed_vertices[i]);
glm::mat4 myModelMatrix = ModelMatrix * Controls->getTranslationMatrix() * Controls->getRotationMatrix() * btTranslationMatrix;
MVP = ProjectionMatrix * ViewMatrix * myModelMatrix;
Controls->setCntrlsViewMatrix(ViewMatrix);
Controls->setCntrlsProjectionMatrix(ProjectionMatrix);
glUniformMatrix4fv(PickingMatrixID, 1, GL_FALSE, &MVP[0][0]);
// Convert "i", the integer mesh ID, into an RGB color
int r = (i & 0x000000FF) >> 0;
int g = (i & 0x0000FF00) >> 8;
int b = (i & 0x00FF0000) >> 16;
// OpenGL expects colors to be in [0,1], so divide by 255.
glUniform4f(pickingColorID, r/255.0f, g/255.0f, b/255.0f, 1.0f);
// 1rst attribute buffer : vertices
glEnableVertexAttribArray(0);
glBindBuffer(GL_ARRAY_BUFFER, gvertexbuffer);
glVertexAttribPointer(
0, // attribute. No particular reason for 0, but must match the layout in the shader.
3, // size
GL_FLOAT, // type
GL_FALSE, // normalized?
0, // stride
(void*)0 // array buffer offset
);
// Index buffer
glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, elementbuffer);
// Draw the triangles !
glDrawElements(
GL_TRIANGLES, // mode
indices.size(), // count
GL_UNSIGNED_SHORT, // type
(void*)0 // element array buffer offset
);
// OpenGL expects colors to be in [0,1], so divide by 255.
}
glDisableVertexAttribArray(0);
glFlush();
glFinish();
glPixelStorei(GL_UNPACK_ALIGNMENT, 1);
// Read the pixel at the center of the screen.
// You can also use glfwGetMousePos().
// Ultra-mega-over slow too, even for 1 pixel,
// because the framebuffer is on the GPU.
unsigned char data[4];
glReadPixels(Controls->get_mx_cur(), Controls->get_my_cur(),1,1, GL_RGBA, GL_UNSIGNED_BYTE, data);
std::cout << "MX: " << Controls->get_mx_cur() << " MY: " << Controls->get_my_cur() << std::endl;
// Convert the color back to an integer ID
int pickedID =
data[0] +
data[1] * 256 +
data[2] * 256*256;
//std::cout << std::hex << pickedID << std::dec<<std::endl;
if (pickedID == 0x00ffffff) { // Full white, must be the background !
printf("Miss\n");
}
else {
std::cout << "mesh " << pickedID << std::endl;
}
// Uncomment these lines to see the picking shader in effect
glutSwapBuffers();
skip = true;
Controls->setPicking(false);
}
if (!skip) {
// White background
glClearColor(0.2f, 0.25f, 0.5f, 0.0f);
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
glEnable(GL_DEPTH_TEST);
//glEnable(GL_CULL_FACE);
glUseProgram(ShaderIDs[0]);
glActiveTexture(GL_TEXTURE0);
glBindTexture(GL_TEXTURE_2D, Texture);
glUniform1i(TextureID, 0);
glm::mat4 myModelMatrix = ModelMatrix * Controls->getTranslationMatrix() * Controls->getRotationMatrix();
MVP = ProjectionMatrix * ViewMatrix * myModelMatrix;
// The inverse transpose of the View Model Matrix will re-normalize the normals if there's
// been any scaling. Otherwise you don't need it.
glm::mat3 NormalMatrix = glm::mat3( glm::transpose(glm::inverse(ViewMatrix * myModelMatrix)));
Controls->setCntrlsViewMatrix(ViewMatrix);
Controls->setCntrlsProjectionMatrix(ProjectionMatrix);
glUniformMatrix4fv(MatrixID, 1, GL_FALSE, &MVP[0][0]);
glUniformMatrix4fv(ViewMatrixID, 1, GL_FALSE, &ViewMatrix[0][0]);
glUniformMatrix4fv(ModelMatrixID, 1, GL_FALSE, &myModelMatrix[0][0]);
// Notice we're passing a 3 by 3 matrix here.
glUniformMatrix3fv(NormalMatrixID, 1, GL_FALSE, &NormalMatrix[0][0]);
glUniform3f(CameraID, cameraLoc.x, cameraLoc.y, cameraLoc.z);
glUniform3f(LightPosID, lightPosition.x, lightPosition.y, lightPosition.z);
// VBO buffer: vertices
// 1rst attribute buffer : vertices
glEnableVertexAttribArray(0);
glBindBuffer(GL_ARRAY_BUFFER, vertexbuffer);
glVertexAttribPointer(
0, // attribute
3, // size
GL_FLOAT, // type
GL_FALSE, // normalized?
0, // stride
(void*)0 // array buffer offset
);
// 2nd attribute buffer : UVs
glEnableVertexAttribArray(1);
glBindBuffer(GL_ARRAY_BUFFER, uvbuffer);
glVertexAttribPointer(
1, // attribute
2, // size
GL_FLOAT, // type
GL_FALSE, // normalized?
0, // stride
(void*)0 // array buffer offset
);
// 2rd attribute buffer : normals
glEnableVertexAttribArray(2);
glBindBuffer(GL_ARRAY_BUFFER, normalbuffer);
glVertexAttribPointer(
2, // attribute
3, // size
GL_FLOAT, // type
GL_FALSE, // normalized?
0, // stride
(void*)0 // array buffer offset
);
// Index buffer
glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, elementbuffer);
if ( Controls->getRenderingMode() == 0 ) {
glDisable(GL_CULL_FACE);
glEnable(GL_DEPTH_TEST);
glPolygonMode(GL_FRONT_AND_BACK, GL_FILL);
//glDisable(GL_POLYGON_OFFSET_FILL);
}
else if (Controls->getRenderingMode() == 1 ) {
glPolygonMode(GL_FRONT_AND_BACK, GL_LINE);
glDisable(GL_CULL_FACE);
//glDisable(GL_POLYGON_OFFSET_FILL);
glUseProgram(ShaderIDs[1]);
glUniformMatrix4fv(MatrixID, 1, GL_FALSE, &MVP[0][0]);
glUniformMatrix4fv(ViewMatrixID, 1, GL_FALSE, &ViewMatrix[0][0]);
glUniformMatrix4fv(ModelMatrixID, 1, GL_FALSE, &myModelMatrix[0][0]);
// Notice we're passing a 3 by 3 matrix here.
glUniformMatrix3fv(NormalMatrixID, 1, GL_FALSE, &NormalMatrix[0][0]);
glUniform3f(CameraID, cameraLoc.x, cameraLoc.y, cameraLoc.z);
glUniform3f(LightPosID, lightPosition.x, lightPosition.y, lightPosition.z);
}
else if (Controls->getRenderingMode() == 2 ) {
glUseProgram(ShaderIDs[1]);
//
glm::mat4 MyOffsetMatrix = glm::scale(glm::mat4(1.0f), glm::vec3(1.025,1.025,1.025));
MyOffsetMatrix = glm::mat4(1.0f);
glm::mat4 myModelMatrix2 = ModelMatrix * Controls->getTranslationMatrix() *
Controls->getRotationMatrix()*MyOffsetMatrix;
glm::mat3 NormalMatrix2 = glm::mat3( glm::transpose(glm::inverse(ViewMatrix *
myModelMatrix2)));
glm::mat4 MVP2 = ProjectionMatrix * ViewMatrix * myModelMatrix2;
glUniformMatrix4fv(MatrixID, 1, GL_FALSE, &MVP2[0][0]);
glUniformMatrix4fv(ViewMatrixID, 1, GL_FALSE, &ViewMatrix[0][0]);
glUniformMatrix4fv(ModelMatrixID, 1, GL_FALSE, &myModelMatrix2[0][0]);
// Notice we're passing a 3 by 3 matrix here.
glUniformMatrix3fv(NormalMatrixID, 1, GL_FALSE, &NormalMatrix2[0][0]);
glUniform3f(CameraID, cameraLoc.x, cameraLoc.y, cameraLoc.z);
glUniform3f(LightPosID, lightPosition.x, lightPosition.y, lightPosition.z);
// The rest is exactly the same as the first object
// 1rst attribute buffer : vertices
glEnableVertexAttribArray(0);
glBindBuffer(GL_ARRAY_BUFFER, vertexbuffer);
glVertexAttribPointer(0, 3, GL_FLOAT, GL_FALSE, 0, (void*)0);
// 2nd attribute buffer : UVs
glEnableVertexAttribArray(1);
glBindBuffer(GL_ARRAY_BUFFER, uvbuffer);
glVertexAttribPointer(1, 2, GL_FLOAT, GL_FALSE, 0, (void*)0);
// 3rd attribute buffer : normals
glEnableVertexAttribArray(2);
glBindBuffer(GL_ARRAY_BUFFER, normalbuffer);
glVertexAttribPointer(2, 3, GL_FLOAT, GL_FALSE, 0, (void*)0);
// Index buffer
glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, elementbuffer);
glPolygonMode(GL_FRONT_AND_BACK, GL_LINE);
glDisable(GL_CULL_FACE);
glEnable(GL_POLYGON_OFFSET_FILL);
// Draw the triangles !
glDrawElements(GL_TRIANGLES, indices.size(), GL_UNSIGNED_SHORT, (void*)0);
glEnable(GL_POLYGON_OFFSET_FILL);
glDisable(GL_CULL_FACE);
glEnable(GL_DEPTH_TEST);
glPolygonOffset(2.0f, 2.0f);
glPolygonMode(GL_FRONT_AND_BACK, GL_FILL);
//glDisable(GL_POLYGON_OFFSET_FILL);
glUseProgram(ShaderIDs[0]);
}
//glUseProgram(ShaderIDs[1]);
// Draw the triangles !
glDrawElements(
GL_TRIANGLES, // mode
indices.size(), // count
GL_UNSIGNED_SHORT, // type
(void*)0 // element array buffer offset
);
glDisableVertexAttribArray(0);
glDisableVertexAttribArray(1);
glDisableVertexAttribArray(2);
glutSwapBuffers();
}
}

Answer 1

The problem was two-fold: Crazy numbers returned is a result of improper multisampling enabled when I wanted specific colour values (how to have my cake and eat it might need a bit of work but right now I don't care), and secondly because glReadPixels() inverts the Y axis and needed to do Height - Current_Mouse_Position for the Y value.
Perhaps glPoints would be faster means of doing what I'm doing, I'll need to look into it.