Picking with glRenderMode - c++

Question

I try to do OpenGL picking so I followed this post (OpenGL Picking on Selection Mode ) which seems to have a working solution but I can't make mine work. I have a 3D textured polygon in my scene and I would like to know when I click on it.
Here is my code :
The mouse method which launches the picking when right click :
void mousePressEvent(QGraphicsSceneMouseEvent *event)
{
if(event->button()==Qt::RightButton){
pickObjects(event->pos().x(),event->pos().y());
}
}
The pickObjects method:
#define BUFSIZE 512
void pickObjects(int x, int y)
{
GLint viewport[4];
GLint hits;
GLuint selectBuf[BUFSIZE];
glSelectBuffer (BUFSIZE, selectBuf);
glRenderMode (GL_SELECT);
glMatrixMode (GL_PROJECTION);
glPushMatrix ();
glLoadIdentity ();
glGetIntegerv (GL_VIEWPORT, viewport);
gluPickMatrix ((GLdouble) x, (GLdouble) (viewport[3] - y), 5.0, 5.0, viewport);
gluPerspective(fov, this->width() / this->height(), 0.1, 1000);
glMatrixMode(GL_MODELVIEW);
glInitNames();
makeCustomAnnot(GL_SELECT);
int hits=0;
glMatrixMode (GL_PROJECTION);
glPopMatrix ();
glMatrixMode(GL_MODELVIEW);
hits = glRenderMode (GL_RENDER);
if (hits != 0)
{
cout<<"FOUND " << hits << " hit(s)"<<endl; //ALWAYS GIVES 0 HITS
processHits(hits,selectBuf);
}
}
The processHits method :
void processHits(GLint inHits, GLuint buffer[])
{
unsigned int i, j;
GLuint names, *ptr, minZ,*ptrNames, numberOfNames;
ptr = (GLuint *) buffer;
minZ = 0xffffffff;
for (i = 0; i < inHits; i++) {
names = *ptr;
ptr++;
if (*ptr < minZ) {
numberOfNames = names;
minZ = *ptr;
ptrNames = ptr+2;
}
ptr += names+2;
}
cout << "Nearest: ";
ptr = ptrNames;
for (j = 0; j < numberOfNames; j++,ptr++) {
cout<< *ptr ;
}
}
The drawing obejct method :
void makeCustomAnnot(GLenum mode){
glEnable(GL_TEXTURE_2D);
glColor3f(1,1,1);
GLuint j=0;
QImage img("img.jpg");
img=QGLWidget::convertToGLFormat(img);
GLuint texturesAnnot[1];
glBindTexture( GL_TEXTURE_2D, texturesAnnot[0] );
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, img.width(), img.height(), 0, GL_RGBA, GL_UNSIGNED_BYTE, img.bits());
glTexParameteri(GL_TEXTURE_2D,GL_TEXTURE_MAG_FILTER,GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D,GL_TEXTURE_MIN_FILTER,GL_LINEAR);
if(mode == GL_SELECT){
glPushName (j);
}
glBegin(GL_QUADS);
glTexCoord2d(0,0);glVertex3d(x4,y4,z4);
glTexCoord2d(1,0);glVertex3d(x3,y3,z3);
glTexCoord2d(1,1);glVertex3d(x2,y2,z2);
glTexCoord2d(0,1);glVertex3d(x1,y1,z1);
glEnd();
if(mode == GL_SELECT)
glPopName ();
}
glDisable(GL_TEXTURE_2D);
}
The rendering method :
void render()
{
glEnable(GL_DEPTH_TEST);
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
glMatrixMode(GL_PROJECTION);
glLoadIdentity();
glViewport(0,0,this->width() , this->height());
gluPerspective(fov, this->width() / this->height(), 0.1, 1000);
glMatrixMode(GL_MODELVIEW);
glLoadIdentity();
glRotatef(xRot / 5.0f, 1.0f, 0.0f, 0.0f);
glRotatef(yRot / 5.0f, 0.0f, 1.0f, 0.0f);
glRotatef(zRot / 5.0f, 0.0f, 0.0f, 1.0f);
glPushMatrix();
makeCustomAnnot(GL_RENDER);
glPopMatrix();
glDisable(GL_DEPTH_TEST);
}
So hits = glRenderMode (GL_RENDER); always returns 0 hits. What am I doing wrong?

Answer 1

So here is what I did to have it work perfectly.
The processHits method :
void processHits(GLint inHits, GLuint buffer[])
{
unsigned int i, j;
GLuint names, *ptr, minZ,*ptrNames, numberOfNames;
ptr = (GLuint *) buffer;
minZ = 0xffffffff;
for (i = 0; i < inHits; i++) {
names = *ptr;
ptr++;
if (*ptr < minZ) {
numberOfNames = names;
minZ = *ptr;
ptrNames = ptr+2;
}
ptr += names+2;
}
cout << "Nearest: ";
ptr = ptrNames;
for (j = 0; j < numberOfNames; j++,ptr++) {
cout<< *ptr ;
}
}
The drawing object method with creates the objects I need to pick :
void makeCustomAnnot(GLenum mode){
glEnable(GL_TEXTURE_2D);
glColor3f(1,1,1);
GLuint j=0;
QImage img("img.jpg");
img=QGLWidget::convertToGLFormat(img);
GLuint texturesAnnot[1];
glBindTexture( GL_TEXTURE_2D, texturesAnnot[0] );
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, img.width(), img.height(), 0, GL_RGBA, GL_UNSIGNED_BYTE, img.bits());
glTexParameteri(GL_TEXTURE_2D,GL_TEXTURE_MAG_FILTER,GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D,GL_TEXTURE_MIN_FILTER,GL_LINEAR);
if(mode == GL_SELECT)
glLoadName (j);
glBegin(GL_QUADS);
glTexCoord2d(0,0);glVertex3d(x4,y4,z4);
glTexCoord2d(1,0);glVertex3d(x3,y3,z3);
glTexCoord2d(1,1);glVertex3d(x2,y2,z2);
glTexCoord2d(0,1);glVertex3d(x1,y1,z1);
glEnd();
glDisable(GL_TEXTURE_2D);
}
The pickObjects method, identical to the rendering function but with instructions for picking :
#define BUFSIZE 512
unsigned int selectBuf[BUFSIZE];
void pickObjects(int x, int y)
{
GLint viewport[4];
GLint hits;
glSelectBuffer (BUFSIZE, selectBuf);
glRenderMode (GL_SELECT);
glMatrixMode (GL_PROJECTION);
glPushMatrix ();
glLoadIdentity ();
glGetIntegerv (GL_VIEWPORT, viewport);
gluPickMatrix ((GLdouble) x, (GLdouble) (viewport[3] - y), 5.0, 5.0, viewport);
gluPerspective(fov, this->width() / this->height(), 0.1, 1000);
glMatrixMode(GL_MODELVIEW);
glLoadIdentity();
glRotatef(xRot / 5.0f, 1.0f, 0.0f, 0.0f);
glRotatef(yRot / 5.0f, 0.0f, 1.0f, 0.0f);
glRotatef(zRot / 5.0f, 0.0f, 0.0f, 1.0f);
glRotatef((float)angleV, 1.0f, 0.0f, 0.0f);
glInitNames();
glPushName( 10000 );
glPushMatrix();
makeCustomAnnot(GL_SELECT);//draw scene in GL_SELECT mode to create the names
glPopMatrix();
hits = glRenderMode (GL_RENDER);//get the hits in GL_RENDER mode
if (hits != 0)
{
processHits(hits,selectBuf);
}
glPopMatrix();
glDisable(GL_DEPTH_TEST);
}
The rendering method :
void render()
{
glEnable(GL_DEPTH_TEST);
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
glMatrixMode(GL_PROJECTION);
glLoadIdentity();
glViewport(0,0,this->width() , this->height());
gluPerspective(fov, this->width() / this->height(), 0.1, 1000);
glMatrixMode(GL_MODELVIEW);
glLoadIdentity();
glRotatef(xRot / 5.0f, 1.0f, 0.0f, 0.0f);
glRotatef(yRot / 5.0f, 0.0f, 1.0f, 0.0f);
glRotatef(zRot / 5.0f, 0.0f, 0.0f, 1.0f);
glRenderMode (GL_RENDER);
glPushMatrix();
makeCustomAnnot(GL_RENDER);
glPopMatrix();
glDisable(GL_DEPTH_TEST);
}
And finally hits = glRenderMode (GL_RENDER); in pickObjects method and then the processHits method return exactly the picked object.