Undesirable serration in surface drawing in OpenGL using Vertex Buffer Objects - c++

Question

First off, I am giving a screenshot of the problematically rendered images in opengl. The fourth surface image is drawn by Matlab and it is what the image supposed to look like in Opengl.
.
.
.
Matlab rendering of the dataset:
(First 3 images are the problematic serrated drawing from OpenGL in different angles, and the 4th one is the MATLAB drawn image which is correct)
The image is a 1024 x 1024 complex matrix. Each element's imaginal part is the height of the point (in a 1024x1024 heightmap), and the real part is the colour of the point.
In matlab we have created a small gaussian shaped mountain. In OpenGL it is rendered with rags and serration. The "raggedness" is spread through the entire image.
Moreover, according to the viewing angle of the object, there appears to be region beyond a line where not only a more weird version of serration happens and also the rendered graphics make a height jump/change.
What can cause this? why is this "raggedness" happenning and what is that line? we have run out of all ideas now and will appreaciate any help. Related parts of the VBO code is given below. We basically create a float4 object for a vertex. first, second and third float numbers in the structure correspond the the coordinations of the point. 4th float (treated as 4 one-byte numbers) is the RGBA color.
also note that the complex matrix which contains the heightmap and the color information is stored in the GPU, so there are calls to CUDA in the code. when all the data is dumped into a file, matlab successfully draws the map, so the data is definitely correct.
#define BUFFER_OFFSET(i) ((char *)NULL + (i))
void initGL()
{
...
glViewport(0, 0, window_width, window_height);
glEnable(GL_BLEND);
glEnable(GL_COLOR_MATERIAL);
glBlendFunc (GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);
// projection
glMatrixMode(GL_PROJECTION);
glLoadIdentity();
gluPerspective(60.0, (GLfloat)window_width / (GLfloat) window_height, 0.1, 15.0);
...
}
void display()
{
camx += camx_v;
camy += camy_v;
camx_v=0;
camy_v=0;
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
// set view matrix
glMatrixMode(GL_MODELVIEW);
glLoadIdentity();
gluLookAt(0, 0, 1, /* look from camera XYZ */
0, 0, 0, /* look at the origin */
0, 1, 0); /* positive Y up vector */
drawGround();
glTranslatef(camx, camy, translate_z);
glRotatef(rotate_x, 1.0, 0.0, 0.0);
glRotatef(rotate_y, 0.0, 1.0, 0.0);
glBindBuffer(GL_ARRAY_BUFFER, vbo);
glEnableClientState(GL_VERTEX_ARRAY);
glVertexPointer(3, GL_FLOAT, 16, BUFFER_OFFSET(0));
glEnableClientState(GL_COLOR_ARRAY);
glColorPointer(4, GL_UNSIGNED_BYTE, 16, BUFFER_OFFSET(12));
glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, vbo_i);
glDrawElements(GL_TRIANGLES, (mesh_width-1) * (mesh_height-1) * 6, GL_UNSIGNED_INT, (GLvoid*)0);
glDisableClientState(GL_VERTEX_ARRAY);
glDisableClientState(GL_COLOR_ARRAY);
glutSwapBuffers();
}
void createVBO(GLuint* vbo, struct cudaGraphicsResource **vbo_res,
unsigned int vbo_res_flags)
{
glGenBuffers(1, vbo);
glBindBuffer(GL_ARRAY_BUFFER, *vbo);
unsigned int size = mesh_width * mesh_height * 4 * sizeof(float);
glBufferData(GL_ARRAY_BUFFER, size, 0, GL_DYNAMIC_DRAW);
glBindBuffer(GL_ARRAY_BUFFER, 0);
cutilSafeCall(cudaGraphicsGLRegisterBuffer(vbo_res, *vbo, vbo_res_flags));
}
void createIBO(GLuint* vbo, struct cudaGraphicsResource **vbo_res,
unsigned int vbo_res_flags, unsigned int numofindice)
{
glGenBuffers(1, vbo);
glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, *vbo);
unsigned int size = (mesh_width-1) * (mesh_height-1) * numofindice * sizeof(GLuint);
glBufferData(GL_ELEMENT_ARRAY_BUFFER, size, 0, GL_STATIC_DRAW);
glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, 0);
cutilSafeCall(cudaGraphicsGLRegisterBuffer(vbo_res, *vbo, vbo_res_flags));
}
void main()
{
initGL();
createVBO(&vbo, &cuda_vbo_resource, cudaGraphicsMapFlagsWriteDiscard);
createIBO(&vbo_i, &cuda_vbo_resource_i, cudaGraphicsMapFlagsWriteDiscard, 6);
glutMainLoop();
}
//KERNEL TO FILL the INDEX BUFFER in GPU, called once at the initialization of the program.
__global__ void fillIBO(unsigned int* pos_i, unsigned int M)
{
unsigned int x = blockIdx.x*blockDim.x + threadIdx.x;
unsigned int y = blockIdx.y*blockDim.y + threadIdx.y;
unsigned int bi;
if(y<M-1 && x<M-1)
{
bi = ((M-1)*y +x)*6;
//TRI
pos_i[bi++] = x + y*M + 1;
pos_i[bi++] = x + y*M + M + 1;
pos_i[bi++] = x + y*M;
pos_i[bi++] = x + y*M;
pos_i[bi++] = x + y*M + M + 1;
pos_i[bi++] = x + y*M + M;
}
}

Answer 1

replace second triangle by :
pos_i[bi++] = x + y*M + 1;
pos_i[bi++] = x + y*M + M + 1;
pos_i[bi++] = x + y*M + M;
also, I'm pretty sure it should be
bi = (M*y +x)*6;