Opencv Mat to opengl with OSVR - c++

Question

I have a problem with opencv and opengl. I need to show with opengl the images retrieved by a webcam with opencv, and put this in Razer OSVR. But with my current code the framerate is around 1 fps or 2 fps, and I don't know what I am doing wrong. Here is my code, I think the wrong thing is in draw_cube() function.
Main.cpp
// Internal Includes
#include <osvr/ClientKit/ClientKit.h>
#include <osvr/ClientKit/Display.h>
#include "SDL2Helpers.h"
#include "OpenGLCube.h"
#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/opencv.hpp>
// Library/third-party includes
#include <SDL.h>
#include <SDL_opengl.h>
// Standard includes
#include <iostream>
static auto const WIDTH = 1920;
static auto const HEIGHT = 1080;
// Forward declarations of rendering functions defined below.
void render(osvr::clientkit::DisplayConfig &disp);
int main(int argc, char *argv[]) {
namespace SDL = osvr::SDL2;
// Open SDL
SDL::Lib lib;
// Use OpenGL 2.1
SDL_GL_SetAttribute(SDL_GL_CONTEXT_MAJOR_VERSION, 2);
SDL_GL_SetAttribute(SDL_GL_CONTEXT_MINOR_VERSION, 1);
// Create a window
auto window = SDL::createWindow("OSVR", SDL_WINDOWPOS_UNDEFINED,
SDL_WINDOWPOS_UNDEFINED, WIDTH, HEIGHT,
SDL_WINDOW_OPENGL | SDL_WINDOW_SHOWN);
if (!window) {
std::cerr << "Could not create window: " << SDL_GetError() << std::endl;
return -1;
}
// Create an OpenGL context and make it current.
SDL::GLContext glctx(window.get());
// Turn on V-SYNC
SDL_GL_SetSwapInterval(1);
// Start OSVR and get OSVR display config
osvr::clientkit::ClientContext ctx("com.osvr.example.SDLOpenGL");
osvr::clientkit::DisplayConfig display(ctx);
if (!display.valid()) {
std::cerr << "\nCould not get display config (server probably not "
"running or not behaving), exiting."
<< std::endl;
return -1;
}
std::cout << "Waiting for the display to fully start up, including "
"receiving initial pose update..."
<< std::endl;
while (!display.checkStartup()) {
ctx.update();
}
std::cout << "OK, display startup status is good!" << std::endl;
// Event handler
SDL_Event e;
#ifndef __ANDROID__ // Don't want to pop up the on-screen keyboard
SDL::TextInput textinput;
#endif
bool quit = false;
while (!quit) {
// Handle all queued events
while (SDL_PollEvent(&e)) {
switch (e.type) {
case SDL_QUIT:
// Handle some system-wide quit event
quit = true;
break;
case SDL_KEYDOWN:
if (SDL_SCANCODE_ESCAPE == e.key.keysym.scancode) {
// Handle pressing ESC
quit = true;
}
break;
}
if (e.type == SDL_QUIT) {
quit = true;
}
}
// Update OSVR
ctx.update();
// Render
render(display);
// Swap buffers
SDL_GL_SwapWindow(window.get());
}
return 0;
}
void render(osvr::clientkit::DisplayConfig &disp) {
/// For each viewer, eye combination...
disp.forEachEye([](osvr::clientkit::Eye eye) {
/// For each display surface seen by the given eye of the given
/// viewer...
eye.forEachSurface([](osvr::clientkit::Surface surface) {
auto viewport = surface.getRelativeViewport();
glViewport(static_cast<GLint>(viewport.left),
static_cast<GLint>(viewport.bottom),
static_cast<GLsizei>(viewport.width),
static_cast<GLsizei>(viewport.height));
glLoadIdentity();
cv::VideoCapture cap(0); // open the default camera
cv::Mat img;
cap >> img; // get a new frame from camera
cv::flip(img,img,0);
//resize(img, img, Size(160, 140), 0, 0, INTER_CUBIC);
draw_cube(img);
});
});
}
OpenGLCube.h
#ifndef INCLUDED_OpenGLVIDEO_h
#define INCLUDED_OpenGLVIDEO_h_
#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/opencv.hpp>
using namespace cv;
GLuint texture;
void draw_cube(cv::Mat img)
{
glGenTextures(1, &texture);
glBindTexture(GL_TEXTURE_2D, texture);
glTexParameteri(GL_TEXTURE_2D,GL_TEXTURE_MAG_FILTER,GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D,GL_TEXTURE_MIN_FILTER,GL_LINEAR);
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB,img.size().width,img.size().height, 0, GL_BGR,GL_UNSIGNED_BYTE,img.data);
glEnable(GL_TEXTURE_2D);
glBegin(GL_QUADS);
glTexCoord2d(0.0, 1.0);
glVertex2d(-1,1);
glTexCoord2d(0.0, 0.0);
glVertex2d(-1,-1);
glTexCoord2d(1.0, 0.0);
glVertex2d(1,-1);
glTexCoord2d(1.0, 1.0);
glVertex2d(1,1);
glEnd();
glDisable(GL_TEXTURE_2D);
glDeleteTextures(1, &texture);
}
#endif
Thanks.

Answer 1

glGenTextures should be called once, before your main loop since you don't need to create a brand new texture every frame. You can simply overwrite the previous data using the same texture object using glTexImage2D just as you are now. Similarly, as a good convention, glDeleteTextures should be called once before exiting the program.
For example, the following should be done outside of your main loop:
glGenTextures(1, &texture);
glBindTexture(GL_TEXTURE_2D, texture);
glTexParameteri(GL_TEXTURE_2D,GL_TEXTURE_MAG_FILTER,GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D,GL_TEXTURE_MIN_FILTER,GL_LINEAR);
You will, of course, need to declare texture outside of your main loop and draw function and pass that in.

Answer 2

This line:
cv::VideoCapture cap(0); // open the default camera
It mustn't be in a mainloop.
Thanks for your replies.