QOpenGLWidget draw 2D texture in another thread - c++

I tried to draw QOpenGLWidget in another thread and here is my code:
// widget.h
class QGLCanvas;
class FrameRenderer: public QThread
{
Q_OBJECT
public:
FrameRenderer(QGLCanvas *parent);
void run() Q_DECL_OVERRIDE;
void stop();
void initialize();
void updateFrame(std::shared_ptr<cv::Mat> frame);
QGLCanvas *parent_;
std::atomic<bool> thread_run_;
bool initialize_;
GLuint texture_;
std::shared_ptr<cv::Mat> frame_;
QMutex mutex_;
};
class QGLCanvas : public QOpenGLWidget, protected QOpenGLFunctions
{
Q_OBJECT
public:
QGLCanvas(QWidget* parent = NULL);
virtual void initializeGL();
virtual void paintGL();
virtual void resizeGL(int width, int height);
void ShowImage(std::shared_ptr<cv::Mat> frame);
// for opengl rendering thread
FrameRenderer* render_thread_;
QTimer* render_timer_;
private slots:
void StartRendering();
};
//widget.cpp
#include "widget.h"
FrameRenderer::FrameRenderer(QGLCanvas* parent):
QThread(), parent_(parent)
{
thread_run_ = true;
initialize_ = false;
frame_ = std::make_shared<cv::Mat>();
cv::Mat img = cv::imread("./res/no_image.jpg");
*frame_ = img;
}
void FrameRenderer::stop()
{
thread_run_ = false;
}
void FrameRenderer::initialize()
{
if(initialize_)
return;
glGenTextures(1, &texture_);
glBindTexture(GL_TEXTURE_2D, texture_);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, frame_->cols,
frame_->rows, 0, GL_RGB, GL_UNSIGNED_BYTE,
NULL);
glBindTexture(GL_TEXTURE_2D, 0);
initialize_ = true;
}
void FrameRenderer::run()
{
while(true)
{
QMutexLocker lock(&mutex_);
if(!thread_run_)
break;
parent_->makeCurrent();
if(!initialize_)
initialize();
int width = parent_->width(), height = parent_->height();
glViewport(0, 0, width, height);
glClearColor(0, 0, 1, 1);
glDisable(GL_DEPTH_TEST);
glEnable(GL_TEXTURE_2D);
glBindTexture(GL_TEXTURE_2D, texture_);
glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, frame_->cols,
frame_->rows, GL_RGB, GL_UNSIGNED_BYTE,
frame_->data);
glBegin(GL_QUADS);
glTexCoord2f(0.0f, 1.0f); glVertex3f(0, 0, 0.0f);
glTexCoord2f(1.0f, 1.0f); glVertex3f(width, 0.0f, 0.0f);
glTexCoord2f(1.0f, 0.0f); glVertex3f(width, height, 0.0f);
glTexCoord2f(0.0f, 0.0f); glVertex3f(0, height, 0.0f);
glEnd();
glDisable(GL_TEXTURE_2D);
parent_->doneCurrent();
QMetaObject::invokeMethod(parent_, "update");
msleep(40);
}
}
void FrameRenderer::updateFrame(std::shared_ptr<cv::Mat> frame)
{
QMutexLocker lock(&mutex_);
frame_ = frame;
}
QGLCanvas::QGLCanvas(QWidget* parent)
: QOpenGLWidget(parent)
{
render_timer_ = new QTimer(this);
QObject::connect(render_timer_, SIGNAL(timeout()), this, SLOT(StartRendering()));
render_timer_->start(1000);
}
void QGLCanvas::initializeGL()
{
}
void QGLCanvas::resizeGL(int width, int height)
{
//stop QGLWidget standard behavior
}
void QGLCanvas::paintGL()
{
}
void QGLCanvas::StartRendering()
{
doneCurrent();
render_thread_ = new FrameRenderer(this);
context()->moveToThread(render_thread_);
render_thread_->start();
}
void QGLCanvas::ShowImage(std::shared_ptr<cv::Mat> frame)
{
if(render_thread_)
render_thread_->updateFrame(frame);
}
// main.cpp
#include "widget.h"
#include <QApplication>
int main(int argc, char *argv[])
{
QApplication app(argc, argv);
// Check that the threaded OpenGL is supported.
if (!QOpenGLContext::supportsThreadedOpenGL())
{
std::cerr << "Threaded OpenGL is not supported" << std::endl;
return EXIT_FAILURE;
}
// Set the wanted surface format.
QSurfaceFormat format;
format.setDepthBufferSize(16);
format.setVersion(4, 4);
format.setProfile(QSurfaceFormat::CoreProfile);
QSurfaceFormat::setDefaultFormat(format);
// Calculate the position of the widget. The widget should be
// located so that the center is also at the center of desktop.
const QDesktopWidget* desktop = QApplication::desktop();
const QSize size(720, 576);
const QPoint position(
desktop->width() / 2 - size.width() / 2,
desktop->height() / 2 - size.height() / 2);
// Create the OpenGL widget
std::shared_ptr<QGLCanvas> widget = std::make_shared<QGLCanvas>();
widget->resize(size);
widget->move(position);
widget->show();
return app.exec();
}
The Problem is I got this error:
Cannot make QOpenglContext current in another thread.
I had a similar code structure using QGLWidget. But when changed to QOpenGLWidget, it cannot work.

Related

Rendering video in QT/OpenGL using updatePaintNode: new frame only appears when I resize screen

I'm writing an OpenGL video renderer in Qt and it's almost working. It'll only render the video if I resize the window. So every time I resize the window, a new frame is drawn on the screen. I think I'm forgetting to call some function after updating the frame.
Here's the first frame, when there's no video data:
After I resize the screen, a frame is drawn:
#include <QtQuick/QQuickItem>
#include <qguiapplication.h>
#include <qsgmaterial.h>
#include <qsgnode.h>
#include <qquickitem.h>
#include <qquickview.h>
#include <qsgsimplerectnode.h>
#include <qsgsimplematerial.h>
#include <QOpenGLContext>
#include <QOpenGLFunctions>
#include "OpenGlMaterialQQuickItem.h"
#include <iostream>
#define GET_STR(x) #x
#define VERTEX_ATTRIBUTE 3
struct State
{
public:
int frameWidth = 1920;
int frameHeight = 1080;
unsigned char *datas[3] = {0};
private:
bool firstRender = true;
public:
void updateData(unsigned char**data, int frameWidth, int frameHeight)
{
this->frameWidth = frameWidth;
this->frameHeight = frameHeight;
if (firstRender) {
datas[0] = new unsigned char[frameWidth*frameHeight]; //Y
datas[1] = new unsigned char[frameWidth*frameHeight/4];//U
datas[2] = new unsigned char[frameWidth*frameHeight/4];//V
firstRender = false;
}
memcpy(datas[0], data[0], frameWidth*frameHeight);
memcpy(datas[1], data[1], frameWidth*frameHeight/4);
memcpy(datas[2], data[2], frameWidth*frameHeight/4);
};
class Shader : public QSGSimpleMaterialShader<State>
{
QSG_DECLARE_SIMPLE_SHADER(Shader, State);
private:
QOpenGLFunctions *glFuncs = nullptr;
GLuint unis[3] = {0};
GLuint texs[3] = {0};
QSize m_viewportSize;
bool firstRender = true;
public:
const char *vertexShader() const override {
return GET_STR(
uniform highp mat4 qt_Matrix;
attribute vec4 vertexIn;
attribute vec2 textureIn;
varying vec2 textureOut;
void main(void)
{
gl_Position = qt_Matrix * vertexIn;
textureOut = textureIn;
}
);
}
const char *fragmentShader() const override {
return GET_STR(
varying vec2 textureOut;
uniform sampler2D tex_y;
uniform sampler2D tex_u;
uniform sampler2D tex_v;
uniform lowp float qt_Opacity;
void main(void)
{
vec3 yuv;
vec3 rgb;
yuv.x = texture2D(tex_y, textureOut).r;
yuv.y = texture2D(tex_u, textureOut).r - 0.5;
yuv.z = texture2D(tex_v, textureOut).r - 0.5;
rgb = mat3(1.0, 1.0, 1.0,
0.0, -0.39465, 2.03211,
1.13983, -0.58060, 0.0) * yuv;
gl_FragColor = vec4(rgb, 1.0) * qt_Opacity;
}
);
}
QList<QByteArray> attributes() const override
{
return {QByteArrayLiteral("vertexIn"), QByteArrayLiteral("textureIn")};
}
void initialize()
{
if (!program()->isLinked()) {
return;
}
QSGSimpleMaterialShader<State>::initialize();
glFuncs = QOpenGLContext::currentContext()->functions();
program()->bind();
glFuncs->glGenTextures(3, texs);
}
void updateState(const State *state, const State *) override
{
//Y
glFuncs->glBindTexture(GL_TEXTURE_2D, texs[0]);
glFuncs->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glFuncs->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glFuncs->glTexImage2D(GL_TEXTURE_2D, 0, GL_RED, state->frameWidth, state->frameHeight, 0, GL_RED, GL_UNSIGNED_BYTE, 0);
//U
glFuncs->glBindTexture(GL_TEXTURE_2D, texs[1]);
glFuncs->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glFuncs->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glFuncs->glTexImage2D(GL_TEXTURE_2D, 0, GL_RED, state->frameWidth/2, state->frameHeight / 2, 0, GL_RED, GL_UNSIGNED_BYTE, 0);
//V
glFuncs->glBindTexture(GL_TEXTURE_2D, texs[2]);
glFuncs->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glFuncs->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glFuncs->glTexImage2D(GL_TEXTURE_2D, 0, GL_RED, state->frameWidth / 2, state->frameHeight / 2, 0, GL_RED, GL_UNSIGNED_BYTE, 0);
glFuncs->glActiveTexture(GL_TEXTURE0);
glFuncs->glBindTexture(GL_TEXTURE_2D, texs[0]);
glFuncs->glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, state->frameWidth, state->frameHeight, GL_RED, GL_UNSIGNED_BYTE, state->datas[0]);
glFuncs->glUniform1i(unis[0], 0);
glFuncs->glActiveTexture(GL_TEXTURE0+1);
glFuncs->glBindTexture(GL_TEXTURE_2D, texs[1]);
glFuncs->glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, state->frameWidth/2, state->frameHeight / 2, GL_RED, GL_UNSIGNED_BYTE, state->datas[1]);
glFuncs->glUniform1i(unis[1],1);
glFuncs->glActiveTexture(GL_TEXTURE0+2);
glFuncs->glBindTexture(GL_TEXTURE_2D, texs[2]);
glFuncs->glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, state->frameWidth / 2, state->frameHeight / 2, GL_RED, GL_UNSIGNED_BYTE, state->datas[2]);
glFuncs->glUniform1i(unis[2], 2);
glFuncs->glDrawArrays(GL_TRIANGLE_STRIP,0,4);
}
void resolveUniforms() override
{
unis[0] = program()->uniformLocation("tex_y");
unis[1] = program()->uniformLocation("tex_u");
unis[2] = program()->uniformLocation("tex_v");
}
};
class Node: public QSGGeometryNode, public FrameUpdater
{
public:
State state;
//QQuickItem* item;
Node()
{
material = Shader::createMaterial();
setMaterial(material);
setFlag(OwnsMaterial, true);
QSGGeometry *g = new QSGGeometry(QSGGeometry::defaultAttributes_TexturedPoint2D(), 4);
QSGGeometry::updateTexturedRectGeometry(g, QRect(), QRect());
setGeometry(g);
setFlag(QSGNode::OwnsGeometry, true);
stream = new MediaStream("rtsp://admin:19929394#192.168.1.178:10554/tcp/av0_0");
stream->setFrameUpdater((FrameUpdater *) this);
boost::thread mediaThread(&MediaStream::run, stream);
}
void updateData(unsigned char**data, int frameWidth, int frameHeight)
{
material->state()->updateData(data, frameWidth, frameHeight);
markDirty(QSGNode::DirtyMaterial);
//item->update();
}
private:
QSGSimpleMaterial<State> *material;
MediaStream* stream;
};
QSGNode * OpenGlMaterialQQuickItem::updatePaintNode(QSGNode *node, UpdatePaintNodeData *) //override
{
std::cout << "updatePaintNode called " << std::endl;
Node *n = static_cast<Node *>(node);
if (!node)
n = new Node();
//n->item = this;
QSGGeometry::updateTexturedRectGeometry(n->geometry(), boundingRect(), QRectF(0, 0, 1, 1));
n->markDirty(QSGNode::DirtyGeometry | QSGNode::DirtyMaterial);
return n;
}
To understand how the code updates a video frame: boost::thread mediaThread(&MediaStream::run, stream); creates a thread that will insert video data through void Node::updateData(unsigned char**data, int frameWidth, int frameHeight). This data is written inside the State struct.
The only thing that is missing is a way to tell Qt that it needs to draw another frame. What is missing?
PS: this is the QQuickItem:
class OpenGlMaterialQQuickItem: public ReactItem
{
//Q_OBJECT
Q_PROPERTY(QString uri WRITE setUri)// NOTIFY uriChanged)
public:
std::string uri;
QSGNode *updatePaintNode(QSGNode *node, UpdatePaintNodeData *) override;
OpenGlMaterialQQuickItem()
{
setFlag(ItemHasContents, true);
}
void setUri(const QString &a) {
uri = a.toStdString();
}
};
According to Qt documentation this is done by calling QQuickItem::update().
Note that you must set the flag QQuickItem::ItemHasContents on the item first.
Update from comments
When you uncomment item->update(); you get QObject::startTimer: Timers cannot be started from another thread, because of 2 things:
The QQuickItem::update() forwards the request to the window and then to the platform adaptation. In some (all ?) cases, the implementation uses a QTimer to schedule the repaint.
You call item->update(); not from the GUI thread. Likely you call it directly from the boost thread you use to read the video.
So in the end, item->update(); fails to start the timer, and nothing happens.
A solution can be to make sure you call QQuickItem::update() from the GUI thread.
Maybe this should work:
QMetaObject::invokeMethod(item, &QQuickItem::update, Qt::QueuedConnection);
This should call QQuickItem::update() from item->thread().

OpenGL strange behaviour in QT when inheriting from QQuickItem versus QQuickPaintedItem (rendering video)

This question is a rewrite of Red video on top of normal video in Qt/OpenGL using QQuickItem but with the code broken to a minimum verifiable example. However, you should see the photos of the old question because they show what happens when real video is rendered into the screen
I have a class class called OpenGlVideoQtQuick2 which I'm testing two possible inheritations: from QQuickItem versus with QQuickPaintedItem. I get the expected behavior (a giant red screen) when OpenGlVideoQtQuick2 inherits from QQuickItem, but not when it inherits from QQuickPaintedItem, which is when I get a black screen with size 640x480, which is the size of the OpenGlVideoQtQuick2 item in main.qml.
Here's what happens when class OpenGlVideoQtQuick2 : public QQuickPaintedItem
Here's what happens when class OpenGlVideoQtQuick2 : public QQuickItem
Here's the code:
OpenGlVideoQtQuick2.h:
#ifndef OpenGlVideoQtQuick2_H
#define OpenGlVideoQtQuick2_H
#include <QtQuick/QQuickItem>
#include <QtGui/QOpenGLShaderProgram>
#include <QtGui/QOpenGLFunctions>
#include <QtQuick/qquickwindow.h>
#include <QtGui/QOpenGLShaderProgram>
#include <QtGui/QOpenGLContext>
#include <QString>
#include <iostream>
#include <QTimer>
#include <QMatrix4x4>
#include <QQmlListProperty>
#include <QQuickPaintedItem>
class OpenGlVideoQtQuick2Renderer2 : public QObject, protected QOpenGLFunctions
{
Q_OBJECT
public:
OpenGlVideoQtQuick2Renderer2() {
}
~OpenGlVideoQtQuick2Renderer2();
void setViewportSize(const QSize &size) { m_viewportSize = size; }
void setWindow(QQuickWindow *window) { m_window = window; }
QMatrix4x4 qQuickVideoMatrix;
public slots:
void render();
private:
QSize m_viewportSize;
QOpenGLShaderProgram* program;
QQuickWindow *m_window;
GLuint unis[3] = {0};
GLuint texs[3] = { 0 };
unsigned char *datas[3] = { 0 };
bool firstRender = true;
int width = 0;
int height = 0;
int x = 0;
int y = 0;
};
//class OpenGlVideoQtQuick2 : public QQuickItem
class OpenGlVideoQtQuick2 : public QQuickPaintedItem
{
Q_OBJECT
protected:
void paint(QPainter* painter){std::cout << "PAINT BEING USED" << std::endl;};
public:
OpenGlVideoQtQuick2();
QMatrix4x4 getModelMatrix();
signals:
void tChanged();
public slots:
void sync();
void cleanup();
void update();//Updates the window
private slots:
void handleWindowChanged(QQuickWindow *win);
private:
OpenGlVideoQtQuick2Renderer2 *openGlVideoQtQuick2Renderer2;
};
#endif // OpenGlVideoQtQuick2_H
OpenGlVideoQtQuick.cpp:
#include "OpenGlVideoQtQuick2.h"
#define GET_STR(x) #x
#define A_VER 3
#define T_VER 4
//Simple shader. Outpus the same location as input, I guess
const char *vString4 = GET_STR(
attribute vec4 vertexIn;
attribute vec2 textureIn;
varying vec2 textureOut;
uniform mat4 u_transform;
void main(void)
{
gl_Position = u_transform * vertexIn;
textureOut = textureIn;
}
);
const char *tString4 = GET_STR(
varying vec2 textureOut;
void main(void)
{
gl_FragColor = vec4(1.0,0,0, 1.0);
}
);
void OpenGlVideoQtQuick2::update()
{
if (window())
window()->update();
}
OpenGlVideoQtQuick2::OpenGlVideoQtQuick2()
: openGlVideoQtQuick2Renderer2(nullptr)
{
connect(this, &QQuickItem::windowChanged, this, &OpenGlVideoQtQuick2::handleWindowChanged);
}
void OpenGlVideoQtQuick2::handleWindowChanged(QQuickWindow *win)
{
if (win) {
connect(win, &QQuickWindow::beforeSynchronizing, this, &OpenGlVideoQtQuick2::sync, Qt::DirectConnection);
win->setClearBeforeRendering(false);
}
}
void OpenGlVideoQtQuick2::cleanup()
{
if (openGlVideoQtQuick2Renderer2) {
delete openGlVideoQtQuick2Renderer2;
openGlVideoQtQuick2Renderer2 = nullptr;
}
}
OpenGlVideoQtQuick2Renderer2::~OpenGlVideoQtQuick2Renderer2()
{
delete program;
}
void OpenGlVideoQtQuick2::sync()
{
//std::cout << "sync called" << std::endl;
if (!openGlVideoQtQuick2Renderer2) {
openGlVideoQtQuick2Renderer2 = new OpenGlVideoQtQuick2Renderer2();
connect(window(), &QQuickWindow::beforeRendering, openGlVideoQtQuick2Renderer2, &OpenGlVideoQtQuick2Renderer2::render, Qt::DirectConnection);
connect(window(), &QQuickWindow::afterRendering, this, &OpenGlVideoQtQuick2::update, Qt::DirectConnection);
}
}
static const GLfloat ver[] = {
-1.0f,-1.0f,
1.0f,-1.0f,
-1.0f, 1.0f,
1.0f, 1.0f
};
static const GLfloat tex[] = {
0.0f, 1.0f,
1.0f, 1.0f,
0.0f, 0.0f,
1.0f, 0.0f
};
//TODO: FIX THIS https://stackoverflow.com/a/54773889/6655884
void OpenGlVideoQtQuick2Renderer2::render()
{
int frameWidth = 1280;
int frameHeight = 720;
if (this->firstRender) {
std::cout << "Creating QOpenGLShaderProgram " << std::endl;
program = new QOpenGLShaderProgram();
initializeOpenGLFunctions();
//this->m_F = QOpenGLContext::currentContext()->functions();
std::cout << "frameWidth: " << frameWidth << + " frameHeight: " << frameHeight << std::endl;
datas[0] = new unsigned char[frameWidth*frameHeight]; //Y
datas[1] = new unsigned char[frameWidth*frameHeight/4]; //U
datas[2] = new unsigned char[frameWidth*frameHeight/4]; //V
std::cout << "Fragment Shader compilation: " << program->addShaderFromSourceCode(QOpenGLShader::Fragment, tString4) << std::endl;
std::cout << "Vertex Shader compilation: " << program->addShaderFromSourceCode(QOpenGLShader::Vertex, vString4) << std::endl;
program->bindAttributeLocation("vertexIn",A_VER);
program->bindAttributeLocation("textureIn",T_VER);
std::cout << "program->link() = " << program->link() << std::endl;
glGenTextures(3, texs);//TODO: ERASE THIS WITH glDeleteTextures
this->firstRender = false;
}
program->bind();
QMatrix4x4 transform;
transform.setToIdentity();
program->setUniformValue("u_transform", this->qQuickVideoMatrix);
glVertexAttribPointer(A_VER, 2, GL_FLOAT, 0, 0, ver);
glEnableVertexAttribArray(A_VER);
glVertexAttribPointer(T_VER, 2, GL_FLOAT, 0, 0, tex);
glEnableVertexAttribArray(T_VER);
unis[0] = program->uniformLocation("tex_y");
unis[1] = program->uniformLocation("tex_u");
unis[2] = program->uniformLocation("tex_v");
//Y
glBindTexture(GL_TEXTURE_2D, texs[0]);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexImage2D(GL_TEXTURE_2D, 0, GL_RED, frameWidth, frameHeight, 0, GL_RED, GL_UNSIGNED_BYTE, 0);
//U
glBindTexture(GL_TEXTURE_2D, texs[1]);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexImage2D(GL_TEXTURE_2D, 0, GL_RED, frameWidth/2, frameHeight / 2, 0, GL_RED, GL_UNSIGNED_BYTE, 0);
//V
glBindTexture(GL_TEXTURE_2D, texs[2]);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexImage2D(GL_TEXTURE_2D, 0, GL_RED, frameWidth / 2, frameHeight / 2, 0, GL_RED, GL_UNSIGNED_BYTE, 0);
glActiveTexture(GL_TEXTURE0);
glBindTexture(GL_TEXTURE_2D, texs[0]);
glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, frameWidth, frameHeight, GL_RED, GL_UNSIGNED_BYTE, datas[0]);
glUniform1i(unis[0], 0);
glActiveTexture(GL_TEXTURE0+1);
glBindTexture(GL_TEXTURE_2D, texs[1]);
glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, frameWidth/2, frameHeight / 2, GL_RED, GL_UNSIGNED_BYTE, datas[1]);
glUniform1i(unis[1],1);
glActiveTexture(GL_TEXTURE0+2);
glBindTexture(GL_TEXTURE_2D, texs[2]);
glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, frameWidth / 2, frameHeight / 2, GL_RED, GL_UNSIGNED_BYTE, datas[2]);
glUniform1i(unis[2], 2);
glDrawArrays(GL_TRIANGLE_STRIP,0,4);
program->disableAttributeArray(A_VER);
program->disableAttributeArray(T_VER);
program->release();
}
main.qml:
import QtQuick 2.0
import OpenGlVideoQtQuick2 1.0
Grid {
columns: 2
spacing: 2
width: 1280
height: 720
OpenGlVideoQtQuick2 {
width: 640
height: 360
}
}
So, I need to make my class derive from QQuickPaintedItem, not QQuickItem, and I need that black screen to not appear on top of my red screen, which is where the actual video will be loaded.
The entire project can be found here: https://github.com/lucaszanella/QQuickPaintedItemBug/tree/c9c2b23d891689a63fbaf2f014142be1f3c5ff0d, where you can compile and test. I recommend to compiling using locally installed cmake and qt folders as explained in the Readme.md file in github
I would not expect the result of using QQuickItem and QQuickPaintedItem to be the same.
When you use QQuickPaintedItem, you are supposed to render the item using the paint() function. Since you call your render() function on beforeRendering(), then just after you do your own rendering, the QQuickPaintedItem will render on top of it what you should have painted in the paint() function.
You can prevent the QQuickPaintedItem from rendering the black rectangle by reimplementing updatePaintNode() as an empty function.

QT5 with OpenGL and Kinect v2: wrong image presentation

I would like to show a color image from the kinect v2 sensor over openGl in a Qt widget. The problem is, that the presentation of this image is incorrect.
Here is the code where I get the color frame:
class:
cv::Mat ColorMap;
std::vector<BYTE> colorBuffer;
int color_width;
int color_height;
code:
if(colorBuffer.empty())
colorBuffer.resize(color_height * color_width * 4 * sizeof(unsigned char));
hr = ColorFrame->CopyConvertedFrameDataToArray((UINT)colorBuffer.size(),
&colorBuffer[0], ColorImageFormat::ColorImageFormat_Bgra );
ColorMap = cv::Mat( color_height, color_width, CV_8UC4, &colorBuffer[0]);
That means, I get the color information as BGR Format with an alpha channel and copy it to a matrix with 4 channels, (BRGA) and each channel has 8Bit from 0 to 255. Correct?
In the next step, I resize it as the same size as the widget:
Kinect->CreateFrame();
cv::Mat GUIColorImage;
Kinect->ColorMap.copyTo(GUIColorImage);
cv::resize(GUIColorImage,GUIColorImage,
cv::Size(ui->Left_Widget_Color->width(),ui->Left_Widget_Color->height()));
Than I've tried two convert methods:
1. convert to BGR
2. convert to 8UC3 (8-Bit, unsigned char, 3 channels (the same as BGR?))
1: GUIColorImage.convertTo(GUIColorImage,CV_BGRA2BGR);
2: GUIColorImage.convertTo(GUIColorImage,CV_8UC3);
But no solutions works.
After the conversion, I try to display it over openGl with:
LeftWidgetColor.UpdateImage(GUIColorImage);
LeftWidgetColor is a QOpenGLWidget:
header:
class RenderWidget : public QOpenGLWidget
{
Q_OBJECT
public:
RenderWidget(QWidget *parent);
~RenderWidget();
void UpdateImage(cv::Mat newimage);
void initializeGL();
void paintGL();
private:
int width;
int height;
GLuint texture;
cv::Mat image;
signals:
void info(QString msg);
void error(QString msg);
void DepthValueAt(cv::Point2i DepthPosition);
protected:
void mousePressEvent(QMouseEvent *event);
void mouseReleaseEvent(QMouseEvent *event);
void mouseMoveEvent(QMouseEvent *event);
};
code:
RenderWidget::RenderWidget(QWidget *parent) : QOpenGLWidget(parent)
{
width = this->size().width();
height = this->size().height();
texture = 0;
initializeGL();
}
RenderWidget::~RenderWidget()
{
glDeleteTextures(1, &texture);
}
void RenderWidget::initializeGL()
{
//Background Color is black
glClearColor(0,0,0,1);
//Storage of Pixelmode for two-dimensional textures
glPixelStorei (GL_UNPACK_ALIGNMENT, 1);
//Create texture
glGenTextures(1, &texture);
glViewport(0, 0, width, height);
}
void RenderWidget::paintGL()
{
// Clear the screen and depth buffer (with black)
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
// Select the model view matrix and reset it
glMatrixMode(GL_MODELVIEW);
glLoadIdentity();
glTranslatef(0.0f, 0.0f, 0.0f);
// Abort drawing if OpenCV was unable to open the camera
if (image.empty())
{
return;
}
glEnable(GL_TEXTURE_RECTANGLE_ARB);
// Typical texture generation using data from the bitmap
glBindTexture(GL_TEXTURE_RECTANGLE_ARB, texture);
// Transfer image data to the GPU
glTexImage2D(GL_TEXTURE_RECTANGLE_ARB, 0,
3, image.cols, image.rows, 0,
GL_BGR, GL_UNSIGNED_BYTE, image.data);
if (glGetError() != GL_NO_ERROR)
{
}
// Draw a 2D face with texture
glBegin(GL_QUADS);
glTexCoord2f(0, 0); glVertex2f(1, 1);
glTexCoord2f(image.cols, 0); glVertex2f(-1, 1);
glTexCoord2f(image.cols, image.rows); glVertex2f(-1, -1);
glTexCoord2f(0, image.rows); glVertex2f(1, -1);
glEnd();
glDisable(GL_TEXTURE_RECTANGLE_ARB);
}
void RenderWidget::UpdateImage(cv::Mat newimage)
{
newimage.copyTo(image);
update();
}
I guess the problem is in:
glTexImage2D(GL_TEXTURE_RECTANGLE_ARB, 0,
3, image.cols, image.rows, 0,
GL_BGR, GL_UNSIGNED_BYTE, image.data);
But I can't find it. I've declared 3-channels, BGR, 8-Bit = 1 Byte as unsigned. Does someone know where the mistake is?
If i show it over imshow (openCv class), it works fine.
My Mistake was, that I used cv::Mat::convertTo(): but here
they wrote, that convertTo() should not used to convert the format (Only to change the data type). cv::cvtColor is the right function and it works well for me. So I've changed the conversion from:
GUIColorImage.convertTo(GUIColorImage,CV_BGRA2BGR);
to
cv::cvtColor(GUIColorImage,GUIColorImage,CV_BGRA2BGR);

Initialize array of objects

If I were to write a working code of what I want to accomplish, it would be this:
Sprite s1(img_path1);
Sprite s2(img_path2);
Sprite s[] = { s1, s2 }
I want to create the objects inside the {} of array declaration instead of first putting the objects into variables and then in the array.
I checked out this post before asking this question and I will clarify a few things first:
I am compiling with -std=c++11
I have not defined a copy constructor of my own
I'm trying to create an array like this:
Sprite s[] =
{
Sprite(img_path1),
Sprite(img_path2)
};
After executing that statement, the contents of s are the same as if I created the array of objects using the default constructor:
Sprite s[2];
The problem seems not to be with the default copy constructor because the following executes perfectly fine:
sp = Sprite(img_path);
Sprite sp1(sp);
std::cout << sp1.getAspectRatio();
Relevant code:
/*
* Sprite.h
*
*/
#ifndef SPRITE_H
#define SPRITE_H
#include <GL/freeglut.h>
#include <FreeImage.h>
#include <iostream>
#include <stdio.h>
class Sprite
{
GLuint texture;
float aspectRatio;
public:
Sprite();
Sprite(std::string path);
Sprite(GLuint texture, float aspectRatio);
float getAspectRatio();
void draw(float x, float y, float alpha, float size);
virtual ~Sprite();
protected:
private:
};
#endif // SPRITE_H
/*
* Sprite.cpp
*
*/
#include "Sprite.h"
Sprite::Sprite()
{
//ctor
}
Sprite::Sprite(std::string file) {
GLuint texture = 0;
int sWidth, sHeight;
if (texture == 0)
{
FIBITMAP* bitmap = FreeImage_Load(
FreeImage_GetFileType(file.c_str(), 0),
file.c_str());
if (bitmap == NULL) {
printf("Could no load image file %s\n", file.c_str());
}
glGenTextures(1, &texture);
printf("%d\n", texture);
glBindTexture(GL_TEXTURE_2D, texture);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
FIBITMAP *pImage = FreeImage_ConvertTo32Bits(bitmap);
int nWidth = sWidth = FreeImage_GetWidth(pImage);
int nHeight = sHeight = FreeImage_GetHeight(pImage);
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, nWidth, nHeight,
0, GL_BGRA_EXT, GL_UNSIGNED_BYTE, (void*)FreeImage_GetBits(pImage));
FreeImage_Unload(pImage);
FreeImage_Unload(bitmap);
}
this->texture = texture;
this->aspectRatio = (float)sWidth / sHeight;
if(texture == 0) {
std::cout << file << std::endl;
}
}
Sprite::Sprite(GLuint t, float as) : texture(t), aspectRatio(as) {}
void Sprite::draw(float x, float y, float alpha, float size) {
float h = size;
float w = size * aspectRatio;
glPushMatrix();
glEnable(GL_TEXTURE_2D);
glBindTexture(GL_TEXTURE_2D, texture);
glTexEnvf(GL_TEXTURE_2D, GL_TEXTURE_ENV_MODE, GL_MODULATE);
glDepthMask(GL_FALSE);
glEnable(GL_BLEND);
glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);
glColor4f(1.0, 1.0, 1.0, alpha);
glBegin(GL_QUADS);
glTexCoord2f(0.0f, 0.0f); glVertex2f(x, y);
glTexCoord2f(0.0f, 1.0f); glVertex2f(x, y + h);
glTexCoord2f(1.0f, 1.0f); glVertex2f(x + w, y + h);
glTexCoord2f(1.0f, 0.0f); glVertex2f(x + w, y);
glEnd();
glDisable(GL_TEXTURE_2D);
glDisable(GL_BLEND);
glDepthMask(GL_TRUE);
glPopMatrix();
}
float Sprite::getAspectRatio() { return aspectRatio; }
Sprite::~Sprite()
{
//dtor
}
/*
* main.cpp (extract)
*
*/
#include "Globals.h"
#include <GL/freeglut.h>
#include "Sprite.h"
void render();
std::string img_path = "/home/saurabh/Dropbox/Code/C/OpenGL_Game/images/";
Sprite s[] =
{
Sprite(img_path + "gait-right-1.gif"),
Sprite(img_path + "gait-right-0.gif"),
Sprite(img_path + "gait-left-1.gif"),
Sprite(img_path + "gait-left-0.gif"),
Sprite(img_path + "gait-top-1.gif"),
Sprite(img_path + "gait-top-0.gif"),
Sprite(img_path + "gait-bottom-1.gif"),
Sprite(img_path + "gait-bottom-0.gif")
};
int main(int argn, char** argc) {
FreeImage_Initialise();
glutInit(&argn, argc);
glutInitDisplayMode(GLUT_RGBA | GLUT_DOUBLE | GLUT_DEPTH);
glutInitWindowSize(GV.windowWidth, GV.windowHeight);
glutCreateWindow("Monogatari");
glutDisplayFunc(render);
glutIdleFunc(render);
glutMainLoop();
FreeImage_DeInitialise();
}
void render() {
s[0].draw(0,0,1,0.5);
}

Using CUDA, SFML, and OpenGL: Texture Refuses to Appear on Quad

Using various tutorials/examples/documentations/forums online, I have typed out code to allow CUDA to manipulate OpenGL textures such that it can be outputted to the screen. My method of displaying is to use PBO and an allocated texture image of uchar4 array. Despite all my attempts at fixing the problem, the texture would not show up on the 2D surface. I cannot seem to pinpoint the problem.
These are all the things I have checked/done thus far: I have created a PBO and registered it with CUDA, called cudaGraphicsResourceGetMappedPointer and the unmapping equivalent before and after the GPU function calls, made sure that glEnable is called for 2D_TEXTURE, glDisable called for any unnecessary values, unbinded textures/buffers when not in need. I have also reset SFML OpenGL states in case SFML was the cause. Square textures have also been employed. My OpenGL verision and CUDA version work for all function calls I use.
There did not seem to be any errors within the program when I checked cudaErrors and OpenGL Errors.
I am not sure if this has something to do with it but when I call:
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
My quad does not seem to display.
I have mainly found inspiration from this website.
Thank you very much!
Here is my code:
Main.cpp
#include <GL/glew.h>
#include <windows.h>
#include <GL/GL.h>
#include <SFML/Window.hpp>
#include <SFML/OpenGL.hpp>
#include <SFML/System.hpp>
#include <SFML/Graphics/RenderWindow.hpp>
#include "GeneralTypedef.h"
#include "OpenGLTest.cuh"
int main()
{
// create the window
sf::RenderWindow window(sf::VideoMode(1024, 1024), "OpenGL");
//window.setVerticalSyncEnabled(true);
sf::Vector2u windowSize;
windowSize = sf::Vector2u(window.getSize());
bool running = true;
glewInit();
window.resetGLStates();
std::printf("OpenGL: %s:", glGetString(GL_VERSION));
// We will not be using SFML's gl states.
OpenGLTest* test = new OpenGLTest(window.getSize());
sf::Time time;
while (running)
{
// handle events
sf::Event event;
while (window.pollEvent(event))
{
if (event.type == sf::Event::Closed)
{
// end the program
running = false;
}
else if (event.type == sf::Event::Resized)
{
// adjust the viewport when the window is resized
glViewport(0, 0, event.size.width, event.size.height);
windowSize = window.getSize();
}
}
// clear the buffers
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
test->createFrame(time.asMicroseconds());
test->drawFrame();
window.display();
}
// release resources...
delete test;
return 0;
}
OpenGLTest.cuh
#ifndef OPENGLTEST_CUH
#define OPENGLTEST_CUH
#include <GL/glew.h>
#include <windows.h>
#include <GL/GL.h>
#include <cuda.h>
#include <cuda_runtime.h>
#include <cuda_gl_interop.h>
#include <SFML/OpenGL.hpp>
#include <SFML/Graphics.hpp>
#include <SFML/System.hpp>
#include "GeneralTypedef.h"
class OpenGLTest
{
public:
uchar4* image;
GLuint gltexture;
GLuint pbo;
cudaGraphicsResource_t cudaPBO;
uchar4* d_textureBufferData;
sf::Vector2u windowSize;
OpenGLTest(sf::Vector2u windowSize)
{
this->windowSize = sf::Vector2u(windowSize);
this->setupOpenGL();
};
~OpenGLTest()
{
delete image;
image == nullptr;
cudaFree(d_textureBufferData);
d_textureBufferData == nullptr;
glDeleteTextures(1, &gltexture);
}
void drawFrame();
void createFrame(float time);
private:
void setupOpenGL();
};
#endif //OPENGLTEST_CUH
OpenGLTest.cu
#include "OpenGLTest.cuh"
__global__ void createGPUTexture(uchar4* d_texture)
{
uint pixelID = blockIdx.x*blockDim.x + threadIdx.x;
d_texture[pixelID].x = 0;
d_texture[pixelID].y = 1;
d_texture[pixelID].z = 1;
d_texture[pixelID].w = 0;
}
__global__ void wow(uchar4* pos, unsigned int width, unsigned int height,
float time)
{
int index = blockIdx.x * blockDim.x + threadIdx.x;
unsigned int x = index%width;
unsigned int y = index / width;
if (index < width*height) {
unsigned char r = (x + (int)time) & 0xff;
unsigned char g = (y + (int)time) & 0xff;
unsigned char b = ((x + y) + (int)time) & 0xff;
// Each thread writes one pixel location in the texture (textel)
pos[index].w = 0;
pos[index].x = r;
pos[index].y = g;
pos[index].z = b;
}
}
void OpenGLTest::drawFrame()
{
glColor3f(1.0f,1.0f,1.0f);
glBindTexture(GL_TEXTURE_2D, gltexture);
glBindBuffer(GL_PIXEL_UNPACK_BUFFER_ARB, pbo);
glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, windowSize.x, windowSize.y, GL_RGBA, GL_UNSIGNED_BYTE, 0);
glBegin(GL_QUADS);
glTexCoord2f(0.0f, 0.0f);
glVertex2f(0.0f, float(windowSize.y));
glTexCoord2f(1.0f, 0.0f);
glVertex2f(float(windowSize.x), float(windowSize.y));
glTexCoord2f(1.0f, 1.0f);
glVertex2f(float(windowSize.x), 0.0f);
glTexCoord2f(0.0f,1.0f);
glVertex2f(0.0f, 0.0f);
glEnd();
glFlush();
// Release
glBindBuffer(GL_PIXEL_UNPACK_BUFFER_ARB, 0);
glBindTexture(GL_TEXTURE_2D, 0);
// Test Triangle
/*
glBegin(GL_TRIANGLES);
glColor3f(0.1, 0.2, 0.3);
glVertex2f(0, 0);
glVertex2f(10, 0);
glVertex2f(0, 100);
glEnd();
*/
}
void OpenGLTest::createFrame(float time)
{
cudaGraphicsMapResources(1, &cudaPBO, 0);
size_t numBytes;
cudaGraphicsResourceGetMappedPointer((void**)&d_textureBufferData, &numBytes, cudaPBO);
int totalThreads = windowSize.x * windowSize.y;
int nBlocks = totalThreads/ 256;
// Run code here.
createGPUTexture << <nBlocks, 256>> >(d_textureBufferData);
//wow << <nBlocks, 256 >> >(d_textureBufferData, windowSize.x, windowSize.y, time);
// Unmap mapping to PBO so that OpenGL can access.
cudaGraphicsUnmapResources(1, &cudaPBO, 0);
}
void OpenGLTest::setupOpenGL()
{
image = new uchar4[1024*1024];
glViewport(0, 0, windowSize.x, windowSize.y);
glMatrixMode(GL_MODELVIEW);
glLoadIdentity();
glMatrixMode(GL_PROJECTION);
glLoadIdentity();
glOrtho(0.0, windowSize.x, windowSize.y, 0.0, -1.0, 1.0);
glEnable(GL_TEXTURE_2D);
glDisable(GL_LIGHTING);
glDisable(GL_DEPTH_TEST);
// Unbind any textures from previous.
glBindTexture(GL_TEXTURE_2D, 0);
glBindBuffer(GL_PIXEL_UNPACK_BUFFER_ARB, 0);
// Create new textures.
glGenTextures(1, &gltexture);
glBindTexture(GL_TEXTURE_2D, gltexture);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
// Create image with same resolution as window.
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, windowSize.x , windowSize.y, 0, GL_RGBA, GL_UNSIGNED_BYTE, image);
// Create pixel buffer boject.
glGenBuffers(1, &pbo);
glBindBuffer(GL_PIXEL_UNPACK_BUFFER_ARB, pbo);
glBufferData(GL_PIXEL_UNPACK_BUFFER_ARB, windowSize.x * windowSize.y * sizeof(uchar4), image, GL_STREAM_COPY);
cudaGraphicsGLRegisterBuffer(&cudaPBO, pbo, cudaGraphicsMapFlagsNone);
glBindBuffer(GL_PIXEL_UNPACK_BUFFER_ARB, 0);
glBindTexture(GL_TEXTURE_2D, 0);
}
GeneralType
#ifndef GENERALTYPEDEF_CUH
#define GENERALTYPEDEF_CUH
typedef unsigned int uint;
#endif // GENERALTYPEDEF_CUH
After rewriting the entire code and understanding it more, I have figured out the reason. The color components for the uchar4 in the kernel function is mapped from 0-255. The w component is transparency. As such, it should be mapped to 255 for the image to show. I hope this helps for those who may have the same problem. Some sites have this value set very low as well.