Painting a QVideoFrame on a QVideoWidget with QT6 - c++

Question

I'd like to process the stream of my webcam frame by frame with QT6. I've checked the internet but since QTMultimedia was heavily reworked with QT6, and since QT6 is pretty new, all the documentation/questions available are outdated.
So, In order to achieve my goal, I'm using a QMediaCaptureSession with a camera set on QMediaDevices::defaultVideoInput(). I checked that this was working by setting the video output of the QMediaCaptureSession to a QVideoWidget with m_session.setVideoOutput(ui->videowidget);, and it's working fine, except that I can't process the frames (basically, it's rendering my webcam on the QVideoWidget).
Now, to process the frames, I have to use a QVideoSink as far as I understand the documentation here and there. So I replaced m_session.setVideoOutput(ui->videowidget); with m_session.setVideoSink(&mysink);, where mysink is a QVideoSink.
Then, since I want to process the frames, I'm connecting the videoFrameChanged signal of mysink to a function processVideoFrame where I want to do 2 things :
process the current frame
render the result on the UI, ideally on ui->videowidget
This is the point where I'm struggling. I do not understand how to use the paint function of the class QVideoFrame to render the processed frame on the QVideoWidget. More precisely :
I do not understand how I'm supposed to instantiate the QPainter. I tried a straightforward new QPainter(ui->videowidget) but it ends up in a QWidget::paintEngine: Should no longer be called exception and nothing is rendered
I do not understand what is actually representing the second parameter rect of QVideoFrame::paint?
I made a MWE, code is below.
mwe_videosinkpainting.h
#ifndef MWE_VIDEOSINKPAINTING_H
#define MWE_VIDEOSINKPAINTING_H
#include <QMainWindow>
#include <QMediaCaptureSession>
#include <QMediaDevices>
#include <QCamera>
#include <QVideoSink>
#include <QPainter>
QT_BEGIN_NAMESPACE
namespace Ui { class MWE_VideoSinkPainting; }
QT_END_NAMESPACE
class MWE_VideoSinkPainting : public QMainWindow
{
Q_OBJECT
public:
MWE_VideoSinkPainting(QWidget *parent = nullptr);
~MWE_VideoSinkPainting();
private slots:
void processVideoFrame();
private:
Ui::MWE_VideoSinkPainting *ui;
QVideoSink mysink;
QMediaCaptureSession m_session;
QScopedPointer<QCamera> m_camera;
};
#endif // MWE_VIDEOSINKPAINTING_H
mwe_videosinking.cpp
#include "mwe_videosinkpainting.h"
#include "ui_mwe_videosinkpainting.h"
MWE_VideoSinkPainting::MWE_VideoSinkPainting(QWidget *parent)
: QMainWindow(parent)
, ui(new Ui::MWE_VideoSinkPainting)
{
ui->setupUi(this);
m_camera.reset(new QCamera(QMediaDevices::defaultVideoInput()));
m_session.setCamera(m_camera.data());
//m_session.setVideoOutput(ui->videowidget);
connect(&mysink, &QVideoSink::videoFrameChanged, this, &MWE_VideoSinkPainting::processVideoFrame);
m_session.setVideoSink(&mysink);
m_camera->start();
}
MWE_VideoSinkPainting::~MWE_VideoSinkPainting()
{
delete ui;
}
void MWE_VideoSinkPainting::processVideoFrame()
{
QVideoFrame videoframe = mysink.videoFrame();
if(videoframe.map(QVideoFrame::ReadOnly))
{
//This is the part I'm struggling to understand and achieve
videoframe.paint(new QPainter(ui->videowidget), QRectF(0.0f,0.0f,100.0f,100.0f), QVideoFrame::PaintOptions());
videoframe.unmap();
}
}
main.cpp
#include "mwe_videosinkpainting.h"
#include <QApplication>
int main(int argc, char *argv[])
{
QApplication a(argc, argv);
MWE_VideoSinkPainting w;
w.show();
return a.exec();
}
ui_mwe_videosinkpainting.h (just so that you have the whole code, it has no value for the question)
#ifndef UI_MWE_VIDEOSINKPAINTING_H
#define UI_MWE_VIDEOSINKPAINTING_H
#include <QtCore/QVariant>
#include <QtMultimediaWidgets/QVideoWidget>
#include <QtWidgets/QApplication>
#include <QtWidgets/QGridLayout>
#include <QtWidgets/QHBoxLayout>
#include <QtWidgets/QMainWindow>
#include <QtWidgets/QMenuBar>
#include <QtWidgets/QStatusBar>
#include <QtWidgets/QWidget>
QT_BEGIN_NAMESPACE
class Ui_MWE_VideoSinkPainting
{
public:
QWidget *centralwidget;
QGridLayout *gridLayout;
QVideoWidget *videowidget;
QHBoxLayout *horizontalLayout;
QMenuBar *menubar;
QStatusBar *statusbar;
void setupUi(QMainWindow *MWE_VideoSinkPainting)
{
if (MWE_VideoSinkPainting->objectName().isEmpty())
MWE_VideoSinkPainting->setObjectName(QString::fromUtf8("MWE_VideoSinkPainting"));
MWE_VideoSinkPainting->resize(800, 600);
centralwidget = new QWidget(MWE_VideoSinkPainting);
centralwidget->setObjectName(QString::fromUtf8("centralwidget"));
gridLayout = new QGridLayout(centralwidget);
gridLayout->setObjectName(QString::fromUtf8("gridLayout"));
videowidget = new QVideoWidget(centralwidget);
videowidget->setObjectName(QString::fromUtf8("videowidget"));
horizontalLayout = new QHBoxLayout(videowidget);
horizontalLayout->setObjectName(QString::fromUtf8("horizontalLayout"));
gridLayout->addWidget(videowidget, 0, 0, 1, 1);
MWE_VideoSinkPainting->setCentralWidget(centralwidget);
menubar = new QMenuBar(MWE_VideoSinkPainting);
menubar->setObjectName(QString::fromUtf8("menubar"));
menubar->setGeometry(QRect(0, 0, 800, 21));
MWE_VideoSinkPainting->setMenuBar(menubar);
statusbar = new QStatusBar(MWE_VideoSinkPainting);
statusbar->setObjectName(QString::fromUtf8("statusbar"));
MWE_VideoSinkPainting->setStatusBar(statusbar);
retranslateUi(MWE_VideoSinkPainting);
QMetaObject::connectSlotsByName(MWE_VideoSinkPainting);
} // setupUi
void retranslateUi(QMainWindow *MWE_VideoSinkPainting)
{
MWE_VideoSinkPainting->setWindowTitle(QCoreApplication::translate("MWE_VideoSinkPainting", "MWE_VideoSinkPainting", nullptr));
} // retranslateUi
};
namespace Ui {
class MWE_VideoSinkPainting: public Ui_MWE_VideoSinkPainting {};
} // namespace Ui
QT_END_NAMESPACE
#endif // UI_MWE_VIDEOSINKPAINTING_H

Answer 1

The answer is quite straightforward : you can use setVideoSink AND setVideoOutput.
The code I gave in OP is good, you just have to uncomment setVideoOutput(ui->videowidget); of mwe_videosinking.cpp and to call setVideoSink BEFORE calling setVideoOutput

Answer 2

Since I cannot format code in a comment...
So, you mean change this...
ui->setupUi(this);
m_camera.reset(new QCamera(QMediaDevices::defaultVideoInput()));
m_session.setCamera(m_camera.data());
//m_session.setVideoOutput(ui->videowidget);
connect(&mysink, &QVideoSink::videoFrameChanged, this, &MWE_VideoSinkPainting::processVideoFrame);
m_session.setVideoSink(&mysink);
m_camera->start();
...to this?
ui->setupUi(this);
m_camera.reset(new QCamera(QMediaDevices::defaultVideoInput()));
m_session.setCamera(m_camera.data());
m_session.setVideoSink(&mysink);
m_session.setVideoOutput(ui->videowidget);
connect(&mysink, &QVideoSink::videoFrameChanged, this, &MWE_VideoSinkPainting::processVideoFrame);
m_camera->start();
I did this and I am still only getting black in my videoWidget.