Display cv::Mat as QVideoFrame in a QML VideoOutput - c++

Question

I have an OpenCV backend which retrieves the video frame from a camera device through cv::VideoCapture, does some processing and then passes the cv::Mat frame to a Qt5 application for display in a QML VideoOutput.
The problem is the frames drawn are empty/white.
The CameraService class is receiving a cv::Mat from a backend object which runs on its own thread using a Qt::QueuedConnection signal. Then I convert to it to a QImage, which I use to initialize a QVideoFrame and pass it to a QAbstractVideoSurface received from a QML VideoOutput, after setting a pixel format on it.
I have checked whether the cv::Mat has valid content before conversion to QVideoFrame, so this is not the case.
Or am I doing it completely wrong and should instead draw an image?
Relevant code:
CameraService.cpp
CameraService::CameraService(Video::Backend *backend)
: QObject(),
surface(nullptr),
isFormatSet(false) {
this->backend = backend;
connect(
backend, &Video::Backend::onFrameReady,
this, &CameraService::onVideoFrameReady,
Qt::QueuedConnection);
}
CameraService::~CameraService() {
backend->deleteLater();
}
QAbstractVideoSurface *CameraService::getVideoSurface() const {
return surface;
}
void CameraService::setVideoSurface(QAbstractVideoSurface *surface) {
if (!this->surface && surface)
backend->start();
if (this->surface && this->surface != surface && this->surface->isActive())
this->surface->stop();
this->surface = surface;
if (this->surface && format.isValid()) {
format = this->surface->nearestFormat(format);
this->surface->start(format);
}
}
void CameraService::setFormat(
int width,
int height,
QVideoFrame::PixelFormat frameFormat
){
QSize size(width, height);
QVideoSurfaceFormat format(size, frameFormat);
this->format = format;
if (surface) {
if (surface->isActive())
surface->stop();
this->format = surface->nearestFormat(this->format);
surface->start(this->format);
}
}
void CameraService::onVideoFrameReady(cv::Mat currentFrame) {
if (!surface || currentFrame.empty())
return;
cv::Mat continuousFrame;
if (!currentFrame.isContinuous())
continuousFrame = currentFrame.clone();
else
continuousFrame = currentFrame;
if (!isFormatSet) {
setFormat(
continuousFrame.cols,
continuousFrame.rows,
QVideoFrame::PixelFormat::Format_BGR32);
isFormatSet = true;
}
frame = QImage(
(uchar *)continuousFrame.data,
continuousFrame.cols,
continuousFrame.rows,
continuousFrame.step,
QVideoFrame::imageFormatFromPixelFormat(
QVideoFrame::PixelFormat::Format_BGR32));
surface->present(QVideoFrame(frame));
}
QML object:
VideoOutput {
objectName: "videoOutput";
anchors.fill: parent;
fillMode: VideoOutput.PreserveAspectCrop;
source: CameraService;
}
The CameraService object is made available as a singleton to QML using this statement:
qmlRegisterSingletonInstance<Application::CameraService>("Application.CameraService", 1, 0, "CameraService", service);

Answer 1

Analyzing the code I have noticed that the conversion is not supported (I recommend you check if the format is valid or). For this I have made some changes:...
#ifndef CAMERASERVICE_H
#define CAMERASERVICE_H
#include "backend.h"
#include <QObject>
#include <QPointer>
#include <QVideoFrame>
#include <QVideoSurfaceFormat>
#include <opencv2/core/mat.hpp>
class QAbstractVideoSurface;
class CameraService : public QObject
{
Q_OBJECT
Q_PROPERTY(QAbstractVideoSurface* videoSurface READ videoSurface WRITE setVideoSurface NOTIFY surfaceChanged)
public:
explicit CameraService(Backend *backend, QObject *parent = nullptr);
QAbstractVideoSurface* videoSurface() const;
public Q_SLOTS:
void setVideoSurface(QAbstractVideoSurface* surface);
Q_SIGNALS:
void surfaceChanged(QAbstractVideoSurface* surface);
private Q_SLOTS:
void onVideoFrameReady(cv::Mat currentFrame);
private:
void setFormat(int width, int height, QVideoFrame::PixelFormat frameFormat);
QPointer<QAbstractVideoSurface> m_surface;
QScopedPointer<Backend> m_backend;
QVideoSurfaceFormat m_format;
bool m_isFormatSet;
QImage m_image;
};
#endif // CAMERASERVICE_H
#include "backend.h"
#include "cameraservice.h"
#include <QAbstractVideoSurface>
#include <iostream>
CameraService::CameraService(Backend *backend, QObject *parent)
: QObject(parent), m_backend(backend), m_isFormatSet(false)
{
connect(m_backend.data(), &Backend::frameReady, this, &CameraService::onVideoFrameReady);
}
QAbstractVideoSurface *CameraService::videoSurface() const
{
return m_surface;
}
void CameraService::setVideoSurface(QAbstractVideoSurface *surface){
if (m_surface == surface)
return;
if(m_surface && m_surface != surface && m_surface->isActive())
m_surface->stop();
m_surface = surface;
Q_EMIT surfaceChanged(m_surface);
m_backend->start();
if (m_surface && m_format.isValid()) {
m_format = m_surface->nearestFormat(m_format);
m_surface->start(m_format);
}
}
void CameraService::setFormat(
int width,
int height,
QVideoFrame::PixelFormat frameFormat
){
QSize size(width, height);
QVideoSurfaceFormat format(size, frameFormat);
m_format = format;
if (m_surface) {
if (m_surface->isActive())
m_surface->stop();
m_format = m_surface->nearestFormat(m_format);
m_surface->start(m_format);
}
}
void CameraService::onVideoFrameReady(cv::Mat currentFrame){
if (!m_surface || currentFrame.empty())
return;
cv::Mat continuousFrame;
if (!currentFrame.isContinuous())
continuousFrame = currentFrame.clone();
else
continuousFrame = currentFrame;
if (!m_isFormatSet) {
setFormat(continuousFrame.cols,
continuousFrame.rows,
QVideoFrame::Format_RGB32);
m_isFormatSet = true;
}
m_image = QImage(continuousFrame.data,
continuousFrame.cols,
continuousFrame.rows,
continuousFrame.step,
QImage::Format_RGB888);
m_image = m_image.rgbSwapped();
m_image.convertTo(QVideoFrame::imageFormatFromPixelFormat(QVideoFrame::Format_RGB32));
m_surface->present(QVideoFrame(m_image));
}
You can find the complete example here.