Problem about saving cv::Mat to mp4 file via appsrc - computer-vision

Question

I want to save CV:: mat as MP4 file through appsrc or forward it through RTMP. I implemented a simple code that can run successfully, but the obtained MP4 file cannot be played. Can someone tell me what the problem is? I think maybe the PTS is set wrong, but I don't know how to solve it.Thank you in advance.
#include <gst/gst.h>
#include <glib.h>
#include <gst/app/gstappsrc.h>
#include <opencv2/opencv.hpp>
#include <unistd.h>
typedef struct {
GstPipeline *pipeline = nullptr;
GstAppSrc *app_src = nullptr;
GstElement *video_convert = nullptr;
GstElement *encoder = nullptr;
GstElement *h264_parser = nullptr;
GstElement *qt_mux = nullptr;
GstElement *file_sink = nullptr;
}CustomData;
int main(int argc,char * argv[]){
CustomData data;
GstBus *bus = nullptr;
GstMessage *msg = nullptr;
GstStateChangeReturn ret;
gboolean terminate = false;
GstClockTime timestamp = 0;
gst_init(&argc, &argv);
data.pipeline = (GstPipeline*)gst_pipeline_new("m_pipeline");
data.app_src = (GstAppSrc*)gst_element_factory_make("appsrc","m_app_src");
data.video_convert = gst_element_factory_make("videoconvert","m_video_convert");
data.encoder = gst_element_factory_make("x264enc","m_x264enc");
data.h264_parser = gst_element_factory_make("h264parse","m_h264_parser");
data.qt_mux = gst_element_factory_make("qtmux","qt_mux");
data.file_sink = gst_element_factory_make("filesink","file_sink");
if (!data.app_src || !data.video_convert || !data.encoder || !data.h264_parser || !data.qt_mux || !data.file_sink || !data.pipeline){
g_printerr("failed to create all elements\n");
return -1;
}
gst_bin_add_many(GST_BIN(data.pipeline), (GstElement*)data.app_src, data.video_convert, data.encoder, data.h264_parser, data.qt_mux, data.file_sink, NULL);
g_assert(gst_element_link_many((GstElement*)data.app_src, data.video_convert, data.encoder, data.h264_parser, data.qt_mux, data.file_sink,NULL));
GstCaps *caps = gst_caps_new_simple("video/x-raw","format",G_TYPE_STRING,"BGR",
"width",G_TYPE_INT,1280,
"height",G_TYPE_INT,720,
"framerate",GST_TYPE_FRACTION,25,1,
NULL);
gst_app_src_set_caps(GST_APP_SRC(data.app_src), caps);
g_object_set(data.app_src,"is_live",true,NULL);
g_object_set(data.app_src,"format",GST_FORMAT_TIME,NULL);
std::string mp4_url = "des.mp4";
g_object_set(data.file_sink,"location",mp4_url.c_str(),NULL);
ret = gst_element_set_state((GstElement*)data.pipeline, GST_STATE_PLAYING);
if (ret == GST_STATE_CHANGE_FAILURE){
g_printerr("Unable to set the pipeline to the playing state. \n");
gst_object_unref(data.pipeline);
return -1;
}
cv::VideoCapture cap;
cap.open("src.mp4");
if(!cap.isOpened())
return -2;
cv::Mat frame;
while(true){
cap.read(frame);
if(frame.empty()){
break;
}
GstBuffer *buffer;
buffer = gst_buffer_new_wrapped(frame.data, frame.size().width * frame.size().height * frame.channels());
GST_BUFFER_PTS (buffer) = timestamp;
GST_BUFFER_DURATION (buffer) = gst_util_uint64_scale_int (1, GST_SECOND, 25);
timestamp += GST_BUFFER_DURATION (buffer);
GstFlowReturn ret;
g_signal_emit_by_name(data.app_src, "push-buffer", buffer, &ret);
usleep(1000000/25);
}
gst_element_set_state((GstElement*)data.pipeline, GST_STATE_NULL);
gst_object_unref(data.pipeline);
return 0;
}

Answer 1

You cannot simply set the pipeline state to NULL. Instead you need to send an EOS event to pipeline and wait until the EOS signal is reported back on the pipeline's bus. If you don't do that then the MP4 file will not be written with some requirement headers and the file cannot play.
See https://gstreamer.freedesktop.org/documentation/application-development/basics/helloworld.html?gi-language=c for how to check for EOS message on the bus.
Check https://gstreamer.freedesktop.org/documentation/applib/gstappsrc.html#gst_app_src_end_of_stream on how to tell your appsrc that the last data buffer has been pushed.