Basler Pylon to GStreamer to multicast UDP - c++

Question

I am attempting to use GStreamer to send captured images from a Basler acA720-290gm GigE camera to multiple clients over UDP multicast. I have been able to acquire the images using the Pylon 7.1.0.25066 API and send them through a GStreamer pipeline consisting of appsrc, videoconvert and ximagesink. However, if I try to add elements to convert the stream to RTP and back, then I only get a single static image.
This code successfully displays the stream on Debian 11. I'm not sure if I can use something other than GRAY8 for the appsrc caps format or not.
Test.cpp
#include <pylon/PylonIncludes.h>
#include <gst/gst.h>
#include "unistd.h"
#include "pthread.h"
pthread_t thread_1;
pthread_t thread_2;
Pylon::CPylonImage image;
bool image_valid = false;
bool gstreamer_thread_done = false;
void* pylon_thread(void*) {
Pylon::CDeviceInfo device_info;
Pylon::CGrabResultPtr ptrGrabResult;
Pylon::PylonAutoInitTerm autoInitTerm;
device_info.SetIpAddress("192.168.1.109");
while (1) {
if (gstreamer_thread_done) { break; }
try {
Pylon::CInstantCamera camera(Pylon::CTlFactory::GetInstance().CreateDevice(device_info));
camera.StartGrabbing(Pylon::GrabStrategy_LatestImageOnly);
while (camera.IsGrabbing()) {
if (gstreamer_thread_done) { break; }
camera.RetrieveResult(5000, ptrGrabResult, Pylon::TimeoutHandling_ThrowException);
if (ptrGrabResult->GrabSucceeded()) {
image.CopyImage(ptrGrabResult);
image_valid = true;
}
else {
fprintf(stderr, "Error: %u %s\n", ptrGrabResult->GetErrorCode(), ptrGrabResult->GetErrorDescription().c_str());
}
}
}
catch (const Pylon::GenericException &e) {
fprintf(stderr, "An exception occurred.\n");
fprintf(stderr, "%s\n", e.GetDescription());
sleep(1);
}
}
pthread_exit(NULL);
}
struct gstreamer_data {
GstElement* appsrc1;
GstElement* videoconvert1;
GstElement* ximagesink1;
GstElement* pipeline;
GMainLoop* main_loop;
guint source_id;
};
static gboolean push_data(gstreamer_data* data) {
GstBuffer* buffer;
GstFlowReturn ret;
buffer = gst_buffer_new_wrapped_full(
(GstMemoryFlags) GST_MEMORY_FLAG_PHYSICALLY_CONTIGUOUS, (gpointer) image.GetBuffer(), image.GetImageSize(), 0, image.GetImageSize(), NULL, NULL
);
g_signal_emit_by_name(data->appsrc1, "push-buffer", buffer, &ret);
if (ret != GST_FLOW_OK) {
printf("Error\n");
return false;
}
return true;
}
static void start_feed(GstElement* source, guint size, gstreamer_data* data) {
(void) source;
(void) size;
if (data->source_id == 0) {
//g_print("Start feeding\n");
data->source_id = g_idle_add((GSourceFunc) push_data, data);
}
}
static void stop_feed(GstElement* source, gstreamer_data* data) {
(void) source;
if (data->source_id != 0) {
//g_print("Stop feeding\n");
g_source_remove(data->source_id);
data->source_id = 0;
}
}
static void error_cb(GstBus* bus, GstMessage* msg, gstreamer_data* data) {
(void) bus;
GError* err;
gchar* debug_info;
gst_message_parse_error(msg, &err, &debug_info);
g_printerr("Error received from element %s: %s\n", GST_OBJECT_NAME (msg->src), err->message);
g_printerr ("Debugging information: %s\n", debug_info ? debug_info : "none");
g_clear_error(&err);
g_free(debug_info);
g_main_loop_quit(data->main_loop);
}
void* gstreamer_thread(void*) {
gstreamer_data data;
GstStateChangeReturn ret;
GstBus* bus;
data.source_id = 0;
gst_init(NULL, NULL);
data.appsrc1 = gst_element_factory_make("appsrc", "appsrc1");
g_object_set(
G_OBJECT(data.appsrc1),
"stream-type", 0,
"format", GST_FORMAT_TIME,
"is-live", TRUE,
NULL
);
g_object_set(
G_OBJECT(data.appsrc1),
"caps", gst_caps_new_simple(
"video/x-raw",
"format", G_TYPE_STRING, "GRAY8",
"width", G_TYPE_INT, image.GetWidth(),
"height", G_TYPE_INT, image.GetHeight(),
"framerate", GST_TYPE_FRACTION, 0, 1,
NULL
),
NULL
);
g_signal_connect(data.appsrc1, "need-data", G_CALLBACK(start_feed), &data);
g_signal_connect(data.appsrc1, "enough-data", G_CALLBACK(stop_feed), &data);
data.videoconvert1 = gst_element_factory_make("videoconvert", "videoconvert1");
data.ximagesink1 = gst_element_factory_make("ximagesink", "ximagesink1");
data.pipeline = gst_pipeline_new("pipeline");
if (
!data.pipeline ||
!data.appsrc1 ||
!data.videoconvert1 ||
!data.ximagesink1
)
{
g_printerr("Not all elements could be created.\n");
exit(-1);
}
gst_bin_add_many (
GST_BIN(data.pipeline),
data.appsrc1,
data.videoconvert1,
data.ximagesink1,
NULL
);
if (
gst_element_link_many (
data.appsrc1,
data.videoconvert1,
data.ximagesink1,
NULL
) != TRUE
)
{
g_printerr("Elements could not be linked.\n");
gst_object_unref(data.pipeline);
exit(-1);
}
bus = gst_element_get_bus(data.pipeline);
gst_bus_add_signal_watch(bus);
g_signal_connect(G_OBJECT (bus), "message::error", (GCallback)error_cb, &data);
gst_object_unref(bus);
ret = gst_element_set_state(data.pipeline, GST_STATE_PLAYING);
if (ret == GST_STATE_CHANGE_FAILURE) {
g_printerr("Unable to set the pipeline to the playing state.\n");
gst_object_unref(data.pipeline);
exit(-1);
}
data.main_loop = g_main_loop_new(NULL, FALSE);
g_main_loop_run(data.main_loop);
printf("Exiting.\n");
gstreamer_thread_done = true;
pthread_exit(NULL);
}
int main() {
int error;
error = pthread_create(&thread_1, NULL, pylon_thread, NULL);
if (error) {
printf("Error: pthread_create: %i\n", error);
exit(EXIT_FAILURE);
}
while (image_valid == false) {
sleep(1);
}
error = pthread_create(&thread_2, NULL, gstreamer_thread, NULL);
if (error) {
printf("Error: pthread_create: %i\n", error);
exit(EXIT_FAILURE);
}
pthread_join(thread_1, NULL);
pthread_join(thread_2, NULL);
return 0;
}
Makefile
# Makefile for Basler pylon sample program
.PHONY: all clean
# The program to build
NAME := Test
# Installation directories for pylon
PYLON_ROOT ?= /opt/pylon
# Build tools and flags
LD := $(CXX)
CPPFLAGS := $(shell $(PYLON_ROOT)/bin/pylon-config --cflags) $(shell pkg-config --cflags gstreamer-1.0) $(shell pkg-config --cflags opencv4) -DUSE_GIGE
CXXFLAGS := #e.g., CXXFLAGS=-g -O0 for debugging
LDFLAGS := $(shell $(PYLON_ROOT)/bin/pylon-config --libs-rpath)
LDLIBS := $(shell $(PYLON_ROOT)/bin/pylon-config --libs) $(shell pkg-config --libs gstreamer-1.0) -lopencv_core -lopencv_imgproc -lpthread
# Rules for building
all: $(NAME)
$(NAME): $(NAME).o
$(LD) $(LDFLAGS) -o $# $^ $(LDLIBS)
$(NAME).o: $(NAME).cpp
$(CXX) $(CPPFLAGS) $(CXXFLAGS) -c -o $# $<
clean:
$(RM) $(NAME).o $(NAME)
This code displays just a single image:
Test.cpp
#include <pylon/PylonIncludes.h>
#include <gst/gst.h>
#include "unistd.h"
#include "pthread.h"
pthread_t thread_1;
pthread_t thread_2;
Pylon::CPylonImage image;
bool image_valid = false;
bool gstreamer_thread_done = false;
void* pylon_thread(void*) {
Pylon::CDeviceInfo device_info;
Pylon::CGrabResultPtr ptrGrabResult;
Pylon::PylonAutoInitTerm autoInitTerm;
device_info.SetIpAddress("192.168.1.109");
while (1) {
if (gstreamer_thread_done) { break; }
try {
Pylon::CInstantCamera camera(Pylon::CTlFactory::GetInstance().CreateDevice(device_info));
camera.StartGrabbing(Pylon::GrabStrategy_LatestImageOnly);
while (camera.IsGrabbing()) {
if (gstreamer_thread_done) { break; }
camera.RetrieveResult(5000, ptrGrabResult, Pylon::TimeoutHandling_ThrowException);
if (ptrGrabResult->GrabSucceeded()) {
image.CopyImage(ptrGrabResult);
image_valid = true;
}
else {
fprintf(stderr, "Error: %u %s\n", ptrGrabResult->GetErrorCode(), ptrGrabResult->GetErrorDescription().c_str());
}
}
}
catch (const Pylon::GenericException &e) {
fprintf(stderr, "An exception occurred.\n");
fprintf(stderr, "%s\n", e.GetDescription());
sleep(1);
}
}
pthread_exit(NULL);
}
struct gstreamer_data {
GstElement* appsrc1;
GstElement* videoconvert1;
GstElement* x264enc1;
GstElement* rtph264pay1;
GstElement* rtph264depay1;
GstElement* avdec_h2641;
GstElement* videoconvert2;
GstElement* ximagesink1;
GstElement* pipeline;
GMainLoop* main_loop;
guint source_id;
};
static gboolean push_data(gstreamer_data* data) {
GstBuffer* buffer;
GstFlowReturn ret;
buffer = gst_buffer_new_wrapped_full(
(GstMemoryFlags) GST_MEMORY_FLAG_PHYSICALLY_CONTIGUOUS, (gpointer) image.GetBuffer(), image.GetImageSize(), 0, image.GetImageSize(), NULL, NULL
);
g_signal_emit_by_name(data->appsrc1, "push-buffer", buffer, &ret);
if (ret != GST_FLOW_OK) {
printf("Error\n");
return false;
}
return true;
}
static void start_feed(GstElement* source, guint size, gstreamer_data* data) {
(void) source;
(void) size;
if (data->source_id == 0) {
g_print("Start feeding\n");
data->source_id = g_idle_add((GSourceFunc) push_data, data);
}
}
static void stop_feed(GstElement* source, gstreamer_data* data) {
(void) source;
if (data->source_id != 0) {
g_print("Stop feeding\n");
g_source_remove(data->source_id);
data->source_id = 0;
}
}
static void error_cb(GstBus* bus, GstMessage* msg, gstreamer_data* data) {
(void) bus;
GError* err;
gchar* debug_info;
gst_message_parse_error(msg, &err, &debug_info);
g_printerr("Error received from element %s: %s\n", GST_OBJECT_NAME (msg->src), err->message);
g_printerr ("Debugging information: %s\n", debug_info ? debug_info : "none");
g_clear_error(&err);
g_free(debug_info);
g_main_loop_quit(data->main_loop);
}
void* gstreamer_thread(void*) {
gstreamer_data data;
GstStateChangeReturn ret;
GstBus* bus;
data.source_id = 0;
gst_init(NULL, NULL);
data.appsrc1 = gst_element_factory_make("appsrc", "appsrc1");
g_object_set(
G_OBJECT(data.appsrc1),
"stream-type", 0,
"format", GST_FORMAT_TIME,
"is-live", TRUE,
NULL
);
g_object_set(
G_OBJECT(data.appsrc1),
"caps", gst_caps_new_simple(
"video/x-raw",
"format", G_TYPE_STRING, "GRAY8",
"width", G_TYPE_INT, image.GetWidth(),
"height", G_TYPE_INT, image.GetHeight(),
"framerate", GST_TYPE_FRACTION, 0, 1,
NULL
),
NULL
);
g_signal_connect(data.appsrc1, "need-data", G_CALLBACK(start_feed), &data);
g_signal_connect(data.appsrc1, "enough-data", G_CALLBACK(stop_feed), &data);
data.videoconvert1 = gst_element_factory_make("videoconvert", "videoconvert1");
data.x264enc1 = gst_element_factory_make("x264enc", "x264enc1");
data.rtph264pay1 = gst_element_factory_make("rtph264pay", "rtph264pay1");
data.rtph264depay1 = gst_element_factory_make("rtph264depay", "rtph264depay1");
data.avdec_h2641 = gst_element_factory_make("avdec_h264", "avdec_h2641");
data.videoconvert2 = gst_element_factory_make("videoconvert", "videoconvert2");
data.ximagesink1 = gst_element_factory_make("ximagesink", "ximagesink1");
data.pipeline = gst_pipeline_new("pipeline");
if (
!data.pipeline ||
!data.appsrc1 ||
!data.videoconvert1 ||
!data.x264enc1 ||
!data.rtph264pay1 ||
!data.rtph264depay1 ||
!data.avdec_h2641 ||
!data.videoconvert2 ||
!data.ximagesink1
)
{
g_printerr("Not all elements could be created.\n");
exit(-1);
}
gst_bin_add_many (
GST_BIN(data.pipeline),
data.appsrc1,
data.videoconvert1,
data.x264enc1,
data.avdec_h2641,
data.videoconvert2,
data.ximagesink1,
NULL
);
if (
gst_element_link_many (
data.appsrc1,
data.videoconvert1,
data.x264enc1,
data.avdec_h2641,
data.videoconvert2,
data.ximagesink1,
NULL
) != TRUE
)
{
g_printerr("Elements could not be linked.\n");
gst_object_unref(data.pipeline);
exit(-1);
}
bus = gst_element_get_bus(data.pipeline);
gst_bus_add_signal_watch(bus);
g_signal_connect(G_OBJECT (bus), "message::error", (GCallback)error_cb, &data);
gst_object_unref(bus);
ret = gst_element_set_state(data.pipeline, GST_STATE_PLAYING);
if (ret == GST_STATE_CHANGE_FAILURE) {
g_printerr("Unable to set the pipeline to the playing state.\n");
gst_object_unref(data.pipeline);
exit(-1);
}
data.main_loop = g_main_loop_new(NULL, FALSE);
g_main_loop_run(data.main_loop);
printf("Exiting.\n");
gstreamer_thread_done = true;
pthread_exit(NULL);
}
int main() {
int error;
error = pthread_create(&thread_1, NULL, pylon_thread, NULL);
if (error) {
printf("Error: pthread_create: %i\n", error);
exit(EXIT_FAILURE);
}
while (image_valid == false) {
sleep(1);
}
error = pthread_create(&thread_2, NULL, gstreamer_thread, NULL);
if (error) {
printf("Error: pthread_create: %i\n", error);
exit(EXIT_FAILURE);
}
pthread_join(thread_1, NULL);
pthread_join(thread_2, NULL);
return 0;
}

Answer 1

Maybe instead of copying image from PtrGrabResult To CPylonImage, try with alternate method of receiving buffers: through image.AttachGrabResultBuffer() method. It is well used in GUI_ImageWindow sample project of Pylon SDK.