Display RTP stream using GstSharp - gstreamer

Question

I want to read a RTP stream and display it in WPF. I have the basic example working with RTSP, but I can't make it work with basic RTP. I believe I have problems with caps settings, but I'm not sure how or why.
There is the input stream encoded with gst-launch:
gst-launch-1.0 dx9screencapsrc width=1920 height=1080 monitor=1 ! "video/x-raw, framerate=30/1" ! videoconvert ! x264enc speed-preset=ultrafast threads=4 sliced-threads=true ! rtph264pay config-interval=1 pt=96 ! udpsink host=127.0.0.1 port = 5600
I have successfully decoded the stream using:
gst-launch-1.0 udpsrc port=5600 ! "application/x-rtp" ! rtph264depay ! avdec_h264 max-threads=12 ! autovideosink
I am therefore trying to use the same pipeline with gstSharp. Here is what I have so far:
using System;
using System.Linq;
using GLib;
using Gst;
using Gst.Video;
using Thread = System.Threading.Thread;
namespace GStreamerD3D.Samples.WPF.D3D11
{
public class Playback
{
private static readonly log4net.ILog _log = log4net.LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType);
public delegate void OnDrawReceivedEventHandler(Element videoSink, GLib.SignalArgs args);
private Pipeline _pipeline;
private MainLoop _mainLoop;
private Thread _mainGlibThread;
private Element _uriDecodeBin, _depay, _avdec, _audioConvert, _videoConvert;
private Element _audioSink, _videoSink;
private VideoOverlayAdapter _adapter;
private IntPtr _handle;
private const bool _enableAudio = false;
private const string _videoDecoder = "d3d11h264dec"; // This decoder will reduce CPU usage significantly
private bool _enableOverlay;
private string _source = "rtsp://localhost:8554/";
public Playback(IntPtr hwnd, bool enableOverlay, string gstDebug = "")
{
_handle = hwnd;
_enableOverlay = enableOverlay;
if (!String.IsNullOrEmpty(gstDebug))
{
Environment.SetEnvironmentVariable("GST_DEBUG", gstDebug);
}
InitGst();
CreatePipeline();
var ret = _pipeline.SetState(State.Playing);
if (ret == StateChangeReturn.Failure)
{
Log("Unable to set the pipeline to the playing state.", LogLevelFlags.Error);
return;
}
}
public OnDrawReceivedEventHandler OnDrawSignalReceived;
public void InitGst()
{
Gst.Application.Init();
GtkSharp.GstreamerSharp.ObjectManager.Initialize();
_mainLoop = new MainLoop();
_mainGlibThread = new Thread(_mainLoop.Run);
_mainGlibThread.Start();
}
private void CreatePipeline()
{
Log("Initializing Pipeline..", LogLevelFlags.Debug);
_pipeline = new Pipeline("pipeline0");
_pipeline.Bus.EnableSyncMessageEmission();
_pipeline.Bus.AddSignalWatch();
_pipeline.Bus.Message += OnBusMessage;
_pipeline.AutoFlushBus = true;
if (!_enableOverlay)
{
_pipeline.Bus.SyncMessage += OnBusSyncMessage;
}
CreateElements();
_pipeline.Add(_uriDecodeBin, _depay, _avdec, _videoConvert, _videoSink);
if (!_videoConvert.Link(_videoSink))
{
Log("Video sink could not be linked", LogLevelFlags.FlagFatal);
return;
}
}
private void OnBusSyncMessage(object o, SyncMessageArgs sargs)
{
Message msg = sargs.Message;
if (!Gst.Video.Global.IsVideoOverlayPrepareWindowHandleMessage(msg))
{
msg.Dispose();
return;
}
Element src = msg.Src as Element;
if (src == null)
{
msg.Dispose();
return;
}
try
{
src["force-aspect-ratio"] = true;
}
catch (PropertyNotFoundException ex)
{
Console.WriteLine(ex.Message, ex);
}
Element overlay = (_pipeline)?.GetByInterface(VideoOverlayAdapter.GType);
_adapter = new VideoOverlayAdapter(overlay.Handle);
_adapter.WindowHandle = _handle;
_adapter.HandleEvents(true);
msg?.Dispose();
overlay?.Dispose();
src?.Dispose();
}
protected bool CreateElements()
{
try
{
_videoSink = ElementFactory.Make("d3d11videosink", "d3d11videosink0");
if (_enableOverlay)
{
_videoSink["draw-on-shared-texture"] = true;
_videoSink.Connect("begin-draw", VideoSink_OnBeginDraw);
}
//SetPrimaryDecoder(_videoDecoder);
_videoConvert = ElementFactory.Make("videoconvert", "videoconvert0");
if (_videoSink == null)
{
Log($"Could not locate Direct3D11", LogLevelFlags.Error);
_videoSink = ElementFactory.Make("autovideosink", "autovideosink0");
}
_uriDecodeBin = ElementFactory.Make("udpsrc", "source");
_uriDecodeBin["port"] = 5600;
var caps = Gst.Global.CapsFromString("application/x-rtp, payload=(int)96");
_uriDecodeBin["caps"] = caps;
_depay = ElementFactory.Make("rtph264depay", "depay");
Log($"Linking rtspsrc: {_uriDecodeBin.Link(_depay)}", LogLevelFlags.Message);
_avdec = ElementFactory.Make("avdec_h264", "avdec");
//_uriDecodeBin.PadAdded += OnPadAdded;
}
catch (Exception ex)
{
Log(ex.ToString(), LogLevelFlags.Critical, ex);
return false;
}
return true;
}
private void OnBusMessage(object o, MessageArgs margs)
{
try
{
Message message = margs.Message;
switch (message.Type)
{
case MessageType.Eos:
Log("Replaying stream...", LogLevelFlags.Info);
var ret = _pipeline.SetState(Gst.State.Ready);
if (ret == StateChangeReturn.Async)
ret = _pipeline.GetState(out var state, out var pending, Gst.Constants.SECOND * 10L);
if (ret == StateChangeReturn.Success)
{
ret = _pipeline.SetState(Gst.State.Playing);
if (ret == StateChangeReturn.Async)
ret = _pipeline.GetState(out var state, out var pending, Gst.Constants.SECOND * 10L);
}
break;
case MessageType.Error:
message.ParseError(out GException err, out string debug);
if (debug == null)
{
debug = "none";
}
Log($"Error! Bus message: {debug}", LogLevelFlags.Error, err);
if (err.Code == 3)
{
System.Windows.Application.Current.Dispatcher.InvokeShutdown();
}
break;
}
}
catch (Exception ex)
{
Log("Bus message error.", LogLevelFlags.Error, ex);
}
}
private void VideoSink_OnBeginDraw(object o, GLib.SignalArgs args)
{
OnDrawSignalReceived?.Invoke((Element)o, args);
}
public void Log(string message, LogLevelFlags logLevel, Exception exception = null)
{
try
{
switch (logLevel)
{
case LogLevelFlags.Debug:
_log.Debug(message, exception);
break;
case LogLevelFlags.Info:
_log.Info(message, exception);
break;
case LogLevelFlags.Warning:
_log.Warn(message, exception);
break;
case LogLevelFlags.Error:
_log.Error(message, exception);
break;
case LogLevelFlags.FlagFatal:
_log.Fatal(message, exception);
break;
default:
_log.Info(message, exception);
break;
}
}
catch (Exception ex)
{
_log.Error($"Log error: {ex.Message}", ex);
}
}
internal void Cleanup()
{
try
{
Log("Cleaning up pipeline..", LogLevelFlags.Info);
_adapter?.HandleEvents(false);
_videoSink?.Disconnect("begin-draw", VideoSink_OnBeginDraw);
if (_pipeline != null)
{
_pipeline.Bus.Message -= OnBusMessage;
_pipeline.Bus.SyncMessage -= OnBusSyncMessage;
}
_pipeline?.SetState(State.Null);
_pipeline?.Dispose();
_mainLoop?.Quit();
GC.Collect();
GC.WaitForPendingFinalizers();
}
catch (Exception ex)
{
Log("Failed to cleanup resources", LogLevelFlags.Error, ex);
}
}
}
}
The most relevant part is extracted below:
protected bool CreateElements()
{
try
{
_videoSink = ElementFactory.Make("d3d11videosink", "d3d11videosink0");
if (_enableOverlay)
{
_videoSink["draw-on-shared-texture"] = true;
_videoSink.Connect("begin-draw", VideoSink_OnBeginDraw);
}
//SetPrimaryDecoder(_videoDecoder);
_videoConvert = ElementFactory.Make("videoconvert", "videoconvert0");
if (_videoSink == null)
{
Log($"Could not locate Direct3D11", LogLevelFlags.Error);
_videoSink = ElementFactory.Make("autovideosink", "autovideosink0");
}
_uriDecodeBin = ElementFactory.Make("udpsrc", "source");
_uriDecodeBin["port"] = 5600;
var caps = Gst.Global.CapsFromString("application/x-rtp, payload=(int)96");
_uriDecodeBin["caps"] = caps;
_depay = ElementFactory.Make("rtph264depay", "depay");
Log($"Linking rtspsrc: {_uriDecodeBin.Link(_depay)}", LogLevelFlags.Message);
_avdec = ElementFactory.Make("avdec_h264", "avdec");
//_uriDecodeBin.PadAdded += OnPadAdded;
}
catch (Exception ex)
{
Log(ex.ToString(), LogLevelFlags.Critical, ex);
return false;
}
return true;
}
This code sends an error
../libs/gst/base/gstbasesrc.c(3127): gst_base_src_loop (): /GstPipeline:pipeline0/GstUDPSrc:source:
streaming stopped, reason not-linked (-1)

Answer 1

I've finally found the answer after hours of research. I will post it in case somebody finds himself in the same situation:
the different elements need to be manually linked. The link must be AFTER the elements are added to the pipeline. In rtsp, the linking is made dynamically after the source pad is added.
private void CreatePipeline()
{
Log("Initializing Pipeline..", LogLevelFlags.Debug);
_pipeline = new Pipeline("pipeline0");
_pipeline.Bus.EnableSyncMessageEmission();
_pipeline.Bus.AddSignalWatch();
_pipeline.Bus.Message += OnBusMessage;
_pipeline.AutoFlushBus = true;
if (!_enableOverlay)
{
_pipeline.Bus.SyncMessage += OnBusSyncMessage;
}
CreateElements();
if (_enableAudio)
{
_pipeline.Add(_uriDecodeBin, _audioConvert, _videoConvert, _audioSink, _videoSink);
if (!_audioConvert.Link(_audioSink))
{
Log("Audio sink could not be linked", LogLevelFlags.Error);
return;
}
}
else
{
_pipeline.Add(_uriDecodeBin, _depay, _avdec, _videoConvert, _videoSink);
}
#### added here ####
_uriDecodeBin.Link(_depay);
_depay.Link(_avdec);
_avdec.Link(_videoConvert);
_videoConvert.Link(_videoSink);
####################
}