Using gstreamer cameraundistort plugin with calculated distortion coefficient - gstreamer

Question

How can i use distortion coefficient
already calculated using a specific python code ( dist [[7.33183864e+03 1.52395233e-01 2.53983049e+00 2.20587897e+00 2.61869010e-06]] ) with cameraundistort plugin.
I have seen the following example pipeline in the documentation :
gst-launch-1.0 -v v4l2src ! videoconvert ! cameraundistort settings= ! autovideosink
But i dindn't get what kind of argument i should give to settings , is this
where i should put my calculated distortion coefficient or my matrix?

Answer 1

New answer:
I eventually found how to do this from the command line. You will need to edit the values in the DATA variable with your own matrix values:
Tested on Ubuntu 20.04. You should see a distorted videotestsrc when compared to running the standard gst-launch-1.0 -vv videotestsrc ! videoconvert ! autovideosink
DATA="<?xml version=\"1.0\"?><opencv_storage><cameraMatrix type_id=\"opencv-matrix\"><rows>3</rows><cols>3</cols><dt>f</dt><data>2.85762378e+03 0. 1.93922961e+03 0. 2.84566113e+03 1.12195850e+03 0. 0. 1.</data></cameraMatrix><distCoeffs type_id=\"opencv-matrix\"><rows>5</rows><cols>1</cols><dt>f</dt><data>-6.14039421e-01 4.00045455e-01 1.47132971e-03 2.46772077e-04 -1.20407566e-01</data></distCoeffs></opencv_storage>"
gst-launch-1.0 -vv videotestsrc ! videoconvert ! cameraundistort settings="$DATA" ! videoconvert ! autovideosink
Original analysis:
I am trying to find the answer to this too.
Based on the following code here: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/blob/324e55a3cdd7c6cef24356fd626deee5fba343df/ext/opencv/gstcameracalibrate.cpp
There is a snippet:
/* set settings property */
g_free (calib->settings);
calib->settings =
camera_serialize_undistort_settings (calib->cameraMatrix,
calib->distCoeffs);
which leads us here to this function: https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/blob/324e55a3cdd7c6cef24356fd626deee5fba343df/ext/opencv/camerautils.cpp
gchar *
camera_serialize_undistort_settings (cv::Mat & cameraMatrix,
cv::Mat & distCoeffs)
{
cv::FileStorage fs (".xml", cv::FileStorage::WRITE + cv::FileStorage::MEMORY);
fs << "cameraMatrix" << cameraMatrix;
fs << "distCoeffs" << distCoeffs;
std::string buf = fs.releaseAndGetString ();
return g_strdup (buf.c_str ());
}
In order to find out what the output of this looks like I wrote the following code:
//
// g++ -o out main.cpp `pkg-config --cflags --libs opencv4 glib-2.0`
//
// https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/blob/324e55a3cdd7c6cef24356fd626deee5fba343df/ext/opencv/camerautils.cpp
//
#include <glib.h>
#include <opencv2/opencv.hpp>
#include <iostream>
gchar *
camera_serialize_undistort_settings (cv::Mat & cameraMatrix,
cv::Mat & distCoeffs)
{
cv::FileStorage fs (".xml", cv::FileStorage::WRITE + cv::FileStorage::MEMORY);
fs << "cameraMatrix" << cameraMatrix;
fs << "distCoeffs" << distCoeffs;
std::string buf = fs.releaseAndGetString ();
return g_strdup (buf.c_str ());
}
int main() {
float camera_matrix_data[9] = { 2.8576236805989838e+03, 0.0, 1.9392296211154603e+03,
0.0, 2.8456611938133528e+03, 1.1219585111073625e+03, 0.0,
0.0, 1.0 };
cv::Mat camera_matrix = cv::Mat(3, 3, CV_32F, camera_matrix_data);
float dist_coefficients_data[5] = {-6.1403941690071129e-01, 4.0004545823187132e-01,
1.4713297080556295e-03, 2.4677208656274745e-04,
-1.2040756970288242e-01};
cv::Mat dist_coefficients = cv::Mat(5, 1, CV_32F, dist_coefficients_data);
std::cout << camera_serialize_undistort_settings(camera_matrix, dist_coefficients);
return 0;
}
and I got this as an output, which as far as I can tell is valid for consumption by the camera_deserialize_undistort_settings() function:
<?xml version="1.0"?>
<opencv_storage>
<cameraMatrix type_id="opencv-matrix">
<rows>3</rows>
<cols>3</cols>
<dt>f</dt>
<data>
2.85762378e+03 0. 1.93922961e+03 0. 2.84566113e+03 1.12195850e+03 0.
0. 1.</data></cameraMatrix>
<distCoeffs type_id="opencv-matrix">
<rows>5</rows>
<cols>1</cols>
<dt>f</dt>
<data>
-6.14039421e-01 4.00045455e-01 1.47132971e-03 2.46772077e-04
-1.20407566e-01</data></distCoeffs>
</opencv_storage>
Unfortunately I cannot overcome the final stumbling block of feeding this to the command line.
edit:
Non command line solution:
Hopefully the following will help somebody - it's a bit hacked together due to time constraints. Basically I gave up on the command line and found an example of how to run a gstreamer pipeline through code. I hacked in the pipeline and was able to feed the parameters in.
Calibration
//
// g++ play.cpp -o play `pkg-config --cflags --libs gstreamer-1.0 opencv4`
//
#include <gst/gst.h>
#include <opencv2/opencv.hpp>
//
// USB Webcam
//
const char *string2 = " \
gst-launch-1.0 -v v4l2src device=/dev/video0 ! videoconvert ! cameraundistort name=undist ! cameracalibrate name=cal ! ximagesink \
";
gchar *
camera_serialize_undistort_settings (cv::Mat & cameraMatrix,
cv::Mat & distCoeffs)
{
cv::FileStorage fs (".xml", cv::FileStorage::WRITE + cv::FileStorage::MEMORY);
fs << "cameraMatrix" << cameraMatrix;
fs << "distCoeffs" << distCoeffs;
std::string buf = fs.releaseAndGetString ();
return g_strdup (buf.c_str ());
}
int
main (int argc, char *argv[])
{
gchar value[100] = {0};
GstElement *pipeline;
GstElement *filesrc;
GstMessage *msg;
GstBus *bus;
GError *error = NULL;
gst_init (&argc, &argv);
pipeline = gst_parse_launch (string2, &error);
if (!pipeline) {
g_print ("Parse error: %s\n", error->message);
exit (1);
}
gst_element_set_state (pipeline, GST_STATE_PLAYING);
bus = gst_element_get_bus (pipeline);
/* wait until we either get an EOS or an ERROR message. Note that in a real
* program you would probably not use gst_bus_poll(), but rather set up an
* async signal watch on the bus and run a main loop and connect to the
* bus's signals to catch certain messages or all messages */
msg = gst_bus_poll (bus, (GstMessageType) ( GST_MESSAGE_EOS | GST_MESSAGE_ERROR ), -1);
switch (GST_MESSAGE_TYPE (msg)) {
case GST_MESSAGE_EOS: {
g_print ("EOS\n");
break;
}
case GST_MESSAGE_ERROR: {
GError *err = NULL; /* error to show to users */
gchar *dbg = NULL; /* additional debug string for developers */
gst_message_parse_error (msg, &err, &dbg);
if (err) {
filesrc = gst_bin_get_by_name (GST_BIN (pipeline), "undist");
gchar *val;
g_object_get (filesrc, "settings", &val, NULL);
g_print("Data:\n");
g_print ("%s", val);
g_object_unref (filesrc);
g_print("\nDone\n");
g_printerr ("ERROR: %s\n", err->message);
g_error_free (err);
}
if (dbg) {
g_printerr ("[Debug details: %s]\n", dbg);
g_free (dbg);
}
}
default:
g_printerr ("Unexpected message of type %d", GST_MESSAGE_TYPE (msg));
break;
}
gst_message_unref (msg);
gst_element_set_state (pipeline, GST_STATE_NULL);
gst_object_unref (pipeline);
gst_object_unref (bus);
return 0;
}
You calibrate as normal using a checkerboard as per the opencv recommended way.
On closing, this will give an output such as this:
<?xml version="1.0"?>
<opencv_storage>
<cameraMatrix type_id="opencv-matrix">
<rows>3</rows>
<cols>3</cols>
<dt>d</dt>
<data>
3.3860583405334381e+02 0. 2.9082938038777263e+02 0.
3.3860583405334381e+02 1.5389814447354675e+02 0. 0. 1.</data></cameraMatrix>
<distCoeffs type_id="opencv-matrix">
<rows>5</rows>
<cols>1</cols>
<dt>d</dt>
<data>
-7.3120392823523372e-01 6.2624904277006888e-01
-1.0205399762451621e-02 5.1857216532169093e-03
-2.0130033675444331e-01</data></distCoeffs>
</opencv_storage>
You can then put this in to another program and load it as such:
Running
//
// g++ play.cpp -o play `pkg-config --cflags --libs gstreamer-1.0 opencv4`
//
#include <gst/gst.h>
#include <opencv2/opencv.hpp>
const char *string2 = " \
gst-launch-1.0 -v v4l2src device=/dev/video0 ! videoconvert ! cameraundistort name=undist1 ! ximagesink \
";
gchar *
camera_serialize_undistort_settings (cv::Mat & cameraMatrix,
cv::Mat & distCoeffs)
{
cv::FileStorage fs (".xml", cv::FileStorage::WRITE + cv::FileStorage::MEMORY);
fs << "cameraMatrix" << cameraMatrix;
fs << "distCoeffs" << distCoeffs;
std::string buf = fs.releaseAndGetString ();
return g_strdup (buf.c_str ());
}
int
main (int argc, char *argv[])
{
float camera_matrix_data[9] = { 3.3860583405334381e+02, 0., 2.9082938038777263e+02, 0.,
3.3860583405334381e+02, 1.5389814447354675e+02, 0., 0., 1. };
cv::Mat camera_matrix = cv::Mat(3, 3, CV_32F, camera_matrix_data);
float dist_coefficients_data[5] = {-7.3120392823523372e-01, 6.2624904277006888e-01,
-1.0205399762451621e-02, 5.1857216532169093e-03,
-2.0130033675444331e-01};
cv::Mat dist_coefficients = cv::Mat(5, 1, CV_32F, dist_coefficients_data);
gchar * dist = camera_serialize_undistort_settings(camera_matrix, dist_coefficients);
GstElement *pipeline;
GstElement *filesrc;
GstMessage *msg;
GstBus *bus;
GError *error = NULL;
gst_init (&argc, &argv);
pipeline = gst_parse_launch (string2, &error);
if (!pipeline) {
g_print ("Parse error: %s\n", error->message);
exit (1);
}
// Feed in our settings
filesrc = gst_bin_get_by_name (GST_BIN (pipeline), "undist1");
g_object_set (filesrc, "settings", dist, NULL);
g_object_unref (filesrc);
gst_element_set_state (pipeline, GST_STATE_PLAYING);
bus = gst_element_get_bus (pipeline);
/* wait until we either get an EOS or an ERROR message. Note that in a real
* program you would probably not use gst_bus_poll(), but rather set up an
* async signal watch on the bus and run a main loop and connect to the
* bus's signals to catch certain messages or all messages */
msg = gst_bus_poll (bus, (GstMessageType) ( GST_MESSAGE_EOS | GST_MESSAGE_ERROR ), -1);
switch (GST_MESSAGE_TYPE (msg)) {
case GST_MESSAGE_EOS: {
g_print ("EOS\n");
break;
}
case GST_MESSAGE_ERROR: {
GError *err = NULL; /* error to show to users */
gchar *dbg = NULL; /* additional debug string for developers */
gst_message_parse_error (msg, &err, &dbg);
if (err) {
g_printerr ("ERROR: %s\n", err->message);
g_error_free (err);
}
if (dbg) {
g_printerr ("[Debug details: %s]\n", dbg);
g_free (dbg);
}
}
default:
g_printerr ("Unexpected message of type %d", GST_MESSAGE_TYPE (msg));
break;
}
gst_message_unref (msg);
gst_element_set_state (pipeline, GST_STATE_NULL);
gst_object_unref (pipeline);
gst_object_unref (bus);
return 0;
}