RealSense OpenCV Depth Image Too Dark - c++

Folks,
I have a realsense SR300, but when I display my depth image in a opencv window, it looks too dark. How can I fix this? When I run the realsense examples the images look good, but the examples use OpenGL. But I need OpenCV for my projects. Here is my code:
int main(int argc, char ** argv)
{
// realsense camera setup
rs::log_to_console(rs::log_severity::warn);
// Create a context object. This object owns the handles to all connected realsense devices
rs::context ctx;
if (ctx.get_device_count() == 0)
{
throw std::runtime_error("No device detected. Is it plugged in?");
}
// Access the first available RealSense device
rs::device * dev = ctx.get_device(0);
// Configure depth to run at VGA resolution at 30 frames per second
dev->enable_stream(rs::stream::depth, 640, 480, rs::format::z16, 30);
rs::intrinsics depth_intrin;
rs::format depth_format;
depth_intrin = dev->get_stream_intrinsics(rs::stream::depth);
depth_format = dev->get_stream_format(rs::stream::depth);
cv::namedWindow("Send Display Image", CV_WINDOW_AUTOSIZE);
/* Set callbacks prior to calling start(). */
auto depth_callback = [depth_intrin, depth_format](rs::frame f)
{
cv::Mat image(cv::Size(640, 480), CV_16UC1,
(void*)f.get_data(), cv::Mat::AUTO_STEP);
cv::imshow("Send Display Image", image);
cv::waitKey(1000/80);
};
/* callback to grab depth fream and publish it. */
dev->set_frame_callback(rs::stream::depth, depth_callback);
// Start streaming
dev->start();
While(1)
{
}
return 0;
}
I am not sure why my image is so dark. I want it to look something like the kinect or the Xtion when I run openni_launch from ROS

Edit:
The normalized function below produces some flickering:
I suspect that is due to the maximal depth value flickering.
The minimal depth value is always 0 as this value is used when the depth is invalid and thus the depth range becomes false.
Instead you should use this:
void make_depth_histogram(const Mat &depth, Mat &normalized_depth) {
normalized_depth = Mat(depth.size(), CV_8U);
int width = depth.cols, height = depth.rows;
static uint32_t histogram[0x10000];
memset(histogram, 0, sizeof(histogram));
for(int i = 0; i < height; ++i) {
for (int j = 0; j < width; ++j) {
++histogram[depth.at<ushort>(i,j)];
}
}
for(int i = 2; i < 0x10000; ++i) histogram[i] += histogram[i-1]; // Build a cumulative histogram for the indices in [1,0xFFFF]
for(int i = 0; i < height; ++i) {
for (int j = 0; j < width; ++j) {
if (uint16_t d = depth.at<ushort>(i,j)) {
int f = histogram[d] * 255 / histogram[0xFFFF]; // 0-255 based on histogram location
normalized_depth.at<uchar>(i,j) = static_cast<uchar>(f);
} else {
normalized_depth.at<uchar>(i,j) = 0;
}
}
}
}
What you observe is because the depth stream is coded on 16 bits (rs::stream::z16) whereas when displayed only 8 bits will be used.
You can normalized your depth map:
double min, max;
minMaxLoc(depth, &min, &max);
Mat depth_normalized;
double alpha = 255.0/(max-min);
depth.convertTo(depth_normalized, CV_8U, alpha, -min*alpha);
Or use a kind of colormap to display the depth: make_depth_histogram().
Full demo code:
inline void make_depth_histogram(const Mat &depth, Mat &color_depth) {
color_depth = Mat(depth.size(), CV_8UC3);
int width = depth.cols, height = depth.rows;
static uint32_t histogram[0x10000];
memset(histogram, 0, sizeof(histogram));
for(int i = 0; i < height; ++i) {
for (int j = 0; j < width; ++j) {
++histogram[depth.at<ushort>(i,j)];
}
}
for(int i = 2; i < 0x10000; ++i) histogram[i] += histogram[i-1]; // Build a cumulative histogram for the indices in [1,0xFFFF]
for(int i = 0; i < height; ++i) {
for (int j = 0; j < width; ++j) {
if (uint16_t d = depth.at<ushort>(i,j)) {
int f = histogram[d] * 255 / histogram[0xFFFF]; // 0-255 based on histogram location
color_depth.at<Vec3b>(i,j) = Vec3b(f, 0, 255 - f);
} else {
color_depth.at<Vec3b>(i,j) = Vec3b(0, 5, 20);
}
}
}
}
int main(int argc, char *argv[]) {
// Create a context object. This object owns the handles to all connected realsense devices
rs::context ctx;
// Access the first available RealSense device
rs::device * dev = ctx.get_device(0);
// Configure Infrared stream to run at VGA resolution at 30 frames per second
dev->enable_stream(rs::stream::depth, 640, 480, rs::format::z16, 30);
// Start streaming
dev->start();
// Camera warmup - Dropped several first frames to let auto-exposure stabilize
for(int i = 0; i < 30; i++)
dev->wait_for_frames();
// Creating OpenCV Matrix from a color image
Mat depth(Size(640, 480), CV_16U, (void*)dev->get_frame_data(rs::stream::depth), Mat::AUTO_STEP);
// Create a color depth
Mat color_depth;
make_depth_histogram(depth, color_depth);
// Create a normalized depth
double min, max;
minMaxLoc(depth, &min, &max);
Mat depth_normalized;
double alpha = 255.0/(max-min);
depth.convertTo(depth_normalized, CV_8U, alpha, -min*alpha);
// Display in a GUI
imshow("Display normalized depth", depth_normalized);
imshow("Display color depth", color_depth);
waitKey(0);
return 0;
}

The only solution I found to this problem which gives satisfactory results is the following:
Save the image as a PNG file. (PNG supports saving 16-bit images)
Use matplotlib to view it in a colored map:
#!/usr/bin/python3
import numpy as np
import cv2
import sys
from matplotlib import pyplot as plt
def printCoordinates(event):
x,y = event.xdata,event.ydata
if x != None:
print("X : ",x," Y: ",y," Value = ",img[np.int(y),np.int(x)])
img = cv2.imread(sys.argv[1],cv2.CV_16UC1)
#img = img/65535
fig = plt.figure()
plt.imshow(img,cmap='nipy_spectral')
cid = fig.canvas.mpl_connect('button_press_event',printCoordinates)
plt.colorbar()
plt.show()
The button_press_event is to print the exact pixel value on the pixel clicked.
RGB Image:
Corresponding Depth Image:

Related

Creating transparent overlay for camera in OpenCV c++

I'm trying to create an overlay for a camera feed, and I want the overlay to be blurred, and about 50% transparent. One way of solving this is to copy each frame from the camera, draw onto it, and merge them together using addWeighted. This doesn't work for me because the blur effect takes up so much resources the output fps drops to 10.
Another solution I thought up is to create the overlay once (It's static after all, why recreate it every frame?) and merge it with the camera feed. However the resulting video gets noticeably darker when doing this, seemingly because the overlay mat refuses to be transparent.
(*cap) >> frameOriginal;
orientationBackground = cv::Mat(frameOriginal.rows, frameOriginal.cols,
frameOriginal.type(), cv::Scalar(0,0,0,0));
cv::Mat headingBackground;
orientationBackground.copyTo(headingBackground);
cv::Point layerpt1(1800, 675);
cv::Point layerpt2(1850, 395);
cv::rectangle(orientationBackground, layerpt1, layerpt2,
cv::Scalar(255,80,80), CV_FILLED, CV_AA);
cv::blur(orientationBackground, orientationBackground, cv::Size(7,30));
double alpha = 0.5;
addWeighted(orientationBackground, alpha, frameOriginal, 1-alpha, 0, frameOriginal);
The before(left) and after(right) adding the overlay:
I'm using OpenCV 3.10 on windows x64 btw
Try this:
cv::Mat input = cv::imread("C:/StackOverflow/Input/Lenna.png");
// define your overlay position
cv::Rect overlay = cv::Rect(400, 100, 50, 300);
float maxFadeRange = 20;
// precompute fading mask:
cv::Size size = input.size();
cv::Mat maskTmp = cv::Mat(size, CV_8UC1, cv::Scalar(255));
// draw black area where overlay is placed, because distance transform will assume 0 = distance 0
cv::rectangle(maskTmp, overlay, 0, -1);
cv::Mat distances;
cv::distanceTransform(maskTmp, distances, CV_DIST_L1, CV_DIST_MASK_PRECISE);
cv::Mat blendingMask = cv::Mat(size, CV_8UC1);
// create blending mask from
for (int j = 0; j < blendingMask.rows; ++j)
for (int i = 0; i < blendingMask.cols; ++i)
{
float dist = distances.at<float>(j, i);
float maskVal = (maxFadeRange - dist)/maxFadeRange * 255; // this will scale from 0 (maxFadeRange distance) to 255 (0 distance)
if (maskVal < 0) maskVal = 0;
blendingMask.at<unsigned char>(j, i) = maskVal;
}
cv::Scalar overlayColor = cv::Scalar(255, 0, 0);
// color a whole image in overlay colors so that rect and blurred area are coverered by that color
cv::Mat overlayImage = cv::Mat(size, CV_8UC3, overlayColor);
// this has created all the stuff that is expensive and can be precomputed for a fixes roi overlay
float transparency = 0.5f; // 50% transparency
// now for each image: just do this:
cv::Mat result = input.clone();
for (int j = 0; j < blendingMask.rows; ++j)
for (int i = 0; i < blendingMask.cols; ++i)
{
const unsigned char & blendingMaskVal = blendingMask.at<unsigned char>(j, i);
if (blendingMaskVal) // only blend in areas where blending is necessary
{
float alpha = transparency * blendingMaskVal / 255.0f;
result.at<cv::Vec3b>(j, i) = (alpha)*overlayImage.at<cv::Vec3b>(j, i) + (1.0f - alpha)*result.at<cv::Vec3b>(j, i);
}
}
Giving this result with 50% transparency and a fading range of 20 pixels:
and this is 20% transparency (variable value = 0.2f) and 100 pixels fading:

opencv how to visualize a non-rectangular region (roi) in a performant way

I have an image (cv::Mat) and a ROI that can be seen as a mask. I want to show the original image with the ROI blended over it.
My mask is smaller than my origiginal image: each element represents a block in the image. Suppose my mask is this (note that my mask is NOT a rectangle)
0 0 1
1 1 1
0 0 0
then I would like to have the parts where (mask == 1) untouched and the rest blended with a color. This is the code I have
cv::Mat blocks = image.clone;
uint npixcol = 32;
uint npixrow = 32;
for (uint ri = 0; ri < 480; ++ri)
for (uint ci = 0; ci < 640; ++ci)
{
if (mask[ri * 640 + ci])
cv::rectangle(blocks, cv::Rect(ci * npixcol, ri * npixrow, npixcol, npixrow), cv::Scalar(0, 0, 0), CV_FILLED, 8, 0);
}
cv::addWeighted(image, 0.5, blocks, 0.5, 0, image, -1);
How can I do this without the extra "clone" command since that is not very performant...
to make it more clear; this is an example of what I want (the color doesn't really matter)!
Is your mask of constant colour? Assuming mask is the same dimension as the image(you can easily scale it) :-
//Manually instead of addWeighted()
for (uint ri = 0; ri < 480; ++ri)
for (uint ci = 0; ci < 640; ++ci)
{
if (mask[ri * 640 + ci])
{
image.at<uchar>(ri,ci) [0] = image.at<uchar>(ri,ci) [0] * weight_blue;
image.at<uchar>(ri,ci) [1] = image.at<uchar>(ri,ci) [1] * weight_green;
image.at<uchar>(ri,ci) [2] = image.at<uchar>(ri,ci) [2] * weight_red;
}
}
Based on your comment, if you can make a mask with the same dimensions as the original image, you could directly modify original image pixel values using iterators. Here is a standalone example:
#include <cstdlib>
#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>
int
main(int argc, char *argv[])
{
cv::Mat image = cv::imread(argv[1]);
cv::Mat mask = cv::Mat::zeros(image.size(), CV_8U);
// let's put some 1 in my test mask.
cv::Mat roi = mask(cv::Rect(0,0,mask.cols/2, mask.rows/2));
roi = 1;
cv::Vec3b blue(255,0,0); // (B,G,R)
float alpha = 0.5;
// Let's have fun with iterators
cv::MatConstIterator_<unsigned char> maskIter = mask.begin<unsigned char>();
const cv::MatConstIterator_<unsigned char> maskIterEnd = mask.end<unsigned char>();
cv::MatIterator_<cv::Vec3b> imageIter = image.begin<cv::Vec3b>();
for (; maskIter != maskIterEnd; ++maskIter, ++imageIter) {
if (*maskIter) {// mask == 1
*imageIter = (1-alpha)*(*imageIter) + alpha*blue; // same as addWeighted
}
}
cv::namedWindow("image", 0);
cv::imshow("image", image);
cv::waitKey(0);
return EXIT_SUCCESS;
}
Basically you want to have a check if your are inside the roi. Then it should return a pointer to your original image. If your are not inside you want to have some kind of colour.
Your could do that with your own wrapper for Mat.
MyMat::at(int x, int y){
if(inRoi(x,y)){
return original.at(x,y);
else
return color(0,0,0);
}
I don't think you can point a subImage of an image onto another image. (That means that i don't think you can redirect the pixel in your blue image onto your original image)
Mat blueImage;
Rect roi;
Mat roiInImage = blueImage(roi);
roiInImage.redirect = originalImage(roi); //don't think something like this is possible

OpenCV: color extraction based on Gaussian mixture model

I am trying to use opencv EM algorithm to do color extraction.I am using the following code based on example in opencv documentation:
cv::Mat capturedFrame ( height, width, CV_8UC3 );
int i, j;
int nsamples = 1000;
cv::Mat samples ( nsamples, 2, CV_32FC1 );
cv::Mat labels;
cv::Mat img = cv::Mat::zeros ( height, height, CV_8UC3 );
img = capturedFrame;
cv::Mat sample ( 1, 2, CV_32FC1 );
CvEM em_model;
CvEMParams params;
samples = samples.reshape ( 2, 0 );
for ( i = 0; i < N; i++ )
{
//from the training samples
cv::Mat samples_part = samples.rowRange ( i*nsamples/N, (i+1)*nsamples/N);
cv::Scalar mean (((i%N)+1)*img.rows/(N1+1),((i/N1)+1)*img.rows/(N1+1));
cv::Scalar sigma (30,30);
cv::randn(samples_part,mean,sigma);
}
samples = samples.reshape ( 1, 0 );
//initialize model parameters
params.covs = NULL;
params.means = NULL;
params.weights = NULL;
params.probs = NULL;
params.nclusters = N;
params.cov_mat_type = CvEM::COV_MAT_SPHERICAL;
params.start_step = CvEM::START_AUTO_STEP;
params.term_crit.max_iter = 300;
params.term_crit.epsilon = 0.1;
params.term_crit.type = CV_TERMCRIT_ITER|CV_TERMCRIT_EPS;
//cluster the data
em_model.train ( samples, Mat(), params, &labels );
cv::Mat probs;
probs = em_model.getProbs();
cv::Mat weights;
weights = em_model.getWeights();
cv::Mat modelIndex = cv::Mat::zeros ( img.rows, img.cols, CV_8UC3 );
for ( i = 0; i < img.rows; i ++ )
{
for ( j = 0; j < img.cols; j ++ )
{
sample.at<float>(0) = (float)j;
sample.at<float>(1) = (float)i;
int response = cvRound ( em_model.predict ( sample ) );
modelIndex.data [ modelIndex.cols*i + j] = response;
}
}
My question here is:
Firstly, I want to extract each model, here totally five, then store those corresponding pixel values in five different matrix. In this case, I could have five different colors seperately. Here I only obtained their indexes, is there any way to achieve their corresponding colors here? To make it easy, I can start from finding the dominant color based on these five GMMs.
Secondly, here my sample datapoints are "100", and it takes about nearly 3 seconds for them. But I want to do all these things in no more than 30 milliseconds. I know OpenCV background extraction, which is using GMM, performs really fast, below 20ms, that means, there must be a way for me to do all these within 30 ms for all 600x800=480000 pixels. I found predict function is the most time consuming one.
First Question:
In order to do color extraction you first need to train the EM with your input pixels. After that you simply loop over all the input pixels again and use predict() to classify each of them. I've attached a small example that utilizes EM for foreground/background separation based on colors. It shows you how to extract the dominant color (mean) of each gaussian and how to access the original pixel color.
#include <opencv2/opencv.hpp>
int main(int argc, char** argv) {
cv::Mat source = cv::imread("test.jpg");
//ouput images
cv::Mat meanImg(source.rows, source.cols, CV_32FC3);
cv::Mat fgImg(source.rows, source.cols, CV_8UC3);
cv::Mat bgImg(source.rows, source.cols, CV_8UC3);
//convert the input image to float
cv::Mat floatSource;
source.convertTo(floatSource, CV_32F);
//now convert the float image to column vector
cv::Mat samples(source.rows * source.cols, 3, CV_32FC1);
int idx = 0;
for (int y = 0; y < source.rows; y++) {
cv::Vec3f* row = floatSource.ptr<cv::Vec3f > (y);
for (int x = 0; x < source.cols; x++) {
samples.at<cv::Vec3f > (idx++, 0) = row[x];
}
}
//we need just 2 clusters
cv::EMParams params(2);
cv::ExpectationMaximization em(samples, cv::Mat(), params);
//the two dominating colors
cv::Mat means = em.getMeans();
//the weights of the two dominant colors
cv::Mat weights = em.getWeights();
//we define the foreground as the dominant color with the largest weight
const int fgId = weights.at<float>(0) > weights.at<float>(1) ? 0 : 1;
//now classify each of the source pixels
idx = 0;
for (int y = 0; y < source.rows; y++) {
for (int x = 0; x < source.cols; x++) {
//classify
const int result = cvRound(em.predict(samples.row(idx++), NULL));
//get the according mean (dominant color)
const double* ps = means.ptr<double>(result, 0);
//set the according mean value to the mean image
float* pd = meanImg.ptr<float>(y, x);
//float images need to be in [0..1] range
pd[0] = ps[0] / 255.0;
pd[1] = ps[1] / 255.0;
pd[2] = ps[2] / 255.0;
//set either foreground or background
if (result == fgId) {
fgImg.at<cv::Point3_<uchar> >(y, x, 0) = source.at<cv::Point3_<uchar> >(y, x, 0);
} else {
bgImg.at<cv::Point3_<uchar> >(y, x, 0) = source.at<cv::Point3_<uchar> >(y, x, 0);
}
}
}
cv::imshow("Means", meanImg);
cv::imshow("Foreground", fgImg);
cv::imshow("Background", bgImg);
cv::waitKey(0);
return 0;
}
I've tested the code with the following image and it performs quite good.
Second Question:
I've noticed that the maximum number of clusters has a huge impact on the performance. So it's better to set this to a very conservative value instead of leaving it empty or setting it to the number of samples like in your example. Furthermore the documentation mentions an iterative procedure to repeatedly optimize the model with less-constrained parameters. Maybe this gives you some speed-up. To read more please have a look at the docs inside the sample code that is provided for train() here.

How to push an OpenCV image viewing window into a QT GUI with Visual studio?

I want to create a GUI with 2 rectangles for viewing videos (one where you see the input video, one where you see the post-processed video).
I want it to be integrated into a QT-made GUI, but I want these video areas to be populated from OpenCV, as an alternative to OpenCV's cv::nameWindow method.
How can I do this?
The basic workflow to do what you desire is:
Open the video with OpenCV API (cvCreateFileCapture, for example)
Grab IplImage frames from video (cvQueryFrame)
Convert them to QImage (see attached code bellow)
Show QImage on within a QLabel (QLabel::setPixmap and QPixmap::fromImage)
Loop the frame update (using a QTimer, for example, with video framerate)
Code to convert IplImage to QImage (assuming RGB32Bits images):
QImage *IplImageToQImage(IplImage *input)
{
if (!input)
return 0;
QImage image(input->width, input->height, QImage::Format_RGB32);
uchar* pBits = image.bits();
int nBytesPerLine = image.bytesPerLine();
for (int n = 0; n < input->height; n++)
{
for (int m = 0; m < input->width; m++)
{
CvScalar s = cvGet2D(input, n, m);
QRgb value = qRgb((uchar)s.val[2], (uchar)s.val[1], (uchar)s.val[0]);
uchar* scanLine = pBits + n * nBytesPerLine;
((uint*)scanLine)[m] = value;
}
}
return image;
}
The understanding of the code above should be straightforward. Any doubts just let us know.
This "low level" option allows you to manipulate each individual frame before displaying it. If you just want to display a video via Qt, you can use the Phonon framework.
Here is code that converts a cv::Mat into a QImage. Methods are for 24bit RGB or grayscale floating point, respectively.
QImage Mat2QImage(const cv::Mat3b &src) {
QImage dest(src.cols, src.rows, QImage::Format_ARGB32);
for (int y = 0; y < src.rows; ++y) {
const cv::Vec3b *srcrow = src[y];
QRgb *destrow = (QRgb*)dest.scanLine(y);
for (int x = 0; x < src.cols; ++x) {
destrow[x] = qRgba(srcrow[x][2], srcrow[x][1], srcrow[x][0], 255);
}
}
return dest;
}
QImage Mat2QImage(const cv::Mat_<double> &src)
{
double scale = 255.0;
QImage dest(src.cols, src.rows, QImage::Format_ARGB32);
for (int y = 0; y < src.rows; ++y) {
const double *srcrow = src[y];
QRgb *destrow = (QRgb*)dest.scanLine(y);
for (int x = 0; x < src.cols; ++x) {
unsigned int color = srcrow[x] * scale;
destrow[x] = qRgba(color, color, color, 255);
}
}
return dest;
}
Then, you can use the QImage inside a Qt widget. See borges' answer.

Detect clip arts or vectorial images using OpenCV in C++

I have a process that detects similar images using SURF and I want to add a check to know which images are real camera photos and which ones are vectorial images like logos of map-screenshots.
Examples:
Photo: http://images.gta-travel.com/HH/Images/J/TYO/TYO-NEW3-8.jpg
Logo: http://estaticos.transhotel.com/img/fotos/hoteles/000137/hft000137578_005.jpg
Logo: http://live.viajesurbis.com/vuweb/content/fichashotel/13127/HOTEL_13127_2.jpg
I tried looking at the grey histogram (and color histogram) but nothing gives me enough info to know which one are vectorials or not.
Ok, solved it, the next code is cleaning the histogram, getting all colors in grey scale and counting the different colors. Maybe in the future I will test if working with the components histograms improves the algorithm.
CvHistogram* wImage::getHistogram() {
IplImage* gray = cvCreateImage(cvGetSize(this->image), 8, 1);
CvHistogram* hist;
int hist_size = 256;
float range[] = {0, 256};
float* ranges[] = {range};
cvCvtColor(this->image, gray, CV_RGB2GRAY);
hist = cvCreateHist(1, &hist_size, CV_HIST_ARRAY, ranges, 1);
cvCalcHist(&gray, hist, 0, NULL);
return hist;
}
bool wImage::isVectorial() {
CvHistogram* hist = this->getHistogram();
int height = 240;
float max_value = 0, min_value = 0;
cvGetMinMaxHistValue(hist, &min_value, &max_value);
int total = 0;
int colors = 0;
float value;
int normalized;
for(int i=0; i < 256; i++){
value = cvQueryHistValue_1D(hist, i);
normalized = cvRound(value * height / max_value);
if(normalized < 2 || normalized > 230) {
continue;
}
colors++;
total += normalized;
}
if((total < 500 && colors < 100) || (total < 1000 && colors < 85)) {
return true;
}
return false;
}