calcHist with multiple channels in openCV - c++

I'm trying to compute a 3D 2x2x2 RGB histogram with openCV with cv::calcHist. However I don't understand how calcHist works with multiple channels. I want it to select one of two red layer of bins i, one of two green layer of bins j and one of two blue layer of bins k and store the result into the corresponding triple-indexed bin bins[i][j][k]. But that's not what it's doing.
The following loop works fine
int bin_index(const float x, unsigned int nbins)
{
return std::min(static_cast<unsigned int>(x*nbins), nbins - 1);
}
//...
for(cv::Point pixel_idx : all possible pixel positions)
{
int red_idx = bin_index(image(pixel_idx)[0], number_of_red_bins);
int green_idx = bin_index(image(pixel_idx)[0], number_of_green_bins);
int blue_idx = bin_index(image(pixel_idx)[0], number_of_blue_bins);
histogram[blue_idx * number_of_green_bins * number_of_red_bins
+ green_idx * number_of_red_bins + red_idx] += 1;
}
But the following usage of cv::calcHist yield a different result
cv::Mat block = image(rect of the patch);
const int channels[] = {0, 1, 2};
const int histSize[] = {number_of_red_bins, number_of_green_bins, number_of_blue_bins};
constexpr int histSizeDims = sizeof(histSize) / sizeof(const int);
const float range[] {0.0f, 1.001f};
const float* histRanges[] = {range, range, range};
cv::Mat result;
cv::calcHist(
&block,
1,
channels,
cv::Mat(),
result,
histSizeDims,
histSize,
histRanges,
true, false);
Can you tell me how how calcHist behaves with multi-dimentional histograms ? Thank you for your help !

Related

How to use ceres::evaluation_callbacks for inner iteration of ceres::cost functions

I am calculation visual image based on a paper and then optimize my parameters which are focal length, rotation and translation. For that reason I am creating cost function by travelling all the pixel bw real image and virtual image. In my ceres cost functions I basically subtracted normalized virtual image from normalized real image. the Virtual Image is calculated in evaluation_callback functor and the cost is calculated in cost function functor. The problem stems from cost functor. Optimization is terminated at first iteration because gradient is equals to 0. I am using ceres::Central for gradient calculation but virtual image creator functor just called once every iteration. However I need that functor to be called for f(x+h) and f(x-h) seperately.When I calculate normalized real image and normalized virtual image by 9 neighbours I have continuing iteration but every iteration takes 25 second which is not acceptable for my case. I need this evaluation_callback function but I could not make it work.
I look at the evaluation_callbacks definition. it is written that "NOTE: Evaluation callbacks are incompatible with inner iterations."
struct RcpAndFpOptimizer {
RcpAndFpOptimizer(cv::Mat &V, const cv::Mat I, int i,int j,double width, double height) : V_(V), I_(I), i_(i),
j_(j), width_(width), height_(height){}
bool operator()(const double* const fp, const double* const rotation, const double* const translation, double* residuals) const {
double intensity = V_.at<double>(j_, i_);
double tmp = (double)I_.at<double>(j_,i_)-(double)intensity;
residuals[0] = tmp;
//std::cout<<"pixels(i,j): "<<i_<<" "<<j_<<" residual: "<<residuals[0]<<std::endl;
return true;
}
const cv::Mat S_;
cv::Mat& V_;
const cv::Mat I_;
const int i_,j_;
double width_, height_;
};
virtual void PrepareForEvaluation(bool evaluateJacobians, bool newEvaluationPoint)
{
if(evaluateJacobians){
std::cout<<"evaluation jacobian is called"<<std::endl;
}
if (newEvaluationPoint)
{
// do your stuff here, e.g. calculate integral image
//Mat V(height_, width_, CV_8UC1);
std::cout<<"preperation is called"<<std::endl;
Intrinsic<double> intrinsicC = INTRINSIC_CAMERA;
Intrinsic<double> intrinsicP= {(double)fP_[0],(double)fP_[0], double(width_/2), double(height_/2), 0, 0};
//Convertion of array to point3d
Point3d bDist = Point3d(translation_[0],translation_[1], translation_[2]);
//Convertion euler array to rotation matrix
const Mat eulerAngles = (cv::Mat_<double>(3,1) << rotArray_[0], rotArray_[1], rotArray_[2]);
Mat rotM = rcpFinder::euler2rot(eulerAngles);
Mat tempVImg(height_, width_, CV_8UC1);
for (int i = 0; i < width_; ++i) {
for (int j = 0; j < height_ ; ++j) {
//std::cout<<"Virtual current x and y pixels: "<<i<<" "<<j<<std::endl;
Point3d unprojPRay = rcpFinder::unprojectPoints(Point2i(i,j),intrinsicC);
//Assigning the intensity from images
tempVImg.at<uchar>(j, i)= rcpFinder::genVirtualImg(S_, intrinsicP, bDist, unprojPRay,
planeNormalAndDistance_, rotM);
auto pixelIntensity = tempVImg.at<uchar>(Point(j, i));
//std::cout<<"pixel intensity "<< pixelIntensity<<std::endl;
}
}
//imshow("Virtual", tempVImg);
Mat integralV;
cv::integral(tempVImg, integralV);
//std::cout<<"integral image type is "<<integralV.type()<<std::endl;
rcpFinder::normalizePixelsImg(tempVImg, integralV, V_);
/*imshow("Normalized Img", V_);
waitKey(0);*/
}
}
// stuff here
const cv::Mat S_;
cv::Mat& V_;
int width_, height_;
map<int, vector<Point3d>> planeNormalAndDistance_;
double *translation_;
double* rotArray_;
double* fP_;
};
//Calling functors is like following
cv::Mat integralImgI;
cv::integral(im1, integralImgI);
cv::Mat normalizedRealImg;
rcpFinder::normalizePixelsImg(im1, integralImgI, normalizedRealImg);
Mat normalizedVirtualImg;
//ceres::CostFunction* total_cost_function = 0;
for (int i = 1; i < width-1; ++i) {
for (int j = 1; j < height-1 ; ++j) {
ceres::CostFunction* cost_function =
new ceres::NumericDiffCostFunction<RcpAndFpOptimizer, ceres::CENTRAL, 1, 1, 3, 3>(
new RcpAndFpOptimizer(normalizedVirtualImg, normalizedRealImg, i, j, width, height));
problem.AddResidualBlock(cost_function, NULL, fp, rotationArray, translation);
}
}
ceres::Solver::Options options;
options.minimizer_progress_to_stdout = true;
options.max_num_iterations = 50;
options.update_state_every_iteration = true;
options.evaluation_callback = (new evaluation_callback_functor(S, normalizedVirtualImg,width, height,
mapNormalAndDist, translation,rotationArray, fp));
ceres::Solver::Summary summary;
ceres::Solve(options, &problem, &summary);
std::cout << summary.BriefReport() << "\n";
I expected to ceres solver run more than one iteration at least and gradient should start from some values and must be decreasing by iteration.
I normalized the pizels with 9 neighbours. The current solution I have found calculating just 9 pixels of virtual image in cost functor and use them for one pixel normalization but that is too slow. I have 640x480 pixels and 9 times calculation for every pixel. Plus jacobian and gradient calculation in NumericalCOstFunction is too much. That's why I want to calculate virtual image in evaluation_callback functor and normalized it inside of that function and useing normalized image in cost functor.
Thank you for your help.
You cannot call evaluationcallback with inneriterations
https://groups.google.com/forum/#!topic/ceres-solver/zjQLIaSuAdQ

Snake active contour algorithm with C++ and OpenCV 3

I am trying to implement the snake algorithm for active contour using C++ and OpenCV 3. I am working with the version that uses the gradient descent. As base test I am trying to draw a contour of a lip. This is the base image.
This is the evolution of the contour without external forces (alpha = 0.001, beta = 3, step-size=0.3).
When I add the external force, this is the result.
As external force I have used just the edge detection with Sobel derivative.
This is the code I use for points update.
array<Mat, 2> edges = edgeMatrices(croppedImage);
const float ALPHA = 0.001, BETA = 3, GAMMA = 0.3, // Gamma is step size.
a = GAMMA * ALPHA, b = GAMMA * BETA;
const uint16_t CYCLES = 1000;
const float p = b, q = -a - 4 * b, r = 1 + 2 * a + 6 * b;
Mat pMatrix = pentadiagonalMatrix(POINTS_NUM, p, q, r).inv();
for (uint16_t i = 0; i < CYCLES; ++i) {
// Extract the x and y derivatives for current points.
auto externalForces = external(edges, x, y);
x = pMatrix * (x + GAMMA * externalForces[0]);
y = pMatrix * (y + GAMMA * externalForces[1]);
// Draw the points.
if (i % 200 == 0 && i > 0)
drawPoints(croppedImage, x, y, { 0.2f * i, 0.2f * i, 0 });
}
This is the code for computing the derivatives.
array<Mat, 2> edgeMatrices(Mat &img) {
// Convert image.
Mat gray;
cvtColor(img, gray, COLOR_BGR2GRAY);
// Apply scharr filter.
Mat grad_x, grad_y, blurred_x, blurred_y;
int scale = 1;
int delta = 0;
int ddepth = CV_16S;
int kernSize = 3;
Sobel(gray, grad_x, ddepth, 1, 0, kernSize, scale, delta, BORDER_DEFAULT);
Sobel(gray, grad_y, ddepth, 0, 1, kernSize, scale, delta, BORDER_DEFAULT);
GaussianBlur(grad_x, blurred_x, Size(5, 5), 30);
GaussianBlur(grad_y, blurred_y, Size(5, 5), 30);
return { blurred_x, blurred_y };
}
array<Mat, 2> external(array<Mat, 2> &edgeMat, Mat &x, Mat &y) {
array<Mat, 2> ext;
ext[0] = { Size{ 1, POINTS_NUM }, CV_32FC1 };
ext[1] = { Size{ 1, POINTS_NUM }, CV_32FC1 };
for (size_t i = 0; i < POINTS_NUM; ++i) {
ext[0].at<float>(0, i) = - edgeMat[0].at<short>(y.at<float>(0, i), x.at<float>(0, i));
ext[1].at<float>(0, i) = - edgeMat[1].at<short>(y.at<float>(0, i), x.at<float>(0, i));
}
return ext;
}
As you can see, the contour points converge in a very strange way and not towards the edge of the lip (that was the result I would expect).
I am not able to understand if it is an error about implementation or about tuning the parameters or it is just is normal behaviour and I misunderstood something about the algorithm.
I have some doubts on the derivative matrices, I think that they should be regularized in some way, but I am not sure which is the right one. Can someone help me?
The only implementations I have found are of the greedy method.

How do I pass an OpenCV Mat into a C++ Tensorflow graph?

In Tensorflow C++ I can load an image file into the graph using
tensorflow::Node* file_reader = tensorflow::ops::ReadFile(tensorflow::ops::Const(IMAGE_FILE_NAME, b.opts()),b.opts().WithName(input_name));
tensorflow::Node* image_reader = tensorflow::ops::DecodePng(file_reader, b.opts().WithAttr("channels", 3).WithName("png_reader"));
tensorflow::Node* float_caster = tensorflow::ops::Cast(image_reader, tensorflow::DT_FLOAT, b.opts().WithName("float_caster"));
tensorflow::Node* dims_expander = tensorflow::ops::ExpandDims(float_caster, tensorflow::ops::Const(0, b.opts()), b.opts());
tensorflow::Node* resized = tensorflow::ops::ResizeBilinear(dims_expander, tensorflow::ops::Const({input_height, input_width},b.opts().WithName("size")),b.opts());
For an embedded application I would like to instead pass an OpenCV Mat into this graph.
How would I convert the Mat to a tensor that could be used as input to tensorflow::ops::Cast or tensorflow::ops::ExpandDims?
It's not directly from a CvMat, but you can see an example of how to initialize a Tensor from an in-memory array in the TensorFlow Android example:
https://github.com/tensorflow/tensorflow/blob/0.6.0/tensorflow/examples/android/jni/tensorflow_jni.cc#L173
You would start off by creating a new tensorflow::Tensor object, with something like this (all code untested):
tensorflow::Tensor input_tensor(tensorflow::DT_FLOAT,
tensorflow::TensorShape({1, height, width, depth}));
This creates a Tensor object with float values, with a batch size of 1, and a size of widthxheight, and with depth channels. For example a 128 wide by 64 high image with 3 channels would pass in a shape of {1, 64, 128, 3}. The batch size is just used when you need to pass in multiple images in a single call, and for simple uses you can leave it as 1.
Then you would get the underlying array behind the tensor using a line like this:
auto input_tensor_mapped = input_tensor.tensor<float, 4>();
The input_tensor_mapped object is an interface to the data in your newly-created tensor, and you can then copy your own data into it. Here I'm assuming you've set source_data as a pointer to your source data, for example:
const float* source_data = some_structure.imageData;
You can then loop through your data and copy it over:
for (int y = 0; y < height; ++y) {
const float* source_row = source_data + (y * width * depth);
for (int x = 0; x < width; ++x) {
const float* source_pixel = source_row + (x * depth);
for (int c = 0; c < depth; ++c) {
const float* source_value = source_pixel + c;
input_tensor_mapped(0, y, x, c) = *source_value;
}
}
}
There are obvious opportunities to optimize this naive approach, and I don't have sample code on hand to show how to deal with the OpenCV side of getting the source data, but hopefully this is helpful to get you started.
Here is complete example to read and feed:
Mat image;
image = imread("flowers.jpg", CV_LOAD_IMAGE_COLOR);
cv::resize(image, image, cv::Size(input_height, input_width), 0, 0, CV_INTER_CUBIC);
int depth = 3;
tensorflow::Tensor input_tensor(tensorflow::DT_FLOAT,
tensorflow::TensorShape({1, image.rows, image.cols, depth}));
for (int y = 0; y < image.rows; y++) {
for (int x = 0; x < image.cols; x++) {
Vec3b pixel = image.at<Vec3b>(y, x);
input_tensor_mapped(0, y, x, 0) = pixel.val[2]; //R
input_tensor_mapped(0, y, x, 1) = pixel.val[1]; //G
input_tensor_mapped(0, y, x, 2) = pixel.val[0]; //B
}
}
auto result = Sub(root.WithOpName("subtract_mean"), input_tensor, {input_mean});
ClientSession session(root);
TF_CHECK_OK(session.Run({result}, out_tensors));
I had tried to run inception model on the opencv Mat file and following code worked for me https://gist.github.com/kyrs/9adf86366e9e4f04addb. Although there are some issue with integration of opencv and tensorflow. Code worked without any issue for .png files but failed to load .jpg and .jpeg. You can follow this for more info https://github.com/tensorflow/tensorflow/issues/1924
Tensor convertMatToTensor(Mat &input)
{
int height = input.rows;
int width = input.cols;
int depth = input.channels();
Tensor imgTensor(tensorflow::DT_FLOAT, tensorflow::TensorShape({height, width, depth}));
float* p = imgTensor.flat<float>().data();
Mat outputImg(height, width, CV_32FC3, p);
input.convertTo(outputImg, CV_32FC3);
return imgTensor;
}

Average values of a MAT channel

I want to obtain the average values of a MAT and MatND variable, just to estimate the sharpness and brightness. However, I have been facing real issues with the vague values I have been encountering. I tried my best, but am still confused. I really do not know, if am doing the right thing.
calcHist(&src_yuv,1,channels,Mat(),hist,1,histSize,ranges,true,false);
Size d = hist.size();
rows = d.height;
cols = d.width;
for(int k=0;k<hbins;k++)
{
for(int l=0;l<sbins;l++)
{
total = total + hist.at<float>(k,l);
}
}
brightness = total/(rows*cols);
Here , am trying to calculate the histogram of the luma channel of src_yuv, which is in YUV format and average the values. Am I doing it the right way? If I change the datatype within <' '>, ranging from uchar to long int, am ending up with different values, which is understandable. But I dunno which is the right data type to use. Moreover, should I loop it within hbins,sbins or rows, cols? Please help me. am stuck at this for a long time.
Laplacian(src_gray,dst,ddepth,kernel_size,scale,delta,BORDER_DEFAULT);
Size s = dst.size();
rows = s.height;
cols = s.width;
total = 0;
max = 0;
for(int k=0;k<rows;k++)
{
for(int l=0;l<cols;l++)
{
total = total + dst.at<>(k,l);
}
}
average = total/(rows*cols);
What is the exact way to compute average in the above case? Could you please help me here? I tried different datatypes, starting from in for the mat, and long int for the total and averages. Its a gray scale image, and the result in the laplacian convoluted image.
convert the input src_yuv to BGR before calcHist and you will get the desired output with the same code.
EDIT: for YUV:-
out = imread("Lena.jpg");
out.convertTo(out, CV_RGB2YCrCb);
MatND hist;
int hbins = 30, sbins = 32;
int histSize[] = {hbins, sbins};
float hranges[] = { 0, 180 };
float sranges[] = { 0, 256 };
int channels[] = {0,1,2};
const float* ranges[] = { hranges, sranges };
calcHist( &out, 1, channels, Mat(),
hist, 2, histSize, ranges,
true,
false );
Size d = hist.size();
int rows = d.height;
int cols = d.width;
float total;
float brightness;
for(int k=0;k<hbins;k++)
{
for(int l=0;l<sbins;l++)
{
total = total + hist.at<float>(k,l);
}
}
brightness = total/(rows*cols);
this gives me brightness to be 246.895

Can normal maps be generated from a texture?

If I have a texture, is it then possible to generate a normal-map for this texture, so it can be used for bump-mapping?
Or how are normal maps usually made?
Yes. Well, sort of. Normal maps can be accurately made from height-maps. Generally, you can also put a regular texture through and get decent results as well. Keep in mind there are other methods of making a normal map, such as taking a high-resolution model, making it low resolution, then doing ray casting to see what the normal should be for the low-resolution model to simulate the higher one.
For height-map to normal-map, you can use the Sobel Operator. This operator can be run in the x-direction, telling you the x-component of the normal, and then the y-direction, telling you the y-component. You can calculate z with 1.0 / strength where strength is the emphasis or "deepness" of the normal map. Then, take that x, y, and z, throw them into a vector, normalize it, and you have your normal at that point. Encode it into the pixel and you're done.
Here's some older incomplete-code that demonstrates this:
// pretend types, something like this
struct pixel
{
uint8_t red;
uint8_t green;
uint8_t blue;
};
struct vector3d; // a 3-vector with doubles
struct texture; // a 2d array of pixels
// determine intensity of pixel, from 0 - 1
const double intensity(const pixel& pPixel)
{
const double r = static_cast<double>(pPixel.red);
const double g = static_cast<double>(pPixel.green);
const double b = static_cast<double>(pPixel.blue);
const double average = (r + g + b) / 3.0;
return average / 255.0;
}
const int clamp(int pX, int pMax)
{
if (pX > pMax)
{
return pMax;
}
else if (pX < 0)
{
return 0;
}
else
{
return pX;
}
}
// transform -1 - 1 to 0 - 255
const uint8_t map_component(double pX)
{
return (pX + 1.0) * (255.0 / 2.0);
}
texture normal_from_height(const texture& pTexture, double pStrength = 2.0)
{
// assume square texture, not necessarily true in real code
texture result(pTexture.size(), pTexture.size());
const int textureSize = static_cast<int>(pTexture.size());
for (size_t row = 0; row < textureSize; ++row)
{
for (size_t column = 0; column < textureSize; ++column)
{
// surrounding pixels
const pixel topLeft = pTexture(clamp(row - 1, textureSize), clamp(column - 1, textureSize));
const pixel top = pTexture(clamp(row - 1, textureSize), clamp(column, textureSize));
const pixel topRight = pTexture(clamp(row - 1, textureSize), clamp(column + 1, textureSize));
const pixel right = pTexture(clamp(row, textureSize), clamp(column + 1, textureSize));
const pixel bottomRight = pTexture(clamp(row + 1, textureSize), clamp(column + 1, textureSize));
const pixel bottom = pTexture(clamp(row + 1, textureSize), clamp(column, textureSize));
const pixel bottomLeft = pTexture(clamp(row + 1, textureSize), clamp(column - 1, textureSize));
const pixel left = pTexture(clamp(row, textureSize), clamp(column - 1, textureSize));
// their intensities
const double tl = intensity(topLeft);
const double t = intensity(top);
const double tr = intensity(topRight);
const double r = intensity(right);
const double br = intensity(bottomRight);
const double b = intensity(bottom);
const double bl = intensity(bottomLeft);
const double l = intensity(left);
// sobel filter
const double dX = (tr + 2.0 * r + br) - (tl + 2.0 * l + bl);
const double dY = (bl + 2.0 * b + br) - (tl + 2.0 * t + tr);
const double dZ = 1.0 / pStrength;
math::vector3d v(dX, dY, dZ);
v.normalize();
// convert to rgb
result(row, column) = pixel(map_component(v.x), map_component(v.y), map_component(v.z));
}
}
return result;
}
There's probably many ways to generate a Normal map, but like others said, you can do it from a Height Map, and 3d packages like XSI/3dsmax/Blender/any of them can output one for you as an image.
You can then output and RGB image with the Nvidia plugin for photoshop, an algorithm to convert it or you might be able to output it directly from those 3d packages with 3rd party plugins.
Be aware that in some case, you might need to invert channels (R, G or B) from the generated normal map.
Here's some resources link with examples and more complete explanation:
http://developer.nvidia.com/object/photoshop_dds_plugins.html
http://en.wikipedia.org/wiki/Normal_mapping
http://www.vrgeo.org/fileadmin/VRGeo/Bilder/VRGeo_Papers/jgt2002normalmaps.pdf
I don't think normal maps are generated from a texture. they are generated from a model.
just as texturing allows you to define complex colour detail with minimal polys (as opposed to just using millions of ploys and just vertex colours to define the colour on your mesh)
A normal map allows you to define complex normal detail with minimal polys.
I believe normal maps are usually generated from a higher res mesh, and then is used with a low res mesh.
I'm sure 3D tools, such as 3ds max or maya, as well as more specific tools will do this for you. unlike textures, I don't think they are usually done by hand.
but they are generated from the mesh, not the texture.
I suggest starting with OpenCV, due to its richness in algorithms. Here's one I wrote that iteratively blurs the normal map and weights those to the overall value, essentially creating more of a topological map.
#define ROW_PTR(img, y) ((uchar*)((img).data + (img).step * y))
cv::Mat normalMap(const cv::Mat& bwTexture, double pStrength)
{
// assume square texture, not necessarily true in real code
int scale = 1.0;
int delta = 127;
cv::Mat sobelZ, sobelX, sobelY;
cv::Sobel(bwTexture, sobelX, CV_8U, 1, 0, 13, scale, delta, cv::BORDER_DEFAULT);
cv::Sobel(bwTexture, sobelY, CV_8U, 0, 1, 13, scale, delta, cv::BORDER_DEFAULT);
sobelZ = cv::Mat(bwTexture.rows, bwTexture.cols, CV_8UC1);
for(int y=0; y<bwTexture.rows; y++) {
const uchar *sobelXPtr = ROW_PTR(sobelX, y);
const uchar *sobelYPtr = ROW_PTR(sobelY, y);
uchar *sobelZPtr = ROW_PTR(sobelZ, y);
for(int x=0; x<bwTexture.cols; x++) {
double Gx = double(sobelXPtr[x]) / 255.0;
double Gy = double(sobelYPtr[x]) / 255.0;
double Gz = pStrength * sqrt(Gx * Gx + Gy * Gy);
uchar value = uchar(Gz * 255.0);
sobelZPtr[x] = value;
}
}
std::vector<cv::Mat>planes;
planes.push_back(sobelX);
planes.push_back(sobelY);
planes.push_back(sobelZ);
cv::Mat normalMap;
cv::merge(planes, normalMap);
cv::Mat originalNormalMap = normalMap.clone();
cv::Mat normalMapBlurred;
for (int i=0; i<3; i++) {
cv::GaussianBlur(normalMap, normalMapBlurred, cv::Size(13, 13), 5, 5);
addWeighted(normalMap, 0.4, normalMapBlurred, 0.6, 0, normalMap);
}
addWeighted(originalNormalMap, 0.3, normalMapBlurred, 0.7, 0, normalMap);
return normalMap;
}