Segmentation of foreground from background - c++

Question

I'm currently working on a project that uses a Lacatan Banana, and I would like to know how to further separate the foreground from the background:
I already got a segmented image of it using erosion, dilation, and thresholding only. The problem is that it is still not properly segmented.
Here is my code:
cv::Mat imggray, imgthresh, fg, bgt, bg;
cv::cvtColor(src, imggray, CV_BGR2GRAY); //Grayscaling the image from RGB color space
cv::threshold(imggray, imgthresh, 0, 255, CV_THRESH_BINARY_INV | CV_THRESH_OTSU); //Create an inverted binary image from the grayscaled image
cv::erode(imgthresh, fg, cv::Mat(), cv::Point(-1, -1), 1); //erosion of the binary image and setting it as the foreground
cv::dilate(imgthresh, bgt, cv::Mat(), cv::Point(-1, -1), 4); //dilation of the binary image to reduce the background region
cv::threshold(bgt, bg, 1, 128, CV_THRESH_BINARY); //we get the background by setting the threshold to 1
cv::Mat markers = cv::Mat::zeros(src.size(), CV_32SC1); //initializing the markers with a size same as the source image and setting its data type as 32-bit Single channel
cv::add(fg, bg, markers); //setting the foreground and background as markers
cv::Mat mask = cv::Mat::zeros(markers.size(), CV_8UC1);
markers.convertTo(mask, CV_8UC1); //converting the 32-bit single channel marker to a 8-bit single channel
cv::Mat mthresh;
cv::threshold(mask, mthresh, 0, 255, CV_THRESH_BINARY | CV_THRESH_OTSU); //threshold further the mask to reduce the noise
// cv::erode(mthresh,mthresh,cv::Mat(), cv::Point(-1,-1),2);
cv::Mat result;
cv::bitwise_and(src, src, result, mthresh); //use the mask to subtrack the banana from the background
for (int x = 0; x < result.rows; x++) { //changing the black background to white
for (int y = 0; y < result.cols; y++) {
if (result.at<Vec3b>(x, y) == Vec3b(0, 0, 0)){
result.at<Vec3b>(x, y)[0] = 255;
result.at<Vec3b>(x, y)[1] = 255;
result.at<Vec3b>(x, y)[2] = 255;
}
}
}
This is my result:

Answer 1

As the background is near gray-color, try using Hue channel and Saturation channel instead of grayscale image.
You can get them easily.
cv::Mat hsv;
cv::cvtColor(src, hsv, CV_BGR2HSV);
std::vector<cv::Mat> channels;
cv::split(src, channels);
cv::Mat hue = channels[0];
cv::Mat saturation = channels[1];
// If you want to combine those channels, use this code.
cv::Mat hs = cv::Mat::zeros(src.size(), CV_8U);
for(int r=0; r<src.rows; r++) {
for(int c=0; c<src.cols; c++) {
int hp = h.at<uchar>(r,c);
int sp = s.at<uchar>(r,c);
hs.at<uchar>(r, c) = static_cast<uchar>((h+s)>>1);
}
}

Answer 2

adaptiveThreshold() should work better than just level-cut threshold(), because it does not consider absolute color levels, but rather a change in color in small area around the point being checked.
Try replacing your thresholding with adaptive one.

Answer 3

Use a top-hat instead of just erosion/dilation. It will take care of the background variations at the same time.
Then in your case a simple thresholding should be good enough to have an accurate segmentation. Else, you can couple it with a watershed.
(I will share some images asap).

Answer 4

Thanks guys, I tried to apply your advises and was able to come up with this
However as you can see there are still bits of the background,any ideas how to "clean" these further, i tried thresholding further but it would still have the bits.The Code I came up with is below and i apologize in advance if the variables and coding style is somewhat confusing didn't have the time to properly sort them.
#include <stdio.h>
#include <iostream>
#include <opencv2\core.hpp>
#include <opencv2\opencv.hpp>
#include <opencv2\highgui.hpp>
using namespace cv;
using namespace std;
Mat COLOR_MAX(Scalar(65, 255, 255));
Mat COLOR_MIN(Scalar(15, 45, 45));
int main(int argc, char** argv){
Mat src,hsv_img,mask,gray_img,initial_thresh;
Mat second_thresh,add_res,and_thresh,xor_thresh;
Mat result_thresh,rr_thresh,final_thresh;
// Load source Image
src = imread("sample11.jpg");
imshow("Original Image", src);
cvtColor(src,hsv_img,CV_BGR2HSV);
imshow("HSV Image",hsv_img);
//imwrite("HSV Image.jpg", hsv_img);
inRange(hsv_img,COLOR_MIN,COLOR_MAX, mask);
imshow("Mask Image",mask);
cvtColor(src,gray_img,CV_BGR2GRAY);
adaptiveThreshold(gray_img, initial_thresh, 255,ADAPTIVE_THRESH_GAUSSIAN_C,CV_THRESH_BINARY_INV,257,2);
imshow("AdaptiveThresh Image", initial_thresh);
add(mask,initial_thresh,add_res);
erode(add_res, add_res, Mat(), Point(-1, -1), 1);
dilate(add_res, add_res, Mat(), Point(-1, -1), 5);
imshow("Bitwise Res",add_res);
threshold(gray_img,second_thresh,170,255,CV_THRESH_BINARY_INV | CV_THRESH_OTSU);
imshow("TreshImge", second_thresh);
bitwise_and(add_res,second_thresh,and_thresh);
imshow("andthresh",and_thresh);
bitwise_xor(add_res, second_thresh, xor_thresh);
imshow("xorthresh",xor_thresh);
bitwise_or(and_thresh,xor_thresh,result_thresh);
imshow("Result image", result_thresh);
bitwise_and(add_res,result_thresh,final_thresh);
imshow("Final Thresh",final_thresh);
erode(final_thresh, final_thresh, Mat(), Point(-1,-1),5);
bitwise_and(src,src,rr_thresh,final_thresh);
imshow("Segmented Image", rr_thresh);
imwrite("Segmented Image.jpg", rr_thresh);
waitKey(0);
return 1;
}