Hole-Filling Filter in OpenCV C++ - c++

Question

I have a basic implementation of hole-filling filter as shown below:
#include <iostream>
#include <opencv2/opencv.hpp>
int main(int argc, char** argv)
{
// please note that the depthImg is (720 x 576) 8UC1
// let's make a smaller one for testing
uchar flatten[6 * 8] = { 140, 185, 48, 235, 201, 192, 131, 57,
55, 87, 82, 0, 6, 201, 0, 38,
6, 239, 82, 142, 46, 33, 172, 72,
133, 0, 232, 226, 66, 59, 10, 204,
214, 123, 202, 100, 0, 32, 6, 147,
105, 191, 50, 21, 87, 117, 118, 244};
cv::Mat depthImg = cv::Mat(6, 8, CV_8UC1, flatten);
// please ignore the border pixels in this case
for (int i = 1; i < depthImg.cols - 1; i++) {
for (int j = 1; j < depthImg.rows - 1; j++) {
unsigned short sumNonZeroAdjs = 0;
uchar countNonZeroAdjs = 0;
if (depthImg.at<uchar>(j, i) == 0) {
uchar iMinus1 = depthImg.at<uchar>(j, i - 1);
uchar iPlus1 = depthImg.at<uchar>(j, i + 1);
uchar jMinus1 = depthImg.at<uchar>(j - 1, i);
uchar jPlus1 = depthImg.at<uchar>(j + 1, i);
if (iMinus1 != 0) {
sumNonZeroAdjs += iMinus1;
countNonZeroAdjs++;
}
if (iPlus1 != 0) {
sumNonZeroAdjs += iPlus1;
countNonZeroAdjs++;
}
if (jMinus1 != 0) {
sumNonZeroAdjs += jMinus1;
countNonZeroAdjs++;
}
if (jPlus1 != 0) {
sumNonZeroAdjs += jPlus1;
countNonZeroAdjs++;
}
depthImg.at<uchar>(j, i) = sumNonZeroAdjs / countNonZeroAdjs;
}
}
}
std::cout << depthImg << std::endl;
return 0;
}
// prints the following:
[140, 185, 48, 235, 201, 192, 131, 57;
55, 87, 82, 116, 6, 201, 135, 38;
6, 239, 82, 142, 46, 33, 172, 72;
133, 181, 232, 226, 66, 59, 10, 204;
214, 123, 202, 100, 71, 32, 6, 147;
105, 191, 50, 21, 87, 117, 118, 244]
The above filter computes an average of adjacent pixels to fill the 0 pixels. The output from this implementation is satisfactory. However, as we can see, the above prototype is not elegant and painfully slow.
I am looking for a similar logic (using adjacent pixels to fill 0 pixels) but faster (execution time) hole-filling filter inbuilt in OpenCV
PS: I am using OpenCV v4.2.0 on Ubuntu 20.04 LTS.
Update 1
Based on the suggestions, I designed pointer style access. Complete code is shown below:
#include <iostream>
#include <opencv2/opencv.hpp>
void inPlaceHoleFillingExceptBorderPtrStyle(cv::Mat& img) {
typedef uchar T;
T* ptr = img.data;
size_t elemStep = img.step / sizeof(T);
for (int i = 1; i < img.rows - 1; i++) {
for (int j = 1; j < img.cols - 1; j++) {
T& curr = ptr[i * elemStep + j];
if (curr != 0) {
continue;
}
ushort sumNonZeroAdjs = 0;
uchar countNonZeroAdjs = 0;
T iM1 = ptr[(i - 1) * elemStep + j];
T iP1 = ptr[(i + 1) * elemStep + j];
T jM1 = ptr[i * elemStep + (j - 1)];
T jP1 = ptr[i * elemStep + (j + 1)];
if (iM1 != 0) {
sumNonZeroAdjs += iM1;
countNonZeroAdjs++;
}
if (iP1 != 0) {
sumNonZeroAdjs += iP1;
countNonZeroAdjs++;
}
if (jM1 != 0) {
sumNonZeroAdjs += jM1;
countNonZeroAdjs++;
}
if (jP1 != 0) {
sumNonZeroAdjs += jP1;
countNonZeroAdjs++;
}
if (countNonZeroAdjs > 0) {
curr = sumNonZeroAdjs / countNonZeroAdjs;
}
}
}
}
void inPlaceHoleFillingExceptBorder(cv::Mat& img) {
typedef uchar T;
for (int i = 1; i < img.cols - 1; i++) {
for (int j = 1; j < img.rows - 1; j++) {
ushort sumNonZeroAdjs = 0;
uchar countNonZeroAdjs = 0;
if (img.at<T>(j, i) != 0) {
continue;
}
T iM1 = img.at<T>(j, i - 1);
T iP1 = img.at<T>(j, i + 1);
T jM1 = img.at<T>(j - 1, i);
T jP1 = img.at<T>(j + 1, i);
if (iM1 != 0) {
sumNonZeroAdjs += iM1;
countNonZeroAdjs++;
}
if (iP1 != 0) {
sumNonZeroAdjs += iP1;
countNonZeroAdjs++;
}
if (jM1 != 0) {
sumNonZeroAdjs += jM1;
countNonZeroAdjs++;
}
if (jP1 != 0) {
sumNonZeroAdjs += jP1;
countNonZeroAdjs++;
}
if (countNonZeroAdjs > 0) {
img.at<T>(j, i) = sumNonZeroAdjs / countNonZeroAdjs;
}
}
}
}
int main(int argc, char** argv) {
// please note that the img is (720 x 576) 8UC1
// let's make a smaller one for testing
// clang-format off
uchar flatten[6 * 8] = { 140, 185, 48, 235, 201, 192, 131, 57,
55, 87, 82, 0, 6, 201, 0, 38,
6, 239, 82, 142, 46, 33, 172, 72,
133, 0, 232, 226, 66, 59, 10, 204,
214, 123, 202, 100, 0, 32, 6, 147,
105, 191, 50, 21, 87, 117, 118, 244};
// clang-format on
cv::Mat img = cv::Mat(6, 8, CV_8UC1, flatten);
cv::Mat img1 = img.clone();
cv::Mat img2 = img.clone();
inPlaceHoleFillingExceptBorderPtrStyle(img1);
inPlaceHoleFillingExceptBorder(img2);
return 0;
}
/*** expected output
[140, 185, 48, 235, 201, 192, 131, 57;
55, 87, 82, 116, 6, 201, 135, 38;
6, 239, 82, 142, 46, 33, 172, 72;
133, 181, 232, 226, 66, 59, 10, 204;
214, 123, 202, 100, 71, 32, 6, 147;
105, 191, 50, 21, 87, 117, 118, 244]
***/
Update 2
Based on the suggestion, the point style code is further improved as shown below:
void inPlaceHoleFillingExceptBorderImpv(cv::Mat& img) {
typedef uchar T;
size_t elemStep = img.step1();
const size_t margin = 1;
for (size_t i = margin; i < img.rows - margin; ++i) {
T* ptr = img.data + i * elemStep;
for (size_t j = margin; j < img.cols - margin; ++j, ++ptr) {
T& curr = ptr[margin];
if (curr != 0) {
continue;
}
T& north = ptr[margin - elemStep];
T& south = ptr[margin + elemStep];
T& east = ptr[margin + 1];
T& west = ptr[margin - 1];
ushort sumNonZeroAdjs = 0;
uchar countNonZeroAdjs = 0;
if (north != 0) {
sumNonZeroAdjs += north;
countNonZeroAdjs++;
}
if (south != 0) {
sumNonZeroAdjs += south;
countNonZeroAdjs++;
}
if (east != 0) {
sumNonZeroAdjs += east;
countNonZeroAdjs++;
}
if (west != 0) {
sumNonZeroAdjs += west;
countNonZeroAdjs++;
}
if (countNonZeroAdjs > 0) {
curr = sumNonZeroAdjs / countNonZeroAdjs;
}
}
}
}

Answer 1

There are three parts: 1) find the zeros, 2) find the mean, and 3) fill the found zeros with mean. So:
/****
* in-place fill zeros with the mean of the surrounding neighborhoods
***/
void fillHoles(Mat gray){
// find the zeros
Mat mask = (gray == 0);
// find the mean with filter2d
Mat kernel = (Mat_<double>(3,3) <<
1/8, 1/8, 1/8
1/8, 0 , 1/8
1/8, 1/8, 1/8
);
Mat avg;
cv::filter2d(gray, avg, CV_8U, kernel)
// then fill the zeros, only where indicated by `mask`
cv::bitwise_or(gray, avg, gray, mask);
}
Note I just realize that this is plainly taking the average, not the non-zero average. For that operation, you might want to do two filters, one for the sum, one for the non-zero counts, then divide the two:
// find the neighborhood sum with filter2d
Mat kernel = (Mat_<double>(3,3) <<
1, 1, 1
1, 0, 1
1, 1, 1
);
Mat sums;
cv::filter2d(gray, sums, CV_64F, kernel);
// find the neighborhood count with filter2d
Mat counts;
cv::filter2d(gray!=0, counts, CV_64F, kernel);
counts /= 255; // because gray!=0 returns 255 where true
// force counts to 1 if 0, so we can divide later
cv::max(counts, 1, counts);
Mat out;
cv::divide(sums, counts, out);
out.convertTo(gray, CV_8U);

Answer 2

#QuangHoang advised a fantastic kernel-based approach. Unfortunately, the result is not matching with the expected output.
Therefore, based on various comments from #CrisLuengo, I managed to design time efficient version of the filter, as shown below:
void inPlaceHoleFillingExceptBorderFast(cv::Mat& img) {
typedef uchar T;
const size_t elemStep = img.step1();
const size_t margin = 1;
for (size_t i = margin; i < img.rows - margin; ++i) {
T* ptr = img.data + i * elemStep;
for (size_t j = margin; j < img.cols - margin; ++j, ++ptr) {
T& curr = ptr[margin];
if (curr != 0) {
continue;
}
T& east = ptr[margin + 1];
T& west = ptr[margin - 1];
T& north = ptr[margin - elemStep];
T& south = ptr[margin + elemStep];
// convert to 0/1 and sum them up
uchar count = static_cast<bool>(north) + static_cast<bool>(south) +
static_cast<bool>(east) + static_cast<bool>(west);
// we do not want to divide by 0
if (count > 0) {
curr = (north + south + east + west) / count;
}
}
}
}