Using Tensorflow Lite C, Digit Classifier - c++

Using Tensorflow Lite, on Android,
Image classification was successful.
Example: https://github.com/tensorflow/examples/tree/master/lite/codelabs/digit_classifier
core : https://github.com/tensorflow/examples/blob/master/lite/codelabs/digit_classifier/android/finish/app/src/main/java/org/tensorflow/lite/codelabs/digitclassifier/DigitClassifier.kt
I want to convert this project to C.
The image(.jpg) will be read using opencv.
The model file is (.tflite)
Give me some advice.
This is my program code.
#include <iostream>
#include <iomanip>
#include <fstream>
#include "tensorflow/lite/interpreter.h"
#include "tensorflow/lite/kernels/register.h"
#include "tensorflow/lite/model.h"
#include "tensorflow/lite/optional_debug_tools.h"
#include "opencv2/opencv.hpp"
using namespace std;
typedef cv::Point3_<float> Pixel;
const uint WIDTH = 28;
const uint HEIGHT = 28;
const uint CHANNEL = 3;
const uint OUTDIM = 10;
void normalize(Pixel &pixel){
pixel.x = (pixel.x / 255.0);
pixel.y = (pixel.y / 255.0);
pixel.z = (pixel.z / 255.0);
}
int main(){
std::vector<std::string> labels;
auto file_name="labels.txt";
std::ifstream input( file_name );
for( std::string line; getline( input, line ); )
{
labels.push_back( line);
}
// read image file
cv::Mat img = cv::imread("sample2.jpg");
cv::Mat inputImg;
img.convertTo(inputImg, CV_32FC3);
cv::cvtColor(inputImg, inputImg, cv::COLOR_BGR2RGB);
// normalize to -1 & 1
Pixel* pixel = inputImg.ptr<Pixel>(0,0);
const Pixel* endPixel = pixel + inputImg.cols * inputImg.rows;
for (; pixel != endPixel; pixel++)
normalize(*pixel);
// resize image as model input
cv::resize(inputImg, inputImg, cv::Size(WIDTH, HEIGHT));
// create model
std::unique_ptr<tflite::FlatBufferModel> model =
tflite::FlatBufferModel::BuildFromFile("mnist.tflite");
tflite::ops::builtin::BuiltinOpResolver resolver;
std::unique_ptr<tflite::Interpreter> interpreter;
tflite::InterpreterBuilder(*model.get(), resolver)(&interpreter);
interpreter->AllocateTensors();
float* inputLayer = interpreter->typed_input_tensor<float>(0);
float* inputImg_ptr = inputImg.ptr<float>(0);
memcpy(inputLayer, inputImg.ptr<float>(0),
WIDTH * HEIGHT * CHANNEL * sizeof(float));
interpreter->Invoke();
float* outputLayer = interpreter->typed_output_tensor<float>(0);
// TODO
return 0;
}
Summary of this question : Android -> C Convert
The result I wantPrediction Result : 6Confidence: 0~1
.tflite model file is here

Related

Function not found tensorflow lite get_top_n

I'm trying to use an "example" code for c++ on qt. In this example, there's a function "get_top_n" from tflite::label_image, in tensorflow/lite/examples/label_image/get_top_n.h. But, qt creator doesn't find the function.
Error: main.cpp:104 (and 107): erreur : no matching function for call to 'get_top_n'
What am I doing wrong here ?
#include <fstream>
#include <string>
#include <vector>
#include <opencv2/opencv.hpp>
#include "tensorflow/lite/interpreter.h"
#include "tensorflow/lite/kernels/register.h"
#include "tensorflow/lite/string_util.h"
#include "tensorflow/lite/examples/label_image/get_top_n.h"
#include "tensorflow/lite/model.h"
std::vector<std::string> load_labels(std::string labels_file)
{
std::ifstream file(labels_file.c_str());
if (!file.is_open())
{
fprintf(stderr, "unable to open label file\n");
exit(-1);
}
std::string label_str;
std::vector<std::string> labels;
while (std::getline(file, label_str))
{
if (label_str.size() > 0)
labels.push_back(label_str);
}
file.close();
return labels;
}
int main(int argc, char *argv[])
{
// Get Model label and input image
if (argc != 4)
{
fprintf(stderr, "TfliteClassification.exe modelfile labels image\n");
exit(-1);
}
const char *modelFileName = argv[1];
const char *labelFile = argv[2];
const char *imageFile = argv[3];
// Load Model
auto model = tflite::FlatBufferModel::BuildFromFile(modelFileName);
if (model == nullptr)
{
fprintf(stderr, "failed to load model\n");
exit(-1);
}
// Initiate Interpreter
std::unique_ptr<tflite::Interpreter> interpreter;
tflite::ops::builtin::BuiltinOpResolver resolver;
tflite::InterpreterBuilder(*model, resolver)(&interpreter);
if (interpreter == nullptr)
{
fprintf(stderr, "Failed to initiate the interpreter\n");
exit(-1);
}
if (interpreter->AllocateTensors() != kTfLiteOk)
{
fprintf(stderr, "Failed to allocate tensor\n");
exit(-1);
}
// Configure the interpreter
interpreter->SetAllowFp16PrecisionForFp32(true);
interpreter->SetNumThreads(1);
// Get Input Tensor Dimensions
int input = interpreter->inputs()[0];
auto height = interpreter->tensor(input)->dims->data[1];
auto width = interpreter->tensor(input)->dims->data[2];
auto channels = interpreter->tensor(input)->dims->data[3];
// Load Input Image
cv::Mat image;
auto frame = cv::imread(imageFile);
if (frame.empty())
{
fprintf(stderr, "Failed to load iamge\n");
exit(-1);
}
// Copy image to input tensor
cv::resize(frame, image, cv::Size(width, height), cv::INTER_NEAREST);
memcpy(interpreter->typed_input_tensor<unsigned char>(0), image.data, image.total() * image.elemSize());
// Inference
std::chrono::steady_clock::time_point start, end;
start = std::chrono::steady_clock::now();
interpreter->Invoke();
end = std::chrono::steady_clock::now();
auto inference_time = std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
// Get Output
int output = interpreter->outputs()[0];
TfLiteIntArray *output_dims = interpreter->tensor(output)->dims;
auto output_size = output_dims->data[output_dims->size - 1];
std::vector<std::pair<float, int>> top_results;
float threshold = 0.01f;
switch (interpreter->tensor(output)->type)
{
case kTfLiteInt32:
tflite::label_image::get_top_n<float>(interpreter->typed_output_tensor<float>(0), output_size, 1, threshold, &top_results, kTfLiteFloat32);
break;
case kTfLiteUInt8:
tflite::label_image::get_top_n<uint8_t>(interpreter->typed_output_tensor<uint8_t>(0), output_size, 1, threshold, &top_results, kTfLiteUInt8);
break;
default:
fprintf(stderr, "cannot handle output type\n");
exit(-1);
}
// Print inference ms in input image
cv::putText(frame, "Infernce Time in ms: " + std::to_string(inference_time), cv::Point(10, 30), cv::FONT_HERSHEY_SIMPLEX, 0.8, cv::Scalar(0, 0, 255), 2);
// Load Labels
auto labels = load_labels(labelFile);
// Print labels with confidence in input image
for (const auto &result : top_results)
{
const float confidence = result.first;
const int index = result.second;
std::string output_txt = "Label :" + labels[index] + " Confidence : " + std::to_string(confidence);
cv::putText(frame, output_txt, cv::Point(10, 60), cv::FONT_HERSHEY_SIMPLEX, 0.8, cv::Scalar(0, 0, 255), 2);
}
// Display image
cv::imshow("Output", frame);
cv::waitKey(0);
return 0;
}
The lines affected :
104: tflite::label_image::get_top_n<float>(interpreter->typed_output_tensor<float>(0), output_size, 1, threshold, &top_results, kTfLiteFloat32);
107: tflite::label_image::get_top_n<uint8_t>(interpreter->typed_output_tensor<uint8_t>(0), output_size, 1, threshold, &top_results, kTfLiteUInt8);
Content of tensorflow/lite/examples/label_image/get_top_n.h:
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. ...*/
#ifndef TENSORFLOW_LITE_EXAMPLES_LABEL_IMAGE_GET_TOP_N_H_
#define TENSORFLOW_LITE_EXAMPLES_LABEL_IMAGE_GET_TOP_N_H_
#include "tensorflow/lite/examples/label_image/get_top_n_impl.h"
namespace tflite {
namespace label_image {
template <class T>
void get_top_n(T* prediction, int prediction_size, size_t num_results,
float threshold, std::vector<std::pair<float, int>>* top_results,
TfLiteType input_type);
// explicit instantiation so that we can use them otherwhere
template void get_top_n<float>(float*, int, size_t, float,
std::vector<std::pair<float, int>>*, TfLiteType);
template void get_top_n<int8_t>(int8_t*, int, size_t, float,
std::vector<std::pair<float, int>>*,
TfLiteType);
template void get_top_n<uint8_t>(uint8_t*, int, size_t, float,
std::vector<std::pair<float, int>>*,
TfLiteType);
} // namespace label_image
} // namespace tflite
#endif // TENSORFLOW_LITE_EXAMPLES_LABEL_IMAGE_GET_TOP_N_H_
Content of tensorflow/lite/examples/label_image/get_top_n_impl.h:
/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. ...*/
#ifndef TENSORFLOW_LITE_EXAMPLES_LABEL_IMAGE_GET_TOP_N_IMPL_H_
#define TENSORFLOW_LITE_EXAMPLES_LABEL_IMAGE_GET_TOP_N_IMPL_H_
#include <algorithm>
#include <functional>
#include <queue>
#include "tensorflow/lite/c/common.h"
namespace tflite {
namespace label_image {
extern bool input_floating;
// Returns the top N confidence values over threshold in the provided vector,
// sorted by confidence in descending order.
template <class T>
void get_top_n(T* prediction, int prediction_size, size_t num_results,
float threshold, std::vector<std::pair<float, int>>* top_results,
TfLiteType input_type) {
// Will contain top N results in ascending order.
std::priority_queue<std::pair<float, int>, std::vector<std::pair<float, int>>,
std::greater<std::pair<float, int>>>
top_result_pq;
const long count = prediction_size; // NOLINT(runtime/int)
float value = 0.0;
for (int i = 0; i < count; ++i) {
switch (input_type) {
case kTfLiteFloat32:
value = prediction[i];
break;
case kTfLiteInt8:
value = (prediction[i] + 128) / 256.0;
break;
case kTfLiteUInt8:
value = prediction[i] / 255.0;
break;
default:
break;
}
// Only add it if it beats the threshold and has a chance at being in
// the top N.
if (value < threshold) {
continue;
}
top_result_pq.push(std::pair<float, int>(value, i));
// If at capacity, kick the smallest value out.
if (top_result_pq.size() > num_results) {
top_result_pq.pop();
}
}
// Copy to output vector and reverse into descending order.
while (!top_result_pq.empty()) {
top_results->push_back(top_result_pq.top());
top_result_pq.pop();
}
std::reverse(top_results->begin(), top_results->end());
}
} // namespace label_image
} // namespace tflite
#endif // TENSORFLOW_LITE_EXAMPLES_LABEL_IMAGE_GET_TOP_N_IMPL_H_
Github link

How do I properly use the OpenH264 Usage Code Example for Encoding?

I have an image and want to encode it with OpenH264.
So far this is the code I derived from their wiki:
#include <fstream>
#include <iterator>
#include <iostream>
#include <codec_api.h> //standard api for openh264
//additional libaries used by sample code
#include <codec_app_def.h>
#include <codec_def.h>
#include <codec_ver.h>
#include <assert.h>
#include <vector>
#include <cstring>
int main()
{
//parameter values
int width = 1920;
int height = 1080;
int framerate = 60;
int bitrate = 5000000;
int total_num = 500; //what does this value do?
//end parameter values
//Read in the File from bmp
std::vector<char> buf; //to store the image information
std::basic_ifstream<char> file("/home/megamol/Git/h264_sample/build/test.bmp", std::ios::binary); //opens bitstream to source
buf = std::vector<char>((std::istreambuf_iterator<char>(file)), std::istreambuf_iterator<char>()); // reads in data to the vector
std::cout << "sizeof buf: " << buf.size() << std::endl;
//Step 1: set up Encoder
ISVCEncoder* encoder_; //declaration of encoder pointer
int rv = WelsCreateSVCEncoder (&encoder_);
//Step 2: initialize with basic parameter
SEncParamBase param;
memset(&param, 0, sizeof (SEncParamBase));
param.iUsageType = EUsageType::SCREEN_CONTENT_REAL_TIME;
param.fMaxFrameRate = framerate;
param.iPicWidth = width;
param.iPicHeight = height;
param.iTargetBitrate = bitrate; //default value of example
encoder_->Initialize(&param);
//Step 3: set video format
int videoFormat = videoFormatI420;
encoder_->SetOption (ENCODER_OPTION_DATAFORMAT, &videoFormat);
//Step 4: encocode and store output bitstream
int frameSize = width * height * 3 / 2;
buf.resize(frameSize);
SFrameBSInfo info;
std::vector<char> compressedData;
memset (&info, 0, sizeof (SFrameBSInfo));
SSourcePicture pic;
memset (&pic, 0, sizeof (SSourcePicture));
pic.iPicWidth = width;
pic.iPicHeight = height;
pic.iColorFormat = videoFormatI420;
pic.iStride[0] = pic.iPicWidth;
pic.iStride[1] = pic.iStride[2] = pic.iPicWidth >> 1;
pic.pData[0] = reinterpret_cast<unsigned char*>(&buf[0]);
pic.pData[1] = pic.pData[0] + width * height;
pic.pData[2] = pic.pData[1] + (width * height >> 2);
//encodes the frame
rv = encoder_->EncodeFrame (&pic, &info); // encodes the Frame
//encoding done encoded Frame should be stored in &info
//begin decoding block
ISVCDecoder *pSvcDecoder;
unsigned char *pBuf= &info;
return 0;
}
I'm not entirely sure whether this is the correct usage of OpenH264 but I'm also not sure how to test it properly.
Now the code example is kind of poorly documented.
What is BufferedData buf; for example? I get that that's supposed to be the input but what is that type? Like how do I load my test.bmp as BufferedData? I don't think that I'm doing that correctly yet.
Another thing I'm pretty confused about is how do I access the output after the encoding? In the example it just says //output bitstream and nothing about saving this output anywhere. I thought the output was info like it says in the codec_api.h header file:
/**
* #brief Encode one frame
* #param kpSrcPic the pointer to the source luminance plane
* chrominance data:
* CbData = kpSrc + m_iMaxPicWidth * m_iMaxPicHeight;
* CrData = CbData + (m_iMaxPicWidth * m_iMaxPicHeight)/4;
* the application calling this interface needs to ensure the data validation between the location
* #param pBsInfo output bit stream
* #return 0 - success; otherwise -failed;
*/
virtual int EXTAPI EncodeFrame (const SSourcePicture* kpSrcPic, SFrameBSInfo* pBsInfo) = 0;
But apparently it only saves informations about the output. I'm just really confused about all of this.
Based on https://github.com/cisco/openh264/blob/master/codec/console/enc/src/welsenc.cpp
#include <codec_api.h>
#include <cassert>
#include <cstring>
#include <vector>
#include <fstream>
#include <iostream>
//Tested with OpenCV 3.3
#include <opencv2/core.hpp>
#include <opencv2/imgcodecs.hpp>
#include <opencv2/highgui.hpp>
#include <opencv2/imgproc.hpp>
using namespace std;
using namespace cv;
int main()
{
ISVCEncoder *encoder_ = nullptr;
int rv = WelsCreateSVCEncoder (&encoder_);
assert (0==rv);
assert (encoder_ != nullptr);
int width = 640;
int height = 480;
int total_num = 100;
SEncParamBase param;
memset (&param, 0, sizeof (SEncParamBase));
param.iUsageType = CAMERA_VIDEO_REAL_TIME;
param.fMaxFrameRate = 30;
param.iPicWidth = width;
param.iPicHeight = height;
param.iTargetBitrate = 5000000;
encoder_->Initialize (&param);
Mat image = imread("test.jpg", IMREAD_COLOR );
Mat imageResized, imageYuv, imageYuvMini;
resize(image, imageResized, Size(width, height));
Mat imageYuvCh[3], imageYuvMiniCh[3];
cvtColor(imageResized, imageYuv, cv::COLOR_BGR2YUV);
split(imageYuv, imageYuvCh);
resize(imageYuv, imageYuvMini, Size(width/2, height/2));
split(imageYuvMini, imageYuvMiniCh);
SFrameBSInfo info;
memset (&info, 0, sizeof (SFrameBSInfo));
SSourcePicture pic;
memset (&pic, 0, sizeof (SSourcePicture));
pic.iPicWidth = width;
pic.iPicHeight = height;
pic.iColorFormat = videoFormatI420;
pic.iStride[0] = imageYuvCh[0].step;
pic.iStride[1] = imageYuvMiniCh[1].step;
pic.iStride[2] = imageYuvMiniCh[2].step;
pic.pData[0] = imageYuvCh[0].data;
pic.pData[1] = imageYuvMiniCh[1].data;
pic.pData[2] = imageYuvMiniCh[2].data;
ofstream outFi;
outFi.open ("test.264", ios::out | ios::binary);
for(int num = 0; num<total_num; num++)
{
//prepare input data
rv = encoder_->EncodeFrame (&pic, &info);
assert (rv == cmResultSuccess);
if (info.eFrameType != videoFrameTypeSkip /*&& cbk != nullptr*/)
{
//output bitstream
for (int iLayer=0; iLayer < info.iLayerNum; iLayer++)
{
SLayerBSInfo* pLayerBsInfo = &info.sLayerInfo[iLayer];
int iLayerSize = 0;
int iNalIdx = pLayerBsInfo->iNalCount - 1;
do {
iLayerSize += pLayerBsInfo->pNalLengthInByte[iNalIdx];
--iNalIdx;
} while (iNalIdx >= 0);
unsigned char *outBuf = pLayerBsInfo->pBsBuf;
outFi.write((char *)outBuf, iLayerSize);
}
}
}
if (encoder_) {
encoder_->Uninitialize();
WelsDestroySVCEncoder (encoder_);
}
outFi.close();
}

How can I convert a CImg image to an itkImage?

I have a CImg image (with double values) with the following form in c++:
CImg<double> image(512,512);
Tcyl.fill(1);
I would like to use ITK functionality to transform this image. So I need to transform this CImg image into itkImage object. How can I convert?
I have never used ITK before, so this was a learning curve. Anyway, I managed to make a radial gradient of floats in CImg and convert that to an ITK image and then write that as a float TIFF.
#include <iostream>
#include <cstdlib>
#define cimg_display 0
#include "CImg.h"
#include "itkImage.h"
#include "itkImportImageFilter.h"
#include "itkImageFileWriter.h"
#include "itkTIFFImageIO.h"
using namespace cimg_library;
using namespace std;
#define W 512
#define H 512
int main() {
// Create and initialise float image with radial gradient
cimg_library::CImg<float> img(W,H);
cimg_forXY(img,x,y) {img(x,y) = hypot((float)(W/2-x),(float)(H/2-y)); }
// Now convert CImg image to ITK image
const unsigned int Dimension = 2;
typedef itk::Image<float,Dimension> InputImageType;
typedef itk::ImportImageFilter<float,Dimension> ImportFilterType;
ImportFilterType::Pointer importFilter = ImportFilterType::New();
InputImageType::SizeType imsize;
imsize[0] = img.width();
imsize[1] = img.height();
ImportFilterType::IndexType start;
start.Fill(0);
ImportFilterType::RegionType region;
region.SetIndex(start);
region.SetSize(imsize);
importFilter->SetRegion(region);
const itk::SpacePrecisionType origin[Dimension] = {0.0,0.0};
importFilter->SetOrigin(origin);
const itk::SpacePrecisionType spacing[Dimension] = {1.0,1.0};
importFilter->SetSpacing(spacing);
// Tell ITK importFilter to take pixels directly from CImg's buffer
importFilter->SetImportPointer(img.data(),imsize[0]*imsize[1],false);
// Write result as a TIFF - so I can check it worked
typedef itk::ImageFileWriter<InputImageType> WriterType;
typedef itk::TIFFImageIO TIFFIOType;
TIFFIOType::Pointer tiffIO = TIFFIOType::New();
tiffIO->SetPixelType(itk::ImageIOBase::SCALAR);
WriterType::Pointer writer = WriterType::New();
writer->SetFileName("result.tif");
writer->SetInput(importFilter->GetOutput());
writer->SetImageIO(tiffIO);
writer->Update();
}
And here's the result (converted to JPEG for SO):
It works just the same if you change float for double except you can't write doubles to a TIFF so the last bit won't work.
I've been messing around some more and managed to load the pixels into a new ITK image ready for further processing, rather than for output:
#include <iostream>
#include <cstdlib>
#define cimg_display 0
#include "CImg.h"
#include "itkImage.h"
#include "itkImageRegionIterator.h"
using namespace cimg_library;
using namespace std;
#define W 5
#define H 3
int main() {
// Create and initialise double image with simple formula
cimg_library::CImg<double> img(W,H);
cimg_forXY(img,x,y) {img(x,y) = (double)x+(10.0*(double)y); }
// Now convert CImg image to ITK image
const unsigned int Dimension = 2;
typedef itk::Image<double,2> ImageType;
ImageType::Pointer image = ImageType::New();
ImageType::SizeType size;
size[0] = img.width();
size[1] = img.height();
ImageType::IndexType start;
start.Fill(0);
ImageType::RegionType region;
region.SetSize(size);
region.SetIndex(start);
image->SetRegions(region);
image->Allocate();
double origin[2];
origin[0]=0;
origin[1]=0;
image->SetOrigin(origin);
double spacing[2];
spacing[0]=1;
spacing[1]=1;
image->SetSpacing(spacing);
typedef itk::ImageRegionIterator<ImageType> IteratorType;
IteratorType it(image,region);
it.GoToBegin();
const double* data = img.data();
while(!it.IsAtEnd()){
it.Set(*data);
++it;
++data;
}
// Display pixels for checking purposes
for(unsigned int r = 0; r < H; r++)
{
for(unsigned int c = 0; c < W; c++)
{
ImageType::IndexType pixelIndex;
pixelIndex[0] = c;
pixelIndex[1] = r;
ImageType::PixelType pixelValue = image->GetPixel( pixelIndex );
cout << "Image[" << r << "," << c << "]: " << pixelValue << endl;
}
}
}
Sample Output
Image[0,0]: 0
Image[0,1]: 1
Image[0,2]: 2
Image[0,3]: 3
Image[0,4]: 4
Image[1,0]: 10
Image[1,1]: 11
Image[1,2]: 12
Image[1,3]: 13
Image[1,4]: 14
Image[2,0]: 20
Image[2,1]: 21
Image[2,2]: 22
Image[2,3]: 23
Image[2,4]: 24

opencv error: assertion failed in cv::Mat::at file: mat.inl.hpp line 930

I have code for svm training phase. I use ms visual studio. I got error while executing below code:
#include <opencv/highgui.h>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/ml/ml.hpp>
#include <fstream>
#include <ctime>
#include <stdio.h>
#include <math.h>
#include <opencv\cv.h>
#include <opencv2\objdetect\objdetect.hpp>
#include <opencv2\imgproc\imgproc.hpp>
#include <opencv2\imgcodecs.hpp>
#include <opencv2\core\core.hpp>
#include <vector>
#include <windows.h>
#include <atlstr.h>
#include <iostream>
#include <sstream>
#include <iomanip>
#include <opencv2\core\core.hpp>
#include <opencv\cvaux.hpp>
using namespace cv;
using namespace cv::ml;
using namespace std;
void readCenters(cv::Mat&, const char *);
cv::Mat shuffleRows(const cv::Mat&,const cv::Mat&);
int CLUSTER_COUNT=5;//number of clusters
Mat train_data1;
Mat test_data;
void splitData(cv::Mat &data,cv::Mat &train_data1,cv::Mat &test_data)
{
int N=data.rows;
float ratio=0.7;
int train_data_length= N*ratio;
int test_data_length=N-train_data_length;
data(cv::Rect(0,0,data.cols,train_data_length)).copyTo(train_data1);
data(cv::Rect(0,train_data_length,data.cols,test_data_length)).copyTo(test_data);
cout<<"length : "<<train_data_length<<endl;
}
void readData(cv::Mat& data)
{
std::vector<const char *> filenames;
filenames.push_back("Data/ik147_1.txt");
filenames.push_back("Data/ik147_2.txt");
filenames.push_back("Data/ik147_3.txt");
filenames.push_back("Data/labels.txt");
std::string line;
std::vector<cv::Mat> raw_data(4);//= new std::vector<cv::Mat>(4);
int row;
double min,max;
for(int i =0;i<4;i++)
{
std::ifstream file( filenames[i] );
while( file>>row )
{
raw_data[i].push_back(row);
}
minMaxLoc(raw_data[i],&min,&max);
cout<<filenames[i]<<" min :"<<min<<", max :"<<max<<std::endl;
}
int N=raw_data[0].rows;
// cv::Mat data(N,3,CV_32FC1);
int columns_to_read=3;
data.create(N,columns_to_read,CV_32FC1);
for(int i=0;i<columns_to_read;i++)
{
raw_data[i](cv::Rect(0,0,1,N)).copyTo(data(cv::Rect(i,0,1,N)));
}
}
void computeLabelledData(cv:: Mat& data,cv::Mat &data_with_labels){
cv::Mat labels,centers;
cv::kmeans(data, CLUSTER_COUNT, labels,
cv::TermCriteria( CV_TERMCRIT_EPS+CV_TERMCRIT_ITER, 10, 1.0),
3, cv::KMEANS_PP_CENTERS, centers);
data_with_labels.create(data.rows,data.cols+1,CV_32FC1);
data.copyTo(data_with_labels(cv::Rect(0,0,data.cols,data.rows)));
labels.copyTo(data_with_labels(cv::Rect(data.cols,0,labels.cols,labels.rows)));
}
int main()
{
Mat data;
readData(data);
Mat data_with_labels,train_data_labels,test_data_labels;
computeLabelledData(data,data_with_labels);
splitData(data,train_data1,test_data);
// Data for visual representation
int width = 512, height = 512;
Mat image = Mat::zeros(height, width, CV_8UC3);
int N=data.rows; //number of data points
int K=5; //number of labels
Mat train_data(train_data1.rows,train_data1.cols,CV_32F);
Mat labels;
int clusterCount=K;
int sampleCount = N;
Mat centers(5,2,CV_32FC1);
readCenters(centers,"centers.txt");
Point center;
center.x = 0;//rng_center.uniform(0, height);
center.y = 0;//rng_center.uniform(0, width);
// Set up training data
Mat labels_converted;
labels.convertTo(labels_converted, CV_32SC1);
Ptr<SVM> svm = SVM::create();
// edit: the params struct got removed,
// we use setter/getter now:
svm->setType(SVM::C_SVC);
// svm->setC(0.1);
svm->setKernel(SVM::LINEAR);
svm->setDegree(1./5);
svm->setGamma(100);
svm->setTermCriteria(TermCriteria(TermCriteria::MAX_ITER, 100, 1e-6));
// Train the SVM
//CvSVM svm;
svm->train(train_data, ROW_SAMPLE, labels_converted);
svm->save("svm_params.xml");
cout<<"svm parameters saved to : svm_params.xml"<<endl;
getchar();
}
void readCenters(cv::Mat &centers, const char * filename){
const int ROWS=5;
const int COLS=2;
cout<<"reading centers "<<filename<<endl;
float array[ROWS][COLS];
std::ifstream file( filename );
std::string line;
int row,col;
int i=0;
while( file>>row>>col )
{
centers.at<float>(i,0)=row;
centers.at<float>(i,1)=col;
i++;
}
}
cv::Mat shuffleRows(const cv::Mat &matrix,const cv::Mat &seeds)
{
cv::Mat output;
for (int cont = 0; cont < matrix.rows; cont++)
output.push_back(matrix.row((int)seeds.at<float>(cont,0)));
return output;
}
while executing this code, I got this error:
I able to read data from text files also centers.txt for define centers for kmeans. It is showing error at Mat::at in mat.inl.hpp file. I followed other references. Also tried to change datatypes from CV_32FC1 to CV_32F. But I cannot solve error.1
This error often happens when you are trying to access a pixel not inside your matrix. Also known as accessing a part of memory your program wasn't expecting to.
Take this example:
I have a matrix that is 10,10 and I try to access the 11,11 pixel. My program will crash and I will get the error that you are showing above. This can also happen even if I try to access 8,8 if I haven't loaded the image properly.
Take your code, wherever you are accessing a matrix could cause this issue, for instance this part of your code:
for(int i=0;i<columns_to_read;i++)
{
raw_data[i](cv::Rect(0,0,1,N)).copyTo(data(cv::Rect(i,0,1,N)));
}
You will need to step through it and see where it is crashing and then figure out what you are doing wrong.

Covariance matrix with opencl and opencv

I recently start to learn opencl and how to create kernel for OpenCV as well.
I am still working with the basics.
I need to implement an opencl kernel for calculate the covariance matrix.
This function don't have any opencl kernel already implemented by opencv's fundation.
Technically what I want to process is that :
cv::RNG tutu(std::time(nullptr)); // This is only for check the improvement of the method over the executions
int sz = tutu.uniform(1,20);
cv::Mat_<float> a1(1,sz);
cv::Mat_<float> a2(a1.size());
cv::Mat_<float> c2;
for(std::size_t i=0;i<sz;i++)
{
a1(i) = tutu.uniform(0,300);
a2(i) = tutu.uniform(300,600);
}
cv::Mat_<float> f;
cv::vconcat(a1,a2,f);
// process the Covariance Matrix :
cv::gemm(one,f,-0.5f,f,1.f,c2,0);
cv::gemm(c2.t(),c2,1.f,cv::noArray(),0.f,c2);
I found OpenCV's fundation have implemented an OpenCL kernel for both gemm and the function transpose.
So I tried to derivate an implementation from an exemple directly from OpenCV's source.
I wrote this :
ocl.h :
void CovarMatrix( cv::Mat_<float>& src,cv::Mat_<float>& covar);
ocl.cpp :
#include <memory>
#include <fstream>
#include <sstream>
#include <iterator>
#include <opencv2/core.hpp>
#include <opencv2/core/ocl.hpp>
namespace test
{
namespace ocl
{
namespace
{
std::unique_ptr<cv::ocl::ProgramSource> cov_src;
void init_cov()
{
std::ifstream stream("../mahalanobis/covarianceMatrix.cl");
std::ostringstream sstream;
sstream << stream.rdbuf();
cv::String norm_file_content = sstream.str();
stream.close();
cov_src.reset(new cv::ocl::ProgramSource(norm_file_content));
}
}
static bool ocl_gemm( cv::Mat_<float>& matA, cv::Mat_<float>& matB, cv::Mat_<float>& CV_OUT matD)
{
cv::Mat_<float> tmp;
cv::Mat_<float> tmp2;
cv::Size sizeA = matA.size(), sizeB = matB.size();
cv::Size sizeD(sizeB.width, sizeA.height);
const cv::ocl::Device & dev = cv::ocl::Device::getDefault();
int max_wg_size = (int)dev.maxWorkGroupSize();
int block_size = (max_wg_size / 32 < 32) ? (max_wg_size / 16 < 16) ? (max_wg_size / 8 < 8) ? 1 : 8 : 16 : 32;
// matD.create(sizeD);
// tmp2.create(matD.t().size());
tmp.create(sizeD);
tmp2.create(tmp.t().size());
matD.create(sizeD.width,sizeD.width);
cv::UMat A = matA.getUMat(cv::ACCESS_READ,cv::USAGE_ALLOCATE_DEVICE_MEMORY);
cv::UMat B = matB.getUMat(cv::ACCESS_READ,cv::USAGE_ALLOCATE_DEVICE_MEMORY);
// cv::UMat D = matD.getUMat(cv::ACCESS_WRITE,cv::USAGE_ALLOCATE_DEVICE_MEMORY);
cv::UMat D = tmp.getUMat(cv::ACCESS_WRITE,cv::USAGE_ALLOCATE_DEVICE_MEMORY);
// cv::UMat E(sizeD.width,sizeD.height,CV_32FC1,cv::Scalar::all(0.),cv::USAGE_ALLOCATE_DEVICE_MEMORY);
cv::UMat E = tmp2.getUMat(cv::ACCESS_WRITE,cv::USAGE_ALLOCATE_DEVICE_MEMORY);
cv::UMat F = matD.getUMat(cv::ACCESS_WRITE,cv::USAGE_ALLOCATE_DEVICE_MEMORY);
matB.copyTo(D);
int vectorWidths[] = { 4, 4, 2, 2, 1, 4, 1, -1 };
int kercn = cv::ocl::checkOptimalVectorWidth(vectorWidths, B, D);
cv::String opts = cv::format(
"-I /home/administrateur/lib_dir/opencv_dir/opencv_304/opencv/modules/core/src/opencl/ -D T=float -D T1=float -D WT=%s -D cn=1 -D kercn=%d -D LOCAL_SIZE=%d %s -D HAVE_C -D TILE_DIM=32 -D BLOCK_ROWS=8 -D rowsPerWI=1 ",
cv::ocl::typeToStr(CV_32FC(kercn)),
kercn, block_size,
(sizeA.width % block_size !=0) ? "-D NO_MULT" : "");
cv::ocl::Kernel k("covarianceMatrix", *cov_src, opts);
k.args(cv::ocl::KernelArg::ReadOnlyNoSize(A),
cv::ocl::KernelArg::ReadOnlyNoSize(B, 1, kercn),
cv::ocl::KernelArg::ReadWrite(D, 1, kercn),
sizeA.width,
cv::ocl::KernelArg::ReadWrite(E,kercn,1),
cv::ocl::KernelArg::ReadWrite(F,kercn,kercn)
);
std::size_t globalsize[2] = { static_cast<std::size_t>(sizeD.width / kercn), static_cast<std::size_t>(sizeD.height)};
std::size_t localsize[2] = { static_cast<std::size_t>(block_size), static_cast<std::size_t>(block_size)};
return k.run(2, globalsize, block_size!=1 ? localsize : nullptr, false);
}
void CovarMatrix( cv::Mat_<float>& src,cv::Mat_<float>& covar)
{
if(!covar.empty())
covar.release();
cv::Mat_<float> o = cv::Mat_<float>::ones(src.rows,src.rows);
if(!cov_src)
init_cov();
ocl_gemm(o,src,covar);
}
covarianceMatrix.cl :
#include "gemm.cl"
#include "transpose.cl"
__kernel void covarianceMatrix
(
__global const uchar * A_ptr, int A_step, int A_offset,
__global const uchar * B_ptr, int B_step, int B_offset,
__global uchar * D_ptr, int D_step, int D_offset, int D_rows, int D_cols,
int n,
__global uchar * E_ptr, int E_step, int E_offset, int E_rows, int E_cols,
__global uchar * F_ptr, int F_step, int F_offset, int F_rows, int F_cols
)
{
// cv::gemm(src2,src1,-0.5,src1,1.f,src2);
// cv::gemm(src2.t(),src2,1.f,cv::noArray(),0.f,dest);
gemm(A_ptr,A_step,A_offset,
B_ptr,B_step,B_offset,
D_ptr,D_step,D_offset,D_rows,D_cols,
n,-0.5f,1.f);
transpose(D_ptr,D_step,D_offset,D_rows,D_cols*sizeof(float),
E_ptr,E_step,E_offset);
gemm(E_ptr,E_step,E_offset,
D_ptr,D_step,D_offset,
F_ptr,F_step,F_offset,F_rows,F_cols,
n,1.f,0.f);
}
If the size of the matrix is fewer than 6 is work perfectly :).
Otherwise ... not really.
It can be check with this code :
cv::RNG tutu(std::time(nullptr));
int sz = tutu.uniform(1,20);
cv::Mat_<float> a1(1,sz);
cv::Mat_<float> a2(a1.size());
for(std::size_t i=0;i<sz;i++)
{
a1(i) = tutu.uniform(0,300);
a2(i) = tutu.uniform(300,600);
}
cv::Mat_<float> f;
cv::vconcat(a1,a2,f);
cv::Mat_<float> c1;
cv::Mat_<float> c2;
cv::Mat_<float> mean;
// reference
cv::calcCovarMatrix(f,c1,mean,cv::COVAR_ROWS | cv::COVAR_NORMAL,CV_32F);
// check
test::ocl::CovarMatrix(f,c2);
std::size_t cnt(0.f);
for(auto it = c1.begin(),it2 = c2.begin();it != c1.end();it++,it2++)
if(*it == *it2)
cnt++;
std::cout<<"check "<<cnt<<" "<<c1.total()<<std::endl;
I am still new in OpenCL and I am interrested to know what I did wrong.
Does someone already implemented an OpenCL kernel for process the covariance matrix with OpenCV ?
Thank in advance for any help.
I resign myself to write this code :
void ocl_CovarMatrix(cv::Mat_<float>& src,cv::Mat_<float>& covar)
{
cv::UMat usrc = src.getUMat(cv::ACCESS_READ,cv::USAGE_ALLOCATE_DEVICE_MEMORY);
cv::UMat ones = cv::UMat::ones(usrc.rows,usrc.rows,usrc.type());
cv::UMat utmp;
double beta = 1.;
double alpha = -1. / static_cast<double>(usrc.rows);
cv::gemm(ones,usrc,alpha,usrc,beta,utmp);
cv::gemm(utmp.t(),utmp,beta,cv::noArray(),0.,utmp);
utmp.copyTo(covar);
ones.release();
utmp.release();
usrc.release();
}
I suspect the GPU memory is updated everytime a function is call, that make that code slower rather than if it has been written in one Kernel.
But it work efficiently.
I am still interested by another solution if maybe someone have an idea.