I need to serialize a JSON string using Avro in C++.
I installed the libserdes library (https://github.com/confluentinc/libserdes) and I used the example code
./examples/kafka-serdes-avro-console-producer.cpp
to write a main,
which serializes a JSON string as follows.
{"s":"BTCUSDT","c":"spread","u":123456789,"a":20000.4,"b":21000.5,"A":1.25,"B":0.58,"p":-1,"P":-1,"st":-1,"rt":1675000000123000}
Here is the code
#include "fh_price_t.h"
#include "avro/Encoder.hh"
#include "avro/Decoder.hh"
#include <avro/Generic.hh>
#include <avro/Specific.hh>
#include <avro/Exception.hh>
#include "libserdes/serdescpp-avro.h"
#include <iostream>
#include <sstream>
#include <string>
#include <fstream>
#define FATAL(reason...) do { \
std::cerr << "% FATAL: " << reason << std::endl; \
exit(1); \
} while (0)
template<class T> std::string to_string_a (const T& x) {
std::ostringstream oss;
oss << x;
return oss.str();
}
/**
* Convert JSON to Avro Datum.
*
* Returns 0 on success or -1 on failure.
*/
int json2avro (Serdes::Schema *schema, const std::string &json,
avro::GenericDatum **datump) {
avro::ValidSchema *avro_schema = schema->object();
/* Input stream from json string */
std::istringstream iss(json);
auto json_is = avro::istreamInputStream(iss);
/* JSON decoder */
avro::DecoderPtr json_decoder = avro::jsonDecoder(*avro_schema);
avro::GenericDatum *datum = new avro::GenericDatum(*avro_schema);
try {
/* Decode JSON to Avro datum */
json_decoder->init(*json_is);
avro::decode(*json_decoder, *datum);
} catch (const avro::Exception &e) {
std::cerr << "% JSON to Avro transformation failed: "
<< e.what() << std::endl;
return -1;
}
*datump = datum;
}
int main (int argc, char* argv) {
izycoinstestavro::fh_price_t price{};
price.s = "BTCUSDT";
price.c = "spread";
price.u = 123456789;
price.a = 20000.4;
price.A = 1.25;
price.b = 21000.52;
price.B = 0.58;
price.p = -1;
price.P = -1;
price.st = -1;
price.rt = 1675000000123000;
std::ifstream file_ifstream("../src/avro/fh_price_t.json");
std::stringstream buffer;
buffer << file_ifstream.rdbuf();
std::string schema_json = buffer.str();
Serdes::Conf *sconf = Serdes::Conf::create();
Serdes::Schema *schema;
std::string errstr;
if (sconf->set("schema.registry.url", "http://localhost:8081", errstr))
FATAL("Conf failed: " << errstr);
if (sconf->set("serializer.framing", "cp1", errstr))
FATAL("Conf failed: " << errstr);
Serdes::Avro *serdes = Serdes::Avro::create(sconf, errstr);
if (!serdes)
FATAL("Failed to create Serdes handle: " << errstr);
schema = Serdes::Schema::add(serdes, "fh_price_t", schema_json, errstr);
if (!schema)
FATAL("Failed to register schema " << "fh_price_t" << ": " << errstr);
std::cout << "% Registered schema " << schema->name() << " with id " << schema->id() << std::endl;
avro::GenericDatum *datum = NULL;
std::vector<char> out;
/* Convert JSON to Avro object */
std::string line = to_string_a(price);
json2avro(schema, line, &datum);
//std::cout << to_string_a(price) << std::endl;
/*if (rr == -1) {
FATAL("Failed to convert JSON to Avro " << to_string_a(price) << ": " << errstr);
}
// Serialize Avro
if (serdes->serialize(schema, datum, out, errstr) == -1) {
std::cerr << "% Avro serialization failed: " << errstr << std::endl;
delete datum;
exit(1);
}
delete datum;
std::cout << "Data Size : " << out.size() << std::endl;*/
return 0;
}
When I run it, I get a double free or corruption (out) error.
The error occurs at the output of the function.
I have located the line that causes the error (because when I remove it, the error disappears)
avro::GenericDatum *datum = new avro::GenericDatum(*avro_schema);
I would like to know why and how to solve it.
Related
I download a tar.gz file with libcurl, and need to decompress the file while downloading, that is, when chunk of the file is downloaded, decompress chunk of the file imediately, instead of decompressing the whole file when the whole file is downloaded. Is there any C/C++ libraries that meet my requirements
I tried to use libarchive to extract the file, but it returned truncated gzip input when extracting the first chunk of the file. It seems that libarchive need the whole file to extract it. Here is my code. I am not sure if i used libarchive correctly as I am new to it.
#include <iostream>
#include <vector>
#include <string>
#include <unistd.h>
#include <fcntl.h>
#include <string.h>
#include <atomic>
#include <thread>
// libarchive
#include <archive.h>
#include <archive_entry.h>
#include <curl/curl.h>
struct mydata {
void *buffer;
ssize_t *size;
};
struct curldata {
void *buffer;
ssize_t *size;
CURL *curl;
};
std::atomic<bool> rd(true);
struct archive *archive, *archivefd;
std::atomic<bool> start_read(false);
la_ssize_t libarchiveRead(struct archive* a, void* client_data, const void** block)
{
if(!rd) {
mydata *my_data = (mydata*)client_data;
std::cout << "calling custom read(), size " << *(my_data->size) << std::endl;
*block = my_data->buffer;
rd=true;
return *(my_data->size);
}
return 0;
}
int libarchiveClose(struct archive* a, void* client_data)
{
std::cout << "calling custom close() for archive" << std::endl;
mydata *my_data = (mydata*)client_data;
delete my_data;
return (ARCHIVE_OK);
}
int libarchiveClosefd(struct archive* a, void* client_data)
{
std::cout << "calling custom close() for archivefd" << std::endl;
mydata *my_data = (mydata*)client_data;
delete my_data;
return (ARCHIVE_OK);
}
static size_t curlWriteFunction(void *ptr, size_t size, size_t nmemb, void *write_data) {
//size is always 1
curldata *my_data = (curldata*)(write_data);
*(my_data->size) = nmemb * size;
std::cout << "calling curlWriteFunction(), size: " << size << " , nmemb: " << nmemb
<< " , my_data->size: " << *(my_data->size) << std::endl;
memcpy(my_data->buffer, ptr, *(my_data->size));
curl_easy_pause(my_data->curl, CURL_WRITEFUNC_PAUSE);
rd=false;
return (*(my_data->size));
}
static size_t progress(void *clientp, double dltotal, double dlnow, double ultotal, double ulnow) {
CURL *curl = (CURL *)clientp;
(void)ultotal;
(void)ulnow;
if(dltotal == 0) {
return 0;
}
if(rd) {
curl_easy_pause(curl, CURLPAUSE_CONT);
std::cout << "progress: " << dlnow/dltotal * 100 << "%" << std::endl;
}
return 0;
}
void readarchive(void *client_data) {
struct archive_entry *entry;
int flags = ARCHIVE_EXTRACT_TIME;
flags |= ARCHIVE_EXTRACT_PERM;
flags |= ARCHIVE_EXTRACT_ACL;
flags |= ARCHIVE_EXTRACT_FFLAGS;
while(rd);
std::cout << "calling archive_read_open for archive.." << std::endl;
int res = archive_read_open(archive,
client_data,
nullptr,
(archive_read_callback*)libarchiveRead,
(archive_close_callback*)libarchiveClose);
std::cout << "called archive_read_open for archive.." << std::endl;
res = archive_read_next_header(archive, &(entry));
while(res == ARCHIVE_OK ) {
std::cout << "Extracting for archive " << archive_entry_pathname(entry) << "..." << std::endl;
// extract current entry
archive_read_extract(archive, entry, flags);
// read next if available
res = archive_read_next_header(archive, &(entry));
}
std::cout << "archive_read_next_header for archive failed, errcode: " << res << " error: " << archive_error_string(archive) << std::endl;
}
//size_t curlWriteFunction(void *ptr, size_t size, size_t nmemb,FILE* fptr) {
// //size is always 1
// std::cout << "calling curlWriteFunction().." << std::endl;
// return fwrite(ptr, size, nmemb, fptr);
//}
int main(int argc, char** argv) {
if(argc < 3)
{
std::cout << argv[0] << "{-r | -w} file[s]" << std::endl;
return 1;
}
std::vector<std::string> filenames;
filenames.reserve(argc);
while (*++argv != nullptr)
{
filenames.emplace_back(*argv);
}
bool modeRead = (filenames[0] == "-r");
std::cout << filenames[0] << " " << filenames[1] << std::endl;
// archive related variables
char buff_archive[16 * 1024], buff_archivefd[16 * 1024];
if(modeRead)
{
archive = archive_read_new();
archive_read_support_filter_gzip(archive);
archive_read_support_format_tar(archive);
mydata *client_data = new mydata();
int res;
char *buff1 = new char[16 * 1024];
client_data->size = new ssize_t;
*(client_data->size) = 0;
client_data->buffer = buff1;
curldata *curl_data = new curldata();
curl_data->size=client_data->size;
curl_data->buffer=buff1;
CURL *curl = curl_easy_init();
curl_data->curl = curl;
curl_easy_setopt(curl, CURLOPT_URL, filenames[1].c_str());
curl_easy_setopt(curl, CURLOPT_WRITEDATA, curl_data);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curlWriteFunction);
curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0L);
curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, curl);
curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION,progress);
std::thread t(readarchive, client_data);
CURLcode result = curl_easy_perform(curl);
if(result != CURLE_OK) {
std::cout << "curl perform failed, errcode; " << result << " err: " << curl_easy_strerror(result) << std::endl;
}
//std::cout << "calling archive_read_open for archivefd.." << std::endl;
//res = archive_read_open(archivefd,
// client_datafd,
// nullptr,
// (archive_read_callback*)libarchiveReadfd,
// (archive_close_callback*)libarchiveClosefd);
//std::cout << "called archive_read_open for archivefd.." << std::endl;
//res = archive_read_next_header(archivefd, &(entry));
//if (res != ARCHIVE_OK) {
// std::cout << "archive_read_next_header for archivefd failed, errcode: " << res << " error: " << archive_error_string(archivefd) << std::endl;
//}
//while(res == ARCHIVE_OK) {
// std::cout << "Extracting for archivefd " << archive_entry_pathname(entry) << "..." << std::endl;
// // extract current entry
// archive_read_extract(archivefd, entry, flags);
// // read next if available
// res = archive_read_next_header(archivefd, &(entry));
//}
t.join();
delete client_data->size;
delete []buff1;
archive_read_close(archive);
archive_read_free(archive);
archive_read_free(archive);
curl_easy_cleanup(curl);
}
return 0;
}
I'm trying to write and read serialized data to file, but it always breaks when reading specific "PNG" struct, with stored "bed_feet_top.png" from minecraft 1.8.9 textures folder in it, and I can't find a way to get the error. Already tried errno and try and catch, but it always says sucess. It seems that it breaks after and not when reading the buggy "PNG" struct, because the read data is uncorrupted and I can recreate the stored image. I'm using "stb_image" library to read the image data and "zpp_bits" library for serializing the data.
the code i would use:
PNG srcImage; //The "PNG" struct it will break at
unsigned char* buffer = stbi_load("bed_feet_top.png", &image.imageWidth, &image.imageHeight, &image.colorChannels, STBI_rgb_alpha);
if (!buffer)
{
std::cout << "Failed to read image: " + path << std::endl;
return 1;
}
image.pixels = (char*)buffer;
DataBuffer srcBuffer;
srcBuffer.serialize("somePath", "someName", srcImage);
DataManager::writeToFile("someFile.dat", srcBuffer);
DataBuffer dstBuffer;
DataManager::readFromFile("someFile.dat", &dstbuffer);
PNG dstImage = dstBuffer.deserialize<PNG>("somePath", "someName"); //The buggy "PNG" struct data is uncorrupted
the "PNG" struct:
struct PNG
{
int imageWidth, imageHeight, colorChannels;
std::string pixels;
};
the struct I'm storing all serialized data in:
struct DataBuffer
{
std::map<std::string, std::vector<std::byte>> data; //All the serialized data is stored in this map
template<typename T>
void serialize(std::string path, std::string name, T obj)
{
path.append("/");
path.append(name);
serialize(path, obj);
}
template<typename T>
void serialize(std::string fullPath, T obj)
{
if (data.contains(fullPath))
{
std::cout << "Data buffer already contains value with te same path!" << std::endl;
return;
}
try
{
zpp::bits::out out(data[fullPath]);
out(obj).or_throw();
}
catch (const std::exception& error)
{
std::cerr << "Serialization failed with error: " << error.what() << std::endl;
}
catch (...) {
std::cerr << "Serialization failed with unknown error!" << std::endl;
}
}
template<typename T>
T deserialize(std::string path, std::string name)
{
path.append("/");
path.append(name);
return deserialize<T>(path);
}
template<typename T>
T deserialize(std::string fullPath)
{
T obj;
if (!data.contains(fullPath))
{
std::cout << "Data buffer doesn't contain value with the given path!" << std::endl;
return obj;
}
try
{
zpp::bits::in in(data[fullPath]);
in(obj).or_throw();
}
catch (const std::exception& error)
{
std::cerr << "Deserialization failed with error: " << error.what() << std::endl;
}
catch (...) {
std::cerr << "Deserialization failed with unknown error!" << std::endl;
}
return obj;
}
};
The write to file function:
void DataManager::writeToFile(std::string fileName, DataBuffer dataBuffer)
{
std::ofstream ofs(fileName, std::ios::out | std::ios::binary);
if (!ofs.is_open())
{
std::cerr << "Failed to open output file stream!" << std::endl;
return;
}
size_t dataCount = dataBuffer.data.size();
ofs.write((char*)&dataCount, 8);
for (auto const& [key, val] : dataBuffer.data)
{
size_t keySize = key.size();
ofs.write((char*)&keySize, sizeof(keySize));
ofs.write(&key[0], keySize);
size_t entryCount = val.size();
ofs.write((char*)&entryCount, 8);
ofs.write((char*)val.data(), entryCount);
if (!ofs.good()) //ofs.good() never returned false to me
{
std::cerr << "Something went wrong when writing " << key << " to the file!" << std::endl;
break;
}
}
ofs.close();
}
the read from file function:
void DataManager::readFromFile(std::string fileName, DataBuffer* dataBuffer)
{
std::ifstream ifs(fileName, std::ios::in, std::ios::binary);
if (!ifs)
{
std::cerr << "File doesnt exist!" << std::endl;
return;
}
if (!ifs.is_open())
{
std::cerr << "Failed to open input file stream!" << std::endl;
return;
}
size_t dataCount;
ifs.read((char*)&dataCount, 8);
for (size_t i = 0; i < dataCount; i++)
{
size_t keySize;
ifs.read((char*)&keySize, sizeof(keySize));
std::string key;
key.resize(keySize);
ifs.read(&key[0], keySize);
size_t entryCount;
ifs.read((char*)&entryCount, 8);
dataBuffer->data[key].resize(entryCount);
ifs.read((char*)dataBuffer->data[key].data(), entryCount);
if (!ifs.good()) //ifs.good() returns false after it reads the buggy "PNG" struct data
{
std::cerr << "Something went wrong when reading " << key << " from the file!" << std::endl;
break;
}
}
ifs.close();
}
EDIT
I added minimal reproducible example as "Igor Tandetnik" requested.
the minimal reproducible example:
#define STB_IMAGE_IMPLEMENTATION
#define STB_IMAGE_WRITE_IMPLEMENTATION
#include <string>
#include <iostream>
#include <vector>
#include <fstream>
#include <stb_image.h>
#include "stb_image_write.h"
#include <zpp_bits.h>
struct PNG
{
int imageWidth, imageHeight, colorChannels;
std::string pixels;
};
int main()
{
std::string fileName = "someFile.dat"; //The file we will store the image data in
std::string srcImageName = "bed_feet_top.png"; //Texture from Minecraft 1.8.9 textures folder
std::string dstImageName = "someImage.png"; //The name of image we will create from the read data
//Reading the image data
PNG srcImage;
unsigned char* buffer = stbi_load(srcImageName.c_str(), &srcImage.imageWidth, &srcImage.imageHeight, &srcImage.colorChannels, STBI_rgb_alpha);
if (!buffer)
{
std::cout << "Failed to read image " + srcImageName << std::endl;
return 1;
}
int dataSize = srcImage.imageWidth * srcImage.imageHeight * srcImage.colorChannels;
srcImage.pixels.assign((char*)buffer, dataSize);
//Serializing the image data
std::vector<std::byte> srcData;
try
{
zpp::bits::out out(srcData);
out(srcImage).or_throw();
}
catch (const std::exception& error)
{
std::cerr << "Serialization failed with error: " << error.what() << std::endl;
}
catch (...) {
std::cerr << "Serialization failed with unknown error!" << std::endl;
}
//Writing the image data to file
std::ofstream ofs(fileName, std::ios::out | std::ios::binary);
if (!ofs.is_open())
{
std::cout << "Failed to open output file stream!" << std::endl;
return 1;
}
size_t srcEntryCount = srcData.size();
ofs.write((char*)&srcEntryCount, 8);
ofs.write((char*)srcData.data(), srcEntryCount);
if (!ofs.good())
{
std::cerr << "Something went wrong when writing" << srcImageName << " to the file!" << std::endl;
ofs.close();
return 1;
}
ofs.close();
//Reading the image data from file
std::ifstream ifs(fileName, std::ios::in, std::ios::binary);
if (!ifs)
{
std::cout << "File doesnt exist!" << std::endl;
return 1;
}
if (!ifs.is_open())
{
std::cout << "Failed to open input file stream!" << std::endl;
return 1;
}
std::vector<std::byte> dstData;
size_t dstEntryCount;
ifs.read((char*)&dstEntryCount, 8);
dstData.resize(dstEntryCount);
ifs.read((char*)dstData.data(), dstEntryCount);
if (!ifs.good())
{
std::cerr << "Something went wrong when reading " << srcImageName << " from the file!" << std::endl;
ifs.close();
return 1; //If you comment this line you will see it will recreate the image from the read data just fine
}
ifs.close();
//Deserializing the read image data
PNG dstImage;
try
{
zpp::bits::in in(dstData);
in(dstImage).or_throw();
} catch (const std::exception& error)
{
std::cerr << "Deserialization failed with error: " << error.what() << std::endl;
} catch (...) {
std::cerr << "Deserialization failed with unknown error!" << std::endl;
}
//Recreating the image from the read image data
stbi_write_png(dstImageName.c_str(), dstImage.imageWidth, dstImage.imageHeight, dstImage.colorChannels, dstImage.pixels.c_str(), dstImage.imageWidth * sizeof(int));
}
I'm caling the objects src and dst, because I don't know how to call them shortly before and after serialization. :D
I'm trying to interpret the output of FasterRCNN in C++ and I'm fighting with the GenericDict type.
My code is as follows:
#include <opencv4/opencv2/opencv.hpp>
#include <opencv4/opencv2/shape.hpp>
#include <opencv4/opencv2/imgcodecs.hpp>
#include <opencv4/opencv2/highgui.hpp>
#include <opencv4/opencv2/imgproc.hpp>
#include <opencv4/opencv2/core/utility.hpp>
#include <opencv4/opencv2/core/mat.hpp>
#include <c10/cuda/CUDAStream.h>
#include <torch/csrc/autograd/grad_mode.h>
#include <torch/csrc/api/include/torch/torch.h>
#include <torch/script.h>
#include <torchvision/vision.h>
#include <torchvision/nms.h>
#include <iostream>
#include <memory>
#include <string>
int main(int argc, const char* argv[])
{
if (argc != 3)
{
printf("usage: %s <path-to-exported-script-module> <image_to_test>\n",argv[0]);
return -1;
}
std::string module_filename = argv[1];
std::string image_file = argv[2];
try
{
cv::Mat input_img = cv::imread(image_file, cv::IMREAD_GRAYSCALE);
torch::autograd::AutoGradMode guard(false);
// Deserialize the ScriptModule from a file using torch::jit::load().
torch::jit::script::Module module = torch::jit::load(module_filename);
assert(module.buffers().size() > 0);
module.eval();
// Assume that the entire model is on the same device.
// We just put input to this device.
auto device = (*std::begin(module.buffers())).device();
const int height = input_img.rows;
const int width = input_img.cols;
const int channels = 1;
auto input = torch::from_blob(input_img.data, {height, width, channels}, torch::kUInt8);
// HWC to CHW
// input = input.to(device, torch::kFloat).permute({2, 0, 1}).contiguous();
input = input.to(device, torch::kFloat).permute({2, 0, 1}).contiguous();
// run the network
std::vector<at::Tensor> inputs;
inputs.push_back(input);
auto output = module.forward({inputs});
if (device.is_cuda())
c10::cuda::getCurrentCUDAStream().synchronize();
std::cout << "output: " << output << std::endl;
auto outputs = output.toTuple()->elements();
std::cout << "outputs: " << outputs << std::endl;
for( auto& elem : outputs )
{
std::cout << "elem: " << elem << std::endl;
if( elem.isGenericDict() )
{
std::cout << "elem is generic dict: " << elem << std::endl;
c10::Dict<c10::IValue, c10::IValue> dict = elem.toGenericDict();
auto elem_vector_0 = dict.at(c10::IValue("scores")).toIntVector();
auto elem_vector_1 = dict.at(c10::IValue("boxes")).toIntVector();
auto elem_vector_2 = dict.at(c10::IValue("labels")).toIntVector();
for( auto& ee0 : elem_vector_0 )
{
std::cout << "elem_vector_0" << ee0 << std::endl;
}
for( auto& ee0 : elem_vector_1 )
{
std::cout << "elem_vector_1" << ee0 << std::endl;
}
for( auto& ee0 : elem_vector_2 )
{
std::cout << "elem_vector_2" << ee0 << std::endl;
}
}
}
cv::namedWindow("Display Image", cv::WINDOW_AUTOSIZE );
cv::imshow("Display Image", input_img);
cv::waitKey(0);
}
catch(const c10::Error& e)
{
std::cerr << e.what() << std::endl;
return -1;
}
catch(const cv::Exception& e)
{
std::cerr << e.what() << std::endl;
return -1;
}
catch(const std::exception& e)
{
std::cerr << e.what() << std::endl;
return -1;
}
catch(...)
{
std::cerr << "Unknown error" << std::endl;
return -1;
}
std::cout << "ok\n";
return 0;
}
and the output is:
(base) fstrati#fstrati-desktop:~/libtorch_shared_cuda_10.1/load_and_run_model/Release$ ./load_and_run_model ./torch_script_v0.2.pt test_img.png
[W faster_rcnn.py:95] Warning: RCNN always returns a (Losses, Detections) tuple in scripting (function )
output: ({}, [{boxes: [ CPUFloatType{0,4} ], labels: [ CPULongType{0} ], scores: [ CPUFloatType{0} ]}])
outputs: {} [{boxes: [ CPUFloatType{0,4} ], labels: [ CPULongType{0} ], scores: [ CPUFloatType{0} ]}]
elem: {}
elem is generic dict: {}
Argument passed to at() was not in the map.
I'm struggling to find a way to extract the boxes, labels and scores from the dictionary
GenericDict.
This map is strange, I cannot iterate on it and I cannot access first and second types...
with it->first it->second
Any ideas ?
Thanks in advance
I think the following method can resolve the main problem here,
output = module.forward(inputs);
auto detections = output.toTuple()->elements().at(1).toList().get(0).toGenericDict();
std::cout << ">>> detections labels: " << detections.at("labels") << std::endl;
std::cout << ">>> detections boxes: " << detections.at("boxes") << std::endl;
std::cout << ">>> detections scores: " << detections.at("scores") << std::endl;
Besides, I've added an executable file https://github.com/zhiqwang/yolov5-rt-stack/tree/master/deployment/libtorch to show how libtorch works.
I try to send and receive a protobuf object by using Microsoft REST SDK. If you are not familiar with protobuf, imagine that it can be any other raw data. On the client side the vector is empty. I think that I do not create correctly the streams on the server side.
The get method of the server looks like this:
void MyServer::handle_get(http_request message)
{
// a class created with protobuf
ns::user *user = new ns::user();
ns::user->set_id(-1);
ns::user->set_name("Dark");
ns::user->set_email("dark#foo.com");
// this schould be fine
int size = user->ByteSize();
void *buffer = malloc(size);
user->SerializeToArray(buffer, size);
// how to do?
concurrency::streams::container_buffer<std::vector<char>> inBuffer;
concurrency::streams::istream bodyStream = message.body();
inBuffer.putn((char*)buffer, size);
Concurrency::task<size_t> read = bodyStream.read_to_end(inBuffer);
message.reply(status_codes::OK, bodyStream, size, L"application/octet-stream");
delete user;
};
The client method like this:
utility::ostringstream_t buf;
buf << "Dark" << "/" << "Foo" << "?" << "id=" << "1";
http_response response = CheckResponse("server/user", server.request(methods::GET, buf.str()).get());
if (response.status_code() == status_codes::OK)
{
cout << "STATUS OK" << endl;
Concurrency::streams::istream iStream = response.body();
auto rslt0 = response.extract_vector();
auto rslt = rslt0.get();
cout << "Result has size: <" << rslt.size() << ">" << endl;
for (unsigned int i = 0; i < rslt.size(); i++)
{
cout << "Data" << i << ": <" << rslt.at(i) << ">" << endl;
}
Server side:
std::vector<uint8_t> data;
//TODO: fill data
//convert the vector into a async istream
auto instream = Concurrency::streams::bytestream::open_istream(data);
message.reply(status_codes::OK, instream);
Client side:
wchar_t urlbuf[256];
//TODO: fill the url
client->request(web::http::methods::GET,urlbuf).then([](web::http::http_response& resp){
if(resp.status_code() != status_codes::OK){
DLogError("error: status != ok");
return;
}
Concurrency::streams::container_buffer<std::vector<uint8_t> >inBuffer;
resp.body().read_to_end(inBuffer).then([](size_t bytesRead){
//TODO: use it
}
}
I answer my own question. There is no need for stream; I can use the binary data by this way:
Server:
void MyServer::handle_get(http_request message)
{
// a class created with protobuf
ns::user *user = new ns::user();
ns::user->set_id(-1);
ns::user->set_name("Dark");
ns::user->set_email("dark#foo.com");
int size = user->ByteSize();
unsigned char *buffer = new unsigned char(size);
user->SerializeToArray(buffer, size);
std::string out;
user->SerializeToString(&out);
std::vector<unsigned char> data(buffer, buffer + size);
string_t outT = utility::conversions::to_base64(data);
message.reply(status_codes::OK, outT);
delete user;
};
Client:
utility::ostringstream_t buf;
buf << "Dark" << "/" << "Foo" << "?" << "id=" << "1";
http_response response = CheckResponse("server/user", server.request(methods::GET, buf.str()).get());
if (response.status_code() == status_codes::OK)
{
cout << "STATUS OK" << endl;
geopp::gnUser *user = new geopp::gnUser();
auto rslt = response.extract_string();
auto rslt0 = utility::conversions::from_base64(rslt.get());
user->ParseFromArray(rslt0.data(), rslt0.size());
cout << "Result: name <" << user->name() << ">, id <" << user->id() << ">" << endl;
};
I am trying to write a simple C++ ping function to see if a network address is responding. I don't need ICMP specifically, I just need to see if the server is there and responding to anything. I have been doing some research and every solution I come up with requires creating a raw socket or something which requires the program to have sudo access. I won't be able to guarantee that the system I am running this on will be able to modify the network stack, so this is not valid.
Here are some related questions I have already looked at.
Opening RAW sockets in linux without being superuser
ICMP sockets (linux)
How to Ping Using Sockets Library - C
Why does ping work without administrator privileges?
C++ Boost.asio Ping
It appears that ping requires superuser access for a good reason. I don't want to purposefully create a security loophole, I just want to see if a server is responding. Is there a good c++ function or resource that can do this? I will make sure to post any sample solution I come up with. I need a Linux (BSD) socket solution. Since almost every unix-like system runs SSH, I can even just test against port 22. I am only going to target Linux systems as a constraint.
Thanks
Here is an example with popen. I would love a better solution, so if there is a socket or other method which does not resort to a shell call, I would greatly appreciate it. This should run with just g++ ping_demo.cpp -o ping_demo. Let me know if it causes compile errors.
// C++ Libraries
#include <cstdio>
#include <iostream>
#include <sstream>
#include <string>
/**
* #brief Convert String to Number
*/
template <typename TP>
TP str2num( std::string const& value ){
std::stringstream sin;
sin << value;
TP output;
sin >> output;
return output;
}
/**
* #brief Convert number to string
*/
template <typename TP>
std::string num2str( TP const& value ){
std::stringstream sin;
sin << value;
return sin.str();
}
/**
* #brief Execute Generic Shell Command
*
* #param[in] command Command to execute.
* #param[out] output Shell output.
* #param[in] mode read/write access
*
* #return 0 for success, 1 otherwise.
*
*/
int Execute_Command( const std::string& command,
std::string& output,
const std::string& mode = "r")
{
// Create the stringstream
std::stringstream sout;
// Run Popen
FILE *in;
char buff[512];
// Test output
if(!(in = popen(command.c_str(), mode.c_str()))){
return 1;
}
// Parse output
while(fgets(buff, sizeof(buff), in)!=NULL){
sout << buff;
}
// Close
int exit_code = pclose(in);
// set output
output = sout.str();
// Return exit code
return exit_code;
}
/**
* #brief Ping
*
* #param[in] address Address to ping.
* #param[in] max_attempts Number of attempts to try and ping.
* #param[out] details Details of failure if one occurs.
*
* #return True if responsive, false otherwise.
*
* #note { I am redirecting stderr to stdout. I would recommend
* capturing this information separately.}
*/
bool Ping( const std::string& address,
const int& max_attempts,
std::string& details )
{
// Format a command string
std::string command = "ping -c " + num2str(max_attempts) + " " + address + " 2>&1";
std::string output;
// Execute the ping command
int code = Execute_Command( command, details );
return (code == 0);
}
/**
* #brief Main Function
*/
int main( int argc, char* argv[] )
{
// Parse input
if( argc < 2 ){
std::cerr << "usage: " << argv[0] << " <address> <max-attempts = 3>" << std::endl;
return 1;
}
// Get the address
std::string host = argv[1];
// Get the max attempts
int max_attempts = 1;
if( argc > 2 ){
max_attempts = str2num<int>(argv[2]);
}
if( max_attempts < 1 ){
std::cerr << "max-attempts must be > 0" << std::endl;
return 1;
}
// Execute the command
std::string details;
bool result = Ping( host, max_attempts, details );
// Print the result
std::cout << host << " ";
if( result == true ){
std::cout << " is responding." << std::endl;
}
else{
std::cout << " is not responding. Cause: " << details << std::endl;
}
return 0;
}
Sample outputs
$> g++ ping_demo.cpp -o ping_demo
$> # Valid Example
$> ./ping_demo localhost
localhost is responding.
$> # Invalid Example
$> ./ping_demo localhostt
localhostt is not responding. Cause: ping: unknown host localhostt
$> # Valid Example
$> ./ping_demo 8.8.8.8
8.8.8.8 is responding.
I created an easy to use ping class with added ethernet port detection capability. I based the code on msmith81886's answer but wanted to condense for those using it in industry.
ping.hpp
#ifndef __PING_HPP_
#define __PING_HPP_
#include <cstdio>
#include <iostream>
#include <sstream>
#include <string>
#include <fstream>
#include <cerrno>
#include <cstring>
class system_ping
{
public:
int test_connection (std::string ip_address, int max_attempts, bool check_eth_port = false, int eth_port_number = 0);
private:
int ping_ip_address(std::string ip_address, int max_attempts, std::string& details);
};
#endif
ping.cpp
#include "../include/ping.hpp"
int system_ping::ping_ip_address(std::string ip_address, int max_attempts, std::string& details)
{
std::stringstream ss;
std::string command;
FILE *in;
char buff[512];
int exit_code;
try
{
command = "ping -c " + std::to_string(max_attempts) + " " + ip_address + " 2>&1";
if(!(in = popen(command.c_str(), "r"))) // open process as read only
{
std::cerr << __FILE__ << "(" << __FUNCTION__ << ":" << __LINE__ << ") | popen error = " << std::strerror(errno) << std::endl;
return -1;
}
while(fgets(buff, sizeof(buff), in) != NULL) // put response into stream
{
ss << buff;
}
exit_code = pclose(in); // blocks until process is done; returns exit status of command
details = ss.str();
}
catch (const std::exception &e)
{
std::cerr << __FILE__ << "(" << __FUNCTION__ << ":" << __LINE__ << ") | e.what() = " << e.what() << std::endl;
return -2;
}
return (exit_code == 0);
}
int system_ping::test_connection (std::string ip_address, int max_attempts, bool check_eth_port, int eth_port_number)
{
std::cout << __FILE__ << "(" << __FUNCTION__ << ":" << __LINE__ << ") | started" << std::endl;
int eth_conn_status_int;
std::string details;
try
{
if (check_eth_port)
{
std::ifstream eth_conn_status ("/sys/class/net/eth" + std::to_string(eth_port_number) + "/carrier");
eth_conn_status >> eth_conn_status_int; // 0: not connected; 1: connected
if (eth_conn_status_int != 1)
{
std::cerr << __FILE__ << "(" << __FUNCTION__ << ":" << __LINE__ << ") | eth" << std::to_string(eth_port_number) << " unplugged";
return -1;
}
}
if (ping_ip_address(ip_address, max_attempts, details) != 1)
{
std::cerr << __FILE__ << "(" << __FUNCTION__ << ":" << __LINE__ << ") | cannot ping " << ip_address << " | " << details << std::endl;
return -2;
}
}
catch (const std::exception &e)
{
std::cerr << __FILE__ << "(" << __FUNCTION__ << ":" << __LINE__ << ") | e.what() = " << e.what() << std::endl;
return -3;
}
std::cout << __FILE__ << "(" << __FUNCTION__ << ":" << __LINE__ << ") | ping " << ip_address << " OK" << std::endl;
std::cout << __FILE__ << "(" << __FUNCTION__ << ":" << __LINE__ << ") | ended" << std::endl;
return 0;
}