Related
I'm having troubles doing experiments on fpga and gpu using sycl.
I'm working online with intel devcloud and I get these 2 runtime errors when performing the executables:
On FPGA:
terminate called after throwing an instance of 'sycl::_V1::runtime_error'
what(): Invalid device program image: size is zero -30 (PI_ERROR_INVALID_VALUE)
on GPU:
terminate called after throwing an instance of 'sycl::_V1::runtime_error'
what(): Native API failed. Native API returns: -1 (PI_ERROR_DEVICE_NOT_FOUND) -1 (PI_ERROR_DEVICE_NOT_FOUND)
Here's the main Codes:
#include <sycl/sycl.hpp>
#include <sycl/ext/intel/fpga_extensions.hpp>
// #include <oneapi/mkl/blas.hpp>
#include <cmath>
#include <chrono>
#include <iostream>
#include <vector>
#include <cmath>
#include "guideline.h"
#include "print_vector.h"
#include "print_time.h"
#include "read_graph.h"
#include "flatVector.h"
using namespace sycl;
int main(int argc, char* argv[]){
// Check Command Line
if(argc < 6){
// NOT ENOUGH PARAMS BY COMMAND LINE -> PROGRAM HALTS
guideline();
}
else{
// Command Line parsing
int device_selected = atoi(argv[1]);
std::string csv_path = argv[2];
double threshold = atof(argv[3]);
double damping = atof(argv[4]);
int verbose;
try{verbose = atoi(argv[5]);}
catch (exception const& e) {verbose = 0;}
device d = device(default_selector());
// Queue
queue q(d);
std::cout << "Device : " << q.get_device().get_info<info::device::name>() << "\n"; // print del device
// Reading and setup Time Calculation
auto start_setup = std::chrono::steady_clock::now();
// Graph Retrieval by csv file
std::vector<std::vector<int>> graph = Read_graph(csv_path);/*Sparse Matrix Representation with the description of each Edge of the Graph*/
std::vector<int> flatGraph = flatten<int>(graph);
// Calculation of the # Nodes
int numNodes = countNodes(graph);
// Calculation of the Degree of each node
std::vector<int> degreesNodes = getDegrees(graph, numNodes+1);
auto end_setup = std::chrono::steady_clock::now();
// Setup Execution Time print
std::cout << "TIME FOR SETUP" << "\n";
print_time(start_setup, end_setup);
// Check Print
//printVector<int>(degreesNodes);
//Creation of Initial and Final Ranks' vectors of PageRank [R(t); R(t+1)]
std::vector<double> ranks_t(numNodes, (double)(1.0/ (double)(numNodes)));
std::vector<double> ranks_t_plus_one(numNodes, 0.0);
std::vector<double> ranksDifferences(numNodes, 0.0);
// PageRank Execution Time calculation
auto start = std::chrono::steady_clock::now();
buffer<int> bufferEdges(flatGraph.data(),flatGraph.size());
buffer<double> bufferRanks(ranks_t.data(),ranks_t.size());
buffer<int> bufferDegrees(degreesNodes.data(),degreesNodes.size());
buffer<double> bufferRanksNext(ranks_t_plus_one.data(),ranks_t_plus_one.size());
buffer<double> bufferRanksDifferences(ranksDifferences.data(),ranksDifferences.size());
double distance = threshold + 1;
int graph_size = flatGraph.size();
int T = 1;
while (distance > threshold) {
q.submit([&](handler &h){
accessor Edges(bufferEdges,h,read_only);
accessor Ranks(bufferRanks,h,read_only);
accessor Degrees(bufferDegrees,h,read_only);
accessor RanksNext(bufferRanksNext,h,write_only);
accessor RanksDifferences(bufferRanksDifferences,h,write_only);
h.parallel_for(range<1>(numNodes),[=] (id<1> i){
RanksNext[i] = (1.0 - damping) / numNodes;
int index_node_i;
int index_node_j;
for (int j = 0; j<graph_size;j+=2) {
index_node_i = j;
index_node_j = j + 1;
if (Edges[index_node_j] == i) {
RanksNext[i] += damping * Ranks[Edges[index_node_i]] / Degrees[Edges[index_node_i]];
}
}
RanksDifferences[i] = (RanksNext[i] - Ranks[i]) * (RanksNext[i] - Ranks[i]);
});
}).wait();
distance = 0;
for (int i = 0; i < numNodes; i++) {
distance += ranksDifferences[i];
ranks_t[i] = ranks_t_plus_one[i];
ranks_t_plus_one[i] = 0.0;
}
distance = sqrt(distance);
std::cout<< "Time:\t" << T << "\tEuclidian Distance:\t" << distance << std::endl;
T++;
}
auto end = std::chrono::steady_clock::now();
// PageRank Results Printing
if(verbose == 1){
for(int i = 0;i<ranks_t.size();i++){
std::cout<<"Final Vector" << i<< "-th component:\t"<<ranks_t[i]<<std::endl;
}
}
std::cout<<std::endl<<std::endl<<std::endl;
std::cout<<"Final Norm:\t"<<distance<<std::endl;
// PageRank Execution Time Printing
std::cout << "TIME FOR PAGERANK" << "\n";
print_time(start, end);
}
return 0;
}
flatVector.h
#include <iostream>
#include <vector>
template<typename T>
std::vector<T> flatten(const std::vector<std::vector<T>>& nestedVector) {
std::vector<T> flatVector;
for (const auto& subVector : nestedVector) {
for (const auto& element : subVector) {
flatVector.push_back(element);
}
}
return flatVector;
}
guideline.h
#include <iostream>
void guideline(){
std::cout<<"Not enough input parameters!\n\n";
std::cout<<"Usage guide:\n\n";
std::cout<<"First parameter:\tDevice code (as int number)\n";
std::cout<<"\t\t1: CPU\n";
std::cout<<"\t\t2: GPU\n";
std::cout<<"\t\t3: FPGA\n";
std::cout<<"Second parameter:\tCsv path of the dataset\n";
std::cout<<"Available Ones:\n\n";
std::cout<<"\t\t\"datasets/cit-Patents.csv\""<<std::endl;
std::cout<<"\t\t\"datasets/soc-LiveJournal1.csv\""<<std::endl;
std::cout<<"\t\t\"datasets/twitter-2010.csv\""<<std::endl;
std::cout<<"\t\t\"datasets/web-uk-2005-all.csv\""<<std::endl;
std::cout<<"Third parameter:\tThreshold (double value)\n";
std::cout<<"Fourth parameter:\tDamping (double value)\n";
std::cout<<"Fifth parameter:\tVerbose (int value)\n";
std::cout<<"Prints all the ranks of each node\n";
std::cout<<"Watch out! The print is huge\n";
}
printTime.h
#include <iostream>
#include <chrono>
void print_time(std::chrono::time_point<std::chrono::steady_clock> start, std::chrono::time_point<std::chrono::steady_clock> end){
std::cout << "Elapsed time in nanoseconds: " << std::chrono::duration_cast<std::chrono::nanoseconds>(end - start).count() << " ns" << std::endl;
std::cout << "Elapsed time in microseconds: " << std::chrono::duration_cast<std::chrono::microseconds>(end - start).count() << " µs" << std::endl;
std::cout << "Elapsed time in milliseconds: " << std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count() << " ms" << std::endl;
std::cout << "Elapsed time in seconds: " << std::chrono::duration_cast<std::chrono::seconds>(end - start).count() << " sec" << std::endl;
}
printVector.h
#include <iostream>
#include <vector>
template <typename T>
void printVector(std::vector<T> vector_like_var){
for(int i = 0; i < vector_like_var.size(); i++){
std::cout<< "element " <<i+1 << "of vector:\t" << vector_like_var[i] <<std::endl;
}
}
read_graph.h
#include <iostream>
#include <fstream>
#include <sstream>
#include <vector>
// #include "print_vector.h"
std::vector<int> getDegrees(const std::vector<std::vector<int>> &graph, int numNodes) {
std::vector<int> degrees(numNodes);
for (auto &edge : graph) {
++degrees[edge[0]];
++degrees[edge[1]];
}
return degrees;
}
std::vector<std::vector<int>> Read_graph(std::string file_name){
// Apertura del file
std::ifstream file(file_name);
if (!file.is_open()) {
std::cerr << "Impossibile aprire il file" << std::endl;
return {};
}
// Lettura del file riga per riga
std::string line;
std::vector<std::vector<int>> graph;
while (getline(file, line)) {
std::stringstream ss(line);
std::string cell;
std::vector<int> edge;
while (getline(ss, cell, ',')) {
edge.push_back(stoi(cell));
}
graph.push_back(edge);
}
file.close();
return graph;
}
int countNodes(std::vector<std::vector<int>> graph){
int numNodes = 0;
for(auto &i : graph){
for(auto &j : i){
numNodes = std::max(numNodes, j);
}
}
return numNodes;
}
I know that it runs on CPU, because I've done experiments and it prints out the ranks.
I tried executing on different devices the experiment, for example by using gen 9 GPUs of intel, but it was no use.
How to put a file with an arbitrary name and arbitrary size into a boost::interprocess::managed_shared_memory?
Note, I donot mean boost::interprocess::managed_mapped_file or
boost::interprocess::file_mapping.
I chose managed_shared_memory because other options require a fixed file name
to be specified but I need to transfer files with different names.
I need to use boost, not Win32 API.
I rummaged through a huge amount of information on the Internet, but did not
find anything suitable.
Therefore, I am asking you for help. I would be very grateful to you.
UPDATE
Added bonus versions at the end. Now this answer presents three complete versions of the code:
Using managed_shared_memory as requested
Using message_queue as a more natural appraoch for upload/transfter
Using TCP sockets (Asio) as to demonstrate the flexibilities of that
All of these are using Boost only
Shared memory managed segments contain arbitrary objects. So you define an object like
struct MyFile {
std::string _filename;
std::vector<char> _contents;
};
And store it there. But, wait, not so quick, because these can only be stored safely with interprocess allocators, so adding some magic sauce (a.k.a lots of interesting typedefs to get the allocators declared, and some constructors):
namespace Shared {
using Mem = bip::managed_shared_memory;
using Mgr = Mem::segment_manager;
template <typename T>
using Alloc = bc::scoped_allocator_adaptor<bip::allocator<T, Mgr>>;
template <typename T> using Vector = bc::vector<T, Alloc<T>>;
using String =
bc::basic_string<char, std::char_traits<char>, Alloc<char>>;
struct MyFile {
using allocator_type = Alloc<char>;
template <typename It>
explicit MyFile(std::string_view name, It b, It e, allocator_type alloc)
String _filename;
Vector<char> _contents;
};
}
Now you can store your files like:
Shared::Mem shm(bip::open_or_create, "shared_mem", 10ull << 30);
std::ifstream ifs("file_name.txt", std::ios::binary);
std::istreambuf_iterator<char> data_begin{ifs}, data_end{};
auto loaded = shm.find_or_construct<Shared::MyFile>("file1")(
file.native(), data_begin, data_end,
shm.get_segment_manager());
Note that the shared memory won't actually take 30GiB right away, even though
that's what 10ull << 30 specifies. On most operating systems this will be
sparesely allocated and only the pages that contain data will be commited.
Improving
You might have wondered what the scoped_allocator_adaptor was for. It doesn't seem we use it?
Well, the idea was to not use find_or_construct directly per file, but to
store a Vector<MyFile so you can harness the full power of BIP allocators.
The following full demo can be invoked
with filename arguments, which will all be loaded (if they exist as
regular files)
without arguments, which will list previously loaded files
Live On Coliru
#include <boost/interprocess/managed_shared_memory.hpp>
#include <boost/interprocess/managed_mapped_file.hpp> // for COLIRU
#include <boost/interprocess/containers/vector.hpp>
#include <boost/interprocess/containers/string.hpp>
#include <boost/container/scoped_allocator.hpp>
#include <fstream>
#include <filesystem>
#include <iostream>
#include <iomanip>
namespace bip = boost::interprocess;
namespace bc = boost::container;
namespace fs = std::filesystem;
namespace Shared {
#ifdef COLIRU
using Mem = bip::managed_mapped_file; // managed_shared_memory not allows
#else
using Mem = bip::managed_shared_memory;
#endif
using Mgr = Mem::segment_manager;
template <typename T>
using Alloc = bc::scoped_allocator_adaptor<bip::allocator<T, Mgr>>;
template <typename T> using Vector = bc::vector<T, Alloc<T>>;
using String = bc::basic_string<char, std::char_traits<char>, Alloc<char>>;
struct MyFile {
using allocator_type = Alloc<char>;
MyFile(MyFile&&) = default;
MyFile(MyFile const& rhs, allocator_type alloc)
: _filename(rhs._filename.begin(), rhs._filename.end(), alloc),
_contents(rhs._contents.begin(), rhs._contents.end(), alloc) {}
MyFile& operator=(MyFile const& rhs) {
_filename.assign(rhs._filename.begin(), rhs._filename.end());
_contents.assign(rhs._contents.begin(), rhs._contents.end());
return *this;
}
template <typename It>
explicit MyFile(std::string_view name, It b, It e, allocator_type alloc)
: _filename(name.data(), name.size(), alloc),
_contents(b, e, alloc) {}
String _filename;
Vector<char> _contents;
friend std::ostream& operator<<(std::ostream& os, MyFile const& mf) {
return os << "Name: " << std::quoted(mf._filename.c_str())
<< " content size: " << mf._contents.size();
}
};
} // namespace Shared
int main(int argc, char** argv) {
Shared::Mem shm(bip::open_or_create, "shared_mem", 512ull << 10);
using FileList = Shared::Vector<Shared::MyFile>;
auto& shared_files =
*shm.find_or_construct<FileList>("FileList")(shm.get_segment_manager());
if (1==argc) {
std::cout << "Displaying previously loaded files: \n";
for (auto& entry : shared_files)
std::cout << entry << std::endl;
} else {
std::cout << "Loading files: \n";
for (auto file : std::vector<fs::path>{argv + 1, argv + argc}) {
if (is_regular_file(file)) {
try {
std::ifstream ifs(file, std::ios::binary);
std::istreambuf_iterator<char> data_begin{ifs}, data_end{};
auto& loaded = shared_files.emplace_back(
file.native(), data_begin, data_end);
std::cout << loaded << std::endl;
} catch (std::system_error const& se) {
std::cerr << "Error: " << se.code().message() << std::endl;
} catch (std::exception const& se) {
std::cerr << "Other: " << se.what() << std::endl;
}
}
}
}
}
When run with
g++ -std=c++17 -O2 -Wall -pedantic -pthread main.cpp -lrt -DCOLIRU
./a.out main.cpp a.out
./a.out
Prints
Loading files:
Name: "main.cpp" content size: 3239
Name: "a.out" content size: 175176
Displaying previously loaded files:
Name: "main.cpp" content size: 3239
Name: "a.out" content size: 175176
BONUS
In response to the comments, I think it's worth actually comparing
Message Queue version
For comparison, here's a message queue implementation
Live On Coliru
#include <boost/interprocess/ipc/message_queue.hpp>
#include <boost/endian/arithmetic.hpp>
#include <fstream>
#include <filesystem>
#include <iostream>
#include <iomanip>
namespace bip = boost::interprocess;
namespace fs = std::filesystem;
using bip::message_queue;
static constexpr auto MAX_FILENAME_LENGH = 512; // 512 bytes max filename length
static constexpr auto MAX_CONTENT_SIZE = 512ull << 10; // 512 KiB max payload size
struct Message {
std::vector<char> _buffer;
using Uint32 = boost::endian::big_uint32_t;
struct header_t {
Uint32 filename_length;
Uint32 content_size;
};
static_assert(std::is_standard_layout_v<header_t> and
std::is_trivial_v<header_t>);
Message() = default;
Message(fs::path file) {
std::string const name = file.native();
std::ifstream ifs(file, std::ios::binary);
std::istreambuf_iterator<char> data_begin{ifs}, data_end{};
_buffer.resize(header_len + name.length());
std::copy(begin(name), end(name), _buffer.data() + header_len);
_buffer.insert(_buffer.end(), data_begin, data_end);
header().filename_length = name.length();
header().content_size = size() - header_len - name.length();
}
Message(char const* buf, size_t size)
: _buffer(buf, buf+size) {}
static constexpr auto header_len = sizeof(header_t);
static constexpr auto max_size =
header_len + MAX_FILENAME_LENGH + MAX_CONTENT_SIZE;
char const* data() const { return _buffer.data(); }
size_t size() const { return _buffer.size(); }
header_t& header() {
assert(_buffer.size() >= header_len);
return *reinterpret_cast<header_t*>(_buffer.data());
}
header_t const& header() const {
assert(_buffer.size() >= header_len);
return *reinterpret_cast<header_t const*>(_buffer.data());
}
std::string_view filename() const {
assert(_buffer.size() >= header_len + header().filename_length);
return { _buffer.data() + header_len, header().filename_length };
}
std::string_view contents() const {
assert(_buffer.size() >=
header_len + header().filename_length + header().content_size);
return {_buffer.data() + header_len + header().filename_length,
header().content_size};
}
friend std::ostream& operator<<(std::ostream& os, Message const& mf) {
return os << "Name: " << std::quoted(mf.filename())
<< " content size: " << mf.contents().size();
}
};
int main(int argc, char** argv) {
message_queue mq(bip::open_or_create, "file_transport", 10, Message::max_size);
if (1==argc) {
std::cout << "Receiving uploaded files: \n";
char rawbuf [Message::max_size];
while (true) {
size_t n;
unsigned prio;
mq.receive(rawbuf, sizeof(rawbuf), n, prio);
Message decoded(rawbuf, n);
std::cout << "Received: " << decoded << std::endl;
}
} else {
std::cout << "Loading files: \n";
for (auto file : std::vector<fs::path>{argv + 1, argv + argc}) {
if (is_regular_file(file)) {
try {
Message encoded(file);
std::cout << "Sending: " << encoded << std::endl;
mq.send(encoded.data(), encoded.size(), 0);
} catch (std::system_error const& se) {
std::cerr << "Error: " << se.code().message() << std::endl;
} catch (std::exception const& se) {
std::cerr << "Other: " << se.what() << std::endl;
}
}
}
}
}
A demo:
Note that there is a filesize limit in this approach because messages have a maximum length
TCP Socket Version
Here's a TCP socket implementation.
Live On Coliru
#include <boost/asio.hpp>
#include <boost/endian/arithmetic.hpp>
#include <vector>
#include <fstream>
#include <filesystem>
#include <iostream>
#include <iomanip>
namespace fs = std::filesystem;
using boost::asio::ip::tcp;
using boost::system::error_code;
static constexpr auto MAX_FILENAME_LENGH = 512; // 512 bytes max filename length
static constexpr auto MAX_CONTENT_SIZE = 512ull << 10; // 512 KiB max payload size
struct Message {
std::vector<char> _buffer;
using Uint32 = boost::endian::big_uint32_t;
struct header_t {
Uint32 filename_length;
Uint32 content_size;
};
static_assert(std::is_standard_layout_v<header_t> and
std::is_trivial_v<header_t>);
Message() = default;
Message(fs::path file) {
std::string const name = file.native();
std::ifstream ifs(file, std::ios::binary);
std::istreambuf_iterator<char> data_begin{ifs}, data_end{};
_buffer.resize(header_len + name.length());
std::copy(begin(name), end(name), _buffer.data() + header_len);
_buffer.insert(_buffer.end(), data_begin, data_end);
header().filename_length = name.length();
header().content_size = actual_size() - header_len - name.length();
}
Message(char const* buf, size_t size)
: _buffer(buf, buf+size) {}
static constexpr auto header_len = sizeof(header_t);
static constexpr auto max_size =
header_len + MAX_FILENAME_LENGH + MAX_CONTENT_SIZE;
char const* data() const { return _buffer.data(); }
size_t actual_size() const { return _buffer.size(); }
size_t decoded_size() const {
return header().filename_length + header().content_size;
}
bool is_complete() const {
return actual_size() >= header_len && actual_size() >= decoded_size();
}
header_t& header() {
assert(actual_size() >= header_len);
return *reinterpret_cast<header_t*>(_buffer.data());
}
header_t const& header() const {
assert(actual_size() >= header_len);
return *reinterpret_cast<header_t const*>(_buffer.data());
}
std::string_view filename() const {
assert(actual_size() >= header_len + header().filename_length);
return std::string_view(_buffer.data() + header_len,
header().filename_length);
}
std::string_view contents() const {
assert(actual_size() >= decoded_size());
return std::string_view(_buffer.data() + header_len +
header().filename_length,
header().content_size);
}
friend std::ostream& operator<<(std::ostream& os, Message const& mf) {
return os << "Name: " << std::quoted(mf.filename())
<< " content size: " << mf.contents().size();
}
};
int main(int argc, char** argv) {
boost::asio::io_context ctx;
u_int16_t port = 8989;
if (1==argc) {
std::cout << "Receiving uploaded files: " << std::endl;
tcp::acceptor acc(ctx, tcp::endpoint{{}, port});
while (true) {
auto s = acc.accept();
std::cout << "Connection accepted from " << s.remote_endpoint() << std::endl;
Message msg;
auto buf = boost::asio::dynamic_buffer(msg._buffer);
error_code ec;
while (auto n = read(s, buf, ec)) {
std::cout << "(read " << n << " bytes, " << ec.message() << ")" << std::endl;
while (msg.is_complete()) {
std::cout << "Received: " << msg << std::endl;
buf.consume(msg.decoded_size() + Message::header_len);
}
}
std::cout << "Connection closed" << std::endl;
}
} else {
std::cout << "Loading files: " << std::endl;
tcp::socket s(ctx);
s.connect(tcp::endpoint{{}, port});
for (auto file : std::vector<fs::path>{argv + 1, argv + argc}) {
if (is_regular_file(file)) {
try {
Message encoded(file);
std::cout << "Sending: " << encoded << std::endl;
write(s, boost::asio::buffer(encoded._buffer));
} catch (std::system_error const& se) {
std::cerr << "Error: " << se.code().message() << std::endl;
} catch (std::exception const& se) {
std::cerr << "Other: " << se.what() << std::endl;
}
}
}
}
}
Demo:
Note how this easily scales to larger files, multiple files in a single connection and even multiple connections simultaneously if you need. It also doesn't do double buffering, which improves performance.
This is why this kind of approach is much more usual than any of your other approaches.
I'm asking myself if it is possible to extend boost-range by an adaptor, which I call adjacentAdaptor. This adaptor should basically iterate over all pairs of adjacent elements in a vector, list and so on.
I think this function is very useful in my use cases, where I often have to iterate over lists representing time steps.
The output of the last for loop should be something like:
0 1
1 2
2 3
A vector having only one element or no elements should produce nothing.
I tried using boost::adaptors::sliced producing the necessary sublist, but then I don't know how boost::range can help me to zip both subranges to one.
I just found a probable solution using boost::iterators, but I really don't like the amount of code one has to write. Also I'm missing the first and second instead I have to write a clumsy get<>. Unfortunately, the program crashes if the vector is empty!
#include <vector>
#include <iostream>
#include <boost/range.hpp>
#include <boost/range/algorithm/transform.hpp>
#include <boost/range/adaptor/sliced.hpp>
#include <boost/iterator.hpp>
#include <boost/iterator/zip_iterator.hpp>
int main()
{
std::vector<int> v = { 0,1,2,3 };
for (auto iter : v | boost::adaptors::sliced(0, v.size() - 1)) {
std::cout << "First: " << iter << std::endl;
}
for (auto iter : v | boost::adaptors::sliced(1, v.size())) {
std::cout << "Second: "<< iter << std::endl;
}
auto s = boost::iterators::make_zip_iterator(boost::make_tuple(v.begin(), v.begin() + 1));
auto e = boost::iterators::make_zip_iterator(boost::make_tuple(v.end()-1, v.end()));
for (auto iter : boost::make_iterator_range(s, e)) {
std::cout << iter.get<0>() << " " << iter.get<1>() << std::endl;
}
// for (auto iter : v | adjacentAdaptor) {
// std::cout << iter.first << " " << iter.second << std::endl;
// }
}
I'm very glad for any help I can receive in this question.
Own partial solution
After some template type deduction I came up with something relatively useable.
#include <vector>
#include <iostream>
#include <boost/range.hpp>
#include <boost/range/algorithm/transform.hpp>
#include <boost/range/adaptor/sliced.hpp>
#include <boost/iterator.hpp>
#include <boost/iterator/zip_iterator.hpp>
template<typename T>
using retHelperType = decltype(boost::iterators::make_zip_iterator(boost::make_tuple(T().begin(), T().begin() + 1)));
template<typename T>
using retType = decltype(boost::make_iterator_range(retHelperType<T>(), retHelperType<T>()));
template<typename T>
retType<T> adjacentIterator(T& v) {
if (v.empty()) {
auto s = boost::iterators::make_zip_iterator(boost::make_tuple(v.end(), v.end()));
auto e = boost::iterators::make_zip_iterator(boost::make_tuple(v.end(), v.end()));
return boost::make_iterator_range(s, e);
}
else {
auto s = boost::iterators::make_zip_iterator(boost::make_tuple(v.begin(), std::next(v.begin())));
auto e = boost::iterators::make_zip_iterator(boost::make_tuple(std::prev(v.end()), v.end()));
return boost::make_iterator_range(s, e);
}
}
int main()
{
retType<std::vector<int>> x;
std::vector<int> v = { };
for (auto iter : adjacentIterator(v)) {
std::cout << iter.get<0>() << " " << iter.get<1>() << std::endl;
}
}
Still, it would be nicer to access the elements with first and second, but I have no idea to achieve this behavior.
I want to share structured data between C++ and Python languages using MessagePack like this one:
{
"t" : [ [t00,...,t0N], ... , [tM0,...,tMN] ],
"x" : [ x0,..,xN],
"P" : [ [P00, ..., P0N], ..., [PM0,...,PMN] ]
}
The number of variables is optional so in some cases I will have for example only:
{
"t" : [ [t00,...,t0N], ... , [tM0,...,tMN] ]
}
Decoding this in Python is pretty simple, my problem is to figure out
how to decode this in C++ if I don't know in advance the structure of
the data ? or the exact number of variables that I would have; is it
possible to iterate the structure in these cases?
I managed to handle a "fixed" data structure ( always with the same
number of variables ) defining a struct for example:
struct variables
{
std::vector< std::vector<double> > t;
std::vector< double > x;
std::vector< std::vector<double> > P;
MSPACK_DEFINE_MAP( t, x, P );
};
std::stringstream inBuffer;
.... (read data )
std::string str( inBuffer.str() );
msgpack::object_handle oh = msgpack::unpack( str.data(), str.size() );
msgpack::object deserialized = oh.get();
variables var;
deserialized.convert( var );
Is there a better way to accomplish this ?, how could manage optional
variables that could not appear in the structure ?; I repeat the
previous question: could I iterate an unknown data structure in C++?,
how ?
Thanks in advance!
Regards, Ernesto
There are two ways to treat unknown data structure.
The first way is using parse/visitor mechanism.
Here is an example:
#include <msgpack.hpp>
#include <sstream>
#include <iostream>
// This is a simple print example visitor.
// You can do any processing in your visitor.
struct my_visitor : msgpack::null_visitor {
bool start_map_key() {
processing_map_key = true;
return true;
}
bool end_map_key() {
processing_map_key = false;
return true;
}
bool start_array(uint32_t size) {
std::cout << "array (size:" << size << ")[" << std::endl;
return true;
}
bool end_array() {
std::cout << "]" << std::endl;
return true;
}
bool visit_str(const char* v, uint32_t size) {
if (processing_map_key) {
std::cout << "map key:" << std::string(v, size) << std::endl;
}
return true;
}
bool visit_positive_integer(uint64_t v) {
std::cout << "found value:" << v << std::endl;
return true;
}
bool processing_map_key = false;
std::string indent;
};
int main() {
// create test data
std::stringstream ss;
msgpack::packer<std::stringstream> pk(ss);
pk.pack_map(1);
pk.pack("t");
pk.pack_array(2);
pk.pack_array(3);
pk.pack(1);
pk.pack(2);
pk.pack(3);
pk.pack_array(3);
pk.pack(4);
pk.pack(5);
pk.pack(6);
// print data (for debug)
{
auto oh = msgpack::unpack(ss.str().data(), ss.str().size());
std::cout << oh.get() << std::endl;
}
// apply visitor
{
my_visitor mv;
msgpack::parse(ss.str().data(), ss.str().size(), mv);
}
}
Running demo: https://wandbox.org/permlink/3NrR4IMDIuLTk9e9
See https://github.com/msgpack/msgpack-c/wiki/v2_0_cpp_visitor.
The other way is using msgpack::type::variant or `msgpack::type::variant_ref.
The former copies data, you can update it. The latter doesn't copy data. You cannot update it.
This approach requires boost. So you need to define MSGPACK_USE_BOOST. I recommend defining as a compiler option.
// Boost is required
#define MSGPACK_USE_BOOST
#include <msgpack.hpp>
#include <sstream>
#include <iostream>
struct my_visitor:boost::static_visitor<void> {
void operator()(uint64_t v) const {
std::cout << "positive insteger:" << v << std::endl;
}
// const is required for map key because std::multimap's key (first) is const.
void operator()(std::string const& v) const {
std::cout << "string:" << v << std::endl;
}
void operator()(std::vector<msgpack::type::variant>& v) const {
std::cout << "array found" << std::endl;
for (auto& e : v) {
boost::apply_visitor(*this, e);
}
}
void operator()(std::multimap<msgpack::type::variant, msgpack::type::variant>& v) const {
std::cout << "map found" << std::endl;
for (auto& e : v) {
std::cout << "key:" << std::endl;
boost::apply_visitor(*this, e.first);
std::cout << "value:" << std::endl;
boost::apply_visitor(*this, e.second);
}
}
template <typename T>
void operator()(T const&) const {
std::cout << " match others" << std::endl;
}
};
int main() {
// create test data
std::stringstream ss;
msgpack::packer<std::stringstream> pk(ss);
pk.pack_map(1);
pk.pack("t");
pk.pack_array(2);
pk.pack_array(3);
pk.pack(1);
pk.pack(2);
pk.pack(3);
pk.pack_array(3);
pk.pack(4);
pk.pack(5);
pk.pack(6);
auto oh = msgpack::unpack(ss.str().data(), ss.str().size());
std::cout << oh.get() << std::endl;
msgpack::type::variant v = oh.get().as<msgpack::type::variant>();
boost::apply_visitor(my_visitor(), v);
}
Running demo: https://wandbox.org/permlink/HQwJjfwW8rLEMi0d
See https://github.com/msgpack/msgpack-c/wiki/v2_0_cpp_variant
Here are exampless:
https://github.com/msgpack/msgpack-c/blob/master/example/boost/msgpack_variant_capitalize.cpp
https://github.com/msgpack/msgpack-c/blob/master/example/boost/msgpack_variant_mapbased.cpp
Both ways can treat unpredictable data structure. You need to do some visitor processing. If the data structure is predictable some extent, your original approach is also good way.
Actually there is a simpler way, if you are dealing with maps (like stated in the question), not arrays.
msgpack::object_handle oh = msgpack::unpack(/* some data */);
std::map<std::string,msgpack::type::variant> map = obj.convert();
This way you will get a map with all the data, no need for a visitor or boost.
I have a vector of custom classes (std::string just for example).
The vector is large and I iterate through often, so I rely on cache locality.
I also have one raw pointer which points at one of the vector elements.
Now is the trick:
The vector is sorted from time to time, so the raw pointer loose the actual pointed element value, and will point to some random element value.
Here is an example to illustrate the same:
#include <iostream>
#include <algorithm>
#include <string>
#include <vector>
#include <memory>
using namespace std;
int main()
{
vector<string> v = {"9","3", "8", "7", "6", "5", "1", "4", "2"};
string* rs = &v[7]; //point to the 7th element
for (size_t i = 0; i < v.size(); ++i)
cerr << v[i];
cerr << endl;
cerr << "Referenced string: " << rs->c_str() << endl;
cerr << "Sort ..." << endl;
sort(v.begin(), v.end(), [](const string& a, const string& b)
{
if (a < b)
return true;
else
return false;
}
);
for (size_t i = 0; i < v.size(); ++i)
cerr << v[i];
cerr << endl;
cerr << "Referenced string: " << rs->c_str() << endl;
cin.get();
return 0;
}
Output:
938765142
Referenced string before sort : 4
Sort ...
123456789
Referenced string after sort : 8
Since I wish the rs pointer to keep pointing to the 7th element value (which is 4) even after the sort, I came up with the following solution (vector of pointers):
#include <iostream>
#include <algorithm>
#include <string>
#include <vector>
#include <memory>
using namespace std;
int main()
{
vector<unique_ptr<string>> v;
v.resize(9);
v[0] = make_unique<string>("9");
v[1] = make_unique<string>("3");
v[2] = make_unique<string>("8");
v[3] = make_unique<string>("7");
v[4] = make_unique<string>("6");
v[5] = make_unique<string>("5");
v[6] = make_unique<string>("1");
v[7] = make_unique<string>("4");
v[8] = make_unique<string>("2");
string* rs = v[7].get();
for (size_t i = 0; i < v.size(); ++i)
cerr << v[i]->c_str();
cerr << endl;
cerr << "Referenced string before sort: " << rs->c_str() << endl;
cerr << "Sort ..." << endl;
sort(v.begin(), v.end(), [](const unique_ptr<string>& a, const unique_ptr<string>& b)
{
if (*a < *b)
return true;
else
return false;
}
);
for (size_t i = 0; i < v.size(); ++i)
cerr << v[i]->c_str();
cerr << endl;
cerr << "Referenced string after sort: " << rs->c_str() << endl;
cin.get();
return 0;
}
Output:
938765142
Referenced string before sort: 4
Sort ...
123456789
Referenced string after sort: 4
While this latter solution works, there is a price: I have lost the cache locality of my vector, since I store pointers in it, rather than the actual objects.
Is there a way to maintain cache locality (e.g.: store my actual objects in the vector), and somehow manage to rs pointer to keep track where its pointed value wander around due to the sorts?
Or from the other perspective, is there a way to achieve cache locality with the vector of pointers?
Solution from Pubby, thanks!:
#include <iostream>
#include <algorithm>
#include <string>
#include <vector>
#include <memory>
using namespace std;
int main()
{
vector<string> data = { "d","e", "f", "g", "i", "b", "c", "a", "h" };
vector<int> indexes = {0,1,2,3,4,5,6,7,8};
int si = 6;
for (size_t i = 0; i < indexes.size(); ++i)
cerr << indexes[i];
cerr << endl;
for (size_t i = 0; i < indexes.size(); ++i)
cerr << data[indexes[i]];
cerr << endl;
cerr << "Referenced string before sort: " << data[si] << endl;
cerr << "Sort ..." << endl;
sort(indexes.begin(), indexes.end(), [&](const int a, const int b)
{
return data[a] < data[b];
}
);
for (size_t i = 0; i < indexes.size(); ++i)
cerr << indexes[i];
cerr << endl;
for (size_t i = 0; i < indexes.size(); ++i)
cerr << data[indexes[i]];
cerr << endl;
cerr << "Referenced string after sort: " << data[si] << endl;
cin.get();
return 0;
}
You can increase locality by storing the strings in a vector which doesn't change, and then store a vector of pointers/indexes to these strings.
Like this:
vector<string> data = {"9","3", "8", "7", "6", "5", "1", "4", "2"};
vector<unsigned> indexes(data.size());
std::iota(indexes.begin(), indexes.end(), 0u);
To sort your data you'd sort indexes using a custom comparator function which retrieves the values from data and compares them. Remember: indexes can change, but data should not!
sort(indexes.begin(), indexes.end(), [&](unsigned a, unsigned b)
{
return data[a] < data[b];
});
Just an idea: Instead of storing std::string in the vector, just append the character arrays of each string to a std::vector<char>.
This packs the strings closely together in memory, improving locality even better than std::string with small string optimization. It will also give better results if the strings exceed the max. size for small string optimization.
For sorting, store index and size of each string in a 2nd vector similar to Pubbys suggestion.
Of course this only works if the string length doesn't need to change dynamically. Otherwise you would have to rebuild the vector<char>.
#include <iostream>
#include <algorithm>
#include <vector>
#include <utility>
#include <string_view>
using namespace std;
using IndexAndSize = pair<size_t,size_t>;
void push_and_index( vector<char>& v, vector<IndexAndSize>& vi, string_view s )
{
vi.emplace_back( v.size(), s.size() );
v.insert( end(v), begin(s), end(s) );
}
string_view make_string_view( vector<char> const& v, IndexAndSize is )
{
return { v.data() + is.first, is.second };
}
int main()
{
vector<char> v;
vector<IndexAndSize> vi;
push_and_index( v, vi, "foo" );
push_and_index( v, vi, "bar" );
push_and_index( v, vi, "foobar" );
push_and_index( v, vi, "barfoo" );
sort( begin(vi), end(vi), [&]( IndexAndSize a, IndexAndSize b )
{
return make_string_view( v, a ) < make_string_view( v, b );
});
for( IndexAndSize is : vi )
{
cout << make_string_view( v, is ) << endl;
}
}
Live demo on Coliru.
Note: C++17's string_view is used only to help with the sorting and output, it's not crucial for this idea.