First of all I am using MSVC 2017 (latest version).
Here is my code for the nonterminal parser:
player.hpp
namespace parse
{
namespace impl
{
namespace x3 = boost::spirit::x3;
struct _tag;
using player_type = x3::rule<_tag, PlayerIterator>;
using player_vector_type = x3::rule<_tag, std::vector<PlayerIterator>>;
BOOST_SPIRIT_DECLARE(player_type);
BOOST_SPIRIT_DECLARE(player_vector_type);
}; //impl
impl::player_type player();
impl::player_vector_type player_vector();
}; //parse
player.cpp
namespace parse
{
namespace impl
{
const player_type player = "player";
const player_vector_type player_vector = "player_vector";
auto player_find = [](auto &ctx)
{
auto &attr = x3::_attr(ctx);
if(attr.which() == 0)
return x3::_val(ctx) = PlayerManager::find(boost::get<int>(attr));
return x3::_val(ctx) = PlayerManager::find(boost::get<std::string>(attr));
};
auto player_vector_find = [](auto &ctx)
{
return x3::_val(ctx) = PlayerManager::vector_find(x3::_attr(ctx));
};
auto const player_def = (x3::int_ | (+x3::char_))[player_find];
auto const player_vector_def = (((+x3::char_)[player_vector_find]));
BOOST_SPIRIT_DEFINE(player);
BOOST_SPIRIT_DEFINE(player_vector);
BOOST_SPIRIT_INSTANTIATE(player_type, iterator_type, context_type);
BOOST_SPIRIT_INSTANTIATE(player_vector_type, iterator_type, context_type);
} //impl
parse::impl::player_type player() { return impl::player; }
parse::impl::player_vector_type player_vector() { return impl::player_vector; }
}//parse
I get linker LNK2019 errors about "unresolved external symbols referenced":
Pastebin.com link with the errors
Any ideas about them?
Thanks in advance.
EDIT:
That's how I call it in my source file:
void test(std::string ¶ms)
{
std::tuple<PlayerIterator, std::vector<PlayerIterator>, std::string> tuple;
if (!x3::phrase_parse(params.begin(), params.end(), parse::player()>> parse::player_vector() >> (+x3::char_), x3::space,tuple))
{
std::cout << "Error: Parsing failed" << std::endl;
return;
}
std::cout << "Parsing succeded" << std::endl;
std::cout << "Found player, size of player vector: "<< std::get<1>(tuple).size() << ", also parsed string:" << std::get<2>(tuple);
return;
};
I'm willing to bet $10 that you mismatched the context or iterator types on the instantiations.
E.g. in your test function, the argument is std::string&, hence params.begin() will be std::string::iterator. If you had the iterator_type configured as follows:
using iterator_type = std::string::const_iterator; // very sensible!
you would have unresolved externals because the iterator type doesn't match the one actually required.
Same thing for the context. To match your invocation it needs to be exactly:
using context_type = x3::phrase_parse_context<x3::space_type>::type;
Sadly you didn't show the whole code, so you'll have to check on your own.
Notes
re-using the tag type is recipe for disaster. I don't think it can work. The rule tags are what dispatches the implementation function in the case of separated compilation units. Fix it:
using player_type = x3::rule<struct player_tag, PlayerIterator>;
using player_vector_type = x3::rule<struct player_vector_tag, std::vector<PlayerIterator>>;
copying the rules seems wasteful, consider returning by reference:
impl::player_type const& player();
impl::player_vector_type const& player_vector();
Note: this should be fine w.r.t. static initialization order fiasco
using which() on a variant is an anti-pattern. You can replace
auto player_find = [](auto &ctx) {
auto &attr = x3::_attr(ctx);
if (attr.which() == 0)
return x3::_val(ctx) = PlayerManager::find(boost::get<int>(attr));
return x3::_val(ctx) = PlayerManager::find(boost::get<std::string>(attr));
};
With
auto find = [](auto const& key) { return PlayerManager::find(key); };
auto player_find = [](auto &ctx) {
return x3::_val(ctx) = boost::apply_visitor(find, x3::_attr(ctx));
};
(+x3::char_) always matches all input
(+x3::graph) still matches all input because of the skipper
Instead you wanted a lexeme:
auto const name = x3::lexeme[+x3::graph];
auto const player_def = (x3::int_ | name) [player_find];
auto const player_vector_def = name[ player_vector_find];
May I suggest to write the test function a lot more concisely:
void test(std::string const ¶ms) {
auto comment_ = x3::lexeme[+x3::char_];
PlayerIterator player;
PlayerIterators vec;
std::string comment;
auto tuple = std::tie(player, vec, comment);
if (phrase_parse(params.cbegin(), params.cend(), parse::player() >> parse::player_vector() >> comment_, x3::space, tuple)) {
std::cout << "Parsing succeded" << std::endl;
std::cout << "Found player, size of player vector: " << vec.size() << "\n";
std::cout << "Also parsed string: " << std::quoted(comment);
} else {
std::cout << "Error: Parsing failed" << std::endl;
}
}
Full Demo
See it Live On Wandbox
stuff.h
Contains mockup PlayerManager
#pragma once
#include <string>
#include <vector>
#include <iostream>
struct PlayerIterator { };
using PlayerIterators = std::vector<PlayerIterator>;
struct PlayerManager {
static PlayerIterator find(std::string const&) { std::cout << __PRETTY_FUNCTION__ << "\n"; return {}; }
static PlayerIterator find(int) { std::cout << __PRETTY_FUNCTION__ << "\n"; return {}; }
static PlayerIterators vector_find(std::string const&) { std::cout << __PRETTY_FUNCTION__ << "\n"; return {}; }
};
test.h
#pragma once
#include <boost/spirit/home/x3.hpp>
#include <boost/fusion/adapted.hpp>
#include "stuff.h"
namespace x3 = boost::spirit::x3;
namespace parse
{
namespace impl
{
using player_type = x3::rule<struct player_tag, PlayerIterator>;
using player_vector_type = x3::rule<struct player_vector_tag, PlayerIterators>;
BOOST_SPIRIT_DECLARE(player_type)
BOOST_SPIRIT_DECLARE(player_vector_type)
} //impl
impl::player_type const& player();
impl::player_vector_type const& player_vector();
} //parse
test.cpp
#include "stuff.h"
#include "test.h"
using iterator_type = std::string::const_iterator;
using context_type = x3::phrase_parse_context<x3::space_type>::type;
namespace parse {
namespace impl {
const player_type player = "player";
const player_vector_type player_vector = "player_vector";
auto find = [](auto const& key) { return PlayerManager::find(key); } ;
auto player_find = [](auto &ctx) { return x3::_val(ctx) = boost::apply_visitor(find, x3::_attr(ctx)); } ;
auto player_vector_find = [](auto &ctx) { return x3::_val(ctx) = PlayerManager::vector_find(x3::_attr(ctx)); } ;
auto const name = x3::lexeme[+x3::graph];
auto const player_def = (x3::int_ | name) [player_find];
auto const player_vector_def = name[ player_vector_find];
BOOST_SPIRIT_DEFINE(player)
BOOST_SPIRIT_DEFINE(player_vector)
BOOST_SPIRIT_INSTANTIATE(player_type, iterator_type, context_type)
BOOST_SPIRIT_INSTANTIATE(player_vector_type, iterator_type, context_type)
} // namespace impl
parse::impl::player_type const& player() { return impl::player; }
parse::impl::player_vector_type const& player_vector() { return impl::player_vector; }
} // namespace parse
main.cpp
#include "stuff.h"
#include "test.h"
#include <iostream>
#include <iomanip>
void test(std::string const ¶ms) {
auto comment_ = x3::lexeme[+x3::char_];
PlayerIterator player;
PlayerIterators vec;
std::string comment;
auto tuple = std::tie(player, vec, comment);
if (phrase_parse(params.cbegin(), params.cend(), parse::player() >> parse::player_vector() >> comment_, x3::space, tuple)) {
std::cout << "Parsing succeded" << std::endl;
std::cout << "Found player, size of player vector: " << vec.size() << "\n";
std::cout << "Also parsed string: " << std::quoted(comment);
} else {
std::cout << "Error: Parsing failed" << std::endl;
}
}
int main() {
test("42 someword # bogus trailing comment");
}
Prints:
static PlayerIterator PlayerManager::find(int)
static PlayerIterators PlayerManager::vector_find(const std::string &)
Parsing succeded
Found player, size of player vector: 0
Also parsed string: "# bogus trailing comment"
Related
How to put a file with an arbitrary name and arbitrary size into a boost::interprocess::managed_shared_memory?
Note, I donot mean boost::interprocess::managed_mapped_file or
boost::interprocess::file_mapping.
I chose managed_shared_memory because other options require a fixed file name
to be specified but I need to transfer files with different names.
I need to use boost, not Win32 API.
I rummaged through a huge amount of information on the Internet, but did not
find anything suitable.
Therefore, I am asking you for help. I would be very grateful to you.
UPDATE
Added bonus versions at the end. Now this answer presents three complete versions of the code:
Using managed_shared_memory as requested
Using message_queue as a more natural appraoch for upload/transfter
Using TCP sockets (Asio) as to demonstrate the flexibilities of that
All of these are using Boost only
Shared memory managed segments contain arbitrary objects. So you define an object like
struct MyFile {
std::string _filename;
std::vector<char> _contents;
};
And store it there. But, wait, not so quick, because these can only be stored safely with interprocess allocators, so adding some magic sauce (a.k.a lots of interesting typedefs to get the allocators declared, and some constructors):
namespace Shared {
using Mem = bip::managed_shared_memory;
using Mgr = Mem::segment_manager;
template <typename T>
using Alloc = bc::scoped_allocator_adaptor<bip::allocator<T, Mgr>>;
template <typename T> using Vector = bc::vector<T, Alloc<T>>;
using String =
bc::basic_string<char, std::char_traits<char>, Alloc<char>>;
struct MyFile {
using allocator_type = Alloc<char>;
template <typename It>
explicit MyFile(std::string_view name, It b, It e, allocator_type alloc)
String _filename;
Vector<char> _contents;
};
}
Now you can store your files like:
Shared::Mem shm(bip::open_or_create, "shared_mem", 10ull << 30);
std::ifstream ifs("file_name.txt", std::ios::binary);
std::istreambuf_iterator<char> data_begin{ifs}, data_end{};
auto loaded = shm.find_or_construct<Shared::MyFile>("file1")(
file.native(), data_begin, data_end,
shm.get_segment_manager());
Note that the shared memory won't actually take 30GiB right away, even though
that's what 10ull << 30 specifies. On most operating systems this will be
sparesely allocated and only the pages that contain data will be commited.
Improving
You might have wondered what the scoped_allocator_adaptor was for. It doesn't seem we use it?
Well, the idea was to not use find_or_construct directly per file, but to
store a Vector<MyFile so you can harness the full power of BIP allocators.
The following full demo can be invoked
with filename arguments, which will all be loaded (if they exist as
regular files)
without arguments, which will list previously loaded files
Live On Coliru
#include <boost/interprocess/managed_shared_memory.hpp>
#include <boost/interprocess/managed_mapped_file.hpp> // for COLIRU
#include <boost/interprocess/containers/vector.hpp>
#include <boost/interprocess/containers/string.hpp>
#include <boost/container/scoped_allocator.hpp>
#include <fstream>
#include <filesystem>
#include <iostream>
#include <iomanip>
namespace bip = boost::interprocess;
namespace bc = boost::container;
namespace fs = std::filesystem;
namespace Shared {
#ifdef COLIRU
using Mem = bip::managed_mapped_file; // managed_shared_memory not allows
#else
using Mem = bip::managed_shared_memory;
#endif
using Mgr = Mem::segment_manager;
template <typename T>
using Alloc = bc::scoped_allocator_adaptor<bip::allocator<T, Mgr>>;
template <typename T> using Vector = bc::vector<T, Alloc<T>>;
using String = bc::basic_string<char, std::char_traits<char>, Alloc<char>>;
struct MyFile {
using allocator_type = Alloc<char>;
MyFile(MyFile&&) = default;
MyFile(MyFile const& rhs, allocator_type alloc)
: _filename(rhs._filename.begin(), rhs._filename.end(), alloc),
_contents(rhs._contents.begin(), rhs._contents.end(), alloc) {}
MyFile& operator=(MyFile const& rhs) {
_filename.assign(rhs._filename.begin(), rhs._filename.end());
_contents.assign(rhs._contents.begin(), rhs._contents.end());
return *this;
}
template <typename It>
explicit MyFile(std::string_view name, It b, It e, allocator_type alloc)
: _filename(name.data(), name.size(), alloc),
_contents(b, e, alloc) {}
String _filename;
Vector<char> _contents;
friend std::ostream& operator<<(std::ostream& os, MyFile const& mf) {
return os << "Name: " << std::quoted(mf._filename.c_str())
<< " content size: " << mf._contents.size();
}
};
} // namespace Shared
int main(int argc, char** argv) {
Shared::Mem shm(bip::open_or_create, "shared_mem", 512ull << 10);
using FileList = Shared::Vector<Shared::MyFile>;
auto& shared_files =
*shm.find_or_construct<FileList>("FileList")(shm.get_segment_manager());
if (1==argc) {
std::cout << "Displaying previously loaded files: \n";
for (auto& entry : shared_files)
std::cout << entry << std::endl;
} else {
std::cout << "Loading files: \n";
for (auto file : std::vector<fs::path>{argv + 1, argv + argc}) {
if (is_regular_file(file)) {
try {
std::ifstream ifs(file, std::ios::binary);
std::istreambuf_iterator<char> data_begin{ifs}, data_end{};
auto& loaded = shared_files.emplace_back(
file.native(), data_begin, data_end);
std::cout << loaded << std::endl;
} catch (std::system_error const& se) {
std::cerr << "Error: " << se.code().message() << std::endl;
} catch (std::exception const& se) {
std::cerr << "Other: " << se.what() << std::endl;
}
}
}
}
}
When run with
g++ -std=c++17 -O2 -Wall -pedantic -pthread main.cpp -lrt -DCOLIRU
./a.out main.cpp a.out
./a.out
Prints
Loading files:
Name: "main.cpp" content size: 3239
Name: "a.out" content size: 175176
Displaying previously loaded files:
Name: "main.cpp" content size: 3239
Name: "a.out" content size: 175176
BONUS
In response to the comments, I think it's worth actually comparing
Message Queue version
For comparison, here's a message queue implementation
Live On Coliru
#include <boost/interprocess/ipc/message_queue.hpp>
#include <boost/endian/arithmetic.hpp>
#include <fstream>
#include <filesystem>
#include <iostream>
#include <iomanip>
namespace bip = boost::interprocess;
namespace fs = std::filesystem;
using bip::message_queue;
static constexpr auto MAX_FILENAME_LENGH = 512; // 512 bytes max filename length
static constexpr auto MAX_CONTENT_SIZE = 512ull << 10; // 512 KiB max payload size
struct Message {
std::vector<char> _buffer;
using Uint32 = boost::endian::big_uint32_t;
struct header_t {
Uint32 filename_length;
Uint32 content_size;
};
static_assert(std::is_standard_layout_v<header_t> and
std::is_trivial_v<header_t>);
Message() = default;
Message(fs::path file) {
std::string const name = file.native();
std::ifstream ifs(file, std::ios::binary);
std::istreambuf_iterator<char> data_begin{ifs}, data_end{};
_buffer.resize(header_len + name.length());
std::copy(begin(name), end(name), _buffer.data() + header_len);
_buffer.insert(_buffer.end(), data_begin, data_end);
header().filename_length = name.length();
header().content_size = size() - header_len - name.length();
}
Message(char const* buf, size_t size)
: _buffer(buf, buf+size) {}
static constexpr auto header_len = sizeof(header_t);
static constexpr auto max_size =
header_len + MAX_FILENAME_LENGH + MAX_CONTENT_SIZE;
char const* data() const { return _buffer.data(); }
size_t size() const { return _buffer.size(); }
header_t& header() {
assert(_buffer.size() >= header_len);
return *reinterpret_cast<header_t*>(_buffer.data());
}
header_t const& header() const {
assert(_buffer.size() >= header_len);
return *reinterpret_cast<header_t const*>(_buffer.data());
}
std::string_view filename() const {
assert(_buffer.size() >= header_len + header().filename_length);
return { _buffer.data() + header_len, header().filename_length };
}
std::string_view contents() const {
assert(_buffer.size() >=
header_len + header().filename_length + header().content_size);
return {_buffer.data() + header_len + header().filename_length,
header().content_size};
}
friend std::ostream& operator<<(std::ostream& os, Message const& mf) {
return os << "Name: " << std::quoted(mf.filename())
<< " content size: " << mf.contents().size();
}
};
int main(int argc, char** argv) {
message_queue mq(bip::open_or_create, "file_transport", 10, Message::max_size);
if (1==argc) {
std::cout << "Receiving uploaded files: \n";
char rawbuf [Message::max_size];
while (true) {
size_t n;
unsigned prio;
mq.receive(rawbuf, sizeof(rawbuf), n, prio);
Message decoded(rawbuf, n);
std::cout << "Received: " << decoded << std::endl;
}
} else {
std::cout << "Loading files: \n";
for (auto file : std::vector<fs::path>{argv + 1, argv + argc}) {
if (is_regular_file(file)) {
try {
Message encoded(file);
std::cout << "Sending: " << encoded << std::endl;
mq.send(encoded.data(), encoded.size(), 0);
} catch (std::system_error const& se) {
std::cerr << "Error: " << se.code().message() << std::endl;
} catch (std::exception const& se) {
std::cerr << "Other: " << se.what() << std::endl;
}
}
}
}
}
A demo:
Note that there is a filesize limit in this approach because messages have a maximum length
TCP Socket Version
Here's a TCP socket implementation.
Live On Coliru
#include <boost/asio.hpp>
#include <boost/endian/arithmetic.hpp>
#include <vector>
#include <fstream>
#include <filesystem>
#include <iostream>
#include <iomanip>
namespace fs = std::filesystem;
using boost::asio::ip::tcp;
using boost::system::error_code;
static constexpr auto MAX_FILENAME_LENGH = 512; // 512 bytes max filename length
static constexpr auto MAX_CONTENT_SIZE = 512ull << 10; // 512 KiB max payload size
struct Message {
std::vector<char> _buffer;
using Uint32 = boost::endian::big_uint32_t;
struct header_t {
Uint32 filename_length;
Uint32 content_size;
};
static_assert(std::is_standard_layout_v<header_t> and
std::is_trivial_v<header_t>);
Message() = default;
Message(fs::path file) {
std::string const name = file.native();
std::ifstream ifs(file, std::ios::binary);
std::istreambuf_iterator<char> data_begin{ifs}, data_end{};
_buffer.resize(header_len + name.length());
std::copy(begin(name), end(name), _buffer.data() + header_len);
_buffer.insert(_buffer.end(), data_begin, data_end);
header().filename_length = name.length();
header().content_size = actual_size() - header_len - name.length();
}
Message(char const* buf, size_t size)
: _buffer(buf, buf+size) {}
static constexpr auto header_len = sizeof(header_t);
static constexpr auto max_size =
header_len + MAX_FILENAME_LENGH + MAX_CONTENT_SIZE;
char const* data() const { return _buffer.data(); }
size_t actual_size() const { return _buffer.size(); }
size_t decoded_size() const {
return header().filename_length + header().content_size;
}
bool is_complete() const {
return actual_size() >= header_len && actual_size() >= decoded_size();
}
header_t& header() {
assert(actual_size() >= header_len);
return *reinterpret_cast<header_t*>(_buffer.data());
}
header_t const& header() const {
assert(actual_size() >= header_len);
return *reinterpret_cast<header_t const*>(_buffer.data());
}
std::string_view filename() const {
assert(actual_size() >= header_len + header().filename_length);
return std::string_view(_buffer.data() + header_len,
header().filename_length);
}
std::string_view contents() const {
assert(actual_size() >= decoded_size());
return std::string_view(_buffer.data() + header_len +
header().filename_length,
header().content_size);
}
friend std::ostream& operator<<(std::ostream& os, Message const& mf) {
return os << "Name: " << std::quoted(mf.filename())
<< " content size: " << mf.contents().size();
}
};
int main(int argc, char** argv) {
boost::asio::io_context ctx;
u_int16_t port = 8989;
if (1==argc) {
std::cout << "Receiving uploaded files: " << std::endl;
tcp::acceptor acc(ctx, tcp::endpoint{{}, port});
while (true) {
auto s = acc.accept();
std::cout << "Connection accepted from " << s.remote_endpoint() << std::endl;
Message msg;
auto buf = boost::asio::dynamic_buffer(msg._buffer);
error_code ec;
while (auto n = read(s, buf, ec)) {
std::cout << "(read " << n << " bytes, " << ec.message() << ")" << std::endl;
while (msg.is_complete()) {
std::cout << "Received: " << msg << std::endl;
buf.consume(msg.decoded_size() + Message::header_len);
}
}
std::cout << "Connection closed" << std::endl;
}
} else {
std::cout << "Loading files: " << std::endl;
tcp::socket s(ctx);
s.connect(tcp::endpoint{{}, port});
for (auto file : std::vector<fs::path>{argv + 1, argv + argc}) {
if (is_regular_file(file)) {
try {
Message encoded(file);
std::cout << "Sending: " << encoded << std::endl;
write(s, boost::asio::buffer(encoded._buffer));
} catch (std::system_error const& se) {
std::cerr << "Error: " << se.code().message() << std::endl;
} catch (std::exception const& se) {
std::cerr << "Other: " << se.what() << std::endl;
}
}
}
}
}
Demo:
Note how this easily scales to larger files, multiple files in a single connection and even multiple connections simultaneously if you need. It also doesn't do double buffering, which improves performance.
This is why this kind of approach is much more usual than any of your other approaches.
I'm trying to parse this type of string
1.2e3ex
1.2e3 ex
And have set up
x3::float_ >> "ex"
Unfortunately, this fails to parse
1ex
Full example code:
#include <iostream>
#include <boost/spirit/home/x3.hpp>
namespace x3 = boost::spirit::x3;
const auto parser = x3::float_ >> "em";
int main()
{
std::string input = "1em";
auto first = input.begin();
auto last = input.end();
float value{};
bool result = x3::phrase_parse(first, last, parser, x3::blank, value);
if(result)
{
if(first == last)
std::cout << "parse succesful: " << value << '\n';
else
std::cout << "incomplete parse: " << value << '\n';
}
else
std::cout << "parse unsuccesful\n";
}
Available live on Coliru as well.
It seems I'd need to jump through some hoops,
struct non_scientific_float_policy : x3::real_policies<float>
{
template <typename Iterator>
static bool parse_exp(Iterator& first, Iterator const& last)
{
return false;
}
};
const auto non_scientific_float = x3::real_parser<float, non_scientific_float_policy>{};
and provide an alternative:
const auto parser = non_scientific_float >> "em" | x3::float_ >> "em";
Is there no other way?
You can solve the issue by tuning the real policy parse_exp in such way that exponent detection must expect not only [eE] character but [eE][-+]?[0-9].
#include <iostream>
#include <boost/spirit/home/x3.hpp>
namespace x3 = boost::spirit::x3;
template <typename T>
struct alt_real_policies : x3::real_policies<T>
{
template <typename Iterator>
static bool parse_exp(Iterator& first, Iterator const& last)
{
Iterator save = first;
if (x3::real_policies<T>::parse_exp(first, last)) {
Iterator iter = first;
if (x3::extract_int<x3::unused_type, 10, 1, 1>::call(iter, last, x3::unused))
return true;
}
first = save;
return false;
}
};
const x3::real_parser<float, alt_real_policies<float>> altfloat;
const auto parser = altfloat >> "em";
int main()
{
std::string input = "1em";
auto first = input.begin();
auto last = input.end();
float value{};
bool result = x3::phrase_parse(first, last, parser, x3::blank, value);
if (result)
{
if (first == last)
std::cout << "parse succesful: " << value << '\n';
else
std::cout << "incomplete parse: " << value << '\n';
}
else
std::cout << "parse unsuccesful\n";
}
http://coliru.stacked-crooked.com/a/f60f334c960cb602
You can use an alternative policy for non-greedy parsing of the exponent. The very simplest I can think of is:
Live On Coliru
#include <boost/spirit/home/x3.hpp>
#include <iostream>
namespace x3 = boost::spirit::x3;
template <typename T>
struct no_exponent : x3::real_policies<T> {
template <typename It>
static bool parse_exp(It, It) { return false; }
};
x3::real_parser<double, no_exponent<double> > noexp_;
const auto parser = (x3::float_ | noexp_) >> "em";
int main() {
std::string input = "-1.67em";
auto first = input.begin();
auto last = input.end();
float value{};
bool result = x3::phrase_parse(first, last, parser, x3::blank, value);
if (result) {
if (first == last)
std::cout << "parse succesful: " << value << '\n';
else
std::cout << "incomplete parse: " << value << '\n';
} else
{
std::cout << "parse unsuccesful\n";
}
}
Printing:
parse succesful: -1.67
I am trying to compose spirit rules but I cannot figure out what the attribute of this new rule would be.
The following code is working as I would expect it.
#include <iostream>
#include <boost/spirit/home/x3.hpp>
#include <boost/fusion/include/adapt_struct.hpp>
#include <boost/fusion/include/io.hpp>
#include <boost/fusion/tuple.hpp>
namespace ast{
struct Record{
int id;
std::string name;
};
struct Document{
Record rec;
Record rec2;
//std::vector<Record> rec;
std::string name;
};
using boost::fusion::operator<<;
}
BOOST_FUSION_ADAPT_STRUCT(ast::Record,
name, id
)
BOOST_FUSION_ADAPT_STRUCT(ast::Document,
rec, rec2,
//rec,
name
)
namespace parser{
namespace x3 = boost::spirit::x3;
namespace ascii = boost::spirit::x3::ascii;
using x3::lit;
using x3::int_;
using ascii::char_;
const auto identifier = +char_("a-z");
const x3::rule<class record, ast::Record> record = "record";
const auto record_def = lit("record") >> identifier >> lit("{") >> int_ >> lit("}");
const x3::rule<class document, ast::Document> document = "document";
const auto document_def =
record >> record
//+record // This should generate a sequence
>> identifier
;
BOOST_SPIRIT_DEFINE(document, record);
}
namespace{
constexpr char g_input[] = R"input(
record foo{42}
record bar{73}
foobar
)input";
}
int main(){
using boost::spirit::x3::ascii::space;
std::string str = g_input;
ast::Document unit;
bool r = phrase_parse(str.begin(), str.end(), parser::document, space, unit);
std::cout << "Got: " << unit << "\n";
return 0;
}
But when I change the rule to parse multiple records(instead of exactly 2) I would expect it to have a std::vector<Record> as an attribute. But all I get is a long compiler error that does not help me very much.
Can someone point me to what I am doing wrong in order to compose the attributes correctly?
I think the whole reason it didn't compile is because you tried to print the result... and std::vector<Record> doesn't know how to be streamed:
namespace ast {
using boost::fusion::operator<<;
static inline std::ostream& operator<<(std::ostream& os, std::vector<Record> const& rs) {
os << "{ ";
for (auto& r : rs) os << r << " ";
return os << "}";
}
}
Some more notes:
adding lexemes where absolutely required (!)
simplifying (no need to BOOST_SPIRIT_DEFINE unless recursive rules/separate TUs)
dropping redundant lit
I arrived at
Live On Coliru
#include <iostream>
#include <boost/spirit/home/x3.hpp>
#include <boost/fusion/include/adapt_struct.hpp>
#include <boost/fusion/include/io.hpp>
namespace ast {
struct Record{
int id;
std::string name;
};
struct Document{
std::vector<Record> rec;
std::string name;
};
}
BOOST_FUSION_ADAPT_STRUCT(ast::Record, name, id)
BOOST_FUSION_ADAPT_STRUCT(ast::Document, rec, name)
namespace ast {
using boost::fusion::operator<<;
static inline std::ostream& operator<<(std::ostream& os, std::vector<Record> const& rs) {
os << "{ ";
for (auto& r : rs) os << r << " ";
return os << "}";
}
}
namespace parser {
namespace x3 = boost::spirit::x3;
namespace ascii = x3::ascii;
const auto identifier = x3::lexeme[+x3::char_("a-z")];
const auto record = x3::rule<class record, ast::Record> {"record"}
= x3::lexeme["record"] >> identifier >> "{" >> x3::int_ >> "}";
const auto document = x3::rule<class document, ast::Document> {"document"}
= +record
>> identifier
;
}
int main(){
std::string const str = "record foo{42} record bar{73} foobar";
auto f = str.begin(), l = str.end();
ast::Document unit;
if (phrase_parse(f, l, parser::document, parser::ascii::space, unit)) {
std::cout << "Got: " << unit << "\n";
} else {
std::cout << "Parse failed\n";
}
if (f != l) {
std::cout << "Remaining unparsed input: '" << std::string(f,l) << "'\n";
}
}
Prints
Got: ({ (foo 42) (bar 73) } foobar)
I'm facing with an issue in defining a grammar with variadic templates.
I started by defining some simple grammars contained into some struct (e.g. Latitude, Longitude) as follows:
#include <boost/fusion/include/adapt_struct.hpp>
#include <boost/spirit/include/qi.hpp>
#include <boost/variant.hpp>
#include <iostream>
#include <string>
using namespace boost::spirit;
template <class Attribute>
using command_rule =
qi::rule<std::string::iterator, Attribute, ascii::space_type>;
template <class Attribute>
using command_grammar =
qi::grammar<std::string::iterator, Attribute, ascii::space_type>;
struct Latitude {
struct return_type {
double lat_;
};
struct grammar : command_grammar<return_type()> {
grammar() : grammar::base_type{latitude_} {
latitude_ = "LAT=" >> qi::double_;
}
private:
command_rule<return_type()> latitude_;
};
};
BOOST_FUSION_ADAPT_STRUCT(Latitude::return_type, (double, lat_))
struct Longitude {
struct return_type {
double lon_;
};
struct grammar : command_grammar<return_type()> {
grammar() : grammar::base_type{longitude_} {
longitude_ = "LON=" >> qi::double_;
}
private:
command_rule<return_type()> longitude_;
};
};
BOOST_FUSION_ADAPT_STRUCT(Longitude::return_type, (double, lon_))
Then, I would like to combine them in a complete grammar that is able to parse strings belonging to any of these simple grammars. To to this, I have defined a variadic template struct that tries to expand a list of sub-grammars into an expression like "grammar1 | grammar2 | ..."
template <class... Commands>
struct device_grammar : boost::spirit::qi::grammar<
std::string::iterator,
boost::variant<typename Commands::return_type...>(),
boost::spirit::ascii::space_type> {
typedef boost::variant<typename Commands::return_type...> return_type;
device_grammar() : device_grammar::base_type{rule_}{
build_rule<typename Commands::grammar...>();
}
private:
template <class CommandGrammar> void build_rule() {
rule_ = CommandGrammar();
}
template <class FirstGrammar, class SecondGrammar, class... Others>
void build_rule() {
build_rule<SecondGrammar, Others...>();
rule_ = rule_ | FirstGrammar();
}
boost::spirit::qi::rule<std::string::iterator, return_type(),
boost::spirit::ascii::space_type>
rule_;
};
typedef device_grammar<Latitude, Longitude> CoordinatesGrammar;
The code compiles (see the complete example below); the problem is that when it try to parse the input string, a segmentation fault is generated.
Can someone please help me to fix this issue?
Thanks a lot in advance.
EXAMPLE CODE (g++-4.9 or clang++-3.9):
#include <boost/fusion/include/adapt_struct.hpp>
#include <boost/spirit/include/qi.hpp>
#include <boost/variant.hpp>
#include <iostream>
#include <string>
template <class... Commands>
struct device_grammar : boost::spirit::qi::grammar<
std::string::iterator,
boost::variant<typename Commands::return_type...>(),
boost::spirit::ascii::space_type> {
typedef boost::variant<typename Commands::return_type...> return_type;
device_grammar() : device_grammar::base_type{rule_}{
build_rule<typename Commands::grammar...>();
}
private:
template <class CommandGrammar> void build_rule() {
rule_ = CommandGrammar();
}
template <class FirstGrammar, class SecondGrammar, class... Others>
void build_rule() {
build_rule<SecondGrammar, Others...>();
rule_ = rule_ | FirstGrammar();
}
boost::spirit::qi::rule<std::string::iterator, return_type(),
boost::spirit::ascii::space_type>
rule_;
};
using namespace boost::spirit;
template <class Attribute>
using command_rule =
qi::rule<std::string::iterator, Attribute, ascii::space_type>;
template <class Attribute>
using command_grammar =
qi::grammar<std::string::iterator, Attribute, ascii::space_type>;
struct Latitude {
struct return_type {
double lat_;
};
struct grammar : command_grammar<return_type()> {
grammar() : grammar::base_type{latitude_} {
latitude_ = "LAT=" >> qi::double_;
}
private:
command_rule<return_type()> latitude_;
};
};
BOOST_FUSION_ADAPT_STRUCT(Latitude::return_type, (double, lat_))
struct Longitude {
struct return_type {
double lon_;
};
struct grammar : command_grammar<return_type()> {
grammar() : grammar::base_type{longitude_} {
longitude_ = "LON=" >> qi::double_;
}
private:
command_rule<return_type()> longitude_;
};
};
BOOST_FUSION_ADAPT_STRUCT(Longitude::return_type, (double, lon_))
typedef device_grammar<Latitude, Longitude> CoordinatesGrammar;
struct print : public boost::static_visitor<> {
void operator()(Latitude::return_type &t) const {
std::cout << "Latitude = " << t.lat_ << " deg" << std::endl;
;
}
void operator()(Longitude::return_type &t) const {
std::cout << "Longitude = " << t.lon_ << " deg" << std::endl;
;
}
};
int main() {
std::string s;
CoordinatesGrammar g;
CoordinatesGrammar::return_type v;
while (1) {
std::getline(std::cin, s);
auto it = s.begin();
if (qi::phrase_parse(it, s.end(), g, ascii::space, v)) {
print p;
boost::apply_visitor(p, v);
}
}
return 0;
}
EDIT:
As far as I understand, the problem is in the lines
rule_ = CommandGrammar();
...
rule_ = rule_ | FirstGrammar();
It seems that the grammar objects can not be temporary and have to be stored as members of the class. How can I do that?
EDIT:
I have also tried to store such objects in a std::tuple, but it seems still not working.
What you are creating closely resembles what qi's auto parser already does: http://www.boost.org/doc/libs/1_64_0/libs/spirit/doc/html/spirit/qi/reference/auto.html
If you specialize create_parser<> for your datatypes you can simply use qi::auto_ straight-away:
Live On Coliru
#include <boost/fusion/include/adapt_struct.hpp>
#include <boost/spirit/include/qi.hpp>
namespace Commands {
namespace qi = boost::spirit::qi;
template <class T> using Rule = qi::rule<std::string::const_iterator, T()>;
template <typename... T>
auto parse(std::string const& s) {
boost::variant<T...> v;
auto it = s.begin();
if (qi::parse(it, s.end(), qi::auto_, v))
return v;
throw std::runtime_error(std::string(__FUNCTION__) + " failed");
}
}
struct Latitude { double lat_; };
BOOST_FUSION_ADAPT_STRUCT(Latitude, lat_)
struct Longitude { double lon_; };
BOOST_FUSION_ADAPT_STRUCT(Longitude, lon_)
namespace boost { namespace spirit { namespace traits {
template <> struct create_parser<Latitude> {
using type = Commands::Rule<Latitude>;
static type const& call() {
static type const s_rule = qi::skip(qi::space)["LAT=" >> qi::auto_];
return s_rule;
};
};
template <> struct create_parser<Longitude> {
using type = Commands::Rule<Longitude>;
static type const& call() {
static type const s_rule = qi::skip(qi::space)["LON=" >> qi::auto_];
return s_rule;
};
};
} } }
struct print {
using result_type = void;
void operator()(Latitude const &t) const { std::cout << "Latitude = " << t.lat_ << " deg" << std::endl; }
void operator()(Longitude const &t) const { std::cout << "Longitude = " << t.lon_ << " deg" << std::endl; }
};
#include <sstream>
int main() {
std::istringstream iss("LAT=4.3\n LON=5.0");
std::string s;
print printer;
while (std::getline(iss, s)) try {
auto v = Commands::parse<Latitude, Longitude>(s);
boost::apply_visitor(printer, v);
}
catch (std::exception const& e) {
std::cout << "'" << s << "': " << e.what() << "\n";
}
}
Prints
Latitude = 4.3 deg
Longitude = 5 deg
Nicer things
If you don't use qi::rule<> you don't need to hard-code the iterator either. Let's go full fun mode and get rid of the visitor too:
[Live On Coliru](http://coliru.stacked-crooked.com/a/84f7a8c9a453fc1b
#include <boost/fusion/include/adapt_struct.hpp>
#include <boost/spirit/include/qi.hpp>
namespace Commands {
template <typename... T>
auto parse(std::string const& s) {
boost::variant<T...> v;
auto it = s.begin();
if (boost::spirit::qi::parse(it, s.end(), boost::spirit::qi::auto_, v))
return v;
throw std::runtime_error(std::string(__FUNCTION__) + " failed");
}
struct Latitude { double lat_; };
struct Longitude { double lon_; };
static inline std::ostream& operator<<(std::ostream& os, Latitude const &t) { return os << "Latitude = " << t.lat_ << " deg"; }
static inline std::ostream& operator<<(std::ostream& os, Longitude const &t) { return os << "Longitude = " << t.lon_ << " deg"; }
}
BOOST_FUSION_ADAPT_STRUCT(Commands::Latitude, lat_)
BOOST_FUSION_ADAPT_STRUCT(Commands::Longitude, lon_)
namespace boost { namespace spirit { namespace traits {
#define MAP_PARSER(T, expr) \
template <> struct create_parser<T> { \
using type = decltype(qi::attr_cast<T, T>(qi::copy(expr))); \
static type const& call() { static type const s_rule = qi::attr_cast<T, T>(qi::copy(expr)); return s_rule; }; \
};
#define AUTO_MAP_PARSER(T, caption) MAP_PARSER(T, qi::skip(qi::space)[qi::lit(caption) >> '=' >> qi::auto_])
AUTO_MAP_PARSER(::Commands::Longitude, "LON")
AUTO_MAP_PARSER(::Commands::Latitude, "LAT")
} } }
#include <sstream>
int main() {
std::istringstream iss("LAT=4.3\n LON=5.0");
std::string s;
while (std::getline(iss, s)) try {
using namespace Commands;
std::cout << "Parsed '" << s << "' into " << parse<Latitude, Longitude>(s) << "\n";
} catch (std::exception const& e) {
std::cout << "'" << s << "': " << e.what() << "\n";
}
}
Prints
Parsed 'LAT=4.3' into Latitude = 4.3 deg
Parsed ' LON=5.0' into Longitude = 5 deg
In addition to the Spirit Qi answer I gave:
If you can afford to enable c++1z, you can use Spirit X3 with fold-expressions:
Live On Coliru
#include <boost/fusion/include/adapt_struct.hpp>
#include <boost/spirit/home/x3.hpp>
namespace Commands {
namespace x3 = boost::spirit::x3;
template <typename... T>
auto parse(std::string const& s) {
using V = boost::variant<T...>;
V v;
auto it = s.begin();
if (x3::parse(it, s.end(), parser_for(v), v))
return v;
throw std::runtime_error(std::string(__FUNCTION__) + " failed");
}
struct Latitude { double lat_; };
struct Longitude { double lon_; };
auto label_for(Latitude) { return "LAT"; }
auto label_for(Longitude) { return "LON"; }
template <typename T, typename P>
auto as_cmd(P p) { return x3::rule<struct _, T>{}
= x3::skip(x3::space)[x3::lit(label_for(T{})) >> '=' >> p]; }
template <typename T> auto parser_for(T) { return as_cmd<T>(x3::double_); }
template <typename... T> auto parser_for(boost::variant<T...> _) { return (parser_for(T{}) | ...); }
static inline std::ostream& operator<<(std::ostream& os, Latitude const &t) { return os << "Latitude = " << t.lat_ << " deg"; }
static inline std::ostream& operator<<(std::ostream& os, Longitude const &t) { return os << "Longitude = " << t.lon_ << " deg"; }
}
BOOST_FUSION_ADAPT_STRUCT(Commands::Latitude, lat_)
BOOST_FUSION_ADAPT_STRUCT(Commands::Longitude, lon_)
#include <iostream>
#include <sstream>
int main() {
std::istringstream iss("LAT=4.3\n LON=5.0");
std::string s;
while (std::getline(iss, s)) try {
using namespace Commands;
std::cout << "Parsed '" << s << "' into " << parse<Latitude, Longitude>(s) << "\n";
} catch (std::exception const& e) {
std::cout << "'" << s << "': " << e.what() << "\n";
}
}
Prints
Parsed 'LAT=4.3' into Latitude = 4.3 deg
Parsed ' LON=5.0' into Longitude = 5 deg
I would like to use a parsed value as the input to a loop parser.
The grammar defines a header that specifies the (variable) size of the following string. For example, say the following string is the input to some parser.
12\r\nTest Payload
The parser should extract the 12, convert it to an unsigned int and then read twelve characters. I can define a boost spirit grammar that compiles, but an assertion in the boost spirit code fails at runtime.
#include <iostream>
#include <boost/spirit.hpp>
using namespace boost::spirit;
struct my_closure : public closure<my_closure, std::size_t> {
member1 size;
};
struct my_grammar : public grammar<my_grammar> {
template <typename ScannerT>
struct definition {
typedef rule<ScannerT> rule_type;
typedef rule<ScannerT, my_closure::context_t> closure_rule_type;
closure_rule_type header;
rule_type payload;
rule_type top;
definition(const my_grammar &self)
{
using namespace phoenix;
header = uint_p[header.size = arg1];
payload = repeat_p(header.size())[anychar_p][assign_a(self.result)];
top = header >> str_p("\r\n") >> payload;
}
const rule_type &start() const { return top; }
};
my_grammar(std::string &p_) : result(p_) {}
std::string &result;
};
int
main(int argc, char **argv)
{
const std::string content = "12\r\nTest Payload";
std::string payload;
my_grammar g(payload);
if (!parse(content.begin(), content.end(), g).full) {
std::cerr << "there was a parsing error!\n";
return -1;
}
std::cout << "Payload: " << payload << std::endl;
return 0;
}
Is it possible to tell spirit that the closure variable should be evaluated lazily? Is this behaviour supported by boost spirit?
This is much easier with the new qi parser available in Spirit 2. The following code snippet provides a full example that mostly works. An unexpected character is being inserted into the final result.
#include <iostream>
#include <string>
#include <boost/version.hpp>
#include <boost/spirit/include/qi.hpp>
#include <boost/spirit/include/qi_repeat.hpp>
#include <boost/spirit/include/qi_grammar.hpp>
#include <boost/spirit/include/phoenix_core.hpp>
#include <boost/spirit/include/phoenix_operator.hpp>
using boost::spirit::qi::repeat;
using boost::spirit::qi::uint_;
using boost::spirit::ascii::char_;
using boost::spirit::ascii::alpha;
using boost::spirit::qi::_1;
namespace phx = boost::phoenix;
namespace qi = boost::spirit::qi;
template <typename P, typename T>
void test_parser_attr(
char const* input, P const& p, T& attr, bool full_match = true)
{
using boost::spirit::qi::parse;
char const* f(input);
char const* l(f + strlen(f));
if (parse(f, l, p, attr) && (!full_match || (f == l)))
std::cout << "ok" << std::endl;
else
std::cout << "fail" << std::endl;
}
static void
straight_forward()
{
std::string str;
int n;
test_parser_attr("12\r\nTest Payload",
uint_[phx::ref(n) = _1] >> "\r\n" >> repeat(phx::ref(n))[char_],
str);
std::cout << "str.length() == " << str.length() << std::endl;
std::cout << n << "," << str << std::endl; // will print "12,Test Payload"
}
template <typename P, typename T>
void
test_phrase_parser(char const* input, P const& p,
T& attr, bool full_match = true)
{
using boost::spirit::qi::phrase_parse;
using boost::spirit::qi::ascii::space;
char const* f(input);
char const* l(f + strlen(f));
if (phrase_parse(f, l, p, space, attr) && (!full_match || (f == l)))
std::cout << "ok" << std::endl;
else
std::cout << "fail" << std::endl;
}
template <typename Iterator>
struct test_grammar
: qi::grammar<Iterator, std::string(), qi::locals<unsigned> > {
test_grammar()
: test_grammar::base_type(my_rule)
{
using boost::spirit::qi::_a;
my_rule %= uint_[_a = _1] >> "\r\n" >> repeat(_a)[char_];
}
qi::rule<Iterator, std::string(), qi::locals<unsigned> > my_rule;
};
static void
with_grammar_local_variable()
{
std::string str;
test_phrase_parser("12\r\nTest Payload", test_grammar<const char*>(), str);
std::cout << str << std::endl; // will print "Test Payload"
}
int
main(int argc, char **argv)
{
std::cout << "boost version: " << BOOST_LIB_VERSION << std::endl;
straight_forward();
with_grammar_local_variable();
return 0;
}
What you are looking for is lazy_p, check the example here: http://www.boost.org/doc/libs/1_35_0/libs/spirit/doc/the_lazy_parser.html