Parsing a command language using Boost Spirit - c++

I am building a parser for a command language that I've pieced together from various samples. I've read the Boost Spirit Qi and Lex docs, and I think I understand the basics, but from what I've read, I should avoid attributes and use utree. What docs I've found on utree basically suck. Given the code below, I have the following questions:
How do I annotate the parser to create an AST using utree?
How do I walk the utree after it is built, to discover what was parsed? e.g. for token-only commands, such as SET DEBUG ON, as well as commands with values, such as LOAD "file.ext" or SET DRIVE C:
I want to add a comment character, "!". So, how can I ignore everything after that - except when it occurs in a quoted string?
Why doesn't my error handler get called when I give it invalid input?
How can I make the command tokens case insensitive, but not change the contents of a quoted string?
#include <Windows.h>
#include <conio.h>
#include <string>
#include <vector>
#include <iostream>
#define BOOST_SPIRIT_DEBUG
#include <boost\spirit\include\qi.hpp>
#include <boost\spirit\include\phoenix.hpp>
#include <boost\spirit\include\lex.hpp>
#include <boost\spirit\include\lex_lexertl.hpp>
using namespace std;
using namespace boost::spirit;
using boost::spirit::utree;
//
// Tokens used by the command grammar
//
template <typename Lexer>
struct command_tokens : lex::lexer <Lexer>
{
command_tokens () :
//
// Verbs, with abbreviation (just enough characters to make each unique)
//
boot ("B(O(O(T)?)?)?"),
exit ("E(X(I(T)?)?)?"),
help ("H(E(L(P)?)?)?"),
dash_help ("-H(E(L(P)?)?)?"),
slash_help ("\\/H(E(L(P)?)?)?"),
load ("L(O(A(D)?)?)?"),
quit ("Q(U(I(T)?)?)?"),
set ("SE(T)?"),
show ("SH(O(W)?)?"),
//
// Nouns, with abbreviation (the minimum number of characters is usually 3, but may be more to ensure uniqueness)
//
debug ("DEB(U(G)?)?"),
drive ("DRI(V(E)?)?"),
trace ("TRA(C(E)?)?"),
//
// Qualifiers
//
on ("ON"),
off ("OFF"),
//
// Tokens to pass back to the grammar
//
quoted_string ("...")
{
using namespace boost::spirit::lex;
//
// Associate the tokens with the lexer
//
this->self
= boot
| exit
| help
| dash_help
| slash_help
| load
| quit
| set
| show
| debug
| drive
| trace
| off
| on
| quoted_string
;
//
// Define whitespace to ignore: space, tab, newline
//
this->self ("WS")
= lex::token_def <> ("[ \\t\\n]+")
;
}
lex::token_def <> boot;
lex::token_def <> dash_help;
lex::token_def <> debug;
lex::token_def <string> drive;
lex::token_def <> exit;
lex::token_def <> help;
lex::token_def <> load;
lex::token_def <> off;
lex::token_def <> on;
lex::token_def <> quit;
lex::token_def <string> quoted_string;
lex::token_def <> set;
lex::token_def <> show;
lex::token_def <> slash_help;
lex::token_def <> trace;
};
//
// Display parse error
//
struct error_handler_
{
template <typename, typename, typename>
struct result
{
typedef void type;
};
template <typename Iterator>
void operator ()
(
qi::info const& What,
Iterator Err_pos,
Iterator Last
) const
{
cout << "Error! Expecting "
<< What
<< " here: \""
<< string (Err_pos, Last)
<< "\""
<< endl;
}
};
boost::phoenix::function <error_handler_> const error_handler = error_handler_ ();
//
// Grammar describing the valid commands
//
template <typename Iterator, typename Lexer>
struct command_grammar : qi::grammar <Iterator>
{
template <typename Lexer>
command_grammar (command_tokens <Lexer> const& Tok) :
command_grammar::base_type (start)
{
using qi::on_error;
using qi::fail;
using qi::char_;
start
= +commands;
commands
= (
boot_command
| exit_command
| help_command
| load_command
| set_command
| show_command
);
boot_command
= Tok.boot;
exit_command
= Tok.exit
| Tok.quit;
help_command
= Tok.help
| Tok.dash_help
| Tok.slash_help;
load_command
= Tok.load >> Tok.quoted_string;
set_command
= Tok.set;
show_command
= Tok.show;
set_property
= debug_property
| drive_property
| trace_property;
debug_property
= Tok.debug >> on_off;
drive_property
= Tok.drive >> char_ ("A-Z") >> char_ (":");
trace_property
= Tok.trace >> on_off;
on_off
= Tok.on
| Tok.off;
BOOST_SPIRIT_DEBUG_NODE (start);
BOOST_SPIRIT_DEBUG_NODE (commands);
BOOST_SPIRIT_DEBUG_NODE (boot_command);
BOOST_SPIRIT_DEBUG_NODE (exit_command);
BOOST_SPIRIT_DEBUG_NODE (help_command);
BOOST_SPIRIT_DEBUG_NODE (load_command);
BOOST_SPIRIT_DEBUG_NODE (quit_command);
BOOST_SPIRIT_DEBUG_NODE (set_command);
BOOST_SPIRIT_DEBUG_NODE (show_command);
BOOST_SPIRIT_DEBUG_NODE (set_property);
BOOST_SPIRIT_DEBUG_NODE (debug_property);
BOOST_SPIRIT_DEBUG_NODE (drive_property);
BOOST_SPIRIT_DEBUG_NODE (trace_property);
BOOST_SPIRIT_DEBUG_NODE (target_property);
on_error <fail> (start, error_handler (_4, _3, _2));
}
qi::rule <Iterator> start;
qi::rule <Iterator> commands;
qi::rule <Iterator> boot_command;
qi::rule <Iterator> exit_command;
qi::rule <Iterator> help_command;
qi::rule <Iterator> load_command;
qi::rule <Iterator> quit_command;
qi::rule <Iterator> set_command;
qi::rule <Iterator> show_command;
qi::rule <Iterator> set_property;
qi::rule <Iterator> debug_property;
qi::rule <Iterator, string ()> drive_property;
qi::rule <Iterator> target_property;
qi::rule <Iterator> trace_property;
qi::rule <Iterator> on_off;
};
int
main
(
int Argc,
PCHAR Argv
)
{
typedef std::string::iterator base_iterator_type;
typedef lex::lexertl::token <base_iterator_type> token_type;
typedef lex::lexertl::lexer <token_type> lexer_type;
typedef command_tokens <lexer_type> command_tokens;
typedef command_tokens::iterator_type iterator_type;
typedef command_grammar <iterator_type, command_tokens::lexer_def> command_grammar;
command_tokens tokens;
command_grammar commands (tokens);
string input = "SET DRIVE C:";
string::iterator it = input.begin ();
iterator_type iter = tokens.begin (it, input.end ());
iterator_type end = tokens.end ();
string ws ("WS");
bool result = lex::tokenize_and_phrase_parse (it, input.end (), tokens, commands, qi::in_state (ws) [tokens.self]);
if (result)
{
cout << "Parse succeeded" << endl;
}
else
{
string rest (it, input.end ());
cout << "Parse failed" << endl;
cout << "Stopped at " << rest << endl;
}
return 0;
} // End of main

I'm going to side-step the majority of your code, for the simple reasons that experience tells me that Lex and utree are generally not what you want to use.
What you do want is define an AST to represent your command language and then come up with a grammar to build it.
AST
namespace Ast {
struct NoValue {
bool operator==(NoValue const &) const { return true; }
};
template <typename Tag> struct GenericCommand {};
namespace tag {
struct boot;
struct help;
struct load;
struct exit;
struct set;
struct show;
};
template <> struct GenericCommand<tag::load> { std::string name; };
template <> struct GenericCommand<tag::set> {
std::string property;
boost::variant<NoValue, std::string, bool> value; // optional
};
using BootCmd = GenericCommand<tag::boot>;
using HelpCmd = GenericCommand<tag::help>;
using ExitCmd = GenericCommand<tag::exit>;
using ShowCmd = GenericCommand<tag::show>;
using LoadCmd = GenericCommand<tag::load>;
using SetCmd = GenericCommand<tag::set>;
using Command = boost::variant<BootCmd, HelpCmd, ExitCmd, ShowCmd, LoadCmd, SetCmd>;
using Commands = std::list<Command>;
}
The full code only adds debug output helpers. And here's the full Fusion Adaption:
BOOST_FUSION_ADAPT_TPL_STRUCT((Tag), (Ast::GenericCommand) (Tag), )
BOOST_FUSION_ADAPT_STRUCT(Ast::LoadCmd, name)
BOOST_FUSION_ADAPT_STRUCT(Ast::SetCmd, property, value)
Grammar
Here I make some choices:
let's make things white-space and case insensitive, allowing line-separated commands: (see also Boost spirit skipper issues)
start = skip(blank) [lazy_command % eol];
let's use Nabialek Trick to associate commands with prefixes. I used a very simple snippet of code to generate the unique prefixes:
std::set<std::string> const verbs { "boot", "exit", "help", "-help", "/help", "load", "quit", "set", "show", };
for (auto const full : verbs)
for (auto partial=full; partial.length(); partial.resize(partial.size()-1)) {
auto n = std::distance(verbs.lower_bound(partial), verbs.upper_bound(full));
if (n < 2) std::cout << "(\"" << partial << "\", &" << full << "_command)\n";
}
you could do the same for properties, but I thought the current setup is simpler:
template <typename Iterator>
struct command_grammar : qi::grammar<Iterator, Ast::Commands()> {
command_grammar() : command_grammar::base_type(start) {
using namespace qi;
start = skip(blank) [lazy_command % eol];
// nabialek trick
lazy_command = no_case [ commands [ _a = _1 ] > lazy(*_a) [ _val = _1 ] ];
on_off.add("on", true)("off", false);
commands.add
("-help", &help_command) ("-hel", &help_command) ("-he", &help_command) ("-h", &help_command)
("/help", &help_command) ("/hel", &help_command) ("/he", &help_command) ("/h", &help_command)
("help", &help_command) ("hel", &help_command) ("he", &help_command) ("h", &help_command)
("boot", &boot_command) ("boo", &boot_command) ("bo", &boot_command) ("b", &boot_command)
("exit", &exit_command) ("exi", &exit_command) ("ex", &exit_command) ("e", &exit_command)
("quit", &exit_command) ("qui", &exit_command) ("qu", &exit_command) ("q", &exit_command)
("load", &load_command) ("loa", &load_command) ("lo", &load_command) ("l", &load_command)
("set", &set_command) ("se", &set_command)
("show", &show_command) ("sho", &show_command) ("sh", &show_command);
quoted_string = '"' >> +~char_('"') >> '"';
// nullary commands
boot_command_ = eps;
exit_command_ = eps;
help_command_ = eps;
show_command_ = eps;
// non-nullary commands
load_command_ = quoted_string;
drive_ = char_("A-Z") >> ':';
set_command_ = no_case[lit("drive")|"driv"|"dri"|"dr"] >> attr("DRIVE") >> drive_
| no_case[ (lit("debug")|"debu"|"deb"|"de") >> attr("DEBUG") >> on_off ]
| no_case[ (lit("trace")|"trac"|"tra"|"tr"|"t") >> attr("TRACE") >> on_off ]
;
BOOST_SPIRIT_DEBUG_NODES(
(start)(lazy_command)
(boot_command) (exit_command) (help_command) (show_command) (set_command) (load_command)
(boot_command_)(exit_command_)(help_command_)(show_command_)(set_command_)(load_command_)
(quoted_string)(drive_)
)
on_error<fail>(start, error_handler_(_4, _3, _2));
on_error<fail>(lazy_command, error_handler_(_4, _3, _2));
boot_command = boot_command_;
exit_command = exit_command_;
help_command = help_command_;
load_command = load_command_;
exit_command = exit_command_;
set_command = set_command_;
show_command = show_command_;
}
private:
struct error_handler_t {
template <typename...> struct result { typedef void type; };
void operator()(qi::info const &What, Iterator Err_pos, Iterator Last) const {
std::cout << "Error! Expecting " << What << " here: \"" << std::string(Err_pos, Last) << "\"" << std::endl;
}
};
boost::phoenix::function<error_handler_t> const error_handler_ = error_handler_t {};
qi::rule<Iterator, Ast::Commands()> start;
using Skipper = qi::blank_type;
using CommandRule = qi::rule<Iterator, Ast::Command(), Skipper>;
qi::symbols<char, bool> on_off;
qi::symbols<char, CommandRule const*> commands;
qi::rule<Iterator, std::string()> drive_property, quoted_string, drive_;
qi::rule<Iterator, Ast::Command(), Skipper, qi::locals<CommandRule const*> > lazy_command;
CommandRule boot_command, exit_command, help_command, load_command, set_command, show_command;
qi::rule<Iterator, Ast::BootCmd(), Skipper> boot_command_;
qi::rule<Iterator, Ast::ExitCmd(), Skipper> exit_command_;
qi::rule<Iterator, Ast::HelpCmd(), Skipper> help_command_;
qi::rule<Iterator, Ast::LoadCmd(), Skipper> load_command_;
qi::rule<Iterator, Ast::SetCmd(), Skipper> set_command_;
qi::rule<Iterator, Ast::ShowCmd(), Skipper> show_command_;
};
Test Cases
Live On Coliru
int main() {
typedef std::string::const_iterator It;
command_grammar<It> const commands;
for (std::string const input : {
"help",
"set drive C:",
"SET DRIVE C:",
"loAD \"XYZ\"",
"load \"anything \nat all\"",
// multiline
"load \"ABC\"\nhelp\n-he\n/H\nsh\nse t off\nse debug ON\nb\nq"
})
{
std::cout << "----- '" << input << "' -----\n";
It f = input.begin(), l = input.end();
Ast::Commands parsed;
bool result = parse(f, l, commands, parsed);
if (result) {
for (auto& cmd : parsed) {
std::cout << "Parsed " << cmd << "\n";
}
} else {
std::cout << "Parse failed\n";
}
if (f != l) {
std::cout << "Remaining unparsed '" << std::string(f, l) << "'\n";
}
}
}
Prints:
----- 'help' -----
Parsed HELP ()
----- 'set drive C:' -----
Parsed SET (DRIVE C)
----- 'SET DRIVE C:' -----
Parsed SET (DRIVE C)
----- 'loAD "XYZ"' -----
Parsed LOAD (XYZ)
----- 'load "anything
at all"' -----
Parsed LOAD (anything
at all)
----- 'load "ABC"
help
-he
/H
sh
se t off
se debug ON
b
q' -----
Parsed LOAD (ABC)
Parsed HELP ()
Parsed HELP ()
Parsed HELP ()
Parsed SHOW ()
Parsed SET (TRACE 0)
Parsed SET (DEBUG 1)
Parsed BOOT ()
Parsed EXIT ()
Full Listing
Live On Coliru
//#define BOOST_SPIRIT_DEBUG
#include <boost/fusion/include/io.hpp>
#include <boost/fusion/adapted/struct.hpp>
#include <boost/spirit/include/phoenix.hpp>
#include <boost/spirit/include/qi.hpp>
namespace qi = boost::spirit::qi;
namespace Ast {
struct NoValue {
bool operator==(NoValue const &) const { return true; }
friend std::ostream& operator<<(std::ostream& os, NoValue) { return os; }
};
template <typename Tag> struct GenericCommand {};
namespace tag {
struct boot {};
struct help {};
struct load {};
struct exit {};
struct set {};
struct show {};
static std::ostream& operator<<(std::ostream& os, boot) { return os << "BOOT"; }
static std::ostream& operator<<(std::ostream& os, help) { return os << "HELP"; }
static std::ostream& operator<<(std::ostream& os, load) { return os << "LOAD"; }
static std::ostream& operator<<(std::ostream& os, exit) { return os << "EXIT"; }
static std::ostream& operator<<(std::ostream& os, set ) { return os << "SET"; }
static std::ostream& operator<<(std::ostream& os, show) { return os << "SHOW"; }
};
template <> struct GenericCommand<tag::load> { std::string name; };
template <> struct GenericCommand<tag::set> {
std::string property;
boost::variant<NoValue, std::string, bool> value; // optional
};
using BootCmd = GenericCommand<tag::boot>;
using HelpCmd = GenericCommand<tag::help>;
using ExitCmd = GenericCommand<tag::exit>;
using ShowCmd = GenericCommand<tag::show>;
using LoadCmd = GenericCommand<tag::load>;
using SetCmd = GenericCommand<tag::set>;
using Command = boost::variant<BootCmd, HelpCmd, ExitCmd, ShowCmd, LoadCmd, SetCmd>;
using Commands = std::list<Command>;
template <typename Tag>
static inline std::ostream& operator<<(std::ostream& os, Ast::GenericCommand<Tag> const& command) {
return os << Tag{} << " " << boost::fusion::as_vector(command);
}
}
BOOST_FUSION_ADAPT_TPL_STRUCT((Tag), (Ast::GenericCommand) (Tag), )
BOOST_FUSION_ADAPT_STRUCT(Ast::LoadCmd, name)
BOOST_FUSION_ADAPT_STRUCT(Ast::SetCmd, property, value)
template <typename Iterator>
struct command_grammar : qi::grammar<Iterator, Ast::Commands()> {
command_grammar() : command_grammar::base_type(start) {
using namespace qi;
start = skip(blank) [lazy_command % eol];
// nabialek trick
lazy_command = no_case [ commands [ _a = _1 ] > lazy(*_a) [ _val = _1 ] ];
on_off.add("on", true)("off", false);
commands.add
("-help", &help_command) ("-hel", &help_command) ("-he", &help_command) ("-h", &help_command)
("/help", &help_command) ("/hel", &help_command) ("/he", &help_command) ("/h", &help_command)
("help", &help_command) ("hel", &help_command) ("he", &help_command) ("h", &help_command)
("boot", &boot_command) ("boo", &boot_command) ("bo", &boot_command) ("b", &boot_command)
("exit", &exit_command) ("exi", &exit_command) ("ex", &exit_command) ("e", &exit_command)
("quit", &exit_command) ("qui", &exit_command) ("qu", &exit_command) ("q", &exit_command)
("load", &load_command) ("loa", &load_command) ("lo", &load_command) ("l", &load_command)
("set", &set_command) ("se", &set_command)
("show", &show_command) ("sho", &show_command) ("sh", &show_command);
quoted_string = '"' >> +~char_('"') >> '"';
// nullary commands
boot_command_ = eps;
exit_command_ = eps;
help_command_ = eps;
show_command_ = eps;
// non-nullary commands
load_command_ = quoted_string;
drive_ = char_("A-Z") >> ':';
set_command_ = no_case[lit("drive")|"driv"|"dri"|"dr"] >> attr("DRIVE") >> drive_
| no_case[ (lit("debug")|"debu"|"deb"|"de") >> attr("DEBUG") >> on_off ]
| no_case[ (lit("trace")|"trac"|"tra"|"tr"|"t") >> attr("TRACE") >> on_off ]
;
BOOST_SPIRIT_DEBUG_NODES(
(start)(lazy_command)
(boot_command) (exit_command) (help_command) (show_command) (set_command) (load_command)
(boot_command_)(exit_command_)(help_command_)(show_command_)(set_command_)(load_command_)
(quoted_string)(drive_)
)
on_error<fail>(start, error_handler_(_4, _3, _2));
on_error<fail>(lazy_command, error_handler_(_4, _3, _2));
boot_command = boot_command_;
exit_command = exit_command_;
help_command = help_command_;
load_command = load_command_;
exit_command = exit_command_;
set_command = set_command_;
show_command = show_command_;
}
private:
struct error_handler_t {
template <typename...> struct result { typedef void type; };
void operator()(qi::info const &What, Iterator Err_pos, Iterator Last) const {
std::cout << "Error! Expecting " << What << " here: \"" << std::string(Err_pos, Last) << "\"" << std::endl;
}
};
boost::phoenix::function<error_handler_t> const error_handler_ = error_handler_t {};
qi::rule<Iterator, Ast::Commands()> start;
using Skipper = qi::blank_type;
using CommandRule = qi::rule<Iterator, Ast::Command(), Skipper>;
qi::symbols<char, bool> on_off;
qi::symbols<char, CommandRule const*> commands;
qi::rule<Iterator, std::string()> drive_property, quoted_string, drive_;
qi::rule<Iterator, Ast::Command(), Skipper, qi::locals<CommandRule const*> > lazy_command;
CommandRule boot_command, exit_command, help_command, load_command, set_command, show_command;
qi::rule<Iterator, Ast::BootCmd(), Skipper> boot_command_;
qi::rule<Iterator, Ast::ExitCmd(), Skipper> exit_command_;
qi::rule<Iterator, Ast::HelpCmd(), Skipper> help_command_;
qi::rule<Iterator, Ast::LoadCmd(), Skipper> load_command_;
qi::rule<Iterator, Ast::SetCmd(), Skipper> set_command_;
qi::rule<Iterator, Ast::ShowCmd(), Skipper> show_command_;
};
int main() {
typedef std::string::const_iterator It;
command_grammar<It> const commands;
for (std::string const input : {
"help",
"set drive C:",
"SET DRIVE C:",
"loAD \"XYZ\"",
"load \"anything \nat all\"",
// multiline
"load \"ABC\"\nhelp\n-he\n/H\nsh\nse t off\nse debug ON\nb\nq"
})
{
std::cout << "----- '" << input << "' -----\n";
It f = input.begin(), l = input.end();
Ast::Commands parsed;
bool result = parse(f, l, commands, parsed);
if (result) {
for (auto& cmd : parsed) {
std::cout << "Parsed " << cmd << "\n";
}
} else {
std::cout << "Parse failed\n";
}
if (f != l) {
std::cout << "Remaining unparsed '" << std::string(f, l) << "'\n";
}
}
}
POST-SCRIPT
Q. How do I annotate the parser to create an AST using utree?
See above
Q. How do I walk the utree after it is built, to discover what was parsed?
See above, see also http://www.boost.org/doc/libs/1_64_0/doc/html/variant/tutorial.html
Q. I want to add a comment character, "!". So, how can I ignore everything after that - except when it occurs in a quoted string?
Simply make the Skipper type a rule that parses e.g.:
qi::rule<Iterator> my_skipper;
my_skipper = blank | '!' >> *(char_ - eol) >> (eol|eoi);
Then use it instead of skip(blank) like skip(my_skipper)
Q. Why doesn't my error handler get called when I give it invalid input?
Because you didn't mark expectation points (operator> instead of operator>>). If you don't, a failure to match a sub-expression simply backtracks.
Q. How can I make the command tokens case insensitive, but not change the contents of a quoted string?
See above

Related

cannot get boost::spirit parser&lexer working for token types other than std::string or int or double

This does not compile (code below).
There was another question here with the same error. But I don't understand the answer. I already tried inserting qi::eps in places -- but without success.
I also tried already adding meta functions (boost::spirit::raits::is_container) for the types used -- but this also does not help.
I also tried using the same variant containing all types I need to use everywhere. Same problem.
Has anybody gotten this working for a lexer returning something else than double or int or string? And for the parser also returning non-trivial objects?
I've tried implementing semantic functions everywhere returning default objects. But this also does not help.
Here comes the code:
// spirit_error.cpp : Defines the entry point for the console application.
//
#include <boost/config/warning_disable.hpp>
#include <boost/spirit/include/qi.hpp>
#include <boost/spirit/include/phoenix_core.hpp>
#include <boost/spirit/include/phoenix_operator.hpp>
#include <boost/phoenix/object.hpp>
#include <boost/spirit/include/qi_char_class.hpp>
#include <boost/spirit/include/phoenix_bind.hpp>
#include <boost/mpl/index_of.hpp>
#include <boost/spirit/include/lex_lexertl.hpp>
#include <boost/intrusive_ptr.hpp>
#include <boost/smart_ptr/intrusive_ref_counter.hpp>
namespace lex = boost::spirit::lex;
namespace qi = boost::spirit::qi;
namespace ascii = boost::spirit::ascii;
namespace frank
{
class ref_counter:public boost::intrusive_ref_counter<ref_counter>
{ public:
virtual ~ref_counter(void)
{
}
};
class symbol:public ref_counter
{ public:
typedef boost::intrusive_ptr<const symbol> symbolPtr;
typedef std::vector<symbolPtr> symbolVector;
struct push_scope
{ push_scope()
{
}
~push_scope(void)
{
}
};
};
class nature:public symbol
{ public:
enum enumAttribute
{ eAbstol,
eAccess,
eDDT,
eIDT,
eUnits
};
struct empty
{ bool operator<(const empty&) const
{ return false;
}
friend std::ostream &operator<<(std::ostream &_r, const empty&)
{ return _r;
}
};
typedef boost::variant<empty, std::string> attributeValue;
};
class discipline:public symbol
{ public:
enum enumDomain
{ eDiscrete,
eContinuous
};
};
class type:public ref_counter
{ public:
typedef boost::intrusive_ptr<type> typePtr;
};
struct myIterator:std::iterator<std::random_access_iterator_tag, char, std::ptrdiff_t, const char*, const char&>
{ std::string *m_p;
std::size_t m_iPos;
myIterator(void)
:m_p(nullptr),
m_iPos(~std::size_t(0))
{
}
myIterator(std::string &_r, const bool _bEnd = false)
:m_p(&_r),
m_iPos(_bEnd ? ~std::size_t(0) : 0)
{
}
myIterator(const myIterator &_r)
:m_p(_r.m_p),
m_iPos(_r.m_iPos)
{
}
myIterator &operator=(const myIterator &_r)
{ if (this != &_r)
{ m_p = _r.m_p;
m_iPos = _r.m_iPos;
}
return *this;
}
const char &operator*(void) const
{ return m_p->at(m_iPos);
}
bool operator==(const myIterator &_r) const
{ return m_p == _r.m_p && m_iPos == _r.m_iPos;
}
bool operator!=(const myIterator &_r) const
{ return m_p != _r.m_p || m_iPos != _r.m_iPos;
}
myIterator &operator++(void)
{ ++m_iPos;
if (m_iPos == m_p->size())
m_iPos = ~std::size_t(0);
return *this;
}
myIterator operator++(int)
{ const myIterator s(*this);
operator++();
return s;
}
myIterator &operator--(void)
{ --m_iPos;
return *this;
}
myIterator operator--(int)
{ const myIterator s(*this);
operator--();
return s;
}
bool operator<(const myIterator &_r) const
{ if (m_p == _r.m_p)
return m_iPos < _r.m_iPos;
else
return m_p < _r.m_p;
}
std::ptrdiff_t operator-(const myIterator &_r) const
{ return m_iPos - _r.m_iPos;
}
};
struct onInclude
{ auto operator()(myIterator &_rStart, myIterator &_rEnd) const
{ // erase what has been matched (the include statement)
_rStart.m_p->erase(_rStart.m_iPos, _rEnd.m_iPos - _rStart.m_iPos);
// and insert the contents of the file
_rStart.m_p->insert(_rStart.m_iPos, "abcd");
_rEnd = _rStart;
return lex::pass_flags::pass_ignore;
}
};
template<typename LEXER>
class lexer:public lex::lexer<LEXER>
{ public:
lex::token_def<type::typePtr> m_sKW_real, m_sKW_integer, m_sKW_string;
lex::token_def<lex::omit> m_sLineComment, m_sCComment;
lex::token_def<lex::omit> m_sWS;
lex::token_def<lex::omit> m_sSemicolon, m_sEqual, m_sColon, m_sInclude, m_sCharOP, m_sCharCP,
m_sComma;
lex::token_def<std::string> m_sIdentifier, m_sString;
lex::token_def<double> m_sReal;
lex::token_def<int> m_sInteger;
lex::token_def<lex::omit> m_sKW_units, m_sKW_access, m_sKW_idt_nature, m_sKW_ddt_nature, m_sKW_abstol,
m_sKW_nature, m_sKW_endnature, m_sKW_continuous, m_sKW_discrete,
m_sKW_potential, m_sKW_flow, m_sKW_domain, m_sKW_discipline, m_sKW_enddiscipline, m_sKW_module,
m_sKW_endmodule, m_sKW_parameter;
//typedef const type *typePtr;
template<typename T>
struct extractValue
{ T operator()(const myIterator &_rStart, const myIterator &_rEnd) const
{ return boost::lexical_cast<T>(std::string(_rStart, _rEnd));
}
};
struct extractString
{ std::string operator()(const myIterator &_rStart, const myIterator &_rEnd) const
{ const auto s = std::string(_rStart, _rEnd);
return s.substr(1, s.size() - 2);
}
};
lexer(void)
:m_sWS("[ \\t\\n\\r]+"),
m_sKW_parameter("\"parameter\""),
m_sKW_real("\"real\""),
m_sKW_integer("\"integer\""),
m_sKW_string("\"string\""),
m_sLineComment("\\/\\/[^\\n]*"),
m_sCComment("\\/\\*"
"("
"[^*]"
"|" "[\\n]"
"|" "([*][^/])"
")*"
"\\*\\/"),
m_sSemicolon("\";\""),
m_sEqual("\"=\""),
m_sColon("\":\""),
m_sCharOP("\"(\""),
m_sCharCP("\")\""),
m_sComma("\",\""),
m_sIdentifier("[a-zA-Z_]+[a-zA-Z0-9_]*"),
m_sString("[\\\"]"
//"("
// "(\\[\"])"
// "|"
//"[^\"]"
//")*"
"[^\\\"]*"
"[\\\"]"),
m_sKW_units("\"units\""),
m_sKW_access("\"access\""),
m_sKW_idt_nature("\"idt_nature\""),
m_sKW_ddt_nature("\"ddt_nature\""),
m_sKW_abstol("\"abstol\""),
m_sKW_nature("\"nature\""),
m_sKW_endnature("\"endnature\""),
m_sKW_continuous("\"continuous\""),
m_sKW_discrete("\"discrete\""),
m_sKW_domain("\"domain\""),
m_sKW_discipline("\"discipline\""),
m_sKW_enddiscipline("\"enddiscipline\""),
m_sKW_potential("\"potential\""),
m_sKW_flow("\"flow\""),
//realnumber ({uint}{exponent})|((({uint}\.{uint})|(\.{uint})){exponent}?)
//exponent [Ee][+-]?{uint}
//uint [0-9][_0-9]*
m_sReal("({uint}{exponent})"
"|"
"("
"(({uint}[\\.]{uint})|([\\.]{uint})){exponent}?"
")"
),
m_sInteger("{uint}"),
m_sInclude("\"`include\""),
m_sKW_module("\"module\""),
m_sKW_endmodule("\"endmodule\"")
{ this->self.add_pattern
("uint", "[0-9]+")
("exponent", "[eE][\\+\\-]?{uint}");
this->self = m_sSemicolon
| m_sEqual
| m_sColon
| m_sCharOP
| m_sCharCP
| m_sComma
| m_sString[lex::_val = boost::phoenix::bind(extractString(), lex::_start, lex::_end)]
| m_sKW_real//[lex::_val = boost::phoenix::bind(&type::getReal)]
| m_sKW_integer//[lex::_val = boost::phoenix::bind(&type::getInteger)]
| m_sKW_string//[lex::_val = boost::phoenix::bind(&type::getString)]
| m_sKW_parameter
| m_sKW_units
| m_sKW_access
| m_sKW_idt_nature
| m_sKW_ddt_nature
| m_sKW_abstol
| m_sKW_nature
| m_sKW_endnature
| m_sKW_continuous
| m_sKW_discrete
| m_sKW_domain
| m_sKW_discipline
| m_sKW_enddiscipline
| m_sReal[lex::_val = boost::phoenix::bind(extractValue<double>(), lex::_start, lex::_end)]
| m_sInteger[lex::_val = boost::phoenix::bind(extractValue<int>(), lex::_start, lex::_end)]
| m_sKW_potential
| m_sKW_flow
| m_sKW_module
| m_sKW_endmodule
| m_sIdentifier
| m_sInclude [ lex::_state = "INCLUDE" ]
;
this->self("INCLUDE") += m_sString [
lex::_state = "INITIAL", lex::_pass = boost::phoenix::bind(onInclude(), lex::_start, lex::_end)
];
this->self("WS") = m_sWS
| m_sLineComment
| m_sCComment
;
}
};
template<typename Iterator, typename Lexer>
class natureParser:public qi::grammar<Iterator, symbol::symbolPtr(void), qi::in_state_skipper<Lexer> >
{ qi::rule<Iterator, symbol::symbolPtr(void), qi::in_state_skipper<Lexer> > m_sStart;
qi::rule<Iterator, std::pair<nature::enumAttribute, nature::attributeValue>(void), qi::in_state_skipper<Lexer> > m_sProperty;
qi::rule<Iterator, std::string(), qi::in_state_skipper<Lexer> > m_sName;
public:
template<typename Tokens>
natureParser(const Tokens &_rTokens)
:natureParser::base_type(m_sStart)
{ m_sProperty = (_rTokens.m_sKW_units
>> _rTokens.m_sEqual
>> _rTokens.m_sString
>> _rTokens.m_sSemicolon
)
| (_rTokens.m_sKW_access
>> _rTokens.m_sEqual
>> _rTokens.m_sIdentifier
>> _rTokens.m_sSemicolon
)
| (_rTokens.m_sKW_idt_nature
>> _rTokens.m_sEqual
>> _rTokens.m_sIdentifier
>> _rTokens.m_sSemicolon
)
| (_rTokens.m_sKW_ddt_nature
>> _rTokens.m_sEqual
>> _rTokens.m_sIdentifier
>> _rTokens.m_sSemicolon
)
| (_rTokens.m_sKW_abstol
>> _rTokens.m_sEqual
>> _rTokens.m_sReal
>> _rTokens.m_sSemicolon
)
;
m_sName = (_rTokens.m_sColon >> _rTokens.m_sIdentifier);
m_sStart = (_rTokens.m_sKW_nature
>> _rTokens.m_sIdentifier
>> -m_sName
>> _rTokens.m_sSemicolon
>> *(m_sProperty)
>> _rTokens.m_sKW_endnature
);
m_sStart.name("start");
m_sProperty.name("property");
}
};
/*
// Conservative discipline
discipline electrical;
potential Voltage;
flow Current;
enddiscipline
*/
// a parser for a discipline declaration
template<typename Iterator, typename Lexer>
class disciplineParser:public qi::grammar<Iterator, symbol::symbolPtr(void), qi::in_state_skipper<Lexer> >
{ qi::rule<Iterator, symbol::symbolPtr(void), qi::in_state_skipper<Lexer> > m_sStart;
typedef std::pair<bool, boost::intrusive_ptr<const nature> > CPotentialAndNature;
struct empty
{ bool operator<(const empty&) const
{ return false;
}
friend std::ostream &operator<<(std::ostream &_r, const empty&)
{ return _r;
}
};
typedef boost::variant<empty, CPotentialAndNature, discipline::enumDomain> property;
qi::rule<Iterator, discipline::enumDomain(), qi::in_state_skipper<Lexer> > m_sDomain;
qi::rule<Iterator, property(void), qi::in_state_skipper<Lexer> > m_sProperty;
public:
template<typename Tokens>
disciplineParser(const Tokens &_rTokens)
:disciplineParser::base_type(m_sStart)
{ m_sDomain = _rTokens.m_sKW_continuous
| _rTokens.m_sKW_discrete
;
m_sProperty = (_rTokens.m_sKW_potential >> _rTokens.m_sIdentifier >> _rTokens.m_sSemicolon)
| (_rTokens.m_sKW_flow >> _rTokens.m_sIdentifier >> _rTokens.m_sSemicolon)
| (_rTokens.m_sKW_domain >> m_sDomain >> _rTokens.m_sSemicolon)
;
m_sStart = (_rTokens.m_sKW_discipline
>> _rTokens.m_sIdentifier
>> _rTokens.m_sSemicolon
>> *m_sProperty
>> _rTokens.m_sKW_enddiscipline
);
}
};
template<typename Iterator, typename Lexer>
class moduleParser:public qi::grammar<Iterator, symbol::symbolPtr(void), qi::in_state_skipper<Lexer> >
{ public:
qi::rule<Iterator, symbol::symbolPtr(void), qi::in_state_skipper<Lexer> > m_sStart;
qi::rule<Iterator, symbol::symbolVector(void), qi::in_state_skipper<Lexer> > m_sModulePortList;
qi::rule<Iterator, symbol::symbolVector(void), qi::in_state_skipper<Lexer> > m_sPortList;
qi::rule<Iterator, symbol::symbolPtr(void), qi::in_state_skipper<Lexer> > m_sPort;
qi::rule<Iterator, std::shared_ptr<symbol::push_scope>(void), qi::in_state_skipper<Lexer> > m_sModule;
typedef boost::intrusive_ptr<const ref_counter> intrusivePtr;
typedef std::vector<intrusivePtr> vectorOfPtr;
qi::rule<Iterator, vectorOfPtr(void), qi::in_state_skipper<Lexer> > m_sModuleItemList;
qi::rule<Iterator, intrusivePtr(void), qi::in_state_skipper<Lexer> > m_sParameter;
qi::rule<Iterator, intrusivePtr(void), qi::in_state_skipper<Lexer> > m_sModuleItem;
qi::rule<Iterator, type::typePtr(void), qi::in_state_skipper<Lexer> > m_sType;
template<typename Tokens>
moduleParser(const Tokens &_rTokens)
:moduleParser::base_type(m_sStart)
{ m_sPort = _rTokens.m_sIdentifier;
m_sPortList %= m_sPort % _rTokens.m_sComma;
m_sModulePortList %= _rTokens.m_sCharOP >> m_sPortList >> _rTokens.m_sCharCP;
m_sModule = _rTokens.m_sKW_module;
m_sType = _rTokens.m_sKW_real | _rTokens.m_sKW_integer | _rTokens.m_sKW_string;
m_sParameter = _rTokens.m_sKW_parameter
>> m_sType
>> _rTokens.m_sIdentifier
;
m_sModuleItem = m_sParameter;
m_sModuleItemList %= *m_sModuleItem;
m_sStart = (m_sModule
>> _rTokens.m_sIdentifier
>> m_sModulePortList
>> m_sModuleItemList
>> _rTokens.m_sKW_endmodule);
}
};
template<typename Iterator, typename Lexer>
class fileParser:public qi::grammar<Iterator, symbol::symbolVector(void), qi::in_state_skipper<Lexer> >
{ public:
disciplineParser<Iterator, Lexer> m_sDiscipline;
natureParser<Iterator, Lexer> m_sNature;
moduleParser<Iterator, Lexer> m_sModule;
qi::rule<Iterator, symbol::symbolVector(void), qi::in_state_skipper<Lexer> > m_sStart;
qi::rule<Iterator, symbol::symbolPtr(void), qi::in_state_skipper<Lexer> > m_sItem;
//public:
template<typename Tokens>
fileParser(const Tokens &_rTokens)
:fileParser::base_type(m_sStart),
m_sNature(_rTokens),
m_sDiscipline(_rTokens),
m_sModule(_rTokens)
{ m_sItem = m_sDiscipline | m_sNature | m_sModule;
m_sStart = *m_sItem;
}
};
}
int main()
{ std::string sInput = "\
nature Current;\n\
units = \"A\";\n\
access = I;\n\
idt_nature = Charge;\n\
abstol = 1e-12;\n\
endnature\n\
\n\
// Charge in coulombs\n\
nature Charge;\n\
units = \"coul\";\n\
access = Q;\n\
ddt_nature = Current;\n\
abstol = 1e-14;\n\
endnature\n\
\n\
// Potential in volts\n\
nature Voltage;\n\
units = \"V\";\n\
access = V;\n\
idt_nature = Flux;\n\
abstol = 1e-6;\n\
endnature\n\
\n\
discipline electrical;\n\
potential Voltage;\n\
flow Current;\n\
enddiscipline\n\
";
typedef lex::lexertl::token<frank::myIterator, boost::mpl::vector<frank::type::typePtr, std::string, double, int> > token_type;
typedef lex::lexertl::actor_lexer<token_type> lexer_type;
typedef frank::lexer<lexer_type>::iterator_type iterator_type;
typedef frank::fileParser<iterator_type, frank::lexer<lexer_type>::lexer_def> grammar_type;
frank::lexer<lexer_type> sLexer;
grammar_type sParser(sLexer);
frank::symbol::push_scope sPush;
auto pStringBegin = frank::myIterator(sInput);
auto pBegin(sLexer.begin(pStringBegin, frank::myIterator(sInput, true)));
const auto b = qi::phrase_parse(pBegin, sLexer.end(), sParser, qi::in_state("WS")[sLexer.self]);
}
Has anybody gotten this working for a lexer returning something else than double or int or string?
Sure. Simple examples might be found on this site
And for the parser also returning non-trivial objects?
Here's your real problem. Spirit is nice for a subset of parsers that are expressed easily in a eDSL, and has the huge benefit of "magically" mapping to a selection of attributes.
Some of the realities are:
attributes are expected to have value-semantic; using polymorphic attributes is hard (How can I use polymorphic attributes with boost::spirit::qi parsers?, e.g.)
using Lex makes most of the sweet-spot disappear since all "highlevel" parsers (like real_parser, [u]int_parser) are out the window. The Spirit devs are on record they prefer not to use Lex. Moreover, Spirit X3 doesn't have Lex support anymore.
Background Information:
I'd very much consider parsing the source as-is, into direct value-typed AST nodes. I know, this is probably what you consider "trivial objects", but don't be deceived by apparent simplicity: recursive variant trees have some expressive power.
Examples
Here's a trivial AST to represent JSON in <20 LoC: Boost Karma generator for composition of classes¹
Here we represent the Graphviz source format with full fidelity: How to use boost spirit list operator with mandatory minimum amount of elements?
I've since created the code to transform that AST into a domain representation with fully correct ownership, cascading lexically scoped node/edge attributes and cross references. I have just recovered that work and put it up on github if you're interested, mainly because the task is pretty similar in many respects, like the overriding/inheriting of properties and resolving identifiers within scopes: https://github.com/sehe/spirit-graphviz/blob/master/spirit-graphviz.cpp#L660
Suggestions, Ideas
In your case I'd take similar approach to retain simplicity. The code shown doesn't (yet) cover the trickiest ingredients (like nature attribute overrides within a discipline).
Once you start implementing use-cases like resolving compatible disciplines and the absolute tolerances at a given node, you want a domain model with full fidelity. Preferrably, there would be no loss of source information, and immutable AST information².
As a middle ground, you could probably avoid building an entire source-AST in memory only to transform it in one big go, at the top-level you could have:
file = qi::skip(skipper) [
*(m_sDiscipline | m_sNature | m_sModule) [process_ast(_1)]
];
Where process_ast would apply the "trivial" AST representation into the domain types, one at a time. That way you keep only small bits of temporary AST representation around.
The domain representation can be arbitrarily sophisticated to support all your logic and use-cases.
Let's "Show, Don't Tell"
Baking the simplest AST that comes to mind matching the grammar³:
namespace frank { namespace ast {
struct nature {
struct empty{};
std::string name;
std::string inherits;
enum class Attribute { units, access, idt, ddt, abstol };
using Value = boost::variant<int, double, std::string>;
std::map<Attribute, Value> attributes;
};
struct discipline {
enum enumDomain { eUnspecified, eDiscrete, eContinuous };
struct properties_t {
enumDomain domain = eUnspecified;
boost::optional<std::string> flow, potential;
};
std::string name;
properties_t properties;
};
// TODO
using module = qi::unused_type;
using file = std::vector<boost::variant<nature, discipline, module> >;
enum class type { real, integer, string };
} }
This is trivial and maps 1:1 onto the grammar productions, which means we have very little impedance.
Tokens? We Don't Need Lex For That
You can have common token parsers without requiring the complexities of Lex
Yes, Lex (especially statically generated) can potentially improve performance, but
if you need that, I wager Spirit Qi is not your best option anyways
premature optimization...
What I did:
struct tokens {
// implicit lexemes
qi::rule<It, std::string()> string, identifier;
qi::rule<It, double()> real;
qi::rule<It, int()> integer;
qi::rule<It, ast::nature::Value()> value;
qi::rule<It, ast::nature::Attribute()> attribute;
qi::rule<It, ast::discipline::enumDomain()> domain;
struct attribute_sym_t : qi::symbols<char, ast::nature::Attribute> {
attribute_sym_t() {
this->add
("units", ast::nature::Attribute::units)
("access", ast::nature::Attribute::access)
("idt_nature", ast::nature::Attribute::idt)
("ddt_nature", ast::nature::Attribute::ddt)
("abstol", ast::nature::Attribute::abstol);
}
} attribute_sym;
struct domain_sym_t : qi::symbols<char, ast::discipline::enumDomain> {
domain_sym_t() {
this->add
("discrete", ast::discipline::eDiscrete)
("continuous", ast::discipline::eContinuous);
}
} domain_sym;
tokens() {
using namespace qi;
auto kw = qr::distinct(copy(char_("a-zA-Z0-9_")));
string = '"' >> *("\\" >> char_ | ~char_('"')) >> '"';
identifier = char_("a-zA-Z_") >> *char_("a-zA-Z0-9_");
real = double_;
integer = int_;
attribute = kw[attribute_sym];
domain = kw[domain_sym];
value = string | identifier | real | integer;
BOOST_SPIRIT_DEBUG_NODES((string)(identifier)(real)(integer)(value)(domain)(attribute))
}
};
Liberating, isn't it? Note how
all attributes are automatically propagated
strings handle escapes (this bit was commented out in your Lex approach). We don't even need semantic actions to (badly) pry out the unquoted/unescaped value
we used distinct to ensure keyword parsing matches only full identifiers. (See How to parse reserved words correctly in boost spirit).
This is actually where you notice the lack of separate lexer.
On the flipside, this makes context-sensitive keywords a breeze (lex can easily prioritizes keywords over identifiers that occur in places where keywords cannot occur.⁴)
What About Skipping Space/Comments?
We could have added a token, but for reasons of convention I made it a parser:
struct skipParser : qi::grammar<It> {
skipParser() : skipParser::base_type(spaceOrComment) {
using namespace qi;
spaceOrComment = space
| ("//" >> *(char_ - eol) >> (eoi|eol))
| ("/*" >> *(char_ - "*/") >> "*/");
BOOST_SPIRIT_DEBUG_NODES((spaceOrComment))
}
private:
qi::rule<It> spaceOrComment;
};
natureParser
We inherit our AST parsers from tokens:
struct natureParser : tokens, qi::grammar<It, ast::nature(), skipParser> {
And from there it is plain sailing:
property = attribute >> '=' >> value >> ';';
nature
= kw["nature"] >> identifier >> -(':' >> identifier) >> ';'
>> *property
>> kw["endnature"];
disciplineParser
discipline = kw["discipline"] >> identifier >> ';'
>> properties
>> kw["enddiscipline"]
;
properties
= kw["domain"] >> domain >> ';'
^ kw["flow"] >> identifier >> ';'
^ kw["potential"] >> identifier >> ';'
;
This shows a competing approach that uses the permutation operator (^) to parse optional alternatives in any order into a fixed frank::ast::discipline properties struct. Of course, you might elect to have a more generic representation here, like we had with ast::nature.
Module AST is left as an exercise for the reader, though the parser rules are implemented below.
Top Level, Encapsulating The Skipper
I hate having to specify the skipper from the calling code (it's more complex than required, and changing the skipper changes the grammar). So, I encapsulate it in the top-level parser:
struct fileParser : qi::grammar<It, ast::file()> {
fileParser() : fileParser::base_type(file) {
file = qi::skip(qi::copy(m_sSkip)) [
*(m_sDiscipline | m_sNature | m_sModule)
];
BOOST_SPIRIT_DEBUG_NODES((file))
}
private:
disciplineParser m_sDiscipline;
natureParser m_sNature;
moduleParser m_sModule;
skipParser m_sSkip;
qi::rule<It, ast::file()> file;
};
Demo Time
This demo adds operator<< for the enums, and a variant visitor to print some AST details for debug/demonstrational purposes (print_em).
Then we have a test driver:
int main() {
using iterator_type = std::string::const_iterator;
iterator_type iter = sInput.begin(), last = sInput.end();
frank::Parsers<iterator_type>::fileParser parser;
print_em print;
frank::ast::file file;
bool ok = qi::parse(iter, last, parser, file);
if (ok) {
for (auto& symbol : file)
print(symbol);
}
else {
std::cout << "Parse failed\n";
}
if (iter != last) {
std::cout << "Remaining unparsed: '" << std::string(iter,last) << "'\n";
}
}
With the sample input from your question we get the following output:
Live On Coliru
-- Nature
name: Current
inherits:
attribute: units = A
attribute: access = I
attribute: idt = Charge
attribute: abstol = 1e-12
-- Nature
name: Charge
inherits:
attribute: units = coul
attribute: access = Q
attribute: ddt = Current
attribute: abstol = 1e-14
-- Nature
name: Voltage
inherits:
attribute: units = V
attribute: access = V
attribute: idt = Flux
attribute: abstol = 1e-06
-- Discipline
name: electrical
domain: (unspecified)
flow: Current
potential: Voltage
Remaining unparsed: '
'
With BOOST_SPIRIT_DEBUG defined, you get rich debug information: Live On Coliru
Full Listing
Live On Coliru
//#define BOOST_SPIRIT_DEBUG
#include <map>
#include <boost/spirit/include/qi.hpp>
#include <boost/fusion/include/adapted.hpp>
#include <boost/spirit/repository/include/qi_distinct.hpp>
namespace qi = boost::spirit::qi;
namespace frank { namespace ast {
struct nature {
struct empty{};
std::string name;
std::string inherits;
enum class Attribute { units, access, idt, ddt, abstol };
using Value = boost::variant<int, double, std::string>;
std::map<Attribute, Value> attributes;
};
struct discipline {
enum enumDomain { eUnspecified, eDiscrete, eContinuous };
struct properties_t {
enumDomain domain = eUnspecified;
boost::optional<std::string> flow, potential;
};
std::string name;
properties_t properties;
};
// TODO
using module = qi::unused_type;
using file = std::vector<boost::variant<nature, discipline, module> >;
enum class type { real, integer, string };
} }
BOOST_FUSION_ADAPT_STRUCT(frank::ast::nature, name, inherits, attributes)
BOOST_FUSION_ADAPT_STRUCT(frank::ast::discipline, name, properties)
BOOST_FUSION_ADAPT_STRUCT(frank::ast::discipline::properties_t, domain, flow, potential)
namespace frank {
namespace qr = boost::spirit::repository::qi;
template <typename It> struct Parsers {
struct tokens {
// implicit lexemes
qi::rule<It, std::string()> string, identifier;
qi::rule<It, double()> real;
qi::rule<It, int()> integer;
qi::rule<It, ast::nature::Value()> value;
qi::rule<It, ast::nature::Attribute()> attribute;
qi::rule<It, ast::discipline::enumDomain()> domain;
struct attribute_sym_t : qi::symbols<char, ast::nature::Attribute> {
attribute_sym_t() {
this->add
("units", ast::nature::Attribute::units)
("access", ast::nature::Attribute::access)
("idt_nature", ast::nature::Attribute::idt)
("ddt_nature", ast::nature::Attribute::ddt)
("abstol", ast::nature::Attribute::abstol);
}
} attribute_sym;
struct domain_sym_t : qi::symbols<char, ast::discipline::enumDomain> {
domain_sym_t() {
this->add
("discrete", ast::discipline::eDiscrete)
("continuous", ast::discipline::eContinuous);
}
} domain_sym;
tokens() {
using namespace qi;
auto kw = qr::distinct(copy(char_("a-zA-Z0-9_")));
string = '"' >> *("\\" >> char_ | ~char_('"')) >> '"';
identifier = char_("a-zA-Z_") >> *char_("a-zA-Z0-9_");
real = double_;
integer = int_;
attribute = kw[attribute_sym];
domain = kw[domain_sym];
value = string | identifier | real | integer;
BOOST_SPIRIT_DEBUG_NODES((string)(identifier)(real)(integer)(value)(domain)(attribute))
}
};
struct skipParser : qi::grammar<It> {
skipParser() : skipParser::base_type(spaceOrComment) {
using namespace qi;
spaceOrComment = space
| ("//" >> *(char_ - eol) >> (eoi|eol))
| ("/*" >> *(char_ - "*/") >> "*/");
BOOST_SPIRIT_DEBUG_NODES((spaceOrComment))
}
private:
qi::rule<It> spaceOrComment;
};
struct natureParser : tokens, qi::grammar<It, ast::nature(), skipParser> {
natureParser() : natureParser::base_type(nature) {
using namespace qi;
auto kw = qr::distinct(copy(char_("a-zA-Z0-9_")));
property = attribute >> '=' >> value >> ';';
nature
= kw["nature"] >> identifier >> -(':' >> identifier) >> ';'
>> *property
>> kw["endnature"];
BOOST_SPIRIT_DEBUG_NODES((nature)(property))
}
private:
using Attribute = std::pair<ast::nature::Attribute, ast::nature::Value>;
qi::rule<It, ast::nature(), skipParser> nature;
qi::rule<It, Attribute(), skipParser> property;
using tokens::attribute;
using tokens::value;
using tokens::identifier;
};
struct disciplineParser : tokens, qi::grammar<It, ast::discipline(), skipParser> {
disciplineParser() : disciplineParser::base_type(discipline) {
auto kw = qr::distinct(qi::copy(qi::char_("a-zA-Z0-9_")));
discipline = kw["discipline"] >> identifier >> ';'
>> properties
>> kw["enddiscipline"]
;
properties
= kw["domain"] >> domain >> ';'
^ kw["flow"] >> identifier >> ';'
^ kw["potential"] >> identifier >> ';'
;
BOOST_SPIRIT_DEBUG_NODES((discipline)(properties))
}
private:
qi::rule<It, ast::discipline(), skipParser> discipline;
qi::rule<It, ast::discipline::properties_t(), skipParser> properties;
using tokens::domain;
using tokens::identifier;
};
struct moduleParser : tokens, qi::grammar<It, ast::module(), skipParser> {
moduleParser() : moduleParser::base_type(module) {
auto kw = qr::distinct(qi::copy(qi::char_("a-zA-Z0-9_")));
m_sPort = identifier;
m_sPortList = m_sPort % ',';
m_sModulePortList = '(' >> m_sPortList >> ')';
m_sModule = kw["module"];
m_sType = kw["real"] | kw["integer"] | kw["string"];
m_sParameter = kw["parameter"] >> m_sType >> identifier;
m_sModuleItem = m_sParameter;
m_sModuleItemList = *m_sModuleItem;
module =
(m_sModule >> identifier >> m_sModulePortList >> m_sModuleItemList >> kw["endmodule"]);
}
private:
qi::rule<It, ast::module(), skipParser> module;
qi::rule<It, skipParser> m_sModulePortList;
qi::rule<It, skipParser> m_sPortList;
qi::rule<It, skipParser> m_sPort;
qi::rule<It, skipParser> m_sModule;
qi::rule<It, skipParser> m_sModuleItemList;
qi::rule<It, skipParser> m_sParameter;
qi::rule<It, skipParser> m_sModuleItem;
qi::rule<It, skipParser> m_sType;
using tokens::identifier;
};
struct fileParser : qi::grammar<It, ast::file()> {
fileParser() : fileParser::base_type(file) {
file = qi::skip(qi::copy(m_sSkip)) [
*(m_sDiscipline | m_sNature | m_sModule)
];
BOOST_SPIRIT_DEBUG_NODES((file))
}
private:
disciplineParser m_sDiscipline;
natureParser m_sNature;
moduleParser m_sModule;
skipParser m_sSkip;
qi::rule<It, ast::file()> file;
};
};
}
extern std::string const sInput;
// just for demo
#include <boost/optional/optional_io.hpp>
namespace frank { namespace ast {
//static inline std::ostream &operator<<(std::ostream &os, const nature::empty &) { return os; }
static inline std::ostream &operator<<(std::ostream &os, nature::Attribute a) {
switch(a) {
case nature::Attribute::units: return os << "units";
case nature::Attribute::access: return os << "access";
case nature::Attribute::idt: return os << "idt";
case nature::Attribute::ddt: return os << "ddt";
case nature::Attribute::abstol: return os << "abstol";
};
return os << "?";
}
static inline std::ostream &operator<<(std::ostream &os, discipline::enumDomain d) {
switch(d) {
case discipline::eDiscrete: return os << "discrete";
case discipline::eContinuous: return os << "continuous";
case discipline::eUnspecified: return os << "(unspecified)";
};
return os << "?";
}
} }
struct print_em {
using result_type = void;
template <typename V>
void operator()(V const& variant) const {
boost::apply_visitor(*this, variant);
}
void operator()(frank::ast::nature const& nature) const {
std::cout << "-- Nature\n";
std::cout << "name: " << nature.name << "\n";
std::cout << "inherits: " << nature.inherits << "\n";
for (auto& a : nature.attributes) {
std::cout << "attribute: " << a.first << " = " << a.second << "\n";
}
}
void operator()(frank::ast::discipline const& discipline) const {
std::cout << "-- Discipline\n";
std::cout << "name: " << discipline.name << "\n";
std::cout << "domain: " << discipline.properties.domain << "\n";
std::cout << "flow: " << discipline.properties.flow << "\n";
std::cout << "potential: " << discipline.properties.potential << "\n";
}
void operator()(frank::ast::module const&) const {
std::cout << "-- Module (TODO)\n";
}
};
int main() {
using iterator_type = std::string::const_iterator;
iterator_type iter = sInput.begin(), last = sInput.end();
frank::Parsers<iterator_type>::fileParser parser;
print_em print;
frank::ast::file file;
bool ok = parse(iter, last, parser, file);
if (ok) {
for (auto& symbol : file)
print(symbol);
}
else {
std::cout << "Parse failed\n";
}
if (iter != last) {
std::cout << "Remaining unparsed: '" << std::string(iter,last) << "'\n";
}
}
std::string const sInput = R"(
nature Current;
units = "A";
access = I;
idt_nature = Charge;
abstol = 1e-12;
endnature
// Charge in coulombs
nature Charge;
units = "coul";
access = Q;
ddt_nature = Current;
abstol = 1e-14;
endnature
// Potential in volts
nature Voltage;
units = "V";
access = V;
idt_nature = Flux;
abstol = 1e-6;
endnature
discipline electrical;
potential Voltage;
flow Current;
enddiscipline
)";
¹ incidentally, the other answer there demonstrates the "impedance mismatch" with polymorphic attributes and Spirit - this time on the Karma side of it
² (to prevent subtle bugs that depend on evaluation order or things like that, e.g.)
³ (gleaning some from here but not importing too much complexity that wasn't reflected in your Lex approach)
⁴ (In fact, this is where you'd need state-switching inside the grammar, an area notoriously underdeveloped and practically unusable in Spirit Lex: e.g. when it works how to avoid defining token which matchs everything in boost::spirit::lex or when it goes badly: Boost.Spirit SQL grammar/lexer failure)
One solution would be to use a std::string everywhere and define a boost::variant with everything needed but not use it anywhere in the parser or lexer directly but only serialize & deserialize it into/from the string.
Is this what the originators of boost::spirit intended?

How to refer to value from symbol table?

I can't figure out how to pass value from symbol table into the function.
template <typename Iterator>
class single_attribute_grammar : public qi::grammar<Iterator, AttributeData(), qi::blank_type>
{
public:
single_attribute_grammar(const word_symbols &words) : single_attribute_grammar::base_type(single_attribute_rule)
{
auto attr_word = phx::bind(&AttributeData::word, qi::_val);
auto grammar_word = phx::bind(&WordGrammar::word, qi::_1);
auto attr_value = phx::bind(&AttributeData::value, qi::_val);
single_attribute_rule = qi::lexeme[words[attr_word = grammar_word] >
qi::int_[attr_value = qi::_1] > (qi::space|qi::eoi)] >>
qi::eps(phx::bind(verify_range, qi::_r1, qi::_val)); // <-- HERE is the problem
BOOST_SPIRIT_DEBUG_NODE(single_attribute_rule);
}
private:
qi::rule<Iterator, AttributeData(), qi::blank_type> single_attribute_rule;
};
I would think I can refer to the value of the found key using qi::_r1 but the code doesn't compile:
main.cpp:72:31: required from ‘single_attribute_grammar<Iterator>::single_attribute_grammar(const word_symbols&) [with Iterator = boost::spirit::classic::position_iterator2<boost::spirit::multi_pass<std::istreambuf_iterator<char, std::char_traits<char> > > >; word_symbols = boost::spirit::qi::symbols<char, WordGrammar>]’
main.cpp:87:16: required from ‘all_attributes_grammar<Iterator>::all_attributes_grammar(const word_symbols&) [with Iterator = boost::spirit::classic::position_iterator2<boost::spirit::multi_pass<std::istreambuf_iterator<char, std::char_traits<char> > > >; word_symbols = boost::spirit::qi::symbols<char, WordGrammar>]’
main.cpp:130:62: required from here
/usr/include/boost/spirit/home/support/context.hpp:180:13: error: static assertion failed: index_is_out_of_bounds
BOOST_SPIRIT_ASSERT_MSG(
^
In file included from /usr/include/boost/spirit/home/qi/domain.hpp:18:0,
from /usr/include/boost/spirit/home/qi/meta_compiler.hpp:15,
from /usr/include/boost/spirit/home/qi/action/action.hpp:14,
from /usr/include/boost/spirit/home/qi/action.hpp:14,
from /usr/include/boost/spirit/home/qi.hpp:14,
from /usr/include/boost/spirit/include/qi.hpp:16,
from main.cpp:11:
/usr/include/boost/spirit/home/support/context.hpp:186:13: error: no type named ‘type’ in ‘struct boost::fusion::result_of::at_c<boost::fusion::cons<AttributeData&, boost::fusion::nil_>, 1>’
type;
^~~~
In case it's helpful here is the MVCE.
#include <iomanip>
#include <string>
#include <vector>
#include <boost/variant.hpp>
#include <boost/optional/optional.hpp>
#define BOOST_SPIRIT_DEBUG
#include <boost/config/warning_disable.hpp>
#include <boost/spirit/include/qi.hpp>
#include <boost/spirit/include/qi_symbols.hpp>
#include <boost/spirit/include/phoenix_core.hpp>
#include <boost/spirit/include/phoenix_operator.hpp>
#include <boost/spirit/include/phoenix_object.hpp> // construct
#include <boost/spirit/include/support_multi_pass.hpp>
#include <boost/spirit/include/classic_position_iterator.hpp>
#include <boost/fusion/include/adapt_struct.hpp>
#include <boost/phoenix/bind.hpp>
namespace qi = boost::spirit::qi;
namespace ascii = boost::spirit::ascii;
namespace classic = boost::spirit::classic;
namespace phx = boost::phoenix;
namespace fusion = boost::fusion;
struct AttributeData
{
std::string word;
int value;
};
using AttributeVariant = boost::variant<
AttributeData
>;
struct WordGrammar
{
std::string word;
int range_from;
int range_to;
};
BOOST_FUSION_ADAPT_STRUCT(
AttributeData,
(std::string, word)
(int, value)
)
using word_symbols = qi::symbols<char, WordGrammar>;
bool verify_range(const WordGrammar &grammar, const AttributeData &data)
{
if(data.value < grammar.range_from || data.value > grammar.range_to)
{
return false;
}
return true;
}
template <typename Iterator>
class single_attribute_grammar : public qi::grammar<Iterator, AttributeData(), qi::blank_type>
{
public:
single_attribute_grammar(const word_symbols &words) : single_attribute_grammar::base_type(single_attribute_rule)
{
auto attr_word = phx::bind(&AttributeData::word, qi::_val);
auto grammar_word = phx::bind(&WordGrammar::word, qi::_1);
auto attr_value = phx::bind(&AttributeData::value, qi::_val);
single_attribute_rule = qi::lexeme[words[attr_word = grammar_word] >
qi::int_[attr_value = qi::_1] > (qi::space|qi::eoi)] >>
qi::eps(phx::bind(verify_range, qi::_r1, qi::_val)); // <-- HERE is the problem
BOOST_SPIRIT_DEBUG_NODE(single_attribute_rule);
}
private:
qi::rule<Iterator, AttributeData(), qi::blank_type> single_attribute_rule;
};
template <typename Iterator>
class all_attributes_grammar : public qi::grammar<Iterator, std::vector<AttributeVariant>(), qi::blank_type>
{
public:
all_attributes_grammar(const word_symbols &words) : all_attributes_grammar::base_type(line_attribute_vec_rule)
, sag(words)
{
line_attribute_rule = (
sag
);
BOOST_SPIRIT_DEBUG_NODE(line_attribute_rule);
line_attribute_vec_rule = (line_attribute_rule % *qi::blank) > qi::eoi;
BOOST_SPIRIT_DEBUG_NODE(line_attribute_vec_rule);
}
private:
single_attribute_grammar<Iterator> sag;
qi::rule<Iterator, AttributeVariant(), qi::blank_type> line_attribute_rule;
qi::rule<Iterator, std::vector<AttributeVariant>(), qi::blank_type> line_attribute_vec_rule;
};
int main()
{
std::vector<AttributeVariant> value;
std::string data{"N100 X-100 AC5"};
std::istringstream input(data);
// iterate over stream input
typedef std::istreambuf_iterator<char> base_iterator_type;
base_iterator_type in_begin(input);
// convert input iterator to forward iterator, usable by spirit parser
typedef boost::spirit::multi_pass<base_iterator_type> forward_iterator_type;
forward_iterator_type fwd_begin = boost::spirit::make_default_multi_pass(in_begin);
forward_iterator_type fwd_end;
// wrap forward iterator with position iterator, to record the position
typedef classic::position_iterator2<forward_iterator_type> pos_iterator_type;
pos_iterator_type position_begin(fwd_begin, fwd_end);
pos_iterator_type position_end;
word_symbols sym;
sym.add
("N", {"N", 1, 9999})
("X", {"X", -999, 999})
("AC", {"AC", -99, 999})
;
all_attributes_grammar<pos_iterator_type> all_attr_gr(sym);
try
{
qi::phrase_parse(position_begin, position_end, all_attr_gr, qi::blank, value);
}
catch (const qi::expectation_failure<pos_iterator_type>& e)
{
const classic::file_position_base<std::string>& pos = e.first.get_position();
std::cout <<
"Parse error at line " << pos.line << " column " << pos.column << ":" << std::endl <<
"'" << e.first.get_currentline() << "'" << std::endl <<
std::setw(pos.column) << " " << "^- here" << std::endl;
}
return 0;
}
Any ideas?
Lots of things to be simplified.
why is AttributeData adapted, when you use semantic actions instead? (see Boost Spirit: "Semantic actions are evil"?)
skipping whitespace is much more elegant when using a skipper. It's also logically contradictory to skip whitespace inside a lexeme (see Boost spirit skipper issues)
In particular, the skipping of whitespace here:
line_attribute_vec_rule = (line_attribute_rule % *qi::blank) > qi::eoi;
is completely impotent (because blanks are already skipped, the *qi::blank will never match any characters). The whole thing reduces to +line_attribute_rule.
line_attribute_vec_rule = +line_attribute_rule > qi::eoi;
Re: your answer
Indeed, you can use more state in the rule. Instead, I'd simplify the AST to support your case. Say:
struct AttrDef {
std::string word;
std::pair<int,int> range;
};
struct AttributeData {
AttrDef def;
int value;
bool is_valid() const {
return std::minmax({value, def.range.first, def.range.second}) == def.range;
}
};
BOOST_FUSION_ADAPT_STRUCT(AttributeData, def, value)
Now, single_attribute_grammar would not care about skipping, and be like:
using attr_defs = qi::symbols<char, AttrDef>;
template <typename Iterator>
struct single_attribute_grammar : public qi::grammar<Iterator, AttributeData()> {
single_attribute_grammar(const attr_defs &defs)
: single_attribute_grammar::base_type(start)
{
using namespace qi;
attribute_data = defs >> int_;
start %= attribute_data [ _pass = is_valid_(_1) ];
BOOST_SPIRIT_DEBUG_NODES((attribute_data));
}
private:
phx::function<std::function<bool(AttributeData const&)> > is_valid_ {&AttributeData::is_valid};
qi::rule<Iterator, AttributeData()> attribute_data;
qi::rule<Iterator, AttributeData()> start;
};
As you can see, there's no state because I didn't use eps. That's right, I flew right in the face of my own guideline ("avoid semantic actions") for the simple reason that it avoids explicit state (instead using the exsting qi::_pass).
Doing a lot of simplifications in main (specifically, using boost::spirit::istream_iterator rather than roll-your-own multi-pass adapting), I would arrive at this:
Live On Coliru
//#define BOOST_SPIRIT_DEBUG
#include <boost/spirit/include/qi.hpp>
#include <boost/spirit/include/phoenix.hpp>
#include <boost/spirit/include/classic_position_iterator.hpp>
#include <boost/fusion/include/adapt_struct.hpp>
#include <boost/fusion/include/io.hpp>
#include <iomanip>
namespace qi = boost::spirit::qi;
namespace phx = boost::phoenix;
struct AttrDef {
std::string word;
std::pair<int,int> range;
};
struct AttributeData {
AttrDef def;
int value;
bool is_valid() const {
return std::minmax({value, def.range.first, def.range.second}) == def.range;
}
};
BOOST_FUSION_ADAPT_STRUCT(AttributeData, def, value)
static inline std::ostream& operator<<(std::ostream& os, const AttrDef& def) {
return os << "AttrDef(" << def.word << ", " << def.range.first << ", " << def.range.second << ")";
}
using attr_defs = qi::symbols<char, AttrDef>;
template <typename Iterator>
struct single_attribute_grammar : public qi::grammar<Iterator, AttributeData()> {
single_attribute_grammar(const attr_defs &defs)
: single_attribute_grammar::base_type(start)
{
using namespace qi;
attribute_data = defs >> int_;
start %= attribute_data [ _pass = is_valid_(_1) ];
BOOST_SPIRIT_DEBUG_NODES((attribute_data));
}
private:
phx::function<std::function<bool(AttributeData const&)> > is_valid_ {&AttributeData::is_valid};
qi::rule<Iterator, AttributeData()> attribute_data;
qi::rule<Iterator, AttributeData()> start;
};
using AttributeVariant = boost::variant<AttributeData>;
template <typename Iterator>
class all_attributes_grammar : public qi::grammar<Iterator, std::vector<AttributeVariant>(), qi::blank_type>
{
public:
all_attributes_grammar(const attr_defs &defs)
: all_attributes_grammar::base_type(line_attribute_vec_rule),
sag(defs)
{
line_attribute_rule = (
sag
);
line_attribute_vec_rule = +line_attribute_rule > qi::eoi;
BOOST_SPIRIT_DEBUG_NODES((line_attribute_rule)(line_attribute_vec_rule));
}
private:
single_attribute_grammar<Iterator> sag;
qi::rule<Iterator, AttributeVariant()> line_attribute_rule;
qi::rule<Iterator, std::vector<AttributeVariant>(), qi::blank_type> line_attribute_vec_rule;
};
int main() {
constexpr bool fail = false, succeed = true;
struct _ { bool expect; std::string data; } const tests[] = {
{ succeed, "N100 X-100 AC5" },
{ fail, "" },
{ fail, " " },
{ succeed, "N1" },
{ succeed, " N1" },
{ succeed, "N1 " },
{ succeed, " N1 " },
{ fail, "N 1" },
{ fail, "N0" },
{ succeed, "N9999" },
{ fail, "N10000" },
};
for (auto test : tests) {
std::istringstream input(test.data);
typedef boost::spirit::classic::position_iterator2<boost::spirit::istream_iterator> pos_iterator_type;
pos_iterator_type position_begin(boost::spirit::istream_iterator{input >> std::noskipws}, {}), position_end;
attr_defs sym;
sym.add
("N", {"N", {1, 9999}})
("X", {"X", {-999, 999}})
("AC", {"AC", {-99, 999}})
;
all_attributes_grammar<pos_iterator_type> all_attr_gr(sym);
try {
std::vector<AttributeVariant> value;
std::cout << " --------- '" << test.data << "'\n";
bool actual = qi::phrase_parse(position_begin, position_end, all_attr_gr, qi::blank, value);
std::cout << ((test.expect == actual)?"PASS":"FAIL");
if (actual) {
std::cout << "\t";
for (auto& attr : value)
std::cout << boost::fusion::as_vector(boost::get<AttributeData>(attr)) << " ";
std::cout << "\n";
} else {
std::cout << "\t(no valid parse)\n";
}
}
catch (const qi::expectation_failure<pos_iterator_type>& e) {
auto& pos = e.first.get_position();
std::cout <<
"Parse error at line " << pos.line << " column " << pos.column << ":" << std::endl <<
"'" << e.first.get_currentline() << "'" << std::endl <<
std::setw(pos.column) << " " << "^- here" << std::endl;
}
if (position_begin != position_end)
std::cout << " -> Remaining '" << std::string(position_begin, position_end) << "'\n";
}
}
Prints:
--------- 'N100 X-100 AC5'
PASS (AttrDef(N, 1, 9999) 100) (AttrDef(X, -999, 999) -100) (AttrDef(AC, -99, 999) 5)
--------- ''
PASS (no valid parse)
--------- ' '
PASS (no valid parse)
-> Remaining ' '
--------- 'N1'
PASS (AttrDef(N, 1, 9999) 1)
--------- ' N1'
PASS (AttrDef(N, 1, 9999) 1)
--------- 'N1 '
PASS (AttrDef(N, 1, 9999) 1)
--------- ' N1 '
PASS (AttrDef(N, 1, 9999) 1)
--------- 'N 1'
PASS (no valid parse)
-> Remaining 'N 1'
--------- 'N0'
PASS (no valid parse)
-> Remaining 'N0'
--------- 'N9999'
PASS (AttrDef(N, 1, 9999) 9999)
--------- 'N10000'
PASS (no valid parse)
-> Remaining 'N10000'
After a bit more studying of boost spirit I've found the solution is to use locals and inherited attributes:
template <typename Iterator>
class single_attribute_grammar : public qi::grammar<Iterator, AttributeData(), qi::locals<WordGrammar>, qi::blank_type>
{
public:
single_attribute_grammar(const word_symbols &words) : single_attribute_grammar::base_type(single_attribute_rule)
{
auto attr_word = phx::bind(&AttributeData::word, qi::_val);
auto grammar_word = phx::bind(&WordGrammar::word, qi::_1);
auto attr_value = phx::bind(&AttributeData::value, qi::_val);
word_rule = words;
BOOST_SPIRIT_DEBUG_NODE(word_rule);
range_check_rule = qi::eps(phx::bind(verify_range, qi::_r1, qi::_r2));
BOOST_SPIRIT_DEBUG_NODE(range_check_rule);
single_attribute_rule = qi::lexeme[word_rule[attr_word = grammar_word,qi::_a=qi::_1] >
qi::int_[attr_value = qi::_1] > (qi::space|qi::eoi)] >>
range_check_rule(qi::_a, qi::_val);
BOOST_SPIRIT_DEBUG_NODE(single_attribute_rule);
}
private:
qi::rule<Iterator, WordGrammar()> word_rule;
qi::rule<Iterator, void(const WordGrammar&, const AttributeData&), qi::blank_type> range_check_rule;
qi::rule<Iterator, AttributeData(), qi::locals<WordGrammar>, qi::blank_type> single_attribute_rule;
};
Also because I use BOOST_SPIRIT_DEBUG_NODE overloading of stream operator for WordGrammar is necessary:
std::ostream& operator<<(std::ostream& ostr, const WordGrammar& grammar)
{
ostr << "WordGrammar(" << grammar.word << ", " << grammar.range_from << ", " << grammar.range_to << ")";
return ostr;
}

Why is boost::recursive_wrapper not working in this case

I have the following three rules:
unary_expression =
( '(' > expression > ')' )
| int_;
operator_expression =
unary_expression >> *(operators > expression);
expression =
( '(' > expression > ')' )
| operator_expression;
Obviously this is recursive, so I use boost::recursive_wrapper and created the following AST:
struct expression;
using unary_expression_node = boost::variant<boost::recursive_wrapper<expression>, int>;
struct unary_expression
{
unary_expression_node m_unary_expression;
};
enum operators { op_eq, op_ne };
struct expression;
struct operator_expression
{
unary_expression first;
using second_type = std::vector<std::pair<operators, expression>>;
second_type second;
};
using expression_node =
boost::variant<boost::recursive_wrapper<expression>, operator_expression>;
struct expression
{
expression_node m_expression;
};
This compiles (see full example below), but when the code attempts to construct an expression object the constructor gets into an infinite loop of calling these three constructors:
#11 0x0000000000466066 in ast::expression::expression ...
#12 0x00000000004682e0 in boost::recursive_wrapper<ast::expression>::recursive_wrapper ...
#13 0x000000000046718d in boost::variant<boost::recursive_wrapper<ast::expression>, ast::operator_expression>::variant
...
Thus, Creating an expression creates a boost::variant<boost::recursive_wrapper<ast::expression>, ast::operator_expression> (aka, an expression_node) which creates a boost::recursive_wrapper<ast::expression> which creates an expression which creates... and so on.
How can I solve this?
Here is a full example that compiles, but segfaults when the stack runs full:
#include <boost/config/warning_disable.hpp>
#include <boost/spirit/include/qi.hpp>
#include <boost/fusion/include/adapt_struct.hpp>
#include <boost/fusion/include/std_pair.hpp>
#include <iostream>
#include <string>
#include <vector>
namespace ast {
struct expression;
using unary_expression_node = boost::variant<boost::recursive_wrapper<expression>, int>;
struct unary_expression
{
unary_expression_node m_unary_expression;
};
enum operators { op_eq, op_ne };
struct expression;
struct operator_expression
{
unary_expression first;
using second_type = std::vector<std::pair<operators, expression>>;
second_type second;
};
using expression_node = boost::variant<boost::recursive_wrapper<expression>, operator_expression>;
struct expression
{
expression_node m_expression;
};
std::ostream& operator<<(std::ostream& os, expression const& expression)
{
return os << expression.m_expression;
}
std::ostream& operator<<(std::ostream& os, unary_expression const& unary_expression)
{
return os << unary_expression.m_unary_expression;
}
std::ostream& operator<<(std::ostream& os, operator_expression const& operator_expression)
{
os << operator_expression.first;
for (auto& l : operator_expression.second)
{
os << ' ' << l.first << ' ' << l.second;
}
return os;
}
} // namespace ast
BOOST_FUSION_ADAPT_STRUCT(
ast::expression,
(ast::expression_node, m_expression)
)
BOOST_FUSION_ADAPT_STRUCT(
ast::unary_expression,
(ast::unary_expression_node, m_unary_expression)
)
BOOST_FUSION_ADAPT_STRUCT(
ast::operator_expression,
(ast::unary_expression, first),
(ast::operator_expression::second_type, second)
)
namespace client
{
namespace qi = boost::spirit::qi;
namespace ascii = boost::spirit::ascii;
template <typename Iterator>
class expression_grammar : public qi::grammar<Iterator, ast::expression(), qi::space_type>
{
private:
qi::symbols<char, ast::operators> operators;
qi::rule<Iterator, ast::unary_expression(), qi::space_type> unary_expression;
qi::rule<Iterator, ast::operator_expression(), qi::space_type> operator_expression;
qi::rule<Iterator, ast::expression(), qi::space_type> expression;
public:
expression_grammar() : expression_grammar::base_type(expression, "expression_grammar")
{
using qi::double_;
using qi::char_;
using qi::int_;
operators.add
("==", ast::op_eq)
("!=", ast::op_ne)
;
unary_expression =
( '(' > expression > ')' )
| int_;
operator_expression =
unary_expression >> *(operators > expression);
expression =
( '(' > expression > ')' )
| operator_expression;
}
};
} // namespace client
int main()
{
std::string const input{"1 == 1 != 0"};
using iterator_type = std::string::const_iterator;
using expression_grammar = client::expression_grammar<iterator_type>;
namespace qi = boost::spirit::qi;
expression_grammar program;
iterator_type iter{input.begin()};
iterator_type const end{input.end()};
ast::expression out;
bool r = qi::phrase_parse(iter, end, program, qi::space, out);
if (!r || iter != end)
{
std::cerr << "Parsing failed." << std::endl;
return 1;
}
std::cout << "Parsed: " << out << std::endl;
}
EDIT:
I tried simplifying things to just two rules (and two 'ast's):
struct expression;
using unary_expression = boost::variant<boost::recursive_wrapper<expression>, int>;
enum operators { op_eq, op_ne };
struct expression
{
unary_expression first;
using second_type = std::vector<std::pair<operators, expression>>;
second_type second;
};
BOOST_FUSION_ADAPT_STRUCT(
ast::expression,
(ast::unary_expression, first),
(ast::expression::second_type, second)
)
[...]
unary_expression =
( '(' > expression > ')' )
| int_;
expression =
unary_expression >> *(operators > expression);
but also this result in an infinite loop.
#18 0x00000000004646f2 in ast::expression::expression
#19 0x00000000004669ac in boost::recursive_wrapper<ast::expression>::recursive_wrapper
#20 0x0000000000465821 in boost::variant<boost::recursive_wrapper<ast::expression>, int>::variant
...
Variants default-construct to their first element type.
This indeed directly leads to an infinite loop. (Demo)
The way to solve it is to make the default variant element not re-entrant or to make it lazily constructed. In this case, you can simply re-arrange to make int the first element.
Better yet, there doesn't seem to be a need to reflect the operator precedence hieararchy (as it is expressed in the rules) in the resultant tree, so why not simplify to:
struct unary_expression;
struct binary_expression;
enum operators { op_eq, op_ne };
using expression = boost::variant<
int,
boost::recursive_wrapper<unary_expression>,
boost::recursive_wrapper<binary_expression>
>;
struct unary_expression {
expression expr;
};
struct binary_expression {
expression first;
std::vector<std::pair<operators, expression>> other;
};
This no longer crashes and seems a bit simpler in adaptation and usage.
Simplified Full Demo
This full demo uses that AST, but adds a true unary expression. A few style things have been fixed:
don't expose the skipper unless you intend for the caller to change it
make the parser const
show unparsed trailing data (or instead assert >> qi::eoi)
Note: I might have changed the precedence rules (specifically, associativity of binary operators). I'm not sure which version you require.
Live On Coliru
//#define BOOST_SPIRIT_DEBUG
#include <boost/spirit/include/qi.hpp>
#include <boost/fusion/include/adapt_struct.hpp>
#include <boost/fusion/include/std_pair.hpp>
#include <iostream>
#include <string>
#include <vector>
namespace ast {
struct unary_expression;
struct binary_expression;
enum operators { op_eq, op_ne };
using expression = boost::variant<
int,
boost::recursive_wrapper<unary_expression>,
boost::recursive_wrapper<binary_expression>
>;
struct unary_expression {
bool negated = false;
expression expr;
};
struct binary_expression {
expression first;
std::vector<std::pair<operators, expression>> other;
};
}
BOOST_FUSION_ADAPT_STRUCT(ast::unary_expression, negated, expr)
BOOST_FUSION_ADAPT_STRUCT(ast::binary_expression, first, other)
namespace ast {
static inline std::ostream& operator<<(std::ostream& os, operators op) { return os << (op==op_eq?"==":"!="); }
static inline std::ostream& operator<<(std::ostream& os, binary_expression const& e) {
os << e.first;
for (auto& oe : e.other)
os << " " << oe.first << " " << oe.second;
return os;
}
static inline std::ostream& operator<<(std::ostream& os, unary_expression const& e) {
return os << (e.negated?"!":"") << "(" << e.expr << ")";
}
}
namespace client
{
namespace qi = boost::spirit::qi;
template <typename Iterator>
class expression_grammar : public qi::grammar<Iterator, ast::expression()> {
private:
qi::symbols<char, ast::operators> operators;
qi::rule<Iterator, ast::expression()> start;
qi::rule<Iterator, ast::expression(), qi::space_type> simple_expression;
qi::rule<Iterator, ast::unary_expression(), qi::space_type> unary_expression;
qi::rule<Iterator, ast::binary_expression(), qi::space_type> binary_expression;
qi::rule<Iterator, ast::expression(), qi::space_type> expression;
public:
expression_grammar() : expression_grammar::base_type(start, "expression") {
using namespace qi;
operators.add
("==", ast::op_eq)
("!=", ast::op_ne)
;
simple_expression =
( '(' > expression > ')' )
| int_;
unary_expression =
matches['!'] >> simple_expression;
binary_expression =
unary_expression >> *(operators > expression);
expression = binary_expression;
start = skip(space) [ expression ];
BOOST_SPIRIT_DEBUG_NODES((expression)(binary_expression)(unary_expression)(simple_expression))
}
};
} // namespace client
int main() {
using It = std::string::const_iterator;
client::expression_grammar<It> const program;
std::string const input{"1 == !(1 != 0)"};
It iter = input.begin(), end = input.end();
ast::expression out;
if (parse(iter, end, program, out)) {
std::cout << "Parsed: " << out << std::endl;
} else {
std::cerr << "Parsing failed." << std::endl;
return 1;
}
if (iter != end) {
std::cout << "Remaining unparsed input: '" << std::string(iter, end) << "'\n";
}
}
Prints
Parsed: (1) == !((1) != (0))

Using lexer token attributes in grammar rules with Lex and Qi from Boost.Spirit

Let's consider following code:
#include <boost/phoenix.hpp>
#include <boost/spirit/include/lex_lexertl.hpp>
#include <boost/spirit/include/qi.hpp>
#include <algorithm>
#include <iostream>
#include <string>
#include <utility>
#include <vector>
namespace lex = boost::spirit::lex;
namespace qi = boost::spirit::qi;
namespace phoenix = boost::phoenix;
struct operation
{
enum type
{
add,
sub,
mul,
div
};
};
template<typename Lexer>
class expression_lexer
: public lex::lexer<Lexer>
{
public:
typedef lex::token_def<operation::type> operator_token_type;
typedef lex::token_def<double> value_token_type;
typedef lex::token_def<std::string> variable_token_type;
typedef lex::token_def<lex::omit> parenthesis_token_type;
typedef std::pair<parenthesis_token_type, parenthesis_token_type> parenthesis_token_pair_type;
typedef lex::token_def<lex::omit> whitespace_token_type;
expression_lexer()
: operator_add('+'),
operator_sub('-'),
operator_mul("[x*]"),
operator_div("[:/]"),
value("\\d+(\\.\\d+)?"),
variable("%(\\w+)"),
parenthesis({
std::make_pair(parenthesis_token_type('('), parenthesis_token_type(')')),
std::make_pair(parenthesis_token_type('['), parenthesis_token_type(']'))
}),
whitespace("[ \\t]+")
{
this->self
+= operator_add [lex::_val = operation::add]
| operator_sub [lex::_val = operation::sub]
| operator_mul [lex::_val = operation::mul]
| operator_div [lex::_val = operation::div]
| value
| variable [lex::_val = phoenix::construct<std::string>(lex::_start + 1, lex::_end)]
| whitespace [lex::_pass = lex::pass_flags::pass_ignore]
;
std::for_each(parenthesis.cbegin(), parenthesis.cend(),
[&](parenthesis_token_pair_type const& token_pair)
{
this->self += token_pair.first | token_pair.second;
}
);
}
operator_token_type operator_add;
operator_token_type operator_sub;
operator_token_type operator_mul;
operator_token_type operator_div;
value_token_type value;
variable_token_type variable;
std::vector<parenthesis_token_pair_type> parenthesis;
whitespace_token_type whitespace;
};
template<typename Iterator>
class expression_grammar
: public qi::grammar<Iterator>
{
public:
template<typename Tokens>
explicit expression_grammar(Tokens const& tokens)
: expression_grammar::base_type(start)
{
start %= expression >> qi::eoi;
expression %= sum_operand >> -(sum_operator >> expression);
sum_operator %= tokens.operator_add | tokens.operator_sub;
sum_operand %= fac_operand >> -(fac_operator >> sum_operand);
fac_operator %= tokens.operator_mul | tokens.operator_div;
if(!tokens.parenthesis.empty())
fac_operand %= parenthesised | terminal;
else
fac_operand %= terminal;
terminal %= tokens.value | tokens.variable;
if(!tokens.parenthesis.empty())
{
parenthesised %= tokens.parenthesis.front().first >> expression >> tokens.parenthesis.front().second;
std::for_each(tokens.parenthesis.cbegin() + 1, tokens.parenthesis.cend(),
[&](typename Tokens::parenthesis_token_pair_type const& token_pair)
{
parenthesised %= parenthesised.copy() | (token_pair.first >> expression >> token_pair.second);
}
);
}
}
private:
qi::rule<Iterator> start;
qi::rule<Iterator> expression;
qi::rule<Iterator> sum_operand;
qi::rule<Iterator> sum_operator;
qi::rule<Iterator> fac_operand;
qi::rule<Iterator> fac_operator;
qi::rule<Iterator> terminal;
qi::rule<Iterator> parenthesised;
};
int main()
{
typedef lex::lexertl::token<std::string::const_iterator, boost::mpl::vector<operation::type, double, std::string>> token_type;
typedef expression_lexer<lex::lexertl::actor_lexer<token_type>> expression_lexer_type;
typedef expression_lexer_type::iterator_type expression_lexer_iterator_type;
typedef expression_grammar<expression_lexer_iterator_type> expression_grammar_type;
expression_lexer_type lexer;
expression_grammar_type grammar(lexer);
while(std::cin)
{
std::string line;
std::getline(std::cin, line);
std::string::const_iterator first = line.begin();
std::string::const_iterator const last = line.end();
bool const result = lex::tokenize_and_parse(first, last, lexer, grammar);
if(!result)
std::cout << "Parsing failed! Reminder: >" << std::string(first, last) << "<" << std::endl;
else
{
if(first != last)
std::cout << "Parsing succeeded! Reminder: >" << std::string(first, last) << "<" << std::endl;
else
std::cout << "Parsing succeeded!" << std::endl;
}
}
}
It is a simple parser for arithmetic expressions with values and variables. It is build using expression_lexer for extracting tokens, and then with expression_grammar to parse the tokens.
Use of lexer for such a small case might seem an overkill and probably is one. But that is the cost of simplified example. Also note that use of lexer allows to easily define tokens with regular expression while that allows to easily define them by external code (and user provided configuration in particular). With the example provided it would be no issue at all to read definition of tokens from an external config file and for example allow user to change variables from %name to $name.
The code seems to be working fine (checked on Visual Studio 2013 with Boost 1.61).
The expression_lexer has attributes attached to tokens. I guess they work since they compile. But I don't really know how to check.
Ultimately I would like the grammar to build me an std::vector with reversed polish notation of the expression. (Where every element would be a boost::variant over either operator::type or double or std::string.)
The problem is however that I failed to use token attributes in my expression_grammar. For example if you try to change sum_operator following way:
qi::rule<Iterator, operation::type ()> sum_operator;
you will get compilation error. I expected this to work since operation::type is the attribute for both operator_add and operator_sub and so also for their alternative. And still it doesn't compile. Judging from the error in assign_to_attribute_from_iterators it seems that parser tries to build the attribute value directly from input stream range. Which means it ignores the [lex::_val = operation::add] I specified in my lexer.
Changing that to
qi::rule<Iterator, operation::type (operation::type)> sum_operator;
didn't help either.
Also I tried changing definition to
sum_operator %= (tokens.operator_add | tokens.operator_sub) [qi::_val = qi::_1];
didn't help either.
How to work around that? I know I could use symbols from Qi. But I want to have the lexer to make it easy to configure regexes for the tokens. I could also extend the assign_to_attribute_from_iterators as described in the documentation but this kind of double the work. I guess I could also skip the attributes on lexer and just have them on grammar. But this again doesn't work well with flexibility on variable token (in my actual case there is slightly more logic there so that it is configurable also which part of the token forms actual name of the variable - while here it is fixed to just skip the first character). Anything else?
Also a side question - maybe anyone knows. Is there a way to get to capture groups of the regular expression of the token from tokens action? So that instead of having
variable [lex::_val = phoenix::construct<std::string>(lex::_start + 1, lex::_end)]
instead I would be able to make a string from the capture group and so easily handle formats like $var$.
Edited! I have improved whitespace skipping along conclusions from Whitespace skipper when using Boost.Spirit Qi and Lex. It is a simplification that does not affect questions asked here.
Ok, here's my take on the RPN 'requirement'. I heavily favor natural (automatic) attribute propagation over semantic actions (see Boost Spirit: "Semantic actions are evil"?)
I consider the other options (uglifying) optimizations. You might do them if you're happy with the overall design and don't mind making it harder to maintain :)
Live On Coliru
Beyond the sample from my comment that you've already studied, I added that RPN transformation step:
namespace RPN {
using cell = boost::variant<AST::operation, AST::value, AST::variable>;
using rpn_stack = std::vector<cell>;
struct transform : boost::static_visitor<> {
void operator()(rpn_stack& stack, AST::expression const& e) const {
boost::apply_visitor(boost::bind(*this, boost::ref(stack), ::_1), e);
}
void operator()(rpn_stack& stack, AST::bin_expr const& e) const {
(*this)(stack, e.lhs);
(*this)(stack, e.rhs);
stack.push_back(e.op);
}
void operator()(rpn_stack& stack, AST::value const& v) const { stack.push_back(v); }
void operator()(rpn_stack& stack, AST::variable const& v) const { stack.push_back(v); }
};
}
That's all! Use it like so, e.g.:
RPN::transform compiler;
RPN::rpn_stack program;
compiler(program, expr);
for (auto& instr : program) {
std::cout << instr << " ";
}
Which makes the output:
Parsing success: (3 + (8 * 9))
3 8 9 * +
Full Listing
Live On Coliru
//#define BOOST_SPIRIT_DEBUG
#include <boost/phoenix.hpp>
#include <boost/bind.hpp>
#include <boost/fusion/adapted/struct.hpp>
#include <boost/spirit/include/lex_lexertl.hpp>
#include <boost/spirit/include/qi.hpp>
#include <algorithm>
#include <iostream>
#include <string>
#include <utility>
#include <vector>
namespace lex = boost::spirit::lex;
namespace qi = boost::spirit::qi;
namespace phoenix = boost::phoenix;
struct operation
{
enum type
{
add,
sub,
mul,
div
};
friend std::ostream& operator<<(std::ostream& os, type op) {
switch (op) {
case type::add: return os << "+";
case type::sub: return os << "-";
case type::mul: return os << "*";
case type::div: return os << "/";
}
return os << "<" << static_cast<int>(op) << ">";
}
};
template<typename Lexer>
class expression_lexer
: public lex::lexer<Lexer>
{
public:
//typedef lex::token_def<operation::type> operator_token_type;
typedef lex::token_def<lex::omit> operator_token_type;
typedef lex::token_def<double> value_token_type;
typedef lex::token_def<std::string> variable_token_type;
typedef lex::token_def<lex::omit> parenthesis_token_type;
typedef std::pair<parenthesis_token_type, parenthesis_token_type> parenthesis_token_pair_type;
typedef lex::token_def<lex::omit> whitespace_token_type;
expression_lexer()
: operator_add('+'),
operator_sub('-'),
operator_mul("[x*]"),
operator_div("[:/]"),
value("\\d+(\\.\\d+)?"),
variable("%(\\w+)"),
parenthesis({
std::make_pair(parenthesis_token_type('('), parenthesis_token_type(')')),
std::make_pair(parenthesis_token_type('['), parenthesis_token_type(']'))
}),
whitespace("[ \\t]+")
{
this->self
+= operator_add [lex::_val = operation::add]
| operator_sub [lex::_val = operation::sub]
| operator_mul [lex::_val = operation::mul]
| operator_div [lex::_val = operation::div]
| value
| variable [lex::_val = phoenix::construct<std::string>(lex::_start + 1, lex::_end)]
| whitespace [lex::_pass = lex::pass_flags::pass_ignore]
;
std::for_each(parenthesis.cbegin(), parenthesis.cend(),
[&](parenthesis_token_pair_type const& token_pair)
{
this->self += token_pair.first | token_pair.second;
}
);
}
operator_token_type operator_add;
operator_token_type operator_sub;
operator_token_type operator_mul;
operator_token_type operator_div;
value_token_type value;
variable_token_type variable;
std::vector<parenthesis_token_pair_type> parenthesis;
whitespace_token_type whitespace;
};
namespace AST {
using operation = operation::type;
using value = double;
using variable = std::string;
struct bin_expr;
using expression = boost::variant<value, variable, boost::recursive_wrapper<bin_expr> >;
struct bin_expr {
expression lhs, rhs;
operation op;
friend std::ostream& operator<<(std::ostream& os, bin_expr const& be) {
return os << "(" << be.lhs << " " << be.op << " " << be.rhs << ")";
}
};
}
BOOST_FUSION_ADAPT_STRUCT(AST::bin_expr, lhs, op, rhs)
template<typename Iterator>
class expression_grammar : public qi::grammar<Iterator, AST::expression()>
{
public:
template<typename Tokens>
explicit expression_grammar(Tokens const& tokens)
: expression_grammar::base_type(start)
{
start = expression >> qi::eoi;
bin_sum_expr = sum_operand >> sum_operator >> expression;
bin_fac_expr = fac_operand >> fac_operator >> sum_operand;
expression = bin_sum_expr | sum_operand;
sum_operand = bin_fac_expr | fac_operand;
sum_operator = tokens.operator_add >> qi::attr(AST::operation::add) | tokens.operator_sub >> qi::attr(AST::operation::sub);
fac_operator = tokens.operator_mul >> qi::attr(AST::operation::mul) | tokens.operator_div >> qi::attr(AST::operation::div);
if(tokens.parenthesis.empty()) {
fac_operand = terminal;
}
else {
fac_operand = parenthesised | terminal;
parenthesised = tokens.parenthesis.front().first >> expression >> tokens.parenthesis.front().second;
std::for_each(tokens.parenthesis.cbegin() + 1, tokens.parenthesis.cend(),
[&](typename Tokens::parenthesis_token_pair_type const& token_pair)
{
parenthesised = parenthesised.copy() | (token_pair.first >> expression >> token_pair.second);
});
}
terminal = tokens.value | tokens.variable;
BOOST_SPIRIT_DEBUG_NODES(
(start) (expression) (bin_sum_expr) (bin_fac_expr)
(fac_operand) (terminal) (parenthesised) (sum_operand)
(sum_operator) (fac_operator)
);
}
private:
qi::rule<Iterator, AST::expression()> start;
qi::rule<Iterator, AST::expression()> expression;
qi::rule<Iterator, AST::expression()> sum_operand;
qi::rule<Iterator, AST::expression()> fac_operand;
qi::rule<Iterator, AST::expression()> terminal;
qi::rule<Iterator, AST::expression()> parenthesised;
qi::rule<Iterator, int()> sum_operator;
qi::rule<Iterator, int()> fac_operator;
// extra rules to help with AST creation
qi::rule<Iterator, AST::bin_expr()> bin_sum_expr;
qi::rule<Iterator, AST::bin_expr()> bin_fac_expr;
};
namespace RPN {
using cell = boost::variant<AST::operation, AST::value, AST::variable>;
using rpn_stack = std::vector<cell>;
struct transform : boost::static_visitor<> {
void operator()(rpn_stack& stack, AST::expression const& e) const {
boost::apply_visitor(boost::bind(*this, boost::ref(stack), ::_1), e);
}
void operator()(rpn_stack& stack, AST::bin_expr const& e) const {
(*this)(stack, e.lhs);
(*this)(stack, e.rhs);
stack.push_back(e.op);
}
void operator()(rpn_stack& stack, AST::value const& v) const { stack.push_back(v); }
void operator()(rpn_stack& stack, AST::variable const& v) const { stack.push_back(v); }
};
}
int main()
{
typedef lex::lexertl::token<std::string::const_iterator, boost::mpl::vector<operation::type, double, std::string>> token_type;
typedef expression_lexer<lex::lexertl::actor_lexer<token_type>> expression_lexer_type;
typedef expression_lexer_type::iterator_type expression_lexer_iterator_type;
typedef expression_grammar<expression_lexer_iterator_type> expression_grammar_type;
expression_lexer_type lexer;
expression_grammar_type grammar(lexer);
RPN::transform compiler;
std::string line;
while(std::getline(std::cin, line) && !line.empty())
{
std::string::const_iterator first = line.begin();
std::string::const_iterator const last = line.end();
AST::expression expr;
bool const result = lex::tokenize_and_parse(first, last, lexer, grammar, expr);
if(!result)
std::cout << "Parsing failed!\n";
else
{
std::cout << "Parsing success: " << expr << "\n";
RPN::rpn_stack program;
compiler(program, expr);
for (auto& instr : program) {
std::cout << instr << " ";
}
}
if(first != last)
std::cout << "Remainder: >" << std::string(first, last) << "<\n";
}
}

boost spirit on_error not triggered

^ No it is not. This was part of the problem, but if review the code as is right now, it already does what the pointed out question/answer shows ... and the errors are still not triggered.
I have this boost spirit parser for string literal. It works. Now I would like to start handle errors when it fail. I copied the on_error handle 1-1 from the mini xml example and it compiles, but it is never triggered (no errors are outputted).
This is the parser:
#define BOOST_SPIRIT_USE_PHOENIX_V3
#define BOOST_SPIRIT_DEBUG
#include <boost/spirit/include/qi.hpp>
#include <boost/spirit/include/phoenix.hpp>
#include <boost/spirit/home/support/iterators/line_pos_iterator.hpp>
#include <boost/spirit/include/phoenix_fusion.hpp>
#include <boost/spirit/include/phoenix_stl.hpp>
#include <boost/fusion/include/adapt_struct.hpp>
namespace qi = boost::spirit::qi;
struct my_handler_f
{
template <typename...> struct result { typedef void type; };
template <typename... T>
void operator()(T&&...) const {
std::cout << "\nmy_handler_f() invoked with " << sizeof...(T) << " arguments\n";
}
};
struct append_utf8_f
{
template <typename, typename>
struct result { typedef void type; };
template <typename INT>
void operator()(INT in, std::string& to) const
{
auto out = std::back_inserter(to);
boost::utf8_output_iterator<decltype(out)> convert(out);
*convert++ = in;
}
};
struct get_line_f
{
template <typename> struct result { typedef size_t type; };
template <typename It> size_t operator()(It const& pos_iter) const
{
return get_line(pos_iter);
}
};
struct RangePosition { size_t beginLine, endLine; };
struct String : public RangePosition
{
String()
: RangePosition()
, value()
, source()
{
}
std::string value;
std::string source;
};
BOOST_FUSION_ADAPT_STRUCT(String,
(std::string, value)
(std::string, source)
(size_t, beginLine)
(size_t, endLine)
)
template <typename Iterator>
struct source_string : qi::grammar<Iterator, String(), qi::space_type>
{
struct escape_symbols : qi::symbols<char, char>
{
escape_symbols()
{
add
("\'" , '\'')
("\"" , '\"')
("\?" , '\?')
("\\" , '\\')
("0" , '\0')
("a" , '\a')
("b" , '\b')
("f" , '\f')
("n" , '\n')
("r" , '\r')
("t" , '\t')
("v" , '\v')
;
}
} escape_symbol;
source_string() : source_string::base_type(start)
{
using qi::raw;
using qi::_val;
using qi::_1;
using qi::_2;
using qi::_3;
using qi::_4;
using qi::space;
using qi::omit;
using qi::no_case;
using qi::print;
using qi::eps;
using qi::on_error;
using qi::fail;
using qi::lit;
namespace phx = boost::phoenix;
using phx::at_c;
using phx::begin;
using phx::end;
using phx::construct;
using phx::ref;
using phx::val;
escape %= escape_symbol;
character %= (no_case["\\x"] > hex12)
| ("\\" > (oct123 | escape))
| (print - (lit('"') | '\\'));
unicode = ("\\u" > hex4[append_utf8(_1, _val)])
| ("\\U" > hex8[append_utf8(_1, _val)]);
string_section %= '"' > *(unicode | character) > '"';
string %= string_section % omit[*space];
main = raw [
string[at_c<0>(_val) = _1]
]
[
at_c<1>(_val) = construct<std::string>(begin(_1), end(_1)),
at_c<2>(_val) = get_line_(begin(_1)),
at_c<3>(_val) = get_line_(end(_1))
];
start %= eps > main;
on_error<fail>(start, my_handler);
}
boost::phoenix::function<my_handler_f> my_handler;
qi::rule<Iterator, std::string()> escape;
qi::uint_parser<char, 16, 1, 2> hex12;
qi::uint_parser<char, 8, 1, 3> oct123;
qi::rule<Iterator, std::string()> character;
qi::uint_parser<uint16_t, 16, 4, 4> hex4;
qi::uint_parser<uint32_t, 16, 8, 8> hex8;
boost::phoenix::function<append_utf8_f> append_utf8;
qi::rule<Iterator, std::string()> unicode;
qi::rule<Iterator, std::string()> string_section;
qi::rule<Iterator, std::string()> string;
boost::phoenix::function<get_line_f> get_line_;
qi::rule<Iterator, String(), qi::space_type> main;
qi::rule<Iterator, String(), qi::space_type> start;
};
and this is the test code
int main()
{
std::string str[] =
{
"\"\\u1234\\U0002345\"",
//"\"te\"\"st\"",
//"\"te\" \"st\"",
//"\"te\" \n \"st\"",
//"\"\"",
//"\"\\\"\"",
//"\"test\"",
//"\"test\" something",
//"\"\\\'\\\"\\\?\\\\\\a\\b\\f\\n\\r\\t\\v\"",
//"\"\\x61cd\\X3012\\x7z\"",
//"\"\\141cd\\06012\\78\\778\"",
"\"te",
//"\"te\nst\"",
//"\"test\\\"",
//"\"te\\st\"",
//
};
typedef boost::spirit::line_pos_iterator<std::string::const_iterator> Iterator;
for (size_t i = 0; i < sizeof(str) / sizeof(str[0]); ++i)
{
source_string<Iterator> g;
Iterator iter(str[i].begin());
Iterator end(str[i].end());
String string;
bool r = phrase_parse(iter, end, g, qi::space, string);
if (r)
std::cout << string.beginLine << "-" << string.endLine << ": " << string.value << " === " << string.source << "\n";
else
std::cout << "Parsing failed\n";
}
}