Left factorisation in Parsing Expression Grammar - c++

Question

I’m trying to write a grammar for a language which allows the following expressions:
Function calls of the form f args (note: no parentheses!)
Addition (and more complex expressions but that’s not the point here) of the form a + b
For example:
f 42 => f(42)
42 + b => (42 + b)
f 42 + b => f(42 + b)
The grammar is unambiguous (every expression can be parsed in exactly one way) but I don’t know how to write this grammar as a PEG since both productions potentially start with the same token, id. This is my wrong PEG. How can I rewrite it to make it valid?
expression ::= call / addition
call ::= id addition*
addition ::= unary
( ('+' unary)
/ ('-' unary) )*
unary ::= primary
/ '(' ( ('+' unary)
/ ('-' unary)
/ expression)
')'
primary ::= number / id
number ::= [1-9]+
id ::= [a-z]+
Now, when this grammar tries to parse the input “a + b” it parses “a” as a function call with zero arguments and chokes on “+ b”.
I’ve uploaded a C++ / Boost.Spirit.Qi implementation of the grammar in case anybody wants to play with it.
(Note that unary disambiguates unary operations and additions: In order to call a function with a negative number as an argument, you need to specify parentheses, e.g. f (-1).)

Answer 1

As proposed in chat you could start out with something like:
expression = addition | simple;
addition = simple >>
( ('+' > expression)
| ('-' > expression)
);
simple = '(' > expression > ')' | call | unary | number;
call = id >> *expression;
unary = qi::char_("-+") > expression;
// terminals
id = qi::lexeme[+qi::char_("a-z")];
number = qi::double_;
Since then I implemented this in C++ with an AST presentation, so you can get a feel for how this grammar actually build the expression tree by pretty printing it.
All source code is on github: https://gist.github.com/2152518
There are two versions (scroll down to 'Alternative' to read more
Grammar:
template <typename Iterator>
struct mini_grammar : qi::grammar<Iterator, expression_t(), qi::space_type>
{
qi::rule<Iterator, std::string(), qi::space_type> id;
qi::rule<Iterator, expression_t(), qi::space_type> addition, expression, simple;
qi::rule<Iterator, number_t(), qi::space_type> number;
qi::rule<Iterator, call_t(), qi::space_type> call;
qi::rule<Iterator, unary_t(), qi::space_type> unary;
mini_grammar() : mini_grammar::base_type(expression)
{
expression = addition | simple;
addition = simple [ qi::_val = qi::_1 ] >>
+(
(qi::char_("+-") > simple) [ phx::bind(&append_term, qi::_val, qi::_1, qi::_2) ]
);
simple = '(' > expression > ')' | call | unary | number;
call = id >> *expression;
unary = qi::char_("-+") > expression;
// terminals
id = qi::lexeme[+qi::char_("a-z")];
number = qi::double_;
}
};
The corresponding AST structures are defined quick-and-dirty using the very powerful Boost Variant:
struct addition_t;
struct call_t;
struct unary_t;
typedef double number_t;
typedef boost::variant<
number_t,
boost::recursive_wrapper<call_t>,
boost::recursive_wrapper<unary_t>,
boost::recursive_wrapper<addition_t>
> expression_t;
struct addition_t
{
expression_t lhs;
char binop;
expression_t rhs;
};
struct call_t
{
std::string id;
std::vector<expression_t> args;
};
struct unary_t
{
char unop;
expression_t operand;
};
BOOST_FUSION_ADAPT_STRUCT(addition_t, (expression_t, lhs)(char,binop)(expression_t, rhs));
BOOST_FUSION_ADAPT_STRUCT(call_t, (std::string, id)(std::vector<expression_t>, args));
BOOST_FUSION_ADAPT_STRUCT(unary_t, (char, unop)(expression_t, operand));
In the full code, I've also overloaded operator<< for these structures.
Full Demo
//#define BOOST_SPIRIT_DEBUG
#include <iostream>
#include <iterator>
#include <string>
#include <boost/spirit/include/qi.hpp>
#include <boost/spirit/include/phoenix.hpp>
#include <boost/fusion/adapted.hpp>
#include <boost/optional.hpp>
namespace qi = boost::spirit::qi;
namespace phx= boost::phoenix;
struct addition_t;
struct call_t;
struct unary_t;
typedef double number_t;
typedef boost::variant<
number_t,
boost::recursive_wrapper<call_t>,
boost::recursive_wrapper<unary_t>,
boost::recursive_wrapper<addition_t>
> expression_t;
struct addition_t
{
expression_t lhs;
char binop;
expression_t rhs;
friend std::ostream& operator<<(std::ostream& os, const addition_t& a)
{ return os << "(" << a.lhs << ' ' << a.binop << ' ' << a.rhs << ")"; }
};
struct call_t
{
std::string id;
std::vector<expression_t> args;
friend std::ostream& operator<<(std::ostream& os, const call_t& a)
{ os << a.id << "("; for (auto& e : a.args) os << e << ", "; return os << ")"; }
};
struct unary_t
{
char unop;
expression_t operand;
friend std::ostream& operator<<(std::ostream& os, const unary_t& a)
{ return os << "(" << a.unop << ' ' << a.operand << ")"; }
};
BOOST_FUSION_ADAPT_STRUCT(addition_t, (expression_t, lhs)(char,binop)(expression_t, rhs));
BOOST_FUSION_ADAPT_STRUCT(call_t, (std::string, id)(std::vector<expression_t>, args));
BOOST_FUSION_ADAPT_STRUCT(unary_t, (char, unop)(expression_t, operand));
void append_term(expression_t& lhs, char op, expression_t operand)
{
lhs = addition_t { lhs, op, operand };
}
template <typename Iterator>
struct mini_grammar : qi::grammar<Iterator, expression_t(), qi::space_type>
{
qi::rule<Iterator, std::string(), qi::space_type> id;
qi::rule<Iterator, expression_t(), qi::space_type> addition, expression, simple;
qi::rule<Iterator, number_t(), qi::space_type> number;
qi::rule<Iterator, call_t(), qi::space_type> call;
qi::rule<Iterator, unary_t(), qi::space_type> unary;
mini_grammar() : mini_grammar::base_type(expression)
{
expression = addition | simple;
addition = simple [ qi::_val = qi::_1 ] >>
+(
(qi::char_("+-") > simple) [ phx::bind(&append_term, qi::_val, qi::_1, qi::_2) ]
);
simple = '(' > expression > ')' | call | unary | number;
call = id >> *expression;
unary = qi::char_("-+") > expression;
// terminals
id = qi::lexeme[+qi::char_("a-z")];
number = qi::double_;
BOOST_SPIRIT_DEBUG_NODE(expression);
BOOST_SPIRIT_DEBUG_NODE(call);
BOOST_SPIRIT_DEBUG_NODE(addition);
BOOST_SPIRIT_DEBUG_NODE(simple);
BOOST_SPIRIT_DEBUG_NODE(unary);
BOOST_SPIRIT_DEBUG_NODE(id);
BOOST_SPIRIT_DEBUG_NODE(number);
}
};
std::string read_input(std::istream& stream) {
return std::string(
std::istreambuf_iterator<char>(stream),
std::istreambuf_iterator<char>());
}
int main() {
std::cin.unsetf(std::ios::skipws);
std::string const code = read_input(std::cin);
auto begin = code.begin();
auto end = code.end();
try {
mini_grammar<decltype(end)> grammar;
qi::space_type space;
std::vector<expression_t> script;
bool ok = qi::phrase_parse(begin, end, *(grammar > ';'), space, script);
if (begin!=end)
std::cerr << "Unparsed: '" << std::string(begin,end) << "'\n";
std::cout << std::boolalpha << "Success: " << ok << "\n";
if (ok)
{
for (auto& expr : script)
std::cout << "AST: " << expr << '\n';
}
}
catch (qi::expectation_failure<decltype(end)> const& ex) {
std::cout << "Failure; parsing stopped after \""
<< std::string(ex.first, ex.last) << "\"\n";
}
}
Alternative:
I have an alternative version that build addition_t iteratively instead of recursively, so to say:
struct term_t
{
char binop;
expression_t rhs;
};
struct addition_t
{
expression_t lhs;
std::vector<term_t> terms;
};
This removes the need to use Phoenix to build the expression:
addition = simple >> +term;
term = qi::char_("+-") > simple;