How to propagate binary operators in boost spirit x3? - c++

Question

I had recently with the help of the amazing sehe managed to advance my boost spirit x3 parser for hlsl (high level shading language) that is a c-like language for writing shader kernels for GPU's. Here is the rough grammar I am following...
https://craftinginterpreters.com/appendix-i.html
Here is the previous question and answer for the curious.
Trying to parse nested expressions with boost spirit x3
I am now trying to implement unary and binary operators and have hit a stumbling block with how they recurse. I am able to get it to compile and a single binary operator is parsed, but having multiple nested ones doesn't seem to be working. I suspect the solution is going to be involving semantic actions again to manually propagate values but I struggle to see how to do that yet as the side effects are hard to understand (still working out how it all works).
Here's my compiling example...
#include <boost/fusion/adapted.hpp>
#include <boost/spirit/home/x3.hpp>
#include <boost/spirit/home/x3/support/ast/variant.hpp>
#include <boost/spirit/home/x3/support/utility/error_reporting.hpp>
#include <iomanip>
#include <iostream>
namespace x3 = boost::spirit::x3;
namespace hlsl
{
namespace ast
{
struct Void
{
};
struct Get;
struct Set;
struct Call;
struct Assign;
struct Binary;
struct Unary;
struct Variable
{
std::string name;
};
using Expr = x3::variant<Void, x3::forward_ast<Get>, x3::forward_ast<Set>, Variable, x3::forward_ast<Call>, x3::forward_ast<Assign>, x3::forward_ast<Binary>, x3::forward_ast<Unary>>;
struct Call
{
Expr name;
std::vector<Expr> arguments_;
};
struct Get
{
Expr object_;
std::string property_;
};
struct Set
{
Expr object_;
Expr value_;
std::string name_;
};
struct Assign
{
std::string name_;
Expr value_;
};
struct Binary
{
Expr left_;
std::string op_;
Expr right_;
};
struct Unary
{
std::string op_;
Expr expr_;
};
} // namespace ast
struct printer
{
std::ostream &_os;
using result_type = void;
void operator()(hlsl::ast::Get const &get) const
{
_os << "get { object_:";
get.object_.apply_visitor(*this);
_os << ", property_:" << quoted(get.property_) << " }";
}
void operator()(hlsl::ast::Set const &set) const
{
_os << "set { object_:";
set.object_.apply_visitor(*this);
_os << ", name_:" << quoted(set.name_);
_os << " equals: ";
set.value_.apply_visitor(*this);
_os << " }";
}
void operator()(hlsl::ast::Assign const &assign) const
{
_os << "assign { ";
_os << "name_:" << quoted(assign.name_);
_os << ", value_:";
assign.value_.apply_visitor(*this);
_os << " }";
}
void operator()(hlsl::ast::Variable const &var) const
{
_os << "var{" << quoted(var.name) << "}";
};
void operator()(hlsl::ast::Binary const &bin) const
{
_os << "binary { ";
bin.left_.apply_visitor(*this);
_os << " " << quoted(bin.op_) << " ";
bin.right_.apply_visitor(*this);
_os << " }";
};
void operator()(hlsl::ast::Unary const &un) const
{
_os << "unary { ";
un.expr_.apply_visitor(*this);
_os << quoted(un.op_);
_os << " }";
};
void operator()(hlsl::ast::Call const &call) const
{
_os << "call{";
call.name.apply_visitor(*this);
_os << ", args: ";
for (auto &arg : call.arguments_)
{
arg.apply_visitor(*this);
_os << ", ";
}
_os << /*quoted(call.name) << */ "}";
};
void operator()(hlsl::ast::Void const &) const { _os << "void{}"; };
};
} // namespace hlsl
BOOST_FUSION_ADAPT_STRUCT(hlsl::ast::Variable, name)
BOOST_FUSION_ADAPT_STRUCT(hlsl::ast::Call, name, arguments_)
BOOST_FUSION_ADAPT_STRUCT(hlsl::ast::Get, object_, property_)
BOOST_FUSION_ADAPT_STRUCT(hlsl::ast::Set, object_, value_)
BOOST_FUSION_ADAPT_STRUCT(hlsl::ast::Assign, name_, value_)
BOOST_FUSION_ADAPT_STRUCT(hlsl::ast::Binary, left_, op_, right_)
BOOST_FUSION_ADAPT_STRUCT(hlsl::ast::Unary, op_, expr_)
namespace hlsl::parser
{
struct eh_tag;
struct error_handler
{
template <typename It, typename Exc, typename Ctx>
auto on_error(It &, It, Exc const &x, Ctx const &context) const
{
x3::get<eh_tag>(context)( //
x.where(), "Error! Expecting: " + x.which() + " here:");
return x3::error_handler_result::fail;
}
};
struct program_ : error_handler
{
};
x3::rule<struct identifier_, std::string> const identifier{"identifier"};
x3::rule<struct variable_, ast::Variable> const variable{"variable"};
x3::rule<struct arguments_, std::vector<ast::Expr>> const arguments{"arguments_"};
x3::rule<struct binary_, hlsl::ast::Binary, true> const binary{"binary"};
x3::rule<struct unary_, hlsl::ast::Unary> const unary{"unary"};
x3::rule<struct unarycallwrapper_, hlsl::ast::Expr> const unarycallwrapper{"unarycallwrapper"};
x3::rule<struct get_, ast::Expr> const get{"get"};
x3::rule<struct call_, ast::Expr> const call{"call"};
x3::rule<struct program_, ast::Expr> const program{"program"};
x3::rule<struct primary_, ast::Expr> const primary{"primary"};
x3::rule<struct expression_, ast::Expr> const expression{"expression"};
x3::rule<struct set_, ast::Set, true> const set{"set"};
x3::rule<struct assign_, ast::Assign> const assign{"assign"};
x3::rule<struct assignment_, ast::Expr> const assignment{"assignment"};
auto get_string_from_variable = [](auto &ctx)
{ _val(ctx).name_ = std::move(_attr(ctx).name); };
auto fix_assignExpr = [](auto &ctx)
{ _val(ctx).value_ = std::move(_attr(ctx)); };
auto as_expr = [](auto &ctx)
{ _val(ctx) = ast::Expr(std::move(_attr(ctx))); };
auto as_unary = [](auto &ctx)
{ _val(ctx) = ast::Unary(std::move(_attr(ctx))); };
auto as_call = [](auto &ctx)
{ _val(ctx) = ast::Call{std::move(_val(ctx)), std::move(_attr(ctx))}; };
auto fold_in_get_to_set = [](auto &ctx)
{
auto &val = x3::_val(ctx);
val.name_ = boost::get<x3::forward_ast<ast::Get>>(val.object_).get().property_;
val.object_ = ast::Expr(boost::get<x3::forward_ast<ast::Get>>(val.object_).get().object_);
};
auto as_string = [](auto &ctx)
{ _val(ctx) = std::move(_attr(ctx).name); };
auto as_assign = [](auto &ctx)
{ _val(ctx) = ast::Assign(std::move(_val(ctx)), std::move(_attr(ctx))); };
auto as_get = [](auto &ctx)
{
_val(ctx) = ast::Get{std::move(_val(ctx)), _attr(ctx)};
};
auto variable_def = identifier;
auto primary_def = variable;
auto identifier_def = x3::lexeme[x3::alpha >> *x3::alnum];
auto expression_def = assignment;
auto assignment_def = (assign | set) | binary; // replace binary with call to see the rest working
auto assign_def = variable[get_string_from_variable] >> '=' >> assignment[fix_assignExpr];
auto set_def = (get >> '=' >> assignment)[fold_in_get_to_set];
auto arguments_def = *(expression % ',');
auto get_def = primary[as_expr] >> *('.' >> identifier)[as_get];
auto call_def = primary[as_expr] >> *((x3::lit('(') >> arguments >> x3::lit(')'))[as_call] | ('.' >> identifier)[as_get]);
auto unary_def = (x3::string("-") >> unary);
auto unarycallwrapper_def = unary | call ;
auto binary_def = unarycallwrapper >> x3::string("*") >> unarycallwrapper;
auto program_def = x3::skip(x3::space)[expression];
BOOST_SPIRIT_DEFINE(primary, assign, binary, unary, unarycallwrapper, assignment, get, set, variable, arguments, expression, call, identifier, program);
} // namespace hlsl::parser
int main()
{
using namespace hlsl;
for (std::string const input :
{
"first",
"first.second",
"first.Second.third",
"first.Second().third",
"first.Second(arg1).third",
"first.Second(arg1, arg2).third",
"first = second",
"first.second = third",
"first.second.third = fourth",
"first.second.third = fourth()",
"first.second.third = fourth(arg1)",
"this * that", //binary { var{"this"} "*" var{"that"} }
"this * -that", // binary { var{"this"} "*" unary{'-', var{"that"}} }
"this * that * there",
}) //
{
std::cout << "===== " << quoted(input) << "\n";
auto f = input.begin(), l = input.end();
// Our error handler
auto const p = x3::with<parser::eh_tag>(
x3::error_handler{f, l, std::cerr})[hlsl::parser::program];
if (hlsl::ast::Expr fs; parse(f, l, p, fs))
{
fs.apply_visitor(hlsl::printer{std::cout << "Parsed: "});
std::cout << "\n";
}
else
{
std::cout << "Parse failed at " << quoted(std::string(f, l)) << "\n";
}
}
}
Any help is appreciated :)

Answer 1

You found out how to jump hoops already :)
To lend some perspective I started from scratch. I copied the specs as a markdown comment. I basically copy pasted stuff and mapped an AST 1:1:
namespace Ast {
//////////////////
// primitive types
struct Nil { };
struct Identifier : std::string { using std::string::string; };
struct String : std::string { using std::string::string; };
enum class Bool { False, True };
using Number = boost::multiprecision::cpp_dec_float_50;
//////////////////
// expressions
enum class Op {
Plus, Minus, Multiply, Divide,
Equal, NotEqual, NOT, OR, AND,
GT, GTE, LT, LTE,
Assign
};
#define FWD(T) boost::recursive_wrapper<struct T>
using boost::optional;
using boost::blank; // std::monostate
using boost::variant;
using Expression = variant< //
Nil, Bool, Number, Identifier, String, //
FWD(FunctionCall), //
FWD(MemberAccess), //
FWD(Unary), //
FWD(Binary) //
>;
using Parameters = std::vector<Identifier>;
using Arguments = std::vector<Expression>;
struct FunctionCall { Expression fun; Arguments args; };
struct MemberAccess { Expression obj; Identifier mem; };
struct Unary { Op op; Expression oper; };
struct Binary { Op op; Expression lhs, rhs; };
//////////////////
// Declarations
struct PrintStmt { Expression value; };
struct ReturnStmt { optional<Expression> value; };
using Statement = variant< //
Expression, PrintStmt, ReturnStmt,
FWD(ForStmt), //
FWD(IfStmt), //
FWD(WhileStmt), //
FWD(Block) //
>;
using Statements = std::vector<Statement>;
struct VarDecl {
Identifier id;
optional<Expression> init;
};
struct ForStmt {
variant<blank, VarDecl, Expression> init;
optional<Expression> cond, incr;
optional<Statement> body;
};
struct IfStmt {
Expression cond;
Statement branch1;
optional<Statement> branch2;
};
struct WhileStmt { // REVIEW might represent as ForStmt
Expression cond;
Statement body;
};
struct Block {
Statements stmts;
};
//////////////////
// Declarations
struct FunDecl {
Identifier id;
Parameters params;
Block body;
};
struct ClassDecl {
Identifier id;
optional<Identifier> super;
std::vector<FunDecl> funcs;
};
using Declaration = boost::variant<ClassDecl, FunDecl, VarDecl, Statement>;
using Declarations = std::vector<Declaration>;
using Program = Declarations;
} // namespace Ast
Notes:
I used decimal number representation to not have to deal with too many representation issues
I changed Block content to be statements instead of declarations. It's unlikely that the script should really allow local class declarations. Allowing it means effectively the Declaration and Statement variant have to merge.
Adapting as Fusion sequences:
BOOST_FUSION_ADAPT_STRUCT(Ast::PrintStmt, value)
BOOST_FUSION_ADAPT_STRUCT(Ast::ReturnStmt, value)
BOOST_FUSION_ADAPT_STRUCT(Ast::ForStmt, init, cond, incr, body)
BOOST_FUSION_ADAPT_STRUCT(Ast::IfStmt, cond, branch1, branch2)
BOOST_FUSION_ADAPT_STRUCT(Ast::WhileStmt, cond, body)
BOOST_FUSION_ADAPT_STRUCT(Ast::Block, stmts)
BOOST_FUSION_ADAPT_STRUCT(Ast::FunDecl, id, params, body)
BOOST_FUSION_ADAPT_STRUCT(Ast::ClassDecl, id, super, funcs)
BOOST_FUSION_ADAPT_STRUCT(Ast::VarDecl, id, init)
// These are not required because they're constructed from semantic actions
//BOOST_FUSION_ADAPT_STRUCT(Ast::Unary, op, oper)
//BOOST_FUSION_ADAPT_STRUCT(Ast::Binary, lhs, rhs)
//BOOST_FUSION_ADAPT_STRUCT(Ast::FunctionCall, fun, args)
//BOOST_FUSION_ADAPT_STRUCT(Ast::MemberAccess, obj, mem)
Next up we declare rules for anything that is gonna recurse:
x3::rule<struct declaration, Ast::Declaration> declaration {"declaration"};
x3::rule<struct statement, Ast::Statement> statement {"statement"};
x3::rule<struct expression, Ast::Expression> expression {"expression"};
x3::rule<struct call, Ast::Expression> call {"call"};
Sadly, due to the operator precedence levels being split up in separate grammar productions, we get a proliferation of these rules:
x3::rule<struct unary, Ast::Expression> unary {"unary"};
x3::rule<struct factor, Ast::Expression> factor {"factor"};
x3::rule<struct term, Ast::Expression> term {"term"};
x3::rule<struct comparison, Ast::Expression> comparison {"comparison"};
x3::rule<struct equality, Ast::Expression> equality {"equality"};
x3::rule<struct logic_and, Ast::Expression> logic_and {"logic_and"};
x3::rule<struct logic_or, Ast::Expression> logic_or {"logic_or"};
x3::rule<struct assignment, Ast::Expression> assignment {"assignment"};
The lexicals are simple enough:
auto number = AST(Number,
x3::raw[x3::lexeme[ //
+x3::digit >> -("." >> +x3::digit) //
]][to_number]);
auto alpha = x3::char_("a-zA-Z_");
auto alnum = x3::char_("a-zA-Z_0-9");
auto identifier = AST(Identifier, x3::lexeme[alpha >> *alnum]);
auto string = AST(String, x3::lexeme['"' >> *~x3::char_('"') >> '"']);
I see I forgot to introduce AST(T, p) macro in time. See below.
Constructing the decimal number from string is fine:
auto to_number = [](auto& ctx) {
auto& raw = _attr(ctx);
_val(ctx) = Ast::Number{std::string(raw.begin(), raw.end())};
};
Keyword Checking
As an advanced feature I added keyword checking. You will find out you need it when you have a function name starting with a keyword, e.g.
def for_each(container, action) {
for (var i = 0; i < = container.size(); ++i) {
action(container.item(i));
}
}
for_each would misparse for as the keyword, unless we check that it is not immediately followed by "identifier" characters. Let's also make this a configuration point for case sensitivity:
// keyword checking
#if CASE_SENSITIVE
auto cs(auto p) { return p; };
#else
auto cs(auto p) { return x3::no_case[p]; };
#endif
auto kw(auto... p) { return x3::lexeme[(cs(p) | ...) >> !alnum]; }
Now we can use kw("for") instead of "for" and it will be properly case sensitive and boundary-checked.
Reserved keywords
The specs don't say, but you may want to avoid creating variables with reserved names. E.g. (return)("key").index would be an expression that invokes a function named return, but return ("key") would be a statement that returns the expression "key" (wrapped in a redundant subexpression).
So, let's add some logic to distinguish non-reserved identifiers:
// utility
auto bool_ = [] {
x3::symbols<Ast::Bool> sym;
sym.add("true", Ast::Bool::True);
sym.add("false", Ast::Bool::False);
return kw(sym);
}();
// Not specified, use `non_reserved = identifier` to allow those
auto reserved = kw("return", bool_, "nil", "fun", "var", "class");
auto non_reserved = !reserved >> identifier;
AST Building
I think I mentioned the at<T>(p) device before.
template <typename T> auto as(auto p, char const* name) {
return x3::rule<struct _, T>{name} = std::move(p);
};
template <typename T> auto as(auto p) {
static auto const name = boost::core::demangle(typeid(T).name());
return as<T>(std::move(p), name.c_str());
};
Making it less verbose with Ast:: types:
#define AST(T, p) as<Ast::T>(p, #T)
Now the utility productions from the grammar can be written as:
auto parameters = AST(Parameters, -(non_reserved % ","));
auto block = AST(Block,"{" >> *statement >> "}");
auto function = AST(FunDecl, non_reserved >> "(" >> parameters >> ")" >> block);
Declarations
// declarations
auto classDecl = AST(ClassDecl, //
kw("class") >> non_reserved >> -("<" >> non_reserved) >> //
"{" >> *function >> "}" //
);
auto funDecl = kw("fun") >> function;
auto varDecl = kw("var") >> AST(VarDecl, non_reserved >> -("=" >> expression) >> ";");
auto declaration_def = AST(Declaration, classDecl | funDecl | varDecl | statement);
auto program = x3::skip(skipper)[AST(Program, *(!x3::eoi >> declaration)) >> x3::eoi];
Not a lot to be said, except note the embedding of the skipper. For fun and exposition, I've customized the skipper to allow C++ style comments:
auto comment //
= ("//" > *(x3::char_ - x3::eol) > (x3::eoi | x3::eol)) //
| ("/*" > *(x3::char_ - "*/") > "*/") //
; //
auto skipper = x3::space | comment;
Statements
It's a bit of tedium, but the Fusion adaptations and previously introduced kw(...) and AST(T, p) helpers do all the heavy lifting:
// statements
auto exprStmt = AST(Expression, expression >> ";");
auto forStmt = AST(ForStmt, //
kw("for") >> "(" >> //
(varDecl | exprStmt | ";") >> //
-expression >> ";" >> //
-expression >> ")" >> statement);
auto ifStmt = AST(IfStmt, //
kw("if") >> ("(" >> expression >> ")") >> statement >>
-(kw("else") >> statement));
auto printStmt = AST(PrintStmt, kw("print") >> expression >> ";");
auto returnStmt = AST(ReturnStmt, kw("return") >> -expression >> ";");
auto whileStmt = AST(WhileStmt, kw("while") >> "(" >> expression >> ")" >> statement);
auto statement_def = AST(Statement, !(x3::eoi | "}") //
>> (forStmt | ifStmt | printStmt | returnStmt |
whileStmt | block | exprStmt));
Note how these are basically carbon copies of the specs.
Expressions
Here is the part that gave trouble.
First let's get the simple things out of way:
auto opsym = [] {
x3::symbols<Ast::Op> sym;
sym.add //
("+", Ast::Op::Plus)("-", Ast::Op::Minus) //
("*", Ast::Op::Multiply)("/", Ast::Op::Divide) //
("==", Ast::Op::Equal)("!=", Ast::Op::NotEqual) //
("!", Ast::Op::NOT)("or", Ast::Op::OR)("and", Ast::Op::AND) //
(">", Ast::Op::GT)(">=", Ast::Op::GTE) //
("<", Ast::Op::LT)("<=", Ast::Op::LTE) //
("=", Ast::Op::Assign);
return as<Ast::Op>( //
&identifier >> kw(sym) // if named operator, require keyword boundary
| sym,
"opsym");
}();
Note here that we conditionally apply the kw() modification on the operator symbol if the input token looks like alphanumeric. That, again, is to prevent andalucia or orlando from misparsing as the logical operators.
The condition &identifier is a bit sloppy, but it saves us from separating the interpunction operators from the named ones. Your profiler will tell you which is better.
auto nil = AST(Nil, kw("nil"));
auto arguments = AST(Arguments, &x3::lit(")") | expression % ",");
// this and super are just builtin identifiers
auto primary = AST(Expression,
bool_ | nil | number | string | non_reserved | "(" >> expression >> ")");
Note that I pruned "this" and "super" from the list as they are
just like other variables. If you opt to make them reserved, you will
need to special-case them here, e.g.
auto this_ = AST(Identifier, kw(x3::string("this")));
auto super_ = AST(Identifier, kw(x3::string("super")));
Smooth Operators
You already noticed the way using semantic actions. I separate out a few semantic action helpers:
auto assign = [](auto& ctx) {
_val(ctx) = _attr(ctx);
};
auto mk_call = [](auto& ctx) {
Ast::Expression expr = _val(ctx);
Ast::Arguments args = _attr(ctx);
_val(ctx) = Ast::FunctionCall{expr, args};
};
auto mk_member = [](auto& ctx) {
Ast::Expression obj = _val(ctx);
Ast::Identifier mem = _attr(ctx);
_val(ctx) = Ast::MemberAccess{obj, mem};
};
auto mk_unary = [](auto& ctx) {
auto& op = at_c<0>(_attr(ctx));
auto& rhs = at_c<1>(_attr(ctx));
_val(ctx) = Ast::Unary{op, rhs};
};
auto mk_binary = [](auto& ctx) {
auto& attr = _attr(ctx);
auto& op = at_c<0>(attr);
auto& rhs = at_c<1>(attr);
_val(ctx) = Ast::Binary{op, _val(ctx), rhs};
};
With these you can do the simples:
auto call_def = primary[assign] >> //
*(("(" >> arguments >> ")")[mk_call] //
| "." >> non_reserved[mk_member] //
);
auto unary_def = (expect_op("!", "-") >> unary)[mk_unary] | call[assign];
auto assignment_def = //
(call[assign] >> (expect_op("=") >> assignment)[mk_binary]) | //
logic_or[assign];
Then the bulk would become e.g.:
auto logic_or_def = logic_and[assign] >> *(&kw("or") >> opsym >> logic_and)[mk_binary];
To avoid the duplication let's make a rule factory:
auto binary_def = [](auto precedent, auto... ops) {
return precedent[assign] >> *(expect_op(ops...) >> precedent)[mk_binary];
};
The expect_op factory handles multiple acceptable operators, and applies proper token boundary checking again:
auto expect_op(auto... ops) {
return &x3::lexeme[
// keyword operator?
(&identifier >> kw((x3::as_parser(ops) | ...))) |
// interpunction operator
((x3::as_parser(ops) | ...) >> !x3::char_("!=><)"))] >>
opsym;
};
Now all the binaries (except the top level assignment, which has special associativity and lhs productions) become:
auto factor_def = binary_def(unary, "/", "*");
auto term_def = binary_def(factor, "-", "+");
auto comparison_def = binary_def(term, ">", ">=", "<", "<=");
auto equality_def = binary_def(comparison, "!=", "==");
auto logic_and_def = binary_def(equality, "and");
auto logic_or_def = binary_def(logic_and, "or");
Tieing it all together:
auto expression_def = assignment;
BOOST_SPIRIT_DEFINE(declaration, statement, expression);
BOOST_SPIRIT_DEFINE(call, unary, factor, term, comparison, equality, logic_and,
logic_or, assignment);
Testing
int main() {
#ifdef COLIRU
std::string input(std::istreambuf_iterator<char>(std::cin), {});
#else
std::string_view input = R"~(
class Cat < Animal {
Cat(name) {
print format("maybe implement member data some day: {}\n", name);
}
bark(volume) {
for (dummy = Nil; volume>0; volume = volume - 1)
print "bark!";
if (dummy or !(dummy == Nil) and universe_sane()) {
while(dummy) {{ print "(just kidding)"; }}
} else if (nesting() == "the shit") {
print("cool beans"); // extra parentheses are fine
return(True != False); // also on return statements
} else brackets = !"required";
return False;
}
bite() { return "pain takes no arguments"; }
}
var pooky = Cat("Pooky");
pooky.bark(10);
pooky = nil; // pooky got offed for being obnoxious :(
)~";
#endif
{
if (Ast::Program parsed;
parse(begin(input), end(input), Grammar::program, parsed))
std::cout << parsed << "\n";
else
std::cout << "Failed\n";
}
}
Live On Coliru Printing
class `Cat` < `Animal`{
[fun] `Cat`(`name`) {
print (`format`("maybe implement member data some day: {}\\n",`name`));
}
[fun] `bark`(`volume`) {
for((`dummy` = Nil); (`volume` > 0); (`volume` = (`volume` - 1)))
print "bark!";
if((`dummy` or ((! (`dummy` == Nil)) and (`universe_sane`())))) {
while(`dummy`)
{
{
print "(just kidding)";
}
}
}
else if(((`nesting`()) == "the shit")) {
print "cool beans";
return (True != False);
}
else (`brackets` = (! "required"))
return False;
}
[fun] `bite`() {
return "pain takes no arguments";
}
}
var `pooky` = (`Cat`("Pooky"));
((`pooky`.`bark`(10))
(`pooky` = Nil)
Locally, interactively:
Full Listing (anti-bitrot)
Sadly [SO] refuses it for length limits. I'll post it on Github. Link coming.
TL;DR
I think the at_c<N> accessor trick to dissect Fusion sequences in semantic action will help a lot.
Also, keep in mind that I don't think this rule structure is good for performant parsers. Just look at how something simple like x = y + (2); will invoke 43 rules (!!!) nested to 32 levels deep (!!!).
That's... not ideal. I've made a fully C++-compatible expression grammar (complete with interpreter) on SO before, and you can witness it here: https://github.com/sehe/qi-extended-parser-evaluator. It's using Spirit Qi, but in spirit it uses an almost X3 approach. I might make an X3 version of it just to compare for myself.
The key difference is that it generically implements operators with some metadata to describe it (token, precedence, associativity). This information is then used to combine expression AST nodes correctly. It even allows to get rid of redundant parentheses, both when building the Ast and when printing.
The interpreter logic (with dynamic type system, some reflection and execution tracing) may be a nice bonus inspiration: https://github.com/sehe/qi-extended-parser-evaluator/blob/master/eval.h#L291

Answer 2

Here is how I have solved the issue.
Instead of having a single binary ast node that stores a string of either "*" or "/", I split it up into separate ast node types for divide and multiply.
I then used the same machinery suggested by #sehe in the linked answer to synthesize the right nodes.
I'm still unsure how you can use semantic actions to synthesize attributes that span accross multiple '>>' operators. I'm guessing that the _val(ctx) in the semantic action refers to the whole ast::Expr across the currently defined rule so maybe you can set one member of a ast::Binary (eg the op string from the x3::string("*"), then in the next term after the '>>' you write _val(ctx) again (copy construct from previous?) and set the next member from the _attr(ctx)? I'll see if I can investigate if that works next. That would allow some more complex synthesizing of Attributes. Although I'm not sure if you could have different types being set accross the rule.
#include <boost/fusion/adapted.hpp>
#include <boost/spirit/home/x3.hpp>
#include <boost/spirit/home/x3/support/ast/variant.hpp>
#include <boost/spirit/home/x3/support/utility/error_reporting.hpp>
#include <iomanip>
#include <iostream>
namespace x3 = boost::spirit::x3;
namespace hlsl
{
namespace ast
{
struct Void
{
};
struct Get;
struct Set;
struct Call;
struct Assign;
struct Divide;
struct Multiply;
struct Unary;
struct Variable
{
std::string name;
// operator std::string() const {
// return name;
// }
};
using Expr = x3::variant<Void, x3::forward_ast<Get>, x3::forward_ast<Set>, Variable, x3::forward_ast<Call>, x3::forward_ast<Assign>, x3::forward_ast<Multiply>, x3::forward_ast<Divide>, x3::forward_ast<Unary>>;
struct Call
{
Expr name;
std::vector<Expr> arguments_;
};
struct Get
{
Expr object_;
std::string property_;
};
struct Set
{
Expr object_;
Expr value_;
std::string name_;
};
struct Assign
{
std::string name_;
Expr value_;
};
// struct Logical
// {
// Expr left_;
// std::string op_;
// Expr right_;
// };
struct Multiply
{
Expr left_;
Expr right_;
};
struct Divide
{
Expr left_;
Expr right_;
};
struct Unary
{
std::string op_;
Expr expr_;
};
} // namespace ast
struct printer
{
std::ostream &_os;
using result_type = void;
void operator()(hlsl::ast::Get const &get) const
{
_os << "get { object_:";
get.object_.apply_visitor(*this);
_os << ", property_:" << quoted(get.property_) << " }";
}
void operator()(hlsl::ast::Set const &set) const
{
_os << "set { object_:";
set.object_.apply_visitor(*this);
_os << ", name_:" << quoted(set.name_);
_os << " equals: ";
set.value_.apply_visitor(*this);
_os << " }";
}
void operator()(hlsl::ast::Assign const &assign) const
{
_os << "assign { ";
_os << "name_:" << quoted(assign.name_);
_os << ", value_:";
assign.value_.apply_visitor(*this);
_os << " }";
}
void operator()(hlsl::ast::Variable const &var) const
{
_os << "var{" << quoted(var.name) << "}";
};
void operator()(hlsl::ast::Divide const &bin) const
{
_os << "divide { ";
bin.left_.apply_visitor(*this);
bin.right_.apply_visitor(*this);
_os << " }";
};
void operator()(hlsl::ast::Multiply const &bin) const
{
_os << "multiply { ";
bin.left_.apply_visitor(*this);
bin.right_.apply_visitor(*this);
_os << " }";
};
void operator()(hlsl::ast::Unary const &un) const
{
_os << "unary { ";
un.expr_.apply_visitor(*this);
_os << quoted(un.op_);
_os << " }";
};
void operator()(hlsl::ast::Call const &call) const
{
_os << "call{";
call.name.apply_visitor(*this);
_os << ", args: ";
for (auto &arg : call.arguments_)
{
arg.apply_visitor(*this);
_os << ", ";
}
_os << /*quoted(call.name) << */ "}";
};
void operator()(hlsl::ast::Void const &) const { _os << "void{}"; };
};
} // namespace hlsl
BOOST_FUSION_ADAPT_STRUCT(hlsl::ast::Variable, name)
BOOST_FUSION_ADAPT_STRUCT(hlsl::ast::Call, name, arguments_)
BOOST_FUSION_ADAPT_STRUCT(hlsl::ast::Get, object_, property_)
BOOST_FUSION_ADAPT_STRUCT(hlsl::ast::Set, object_, value_)
BOOST_FUSION_ADAPT_STRUCT(hlsl::ast::Assign, name_, value_)
BOOST_FUSION_ADAPT_STRUCT(hlsl::ast::Multiply, left_, right_)
BOOST_FUSION_ADAPT_STRUCT(hlsl::ast::Divide, left_, right_)
BOOST_FUSION_ADAPT_STRUCT(hlsl::ast::Unary, op_, expr_)
namespace hlsl::parser
{
struct eh_tag;
struct error_handler
{
template <typename It, typename Exc, typename Ctx>
auto on_error(It &, It, Exc const &x, Ctx const &context) const
{
x3::get<eh_tag>(context)( //
x.where(), "Error! Expecting: " + x.which() + " here:");
return x3::error_handler_result::fail;
}
};
struct program_ : error_handler
{
};
x3::rule<struct identifier_, std::string> const identifier{"identifier"};
x3::rule<struct variable_, ast::Variable> const variable{"variable"};
x3::rule<struct arguments_, std::vector<ast::Expr>> const arguments{"arguments_"};
x3::rule<struct binary_, hlsl::ast::Expr> const binary{"binary"};
x3::rule<struct multiply_, hlsl::ast::Expr> const multiply{"multiply"};
x3::rule<struct divide_, hlsl::ast::Expr> const divide{"divide"};
x3::rule<struct unary_, hlsl::ast::Unary> const unary{"unary"};
x3::rule<struct unarycallwrapper_, hlsl::ast::Expr> const unarycallwrapper{"unarycallwrapper"};
x3::rule<struct get_, ast::Expr> const get{"get"};
x3::rule<struct call_, ast::Expr> const call{"call"};
x3::rule<struct program_, ast::Expr> const program{"program"};
x3::rule<struct primary_, ast::Expr> const primary{"primary"};
x3::rule<struct expression_, ast::Expr> const expression{"expression"};
x3::rule<struct set_, ast::Set, true> const set{"set"};
x3::rule<struct assign_, ast::Assign> const assign{"assign"};
x3::rule<struct assignment_, ast::Expr> const assignment{"assignment"};
auto get_string_from_variable = [](auto &ctx)
{ _val(ctx).name_ = std::move(_attr(ctx).name); };
auto fix_assignExpr = [](auto &ctx)
{ _val(ctx).value_ = std::move(_attr(ctx)); };
auto as_expr = [](auto &ctx)
{ _val(ctx) = ast::Expr(std::move(_attr(ctx))); };
auto as_unary = [](auto &ctx)
{ _val(ctx) = ast::Unary(std::move(_attr(ctx))); };
auto as_call = [](auto &ctx)
{ _val(ctx) = ast::Call{std::move(_val(ctx)), std::move(_attr(ctx))}; };
auto as_multiply = [](auto &ctx)
{ _val(ctx) = ast::Multiply{std::move(_val(ctx)), std::move(_attr(ctx))}; };
auto as_divide = [](auto &ctx)
{ _val(ctx) = ast::Divide{std::move(_val(ctx)), std::move(_attr(ctx))}; };
auto fold_in_get_to_set = [](auto &ctx)
{
auto &val = x3::_val(ctx);
val.name_ = boost::get<x3::forward_ast<ast::Get>>(val.object_).get().property_;
val.object_ = ast::Expr(boost::get<x3::forward_ast<ast::Get>>(val.object_).get().object_);
};
auto as_string = [](auto &ctx)
{ _val(ctx) = std::move(_attr(ctx).name); };
auto as_assign = [](auto &ctx)
{ _val(ctx) = ast::Assign(std::move(_val(ctx)), std::move(_attr(ctx))); };
auto as_get = [](auto &ctx)
{
_val(ctx) = ast::Get{std::move(_val(ctx)), _attr(ctx)};
};
auto variable_def = identifier;
auto primary_def = variable;
auto identifier_def = x3::lexeme[x3::alpha >> *x3::alnum];
auto expression_def = assignment;
auto assignment_def = (assign | set) | binary; // replace binary with call to see the rest working
auto assign_def = variable[get_string_from_variable] >> '=' >> assignment[fix_assignExpr];
auto set_def = (get >> '=' >> assignment)[fold_in_get_to_set];
auto arguments_def = *(expression % ',');
auto get_def = primary[as_expr] >> *('.' >> identifier)[as_get];
auto call_def = primary[as_expr] >> *((x3::lit('(') >> arguments >> x3::lit(')'))[as_call] | ('.' >> identifier)[as_get]);
auto unary_def = (x3::string("-") >> unarycallwrapper);
auto unarycallwrapper_def = call | unary;
auto binary_def = unarycallwrapper[as_expr] >> *((x3::lit('/') >> unarycallwrapper[as_divide]) | (x3::lit('*') >> unarycallwrapper[as_multiply]));
auto program_def = x3::skip(x3::space)[expression];
BOOST_SPIRIT_DEFINE(primary, assign, binary, multiply, divide, unary, unarycallwrapper, assignment, get, set, variable, arguments, expression, call, identifier, program);
} // namespace hlsl::parser
int main()
{
using namespace hlsl;
for (std::string const input :
{
"first",
"first.second",
"first.Second.third",
"first.Second().third",
"first.Second(arg1).third",
"first.Second(arg1, arg2).third",
"first = second",
"first.second = third",
"first.second.third = fourth",
"first.second.third = fourth()",
"first.second.third = fourth(arg1)",
"this * that", // binary { var{"this"} "*" var{"that"} }
"this * -that", // binary { var{"this"} "*" unary{'-', var{"that"}} }
"this * that * there",
"this * that / there",
"this.inner * that * there.inner2",
}) //
{
std::cout << "===== " << quoted(input) << "\n";
auto f = input.begin(), l = input.end();
// Our error handler
auto const p = x3::with<parser::eh_tag>(
x3::error_handler{f, l, std::cerr})[hlsl::parser::program];
if (hlsl::ast::Expr fs; parse(f, l, p, fs))
{
fs.apply_visitor(hlsl::printer{std::cout << "Parsed: "});
std::cout << "\n";
}
else
{
std::cout << "Parse failed at " << quoted(std::string(f, l)) << "\n";
}
}
}

Answer 3

I also figured out how the semantic actions write to _val(ctx) across multiple sequence '>>' operators. You can write to them with the type that you need and it gets passed to the next one!
See binary2 rule and how it's def uses two semantic actions to write a Binary2 ast node and set different members each time.
#include <boost/fusion/adapted.hpp>
#include <boost/spirit/home/x3.hpp>
#include <boost/spirit/home/x3/support/ast/variant.hpp>
#include <boost/spirit/home/x3/support/utility/error_reporting.hpp>
#include <iomanip>
#include <iostream>
namespace x3 = boost::spirit::x3;
namespace hlsl
{
namespace ast
{
struct Void
{
};
struct Get;
struct Set;
struct Call;
struct Assign;
struct Divide;
struct Multiply;
struct Unary;
struct Binary2;
struct Variable
{
std::string name;
// operator std::string() const {
// return name;
// }
};
using Expr = x3::variant<Void, x3::forward_ast<Get>, x3::forward_ast<Set>, Variable, x3::forward_ast<Call>, x3::forward_ast<Assign>, x3::forward_ast<Multiply>, x3::forward_ast<Binary2>, x3::forward_ast<Divide>, x3::forward_ast<Unary>>;
struct Call
{
Expr name;
std::vector<Expr> arguments_;
};
struct Get
{
Expr object_;
std::string property_;
};
struct Set
{
Expr object_;
Expr value_;
std::string name_;
};
struct Assign
{
std::string name_;
Expr value_;
};
// struct Logical
// {
// Expr left_;
// std::string op_;
// Expr right_;
// };
struct Multiply
{
Expr left_;
Expr right_;
};
struct Binary2
{
Expr left_;
std::string op_;
Expr right_;
};
struct Divide
{
Expr left_;
Expr right_;
};
struct Unary
{
std::string op_;
Expr expr_;
};
} // namespace ast
struct printer
{
std::ostream &_os;
using result_type = void;
void operator()(hlsl::ast::Get const &get) const
{
_os << "get { object_:";
get.object_.apply_visitor(*this);
_os << ", property_:" << quoted(get.property_) << " }";
}
void operator()(hlsl::ast::Set const &set) const
{
_os << "set { object_:";
set.object_.apply_visitor(*this);
_os << ", name_:" << quoted(set.name_);
_os << " equals: ";
set.value_.apply_visitor(*this);
_os << " }";
}
void operator()(hlsl::ast::Assign const &assign) const
{
_os << "assign { ";
_os << "name_:" << quoted(assign.name_);
_os << ", value_:";
assign.value_.apply_visitor(*this);
_os << " }";
}
void operator()(hlsl::ast::Variable const &var) const
{
_os << "var{" << quoted(var.name) << "}";
};
void operator()(hlsl::ast::Divide const &bin) const
{
_os << "divide { ";
bin.left_.apply_visitor(*this);
bin.right_.apply_visitor(*this);
_os << " }";
};
void operator()(hlsl::ast::Multiply const &bin) const
{
_os << "multiply { ";
bin.left_.apply_visitor(*this);
bin.right_.apply_visitor(*this);
_os << " }";
};
void operator()(hlsl::ast::Binary2 const &bin) const
{
_os << "binary2 { ";
bin.left_.apply_visitor(*this);
_os << bin.op_ << ", ";
bin.right_.apply_visitor(*this);
_os << " }";
};
void operator()(hlsl::ast::Unary const &un) const
{
_os << "unary { ";
un.expr_.apply_visitor(*this);
_os << quoted(un.op_);
_os << " }";
};
void operator()(hlsl::ast::Call const &call) const
{
_os << "call{";
call.name.apply_visitor(*this);
_os << ", args: ";
for (auto &arg : call.arguments_)
{
arg.apply_visitor(*this);
_os << ", ";
}
_os << /*quoted(call.name) << */ "}";
};
void operator()(hlsl::ast::Void const &) const { _os << "void{}"; };
};
} // namespace hlsl
BOOST_FUSION_ADAPT_STRUCT(hlsl::ast::Variable, name)
BOOST_FUSION_ADAPT_STRUCT(hlsl::ast::Call, name, arguments_)
BOOST_FUSION_ADAPT_STRUCT(hlsl::ast::Get, object_, property_)
BOOST_FUSION_ADAPT_STRUCT(hlsl::ast::Set, object_, value_)
BOOST_FUSION_ADAPT_STRUCT(hlsl::ast::Assign, name_, value_)
BOOST_FUSION_ADAPT_STRUCT(hlsl::ast::Multiply, left_, right_)
BOOST_FUSION_ADAPT_STRUCT(hlsl::ast::Binary2, left_, op_, right_)
BOOST_FUSION_ADAPT_STRUCT(hlsl::ast::Divide, left_, right_)
BOOST_FUSION_ADAPT_STRUCT(hlsl::ast::Unary, op_, expr_)
namespace hlsl::parser
{
struct eh_tag;
struct error_handler
{
template <typename It, typename Exc, typename Ctx>
auto on_error(It &, It, Exc const &x, Ctx const &context) const
{
x3::get<eh_tag>(context)( //
x.where(), "Error! Expecting: " + x.which() + " here:");
return x3::error_handler_result::fail;
}
};
struct program_ : error_handler
{
};
x3::rule<struct identifier_, std::string> const identifier{"identifier"};
x3::rule<struct binop_, std::string> const binop{"binop"};
x3::rule<struct variable_, ast::Variable> const variable{"variable"};
x3::rule<struct arguments_, std::vector<ast::Expr>> const arguments{"arguments_"};
x3::rule<struct binary_, hlsl::ast::Expr> const binary{"binary"};
x3::rule<struct binary2_, hlsl::ast::Expr> const binary2{"binary2"};
x3::rule<struct multiply_, hlsl::ast::Expr> const multiply{"multiply"};
x3::rule<struct divide_, hlsl::ast::Expr> const divide{"divide"};
x3::rule<struct unary_, hlsl::ast::Unary> const unary{"unary"};
x3::rule<struct unarycallwrapper_, hlsl::ast::Expr> const unarycallwrapper{"unarycallwrapper"};
x3::rule<struct get_, ast::Expr> const get{"get"};
x3::rule<struct call_, ast::Expr> const call{"call"};
x3::rule<struct program_, ast::Expr> const program{"program"};
x3::rule<struct primary_, ast::Expr> const primary{"primary"};
x3::rule<struct expression_, ast::Expr> const expression{"expression"};
x3::rule<struct set_, ast::Set, true> const set{"set"};
x3::rule<struct assign_, ast::Assign> const assign{"assign"};
x3::rule<struct assignment_, ast::Expr> const assignment{"assignment"};
auto get_string_from_variable = [](auto &ctx)
{ _val(ctx).name_ = std::move(_attr(ctx).name); };
auto fix_assignExpr = [](auto &ctx)
{ _val(ctx).value_ = std::move(_attr(ctx)); };
auto as_expr = [](auto &ctx)
{ _val(ctx) = ast::Expr(std::move(_attr(ctx))); };
auto as_unary = [](auto &ctx)
{ _val(ctx) = ast::Unary(std::move(_attr(ctx))); };
auto as_call = [](auto &ctx)
{ _val(ctx) = ast::Call{std::move(_val(ctx)), std::move(_attr(ctx))}; };
auto as_multiply = [](auto &ctx)
{ _val(ctx) = ast::Multiply{std::move(_val(ctx)), std::move(_attr(ctx))}; };
auto as_divide = [](auto &ctx)
{ _val(ctx) = ast::Divide{std::move(_val(ctx)), std::move(_attr(ctx))}; };
auto as_binary2A = [](auto &ctx)
{ _val(ctx) = ast::Binary2{std::move(_val(ctx)), std::move(_attr(ctx)), ast::Expr{}}; };
auto as_binary2B = [](auto &ctx)
{ //_val(ctx) = std::move(_val(ctx));
boost::get<x3::forward_ast<ast::Binary2>>(_val(ctx)).get().right_ = std::move(_attr(ctx)); };
auto fold_in_get_to_set = [](auto &ctx)
{
auto &val = x3::_val(ctx);
val.name_ = boost::get<x3::forward_ast<ast::Get>>(val.object_).get().property_;
val.object_ = ast::Expr(boost::get<x3::forward_ast<ast::Get>>(val.object_).get().object_);
};
auto as_string = [](auto &ctx)
{ _val(ctx) = std::move(_attr(ctx).name); };
auto as_assign = [](auto &ctx)
{ _val(ctx) = ast::Assign(std::move(_val(ctx)), std::move(_attr(ctx))); };
auto as_get = [](auto &ctx)
{
_val(ctx) = ast::Get{std::move(_val(ctx)), _attr(ctx)};
};
auto variable_def = identifier;
auto primary_def = variable;
auto identifier_def = x3::lexeme[x3::alpha >> *x3::alnum];
auto expression_def = assignment;
auto assignment_def = (assign | set) | binary2; // replace binary with call to see the rest working
auto assign_def = variable[get_string_from_variable] >> '=' >> assignment[fix_assignExpr];
auto set_def = (get >> '=' >> assignment)[fold_in_get_to_set];
auto arguments_def = *(expression % ',');
auto get_def = primary[as_expr] >> *('.' >> identifier)[as_get];
auto call_def = primary[as_expr] >> *((x3::lit('(') >> arguments >> x3::lit(')'))[as_call] | ('.' >> identifier)[as_get]);
auto unary_def = (x3::string("-") >> unarycallwrapper);
auto unarycallwrapper_def = unary | call;
auto binop_def = x3::string("*") | x3::string("/");
auto binary_def = unarycallwrapper[as_expr] >> *((x3::lit('/') >> unarycallwrapper[as_divide]) | (x3::lit('*') >> unarycallwrapper[as_multiply]));
auto binary2_def = unarycallwrapper[as_expr] >> *(binop[as_binary2A] >> unarycallwrapper[as_binary2B]);
auto program_def = x3::skip(x3::space)[expression];
BOOST_SPIRIT_DEFINE(primary, assign, binop, binary, binary2, unary, unarycallwrapper, assignment, get, set, variable, arguments, expression, call, identifier, program);
} // namespace hlsl::parser
int main()
{
using namespace hlsl;
for (std::string const input :
{
"first",
"first.second",
"first.Second.third",
"first.Second().third",
"first.Second(arg1).third",
"first.Second(arg1, arg2).third",
"first = second",
"first.second = third",
"first.second.third = fourth",
"first.second.third = fourth()",
"first.second.third = fourth(arg1)",
"this * that", // binary { var{"this"} "*" var{"that"} }
"this * -that", // binary { var{"this"} "*" unary{'-', var{"that"}} }
"this * that * there",
"this * that / there",
"this.inner * that * there.inner2",
}) //
{
std::cout << "===== " << quoted(input) << "\n";
auto f = input.begin(), l = input.end();
// Our error handler
auto const p = x3::with<parser::eh_tag>(
x3::error_handler{f, l, std::cerr})[hlsl::parser::program];
if (hlsl::ast::Expr fs; parse(f, l, p, fs))
{
fs.apply_visitor(hlsl::printer{std::cout << "Parsed: "});
std::cout << "\n";
}
else
{
std::cout << "Parse failed at " << quoted(std::string(f, l)) << "\n";
}
}
}

Answer 4

One more post...
I'm almost finished with the full stack of expression operators that hlsl uses (bitwise, logical, compound assignment etc).
I even figured out nested ternary operators which I thought would be really hard but didn't turn out too bad.
#include <boost/fusion/adapted.hpp>
#include <boost/spirit/home/x3.hpp>
#include <boost/spirit/home/x3/support/ast/variant.hpp>
#include <boost/spirit/home/x3/support/utility/error_reporting.hpp>
#include <iomanip>
#include <iostream>
namespace x3 = boost::spirit::x3;
namespace hlsl
{
namespace ast
{
struct Void
{
};
struct Get;
struct Set;
struct Call;
struct Assign;
struct CompoundAssign;
struct Divide;
struct Multiply;
struct Unary;
struct Binary;
struct Logical;
struct Bitwise;
struct Ternary;
struct Variable
{
std::string name;
// operator std::string() const {
// return name;
// }
};
using Expr = x3::variant< Void,
x3::forward_ast<Get>,
x3::forward_ast<Set>,
Variable,
x3::forward_ast<Call>,
x3::forward_ast<Assign>,
x3::forward_ast<CompoundAssign>,
x3::forward_ast<Multiply>,
x3::forward_ast<Binary>,
x3::forward_ast<Logical>,
x3::forward_ast<Ternary>,
x3::forward_ast<Bitwise>,
x3::forward_ast<Divide>,
x3::forward_ast<Unary>>;
struct Call
{
Expr name;
std::vector<Expr> arguments_;
};
struct Get
{
Expr object_;
std::string property_;
};
struct Set
{
Expr object_;
Expr value_;
std::string name_;
};
struct Assign
{
std::string name_;
Expr value_;
};
struct CompoundAssign
{
std::string name_;
std::string op_;
Expr value_;
};
struct Multiply
{
Expr left_;
Expr right_;
};
struct Binary
{
Expr left_;
std::string op_;
Expr right_;
};
struct Logical
{
Expr left_;
std::string op_;
Expr right_;
};
struct Bitwise
{
Expr left_;
std::string op_;
Expr right_;
};
struct Divide
{
Expr left_;
Expr right_;
};
struct Unary
{
std::string op_;
Expr expr_;
};
struct Ternary
{
Expr condition_;
Expr ifexpr_;
Expr elseexpr_;
};
} // namespace ast
struct printer
{
std::ostream &_os;
using result_type = void;
void operator()(hlsl::ast::Get const &get) const
{
_os << "get { object_:";
get.object_.apply_visitor(*this);
_os << ", property_:" << quoted(get.property_) << " }";
}
void operator()(hlsl::ast::Set const &set) const
{
_os << "set { object_:";
set.object_.apply_visitor(*this);
_os << ", name_:" << quoted(set.name_);
_os << " equals: ";
set.value_.apply_visitor(*this);
_os << " }";
}
void operator()(hlsl::ast::Assign const &assign) const
{
_os << "assign { ";
_os << "name_:" << quoted(assign.name_);
_os << ", value_:";
assign.value_.apply_visitor(*this);
_os << " }";
}
void operator()(hlsl::ast::CompoundAssign const &assign) const
{
_os << "compoundAssign { ";
_os << "name_:" << quoted(assign.name_);
_os << "op_:" << quoted(assign.op_);
_os << ", value_:";
assign.value_.apply_visitor(*this);
_os << " }";
}
void operator()(hlsl::ast::Variable const &var) const
{
_os << "var{" << quoted(var.name) << "}";
};
void operator()(hlsl::ast::Divide const &bin) const
{
_os << "divide { ";
bin.left_.apply_visitor(*this);
bin.right_.apply_visitor(*this);
_os << " }";
};
void operator()(hlsl::ast::Multiply const &bin) const
{
_os << "multiply { ";
bin.left_.apply_visitor(*this);
bin.right_.apply_visitor(*this);
_os << " }";
};
void operator()(hlsl::ast::Binary const &bin) const
{
_os << "binary { ";
bin.left_.apply_visitor(*this);
_os << bin.op_ << ", ";
bin.right_.apply_visitor(*this);
_os << " }";
};
void operator()(hlsl::ast::Logical const &bin) const
{
_os << "logical { ";
bin.left_.apply_visitor(*this);
_os << bin.op_ << ", ";
bin.right_.apply_visitor(*this);
_os << " }";
};
void operator()(hlsl::ast::Bitwise const &bin) const
{
_os << "bitwise { ";
bin.left_.apply_visitor(*this);
_os << bin.op_ << ", ";
bin.right_.apply_visitor(*this);
_os << " }";
};
void operator()(hlsl::ast::Unary const &un) const
{
_os << "unary { ";
un.expr_.apply_visitor(*this);
_os << quoted(un.op_);
_os << " }";
};
void operator()(hlsl::ast::Ternary const &tern) const
{
_os << "ternary { ";
tern.condition_.apply_visitor(*this);
tern.ifexpr_.apply_visitor(*this);
tern.elseexpr_.apply_visitor(*this);
_os << " }";
};
void operator()(hlsl::ast::Call const &call) const
{
_os << "call{";
call.name.apply_visitor(*this);
_os << ", args: ";
for (auto &arg : call.arguments_)
{
arg.apply_visitor(*this);
_os << ", ";
}
_os << /*quoted(call.name) << */ "}";
};
void operator()(hlsl::ast::Void const &) const { _os << "void{}"; };
};
} // namespace hlsl
BOOST_FUSION_ADAPT_STRUCT(hlsl::ast::Variable, name)
BOOST_FUSION_ADAPT_STRUCT(hlsl::ast::Call, name, arguments_)
BOOST_FUSION_ADAPT_STRUCT(hlsl::ast::Get, object_, property_)
BOOST_FUSION_ADAPT_STRUCT(hlsl::ast::Set, object_, value_)
BOOST_FUSION_ADAPT_STRUCT(hlsl::ast::Assign, name_, value_)
BOOST_FUSION_ADAPT_STRUCT(hlsl::ast::CompoundAssign, name_, op_, value_)
BOOST_FUSION_ADAPT_STRUCT(hlsl::ast::Multiply, left_, right_)
BOOST_FUSION_ADAPT_STRUCT(hlsl::ast::Binary, left_, op_, right_)
BOOST_FUSION_ADAPT_STRUCT(hlsl::ast::Logical, left_, op_, right_)
BOOST_FUSION_ADAPT_STRUCT(hlsl::ast::Bitwise, left_, op_, right_)
BOOST_FUSION_ADAPT_STRUCT(hlsl::ast::Divide, left_, right_)
BOOST_FUSION_ADAPT_STRUCT(hlsl::ast::Unary, op_, expr_)
BOOST_FUSION_ADAPT_STRUCT(hlsl::ast::Ternary, condition_, ifexpr_, elseexpr_)
namespace hlsl::parser
{
struct eh_tag;
struct error_handler
{
template <typename It, typename Exc, typename Ctx>
auto on_error(It &, It, Exc const &x, Ctx const &context) const
{
x3::get<eh_tag>(context)( //
x.where(), "Error! Expecting: " + x.which() + " here:");
return x3::error_handler_result::fail;
}
};
struct program_ : error_handler
{
};
x3::rule<struct identifier_, std::string> const identifier{"identifier"};
x3::rule<struct factor_, std::string> const factor{"factor"};
x3::rule<struct term_, std::string> const term{"term"};
x3::rule<struct compare_op_, std::string> const compare_op{"compare_op"};
x3::rule<struct equality_op_, std::string> const equality_op{"equality_op"};
x3::rule<struct compoundassign_op_, std::string> const compoundassign_op{"compoundassign_op"};
x3::rule<struct bitwise_shift_op_, std::string> const bitwise_shift_op{"bitwise_shift_op"};
x3::rule<struct variable_, ast::Variable> const variable{"variable"};
x3::rule<struct arguments_, std::vector<ast::Expr>> const arguments{"arguments_"};
x3::rule<struct bitwise_or_, hlsl::ast::Expr> const bitwise_or{"bitwise_or"};
x3::rule<struct bitwise_xor_, hlsl::ast::Expr> const bitwise_xor{"bitwise_xor"};
x3::rule<struct bitwise_and_, hlsl::ast::Expr> const bitwise_and{"bitwise_and"};
x3::rule<struct bitwise_shift_, hlsl::ast::Expr> const bitwise_shift{"bitwise_shift"};
x3::rule<struct addition_, hlsl::ast::Expr> const addition{"addition"};
x3::rule<struct comparison_, hlsl::ast::Expr> const comparison{"comparison"};
x3::rule<struct equality_, hlsl::ast::Expr> const equality{"equality"};
x3::rule<struct logical_or_, hlsl::ast::Expr> const logical_or{"logical_or"};
x3::rule<struct logical_and_, hlsl::ast::Expr> const logical_and{"logical_and"};
x3::rule<struct multiply_, hlsl::ast::Expr> const multiply{"multiply"};
x3::rule<struct unary_, hlsl::ast::Unary> const unary{"unary"};
x3::rule<struct unarycallwrapper_, hlsl::ast::Expr> const unarycallwrapper{"unarycallwrapper"};
x3::rule<struct get_, ast::Expr> const get{"get"};
x3::rule<struct call_, ast::Expr> const call{"call"};
x3::rule<struct program_, ast::Expr> const program{"program"};
x3::rule<struct primary_, ast::Expr> const primary{"primary"};
x3::rule<struct expression_, ast::Expr> const expression{"expression"};
x3::rule<struct set_, ast::Set, true> const set{"set"};
x3::rule<struct assign_, ast::Assign> const assign{"assign"};
x3::rule<struct compoundassign_, ast::CompoundAssign> const compoundassign{"compoundassign"};
x3::rule<struct ternary_, ast::Expr> const ternary{"ternary"};
x3::rule<struct assignment_, ast::Expr> const assignment{"assignment"};
auto get_string_from_variable = [](auto &ctx)
{ _val(ctx).name_ = std::move(_attr(ctx).name); };
auto get_string_from_variable_cast = [](auto &ctx)
{ _val(ctx).name_ = std::move(_attr(ctx).name); };
auto fix_assignExpr = [](auto &ctx)
{ _val(ctx).value_ = std::move(_attr(ctx)); };
auto as_expr = [](auto &ctx)
{ _val(ctx) = ast::Expr(std::move(_attr(ctx))); };
auto as_unary = [](auto &ctx)
{ _val(ctx) = ast::Unary(std::move(_attr(ctx))); };
auto as_call = [](auto &ctx)
{ _val(ctx) = ast::Call{std::move(_val(ctx)), std::move(_attr(ctx))}; };
auto as_binary_op = [](auto &ctx)
{ _val(ctx) = ast::Binary{std::move(_val(ctx)), std::move(_attr(ctx)), ast::Expr{}}; };
auto as_binary_wrap = [](auto &ctx)
{ boost::get<x3::forward_ast<ast::Binary>>(_val(ctx)).get().right_ = std::move(_attr(ctx)); };
auto as_logical_op = [](auto &ctx)
{ _val(ctx) = ast::Logical{std::move(_val(ctx)), std::move(_attr(ctx)), ast::Expr{}}; };
auto as_logical_wrap = [](auto &ctx)
{ boost::get<x3::forward_ast<ast::Logical>>(_val(ctx)).get().right_ = std::move(_attr(ctx)); };
auto as_bitwise_op = [](auto &ctx)
{ _val(ctx) = ast::Bitwise{std::move(_val(ctx)), std::move(_attr(ctx)), ast::Expr{}}; };
auto as_bitwise_wrap = [](auto &ctx)
{ boost::get<x3::forward_ast<ast::Bitwise>>(_val(ctx)).get().right_ = std::move(_attr(ctx)); };
auto as_compound_op = [](auto &ctx)
{ _val(ctx).op_ = std::move(_attr(ctx)); };
auto as_ternary_ifexpr = [](auto &ctx)
{ _val(ctx) = ast::Ternary{std::move(_val(ctx)), std::move(_attr(ctx)), ast::Expr{}}; };
auto as_ternary_elseexpr = [](auto &ctx)
{ boost::get<x3::forward_ast<ast::Ternary>>(_val(ctx)).get().elseexpr_ = std::move(_attr(ctx)); };
auto as_compound_wrap = [](auto &ctx)
{ boost::get<x3::forward_ast<ast::CompoundAssign>>(_val(ctx)).get().value_ = std::move(_attr(ctx)); };
auto fold_in_get_to_set = [](auto &ctx)
{
auto &val = x3::_val(ctx);
val.name_ = boost::get<x3::forward_ast<ast::Get>>(val.object_).get().property_;
val.object_ = ast::Expr(boost::get<x3::forward_ast<ast::Get>>(val.object_).get().object_);
};
auto as_string = [](auto &ctx)
{ _val(ctx) = std::move(_attr(ctx).name); };
auto as_assign = [](auto &ctx)
{ _val(ctx) = ast::Assign(std::move(_val(ctx)), std::move(_attr(ctx))); };
auto as_get = [](auto &ctx)
{
_val(ctx) = ast::Get{std::move(_val(ctx)), _attr(ctx)};
};
auto expression_def = assignment;
auto variable_def = identifier;
auto identifier_def = x3::lexeme[x3::alpha >> *x3::alnum];
auto set_def = (get >> '=' >> assignment)[fold_in_get_to_set];
auto arguments_def = *(expression % ',');
auto factor_def = x3::string("*") | x3::string("/");
auto term_def = x3::string("+") | x3::string("-");
auto compare_op_def = x3::string("<=") | x3::string(">=") | x3::string("<") | x3::string(">");
auto equality_op_def = x3::string("!=") | x3::string("==");
auto compoundassign_op_def = x3::string("*=") | x3::string("/=") | x3::string("%=") | x3::string("+=")
| x3::string("-=") | x3::string("<<=") | x3::string(">>=")
| x3::string("&=") | x3::string("^=") | x3::string("|=");
auto bitwise_shift_op_def = x3::string(">>") | x3::string("<<");
//auto binary_def = unarycallwrapper[as_expr] >> *((x3::lit('/') >> unarycallwrapper[as_divide]) | (x3::lit('*') >> unarycallwrapper[as_multiply]));
auto assign_def = variable[get_string_from_variable] >> '=' >> assignment[fix_assignExpr];
auto compoundassign_def = variable[get_string_from_variable] >> compoundassign_op[as_compound_op] >> assignment[fix_assignExpr];
auto assignment_def = (assign | compoundassign | set) | ternary;
auto ternary_def = logical_or[as_expr] >> *('?' >> expression[as_ternary_ifexpr] >> ':' >> ternary[as_ternary_elseexpr]);
auto logical_or_def = logical_and[as_expr] >> *(x3::string("||")[as_logical_op] >> logical_and[as_logical_wrap]);
auto logical_and_def = bitwise_or[as_expr] >> *(x3::string("&&")[as_logical_op] >> bitwise_or[as_logical_wrap]);
auto bitwise_or_def = bitwise_xor[as_expr] >> *((x3::string("|") >> !(x3::lit('|') | x3::lit('=')))[as_bitwise_op] >> bitwise_xor[as_bitwise_wrap]);
auto bitwise_xor_def = bitwise_and[as_expr] >> *((x3::string("^") > !(x3::lit('^') | x3::lit('=')))[as_bitwise_op] >> bitwise_and[as_bitwise_wrap]);
auto bitwise_and_def = equality[as_expr] >> *((x3::string("&") >> !(x3::lit('&') | x3::lit('=')))[as_bitwise_op] >> equality[as_bitwise_wrap]);
auto equality_def = comparison[as_expr] >> *(equality_op[as_binary_op] >> comparison[as_binary_wrap]);
auto comparison_def = bitwise_shift[as_expr] >> *(compare_op[as_binary_op] >> bitwise_shift[as_binary_wrap]);
auto bitwise_shift_def = addition[as_expr] >> *(bitwise_shift_op[as_binary_op] >> addition[as_binary_wrap]);
auto addition_def = multiply[as_expr] >> *(term[as_binary_op] >> multiply[as_binary_wrap]);
auto multiply_def = unarycallwrapper[as_expr] >> *(factor[as_binary_op] >> unarycallwrapper[as_binary_wrap]);
auto unarycallwrapper_def = unary | call;
auto unary_def = (x3::string("-") >> unarycallwrapper);
auto get_def = primary[as_expr] >> *('.' >> identifier)[as_get];
auto call_def = primary[as_expr] >> *((x3::lit('(') >> arguments >> x3::lit(')'))[as_call] | ('.' >> identifier)[as_get]);
auto primary_def = variable;
auto program_def = x3::skip(x3::space)[expression];
BOOST_SPIRIT_DEFINE(primary, assign,
compoundassign, compoundassign_op, bitwise_or, bitwise_xor,
ternary,
bitwise_and, bitwise_shift, bitwise_shift_op,
logical_and, logical_or, equality_op,
equality, factor, compare_op, comparison,
term, addition, multiply, unary, unarycallwrapper,
assignment, get, set, variable, arguments, expression, call, identifier, program);
} // namespace hlsl::parser
int main()
{
using namespace hlsl;
for (std::string const input :
{
"first",
"first.second",
"first.Second.third",
"first.Second().third",
"first.Second(arg1).third",
"first.Second(arg1, arg2).third",
"first = second",
"first.second = third",
"first.second.third = fourth",
"first.second.third = fourth()",
"first.second.third = fourth(arg1)",
"this * that", // binary { var{"this"} "*" var{"that"} }
"this * -that", // binary { var{"this"} "*" unary{'-', var{"that"}} }
"this * that * there",
"this * that / there",
"this.inner * that * there.inner2",
"first + second",
"first + second * third",
"first < second",
"first <= second * third",
"first - second > third",
"first != second",
"first == second * third",
"first || second",
"first || second && third"
"first |= second",
"first |= second.third",
"first & second",
"first & second && third",
"first &= second && third",
"first << second && third",
"first ^ second",
"first ^ second ^^ third", //fails on purpose because this operator doesn't exist!
"zero |= first | second || third",
"first ? second : third",
"first > second ? third : fourth",
"first > second ? third : fourth > fifth ? sixth : seventh"
}) //
{
std::cout << "===== " << quoted(input) << "\n";
auto f = input.begin(), l = input.end();
// Our error handler
auto const p = x3::with<parser::eh_tag>(
x3::error_handler{f, l, std::cerr})[hlsl::parser::program];
if (hlsl::ast::Expr fs; parse(f, l, p, fs))
{
fs.apply_visitor(hlsl::printer{std::cout << "Parsed: "});
std::cout << "\n";
}
else
{
std::cout << "Parse failed at " << quoted(std::string(f, l)) << "\n";
}
}
}