Given the following x3 grammar that parses correctly, I want to add validation of parameters, qualifiers, and properties. This would seem to indicate some method of dynamically switching which symbol table is being used within the various rules. What is the best way to implement this? It would seem to be some mixture of semantic actions and attributes, but it is not clear to me how.
#include <string>
#include <vector>
#include <iostream>
#include <iomanip>
#include <map>
#include <boost/config/warning_disable.hpp>
#include <boost/spirit/home/x3.hpp>
#include <boost/variant.hpp>
#include <boost/fusion/adapted/struct.hpp>
namespace x3 = boost::spirit::x3;
namespace scl
{
//
// This will take a symbol value and return the string associated with that value. From an example by sehe
// TODO: There is probably a better C++14/17 way to do this with the symbol.for_each operator and a lambda,
// but I haven't figured it out yet
//
template <typename T>
struct SYMBOL_LOOKUP
{
SYMBOL_LOOKUP (T Symbol, std::string& String) : _sought (Symbol), _found (String)
{
}
void operator () (std::basic_string <char> s, T ct)
{
if (_sought == ct)
{
_found = s;
}
}
std::string found () const { return _found; }
private:
T _sought;
std::string& _found;
};
//
// This section describes the valid verbs, the parameters that are valid for each verb, and
// the qualifiers that are valid for each verb or parameter of a verb.
// TODO: There is probably some complicated C++11/14/17 expression template for generating all
// of this as a set of linked tables, where each verb points to a parameter table, which points
// to a qualifier table, but that is currently beyond my ability to implement, so each structure
// is implemented discretely
//
//
// Legal verbs
//
enum class VERBS
{
load, //
set, //
show, //
};
struct VALID_VERBS : x3::symbols <VERBS>
{
VALID_VERBS ()
{
add
("load", VERBS::load) //
("set", VERBS::set) //
("show", VERBS::show) //
;
}
} const valid_verbs;
//
// LOAD parameter 1
//
enum class LOAD_PARAMETER1
{
dll, // LOAD DLL <file-spec>
pdb, // LOAD PDB <file-spec>
};
struct VALID_LOAD_PARAMETER1 : x3::symbols <LOAD_PARAMETER1>
{
VALID_LOAD_PARAMETER1 ()
{
add
("dll", LOAD_PARAMETER1::dll) //
("pdb", LOAD_PARAMETER1::pdb) //
;
}
} const valid_load_parameter1;
//
// SET parameter 1
//
enum class SET_PARAMETER1
{
debug, // SET DEBUG {/ON | /OFF}
trace, // SET TRACE {/ON | OFF}
};
struct VALID_SET_PARAMETER1 : x3::symbols <SET_PARAMETER1>
{
VALID_SET_PARAMETER1 ()
{
add
("debug", SET_PARAMETER1::debug) //
("trace", SET_PARAMETER1::trace) //
;
}
} const valid_set_parameter1;
//
// SET qualifiers
//
enum class SET_QUALIFIERS
{
off, //
on //
};
struct VALID_SET_QUALIFIERS : x3::symbols <SET_QUALIFIERS>
{
VALID_SET_QUALIFIERS ()
{
add
("off", SET_QUALIFIERS::off) //
("on", SET_QUALIFIERS::on) //
;
}
} const valid_set_qualifiers;
//
// SHOW parameter 1
//
enum class SHOW_PARAMETER1
{
debug, // SHOW DEBUG
module, // SHOW MODULE <wildcard-expression> [/SYMBOLS]
symbols, // SHOW SYMBOLS *{/ALL /FULL /OUT=<file-spec> /TYPE=(+{all,exports,imports})} [wild-card-expression]
trace, // SHOW TRACE
};
struct VALID_SHOW_PARAMETER1 : x3::symbols <SHOW_PARAMETER1>
{
VALID_SHOW_PARAMETER1 ()
{
add
("debug", SHOW_PARAMETER1::debug) //
("module", SHOW_PARAMETER1::module) //
("symbols", SHOW_PARAMETER1::symbols) //
("trace", SHOW_PARAMETER1::trace) //
;
}
} const valid_show_parameter1;
//
// SHOW qualifiers
//
enum class SHOW_QUALIFIERS
{
all, // Display all objects of the specified type
full, // Display all information about the specified object(s)
out, // Write output to the specified file (/out=<file spec>)
type, // List of properties to display
};
struct VALID_SHOW_QUALIFIERS : x3::symbols <SHOW_QUALIFIERS>
{
VALID_SHOW_QUALIFIERS ()
{
add
("all", SHOW_QUALIFIERS::all) //
("full", SHOW_QUALIFIERS::full) //
("out", SHOW_QUALIFIERS::out) //
("type", SHOW_QUALIFIERS::type) // Valid properties in VALID_SHOW_TYPE_PROPERTIES
;
}
} const valid_show_qualifiers;
//
// SHOW /TYPE=(property_list)
//
enum class SHOW_TYPE_PROPERTIES
{
all, //
exports, //
imports, //
};
struct VALID_SHOW_TYPE_PROPERTIES : x3::symbols <SHOW_TYPE_PROPERTIES>
{
VALID_SHOW_TYPE_PROPERTIES ()
{
add
("all", SHOW_TYPE_PROPERTIES::all) //
("exports", SHOW_TYPE_PROPERTIES::exports) //
("imports", SHOW_TYPE_PROPERTIES::imports) //
;
}
} const valid_show_type_properties;
//
// Convert a verb value to its string representation
//
std::string to_string (const VERBS Verb)
{
std::string result;
SYMBOL_LOOKUP <VERBS> lookup (Verb, result);
//
// Loop through all the entries in the symbol table looking for the specified value
// Is there a better way to use this for_each with a lambda?
//
valid_verbs.for_each (lookup);
return result;
} // End to_string
} // End namespace scl
namespace scl_ast
{
struct KEYWORD : std::string
{
using std::string::string;
using std::string::operator=;
};
struct NIL
{
};
using VALUE = boost::variant <NIL, std::string, int, double, KEYWORD>;
struct PROPERTY
{
KEYWORD name;
VALUE value;
};
struct QUALIFIER
{
enum KIND
{
positive,
negative
} kind;
std::string identifier;
std::vector <PROPERTY> properties;
};
struct PARAMETER
{
KEYWORD keyword;
std::vector <QUALIFIER> qualifiers;
};
struct COMMAND
{
scl::VERBS verb;
std::vector <QUALIFIER> qualifiers;
std::vector <PARAMETER> parameters;
};
//
// Overloads for printing the AST to the console
//
#pragma region debug
static inline std::ostream& operator<< (std::ostream& os, VALUE const& v)
{
struct
{
std::ostream& _os;
void operator() (std::string const& s) const { _os << std::quoted (s); }
void operator() (int i) const { _os << i; }
void operator() (double d) const { _os << d; }
void operator() (KEYWORD const& kwv) const { _os << kwv; }
void operator() (NIL) const { }
} vis { os };
boost::apply_visitor (vis, v);
return os;
}
static inline std::ostream& operator<< (std::ostream& os, PROPERTY const& prop)
{
os << prop.name;
if (prop.value.which ())
{
os << "=" << prop.value;
}
return os;
}
static inline std::ostream& operator<< (std::ostream& os, QUALIFIER const& q)
{
os << "/" << (q.kind == QUALIFIER::negative ? "no" : "") << q.identifier;
if (!q.properties.empty ())
{
os << "=(";
}
for (auto const& prop : q.properties)
{
os << prop << " ";
}
if (!q.properties.empty ())
{
os << ")";
}
return os;
}
static inline std::ostream& operator<< (std::ostream& os, std::vector <QUALIFIER> const& qualifiers)
{
for (auto const& qualifier : qualifiers)
{
os << " " << qualifier;
}
return os;
}
static inline std::ostream& operator<< (std::ostream& os, PARAMETER const& p)
{
return os << p.keyword << " " << p.qualifiers;
}
static inline std::ostream& operator<< (std::ostream& os, COMMAND const& cmd)
{
os << scl::to_string (cmd.verb) << cmd.qualifiers;
for (auto& param : cmd.parameters)
{
os << " " << param;
}
return os;
}
#pragma endregion debug
}; // End namespace scl_ast
BOOST_FUSION_ADAPT_STRUCT (scl_ast::PROPERTY, name, value);
BOOST_FUSION_ADAPT_STRUCT (scl_ast::QUALIFIER, kind, identifier, properties);
BOOST_FUSION_ADAPT_STRUCT (scl_ast::PARAMETER, keyword, qualifiers);
BOOST_FUSION_ADAPT_STRUCT (scl_ast::COMMAND, verb, qualifiers, parameters);
//
// Grammar for simple command language
//
namespace scl
{
using namespace x3;
auto const param = rule <struct _keyword, scl_ast::KEYWORD> { "param" }
= lexeme [+char_ ("a-zA-Z0-9$_.\\*?+-")];
auto const identifier
= lexeme [+char_ ("a-zA-Z0-9_")];
auto const quoted_string
= lexeme ['"' >> *('\\' > char_ | ~char_ ('"')) >> '"'];
auto const property_value
= quoted_string
| real_parser <double, x3::strict_real_policies <double>> {}
| int_
| param;
auto const property = rule <struct _property, scl_ast::PROPERTY> { "property" }
= identifier >> -('=' >> property_value);
auto const property_list = rule <struct _property_list, std::vector <scl_ast::PROPERTY>> { "property_list" }
= '(' >> property % ',' >> ')';
auto const qual
= attr (scl_ast::QUALIFIER::positive) >> lexeme ['/' >> identifier] >> -( '=' >> (property_list | repeat (1) [property]));
auto const neg_qual
= attr (scl_ast::QUALIFIER::negative) >> lexeme [no_case ["/no"] >> identifier] >> repeat (0) [property]; // Negated qualifiers never have properties (repeat(0) keeps the compiler happy)
auto const qualifier
= neg_qual | qual;
auto const verb
= no_case [valid_verbs]; // Uses static list of allowed verbs
auto const parameter = rule <struct _parameter, scl_ast::PARAMETER> { "parameter" }
= param >> *qualifier;
auto const command = rule <struct _command, scl_ast::COMMAND> { "command" }
= skip (blank) [verb >> *qualifier >> *parameter];
}; // End namespace scl
int
main ()
{
std::vector <std::string> input =
{
"load dll test.dll",
"LOAD pdb test.pdb",
"set debug /on",
"show debug",
"SHOW module test.dll/symbols",
"show symbols/type=export test*",
"show symbols test.dll/type=(import,export)",
"show symbols s*/out=s.txt",
"show symbols /all /full",
};
for (auto const& str : input)
{
scl_ast::COMMAND cmd;
auto b = str.begin ();
auto e = str.end ();
bool ok = parse (b, e, scl::command, cmd);
std::cout << (ok ? "OK" : "FAIL") << '\t' << std::quoted (str) << std::endl;
if (ok)
{
std::cout << " -- Full AST: " << cmd << std::endl;
std::cout << " -- Verb + Qualifiers: " << scl::to_string (cmd.verb) << cmd.qualifiers << std::endl;
for (auto const& param : cmd.parameters)
{
std::cout << " -- Parameter + Qualifiers: " << param << std::endl;
}
if (b != e)
{
std::cout << "*** Remaining unparsed: " << std::quoted (std::string (b, e)) << std::endl;
}
}
std::cout << std::endl;
} // End for
return 0;
} // End main
So, I spent quite some time thinking about this.
I admit most of the thoughts didn't escape brainstorm. However, I made a proof-of-concept, from scratch, starting from /just/ the bare minimum:
/* Synopsis:
*
* LOAD DLL <file-spec>
* LOAD PDB <file-spec>
* SET DEBUG {/ON | /OFF}
* SET TRACE {/ON | /OFF}
*
* SHOW DEBUG
* SHOW MODULE <wildcard-expression> [/SYMBOLS]
* SHOW SYMBOLS { [/ALL] [/FULL] [/OUT=<file-spec>] [/TYPE=(+{all,exports,imports})] [wild-card-expression] }...
* SHOW TRACE
*/
Utilities
Since we have several domains that can have sets of options that are to be treated as (case-insensitive) keyword identifiers, I thought of creating a facility for those:
Note: for brevity this keeps all values as int for now. In that, it falls short of "Better Enum". But given a few macros you should be able to make Options::type (and Enum<TagType>) resolve to a proper enum type.
namespace util {
template <typename Tag> struct FlavouredString : std::string {
using std::string::string;
using std::string::operator=;
};
template <typename Tag> struct Options {
using type = int; // TODO typed enums? Requires macro tedium
std::vector<char const*> _options;
Options(std::initializer_list<char const*> options) : _options(options) {}
Options(std::vector<char const*> options) : _options(options) {}
std::string to_string(type id) const { return _options.at(id); }
type to_id(std::string const& name) const { return find(_options.begin(), _options.end(), name) - _options.begin(); }
};
template <typename Tag> using Enum = typename Options<Tag>::type;
template <typename Tag> struct IcOptions : Options<Tag> { using Options<Tag>::Options; };
}
To support our AST types, we will create instances of these utilities like:
IcOptions<struct DllPdb> static const dll_pdb { "DLL", "PDB" };
IcOptions<struct Setting> static const setting { "DEBUG", "TRACE" };
IcOptions<struct OnOff> static const on_off { "OFF", "ON" };
IcOptions<struct SymType> static const sym_type{ "all", "imports", "exports" };
using Wildcard = FlavouredString<struct _Wild>;
using Filespec = FlavouredString<struct _Filespec>;
AST Types
This goes a completely different route from before: instead of defining a general-purpose AST with arbitrary numbers of arbitrary-type arguments and values, I've opted to define the commands strongly-typed:
namespace ast {
struct LoadCommand {
Enum<DllPdb> kind = {};
Filespec filespec;
};
struct SetCommand {
Enum<Setting> setting = {};
Enum<OnOff> value = {};
};
struct ShowSettingCommand {
Enum<Setting> setting;
};
struct ShowModuleCommand {
Wildcard wildcard;
bool symbols = false;
};
using SymbolTypes = std::vector<Enum<SymType> >;
struct ShowSymbolsCommand {
bool all = false;
bool full = false;
Filespec out;
SymbolTypes types;
Wildcard wildcard;
};
using Command = boost::variant<
LoadCommand,
SetCommand,
ShowSettingCommand,
ShowModuleCommand,
ShowSymbolsCommand
>;
}
Adaptation is like before:
BOOST_FUSION_ADAPT_STRUCT(scl::ast::LoadCommand, kind, filespec)
BOOST_FUSION_ADAPT_STRUCT(scl::ast::SetCommand, setting, value)
BOOST_FUSION_ADAPT_STRUCT(scl::ast::ShowSettingCommand, setting)
BOOST_FUSION_ADAPT_STRUCT(scl::ast::ShowModuleCommand, wildcard, symbols)
Note that ShowSymbolsCommand is not adapted because the rule doesn't follow the struct layout
Parser Utilities
Let's support our core concepts with some composable parser factories:
// (case insensitive) keyword handling
static auto kw = [](auto p) { return x3::lexeme[p >> !(x3::graph - x3::char_("/=,()"))]; };
static auto ikw = [](auto p) { return x3::no_case [kw(p)]; };
static auto qualifier = [](auto p) { return x3::lexeme['/' >> ikw(p)]; };
I could explain these, but the usage below will be more clear. So, without further ado, presenting the trick that allows us to use any Options or CiOptions instance directly in a parser expression:
// Options and CiOptions
namespace util {
template <typename Tag>
auto as_spirit_parser(Options<Tag> const& o, bool to_lower = false) {
x3::symbols<typename Options<Tag>::type> p;
int n = 0;
for (std::string el : o._options) {
if (to_lower) boost::to_lower(el);
p.add(el, n++);
}
return kw(p);
}
template <typename Tag>
auto as_spirit_parser(IcOptions<Tag> const& o) {
return x3::no_case [ as_spirit_parser(o, true) ];
}
}
Nothing unexpected there, I suppose, but it does allow for elegant rule definitions:
The Rule Definitions
DEF_RULE(Filespec) = quoted_string | bare_string;
DEF_RULE(Wildcard) = lexeme[+char_("a-zA-Z0-9$_.\\*?+-")];
DEF_RULE(LoadCommand)
= ikw("load") >> ast::dll_pdb >> Filespec;
DEF_RULE(SetCommand)
= ikw("set") >> ast::setting >> qualifier(ast::on_off);
DEF_RULE(ShowSettingCommand)
= ikw("show") >> ast::setting;
DEF_RULE(ShowModuleCommand)
= ikw("show") >> ikw("module") >> Wildcard >> matches[qualifier("symbols")];
// ... ShowSymbolsQualifiers (see below) ...
DEF_RULE(ShowSymbolsCommand)
= ikw("show") >> ikw("symbols") >> *ShowSymbolsQualifiers;
DEF_RULE(Command)
= skip(blank)[ LoadCommand | SetCommand | ShowSettingCommand | ShowModuleCommand | ShowSymbolsCommand ];
The Heavier Lifting
You'll note I skipped ShowSymbolsQualifiers. That's because that's the only rule that cannot benefit from automatic attribute propagation, so I've resorted to using semantic actions:
Note the IIFE idiom allows for "very local" helper definitions
DEF_RULE(SymbolTypes) = [] {
auto type = as_parser(ast::sym_type);
return '(' >> (type % ',') >> ')' | repeat(1) [ type ];
}(); // IIFE pattern
RULE(ShowSymbolsQualifiers, ShowSymbolsCommand)
= [] {
auto set = [](auto member, auto p) {
auto propagate = [member](auto& ctx) {
traits::move_to(_attr(ctx), _val(ctx).*(member));
};
return as_parser(p)[propagate];
};
using T = ast::ShowSymbolsCommand;;
return qualifier("all") >> set(&T::all, attr(true))
| qualifier("full") >> set(&T::full, attr(true))
| qualifier("out") >> set(&T::out, '=' >> Filespec)
| qualifier("type") >> set(&T::types, '=' >> SymbolTypes)
| set(&T::wildcard, Wildcard);
}(); // IIFE pattern
FULL DEMO
Live On Coliru
//#define BOOST_SPIRIT_X3_DEBUG
#include <iomanip>
#include <iostream>
#include <string>
#include <vector>
#include <boost/algorithm/string/case_conv.hpp> // to_lower
#include <boost/fusion/adapted/struct.hpp>
#include <boost/spirit/home/x3.hpp>
/* Synopsis:
*
* LOAD DLL <file-spec>
* LOAD PDB <file-spec>
* SET DEBUG {/ON | /OFF}
* SET TRACE {/ON | /OFF}
*
* SHOW DEBUG
* SHOW MODULE <wildcard-expression> [/SYMBOLS]
* SHOW SYMBOLS { [/ALL] [/FULL] [/OUT=<file-spec>] [/TYPE=(+{all,exports,imports})] [wild-card-expression] }...
* SHOW TRACE
*/
namespace scl {
namespace util {
template <typename Tag> struct FlavouredString : std::string {
using std::string::string;
using std::string::operator=;
};
template <typename Tag> struct Options {
using type = int; // TODO typed enums? Requires macro tedium
std::vector<char const*> _options;
Options(std::initializer_list<char const*> options) : _options(options) {}
Options(std::vector<char const*> options) : _options(options) {}
std::string to_string(type id) const { return _options.at(id); }
type to_id(std::string const& name) const { return find(_options.begin(), _options.end(), name) - _options.begin(); }
};
template <typename Tag> using Enum = typename Options<Tag>::type;
template <typename Tag> struct IcOptions : Options<Tag> { using Options<Tag>::Options; };
}
namespace ast {
using namespace util;
IcOptions<struct DllPdb> static const dll_pdb { "DLL", "PDB" };
IcOptions<struct Setting> static const setting { "DEBUG", "TRACE" };
IcOptions<struct OnOff> static const on_off { "OFF", "ON" };
IcOptions<struct SymType> static const sym_type{ "all", "imports", "exports" };
using Wildcard = FlavouredString<struct _Wild>;
using Filespec = FlavouredString<struct _Filespec>;
struct LoadCommand {
Enum<DllPdb> kind = {};
Filespec filespec;
};
struct SetCommand {
Enum<Setting> setting = {};
Enum<OnOff> value = {};
};
struct ShowSettingCommand {
Enum<Setting> setting;
};
struct ShowModuleCommand {
Wildcard wildcard;
bool symbols = false;
};
using SymbolTypes = std::vector<Enum<SymType> >;
struct ShowSymbolsCommand {
bool all = false;
bool full = false;
Filespec out;
SymbolTypes types;
Wildcard wildcard;
};
using Command = boost::variant<
LoadCommand,
SetCommand,
ShowSettingCommand,
ShowModuleCommand,
ShowSymbolsCommand
>;
}
}
#ifndef NDEBUG // for debug printing
namespace scl { namespace ast {
static inline std::ostream &operator<<(std::ostream &os, Wildcard const &w) { return os << std::quoted(w); }
static inline std::ostream &operator<<(std::ostream &os, Filespec const &s) { return os << std::quoted(s); }
static inline std::ostream &operator<<(std::ostream &os, LoadCommand const &cmd) {
return os << "LOAD " << dll_pdb.to_string(cmd.kind) << " " << cmd.filespec ;
}
static inline std::ostream &operator<<(std::ostream &os, SetCommand const &cmd) {
return os << "SET " << setting.to_string(cmd.setting) << " /" << on_off.to_string(cmd.value);
}
static inline std::ostream &operator<<(std::ostream &os, ShowSettingCommand const &cmd) {
return os << "SHOW " << setting.to_string(cmd.setting);
}
static inline std::ostream &operator<<(std::ostream &os, ShowModuleCommand const &cmd) {
return os << "SHOW MODULE " << cmd.wildcard << (cmd.symbols?" /SYMBOLS":"");
}
static inline std::ostream &operator<<(std::ostream &os, ShowSymbolsCommand const &cmd) {
os << "SHOW SYMBOLS";
if (cmd.all) os << " /ALL";
if (cmd.full) os << " /FULL";
if (cmd.out.size()) os << " /OUT=" << cmd.out;
if (cmd.types.size()) {
os << " /TYPE=(";
bool first = true;
for (auto type : cmd.types)
os << (std::exchange(first, false)?"":",") << sym_type.to_string(type);
os << ")";
}
return os << " " << cmd.wildcard;
}
} }
#endif
BOOST_FUSION_ADAPT_STRUCT(scl::ast::LoadCommand, kind, filespec)
BOOST_FUSION_ADAPT_STRUCT(scl::ast::SetCommand, setting, value)
BOOST_FUSION_ADAPT_STRUCT(scl::ast::ShowSettingCommand, setting)
BOOST_FUSION_ADAPT_STRUCT(scl::ast::ShowModuleCommand, wildcard, symbols)
// Grammar for simple command language
namespace scl {
namespace x3 = boost::spirit::x3;
// (case insensitive) keyword handling
static auto kw = [](auto p) { return x3::lexeme[p >> !(x3::graph - x3::char_("/=,()"))]; };
static auto ikw = [](auto p) { return x3::no_case [kw(p)]; };
static auto qualifier = [](auto p) { return x3::lexeme['/' >> ikw(p)]; };
// Options and CiOptions
namespace util {
template <typename Tag>
auto as_spirit_parser(Options<Tag> const& o, bool to_lower = false) {
x3::symbols<typename Options<Tag>::type> p;
int n = 0;
for (std::string el : o._options) {
if (to_lower) boost::to_lower(el);
p.add(el, n++);
}
return kw(p);
}
template <typename Tag>
auto as_spirit_parser(IcOptions<Tag> const& o) {
return x3::no_case [ as_spirit_parser(o, true) ];
}
}
// shorthand rule declarations
#define RULE(name, Attr) static auto const name = x3::rule<struct _##Attr, ast::Attr>{#Attr}
#define DEF_RULE(Attr) RULE(Attr, Attr)
using namespace x3;
auto const bare_string
= lexeme[+char_("a-zA-Z0-9$_.\\*?+-")]; // bare string taken from old "param" rule
auto const quoted_string
= lexeme['"' >> *(('\\' > char_) | ~char_('"')) >> '"'];
DEF_RULE(Filespec) = quoted_string | bare_string;
DEF_RULE(Wildcard) = lexeme[+char_("a-zA-Z0-9$_.\\*?+-")];
DEF_RULE(LoadCommand)
= ikw("load") >> ast::dll_pdb >> Filespec;
DEF_RULE(SetCommand)
= ikw("set") >> ast::setting >> qualifier(ast::on_off);
DEF_RULE(ShowSettingCommand)
= ikw("show") >> ast::setting;
DEF_RULE(ShowModuleCommand)
= ikw("show") >> ikw("module") >> Wildcard >> matches[qualifier("symbols")];
// Note the IIFE idiom allows for "very local" helper definitions
DEF_RULE(SymbolTypes) = [] {
auto type = as_parser(ast::sym_type);
return '(' >> (type % ',') >> ')' | repeat(1) [ type ];
}(); // IIFE idiom
RULE(ShowSymbolsQualifiers, ShowSymbolsCommand)
= [] {
auto set = [](auto member, auto p) {
auto propagate = [member](auto& ctx) {
traits::move_to(_attr(ctx), _val(ctx).*(member));
};
return as_parser(p)[propagate];
};
using T = ast::ShowSymbolsCommand;;
return qualifier("all") >> set(&T::all, attr(true))
| qualifier("full") >> set(&T::full, attr(true))
| qualifier("out") >> set(&T::out, '=' >> Filespec)
| qualifier("type") >> set(&T::types, '=' >> SymbolTypes)
| set(&T::wildcard, Wildcard);
}(); // IIFE idiom
DEF_RULE(ShowSymbolsCommand)
= ikw("show") >> ikw("symbols") >> *ShowSymbolsQualifiers;
DEF_RULE(Command)
= skip(blank)[ LoadCommand | SetCommand | ShowSettingCommand | ShowModuleCommand | ShowSymbolsCommand ];
#undef DEF_RULE
#undef RULE
} // End namespace scl
int main() {
for (std::string const str : {
"load dll test.dll",
"LOAD pdb \"test special.pdb\"",
"LOAD pDb test.pdb",
"set debug /on",
"show debug",
"SHOW module test.dll/symbols",
"SHOW MODULE TEST.DLL /SYMBOLS",
"SHOW module test.dll / symbols",
"SHOW module test.dll",
"show symbols/type=exports test*",
"show symbols/type=(exports,imports) test*",
"show symbols test.dll/type=(imports,exports)",
"show symbols test.dll/tyPE=(imports,exports)",
"show symbols s*/out=s.txt",
"show symbols /all /full",
}) {
std::cout << " ======== " << std::quoted(str) << std::endl;
auto b = str.begin(), e = str.end();
scl::ast::Command cmd;
if (parse(b, e, scl::Command, cmd))
std::cout << " - Parsed: " << cmd << std::endl;
if (b != e)
std::cout << " - Remaining unparsed: " << std::quoted(std::string(b, e)) << std::endl;
}
}
Prints
======== "load dll test.dll"
- Parsed: LOAD DLL "test.dll"
======== "LOAD pdb \"test special.pdb\""
- Parsed: LOAD PDB "test special.pdb"
======== "LOAD pDb test.pdb"
- Parsed: LOAD PDB "test.pdb"
======== "set debug /on"
- Parsed: SET DEBUG /ON
======== "show debug"
- Parsed: SHOW DEBUG
======== "SHOW module test.dll/symbols"
- Parsed: SHOW MODULE "test.dll" /SYMBOLS
======== "SHOW MODULE TEST.DLL /SYMBOLS"
- Parsed: SHOW MODULE "TEST.DLL" /SYMBOLS
======== "SHOW module test.dll / symbols"
- Parsed: SHOW MODULE "test.dll"
- Remaining unparsed: "/ symbols"
======== "SHOW module test.dll"
- Parsed: SHOW MODULE "test.dll"
======== "show symbols/type=exports test*"
- Parsed: SHOW SYMBOLS /TYPE=(exports) "test*"
======== "show symbols/type=(exports,imports) test*"
- Parsed: SHOW SYMBOLS /TYPE=(exports,imports) "test*"
======== "show symbols test.dll/type=(imports,exports)"
- Parsed: SHOW SYMBOLS /TYPE=(imports,exports) "test.dll"
======== "show symbols test.dll/tyPE=(imports,exports)"
- Parsed: SHOW SYMBOLS /TYPE=(imports,exports) "test.dll"
======== "show symbols s*/out=s.txt"
- Parsed: SHOW SYMBOLS /OUT="s.txt" "s*"
======== "show symbols /all /full"
- Parsed: SHOW SYMBOLS /ALL /FULL ""
Related
The examples in the Boost.Spirit documentation seem to fall in two cases:
1/ Define a parser in a function: semantic actions can access local variables and data as they are local lambdas. Like push_back here: https://www.boost.org/doc/libs/master/libs/spirit/doc/x3/html/spirit_x3/tutorials/number_list___stuffing_numbers_into_a_std__vector.html
2/ Define a parser in a namespace, like here: https://www.boost.org/doc/libs/1_69_0/libs/spirit/doc/x3/html/spirit_x3/tutorials/minimal.html
which seems to be necessary to be able to invoke BOOST_SPIRIT_DEFINE.
My question is: how to combine both (properly, without globals) ? My dream API would be to pass some argument to phrase_parse and then do some x3::_arg(ctx) but I couldn't find anything like this.
Here is for instance my parser: for now the actions are writing to std::cerr. What if I wanted to write to a custom std::ostream& instead, that would be passed to the parse function?
using namespace boost::spirit;
using namespace boost::spirit::x3;
rule<struct id_action> action = "action";
rule<struct id_array> array = "array";
rule<struct id_empty_array> empty_array = "empty_array";
rule<struct id_atom> atom = "atom";
rule<struct id_sequence> sequence = "sequence";
rule<struct id_root> root = "root";
auto access_index_array = [] (const auto& ctx) { std::cerr << "access_array: " << x3::_attr(ctx) << "\n" ;};
auto access_empty_array = [] (const auto& ctx) { std::cerr << "access_empty_array\n" ;};
auto access_named_member = [] (const auto& ctx) { std::cerr << "access_named_member: " << x3::_attr(ctx) << "\n" ;};
auto start_action = [] (const auto& ctx) { std::cerr << "start action\n" ;};
auto finish_action = [] (const auto& ctx) { std::cerr << "finish action\n" ;};
auto create_array = [] (const auto& ctx) { std::cerr << "create_array\n" ;};
const auto action_def = +(lit('.')[start_action]
>> -((+alnum)[access_named_member])
>> *(('[' >> x3::int_ >> ']')[access_index_array] | lit("[]")[access_empty_array]));
const auto sequence_def = (action[finish_action] % '|');
const auto array_def = ('[' >> sequence >> ']')[create_array];
const auto root_def = array | action;
BOOST_SPIRIT_DEFINE(action)
BOOST_SPIRIT_DEFINE(array)
BOOST_SPIRIT_DEFINE(sequence)
BOOST_SPIRIT_DEFINE(root)
bool parse(std::string_view str)
{
using ascii::space;
auto first = str.begin();
auto last = str.end();
bool r = phrase_parse(
first, last,
parser::array_def | parser::sequence_def,
ascii::space
);
if (first != last)
return false;
return r;
}
About the approaches:
1/ Yes, this is viable for small, contained parsers. Typically only used in a single TU, and exposed via non-generic interface.
2/ This is the approach for (much) larger grammars, that you might wish to spread across TUs, and/or are instantiated across several TU's generically.
Note that you do NOT need BOOST_SPIRIT_DEFINE unless you
have recursive rules
want to split declaration from definition. [This becomes pretty complicated, and I recommend against using that for X3.]
The Question
My question is: how to combine both (properly, without globals) ?
You can't combine something with namespace level declarations, if one of the requiremenents is "without globals".
My dream API would be to pass some argument to phrase_parse and then do some x3::_arg(ctx) but I couldn't find anything like this.
I don't know what you think x3::_arg(ctx) would do, in that particular dream :)
Here is for instance my parser: for now the actions are writing to std::cerr. What if I wanted to write to a custom std::ostream& instead, that would be passed to the parse function?
Now that's a concrete question. I'd say: use the context.
You could make it so that you can use x3::get<ostream>(ctx) returns the stream:
struct ostream{};
auto access_index_array = [] (const auto& ctx) { x3::get<ostream>(ctx) << "access_array: " << x3::_attr(ctx) << "\n" ;};
auto access_empty_array = [] (const auto& ctx) { x3::get<ostream>(ctx) << "access_empty_array\n" ;};
auto access_named_member = [] (const auto& ctx) { x3::get<ostream>(ctx) << "access_named_member: " << x3::_attr(ctx) << "\n" ;};
auto start_action = [] (const auto& ctx) { x3::get<ostream>(ctx) << "start action\n" ;};
auto finish_action = [] (const auto& ctx) { x3::get<ostream>(ctx) << "finish action\n" ;};
auto create_array = [] (const auto& ctx) { x3::get<ostream>(ctx) << "create_array\n";};
Now you need to put the tagged param in the context during parsing:
bool r = phrase_parse(
f, l,
x3::with<parser::ostream>(std::cerr)[parser::array_def | parser::sequence_def],
x3::space);
Live Demo: http://coliru.stacked-crooked.com/a/a26c8eb0af6370b9
Prints
start action
access_named_member: a
finish action
start action
access_named_member: b
start action
start action
access_array: 2
start action
access_named_member: foo
start action
access_empty_array
finish action
start action
access_named_member: c
finish action
create_array
true
Intermixed with the standard X3 debug output:
<sequence>
<try>.a|.b..[2].foo.[]|.c</try>
<action>
<try>.a|.b..[2].foo.[]|.c</try>
<success>|.b..[2].foo.[]|.c]</success>
</action>
<action>
<try>.b..[2].foo.[]|.c]</try>
<success>|.c]</success>
</action>
<action>
<try>.c]</try>
<success>]</success>
</action>
<success>]</success>
</sequence>
But Wait #1 - Event Handlers
It looks like you're parsing something similar to JSON Pointer or jq syntax. In the case that you wanted to provide a callback-interface (SAX-events), why not bind the callback interface instead of the actions:
struct handlers {
using N = x3::unused_type;
virtual void index(int) {}
virtual void index(N) {}
virtual void property(std::string) {}
virtual void start(N) {}
virtual void finish(N) {}
virtual void create_array(N) {}
};
#define EVENT(e) ([](auto& ctx) { x3::get<handlers>(ctx).e(x3::_attr(ctx)); })
const auto action_def =
+(x3::lit('.')[EVENT(start)] >> -((+x3::alnum)[EVENT(property)]) >>
*(('[' >> x3::int_ >> ']')[EVENT(index)] | x3::lit("[]")[EVENT(index)]));
const auto sequence_def = action[EVENT(finish)] % '|';
const auto array_def = ('[' >> sequence >> ']')[EVENT(create_array)];
const auto root_def = array | action;
Now you can implement all handlers neatly in one interface:
struct default_handlers : parser::handlers {
std::ostream& os;
default_handlers(std::ostream& os) : os(os) {}
void index(int i) override { os << "access_array: " << i << "\n"; };
void index(N) override { os << "access_empty_array\n" ; };
void property(std::string n) override { os << "access_named_member: " << n << "\n" ; };
void start(N) override { os << "start action\n" ; };
void finish(N) override { os << "finish action\n" ; };
void create_array(N) override { os << "create_array\n"; };
};
auto f = str.begin(), l = str.end();
bool r = phrase_parse(f, l,
x3::with<parser::handlers>(default_handlers{std::cout}) //
[parser::array_def | parser::sequence_def],
x3::space);
See it Live On Coliru once again:
start action
access_named_member: a
finish action
start action
access_named_member: b
start action
start action
access_array: 2
start action
access_named_member: foo
start action
access_empty_array
finish action
start action
access_named_member: c
finish action
create_array
true
But Wait #2 - No Actions
The natural way to expose attributes would be to build an AST. See also Boost Spirit: "Semantic actions are evil"?
Without further ado:
namespace AST {
using Id = std::string;
using Index = int;
struct Member {
std::optional<Id> name;
};
struct Indexer {
std::optional<int> index;
};
struct Action {
Member member;
std::vector<Indexer> indexers;
};
using Actions = std::vector<Action>;
using Sequence = std::vector<Actions>;
struct ArrayCtor {
Sequence actions;
};
using Root = boost::variant<ArrayCtor, Actions>;
}
Of course, I'm making some assumptions. The rules can be much simplified:
namespace parser {
template <typename> struct Tag {};
#define AS(T, p) (x3::rule<Tag<AST::T>, AST::T>{#T} = p)
auto id = AS(Id, +x3::alnum);
auto member = AS(Member, x3::lit('.') >> -id);
auto indexer = AS(Indexer,'[' >> -x3::int_ >> ']');
auto action = AS(Action, member >> *indexer);
auto actions = AS(Actions, +action);
auto sequence = AS(Sequence, actions % '|');
auto array = AS(ArrayCtor, '[' >> -sequence >> ']'); // covers empty array
auto root = AS(Root, array | actions);
} // namespace parser
And the parsing function returns the AST:
AST::Root parse(std::string_view str) {
auto f = str.begin(), l = str.end();
AST::Root parsed;
phrase_parse(f, l, x3::expect[parser::root >> x3::eoi], x3::space, parsed);
return parsed;
}
(Note that it now throws x3::expection_failure if the input is invalid or not completely parsed)
int main() {
std::cout << parse("[.a|.b..[2].foo.[]|.c]");
}
Now prints:
[.a|.b./*none*/./*none*/[2].foo./*none*/[/*none*/]|.c]
See it Live On Coliru
//#define BOOST_SPIRIT_X3_DEBUG
#include <boost/fusion/adapted.hpp>
#include <boost/spirit/home/x3.hpp>
#include <ostream>
#include <optional>
namespace x3 = boost::spirit::x3;
namespace AST {
using Id = std::string;
using Index = int;
struct Member {
std::optional<Id> name;
};
struct Indexer {
std::optional<int> index;
};
struct Action {
Member member;
std::vector<Indexer> indexers;
};
using Actions = std::vector<Action>;
using Sequence = std::vector<Actions>;
struct ArrayCtor {
Sequence actions;
};
using Root = boost::variant<ArrayCtor, Actions>;
}
BOOST_FUSION_ADAPT_STRUCT(AST::Member, name)
BOOST_FUSION_ADAPT_STRUCT(AST::Indexer, index)
BOOST_FUSION_ADAPT_STRUCT(AST::Action, member, indexers)
BOOST_FUSION_ADAPT_STRUCT(AST::ArrayCtor, actions)
namespace parser {
template <typename> struct Tag {};
#define AS(T, p) (x3::rule<Tag<AST::T>, AST::T>{#T} = p)
auto id = AS(Id, +x3::alnum);
auto member = AS(Member, x3::lit('.') >> -id);
auto indexer = AS(Indexer,'[' >> -x3::int_ >> ']');
auto action = AS(Action, member >> *indexer);
auto actions = AS(Actions, +action);
auto sequence = AS(Sequence, actions % '|');
auto array = AS(ArrayCtor, '[' >> -sequence >> ']'); // covers empty array
auto root = AS(Root, array | actions);
} // namespace parser
AST::Root parse(std::string_view str) {
auto f = str.begin(), l = str.end();
AST::Root parsed;
phrase_parse(f, l, x3::expect[parser::root >> x3::eoi], x3::space, parsed);
return parsed;
}
// for debug output
#include <iostream>
#include <iomanip>
namespace AST {
static std::ostream& operator<<(std::ostream& os, Member const& m) {
return os << "." << m.name.value_or("/*none*/");
}
static std::ostream& operator<<(std::ostream& os, Indexer const& i) {
if (i.index)
return os << "[" << *i.index << "]";
else
return os << "[/*none*/]";
}
static std::ostream& operator<<(std::ostream& os, Action const& a) {
os << a.member;
for (auto& i : a.indexers)
os << i;
return os;
}
static std::ostream& operator<<(std::ostream& os, Actions const& aa) {
for (auto& a : aa)
os << a;
return os;
}
static std::ostream& operator<<(std::ostream& os, Sequence const& s) {
bool first = true;
for (auto& a : s)
os << (std::exchange(first, false) ? "" : "|") << a;
return os;
}
static std::ostream& operator<<(std::ostream& os, ArrayCtor const& ac) {
return os << "[" << ac.actions << "]";
}
}
int main() {
std::cout << parse("[.a|.b..[2].foo.[]|.c]");
}
I'd like to make a keyword parser that matches i.e. int, but does not match int in integer with eger left over. I use x3::symbols to get automatically get the parsed keyword represented as an enum value.
Minimal example:
#include <boost/spirit/home/x3.hpp>
#include <boost/spirit/home/x3/support/utility/error_reporting.hpp>
namespace x3 = boost::spirit::x3;
enum class TypeKeyword { Int, Float, Bool };
struct TypeKeywordSymbolTable : x3::symbols<TypeKeyword> {
TypeKeywordSymbolTable()
{
add("float", TypeKeyword::Float)
("int", TypeKeyword::Int)
("bool", TypeKeyword::Bool);
}
};
const TypeKeywordSymbolTable type_keyword_symbol_table;
struct TypeKeywordRID {};
using TypeKeywordRule = x3::rule<TypeKeywordRID, TypeKeyword>;
const TypeKeywordRule type_keyword = "type_keyword";
const auto type_keyword_def = type_keyword_symbol_table;
BOOST_SPIRIT_DEFINE(type_keyword);
using Iterator = std::string_view::const_iterator;
/* Thrown when the parser has failed to parse the whole input stream. Contains
* the part of the input stream that has not been parsed. */
class LeftoverError : public std::runtime_error {
public:
LeftoverError(Iterator begin, Iterator end)
: std::runtime_error(std::string(begin, end))
{}
std::string_view get_leftover_data() const noexcept { return what(); }
};
template<typename Rule>
typename Rule::attribute_type parse(std::string_view input, const Rule& rule)
{
Iterator begin = input.begin();
Iterator end = input.end();
using ExpectationFailure = boost::spirit::x3::expectation_failure<Iterator>;
typename Rule::attribute_type result;
try {
bool r = x3::phrase_parse(begin, end, rule, x3::space, result);
if (r && begin == end) {
return result;
} else { // Occurs when the whole input stream has not been consumed.
throw LeftoverError(begin, end);
}
} catch (const ExpectationFailure& exc) {
throw LeftoverError(exc.where(), end);
}
}
int main()
{
// TypeKeyword::Bool is parsed and "ean" is leftover, but failed parse with
// "boolean" leftover is desired.
parse("boolean", type_keyword);
// TypeKeyword::Int is parsed and "eger" is leftover, but failed parse with
// "integer" leftover is desired.
parse("integer", type_keyword);
// TypeKeyword::Int is parsed successfully and this is the desired behavior.
parse("int", type_keyword);
}
Basicly, I want integer not to be recognized as a keyword with additional eger left to parse.
I morphed the test cases into self-describing expectations:
Live On Compiler Explorer
Prints:
FAIL "boolean" -> TypeKeyword::Bool (expected Leftover:"boolean")
FAIL "integer" -> TypeKeyword::Int (expected Leftover:"integer")
OK "int" -> TypeKeyword::Int
Now, the simplest, naive approach would be to make sure you parse till the eoi, by simply changing
auto actual = parse(input, Parser::type_keyword);
To
auto actual = parse(input, Parser::type_keyword >> x3::eoi);
And indeed the tests pass: Live
OK "boolean" -> Leftover:"boolean"
OK "integer" -> Leftover:"integer"
OK "int" -> TypeKeyword::Int
However, this fits the tests, but not the goal. Let's imagine a more involved grammar, where type identifier; is to be parsed:
auto identifier
= x3::rule<struct id_, Ast::Identifier> {"identifier"}
= x3::lexeme[x3::char_("a-zA-Z_") >> *x3::char_("a-zA-Z_0-9")];
auto type_keyword
= x3::rule<struct tk_, Ast::TypeKeyword> {"type_keyword"}
= type_;
auto declaration
= x3::rule<struct decl_, Ast::Declaration>{"declaration"}
= type_keyword >> identifier >> ';';
I'll leave the details for Compiler Explorer:
OK "boolean b;" -> Leftover:"boolean b;"
OK "integer i;" -> Leftover:"integer i;"
OK "int j;" -> (TypeKeyword::Int j)
Looks good. But what if we add some interesting tests:
{"flo at f;", LeftoverError("flo at f;")},
{"boolean;", LeftoverError("boolean;")},
It prints (Live)
OK "boolean b;" -> Leftover:"boolean b;"
OK "integer i;" -> Leftover:"integer i;"
OK "int j;" -> (TypeKeyword::Int j)
FAIL "boolean;" -> (TypeKeyword::Bool ean) (expected Leftover:"boolean;")
So, the test cases were lacking. Your prose description is actually closer:
I'd like to make a keyword parser that matches i.e. int, but does not match int in integer with eger left over
That correctly implies you want to check the lexeme inside the type_keyword rule. A naive try might be checking that no identifier character follows the type keyword:
auto type_keyword
= x3::rule<struct tk_, Ast::TypeKeyword> {"type_keyword"}
= type_ >> !identchar;
Where identchar was factored out of identifier like so:
auto identchar = x3::char_("a-zA-Z_0-9");
auto identifier
= x3::rule<struct id_, Ast::Identifier> {"identifier"}
= x3::lexeme[x3::char_("a-zA-Z_") >> *identchar];
However, this doesn't work. Can you see why (peeking allowed: https://godbolt.org/z/jb4zfhfWb)?
Our latest devious test case now passes (yay), but int j; is now rejected! If you think about it, it only makes sense, because you have spaced skipped.
The essential word I used a moment ago was lexeme: you want to treat some units as lexemes (and whitespace stops the lexeme. Or rather, whitespace isn't automatically skipped inside the lexeme¹). So, a fix would be:
auto type_keyword
// ...
= x3::lexeme[type_ >> !identchar];
(Note how I sneakily already included that on the identifier rule earlier)
Lo and behold (Live):
OK "boolean b;" -> Leftover:"boolean b;"
OK "integer i;" -> Leftover:"integer i;"
OK "int j;" -> (TypeKeyword::Int j)
OK "boolean;" -> Leftover:"boolean;"
Summarizing
This topic is a frequently recurring one, and it requires a solid understanding of skippers, lexemes first and foremost. Here are some other posts for inspiration:
Stop X3 symbols from matching substrings
parsing identifiers except keywords
Boost Spirit x3: parse delimited string Where I introduce a more general helper you might find useful:
auto kw = [](auto p) {
return x3::lexeme [ x3::as_parser(p) >> !x3::char_("a-zA-Z0-9_") ];
};
Stop X3 symbols from matching substrings
Dynamically switching symbol tables in x3
Good luck!
Complete Listing
Anti-Bitrot, the final listing:
#include <boost/fusion/adapted.hpp>
#include <boost/fusion/include/as_vector.hpp>
#include <boost/fusion/include/io.hpp>
#include <boost/lexical_cast.hpp>
#include <boost/spirit/home/x3.hpp>
#include <iomanip>
#include <iostream>
namespace x3 = boost::spirit::x3;
namespace Ast {
enum class TypeKeyword { Int, Float, Bool };
static std::ostream& operator<<(std::ostream& os, TypeKeyword tk) {
switch (tk) {
case TypeKeyword::Int: return os << "TypeKeyword::Int";
case TypeKeyword::Float: return os << "TypeKeyword::Float";
case TypeKeyword::Bool: return os << "TypeKeyword::Bool";
};
return os << "?";
}
using Identifier = std::string;
struct Declaration {
TypeKeyword type;
Identifier id;
bool operator==(Declaration const&) const = default;
};
} // namespace Ast
BOOST_FUSION_ADAPT_STRUCT(Ast::Declaration, type, id)
namespace Ast{
static std::ostream& operator<<(std::ostream& os, Ast::Declaration const& d) {
return os << boost::lexical_cast<std::string>(boost::fusion::as_vector(d));
}
} // namespace Ast
namespace Parser {
struct Type : x3::symbols<Ast::TypeKeyword> {
Type() {
add("float", Ast::TypeKeyword::Float) //
("int", Ast::TypeKeyword::Int) //
("bool", Ast::TypeKeyword::Bool); //
}
} const static type_;
auto identchar = x3::char_("a-zA-Z_0-9");
auto identifier
= x3::rule<struct id_, Ast::Identifier> {"identifier"}
= x3::lexeme[x3::char_("a-zA-Z_") >> *identchar];
auto type_keyword
= x3::rule<struct tk_, Ast::TypeKeyword> {"type_keyword"}
= x3::lexeme[type_ >> !identchar];
auto declaration
= x3::rule<struct decl_, Ast::Declaration>{"declaration"}
= type_keyword >> identifier >> ';';
} // namespace Parser
struct LeftoverError : std::runtime_error {
using std::runtime_error::runtime_error;
friend std::ostream& operator<<(std::ostream& os, LeftoverError const& e) {
return os << (std::string("Leftover:\"") + e.what() + "\"");
}
bool operator==(LeftoverError const& other) const {
return std::string_view(what()) == other.what();
}
};
template<typename T>
using Maybe = boost::variant<T, LeftoverError>;
template <typename Rule,
typename Attr = typename x3::traits::attribute_of<Rule, x3::unused_type>::type,
typename R = Maybe<Attr>>
R parse(std::string_view input, Rule const& rule) {
Attr result;
auto f = input.begin(), l = input.end();
return x3::phrase_parse(f, l, rule, x3::space, result)
? R(std::move(result))
: LeftoverError({f, l});
}
int main()
{
using namespace Ast;
struct {
std::string_view input;
Maybe<Declaration> expected;
} cases[] = {
{"boolean b;", LeftoverError("boolean b;")},
{"integer i;", LeftoverError("integer i;")},
{"int j;", Declaration{TypeKeyword::Int, "j"}},
{"boolean;", LeftoverError("boolean;")},
};
for (auto [input, expected] : cases) {
auto actual = parse(input, Parser::declaration >> x3::eoi);
bool ok = expected == actual;
std::cout << std::left << std::setw(6) << (ok ? "OK" : "FAIL")
<< std::setw(12) << std::quoted(input) << " -> "
<< std::setw(20) << actual;
if (not ok)
std::cout << " (expected " << expected << ")";
std::cout << "\n";
}
}
¹ see Boost spirit skipper issues
What is the algorithm for developing a string parser to create a geometry? The geometry is generated in 2 steps: at the first step, we create primitives; at the second, we combine primitives into objects.
The syntax is presented in the string below.
string str="[GEOMETRY]
PRIMITIVE1=SPHERE(RADIUS=5.5);
PRIMITIVE2=BOX(A=-5.2, B=7.3);
//...
OBJECT1=PRIMITIVE2*(-PRIMITIVE1);
//..."
class PRIMITIVE{
int number;
public:
Primitive& operator+ (Primitive& primitive) {}; //overloading arithmetic operations
Primitive& operator* (Primitive& primitive) {};
Primitive& operator- (Primitive& primitive) {};
virtual bool check_in_point_inside_primitive = 0;
};
class SPHERE:public PRIMITIVE{
double m_radius;
public:
SPHERE(double radius): m_radius(radius) {}; //In which part of the parser to create objects?
bool check_in_point_inside_sphere(Point& point){};
};
class BOX:public PRIMITIVE{
double m_A;
double m_B;
public:
BOX(double A, double B): m_A(A), m_B(B) {};
bool check_in_point_inside_box(Point& point){};
};
class OBJECT{
int number;
PRIMITIVE& primitive;
public:
OBJECT(){};
bool check_in_point_inside_object(Primitive& PRIMITIVE1, Primitive& PRIMITIVE2, Point& point){
//>How to construct a function from an expression 'PRIMITIVE2*(-PRIMITIVE1)' when parsing?
}
};
How to analyze the string PRIMITIVE1=SPHERE(RADIUS=5.5) and pass a parameter to the constructor of SPHERE()? How to identify this object with the name PRIMITIVE 1 to call to it in OBJECT? Is it possible to create a pair<PRIMITIVE1,SPHERE(5.5)> and store all primitives in map?
How to parse the string of the OBJECT1 and to construct a function from an expression PRIMITIVE2*(-PRIMITIVE1) inside an OBJECT1? This expression will be required multiple times when determining the position of each point relative to the object.
How to use boost::spirit for this task? Tokenize a string using boost::spirit::lex, and then develop rules using boost::spirit::qi?
As a finger exercise, and despite the serious problems I see with the chosen virtual type hierarchy, let's try to make a value-oriented container of Primitives that can be indexed by their id (ById):
Live On Coliru
#include <boost/intrusive/set.hpp>
#include <boost/poly_collection/base_collection.hpp>
#include <iostream>
namespace bi = boost::intrusive;
struct Point {
};
using IndexHook = bi::set_member_hook<bi::link_mode<bi::auto_unlink>>;
class Primitive {
int _id;
public:
struct ById {
bool operator()(auto const&... oper) const { return std::less<>{}(access(oper)...); }
private:
static int access(int id) { return id; }
static int access(Primitive const& p) { return p._id; }
};
IndexHook _index;
Primitive(int id) : _id(id) {}
virtual ~Primitive() = default;
int id() const { return _id; }
Primitive& operator+= (Primitive const& primitive) { return *this; } //overloading arithmetic operations
Primitive& operator*= (Primitive const& primitive) { return *this; }
Primitive& operator-= (Primitive const& primitive) { return *this; }
virtual bool check_in_point_inside(Point const&) const = 0;
};
using Index =
bi::set<Primitive, bi::constant_time_size<false>,
bi::compare<Primitive::ById>,
bi::member_hook<Primitive, IndexHook, &Primitive::_index>>;
class Sphere : public Primitive {
double _radius;
public:
Sphere(int id, double radius)
: Primitive(id)
, _radius(radius) {} // In which part of the parser to create objects?
bool check_in_point_inside(Point const& point) const override { return false; }
};
class Box : public Primitive {
double _A;
double _B;
public:
Box(int id, double A, double B) : Primitive(id), _A(A), _B(B) {}
bool check_in_point_inside(Point const& point) const override { return false; }
};
class Object{
int _id;
Primitive& _primitive;
public:
Object(int id, Primitive& p) : _id(id), _primitive(p) {}
bool check_in_point_inside_object(Primitive const& p1, Primitive const& p2,
Point const& point) const
{
//>How to construct a function from an expression
//'PRIMITIVE2*(-PRIMITIVE1)' when parsing?
return false;
}
};
using Primitives = boost::poly_collection::base_collection<Primitive>;
int main() {
Primitives test;
test.insert(Sphere{2, 4.0});
test.insert(Sphere{4, 4.0});
test.insert(Box{2, 5, 6});
test.insert(Sphere{1, 4.0});
test.insert(Box{3, 5, 6});
Index idx;
for (auto& p : test)
if (not idx.insert(p).second)
std::cout << "Duplicate id " << p.id() << " not indexed\n";
for (auto& p : idx)
std::cout << typeid(p).name() << " " << p.id() << "\n";
std::cout << "---\n";
for (auto& p : test)
std::cout << typeid(p).name() << " " << p.id() << "\n";
}
Prints
Duplicate id 2 not indexed
6Sphere 1
3Box 2
3Box 3
6Sphere 4
---
3Box 2
3Box 3
6Sphere 2
6Sphere 4
6Sphere 1
So far so good. This is an important building block to prevent all manner of pain when dealing with virtual types in Spirit grammars¹
PS: I've since dropped the idea of intrusive_set. It doesn't work because the base_container moves items around on reallocation, and that unlinks the items from their intrusive set.
Instead, see below for an approach that doesn't try to resolve ids during the parse.
Parsing primitives
We get the ID from the PRIMITIVE1. We could store it somewhere before naturally parsing the primitives themselves, then set the id on it on commit.
Let's start with defining a State object for the parser:
struct State {
Ast::Id next_id;
Ast::Primitives primitives;
Ast::Objects objects;
template <typename... T> void commit(boost::variant<T...>& val) {
boost::apply_visitor([this](auto& obj) { commit(obj); }, val);
}
template <typename T> void commit(T& primitiveOrExpr) {
auto id = std::exchange(next_id, 0);
if constexpr (std::is_base_of_v<Ast::Primitive, T>) {
primitiveOrExpr.id = id;
primitives.insert(std::move(primitiveOrExpr));
} else {
objects.push_back(Ast::Object{id, std::move(primitiveOrExpr)});
}
}
};
As you can see, we just have a place to store the primitives, objects. And then there is the temporary storage for our next_id while we're still parsing the next entity.
The commit function helps sorting the products of the parser rules. As it happens, they can be variant, which is why we have the apply_visitor dispatch for commit on a variant.
Again, as the footnote¹ explains, Spirit's natural attribute synthesis favors static polymorphism.
The semantic actions we need are now:
static inline auto& state(auto& ctx) { return get<State>(ctx); }
auto draft = [](auto& ctx) { state(ctx).next_id = _attr(ctx); };
auto commit = [](auto& ctx) { state(ctx).commit(_attr(ctx)); };
Now let's jump ahead to the primitives:
auto sphere = as<Ast::Sphere>(eps >> "sphere" >>'(' >> param("radius") >> ')');
auto box = as<Ast::Box>(eps >> "box" >> '(' >> param('a') >> ',' >> param('b') >> ')');
auto primitive =
("primitive" >> uint_[draft] >> '=' >> (sphere | box)[commit]) > ';';
That's still cheating a little, as I've used the param helper to reduce typing:
auto number = as<Ast::Number>(double_, "number");
auto param(auto name, auto p) { return eps >> omit[name] >> '=' >> p; }
auto param(auto name) { return param(name, number); }
As you can see I've already assumed most parameters will have numerical nature.
What Are Objects Really?
Looking at it for a while, I concluded that really an Object is defined as an id number (OBJECT1, OBJECT2...) which is tied to an expression. The expression can reference primitives and have some unary and binary operators.
Let's sketch an AST for that:
using Number = double;
struct RefPrimitive { Id id; };
struct Binary;
struct Unary;
using Expr = boost::variant< //
Number, //
RefPrimitive, //
boost::recursive_wrapper<Unary>, //
boost::recursive_wrapper<Binary> //
>;
struct Unary { char op; Expr oper; };
struct Binary { Expr lhs; char op; Expr rhs; };
struct Object { Id id; Expr expr; };
Now To Parse Into That Expression AST
It's really 1:1 rules for each Ast node type. E.g.:
auto ref_prim = as<Ast::RefPrimitive>(lexeme["primitive" >> uint_]);
Now many of the expression rules can recurse, so we need declared rules with definitions via BOOST_SPIRIT_DEFINE:
// object expression grammar
rule<struct simple_tag, Ast::Expr> simple{"simple"};
rule<struct unary_tag, Ast::Unary> unary{"unary"};
rule<struct expr_tag, Ast::Expr> expr{"expr"};
rule<struct term_tag, Ast::Expr> term{"term"};
rule<struct factor_tag, Ast::Expr> factor{"factor"};
As you can tell, some of these are not 1:1 with the Ast nodes, mainly because of the recursion and the difference in operator precedence (term vs factor vs. simple). It's easier to see with the rule definition:
auto unary_def = char_("-+") >> simple;
auto simple_def = ref_prim | unary | '(' >> expr >> ")";
auto factor_def = simple;
auto term_def = factor[assign] >> *(char_("*/") >> term)[make_binary];
auto expr_def = term[assign] >> *(char_("-+") >> expr)[make_binary];
Because none of the rules actually expose a Binary, automatic attribute propagation is not convenient there². Instead, we use assign and make_binary semantic actions:
auto assign = [](auto& ctx) { _val(ctx) = _attr(ctx); };
auto make_binary = [](auto& ctx) {
using boost::fusion::at_c;
auto& attr = _attr(ctx);
auto op = at_c<0>(attr);
auto& rhs = at_c<1>(attr);
_val(ctx) = Ast::Binary { _val(ctx), op, rhs };
};
Finally, let's tie the defintions to the declared rules (using their tag types):
BOOST_SPIRIT_DEFINE(simple, unary, expr, term, factor)
All we need is a similar line to primitive:
auto object =
("object" >> uint_[draft] >> '=' >> (expr)[commit]) > ';';
And we can finish up by defining each line as a primitive|object:
auto line = primitive | object;
auto file = no_case[skip(ws_comment)[*eol >> "[geometry]" >> (-line % eol) >> eoi]];
At the top level we expect the [GEOMETRY] header, specify that we want to be case insensitive and ... that ws_comment is to be skipped³:
auto ws_comment = +(blank | lexeme["//" >> *(char_ - eol) >> eol]);
This allows us to ignore the // comments as well.
Live Demo Time
Live On Compiler Explorer
//#define BOOST_SPIRIT_X3_DEBUG
#include <boost/fusion/adapted.hpp>
#include <boost/poly_collection/base_collection.hpp>
#include <boost/spirit/home/x3.hpp>
#include <iostream>
#include <list>
#include <map>
namespace x3 = boost::spirit::x3;
namespace Ast {
using Id = uint32_t;
struct Point { }; // ?? where does this belong?
struct Primitive {
Id id;
virtual ~Primitive() = default;
};
struct Sphere : Primitive { double radius; };
struct Box : Primitive { double a, b; };
using Number = double;
struct RefPrimitive { Id id; };
struct Binary;
struct Unary;
using Expr = boost::variant< //
Number, //
RefPrimitive, //
boost::recursive_wrapper<Unary>, //
boost::recursive_wrapper<Binary> //
>;
struct Unary { char op; Expr oper; };
struct Binary { Expr lhs; char op; Expr rhs; };
struct Object { Id id; Expr expr; };
using Primitives = boost::poly_collection::base_collection<Primitive>;
using Objects = std::list<Object>;
using Index = std::map<Id, std::reference_wrapper<Primitive const>>;
std::ostream& operator<<(std::ostream& os, Primitive const& p) {
return os << boost::core::demangle(typeid(p).name()) << " "
<< "(id: " << p.id << ")";
}
std::ostream& operator<<(std::ostream& os, Object const& o) {
return os << "object(id:" << o.id << ", expr:" << o.expr << ")";
}
std::ostream& operator<<(std::ostream& os, RefPrimitive ref) {
return os << "reference(prim:" << ref.id << ")";
}
std::ostream& operator<<(std::ostream& os, Binary const& b) {
return os << '(' << b.lhs << b.op << b.rhs << ')';
}
std::ostream& operator<<(std::ostream& os, Unary const& u) {
return os << '(' << u.op << u.oper << ')';
}
} // namespace Ast
BOOST_FUSION_ADAPT_STRUCT(Ast::Primitive, id)
BOOST_FUSION_ADAPT_STRUCT(Ast::Sphere, radius)
BOOST_FUSION_ADAPT_STRUCT(Ast::Box, a, b)
BOOST_FUSION_ADAPT_STRUCT(Ast::Object, id)
BOOST_FUSION_ADAPT_STRUCT(Ast::RefPrimitive, id)
BOOST_FUSION_ADAPT_STRUCT(Ast::Unary, op, oper)
namespace Parser {
using namespace x3;
struct State {
Ast::Id next_id;
Ast::Primitives primitives;
Ast::Objects objects;
template <typename... T> void commit(boost::variant<T...>& val) {
boost::apply_visitor([this](auto& obj) { commit(obj); }, val);
}
template <typename T> void commit(T& val) {
auto id = std::exchange(next_id, 0);
if constexpr (std::is_base_of_v<Ast::Primitive, T>) {
val.id = id;
primitives.insert(std::move(val));
} else {
objects.push_back(Ast::Object{id, std::move(val)});
}
}
};
static inline auto& state(auto& ctx) { return get<State>(ctx); }
auto draft = [](auto& ctx) { state(ctx).next_id = _attr(ctx); };
auto commit = [](auto& ctx) { state(ctx).commit(_attr(ctx)); };
template <typename T>
auto as = [](auto p, char const* name = "as") {
return rule<struct _, T>{name} = p;
};
auto ws_comment = +(blank | lexeme["//" >> *(char_ - eol) >> (eol | eoi)]);
auto number = as<Ast::Number>(double_, "number");
auto param(auto name, auto p) { return eps >> omit[name] >> '=' >> p; }
auto param(auto name) { return param(name, number); }
auto sphere = as<Ast::Sphere>(eps >> "sphere" >>'(' >> param("radius") >> ')');
auto box = as<Ast::Box>(eps >> "box" >> '(' >> param('a') >> ',' >> param('b') >> ')');
auto primitive =
("primitive" >> uint_[draft] >> '=' >> (sphere | box)[commit]) > ';';
auto ref_prim = as<Ast::RefPrimitive>(lexeme["primitive" >> uint_], "ref_prim");
// object expression grammar
rule<struct simple_tag, Ast::Expr> simple{"simple"};
rule<struct unary_tag, Ast::Unary> unary{"unary"};
rule<struct expr_tag, Ast::Expr> expr{"expr"};
rule<struct term_tag, Ast::Expr> term{"term"};
rule<struct factor_tag, Ast::Expr> factor{"factor"};
auto assign = [](auto& ctx) { _val(ctx) = _attr(ctx); };
auto make_binary = [](auto& ctx) {
using boost::fusion::at_c;
auto& attr = _attr(ctx);
auto op = at_c<0>(attr);
auto& rhs = at_c<1>(attr);
_val(ctx) = Ast::Binary { _val(ctx), op, rhs };
};
auto unary_def = char_("-+") >> simple;
auto simple_def = ref_prim | unary | '(' >> expr >> ")";
auto factor_def = simple;
auto term_def = factor[assign] >> *(char_("*/") >> term)[make_binary];
auto expr_def = term[assign] >> *(char_("-+") >> expr)[make_binary];
BOOST_SPIRIT_DEFINE(simple, unary, expr, term, factor)
auto object =
("object" >> uint_[draft] >> '=' >> (expr)[commit]) > ';';
auto line = primitive | object;
auto file = no_case[skip(ws_comment)[*eol >> "[geometry]" >> (-line % eol) >> eoi]];
} // namespace Parser
int main() {
for (std::string const input :
{
R"(
[geometry]
primitive1=sphere(radius=5.5);
primitive2=box(a=-5.2, b=7.3);
//...
object1=primitive2*(-primitive1);
//...)",
R"(
[GEOMETRY]
PRIMITIVE1=SPHERE(RADIUS=5.5);
PRIMITIVE2=BOX(A=-5.2, B=7.3);
//...
OBJECT1=PRIMITIVE2*(-PRIMITIVE1);
//...)",
}) //
{
Parser::State state;
bool ok = parse(begin(input), end(input),
x3::with<Parser::State>(state)[Parser::file]);
std::cout << "Parse success? " << std::boolalpha << ok << "\n";
Ast::Index index;
for (auto& p : state.primitives)
if (auto[it,ok] = index.emplace(p.id, p); not ok) {
std::cout << "Duplicate id " << p
<< " (conflicts with existing " << it->second.get()
<< ")\n";
}
std::cout << "Primitives by ID:\n";
for (auto& [id, prim] : index)
std::cout << " - " << prim << "\n";
std::cout << "Objects in definition order:\n";
for (auto& obj: state.objects)
std::cout << " - " << obj << "\n";
}
}
Prints
Parse success? true
Primitives by ID:
- Ast::Sphere (id: 1)
- Ast::Box (id: 2)
Objects in definition order:
- object(id:1, expr:(reference(prim:2)*(-reference(prim:1))))
Parse success? true
Primitives by ID:
- Ast::Sphere (id: 1)
- Ast::Box (id: 2)
Objects in definition order:
- object(id:1, expr:(reference(prim:2)*(-reference(prim:1))))
¹ How can I use polymorphic attributes with boost::spirit::qi parsers?
² and insisting on that leads to classical in-efficiency with rules that cause a lot of backtracking
³ outside of lexemes
I have a parser in which I want to capture certain types of whitespace as enum values and preserve the spaces for the "text" values.
My whitespace parser is pretty basic (Note: I've only added the pipe character here for test/dev purposes):
struct whitespace_p : x3::symbols<Whitespace>
{
whitespace_p()
{
add
("\n", Whitespace::NEWLINE)
("\t", Whitespace::TAB)
("|", Whitespace::PIPE)
;
}
} whitespace;
And I want to capture everything either into my enum or into std::strings:
struct Element : x3::variant<Whitespace, std::string>
{
using base_type::base_type;
using base_type::operator=;
};
And to parse my input I use something like this:
const auto contentParser
= x3::rule<class ContentParserID, Element, true> { "contentParser" }
= x3::no_skip[+(x3::char_ - (whitespace))]
| whitespace
;
using Elements = std::vector<Element>;
const auto elementsParser
= x3::rule<class ContentParserID, Elements, true> { "elementsParser" }
= contentParser >> *(contentParser);
The problem though is that the parser stops at the first tab or newline it hits.
Code: http://coliru.stacked-crooked.com/a/d2cda4ce721279a4
#include <boost/spirit/home/x3.hpp>
#include <boost/spirit/home/x3/support/ast/variant.hpp>
#include <iostream>
namespace x3 = boost::spirit::x3;
enum Whitespace
{
NEWLINE,
TAB,
PIPE
};
struct whitespace_p : x3::symbols<Whitespace>
{
whitespace_p()
{
add
("\n", Whitespace::NEWLINE)
("\t", Whitespace::TAB)
("|", Whitespace::PIPE)
;
}
} whitespace;
struct Element : x3::variant<Whitespace, std::string>
{
using base_type::base_type;
using base_type::operator=;
};
const auto contentParser
= x3::rule<class ContentParserID, Element, true> { "contentParser" }
= x3::no_skip[+(x3::char_ - (whitespace))]
| whitespace
;
using Elements = std::vector<Element>;
const auto elementsParser
= x3::rule<class ContentParserID, Elements, true> { "elementsParser" }
= contentParser >> *(contentParser);
struct print_visitor
: public boost::static_visitor<std::string>
{
std::string operator()(const Whitespace& ws) const
{
if (ws == Whitespace::NEWLINE)
{
return "newline";
}
else if (ws == Whitespace::PIPE)
{
return "pipe";
}
else
{
return "tab";
}
}
std::string operator()(const std::string& str) const
{
return str;
}
};
int main()
{
const std::string text = "Hello \n World";
std::string::const_iterator start = std::begin(text);
const std::string::const_iterator stop = std::end(text);
Elements elements{};
bool result =
phrase_parse(start, stop, elementsParser, x3::ascii::space, elements);
if (!result)
{
std::cout << "failed to parse!\n";
}
else if (start != stop)
{
std::cout << "unparsed: " << std::string{start, stop} << '\n';
}
else
{
for (const auto& e : elements)
{
std::cout << "element: [" << boost::apply_visitor(print_visitor{}, e) << "]\n";
}
}
}
If I parse the text Hello | World then I get the results I'm expecting. But if I instead use Hello \n World the whitespace after the \n is swallowed and the World is never parsed. Ideally I'd like to see this output:
element: [Hello ]
element: [newline]
element: [ World]
How can I accomplish this? Thank you!
My goto reference on skipper issues: Boost spirit skipper issues
In this case you made it work with no_skip[]. That's correct.
no_skip is like lexeme except it doesn't pre-skip, from the source (boost/spirit/home/x3/directive/no_skip.hpp):
// same as lexeme[], but does not pre-skip
Alternative Take
In your case I would flip the logic: just adjust the skipper itself.
Also, don't supply the skipper with phrase_parse, because your grammar is highly sensitive to the correct value of the skipper.
Your whole grammar could be:
const auto p = x3::skip(x3::space - whitespace) [
*(+x3::graph | whitespace)
];
Here's a Live Demo On Coliru
#include <boost/spirit/home/x3.hpp>
#include <boost/spirit/home/x3/support/ast/variant.hpp>
#include <iostream>
#include <iomanip>
namespace x3 = boost::spirit::x3;
enum Whitespace { NEWLINE, TAB, PIPE };
struct whitespace_p : x3::symbols<Whitespace> {
whitespace_p() {
add
("\n", Whitespace::NEWLINE)
("\t", Whitespace::TAB)
("|", Whitespace::PIPE)
;
}
} static const whitespace;
struct Element : x3::variant<Whitespace, std::string> {
using base_type::base_type;
using base_type::operator=;
};
using Elements = std::vector<Element>;
static inline std::ostream& operator<<(std::ostream& os, Element const& el) {
struct print_visitor {
std::ostream& os;
auto& operator()(Whitespace ws) const {
switch(ws) {
case Whitespace::NEWLINE: return os << "[newline]";
case Whitespace::PIPE: return os << "[pipe]";
case Whitespace::TAB: return os << "[tab]";
}
return os << "?";
}
auto& operator()(const std::string& str) const { return os << std::quoted(str); }
} vis{os};
return boost::apply_visitor(vis, el);
}
int main() {
std::string const text = "\tHello \n World";
auto start = begin(text), stop = end(text);
const auto p = x3::skip(x3::space - whitespace) [
*(+x3::graph | whitespace)
];
Elements elements;
if (!parse(start, stop, p, elements)) {
std::cout << "failed to parse!\n";
} else {
std::copy(begin(elements), end(elements), std::ostream_iterator<Element>(std::cout, "\n"));
}
if (start != stop) {
std::cout << "unparsed: " << std::quoted(std::string(start, stop)) << '\n';
}
}
Prints
[tab]
"Hello"
[newline]
"World"
Even Simpler?
It doesn't seem like you'd need any skipper here at all. Why not:
const auto p = *(+~x3::char_("\n\t|") | whitespace);
While we're at it, there's no need for symbols to map enums:
struct Element : x3::variant<char, std::string> {
// ...
};
using Elements = std::vector<Element>;
And then
const auto p
= x3::rule<struct ID, Element> {}
= +~x3::char_("\n\t|") | x3::char_;
Live On Coliru
#include <boost/spirit/home/x3.hpp>
#include <boost/spirit/home/x3/support/ast/variant.hpp>
#include <iostream>
#include <iomanip>
namespace x3 = boost::spirit::x3;
struct Element : x3::variant<char, std::string> {
using variant = x3::variant<char, std::string>;
using variant::variant;
using variant::operator=;
friend std::ostream& operator<<(std::ostream& os, Element const& el) {
struct print_visitor {
std::ostream& os;
auto& operator()(char ws) const {
switch(ws) {
case '\n': return os << "[newline]";
case '\t': return os << "[pipe]";
case '|': return os << "[tab]";
}
return os << "?";
}
auto& operator()(const std::string& str) const { return os << std::quoted(str); }
} vis{os};
return boost::apply_visitor(vis, el);
}
};
using Elements = std::vector<Element>;
int main() {
std::string const text = "\tHello \n World";
auto start = begin(text);
auto const stop = end(text);
Elements elements;
const auto p
= x3::rule<struct ID, Element> {}
= +~x3::char_("\n\t|") | x3::char_;
if (!parse(start, stop, *p, elements)) {
std::cout << "failed to parse!\n";
} else {
std::copy(begin(elements), end(elements), std::ostream_iterator<Element>(std::cout, "\n"));
}
if (start != stop) {
std::cout << "unparsed: " << std::quoted(std::string(start, stop)) << '\n';
}
}
Prints
[pipe]
"Hello "
[newline]
" World"
The problems are that you are using a phrase_parser instead of a parser at line 76.
Try to use something like
bool result =
parse(start, stop, elementsParser, elements);
Your phrase_parser was instructed to skip spaces, what you really don't want.
Look the first answer of How to use boost::spirit to parse a sequence of words into a vector?
In the grammar below, when I add the alternative (| property) to the start rule, I get this error
'boost::spirit::x3::traits::detail::move_to': none of the 3 overloads
could convert all the argument types
e:\data\boost\boost_1_65_1\boost\spirit\home\x3\support\traits\move_to.hpp 180
I suspect that the problem is that the property attribute is a struct, and property_list is a vector (shouldn't x3 create a vector of one entry?). What is the recommended way to design the AST structures to support alternatives? A Boost variant?
#include <string>
#include <vector>
#include <iostream>
#include <iomanip>
#include <map>
#pragma warning(push)
#pragma warning(disable : 4348)
#include <boost/config/warning_disable.hpp>
#include <boost/spirit/home/x3.hpp>
#include <boost/variant.hpp>
#include <boost/fusion/adapted/struct.hpp>
#pragma warning(pop)
namespace x3 = boost::spirit::x3;
namespace scl_ast
{
struct KEYWORD : std::string
{
using std::string::string;
using std::string::operator=;
};
struct NIL
{
};
using VALUE = boost::variant <NIL, std::string, int, double, KEYWORD>;
struct PROPERTY
{
KEYWORD name;
VALUE value;
};
static inline std::ostream& operator<< (std::ostream& os, VALUE const& v)
{
struct
{
std::ostream& _os;
void operator () (std::string const& s) const { _os << std::quoted (s); }
void operator () (int i) const { _os << i; }
void operator () (double d) const { _os << d; }
void operator () (KEYWORD const& k) const { _os << k; }
void operator () (NIL) const { }
} vis { os };
boost::apply_visitor (vis, v);
return os;
}
static inline std::ostream& operator<< (std::ostream& os, PROPERTY const& prop)
{
os << prop.name;
if (prop.value.which ())
{
os << "=" << prop.value;
}
return os;
}
static inline std::ostream& operator<< (std::ostream& os, std::vector <PROPERTY> const& props)
{
for (auto const& prop : props)
{
os << prop << " ";
}
return os;
}
}; // End namespace scl_ast
BOOST_FUSION_ADAPT_STRUCT (scl_ast::PROPERTY, name, value)
//
// Keyword-value grammar for simple command language
//
namespace scl
{
using namespace x3;
auto const keyword = rule <struct _keyword, std::string> { "keyword" }
= lexeme [+char_ ("a-zA-Z0-9$_")];
auto const quoted_string
= lexeme ['"' >> *('\\' > char_ | ~char_ ('"')) >> '"'];
auto const value
= quoted_string
| x3::real_parser<double, x3::strict_real_policies<double>>{}
| x3::int_
| keyword;
auto const property = rule <struct _property, scl_ast::PROPERTY> { "property" }
= keyword >> -(("=" >> value));
auto const property_list = rule <struct _property_list, std::vector <scl_ast::PROPERTY>> { "property_list" }
= lit ('(') >> property % ',' >> lit (')');
auto const start = skip (blank) [property_list | property];
}; // End namespace scl
int
main ()
{
std::vector <std::string> input =
{
"(abc=1.,def=.5,ghi=2.0)",
"(ghi = 1, jkl = 3)",
"(abc,def=1,ghi=2.4,jkl=\"mno 123\", pqr = stu)",
"(abc = test, def, ghi=2)",
"abc=1",
"def = 2.7",
"ghi"
};
for (auto const& str : input)
{
std::vector <scl_ast::PROPERTY> result;
auto b = str.begin (), e = str.end ();
bool ok = x3::parse (b, e, scl::start, result);
std::cout << (ok ? "OK" : "FAIL") << '\t' << std::quoted (str) << std::endl;
if (ok)
{
std::cout << " -- Parsed: " << result << std::endl;
if (b != e)
{
std::cout << " -- Unparsed: " << std::quoted (std::string (b, e)) << std::endl;
}
}
std::cout << std::endl;
} // End for
return 0;
} // End main
I suspect that the problem is that the property attribute is a struct, and property_list is a vector (shouldn't x3 create a vector of one entry?)
Yes, yes, and yes, depending.
I see roughly 3 approaches to tackle this. Let's start with the simples:
Force Container-like synthesis: Live On Coliru
= skip(blank)[property_list | repeat(1)[property] ];
Coercing the attribute type. Turns out I was wrong here: It used to work for Qi, but apparently that has been dropped. Here it is, not-working and all:
auto coerce = [](auto p) { return rule<struct _, std::vector<scl_ast::PROPERTY> > {} = p; };
auto const start
= skip(blank)[property_list | coerce(property)];
Third is actually moot because the same problem. So I guess I owe you a contrived workaround, using semantic actions: Live On Coliru
auto push_back = [](auto& ctx) {
_val(ctx).push_back(_attr(ctx));
};
auto const start
= rule<struct _start, std::vector<scl_ast::PROPERTY>, true>{ "start" }
= skip(blank)[property_list | omit[property[push_back]]];