In my boost::spirit grammar I have the following snippet;
implicit_method_declaration = (-(qi::token(ABSTRACT)) >> ...)
The type of -(qi::token(ABSTRACT) is boost::optional<boost::iterator_range<std::string::iterator>> however I'm only using this construct to check if the abstract keyword, is actually present, that is, I'd rather have -(qi::token(ABSTRACT) have the type bool with the value boost::optional<...> operator bool() const.
How would I go about achieving this?
I think you're looking for qi::matches[]:
implicit_method_declaration =
qi::matches[qi::token(ABSTRACT)] >> ...;
An alternative would be to use qi::attr() with alternatives:
implicit_method_declaration =
(
qi::token(ABSTRACT) >> qi::attr(true)
| qi::attr(false)
) >> ...;
A quick demo again: http://coliru.stacked-crooked.com/a/ed8bbad53e8c1943
#include <boost/spirit/include/qi.hpp>
namespace qi = boost::spirit::qi;
template <typename It, typename Skipper = qi::space_type>
struct parser : qi::grammar<It, bool(), Skipper>
{
parser() : parser::base_type(implicit_method_declaration)
{
using namespace qi;
implicit_method_declaration = matches["abstract"];
BOOST_SPIRIT_DEBUG_NODES((implicit_method_declaration));
}
private:
qi::rule<It, bool(), Skipper> implicit_method_declaration;
};
bool doParse(const std::string& input)
{
typedef std::string::const_iterator It;
auto f(begin(input)), l(end(input));
parser<It, qi::space_type> p;
bool data;
try
{
bool ok = qi::phrase_parse(f,l,p,qi::space,data);
if (ok)
{
std::cout << "parse success\n";
std::cout << "data: " << data << "\n";
}
else std::cerr << "parse failed: '" << std::string(f,l) << "'\n";
if (f!=l) std::cerr << "trailing unparsed: '" << std::string(f,l) << "'\n";
return ok;
} catch(const qi::expectation_failure<It>& e)
{
std::string frag(e.first, e.last);
std::cerr << e.what() << "'" << frag << "'\n";
}
return false;
}
int main()
{
doParse("abstract");
doParse("static final");
}
Output
parse success
data: 1
parse success
data: 0
trailing unparsed: 'static final'
Related
I am trying to parse the following messages with Spirit Qi:
"A/B AND C/D", "A/B", "A/B AND C/D AND E/F"
I am able to parse "A/B" but cannot get the correct results for the other strings.
I tried to following code:
qi::rule<It, AstNodeVector()> entries;
qi::rule<It, AstNodeVector()> lists;
qi::rule<It, std::string()> element;
this->entries= *(this->lists % " AND ");
this->lists= this->element >> '/' >> this->element;
this->element = qi::char_("A-Z");
What is wrong with my grammar?
It seems you're not skipping whitespace. Maybe that's a conceptual problem (see Boost spirit skipper issues).
Regardless, it does parse:
Live On Coliru
#include <boost/spirit/include/qi.hpp>
#include <iomanip>
namespace qi = boost::spirit::qi;
using AstNodeVector = std::vector<std::string>;
template <typename It>
struct P : qi::grammar<It, AstNodeVector()> {
P() : P::base_type(entries) {
entries = *(lists % " AND ");
lists = element >> '/' >> element;
element = qi::char_("A-Z");
}
private:
qi::rule<It, AstNodeVector()> entries;
qi::rule<It, AstNodeVector()> lists;
qi::rule<It, std::string()> element;
};
int main() {
using It = std::string::const_iterator;
P<It> const p {};
for (std::string const input: {
"A/B AND C/D",
"A/B",
"A/B AND C/D AND E/F",
})
{
It f = begin(input), l = end(input);
AstNodeVector results;
if (phrase_parse(f, l, p, qi::space, results)) {
std::cout << "Success: " << std::quoted(input) << "\n";
for (auto& el : results) {
std::cout << " -- " << std::quoted(el) << "\n";
}
} else {
std::cout << "FAIL: " << std::quoted(input) << "\n";
}
if (f != l) {
std::cout << "Remaining input: " << std::quoted(std::string(f,l)) << "\n";
}
}
}
Prints
Success: "A/B AND C/D"
-- "A"
-- "B"
-- "C"
-- "D"
Success: "A/B"
-- "A"
-- "B"
Success: "A/B AND C/D AND E/F"
-- "A"
-- "B"
-- "C"
-- "D"
-- "E"
-- "F"
Perhaps you should have included self-contained code, or elaborate on what exactly is the problem.
Assume I already have a struct that looks like this:
struct LETTER
{
double one;
char[12] two;
double three;
char[12] four;
};
And my inputs are comma separated, for example:
"32,CATSANDDOGS,42,WHAT"
"43,BATANDZEBRAS,23,PARROT"
I've been trying to adapt this example (Spirit Qi : rule for char [5]) to to roll through BOOST_FUSION_ADAPT_STRUCT but haven't had any luck. I tried using std::array as shown here (http://www.boost.org/doc/libs/1_64_0/libs/spirit/example/qi/boost_array.cpp) but I haven't been able to make it work in a struct. Is what I'm trying to do even possible? What am I doing wrong here? I would think this would be the most obvious use case.
Is what I'm trying to do even possible?
I'm going to assume you want to write idiomatic C++ code (which is obviously the target domain for Spirit Qi), so you can use std::string:
Live On Coliru
#include <boost/fusion/adapted.hpp>
#include <boost/fusion/include/io.hpp>
#include <boost/spirit/include/qi.hpp>
#include <iostream>
namespace qi = boost::spirit::qi;
struct Letter {
double one;
std::string two;
double three;
std::string four;
};
BOOST_FUSION_ADAPT_STRUCT(Letter, one, two, three, four)
template <typename Iterator> struct LETTERParser : qi::grammar<Iterator, Letter()> {
LETTERParser() : LETTERParser::base_type(start) {
using namespace qi;
_11c = repeat(11) [char_];
start = skip(space) [ "LETTER" >> double_ >> _11c >> double_ >> _11c ];
}
private:
qi::rule<Iterator, Letter()> start;
qi::rule<Iterator, std::string()> _11c;
};
int main() {
const std::string input("LETTER 42 12345678901 +Inf abcdefghijk ");
using It = std::string::const_iterator;
LETTERParser<It> parser;
Letter example;
It f = input.begin(), l = input.end();
if (phrase_parse(f, l, parser, qi::ascii::space, example)) {
std::cout << "parsed: " << boost::fusion::as_vector(example) << "\n";
std::cout << " example.one: " << example.one << "\n";
std::cout << " example.two: '" << example.two << "'\n";
std::cout << " example.three: " << example.three << "\n";
std::cout << " example.four: '" << example.four << "'\n";
} else {
std::cout << "couldn't parse '" << input << "'\n";
}
if (f != l)
std::cout << "Remaining unparsed input: '" << std::string(f,l) << "'\n";
}
Prints
parsed: (42 12345678901 inf abcdefghijk)
example.one: 42
example.two: '12345678901'
example.three: inf
example.four: 'abcdefghijk'
I have the following code:
#include <boost/any.hpp>
#include <boost/spirit/include/qi.hpp>
#include <iostream>
#include <string>
template <typename Iterator>
struct parser : boost::spirit::qi::grammar<Iterator, boost::any(), boost::spirit::qi::ascii::space_type>
{
parser() : parser::base_type(start)
{
start %= boost::spirit::qi::int_ | boost::spirit::qi::lexeme['"' >> +(boost::spirit::qi::char_ - '"') >> '"']; // example: 0 or "str"
}
boost::spirit::qi::rule<Iterator, boost::any(), boost::spirit::qi::ascii::space_type> start;
};
int main()
{
const std::string input_data("\"str\"");
boost::any var = 0;
auto itr = input_data.begin();
auto end = input_data.end();
parser<decltype(itr)> g;
bool res = boost::spirit::qi::phrase_parse(itr, end, g, boost::spirit::ascii::space, var);
if (res && itr == end)
{
std::cout << "Parsing succeeded \n";
try
{
std::cout << boost::any_cast<std::string>(var) << '\n';
}
catch (const boost::bad_any_cast& ex)
{
std::cerr << ex.what() << '\n';
}
}
else
{
std::cout << "Parsing failed \n";
}
}
It compiles fine until I add
boost::spirit::qi::lexeme['"' >> +(boost::spirit::qi::char_ - '"') >> '"']
I can't post the errors here and on pastie.org because of the characters amount limitations.
What am I doing wrong? How can I fix it?
Again, why would you want to complicate matters and make it slow by using boost::any?
Anyways, +char_ exposes std::vector<char> and apparently the attribute compatibility rules don't want to decide what to transform this to.
Make it explicit with as_string: Live On Coliru
#include <boost/any.hpp>
#include <boost/spirit/include/qi.hpp>
#include <iostream>
#include <string>
template <typename Iterator>
struct parser : boost::spirit::qi::grammar<Iterator, boost::any(), boost::spirit::qi::ascii::space_type>
{
parser() : parser::base_type(start)
{
using namespace boost::spirit::qi;
start %= int_ | as_string [ lexeme['"' >> +(char_ - '"') >> '"'] ]; // example: 0 or "str"
}
boost::spirit::qi::rule<Iterator, boost::any(), boost::spirit::qi::ascii::space_type> start;
};
int main()
{
// const std::string input_data("\"str\"");
const std::string input_data("123");
for (std::string const& input_data : { "\"str\"", "123" })
{
boost::any var = boost::none;
auto itr = input_data.begin();
auto end = input_data.end();
parser<decltype(itr)> g;
bool res = boost::spirit::qi::phrase_parse(itr, end, g, boost::spirit::ascii::space, var);
if (res && itr == end)
{
std::cout << "Parsing succeeded \n";
try { std::cout << boost::any_cast<int> (var) << '\n'; } catch (const boost::bad_any_cast& ex) { std::cerr << ex.what() << '\n'; }
try { std::cout << boost::any_cast<std::string>(var) << '\n'; } catch (const boost::bad_any_cast& ex) { std::cerr << ex.what() << '\n'; }
}
else
{
std::cout << "Parsing failed \n";
}
}
}
edit : I have ripped out the lexer as it does not cleanly integrate with Qi and just obfuscates grammars (see here).
I'm trying to grow a grammar on top of the spirit lex framework. When I try to move a skip parser into the grammar I begin getting errors.
So, changing the qi::grammar<> and qi::rule<> event signatures from <Iterator> to <Iterator,void(),ascii::space_type>. In the grammar struct. What do I need to do?
Additionally I have set the token_def to omit its attribute for the optional token, and some others. Why is it still providing me with a valid _val in the semantic action for optional in the lexer? Reason I ask is because I thought the problem had to do with the string attribute of the optional token on the rhs of the event rule in qi not unifying with the void() Attribute signature of the rule.
#include <boost/spirit/include/phoenix_core.hpp>
#include <boost/spirit/include/lex_lexertl.hpp>
#include <boost/spirit/include/qi.hpp>
#include <boost/cstdint.hpp>
#include <string>
#include<exception>
namespace lex = boost::spirit::lex;
namespace px = boost::phoenix;
namespace qi = boost::spirit::qi;
namespace ascii = boost::spirit::ascii;
template <typename Lexer>
struct tokens : lex::lexer<Lexer>
{
tokens()
: left_paranthesis("\"{\""),
right_paranthesis("\"}\""),
colon(":"),
namespace_("(?i:namespace)"),
event("(?i:event)"),
optional("(?i:optional)"),
required("(?i:required)"),
ordinal("\\d+"),
identifier("\\w+")
{
using boost::spirit::lex::_val;
this->self
= " "
| left_paranthesis [ std::cout << px::val("lpar") << std::endl]
| right_paranthesis [ std::cout << px::val("rpar") << std::endl]
| colon [ std::cout << px::val("colon") << std::endl]
| namespace_ [ std::cout << px::val("kw namesapce") << std::endl]
| event [ std::cout << px::val("kw event") << std::endl]
| optional [ std::cout << px::val("optional ") << "-->" << _val << "<--" << std::endl]
| required [ std::cout << px::val("required") << std::endl]
| ordinal [ std::cout << px::val("val ordinal (") << _val << ")" << std::endl]
| identifier [std::cout << px::val("val identifier(") << _val << ")" << std::endl];
}
lex::token_def<> left_paranthesis, right_paranthesis, colon;
lex::token_def<lex::omit> namespace_, event, optional, required;
lex::token_def<boost::uint32_t> ordinal;
lex::token_def<> identifier;
};
template <typename Iterator>
struct grammar : qi::grammar<Iterator>
{
template <typename TokenDef>
grammar(TokenDef const& tok)
: grammar::base_type(event)
{
//start = event;
event = tok.optional [ std::cout << px::val("== OPTIONAL") << std::endl];
}
qi::rule<Iterator> start;
qi::rule<Iterator> event;
};
// std::string test = "namespace{ event { OPtiONAL 124:hello_world RequireD} } ";
std::string test = "OPTIONAL";
int main()
{
typedef lex::lexertl::token<std::string::iterator, boost::mpl::vector<boost::uint32_t, std::string> > token_type;
typedef lex::lexertl::actor_lexer<token_type> lexer_type;
typedef tokens<lexer_type>::iterator_type iterator_type;
tokens<lexer_type> token_lexer;
grammar<iterator_type> grammar(token_lexer);
std::string::iterator first = test.begin();
std::string::iterator last = test.end();
bool r;
r = lex::tokenize_and_parse(first, last, token_lexer, grammar);
if(r)
;
else
{
std::cout << "parsing failed" << std::endl;
}
/*
lexer_type::iterator_type iter;
try
{
iter = token_lexer.begin(first,last);
}
catch(std::exception & e)
{
std::cout << e.what() << std::endl;
}
lexer_type::iterator_type end = token_lexer.end();
while (iter != end && token_is_valid(*iter))
++iter;
*/
}
This grammar fails :
template <typename Iterator>
struct grammar : qi::grammar<Iterator,void(),ascii::space_type>
{
template <typename TokenDef>
grammar(TokenDef const& tok)
: grammar::base_type(event)
{
//start = event;
event = tok.optional [ std::cout << px::val("== OPTIONAL") << std::endl];
}
qi::rule<Iterator> start;
qi::rule<Iterator,void(),ascii::space_type> event;
};
As with most of spirit. If you wanna do something REALISTIC you gotta spend hours looking for a solution which isn't documented but buried in examples and mailing lists. Seriously considering moving to ragel or flex/bison. The problem isn't that the machinery isn't available it's that it is undocumented.
In this case when looking at the lex documentation, one get's generously mislead by looking at the lex parser api calls which has a tokenize_and_phrase_parse function. Which doesn't really work when you try to use it like the qi::phrase_parse neither does the documentation explain how to wire up a skipper using this function.
The wire-up of getting a space skipper into the parser is done by altering the lexer, and then using some undocumented qi-skipper construct initialising the grammar and rules. You can see this in action in the lex example directory ( example 5). Code that compiles and works:
#include <boost/spirit/include/phoenix_core.hpp>
#include <boost/spirit/include/lex_lexertl.hpp>
#include <boost/spirit/include/qi.hpp>
#include <boost/cstdint.hpp>
#include <string>
#include<exception>
namespace lex = boost::spirit::lex;
namespace px = boost::phoenix;
namespace qi = boost::spirit::qi;
namespace ascii = boost::spirit::ascii;
template <typename Lexer>
struct tokens : lex::lexer<Lexer>
{
tokens()
: left_paranthesis("\"{\""),
right_paranthesis("\"}\""),
colon(":"),
namespace_("(?i:namespace)"),
event("(?i:event)"),
optional("(?i:optional)"),
required("(?i:required)"),
ordinal("\\d+"),
identifier("\\w+")
{
using boost::spirit::lex::_val;
this->self
=
left_paranthesis [ std::cout << px::val("lpar") << std::endl]
| right_paranthesis [ std::cout << px::val("rpar") << std::endl]
| colon [ std::cout << px::val("colon") << std::endl]
| namespace_ [ std::cout << px::val("kw namesapce") << std::endl]
| event [ std::cout << px::val("kw event") << std::endl]
| optional [ std::cout << px::val("optional ") << "-->" << _val << "<--" << std::endl]
| required [ std::cout << px::val("required") << std::endl]
| ordinal [ std::cout << px::val("val ordinal (") << _val << ")" << std::endl]
| identifier [std::cout << px::val("val identifier(") << _val << ")" << std::endl];
this->self("WS") = lex::token_def<>("[ \\t\\n]+");
}
lex::token_def<> left_paranthesis, right_paranthesis, colon;
lex::token_def<lex::omit> namespace_, event, optional, required;
lex::token_def<boost::uint32_t> ordinal;
lex::token_def<> identifier;
};
template <typename Iterator, typename Lexer>
struct grammar : qi::grammar<Iterator,qi::in_state_skipper<Lexer> >
{
template <typename TokenDef>
grammar(TokenDef const& tok)
: grammar::base_type(event)
{
//start = event;
event = tok.optional [ std::cout << px::val("== OPTIONAL") << std::endl];
}
qi::rule<Iterator> start;
qi::rule<Iterator, qi::in_state_skipper<Lexer> > event;
};
// std::string test = "namespace{ event { OPtiONAL 124:hello_world RequireD} } ";
std::string test = " OPTIONAL ";
int main()
{
typedef lex::lexertl::token<std::string::iterator, boost::mpl::vector<boost::uint32_t, std::string> > token_type;
typedef lex::lexertl::actor_lexer<token_type> lexer_type;
typedef tokens<lexer_type>::iterator_type iterator_type;
tokens<lexer_type> token_lexer;
grammar<iterator_type,tokens<lexer_type>::lexer_def> grammar(token_lexer);
std::string::iterator it = test.begin();
iterator_type first = token_lexer.begin(it, test.end());
iterator_type last = token_lexer.end();
bool r;
r = qi::phrase_parse(first, last, grammar, qi::in_state("WS")[token_lexer.self]);
if(r)
;
else
{
std::cout << "parsing failed" << std::endl;
}
/*
lexer_type::iterator_type iter;
try
{
iter = token_lexer.begin(first,last);
}
catch(std::exception & e)
{
std::cout << e.what() << std::endl;
}
lexer_type::iterator_type end = token_lexer.end();
while (iter != end && token_is_valid(*iter))
++iter;
*/
}
Why the parser using the rules below returns an empty container? There're 3 rules. One is for parsing a string of characters except double quote, the second parses a pair (e.g. "col1" : 2) and the third parses the vector of such pairs. The ouput of the program below in MSVS2012 is
parse success
result: '' : 0
result: '' : 0
result: '' : 0
.
namespace parsers
{
spirit::qi::rule< iterator, column_name_t() > quoted_string =
spirit::qi::lexeme["\"" >> +~spirit::qi::char_("\"") >> "\""];
spirit::qi::rule< iterator, column_and_aggregate(), spirit::qi::space_type > agg_pair =
quoted_string//::boost::bind( &apply_col_and_aggr_visitor, spirit::qi::_val, spirit::qi::_1 )]
> ':'
// A rule validation technic is used below.
> spirit::int_[spirit::qi::_pass = (spirit::qi::_1 >=AVG && spirit::qi::_1<=SUM)];//::boost::bind( &apply_col_and_aggr_visitor, spirit::qi::_val, spirit::qi::_1 )];
spirit::qi::rule< iterator, column_and_aggregate_container(), spirit::qi::space_type > aggregates_parser =
'{'
> agg_pair/*[phoenix::push_back(spirit::qi::_val, spirit::qi::_1)]*/ % ',' // N.B.!!! list-redux technic
> '}';
}
using namespace parsers;
using namespace boost::spirit;
bool doParse(const std::string& input)
{
typedef std::string::const_iterator It;
auto f(begin(input)), l(end(input));
//parser<It, qi::space_type> p;
column_and_aggregate_container data;
typedef BOOST_TYPEOF(qi::space) skipper_type;
try
{
bool ok = qi::phrase_parse(f,l,aggregates_parser,qi::space,data);
if (ok)
{
std::cout << "parse success\n";
for (auto& pair : data)
std::cout << "result: '" << pair.first << "' : " << (int) pair.second << "\n";
}
else std::cerr << "parse failed: '" << std::string(f,l) << "'\n";
if (f!=l) std::cerr << "trailing unparsed: '" << std::string(f,l) << "'\n";
return ok;
}
catch(const qi::expectation_failure<It>& e)
{
std::string frag(e.first, e.last);
std::cerr << e.what() << "'" << frag << "'\n";
}
return false;
}
int main()
{
//bool ok = doParse("{ 'column 1' : 1, 'column 2' : 0, 'column 3' : 1 }");
doParse("{ \"column 1\" : 1, \"column 2\" : 0, \"column 3\" : 1 }");
//return ok? 0 : 255;
}
template <typename it, typename skipper = qi::space_type>
struct quoted_string_parser
{
quoted_string_parser()
{
using namespace qi;
quoted_string %= lexeme['"' >> *~char_('"') >> '"'];
BOOST_SPIRIT_DEBUG_NODE(quoted_string);
}
qi::rule<it, std::string(), skipper> quoted_string;
};
template <typename it, typename skipper = qi::space_type>
struct aggregates_parser : qi::grammar<it, column_and_aggregate_container(), skipper>
{
aggregates_parser() : aggregates_parser::base_type(aggregates_parser_)
{
using namespace qi;
agg_pair %= quoted_string_parser<it,skipper> > ':' > int_[_pass = (qi::_1 >= AVG && qi::_1 <= SUM)];
aggregates_parser_ = '{' > agg_pair % ',' > '}';
BOOST_SPIRIT_DEBUG_NODE(aggregates_parser_);
}
private:
qi::rule<it, sql_faggregate(), skipper> faggr;
qi::rule<it, column_and_aggregate(), skipper> agg_pair;
qi::rule<it, column_and_aggregate_container(), skipper> aggregates_parser_;
};
Like I said in the answer right where I suggested this semantic action for validation:
When a semantic action is present, automatic attribute propagation does not normally occur. Using %= forces automatic attribute propagation (we want this because the semantic action doesn't assign the attribute value(s), it just validates them).
Again, a fully working demonstration, incorporating your rules:
#include <boost/fusion/adapted/std_pair.hpp>
#include <boost/spirit/include/qi.hpp>
#include <boost/spirit/include/phoenix.hpp>
namespace qi = boost::spirit::qi;
namespace phx = boost::phoenix;
typedef std::string column_name_t;
enum sql_faggregate
{
AVG,
// ....
SUM,
};
typedef std::pair<column_name_t, sql_faggregate> column_and_aggregate;
typedef std::vector<column_and_aggregate> column_and_aggregate_container;
template <typename It, typename Skipper = qi::space_type>
struct parser : qi::grammar<It, column_and_aggregate_container(), Skipper>
{
parser() : parser::base_type(aggregates_parser)
{
using namespace qi;
quoted_string = lexeme['"' >> +~char_('"') >> '"'];
agg_pair %= quoted_string > ':' // A rule validation technic is used below.
> int_[_pass = (_1 >=AVG && _1<=SUM)];
aggregates_parser = '{' > agg_pair % ',' > '}';
BOOST_SPIRIT_DEBUG_NODE(aggregates_parser);
}
private:
qi::rule<It, std::string(), qi::space_type> quoted_string;
qi::rule<It, sql_faggregate(), qi::space_type> faggr;
qi::rule<It, column_and_aggregate(), qi::space_type> agg_pair;
qi::rule<It, column_and_aggregate_container(), qi::space_type> aggregates_parser;
};
bool doParse(const std::string& input)
{
typedef std::string::const_iterator It;
auto f(begin(input)), l(end(input));
parser<It, qi::space_type> p;
column_and_aggregate_container data;
try
{
bool ok = qi::phrase_parse(f,l,p,qi::space,data);
if (ok)
{
std::cout << "parse success\n";
for (auto& pair : data)
std::cout << "result: '" << pair.first << "' : " << (int) pair.second << "\n";
}
else std::cerr << "parse failed: '" << std::string(f,l) << "'\n";
if (f!=l) std::cerr << "trailing unparsed: '" << std::string(f,l) << "'\n";
return ok;
} catch(const qi::expectation_failure<It>& e)
{
std::string frag(e.first, e.last);
std::cerr << e.what() << "'" << frag << "'\n";
}
return false;
}
int main()
{
bool ok = doParse("{ \"column 1\" : 1, \"column 2\" : 0, \"column 3\" : 1 }");
return ok? 0 : 255;
}
Prints
parse success
result: 'column 1' : 1
result: 'column 2' : 0
result: 'column 3' : 1
as expected