Boost spirit revert parsing - c++

I want to parse a file containing the following structure:
some
garbage *&%
section1 {
section_content
}
section2 {
section_content
}
The rule parsing section_name1 { ... } section_name2 { ... } is already defined:
section_name_rule = lexeme[+char_("A-Za-z0-9_")];
section = section_name_rule > lit("{") > /*some complicated things*/... > lit("}");
sections %= +section;
So I need to skip any garbage until the sections rule is met.
Is there any way to accomplish this? I have tried seek[sections], but it seems not to work.
EDIT:
I localized the reason why seek is not working: if I use follows operator(>>), then it works. If expectation parser is used (>), then it throws an exception. Here is a sample code:
#define BOOST_SPIRIT_DEBUG
#include <boost/fusion/adapted/struct.hpp>
#include <boost/spirit/include/qi.hpp>
#include <boost/spirit/repository/include/qi_seek.hpp>
#include <boost/spirit/include/phoenix.hpp>
namespace qi = boost::spirit::qi;
using boost::phoenix::push_back;
struct section_t {
std::string name, contents;
friend std::ostream& operator<<(std::ostream& os, section_t const& s) { return os << "section_t[" << s.name << "] {" << s.contents << "}"; }
};
BOOST_FUSION_ADAPT_STRUCT(section_t, (std::string, name)(std::string, contents))
typedef std::vector<section_t> sections_t;
template <typename It, typename Skipper = qi::space_type>
struct grammar : qi::grammar<It, sections_t(), Skipper>
{
grammar() : grammar::base_type(start) {
using namespace qi;
using boost::spirit::repository::qi::seek;
section_name_rule = lexeme[+char_("A-Za-z0-9_")];
//Replacing '>>'s with '>'s throws an exception, while this works as expected!!
section = section_name_rule
>>
lit("{") >> lexeme[*~char_('}')] >> lit("}");
start = seek [ hold[section[push_back(qi::_val, qi::_1)]] ]
>> *(section[push_back(qi::_val, qi::_1)]);
}
private:
qi::rule<It, sections_t(), Skipper> start;
qi::rule<It, section_t(), Skipper> section;
qi::rule<It, std::string(), Skipper> section_name_rule;
};
int main() {
typedef std::string::const_iterator iter;
std::string storage("sdfsdf\n sd:fgdfg section1 {dummy } section2 {dummy } section3 {dummy }");
iter f(storage.begin()), l(storage.end());
sections_t sections;
if (qi::phrase_parse(f, l, grammar<iter>(), qi::space, sections))
{
for(auto& s : sections)
std::cout << "Parsed: " << s << "\n";
}
if (f != l)
std::cout << "Remaining unparsed: '" << std::string(f,l) << "'\n";
}
So in the real example my entire grammar is constructed with expectation operators. Do I have to change everything to make the "seek" work, or is there any other way (let's say, seek a simple "{", and revert one section_name_rule back)??

Here's a demonstration, using Hamlet for inspiration: Live On Coliru
start = *seek [ no_skip[eol] >> hold [section] ];
Notes:
drop the expectation points
optimize by requiring start of line before section name
Example input:
some
garbage *&%
section1 {
Claudius: ...But now, my cousin Hamlet, and my son —
Hamlet: A little more than kin, and less than kind.
}
WE CAN DO MOAR GARBAGE
section2 {
Claudius: How is it that the clouds still hang on you?
Hamlet: Not so my lord; I am too much i' the sun
}
Output:
Parsed: section_t[section1] {Claudius: ...But now, my cousin Hamlet, and my son —
Hamlet: A little more than kin, and less than kind.
}
Parsed: section_t[section2] {Claudius: How is it that the clouds still hang on you?
Hamlet: Not so my lord; I am too much i' the sun
}
Reference Listing
// #define BOOST_SPIRIT_DEBUG
#include <boost/fusion/adapted/struct.hpp>
#include <boost/spirit/include/qi.hpp>
#include <boost/spirit/repository/include/qi_seek.hpp>
namespace qi = boost::spirit::qi;
struct section_t {
std::string name, contents;
friend std::ostream& operator<<(std::ostream& os, section_t const& s) { return os << "section_t[" << s.name << "] {" << s.contents << "}"; }
};
BOOST_FUSION_ADAPT_STRUCT(section_t, (std::string, name)(std::string, contents))
typedef std::vector<section_t> sections_t;
template <typename It, typename Skipper = qi::space_type>
struct grammar : qi::grammar<It, sections_t(), Skipper>
{
grammar() : grammar::base_type(start) {
using namespace qi;
using boost::spirit::repository::qi::seek;
section_name_rule = lexeme[+char_("A-Za-z0-9_")];
section = section_name_rule >> '{' >> lexeme[*~char_('}')] >> '}';
start = *seek [ no_skip[eol] >> hold [section] ];
BOOST_SPIRIT_DEBUG_NODES((start)(section)(section_name_rule))
}
private:
qi::rule<It, sections_t(), Skipper> start;
qi::rule<It, section_t(), Skipper> section;
qi::rule<It, std::string(), Skipper> section_name_rule;
};
int main() {
using It = boost::spirit::istream_iterator;
It f(std::cin >> std::noskipws), l;
sections_t sections;
if (qi::phrase_parse(f, l, grammar<It>(), qi::space, sections))
{
for(auto& s : sections)
std::cout << "Parsed: " << s << "\n";
}
if (f != l)
std::cout << "Remaining unparsed: '" << std::string(f,l) << "'\n";
}

Related

boost spirit: copy the result in a vector of strings

I want to parse a function (with an arbitrary name and an arbitrary numbers af arguments) in this form:
function(bye, 1, 3, 4, foo)
The arguments could be generic strings comma separated.
And I want to copy the name of the function and the arguments in a vector of strings.
like this
std::vector<std::string> F;
std::string fun = "function(bye, 1, 3, 4, foo)";
// The parser must produce this vector from the example
F[0] == "function"
F[1] == "1"
F[2] == "3"
F[3] == "4"
F[4] == "foo"
I've written the following code by after reading some tutorial but it does not work (In the sense that it not compile).
#include <boost/config/warning_disable.hpp>
#include <boost/spirit/include/qi.hpp>
#include <boost/spirit/include/phoenix_core.hpp>
#include <boost/spirit/include/phoenix_operator.hpp>
#include <boost/spirit/include/phoenix_object.hpp>
#include <boost/fusion/include/adapt_struct.hpp>
#include <boost/fusion/include/io.hpp>
#include <iostream>
#include <string>
namespace client
{
namespace qi = boost::spirit::qi;
namespace ascii = boost::spirit::ascii;
///////////////////////////////////////////////////////////////////////////////
template <typename Iterator>
struct command_parser : qi::grammar<Iterator, std::vector<std::string>(), ascii::space_type>
{
command_parser() : command_parser::base_type(start)
{
using qi::int_;
using qi::lit;
using qi::double_;
using qi::lexeme;
using ascii::char_;
fn_name = +qi::char_("a-zA-Z");
string = +qi::char_("a-zA-Z_0-9");
rec = *( lit(",") >> string );
start %= fn_name >> lit("(") >> string >> rec >> lit(")") ;
}
qi::rule<Iterator, std::string(), ascii::space_type> fn_name;
qi::rule<Iterator, std::string(), ascii::space_type> string;
qi::rule<Iterator, std::string(), ascii::space_type> rec;
qi::rule<Iterator, std::vector<std::string>, ascii::space_type> start;
};
}
////////////////////////////////////////////////////////////////////////////
// Main program
////////////////////////////////////////////////////////////////////////////
int
main()
{
namespace qi = boost::spirit::qi;
std::cout << "/////////////////////////////////////////////////////////\n\n";
client::command_parser<std::string::iterator> CP;
std::string cmd("fun(1,2,3,4 , 5, foo) ");
std::vector<std::string> VV;
bool result = qi::parse(cmd.begin(), cmd.end(), CP, VV);
if (result) {
for ( auto sss : VV ){
std::cout << sss << std::endl;
}
} else {
std::cout << "Fail" << std::endl;
}
return 0 ;
}
Just for fun, here's my minimalist take on this grammar:
using CallList = std::vector<std::string>;
struct ParseError : std::runtime_error {
ParseError() : std::runtime_error("ParseError") {}
};
// The parse implementation
CallList parse_function_call(std::string const& fun) {
CallList elements;
using namespace boost::spirit::qi;
using It = decltype(begin(fun));
static const rule<It, std::string()> identifier = alpha >> +(alnum | char_('_'));
if (!phrase_parse(begin(fun), end(fun),
identifier >> '(' >> -(lexeme[+~char_(",)")] % ",") >> ')' >> eoi,
space, elements))
throw ParseError{};
return elements;
}
With a little bit of plumbing
// just for test output
using TestResult = std::variant<CallList, ParseError>;
// exceptions are equivalent
static constexpr bool operator==(ParseError const&, ParseError const&)
{ return true; }
static inline std::ostream& operator<<(std::ostream& os, TestResult const& tr) {
using namespace std;
if (holds_alternative<ParseError>(tr)) {
return os << "ParseError";
} else {
auto& list = get<CallList>(tr);
copy(begin(list), end(list), std::experimental::make_ostream_joiner(os << "{", ","));
return os << "}";
}
}
TestResult try_parse(std::string const& fun) {
try { return parse_function_call(fun); }
catch(ParseError const& e) { return e; }
}
Here's a test runner:
for (auto const& [input, expected]: {
Case("function(bye, 1, 3, 4, foo)", CallList{"function", "1", "3", "4", "foo"}),
{"liar(pants on fire)", CallList{"liar", "pants on fire"}},
{"liar('pants on fire')", CallList{"liar", "'pants on fire'"}},
{"nullary()", CallList{"nullary"}},
{"nullary( )", CallList{"nullary"}},
{"zerolength(a,,b)", ParseError{}},
{"zerolength(a, ,b)", ParseError{}},
{"noarglust", ParseError{}},
{"", ParseError{}},
{"()", ParseError{}},
{"1(invalidfunctionname)", ParseError{}},
{"foo(bar) BOGUS", ParseError{}},
})
{
auto const actual = try_parse(input);
bool const ok = (actual == expected);
cout << std::quoted(input) << ": " << (ok? "PASS":"FAIL") << "\n";
if (!ok) {
std::cout << " -- expected: " << expected << "\n";
std::cout << " -- actual: " << actual << "\n";
}
}
Which prints Live On Coliru
"function(bye, 1, 3, 4, foo)": FAIL
-- expected: {function,1,3,4,foo}
-- actual: {function,bye,1,3,4,foo}
"liar(pants on fire)": PASS
"liar('pants on fire')": PASS
"nullary()": PASS
"nullary( )": PASS
"zerolength(a,,b)": PASS
"zerolength(a, ,b)": PASS
"noarglust": PASS
"": PASS
"()": PASS
"1(invalidfunctionname)": PASS
"foo(bar) BOGUS": PASS
Note that your example test-case doesn't pass, but I think that was a mistake in the test case.
Full Listing
Live On Coliru
//#define BOOST_SPIRIT_DEBUG
#include <boost/spirit/include/qi.hpp>
#include <experimental/iterator>
#include <variant>
#include <iomanip>
using CallList = std::vector<std::string>;
struct ParseError : std::runtime_error {
ParseError() : std::runtime_error("ParseError") {}
};
// The parse implementation
CallList parse_function_call(std::string const& fun) {
CallList elements;
using namespace boost::spirit::qi;
using It = decltype(begin(fun));
static const rule<It, std::string()> identifier = alpha >> +(alnum | char_('_'));
if (!phrase_parse(begin(fun), end(fun),
identifier >> '(' >> -(lexeme[+~char_(",)")] % ",") >> ')' >> eoi,
space, elements))
throw ParseError{};
return elements;
}
// just for test output
using TestResult = std::variant<CallList, ParseError>;
// exceptions are equivalent
static constexpr bool operator==(ParseError const&, ParseError const&)
{ return true; }
static inline std::ostream& operator<<(std::ostream& os, TestResult const& tr) {
using namespace std;
if (holds_alternative<ParseError>(tr)) {
return os << "ParseError";
} else {
auto& list = get<CallList>(tr);
copy(begin(list), end(list), std::experimental::make_ostream_joiner(os << "{", ","));
return os << "}";
}
}
TestResult try_parse(std::string const& fun) {
try { return parse_function_call(fun); }
catch(ParseError const& e) { return e; }
}
int main() {
using namespace std;
using Case = pair<std::string, TestResult>;
for (auto const& [input, expected]: {
Case("function(bye, 1, 3, 4, foo)", CallList{"function", "1", "3", "4", "foo"}),
{"liar(pants on fire)", CallList{"liar", "pants on fire"}},
{"liar('pants on fire')", CallList{"liar", "'pants on fire'"}},
{"nullary()", CallList{"nullary"}},
{"nullary( )", CallList{"nullary"}},
{"zerolength(a,,b)", ParseError{}},
{"zerolength(a, ,b)", ParseError{}},
{"noarglust", ParseError{}},
{"", ParseError{}},
{"()", ParseError{}},
{"1(invalidfunctionname)", ParseError{}},
{"foo(bar) BOGUS", ParseError{}},
})
{
auto const actual = try_parse(input);
bool const ok = (actual == expected);
cout << std::quoted(input) << ": " << (ok? "PASS":"FAIL") << "\n";
if (!ok) {
std::cout << " -- expected: " << expected << "\n";
std::cout << " -- actual: " << actual << "\n";
}
}
}
I'm correcting my answer per suggestions made by #sehe. All the credit for these corrections go to him. I am referencing your line numbers below. So the first error is from spirit and it says:
incompatible_start_rule:
// If you see the assertion below failing then the start rule
// passed to the constructor of the grammar is not compatible with
// the grammar (i.e. it uses different template parameters).
The signature of the start parser does not match that of the parser deceleration.
22. struct command_parser : qi::grammar<Iterator, std::vector<std::string>(), ascii::space_type>
43. qi::rule<Iterator, std::vector<std::string>, ascii::space_type> start;
I googled this and could not find an explanation but using an object rather than a type is preferable. I did it the other way in my first answer. The proper fix is at line 43:
43. qi::rule<Iterator, std::vector<std::string>(), ascii::space_type> start;
The next spirit error is:
The rule was instantiated with a skipper type but you have not pass
any. Did you use parse instead of phrase_parse?");
So a phrase_parse is required with a skipper. Note that we need a skipper to pass along.
64. using qi::ascii::space;
65. bool result = qi::phrase_parse(cmd.begin(), cmd.end(), CP, space, VV);
Now it compiles and the output is:
fun
1
2345foo
I see that won't do and you are looking to stuff the vector with each of the passed parameters. So you need a rule that is compatible with your attribute and intention. The kleene operator working with a std::string will put all the data into one string. So use your attribute:
41. qi::rule<Iterator, std::vector<std::string>(), ascii::space_type> rec;``
Now as #sehe points out, the skipper with fn_name and string will just concatenate names with spaces and newlines. So don't use skippers there.
39. qi::rule<Iterator, std::string()> fn_name;
40. qi::rule<Iterator, std::string()> string;
The other error I made was to see the %= and call it a list operator. From here, it is a definition operator. I'm not sure why there are two but playing around, it seems you need to use %= with semantic action. Here is the corrected code:
#include <boost/config/warning_disable.hpp>
#include <boost/spirit/include/qi.hpp>
#include <boost/spirit/include/phoenix_core.hpp>
#include <boost/spirit/include/phoenix_operator.hpp>
#include <boost/spirit/include/phoenix_object.hpp>
#include <boost/fusion/include/adapt_struct.hpp>
#include <boost/fusion/include/io.hpp>
#include <iostream>
#include <string>
namespace client
{
namespace qi = boost::spirit::qi;
namespace ascii = boost::spirit::ascii;
template <typename Iterator>
struct command_parser : qi::grammar<Iterator, std::vector<std::string>(), ascii::space_type>
{
command_parser() : command_parser::base_type(start)
{
using qi::int_;
using qi::lit;
using qi::double_;
using qi::lexeme;
using ascii::char_;
fn_name = +qi::char_("a-zA-Z");
string = +qi::char_("a-zA-Z_0-9");
rec = *(lit(",") >> string);
start %= fn_name >> lit("(") >> string >> rec >> lit(")");
}
qi::rule<Iterator, std::string()> fn_name;
qi::rule<Iterator, std::string()> string;
qi::rule<Iterator, std::vector<std::string>(), ascii::space_type> rec;
qi::rule<Iterator, std::vector<std::string>(), ascii::space_type> start;
};
}
int main()
{
namespace qi = boost::spirit::qi;
client::command_parser<std::string::iterator> CP;
std::string cmd("function(1,2,3,4 , 5, foo) ");
std::vector<std::string> VV;
bool result = qi::phrase_parse(cmd.begin(), cmd.end(), CP, qi::ascii::space, VV);
if (result) {
for (auto sss : VV) {
std::cout << sss << std::endl;
}
}
else {
std::cout << "Fail" << std::endl;
}
return 0;
}
And here is an example using X3:
#include <boost/spirit/home/x3.hpp>
#include <iostream>
#include <vector>
//your attribute, could be more complex, might use namespace
using attr = std::vector<std::string>;
namespace parser {
namespace x3 = boost::spirit::x3;
const auto fn_name = +x3::char_("a-zA-Z");
const auto string = +x3::char_("a-zA-Z_0-9");
const auto start = x3::rule<struct _, attr>() = fn_name >> "(" >> string % ',' >> ")";
}
int main()
{
namespace x3 = boost::spirit::x3;
std::string cmd("fun(1,.2,3,4 , 5, foo) ");
attr VV;
auto it = cmd.begin();
bool result = phrase_parse(it, cmd.end(), parser::start, x3::space, VV);
if (result) {
for (auto sss : VV) {
std::cout << "-> " << sss << std::endl;
}
}
else
std::cout << "Fail at" << std::endl;
return 0;
}

Treat escaped newline as line continuation

Here is an example of the syntax -- two groups of items:
I_name m_name parameter1=value parameter2=value
I_name m_name parameter1=value \
parameter2=value
My question is how to define the skip-type.
It is not just space_type but space_type minus newline.
But newline followed by backslash is a skip-type.
E.g.
I define name like that:
qi::rule<Iterator, std::string(), ascii::space_type> m_sName;
m_sName %= qi::lexeme[ascii::alpha >> *ascii::alnum];
This is obviously not correct, as the space_type must include newline-backslash.
The following grammar works for me.
*("\\\n" | ~qi::char_('\n')) % '\n'
It will ignore any newline after the backslash. And the following is a simple test.
#include <vector>
#include <string>
#include <boost/config/warning_disable.hpp>
#include <boost/spirit/include/qi.hpp>
#define BOOST_TEST_MODULE example
#include <boost/test/unit_test.hpp>
typedef std::vector<std::string> Lines;
inline auto ParseLines(std::string const& str) {
Lines lines;
namespace qi = boost::spirit::qi;
if (qi::parse(
str.begin(), str.end(),
*("\\\n" | ~qi::char_('\n')) % '\n',
lines)) {
return lines;
}
else {
throw std::invalid_argument("Parse error at ParseLines");
}
}
BOOST_AUTO_TEST_CASE(TestParseLines) {
std::string const str =
"I_name m_name parameter1=value parameter2=value\n"
"I_name m_name parameter1 = value \\\n"
"parameter2 = value";
Lines const expected{
"I_name m_name parameter1=value parameter2=value",
"I_name m_name parameter1 = value parameter2 = value"
};
BOOST_TEST(ParseLines(str) == expected);
}
You should use "-std=c++14 -lboost_unit_test_framework" for compilation. Anyway, it is easy to convert the code for c++03.
qi::blank is exactly that. It's qi::space without newlines.
You can do this too: ("\\\n" | qi::blank)
To be able to declare a rule with such a skipper, define a skipper grammar:
template <typename It>
struct my_skipper : qi::grammar<It> {
my_skipper() : my_skipper::base_type(start) {}
qi::rule<It> start = ("\\\n" | qi::blank);
};
Full Demo
Live On Coliru
#include <boost/spirit/include/qi.hpp>
#include <boost/fusion/include/adapted.hpp>
#include <map>
namespace qi = boost::spirit::qi;
namespace ast {
struct record {
std::string iname, mname;
std::map<std::string, std::string> params;
};
using records = std::vector<record>;
}
BOOST_FUSION_ADAPT_STRUCT(ast::record, iname, mname, params)
template <typename It>
struct my_parser : qi::grammar<It, ast::records()> {
using Skipper = qi::rule<It>;
my_parser() : my_parser::base_type(start) {
skipper = ("\\\n" | qi::blank);
name = +qi::graph;
key = +(qi::graph - '=');
param = key >> '=' >> name;
record = name >> name >> *param;
records = *(record >> +qi::eol);
start = qi::skip(qi::copy(skipper)) [ records ];
}
private:
Skipper skipper;
qi::rule<It, ast::records(), Skipper> records;
qi::rule<It, ast::record(), Skipper> record;
qi::rule<It, ast::records()> start;
qi::rule<It, std::pair<std::string, std::string>()> param;
qi::rule<It, std::string()> name, key;
};
int main() {
#if 1
using It = boost::spirit::istream_iterator;
It f(std::cin >> std::noskipws), l;
#else
using It = std::string::const_iterator;
std::string const input = "something here a=1\n";
It f = input.begin(), l = input.end();
#endif
ast::records data;
bool ok = qi::parse(f, l, my_parser<It>(), data);
if (ok) {
std::cout << "Parsed:\n";
for (auto& r : data) {
std::cout << "\t" << r.iname << " " << r.mname;
for (auto& p : r.params)
std::cout << " [" << p.first << ": " << p.second << "]";
std::cout << "\n";
}
} else {
std::cout << "Parse failed\n";
}
if (f!=l)
std::cout << "Remaining input: '" << std::string(f,l) << "'\n";
}
Prints (for the input in your question):
Parsed:
I_name m_name [parameter1: value] [parameter2: value]
I_name m_name [parameter1: value] [parameter2: value]

Parsing nested key value pairs in Boost Spirit

I am having trouble writing what I think should be a simple parser using Boost::Spirit. (I'm using Spirit instead of just using string functions as this is partly a learning exercise for me).
Data
The data to parse takes the form of key value pairs, where a value can itself be a key value pair. Keys are alphanumeric (with underscores and no digit as first character); values are alphanumeric plus .-_ - the values can be dates in the format DD-MMM-YYYY e.g. 01-Jan-2015 and floating point numbers like 3.1415 in addition to plain old alphanumeric strings. Keys and values are separated with =; pairs are separated with ;; structured values are delimited with {...}. At the moment I am erasing all spaces from the user input before passing it to Spirit.
Example input:
Key1 = Value1; Key2 = { NestedKey1=Alan; NestedKey2 = 43.1232; }; Key3 = 15-Jul-1974 ;
I would then strip all spaces to give
Key1=Value1;Key2={NestedKey1=Alan;NestedKey2=43.1232;};Key3=15-Jul-1974;
and then I actually pass it to Spirit.
Problem
What I currently have works just dandy when values are simply values. When I start encoding structured values in the input then Spirit stops after the first structured value. A workaround if there is only one structured value is to put it at the end of the input... but I will need two or more structured values on occasion.
The code
The below compiles in VS2013 and illustrates the errors:
#include <boost/spirit/include/qi.hpp>
#include <boost/fusion/include/pair.hpp>
#include <boost/fusion/adapted.hpp>
#include <map>
#include <string>
#include <iostream>
typedef std::map<std::string, std::string> ARGTYPE;
#define BOOST_SPIRIT_DEBUG
namespace qi = boost::spirit::qi;
namespace fusion = boost::fusion;
template < typename It, typename Skipper>
struct NestedGrammar : qi::grammar < It, ARGTYPE(), Skipper >
{
NestedGrammar() : NestedGrammar::base_type(Sequence)
{
using namespace qi;
KeyName = qi::char_("a-zA-Z_") >> *qi::char_("a-zA-Z0-9_");
Value = +qi::char_("-.a-zA-Z_0-9");
Pair = KeyName >> -(
'=' >> ('{' >> raw[Sequence] >> '}' | Value)
);
Sequence = Pair >> *((qi::lit(';') | '&') >> Pair);
BOOST_SPIRIT_DEBUG_NODE(KeyName);
BOOST_SPIRIT_DEBUG_NODE(Value);
BOOST_SPIRIT_DEBUG_NODE(Pair);
BOOST_SPIRIT_DEBUG_NODE(Sequence);
}
private:
qi::rule<It, ARGTYPE(), Skipper> Sequence;
qi::rule<It, std::string()> KeyName;
qi::rule<It, std::string(), Skipper> Value;
qi::rule<It, std::pair < std::string, std::string>(), Skipper> Pair;
};
template <typename Iterator>
ARGTYPE Parse2(Iterator begin, Iterator end)
{
NestedGrammar<Iterator, qi::space_type> p;
ARGTYPE data;
qi::phrase_parse(begin, end, p, qi::space, data);
return data;
}
// ARGTYPE is std::map<std::string,std::string>
void NestedParse(std::string Input, ARGTYPE& Output)
{
Input.erase(std::remove_if(Input.begin(), Input.end(), isspace), Input.end());
Output = Parse2(Input.begin(), Input.end());
}
int main(int argc, char** argv)
{
std::string Example1, Example2, Example3;
ARGTYPE Out;
Example1 = "Key1=Value1 ; Key2 = 01-Jan-2015; Key3 = 2.7181; Key4 = Johnny";
Example2 = "Key1 = Value1; Key2 = {InnerK1 = one; IK2 = 11-Nov-2011;};";
Example3 = "K1 = V1; K2 = {IK1=IV1; IK2=IV2;}; K3=V3; K4 = {JK1=JV1; JK2=JV2;};";
NestedParse(Example1, Out);
for (ARGTYPE::iterator i = Out.begin(); i != Out.end(); i++)
std::cout << i->first << "|" << i->second << std::endl;
std::cout << "=====" << std::endl;
/* get the following, as expected:
Key1|Value1
Key2|01-Jan-2015
Key3|2.7181
Key4|Johnny
*/
NestedParse(Example2, Out);
for (ARGTYPE::iterator i = Out.begin(); i != Out.end(); i++)
std::cout << i->first << "|" << i->second << std::endl;
std::cout << "=====" << std::endl;
/* get the following, as expected:
Key1|Value1
key2|InnerK1=one;IK2=11-Nov-2011
*/
NestedParse(Example3, Out);
for (ARGTYPE::iterator i = Out.begin(); i != Out.end(); i++)
std::cout << i->first << "|" << i->second << std::endl;
/* Only get the first two lines of the expected output:
K1|V1
K2|IK1=IV1;IK2=IV2
K3|V3
K4|JK1=JV1;JK2=JV2
*/
return 0;
}
I'm not sure if the problem is down to my ignorance of BNF, my ignorance of Spirit, or perhaps my ignorance of both at this point.
Any help appreciated. I've read e.g. Spirit Qi sequence parsing issues and links therein but I still can't figure out what I am doing wrong.
Indeed this precisely a simple grammar that Spirit excels at.
Moreover there is absolutely no need to skip whitespace up front: Spirit has skippers built in for the purpose.
To your explicit question, though:
The Sequence rule is overcomplicated. You could just use the list operator (%):
Sequence = Pair % char_(";&");
Now your problem is that you end the sequence with a ; that isn't expected, so both Sequence and Value fail the parse eventually. This isn't very clear unless you #define BOOST_SPIRIT_DEBUG¹ and inspect the debug output.
So to fix it use:
Sequence = Pair % char_(";&") >> -omit[char_(";&")];
Fix Live On Coliru (or with debug info)
Prints:
Key1|Value1
Key2|01-Jan-2015
Key3|2.7181
Key4|Johnny
=====
Key1|Value1
Key2|InnerK1=one;IK2=11-Nov-2011;
=====
K1|V1
K2|IK1=IV1;IK2=IV2;
K3|V3
K4|JK1=JV1;JK2=JV2;
Bonus Cleanup
Actually, that was simple. Just remove the redundant line removing whitespace. The skipper was already qi::space.
(Note though that the skipper doesn't apply to your Value rule, so values cannot contain whitespace but the parser will not silently skip it either; I suppose this is likely what you want. Just be aware of it).
Recursive AST
You would actually want to have a recursive AST, instead of parsing into a flat map.
Boost recursive variants make this a breeze:
namespace ast {
typedef boost::make_recursive_variant<std::string, std::map<std::string, boost::recursive_variant_> >::type Value;
typedef std::map<std::string, Value> Sequence;
}
To make this work you just change the declared attribute types of the rules:
qi::rule<It, ast::Sequence(), Skipper> Sequence;
qi::rule<It, std::pair<std::string, ast::Value>(), Skipper> Pair;
qi::rule<It, std::string(), Skipper> String;
qi::rule<It, std::string()> KeyName;
The rules themselves don't even have to change at all. You will need to write a little visitor to stream the AST:
static inline std::ostream& operator<<(std::ostream& os, ast::Value const& value) {
struct vis : boost::static_visitor<> {
vis(std::ostream& os, std::string indent = "") : _os(os), _indent(indent) {}
void operator()(std::map<std::string, ast::Value> const& map) const {
_os << "map {\n";
for (auto& entry : map) {
_os << _indent << " " << entry.first << '|';
boost::apply_visitor(vis(_os, _indent+" "), entry.second);
_os << "\n";
}
_os << _indent << "}\n";
}
void operator()(std::string const& s) const {
_os << s;
}
private:
std::ostream& _os;
std::string _indent;
};
boost::apply_visitor(vis(os), value);
return os;
}
Now it prints:
map {
Key1|Value1
Key2|01-Jan-2015
Key3|2.7181
Key4|Johnny
}
=====
map {
Key1|Value1
Key2|InnerK1 = one; IK2 = 11-Nov-2011;
}
=====
map {
K1|V1
K2|IK1=IV1; IK2=IV2;
K3|V3
K4|JK1=JV1; JK2=JV2;
}
Of course, the clincher is when you change raw[Sequence] to just Sequence now:
map {
Key1|Value1
Key2|01-Jan-2015
Key3|2.7181
Key4|Johnny
}
=====
map {
Key1|Value1
Key2|map {
IK2|11-Nov-2011
InnerK1|one
}
}
=====
map {
K1|V1
K2|map {
IK1|IV1
IK2|IV2
}
K3|V3
K4|map {
JK1|JV1
JK2|JV2
}
}
Full Demo Code
Live On Coliru
//#define BOOST_SPIRIT_DEBUG
#include <boost/variant.hpp>
#include <boost/spirit/include/qi.hpp>
#include <boost/fusion/adapted/std_pair.hpp>
#include <iostream>
#include <string>
#include <map>
namespace ast {
typedef boost::make_recursive_variant<std::string, std::map<std::string, boost::recursive_variant_> >::type Value;
typedef std::map<std::string, Value> Sequence;
}
namespace qi = boost::spirit::qi;
template <typename It, typename Skipper>
struct NestedGrammar : qi::grammar <It, ast::Sequence(), Skipper>
{
NestedGrammar() : NestedGrammar::base_type(Sequence)
{
using namespace qi;
KeyName = qi::char_("a-zA-Z_") >> *qi::char_("a-zA-Z0-9_");
String = +qi::char_("-.a-zA-Z_0-9");
Pair = KeyName >> -(
'=' >> ('{' >> Sequence >> '}' | String)
);
Sequence = Pair % char_(";&") >> -omit[char_(";&")];
BOOST_SPIRIT_DEBUG_NODES((KeyName) (String) (Pair) (Sequence))
}
private:
qi::rule<It, ast::Sequence(), Skipper> Sequence;
qi::rule<It, std::pair<std::string, ast::Value>(), Skipper> Pair;
qi::rule<It, std::string(), Skipper> String;
qi::rule<It, std::string()> KeyName;
};
template <typename Iterator>
ast::Sequence DoParse(Iterator begin, Iterator end)
{
NestedGrammar<Iterator, qi::space_type> p;
ast::Sequence data;
qi::phrase_parse(begin, end, p, qi::space, data);
return data;
}
static inline std::ostream& operator<<(std::ostream& os, ast::Value const& value) {
struct vis : boost::static_visitor<> {
vis(std::ostream& os, std::string indent = "") : _os(os), _indent(indent) {}
void operator()(std::map<std::string, ast::Value> const& map) const {
_os << "map {\n";
for (auto& entry : map) {
_os << _indent << " " << entry.first << '|';
boost::apply_visitor(vis(_os, _indent+" "), entry.second);
_os << "\n";
}
_os << _indent << "}\n";
}
void operator()(std::string const& s) const {
_os << s;
}
private:
std::ostream& _os;
std::string _indent;
};
boost::apply_visitor(vis(os), value);
return os;
}
int main()
{
std::string const Example1 = "Key1=Value1 ; Key2 = 01-Jan-2015; Key3 = 2.7181; Key4 = Johnny";
std::string const Example2 = "Key1 = Value1; Key2 = {InnerK1 = one; IK2 = 11-Nov-2011;};";
std::string const Example3 = "K1 = V1; K2 = {IK1=IV1; IK2=IV2;}; K3=V3; K4 = {JK1=JV1; JK2=JV2;};";
std::cout << DoParse(Example1.begin(), Example1.end()) << "\n";
std::cout << DoParse(Example2.begin(), Example2.end()) << "\n";
std::cout << DoParse(Example3.begin(), Example3.end()) << "\n";
}
¹ You "had" it, but not in the right place! It should go before any Boost includes.

Iterative update of abstract syntax tree with boost spirit

I have a working boost spirit parser and was thinking if it is possible to do iterative update of an abstract syntax tree with boost spirit?
I have a struct similar to:
struct ast;
typedef boost::variant< boost::recursive_wrapper<ast> > node;
struct ast
{
std::vector<int> value;
std::vector<node> children;
};
Which is being parsed by use of:
bool r = phrase_parse(begin, end, grammar, space, ast);
Would it be possible to do iterative update of abstract syntax tree with boost spirit? I have not found any documentation on this, but I was thinking if the parsers semantic actions could push_back on an already existing AST. Has anyone tried this?
This would allow for parsing like this:
bool r = phrase_parse(begin, end, grammar, space, ast); //initial parsing
//the second parse will be called at a later state given some event/timer/io/something
bool r = phrase_parse(begin, end, grammar, space, ast); //additional parsing which will update the already existing AST
How would you know which nodes to merge? Or would you always add ("graft") at the root level? In that case, why don't you just parse another and merge moving the elements into the existing ast?
ast& operator+=(ast&& other) {
std::move(other.value.begin(), other.value.end(), back_inserter(value));
std::move(other.children.begin(), other.children.end(), back_inserter(children));
return *this;
}
Demo Time
Let's devise the simplest grammar I can think of for this AST:
start = '{' >> -(int_ % ',') >> ';' >> -(start % ',') >> '}';
Note I didn't even make the ; optional. Oh well. Samples. Exercises for readers. ☡ You know the drill.
We implement the trivial function ast parse(It f, It l), and then we can simply merge the asts:
int main() {
ast merged;
for(std::string const& input : {
"{1 ,2 ,3 ;{4 ;{9 , 8 ;}},{5 ,6 ;}}",
"{10,20,30;{40;{90, 80;}},{50,60;}}",
})
{
merged += parse(input.begin(), input.end());
std::cout << "merged + " << input << " --> " << merged << "\n";
}
}
Live On Coliru
//#define BOOST_SPIRIT_DEBUG
#include <boost/fusion/adapted/struct.hpp>
#include <boost/spirit/include/qi.hpp>
#include <boost/spirit/include/karma.hpp>
namespace qi = boost::spirit::qi;
namespace karma = boost::spirit::karma;
struct ast;
//typedef boost::make_recursive_variant<boost::recursive_wrapper<ast> >::type node;
typedef boost::variant<boost::recursive_wrapper<ast> > node;
struct ast {
std::vector<int> value;
std::vector<node> children;
ast& operator+=(ast&& other) {
std::move(other.value.begin(), other.value.end(), back_inserter(value));
std::move(other.children.begin(), other.children.end(), back_inserter(children));
return *this;
}
};
BOOST_FUSION_ADAPT_STRUCT(ast,
(std::vector<int>,value)
(std::vector<node>,children)
)
template <typename It, typename Skipper = qi::space_type>
struct grammar : qi::grammar<It, ast(), Skipper>
{
grammar() : grammar::base_type(start) {
using namespace qi;
start = '{' >> -(int_ % ',') >> ';' >> -(start % ',') >> '}';
BOOST_SPIRIT_DEBUG_NODES((start));
}
private:
qi::rule<It, ast(), Skipper> start;
};
// for output:
static inline std::ostream& operator<<(std::ostream& os, ast const& v) {
using namespace karma;
rule<boost::spirit::ostream_iterator, ast()> r;
r = '{' << -(int_ % ',') << ';' << -((r|eps) % ',') << '}';
return os << format(r, v);
}
template <typename It> ast parse(It f, It l)
{
ast parsed;
static grammar<It> g;
bool ok = qi::phrase_parse(f,l,g,qi::space,parsed);
if (!ok || (f!=l)) {
std::cout << "Parse failure\n";
std::cout << "Remaining unparsed: '" << std::string(f,l) << "'\n";
exit(255);
}
return parsed;
}
int main() {
ast merged;
for(std::string const& input : {
"{1 ,2 ,3 ;{4 ;{9 , 8 ;}},{5 ,6 ;}}",
"{10,20,30;{40;{90, 80;}},{50,60;}}",
})
{
merged += parse(input.begin(), input.end());
std::cout << "merged + " << input << " --> " << merged << "\n";
}
}
Of course, it prints:
merged + {1 ,2 ,3 ;{4 ;{9 , 8 ;}},{5 ,6 ;}} --> {1,2,3;{4;{9,8;}},{5,6;}}
merged + {10,20,30;{40;{90, 80;}},{50,60;}} --> {1,2,3,10,20,30;{4;{9,8;}},{5,6;},{40;{90,80;}},{50,60;}}
UPDATE
In this - trivial - example, you can just bind the collections to the attributes in the parse call. The same thing will happen without the operator+= call needed to move the elements, because the rules are written to automatically append to the bound container attribute.
CAVEAT: A distinct disadvantage of modifying the target value in-place is what happens if parsing fails. In the version the merged value will then be "undefined" (has received partial information from the failed parse).
So if you want to parse inputs "atomically", the first, more explicit approach is a better fit.
So the following is a slightly shorter way to write the same:
Live On Coliru
// #define BOOST_SPIRIT_DEBUG
#include <boost/fusion/adapted/struct.hpp>
#include <boost/spirit/include/qi.hpp>
#include <boost/spirit/include/karma.hpp>
namespace qi = boost::spirit::qi;
namespace karma = boost::spirit::karma;
struct ast;
//typedef boost::make_recursive_variant<boost::recursive_wrapper<ast> >::type node;
typedef boost::variant<boost::recursive_wrapper<ast> > node;
struct ast {
std::vector<int> value;
std::vector<node> children;
};
BOOST_FUSION_ADAPT_STRUCT(ast,
(std::vector<int>,value)
(std::vector<node>,children)
)
template <typename It, typename Skipper = qi::space_type>
struct grammar : qi::grammar<It, ast(), Skipper>
{
grammar() : grammar::base_type(start) {
using namespace qi;
start = '{' >> -(int_ % ',') >> ';' >> -(start % ',') >> '}';
BOOST_SPIRIT_DEBUG_NODES((start));
}
private:
qi::rule<It, ast(), Skipper> start;
};
// for output:
static inline std::ostream& operator<<(std::ostream& os, ast const& v) {
using namespace karma;
rule<boost::spirit::ostream_iterator, ast()> r;
r = '{' << -(int_ % ',') << ';' << -((r|eps) % ',') << '}';
return os << format(r, v);
}
template <typename It> void parse(It f, It l, ast& into)
{
static grammar<It> g;
bool ok = qi::phrase_parse(f,l,g,qi::space,into);
if (!ok || (f!=l)) {
std::cout << "Parse failure\n";
std::cout << "Remaining unparsed: '" << std::string(f,l) << "'\n";
exit(255);
}
}
int main() {
ast merged;
for(std::string const& input : {
"{1 ,2 ,3 ;{4 ;{9 , 8 ;}},{5 ,6 ;}}",
"{10,20,30;{40;{90, 80;}},{50,60;}}",
})
{
parse(input.begin(), input.end(), merged);
std::cout << "merged + " << input << " --> " << merged << "\n";
}
}
Still prints

What is the correct way to use boost::qi::rule with BOOST_FUSION_ADAPT_STRUCT?

I am attempting to get a qi::rule<> to emit a struct with BOOST_FUSION_ADAPT_STRUCT based on the boost employee example.
I have the following struct and its associated fusion macro:
struct LineOnCommand
{
int lineNum;
std::vector<char> humpType;
};
BOOST_FUSION_ADAPT_STRUCT(
LineOnCommand,
(int, lineNum)
(std::vector<char>, humpType)
)
The associated parsing rules are:
qi::rule<Iterator, std::vector<char> ascii::space_type> humpIdentifer = qi::lit("BH") | qi::lit("DH");
qi::rule<Iterator, LineOnCommand(), ascii::space_type> Cmd_LNON = qi::int_ >> -humpIdentifier >> qi::lit("LNON");
I then have a compound rule, of which all others (including this simple test case) are a part which is passed to the parser:
qi::rule<Iterator, qi::unused_type, ascii::space_type> commands =
+( /* other rules | */ Cmd_LNON /*| other rules */);
bool success = qi::phrase_parse(StartIterator, EndIterator, commands, ascii::space);
The problem comes when I attempt to compile, and I get the error:
<boostsource>/spirit/home/qi/detail/assign_to.hpp(152): error: no suitable constructor exists to convert form "const int" to "LineOnCommand"
attr = static_cast<Attribute>(val);
Clearly I'm doing something wrong, but I'm not sure what. If I understand the way spirit works, the 2nd argument to the template of the rule represents the attribute (i.e. the data type emitted by the rule), and the BOOST_FUSION_ADAPT_STRUCT macro will adapt my struct so that boost knows how to convert a stream that is "int, std::vector" to it.
The only difference between what I'm doing here and the boost employee example is that I'm not using an explicit grammar to do the parsing. My understanding is this is not necessary, and that a rule by itself is sufficient.
What am I doing wrong?
I'm not sure. I think I'm missing the problem. Perhaps, I "naturally" sidestep the problem because your sample is not self-contained.
So, here's my take on it: See it Live On Coliru, in the hope that just comparing things helps you:
I fixed the obvious typos in your rule declaration
I suggested something other than qi::unused_type; if there's no attribute, there's no need to state it; beyond the iterator type, the template arguments to qi::rule and qi::grammar are not positional. So
qi::rule<It, qi::unused_type(), ascii::space_type> r;
qi::rule<It, ascii::space_type, qi::unused_type()> r;
qi::rule<It, ascii::space_type> r;
are all /logically/ equivalent.
Full listing:
#include <boost/fusion/adapted/struct.hpp>
#include <boost/spirit/include/qi.hpp>
namespace qi = boost::spirit::qi;
namespace ascii = boost::spirit::ascii;
struct LineOnCommand
{
int lineNum;
std::vector<char> humpType;
};
BOOST_FUSION_ADAPT_STRUCT(
LineOnCommand,
(int, lineNum)
(std::vector<char>, humpType)
)
template <typename It, typename Skipper = ascii::space_type>
struct parser : qi::grammar<It, std::vector<LineOnCommand>(), Skipper>
{
parser() : parser::base_type(commands)
{
using namespace qi;
humpIdentifier = string("BH") | string("DH");
Cmd_LNON = int_ >> -humpIdentifier >> "LNON";
commands = +( /* other rules | */ Cmd_LNON /*| other rules */ );
}
private:
qi::rule<It, std::vector<char>(), Skipper> humpIdentifier;
qi::rule<It, LineOnCommand(), Skipper> Cmd_LNON;
qi::rule<It, std::vector<LineOnCommand>(), Skipper> commands;
};
int main()
{
typedef std::string::const_iterator Iterator;
parser<Iterator> p;
std::string const input =
"123 BH LNON\n"
"124 LNON\t\t\t"
"125 DH LNON\n"
"126 INVALID LNON";
auto f(input.begin()), l(input.end());
std::vector<LineOnCommand> data;
bool success = qi::phrase_parse(f, l, p, ascii::space, data);
std::cout << "success:" << std::boolalpha << success << ", "
<< "elements: " << data.size() << "\n";
if (success)
{
for (auto& el : data)
{
std::cout << "Item: " << el.lineNum << ", humpType '" << std::string(el.humpType.begin(), el.humpType.end()) << "'\n";
}
}
if (f!=l)
std::cout << "Trailing unparsed: '" << std::string(f,l) << "'\n";
return success? 0 : 1;
}
Output:
success:true, elements: 3
Item: 123, humpType 'BH'
Item: 124, humpType ''
Item: 125, humpType 'DH'
Trailing unparsed: '126 INVALID LNON'