Subject: [Boost-commit] svn:boost r73132 - trunk/libs/spirit/example/qi/compiler_tutorial/conjure2
From: hartmut.kaiser_at_[hidden]
Date: 2011-07-15 21:55:48
Author: hkaiser
Date: 2011-07-15 21:55:47 EDT (Fri, 15 Jul 2011)
New Revision: 73132
URL: http://svn.boost.org/trac/boost/changeset/73132
Log:
Spirit: simplified token-id interface for lexer
Text files modified:
trunk/libs/spirit/example/qi/compiler_tutorial/conjure2/function_def.hpp | 2 +-
trunk/libs/spirit/example/qi/compiler_tutorial/conjure2/lexer.cpp | 2 ++
trunk/libs/spirit/example/qi/compiler_tutorial/conjure2/lexer.hpp | 40 ++++++++++++++++++++++++++++++++++++++++
trunk/libs/spirit/example/qi/compiler_tutorial/conjure2/lexer_def.hpp | 28 ++++++++++++++++++++--------
trunk/libs/spirit/example/qi/compiler_tutorial/conjure2/statement_def.hpp | 10 +++++-----
trunk/libs/spirit/example/qi/compiler_tutorial/conjure2/token_ids.hpp | 8 ++------
6 files changed, 70 insertions(+), 20 deletions(-)
Modified: trunk/libs/spirit/example/qi/compiler_tutorial/conjure2/function_def.hpp
==============================================================================
--- trunk/libs/spirit/example/qi/compiler_tutorial/conjure2/function_def.hpp (original)
+++ trunk/libs/spirit/example/qi/compiler_tutorial/conjure2/function_def.hpp 2011-07-15 21:55:47 EDT (Fri, 15 Jul 2011)
@@ -39,7 +39,7 @@
identifier = body.expr.identifier;
argument_list = -(identifier % ',');
- start = (token(lexer::ID_VOID_KWD) | token(lexer::ID_INT_KWD))
+ start = (l.token("void") | l.token("int"))
> identifier
> '(' > argument_list > ')'
> '{' > body > '}'
Modified: trunk/libs/spirit/example/qi/compiler_tutorial/conjure2/lexer.cpp
==============================================================================
--- trunk/libs/spirit/example/qi/compiler_tutorial/conjure2/lexer.cpp (original)
+++ trunk/libs/spirit/example/qi/compiler_tutorial/conjure2/lexer.cpp 2011-07-15 21:55:47 EDT (Fri, 15 Jul 2011)
@@ -13,3 +13,5 @@
typedef std::string::const_iterator base_iterator_type;
template client::lexer::conjure_tokens<base_iterator_type>::conjure_tokens();
+template bool client::lexer::conjure_tokens<base_iterator_type>::add_keyword(
+ std::string const&);
Modified: trunk/libs/spirit/example/qi/compiler_tutorial/conjure2/lexer.hpp
==============================================================================
--- trunk/libs/spirit/example/qi/compiler_tutorial/conjure2/lexer.hpp (original)
+++ trunk/libs/spirit/example/qi/compiler_tutorial/conjure2/lexer.hpp 2011-07-15 21:55:47 EDT (Fri, 15 Jul 2011)
@@ -17,6 +17,7 @@
#include <boost/spirit/include/lex_static_lexertl.hpp>
#include "conjure_static_switch_lexer.hpp"
#endif
+#include <boost/assert.hpp>
namespace client { namespace lexer
{
@@ -67,13 +68,52 @@
struct conjure_tokens
: lex::lexer<typename detail::get_lexer_type<BaseIterator>::type>
{
+ private:
+ // get the type of any qi::raw_token(...) and qi::token(...) constructs
+ typedef typename boost::spirit::result_of::terminal<
+ boost::spirit::tag::raw_token(tokenids)
+ >::type raw_token_spec;
+
+ typedef typename boost::spirit::result_of::terminal<
+ boost::spirit::tag::token(tokenids)
+ >::type token_spec;
+
+ typedef std::map<std::string, tokenids> keyword_map_type;
+
+ protected:
+ // add a keyword to the mapping table
+ bool add_keyword(std::string const& keyword);
+
+ public:
typedef BaseIterator base_iterator_type;
conjure_tokens();
+ // extract a raw_token(id) when given a registered keyword
+ raw_token_spec raw_token (std::string const& kwd) const
+ {
+ namespace qi = boost::spirit::qi;
+ qi::raw_token_type raw_token;
+
+ typename keyword_map_type::const_iterator it = keywords_.find(kwd);
+ BOOST_ASSERT(it != keywords_.end());
+ return qi::raw_token((it != keywords_.end()) ? (*it).second : ID_INVALID);
+ }
+
+ token_spec token (std::string const& kwd) const
+ {
+ namespace qi = boost::spirit::qi;
+ qi::token_type token;
+
+ typename keyword_map_type::const_iterator it = keywords_.find(kwd);
+ BOOST_ASSERT(it != keywords_.end());
+ return qi::token((it != keywords_.end()) ? (*it).second : ID_INVALID);
+ }
+
lex::token_def<std::string> identifier;
lex::token_def<unsigned int> uint_;
lex::token_def<bool> bool_;
+ keyword_map_type keywords_;
};
}}
Modified: trunk/libs/spirit/example/qi/compiler_tutorial/conjure2/lexer_def.hpp
==============================================================================
--- trunk/libs/spirit/example/qi/compiler_tutorial/conjure2/lexer_def.hpp (original)
+++ trunk/libs/spirit/example/qi/compiler_tutorial/conjure2/lexer_def.hpp 2011-07-15 21:55:47 EDT (Fri, 15 Jul 2011)
@@ -19,14 +19,12 @@
this->self = uint_ | bool_;
- this->self.add
- ("void", ID_VOID_KWD)
- ("int", ID_INT_KWD)
- ("if", ID_IF_KWD)
- ("else", ID_ELSE_KWD)
- ("while", ID_WHILE_KWD)
- ("return", ID_RETURN_KWD)
- ;
+ add_keyword("void");
+ add_keyword("int");
+ add_keyword("if");
+ add_keyword("else");
+ add_keyword("while");
+ add_keyword("return");
this->self.add
("\\|\\|", ID_OP_LOGICAL_OR)
@@ -58,5 +56,19 @@
]
;
}
+
+ template <typename BaseIterator>
+ bool conjure_tokens<BaseIterator>::add_keyword(std::string const& keyword)
+ {
+ // ad the token to the lexer
+ tokenids id = tokenids(this->get_next_id());
+ this->self.add(keyword, id);
+
+ // store the mapping for later retrieval
+ std::pair<typename keyword_map_type::iterator, bool> p =
+ keywords_.insert(typename keyword_map_type::value_type(keyword, id));
+
+ return p.second;
+ }
}}
Modified: trunk/libs/spirit/example/qi/compiler_tutorial/conjure2/statement_def.hpp
==============================================================================
--- trunk/libs/spirit/example/qi/compiler_tutorial/conjure2/statement_def.hpp (original)
+++ trunk/libs/spirit/example/qi/compiler_tutorial/conjure2/statement_def.hpp 2011-07-15 21:55:47 EDT (Fri, 15 Jul 2011)
@@ -50,7 +50,7 @@
;
variable_declaration =
- raw_token(lexer::ID_INT_KWD)
+ l.raw_token("int")
> expr.identifier
> -('=' > expr)
> ';'
@@ -64,20 +64,20 @@
;
if_statement =
- raw_token(lexer::ID_IF_KWD)
+ l.raw_token("if")
> '('
> expr
> ')'
> statement_
>
-(
- raw_token(lexer::ID_ELSE_KWD)
+ l.raw_token("else")
> statement_
)
;
while_statement =
- raw_token(lexer::ID_WHILE_KWD)
+ l.raw_token("while")
> '('
> expr
> ')'
@@ -89,7 +89,7 @@
;
return_statement =
- raw_token(lexer::ID_RETURN_KWD)
+ l.raw_token("return")
> -expr
> ';'
;
Modified: trunk/libs/spirit/example/qi/compiler_tutorial/conjure2/token_ids.hpp
==============================================================================
--- trunk/libs/spirit/example/qi/compiler_tutorial/conjure2/token_ids.hpp (original)
+++ trunk/libs/spirit/example/qi/compiler_tutorial/conjure2/token_ids.hpp 2011-07-15 21:55:47 EDT (Fri, 15 Jul 2011)
@@ -16,6 +16,8 @@
enum tokenids
{
+ ID_INVALID = - 1,
+
ID_OP_OPERATOR = 0x10000,
ID_OP_BINARY = 0x20000,
ID_OP_UNARY = 0x40000,
@@ -39,12 +41,6 @@
ID_IDENTIFIER = ID_OP_OPERATOR + 100,
ID_COMMENT,
ID_WHITESPACE,
- ID_VOID_KWD,
- ID_INT_KWD,
- ID_IF_KWD,
- ID_ELSE_KWD,
- ID_WHILE_KWD,
- ID_RETURN_KWD,
ID_UINT,
ID_BOOL
};