Subject: [Boost-commit] svn:boost r54689 - in trunk/libs/spirit: doc example/lex test/lex
From: hartmut.kaiser_at_[hidden]
Date: 2009-07-05 17:28:27
Author: hkaiser
Date: 2009-07-05 17:28:26 EDT (Sun, 05 Jul 2009)
New Revision: 54689
URL: http://svn.boost.org/trac/boost/changeset/54689
Log:
Spirit.Lex: removed token_set class
Text files modified:
trunk/libs/spirit/doc/what_s_new.qbk | 11 ++++++-----
trunk/libs/spirit/example/lex/example3.cpp | 24 +++++++++---------------
trunk/libs/spirit/example/lex/example4.cpp | 33 +++++++++++++--------------------
trunk/libs/spirit/example/lex/example5.cpp | 27 ++++++++++-----------------
trunk/libs/spirit/example/lex/example6.cpp | 37 ++++++++++++++-----------------------
trunk/libs/spirit/test/lex/lexertl3.cpp | 10 +++-------
trunk/libs/spirit/test/lex/lexertl4.cpp | 16 +++++++---------
trunk/libs/spirit/test/lex/lexertl5.cpp | 14 +++++---------
8 files changed, 67 insertions(+), 105 deletions(-)
Modified: trunk/libs/spirit/doc/what_s_new.qbk
==============================================================================
--- trunk/libs/spirit/doc/what_s_new.qbk (original)
+++ trunk/libs/spirit/doc/what_s_new.qbk 2009-07-05 17:28:26 EDT (Sun, 05 Jul 2009)
@@ -116,11 +116,12 @@
Both take a possibly arbitrary number of attribute arguments as its last
parameters (well, the number of attributes is limited by the macro
`SPIRIT_ARGUMENTS_LIMIT`, which defaults to `PHOENIX_LIMIT`).
-* The `lex::lexertl_lexer`, `lex::lexertl_token_set`, and `lex::lexertl_token`
+* The `lex::lexertl_lexer`, and `lex::lexertl_token`
classes have been moved to the `lex::lexertl` namespace and the names have been
- changed to `lex::lexertl::lexer`, `lex::lexertl::token_set`, `lex::lexertl::token`
- (the same applies to the `lex::lexert_actor_lexer`, and the `static_lexertl_*`
- family of types).
+ changed to `lex::lexertl::lexer`, `lex::lexertl::token` (the same applies to
+ the `lex::lexert_actor_lexer`, and the `static_lexertl_*` family of types).
+* The class `lex::lexertl_token_set` has been removed alltogether as its
+ complete functionality is available from the lexer class.
* The __lex__ library has been updated to use the newest version of Ben
Hansons __lexertl__ lexer construction library (Boost review pending).
* The `lex::lexer<Lexer>` template constructor now takes an optional parameter
@@ -139,7 +140,7 @@
* Lexer semantic actions now have to conform to a changed interface (see
__sec_lex_semactions__ for details).
* Added placeholder symbols usable from the inside of lexer semantic actions
- while using Phoenix: _start, _end, _eoi, _state, and _pass (see
+ while using Phoenix: _start, _end, _eoi, _state, _value, and _pass (see
__sec_lex_semactions__ for more details).
* Added (lazy) support functions usable from the inside of lexer semantic
actions while using Phoenix: `lex::more()`, `lex::less()`, and
Modified: trunk/libs/spirit/example/lex/example3.cpp
==============================================================================
--- trunk/libs/spirit/example/lex/example3.cpp (original)
+++ trunk/libs/spirit/example/lex/example3.cpp 2009-07-05 17:28:26 EDT (Sun, 05 Jul 2009)
@@ -41,29 +41,25 @@
template <typename Lexer>
struct example3_tokens : lexer<Lexer>
{
- typedef typename Lexer::token_set token_set;
-
example3_tokens()
{
// define the tokens to match
ellipses = "\\.\\.\\.";
number = "[0-9]+";
+ // associate the tokens and the token set with the lexer
+ this->self = ellipses | '(' | ')' | number;
+
// define the whitespace to ignore (spaces, tabs, newlines and C-style
// comments)
- white_space
+ this->self("WS")
= token_def<>("[ \\t\\n]+") // whitespace
| "\\/\\*[^*]*\\*+([^/*][^*]*\\*+)*\\/" // C style comments
;
-
- // associate the tokens and the token set with the lexer
- this->self = ellipses | '(' | ')' | number;
- this->self("WS") = white_space;
}
// these tokens expose the iterator_range of the matched input sequence
token_def<> ellipses, identifier, number;
- token_set white_space;
};
///////////////////////////////////////////////////////////////////////////////
@@ -71,7 +67,7 @@
///////////////////////////////////////////////////////////////////////////////
template <typename Iterator, typename Lexer>
struct example3_grammar
- : grammar<Iterator, in_state_skipper<typename Lexer::token_set> >
+ : grammar<Iterator, in_state_skipper<Lexer> >
{
template <typename TokenDef>
example3_grammar(TokenDef const& tok)
@@ -95,8 +91,7 @@
BOOST_SPIRIT_DEBUG_NODE(couplet);
}
- typedef typename Lexer::token_set token_set;
- rule<Iterator, in_state_skipper<token_set> > start, couplet;
+ rule<Iterator, in_state_skipper<Lexer> > start, couplet;
};
///////////////////////////////////////////////////////////////////////////////
@@ -119,7 +114,7 @@
typedef example3_tokens::iterator_type iterator_type;
// this is the type of the grammar to parse
- typedef example3_grammar<iterator_type, lexer_type> example3_grammar;
+ typedef example3_grammar<iterator_type, example3_tokens::lexer_def> example3_grammar;
// now we use the types defined above to create the lexer and grammar
// object instances needed to invoke the parsing process
@@ -136,9 +131,8 @@
// Parsing is done based on the the token stream, not the character
// stream read from the input.
- // Note, how we use the token_set defined above as the skip parser.
- std::string ws("WS");
- bool r = phrase_parse(iter, end, calc, in_state(ws)[tokens.white_space]);
+ // Note how we use the lexer defined above as the skip parser.
+ bool r = phrase_parse(iter, end, calc, in_state("WS")[tokens.self]);
if (r && iter == end)
{
Modified: trunk/libs/spirit/example/lex/example4.cpp
==============================================================================
--- trunk/libs/spirit/example/lex/example4.cpp (original)
+++ trunk/libs/spirit/example/lex/example4.cpp 2009-07-05 17:28:26 EDT (Sun, 05 Jul 2009)
@@ -44,8 +44,6 @@
template <typename Lexer>
struct example4_tokens : lexer<Lexer>
{
- typedef typename Lexer::token_set token_set;
-
example4_tokens()
{
// define the tokens to match
@@ -55,17 +53,16 @@
else_ = "else";
while_ = "while";
+ // associate the tokens and the token set with the lexer
+ this->self = token_def<>('(') | ')' | '{' | '}' | '=' | ';' | constant;
+ this->self += if_ | else_ | while_ | identifier;
+
// define the whitespace to ignore (spaces, tabs, newlines and C-style
// comments)
- white_space
+ this->self("WS")
= token_def<>("[ \\t\\n]+")
| "\\/\\*[^*]*\\*+([^/*][^*]*\\*+)*\\/"
;
-
- // associate the tokens and the token set with the lexer
- this->self = token_def<>('(') | ')' | '{' | '}' | '=' | ';' | constant;
- this->self += if_ | else_ | while_ | identifier;
- this->self("WS") = white_space;
}
//[example4_token_def
@@ -88,9 +85,6 @@
token_def<std::string> identifier;
token_def<unsigned int> constant;
//]
-
- // token set to be used as the skip parser
- token_set white_space;
};
///////////////////////////////////////////////////////////////////////////////
@@ -98,7 +92,7 @@
///////////////////////////////////////////////////////////////////////////////
template <typename Iterator, typename Lexer>
struct example4_grammar
- : grammar<Iterator, in_state_skipper<typename Lexer::token_set> >
+ : grammar<Iterator, in_state_skipper<Lexer> >
{
template <typename TokenDef>
example4_grammar(TokenDef const& tok)
@@ -150,15 +144,14 @@
;
}
- typedef typename Lexer::token_set token_set;
typedef boost::variant<unsigned int, std::string> expression_type;
- rule<Iterator, in_state_skipper<token_set> > program, block, statement;
- rule<Iterator, in_state_skipper<token_set> > assignment, if_stmt;
- rule<Iterator, in_state_skipper<token_set> > while_stmt;
+ rule<Iterator, in_state_skipper<Lexer> > program, block, statement;
+ rule<Iterator, in_state_skipper<Lexer> > assignment, if_stmt;
+ rule<Iterator, in_state_skipper<Lexer> > while_stmt;
// the expression is the only rule having a return value
- rule<Iterator, expression_type(), in_state_skipper<token_set> > expression;
+ rule<Iterator, expression_type(), in_state_skipper<Lexer> > expression;
};
///////////////////////////////////////////////////////////////////////////////
@@ -195,7 +188,7 @@
typedef example4_tokens::iterator_type iterator_type;
// this is the type of the grammar to parse
- typedef example4_grammar<iterator_type, lexer_type> example4_grammar;
+ typedef example4_grammar<iterator_type, example4_tokens::lexer_def> example4_grammar;
// now we use the types defined above to create the lexer and grammar
// object instances needed to invoke the parsing process
@@ -212,10 +205,10 @@
// Parsing is done based on the the token stream, not the character
// stream read from the input.
- // Note, how we use the token_set defined above as the skip parser. It must
+ // Note how we use the lexer defined above as the skip parser. It must
// be explicitly wrapped inside a state directive, switching the lexer
// state for the duration of skipping whitespace.
- bool r = phrase_parse(iter, end, calc, in_state("WS")[tokens.white_space]);
+ bool r = phrase_parse(iter, end, calc, in_state("WS")[tokens.self]);
if (r && iter == end)
{
Modified: trunk/libs/spirit/example/lex/example5.cpp
==============================================================================
--- trunk/libs/spirit/example/lex/example5.cpp (original)
+++ trunk/libs/spirit/example/lex/example5.cpp 2009-07-05 17:28:26 EDT (Sun, 05 Jul 2009)
@@ -46,8 +46,6 @@
template <typename Lexer>
struct example5_base_tokens : lexer<Lexer>
{
- typedef typename Lexer::token_set token_set;
-
protected:
// this lexer is supposed to be used as a base type only
example5_base_tokens() {}
@@ -61,17 +59,16 @@
if_ = "if";
while_ = "while";
+ // associate the tokens and the token set with the lexer
+ this->self += token_def<>('(') | ')' | '{' | '}' | '=' | ';' | constant;
+ this->self += if_ | while_ | identifier;
+
// define the whitespace to ignore (spaces, tabs, newlines and C-style
// comments)
- white_space
+ this->self("WS")
= token_def<>("[ \\t\\n]+")
| "\\/\\*[^*]*\\*+([^/*][^*]*\\*+)*\\/"
;
-
- // associate the tokens and the token set with the lexer
- this->self += token_def<>('(') | ')' | '{' | '}' | '=' | ';' | constant;
- this->self += if_ | while_ | identifier;
- this->self("WS") = white_space;
}
// these tokens have no attribute
@@ -92,9 +89,6 @@
// avoiding them being copied around.
token_def<std::string> identifier;
token_def<unsigned int> constant;
-
- // token set to be used as the skip parser
- token_set white_space;
};
///////////////////////////////////////////////////////////////////////////////
@@ -102,7 +96,7 @@
///////////////////////////////////////////////////////////////////////////////
template <typename Iterator, typename Lexer>
struct example5_base_grammar
- : grammar<Iterator, in_state_skipper<typename Lexer::token_set> >
+ : grammar<Iterator, in_state_skipper<Lexer> >
{
template <typename TokenDef>
example5_base_grammar(TokenDef const& tok)
@@ -152,7 +146,7 @@
;
}
- typedef in_state_skipper<typename Lexer::token_set> skipper_type;
+ typedef in_state_skipper<Lexer> skipper_type;
rule<Iterator, skipper_type> program, block, statement;
rule<Iterator, skipper_type> assignment, if_stmt;
@@ -170,7 +164,6 @@
struct example5_tokens : example5_base_tokens<Lexer>
{
typedef example5_base_tokens<Lexer> base_type;
- typedef typename Lexer::token_set token_set;
example5_tokens()
{
@@ -240,7 +233,7 @@
typedef example5_tokens::iterator_type iterator_type;
// this is the type of the grammar to parse
- typedef example5_grammar<iterator_type, lexer_type> example5_grammar;
+ typedef example5_grammar<iterator_type, example5_tokens::lexer_def> example5_grammar;
// now we use the types defined above to create the lexer and grammar
// object instances needed to invoke the parsing process
@@ -257,11 +250,11 @@
// Parsing is done based on the the token stream, not the character
// stream read from the input.
- // Note, how we use the token_set defined above as the skip parser. It must
+ // Note how we use the lexer defined above as the skip parser. It must
// be explicitly wrapped inside a state directive, switching the lexer
// state for the duration of skipping whitespace.
std::string ws("WS");
- bool r = phrase_parse(iter, end, calc, in_state(ws)[tokens.white_space]);
+ bool r = phrase_parse(iter, end, calc, in_state(ws)[tokens.self]);
if (r && iter == end)
{
Modified: trunk/libs/spirit/example/lex/example6.cpp
==============================================================================
--- trunk/libs/spirit/example/lex/example6.cpp (original)
+++ trunk/libs/spirit/example/lex/example6.cpp 2009-07-05 17:28:26 EDT (Sun, 05 Jul 2009)
@@ -60,21 +60,12 @@
template <typename Lexer>
struct example6_tokens : lexer<Lexer>
{
- typedef typename Lexer::token_set token_set;
-
example6_tokens()
{
// define the tokens to match
identifier = "[a-zA-Z_][a-zA-Z0-9_]*";
constant = "[0-9]+";
- // define the whitespace to ignore (spaces, tabs, newlines and C-style
- // comments)
- white_space
- = token_def<>("[ \\t\\n]+")
- | "\\/\\*[^*]*\\*+([^/*][^*]*\\*+)*\\/"
- ;
-
// associate the tokens and the token set with the lexer
this->self = token_def<>('(') | ')' | '{' | '}' | '=' | ';';
@@ -90,8 +81,12 @@
(identifier, ID_IDENTIFIER)
;
- // add whitespace tokens to another lexer state (here: "WS")
- this->self("WS") = white_space;
+ // define the whitespace to ignore (spaces, tabs, newlines and C-style
+ // comments) and add those to another lexer state (here: "WS")
+ this->self("WS")
+ = token_def<>("[ \\t\\n]+")
+ | "\\/\\*[^*]*\\*+([^/*][^*]*\\*+)*\\/"
+ ;
}
// The following two tokens have an associated attribute type, identifier
@@ -109,9 +104,6 @@
// avoiding them being copied around.
token_def<std::string> identifier;
token_def<unsigned int> constant;
-
- // token set to be used as the skip parser
- token_set white_space;
};
///////////////////////////////////////////////////////////////////////////////
@@ -119,7 +111,7 @@
///////////////////////////////////////////////////////////////////////////////
template <typename Iterator, typename Lexer>
struct example6_grammar
- : grammar<Iterator, in_state_skipper<typename Lexer::token_set> >
+ : grammar<Iterator, in_state_skipper<Lexer> >
{
template <typename TokenDef>
example6_grammar(TokenDef const& tok)
@@ -174,15 +166,14 @@
;
}
- typedef typename Lexer::token_set token_set;
typedef boost::variant<unsigned int, std::string> expression_type;
- rule<Iterator, in_state_skipper<token_set> > program, block, statement;
- rule<Iterator, in_state_skipper<token_set> > assignment, if_stmt;
- rule<Iterator, in_state_skipper<token_set> > while_stmt;
+ rule<Iterator, in_state_skipper<Lexer> > program, block, statement;
+ rule<Iterator, in_state_skipper<Lexer> > assignment, if_stmt;
+ rule<Iterator, in_state_skipper<Lexer> > while_stmt;
// the expression is the only rule having a return value
- rule<Iterator, expression_type(), in_state_skipper<token_set> > expression;
+ rule<Iterator, expression_type(), in_state_skipper<Lexer> > expression;
};
///////////////////////////////////////////////////////////////////////////////
@@ -218,7 +209,7 @@
typedef example6_tokens::iterator_type iterator_type;
// this is the type of the grammar to parse
- typedef example6_grammar<iterator_type, lexer_type> example6_grammar;
+ typedef example6_grammar<iterator_type, example6_tokens::lexer_def> example6_grammar;
// now we use the types defined above to create the lexer and grammar
// object instances needed to invoke the parsing process
@@ -235,11 +226,11 @@
// Parsing is done based on the the token stream, not the character
// stream read from the input.
- // Note, how we use the token_def defined above as the skip parser. It must
+ // Note how we use the lexer defined above as the skip parser. It must
// be explicitly wrapped inside a state directive, switching the lexer
// state for the duration of skipping whitespace.
std::string ws("WS");
- bool r = phrase_parse(iter, end, calc, in_state(ws)[tokens.white_space]);
+ bool r = phrase_parse(iter, end, calc, in_state(ws)[tokens.self]);
if (r && iter == end)
{
Modified: trunk/libs/spirit/test/lex/lexertl3.cpp
==============================================================================
--- trunk/libs/spirit/test/lex/lexertl3.cpp (original)
+++ trunk/libs/spirit/test/lex/lexertl3.cpp 2009-07-05 17:28:26 EDT (Sun, 05 Jul 2009)
@@ -21,18 +21,17 @@
std::size_t const CPPCOMMENT = 2;
token_def c_comment ("\\/\\*[^*]*\\*+([^/*][^*]*\\*+)*\\/", CCOMMENT);
token_def cpp_comment ("\\/\\/[^\\n\\r]*(\\n|\\r|\\r\\n)", CPPCOMMENT);
- token_def ws_tok ("[\\v\\f\\n\\r]*");
typedef std::string::iterator base_iterator_type;
typedef lex::lexertl::token<base_iterator_type> token_type;
typedef lex::lexertl::lexer<token_type> lexer_type;
typedef lex::lexer<lexer_type> lexer_def;
- typedef lexer_def::token_set token_set;
{
// initialize lexer
std::string str("def");
+ token_def ws_tok ("[\\v\\f\\n\\r]*");
lexer_def lex;
lex.self = c_comment;
lex.self += cpp_comment | '1' | '2' | '3' | "abc" | str;
@@ -49,15 +48,12 @@
}
{
- // init a token set
- token_set ws;
- ws = token_def(' ') | '\t' | ws_tok;
-
// initialize lexer
lexer_def lex;
+ token_def ws_tok ("[\\v\\f\\n\\r]*");
lex.self = c_comment;
lex.self += cpp_comment | '1' | '2' | '3';
- lex.self("WHITESPACE") = ws;
+ lex.self("WHITESPACE") = token_def(' ') | '\t' | ws_tok;
// test lexer for two different input strings
BOOST_TEST(test (lex, "/* this is a comment */", CCOMMENT));
Modified: trunk/libs/spirit/test/lex/lexertl4.cpp
==============================================================================
--- trunk/libs/spirit/test/lex/lexertl4.cpp (original)
+++ trunk/libs/spirit/test/lex/lexertl4.cpp 2009-07-05 17:28:26 EDT (Sun, 05 Jul 2009)
@@ -32,13 +32,13 @@
typedef lex::lexertl::lexer<token_type> lexer_type;
typedef lex::lexer<lexer_type> lexer_def;
- typedef lexer_def::token_set token_set;
std::string str("def");
{
// initialize lexer
lexer_def lex;
+ token_def ws_tok ("[\\v\\f\\n\\r]*", TOKEN_ID_WS);
lex.self.add
(c_comment)(cpp_comment)
('1')('2')('3')
@@ -58,15 +58,10 @@
}
{
- // init a token set
- token_set ws;
- ws.add
- (' ')('\t')
- (ws_tok)
- ;
-
// initialize lexer
lexer_def lex;
+ token_def ws_tok ("[\\v\\f\\n\\r]*", TOKEN_ID_WS);
+
lex.self.add
(c_comment)(cpp_comment)
('1')('2')('3')
@@ -74,7 +69,10 @@
(str, TOKEN_ID_STR)
;
- lex.self("WHITESPACE").add(ws);
+ lex.self("WHITESPACE").add
+ (' ')('\t')
+ (ws_tok)
+ ;
// test lexer for different input strings
BOOST_TEST(test (lex, "/* this is a comment */", CCOMMENT));
Modified: trunk/libs/spirit/test/lex/lexertl5.cpp
==============================================================================
--- trunk/libs/spirit/test/lex/lexertl5.cpp (original)
+++ trunk/libs/spirit/test/lex/lexertl5.cpp 2009-07-05 17:28:26 EDT (Sun, 05 Jul 2009)
@@ -29,7 +29,6 @@
typedef lex::lexertl::lexer<token_type> lexer_type;
typedef lex::lexer<lexer_type> lexer_def;
- typedef lexer_def::token_set token_set;
std::string str("def");
@@ -79,13 +78,7 @@
token_def cpp_comment ("{CPPCOMMENT}", CPPCOMMENT);
token_def ws_tok ("{WS}");
- // init a token set
- token_set ws;
- ws.add
- (' ')('\t')
- (ws_tok, TOKEN_ID_WS)
- ;
-
+ // init lexer
lex.self.add
(c_comment)(cpp_comment)
('1')('2')('3')
@@ -93,7 +86,10 @@
(str, TOKEN_ID_STR)
;
- lex.self("WHITESPACE").add(ws);
+ lex.self("WHITESPACE").add
+ (' ')('\t')
+ (ws_tok, TOKEN_ID_WS)
+ ;
// test lexer for different input strings
BOOST_TEST(test (lex, "/* this is a comment */", CCOMMENT));