From: hartmut.kaiser_at_[hidden]
Date: 2008-04-17 21:12:11
Author: hkaiser
Date: 2008-04-17 21:12:10 EDT (Thu, 17 Apr 2008)
New Revision: 44528
URL: http://svn.boost.org/trac/boost/changeset/44528
Log:
Spirit.Lex: Added support for token ids other than std::size_t.
Text files modified:
trunk/boost/spirit/home/lex/lexer/lexer.hpp | 24 ++++++++++++---------
trunk/boost/spirit/home/lex/lexer/lexer_fwd.hpp | 3 +
trunk/boost/spirit/home/lex/lexer/lexertl/lexertl_lexer.hpp | 4 +++
trunk/boost/spirit/home/lex/lexer/lexertl/lexertl_token.hpp | 3 ++
trunk/boost/spirit/home/lex/lexer/meta_grammar.hpp | 9 +------
trunk/boost/spirit/home/lex/lexer/string_token_def.hpp | 4 ++
trunk/boost/spirit/home/lex/lexer/token_def.hpp | 44 +++++++++++++++++++++++++++------------
trunk/boost/spirit/home/lex/lexer/token_set.hpp | 20 ++++++++++--------
8 files changed, 69 insertions(+), 42 deletions(-)
Modified: trunk/boost/spirit/home/lex/lexer/lexer.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/lexer.hpp (original)
+++ trunk/boost/spirit/home/lex/lexer/lexer.hpp 2008-04-17 21:12:10 EDT (Thu, 17 Apr 2008)
@@ -47,6 +47,8 @@
typedef typename proto::terminal<terminal_holder>::type tag;
typedef proto::extends<tag, lexer_def_> base_type;
+ typedef typename LexerDef::id_type id_type;
+
tag make_tag() const
{
tag xpr = {{ this }};
@@ -65,7 +67,7 @@
// id, and the corresponding pair of iterators
typedef typename Iterator::base_iterator_type iterator_type;
typedef
- fusion::vector<std::size_t, iterator_range<iterator_type> >
+ fusion::vector<id_type, iterator_range<iterator_type> >
type;
};
@@ -84,7 +86,7 @@
token_type;
token_type &t = *first;
- if (0 != t.id()) {
+ if (token_is_valid(t)) {
// any of the token definitions matched
qi::detail::assign_to(t, attr);
++first;
@@ -109,31 +111,31 @@
{}
adder const&
- operator()(char_type c, std::size_t token_id = 0) const
+ operator()(char_type c, id_type token_id = 0) const
{
if (0 == token_id)
- token_id = static_cast<std::size_t>(c);
+ token_id = static_cast<id_type>(c);
def.def.add_token (def.state.c_str(), lex::detail::escape(c),
token_id);
return *this;
}
adder const&
- operator()(string_type const& s, std::size_t token_id = 0) const
+ operator()(string_type const& s, id_type token_id = id_type()) const
{
if (0 == token_id)
- token_id = next_id();
+ token_id = next_id<id_type>::get();
def.def.add_token (def.state.c_str(), s, token_id);
return *this;
}
template <typename Attribute>
adder const&
- operator()(token_def<Attribute, char_type>& tokdef,
- std::size_t token_id = 0) const
+ operator()(token_def<Attribute, char_type, id_type>& tokdef,
+ id_type token_id = id_type()) const
{
// make sure we have a token id
if (0 == token_id) {
if (0 == tokdef.id()) {
- token_id = next_id();
+ token_id = next_id<id_type>::get();
tokdef.id(token_id);
}
else {
@@ -281,13 +283,14 @@
class lexer_def : noncopyable, public Lexer
{
private:
- typedef lexer_def<Lexer> self_type;
+ typedef lexer_def self_type;
// avoid warnings about using 'this' in constructor
lexer_def& this_() { return *this; }
public:
typedef Lexer lexer_type;
+ typedef typename Lexer::id_type id_type;
typedef detail::lexer_def_<self_type> token_set;
typedef typename Lexer::char_type char_type;
typedef std::basic_string<char_type> string_type;
@@ -317,6 +320,7 @@
typedef typename Definition::lexer_type lexer_type;
typedef typename Definition::char_type char_type;
typedef typename Definition::iterator_type iterator_type;
+ typedef typename Definition::id_type id_type;
lexer(Definition& token_def_)
: token_def(token_def_)
Modified: trunk/boost/spirit/home/lex/lexer/lexer_fwd.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/lexer_fwd.hpp (original)
+++ trunk/boost/spirit/home/lex/lexer/lexer_fwd.hpp 2008-04-17 21:12:10 EDT (Thu, 17 Apr 2008)
@@ -17,7 +17,8 @@
///////////////////////////////////////////////////////////////////////////
// This component represents a token definition
///////////////////////////////////////////////////////////////////////////
- template<typename Attribute = unused_type, typename Char = char>
+ template<typename Attribute = unused_type, typename Char = char,
+ typename Idtype = std::size_t>
class token_def;
///////////////////////////////////////////////////////////////////////////
Modified: trunk/boost/spirit/home/lex/lexer/lexertl/lexertl_lexer.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/lexertl/lexertl_lexer.hpp (original)
+++ trunk/boost/spirit/home/lex/lexer/lexertl/lexertl_lexer.hpp 2008-04-17 21:12:10 EDT (Thu, 17 Apr 2008)
@@ -83,6 +83,9 @@
typedef std::basic_string<char_type> string_type;
public:
+ typedef Token token_type;
+ typedef typename Token::id_type id_type;
+
// interface for token definition management
void add_token (char_type const* state, string_type const& tokendef,
std::size_t token_id)
@@ -192,6 +195,7 @@
// Every lexer type to be used as a lexer for Spirit has to conform to
// a public interface .
typedef Token token_type;
+ typedef typename Token::id_type id_type;
typedef TokenSet token_set;
typedef lexertl_iterator<Functor> iterator_type;
Modified: trunk/boost/spirit/home/lex/lexer/lexertl/lexertl_token.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/lexertl/lexertl_token.hpp (original)
+++ trunk/boost/spirit/home/lex/lexer/lexertl/lexertl_token.hpp 2008-04-17 21:12:10 EDT (Thu, 17 Apr 2008)
@@ -53,6 +53,8 @@
// iterator_type The type of the iterator used to access the
// underlying character stream.
//
+ // id_type The type of the token id used.
+ //
// methods
// default constructor
// This should initialize the token as an end of
@@ -107,6 +109,7 @@
{
typedef Iterator iterator_type;
typedef mpl::false_ has_state;
+ typedef std::size_t id_type;
// default constructed tokens correspond to EOI tokens
lexertl_token()
Modified: trunk/boost/spirit/home/lex/lexer/meta_grammar.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/meta_grammar.hpp (original)
+++ trunk/boost/spirit/home/lex/lexer/meta_grammar.hpp 2008-04-17 21:12:10 EDT (Thu, 17 Apr 2008)
@@ -12,6 +12,7 @@
#endif
#include <boost/spirit/home/lex/domain.hpp>
+#include <boost/spirit/home/lex/lexer/lexer_fwd.hpp>
#include <boost/spirit/home/lex/lexer/terminal_holder.hpp>
#include <boost/spirit/home/support/placeholders.hpp>
#include <boost/spirit/home/support/meta_grammar.hpp>
@@ -29,12 +30,6 @@
struct string_token_def;
struct char_token_def;
- template<typename Attribute, typename Char>
- class token_def;
-
- template <typename TokenSet>
- class token_set;
-
struct lexer_meta_grammar;
template <typename Expr, typename Enable>
@@ -52,7 +47,7 @@
// token_def<>
meta_grammar::terminal_rule<
lex::domain,
- terminal_holder<proto::_, lex::token_def<proto::_, proto::_> >,
+ terminal_holder<proto::_, lex::token_def<proto::_, proto::_, proto::_> >,
terminal_director
>,
// token_set
Modified: trunk/boost/spirit/home/lex/lexer/string_token_def.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/string_token_def.hpp (original)
+++ trunk/boost/spirit/home/lex/lexer/string_token_def.hpp 2008-04-17 21:12:10 EDT (Thu, 17 Apr 2008)
@@ -27,7 +27,9 @@
collect(Component const& component, LexerDef& lexdef,
String const& state)
{
- lexdef.add_token (state.c_str(), subject(component), next_id());
+ typedef typename LexerDef::id_type id_type;
+ lexdef.add_token (state.c_str(), subject(component),
+ next_id<id_type>::get());
}
};
Modified: trunk/boost/spirit/home/lex/lexer/token_def.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/token_def.hpp (original)
+++ trunk/boost/spirit/home/lex/lexer/token_def.hpp 2008-04-17 21:12:10 EDT (Thu, 17 Apr 2008)
@@ -34,22 +34,37 @@
};
///////////////////////////////////////////////////////////////////////////
- inline std::size_t next_id()
+ // The next_id template needs to be specialized for any non-default token
+ // id type used by a custom token type. It need to expose a function
+ // 'static Idtype get()' returning the next available token id each time
+ // it is called.
+ template <typename Idtype>
+ struct next_id;
+
+ ///////////////////////////////////////////////////////////////////////////
+ // Default specialization for the next_id template returning the next
+ // available token id.
+ template <>
+ struct next_id<std::size_t>
{
- static std::size_t next_token_id = min_token_id;
- return next_token_id++;
- }
-
+ static std::size_t get()
+ {
+ static std::size_t next_token_id = min_token_id;
+ return next_token_id++;
+ }
+ };
+
///////////////////////////////////////////////////////////////////////////
// This component represents a token definition
///////////////////////////////////////////////////////////////////////////
- template<typename Attribute, typename Char>
+ template<typename Attribute, typename Char, typename Idtype>
class token_def
: public proto::extends<
typename make_terminal_holder<
- token_def<Attribute, Char>*, token_def<Attribute, Char>
+ token_def<Attribute, Char, Idtype>*,
+ token_def<Attribute, Char, Idtype>
>::type,
- token_def<Attribute, Char>
+ token_def<Attribute, Char, Idtype>
>
{
private:
@@ -132,23 +147,24 @@
{
token_state = lexdef.add_state(state.c_str());
if (0 == token_id)
- token_id = next_id();
+ token_id = next_id<Idtype>::get();
lexdef.add_token (state.c_str(), def, token_id);
}
public:
typedef Char char_type;
+ typedef Idtype id_type;
typedef std::basic_string<char_type> string_type;
// Lex interface: constructing token definitions
token_def()
: base_type(make_tag()), token_id(0), token_state(~0)
{}
- explicit token_def(char_type def_, std::size_t id_ = 0)
+ explicit token_def(char_type def_, Idtype id_ = Idtype())
: base_type(make_tag()), def(lex::detail::escape(def_)),
token_id(0 == id_ ? def_ : id_), token_state(~0)
{}
- explicit token_def(string_type def_, std::size_t id_ = 0)
+ explicit token_def(string_type def_, Idtype id_ = Idtype())
: base_type(make_tag()), def(def_), token_id(id_), token_state(~0)
{}
@@ -167,14 +183,14 @@
}
// general accessors
- std::size_t id() const { return token_id; }
- void id(std::size_t id) { token_id = id; }
+ Idtype id() const { return token_id; }
+ void id(Idtype id) { token_id = id; }
string_type const& definition() const { return def; }
std::size_t state() const { return token_state; }
private:
string_type def;
- std::size_t token_id;
+ Idtype token_id;
std::size_t token_state;
};
Modified: trunk/boost/spirit/home/lex/lexer/token_set.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/token_set.hpp (original)
+++ trunk/boost/spirit/home/lex/lexer/token_set.hpp 2008-04-17 21:12:10 EDT (Thu, 17 Apr 2008)
@@ -42,7 +42,7 @@
// avoid warnings about using 'this' in constructor
token_set& this_() { return *this; }
- typedef token_set<TokenSet> self_type;
+ typedef token_set self_type;
typedef TokenSet base_token_set;
// initialize proto base class
@@ -57,6 +57,8 @@
}
public:
+ typedef typename TokenSet::id_type id_type;
+
// Qi interface: metafunction calculating parser return type
template <typename Component, typename Context, typename Iterator>
struct attribute
@@ -65,7 +67,7 @@
// and the corresponding pair of iterators
typedef typename Iterator::base_iterator_type iterator_type;
typedef
- fusion::vector<std::size_t, iterator_range<iterator_type> >
+ fusion::vector<id_type, iterator_range<iterator_type> >
type;
};
@@ -88,7 +90,7 @@
BOOST_ASSERT(~0 != token_state);
token_type &t = *first;
- if (0 != t.id() && token_state == t.state()) {
+ if (token_is_valid(t) && token_state == t.state()) {
// any of the token definitions matched
qi::detail::assign_to(t, attr);
++first;
@@ -123,7 +125,7 @@
{}
adder const&
- operator()(char_type c, std::size_t token_id = 0) const
+ operator()(char_type c, id_type token_id = id_type()) const
{
if (0 == token_id)
token_id = static_cast<std::size_t>(c);
@@ -132,22 +134,22 @@
return *this;
}
adder const&
- operator()(string_type const& s, std::size_t token_id = 0) const
+ operator()(string_type const& s, id_type token_id = id_type()) const
{
if (0 == token_id)
- token_id = next_id();
+ token_id = next_id<id_type>::get();
def.add_token (def.initial_state().c_str(), s, token_id);
return *this;
}
template <typename Attribute>
adder const&
- operator()(token_def<Attribute, char_type>& tokdef,
- std::size_t token_id = 0) const
+ operator()(token_def<Attribute, char_type, id_type>& tokdef,
+ id_type token_id = id_type()) const
{
// make sure we have a token id
if (0 == token_id) {
if (0 == tokdef.id()) {
- token_id = next_id();
+ token_id = next_id<id_type>::get();
tokdef.id(token_id);
}
else {