Subject: [Boost-commit] svn:boost r66778 - trunk/boost/spirit/home/lex/lexer/lexertl
From: hartmut.kaiser_at_[hidden]
Date: 2010-11-26 14:01:18
Author: hkaiser
Date: 2010-11-26 14:01:13 EST (Fri, 26 Nov 2010)
New Revision: 66778
URL: http://svn.boost.org/trac/boost/changeset/66778
Log:
Spirit: adding another template parameter to lex::lexertl::token<>: the Idtype
Text files modified:
trunk/boost/spirit/home/lex/lexer/lexertl/token.hpp | 87 ++++++++++++++++++++++-----------------
1 files changed, 49 insertions(+), 38 deletions(-)
Modified: trunk/boost/spirit/home/lex/lexer/lexertl/token.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/lexertl/token.hpp (original)
+++ trunk/boost/spirit/home/lex/lexer/lexertl/token.hpp 2010-11-26 14:01:13 EST (Fri, 26 Nov 2010)
@@ -53,6 +53,8 @@
// by this token type.
// HasState A mpl::bool_ indicating, whether this token type
// should support lexer states.
+ // Idtype The type to use for the token id (defaults to
+ // std::size_t).
//
// It is possible to use other token types with the spirit::lex
// framework as well. If you plan to use a different type as your token
@@ -106,19 +108,20 @@
///////////////////////////////////////////////////////////////////////////
template <typename Iterator = char const*
, typename AttributeTypes = mpl::vector0<>
- , typename HasState = mpl::true_>
+ , typename HasState = mpl::true_
+ , typename Idtyep = std::size_t>
struct token;
///////////////////////////////////////////////////////////////////////////
// This specialization of the token type doesn't contain any item data and
// doesn't support working with lexer states.
///////////////////////////////////////////////////////////////////////////
- template <typename Iterator>
- struct token<Iterator, lex::omit, mpl::false_>
+ template <typename Iterator, typename Idtype>
+ struct token<Iterator, lex::omit, mpl::false_, Idtype>
{
typedef Iterator iterator_type;
typedef mpl::false_ has_state;
- typedef std::size_t id_type;
+ typedef Idtype id_type;
typedef unused_type token_value_type;
// default constructed tokens correspond to EOI tokens
@@ -188,10 +191,10 @@
#if defined(BOOST_SPIRIT_DEBUG)
template <typename Char, typename Traits, typename Iterator
- , typename AttributeTypes, typename HasState>
+ , typename AttributeTypes, typename HasState, typename Idtype>
inline std::basic_ostream<Char, Traits>&
operator<< (std::basic_ostream<Char, Traits>& os
- , token<Iterator, AttributeTypes, HasState> const& t)
+ , token<Iterator, AttributeTypes, HasState, Idtype> const& t)
{
if (t) {
Iterator end = t.matched_.second;
@@ -209,12 +212,12 @@
// This specialization of the token type doesn't contain any item data but
// supports working with lexer states.
///////////////////////////////////////////////////////////////////////////
- template <typename Iterator>
- struct token<Iterator, lex::omit, mpl::true_>
- : token<Iterator, lex::omit, mpl::false_>
+ template <typename Iterator, typename Idtype>
+ struct token<Iterator, lex::omit, mpl::true_, Idtype>
+ : token<Iterator, lex::omit, mpl::false_, Idtype>
{
private:
- typedef token<Iterator, lex::omit, mpl::false_> base_type;
+ typedef token<Iterator, lex::omit, mpl::false_, Idtype> base_type;
public:
typedef typename base_type::id_type id_type;
@@ -305,15 +308,16 @@
{};
}
- template <typename Iterator, typename AttributeTypes, typename HasState>
- struct token : token<Iterator, lex::omit, HasState>
+ template <typename Iterator, typename AttributeTypes, typename HasState
+ , typename Idtype>
+ struct token : token<Iterator, lex::omit, HasState, Idtype>
{
private: // precondition assertions
#if !BOOST_WORKAROUND(BOOST_MSVC, <= 1300)
BOOST_STATIC_ASSERT((mpl::is_sequence<AttributeTypes>::value ||
is_same<AttributeTypes, lex::omit>::value));
#endif
- typedef token<Iterator, lex::omit, HasState> base_type;
+ typedef token<Iterator, lex::omit, HasState, Idtype> base_type;
protected:
// If no additional token value types are given, the the token will
@@ -374,10 +378,11 @@
///////////////////////////////////////////////////////////////////////////
// tokens are considered equal, if their id's match (these are unique)
- template <typename Iterator, typename AttributeTypes, typename HasState>
+ template <typename Iterator, typename AttributeTypes, typename HasState
+ , typename Idtype>
inline bool
- operator== (token<Iterator, AttributeTypes, HasState> const& lhs,
- token<Iterator, AttributeTypes, HasState> const& rhs)
+ operator== (token<Iterator, AttributeTypes, HasState, Idtype> const& lhs,
+ token<Iterator, AttributeTypes, HasState, Idtype> const& rhs)
{
return lhs.id() == rhs.id();
}
@@ -387,9 +392,10 @@
// validate a token instance. It has to be defined in the same namespace
// as the token class itself to allow ADL to find it.
///////////////////////////////////////////////////////////////////////////
- template <typename Iterator, typename AttributeTypes, typename HasState>
+ template <typename Iterator, typename AttributeTypes, typename HasState
+ , typename Idtype>
inline bool
- token_is_valid(token<Iterator, AttributeTypes, HasState> const& t)
+ token_is_valid(token<Iterator, AttributeTypes, HasState, Idtype> const& t)
{
return t.is_valid();
}
@@ -407,12 +413,12 @@
// This is called from the parse function of token_def if the token_def
// has been defined to carry a special attribute type
template <typename Attribute, typename Iterator, typename AttributeTypes
- , typename HasState>
+ , typename HasState, typename Idtype>
struct assign_to_attribute_from_value<Attribute
- , lex::lexertl::token<Iterator, AttributeTypes, HasState> >
+ , lex::lexertl::token<Iterator, AttributeTypes, HasState, Idtype> >
{
static void
- call(lex::lexertl::token<Iterator, AttributeTypes, HasState> const& t
+ call(lex::lexertl::token<Iterator, AttributeTypes, HasState, Idtype> const& t
, Attribute& attr)
{
// The goal of this function is to avoid the conversion of the pair of
@@ -462,7 +468,7 @@
// tokenization.
typedef lex::lexertl::token<
- Iterator, AttributeTypes, HasState> token_type;
+ Iterator, AttributeTypes, HasState, Idtype> token_type;
const_cast<token_type&>(t).value() = attr; // re-assign value
}
else {
@@ -474,12 +480,13 @@
// These are called from the parse function of token_def if the token type
// has no special attribute type assigned
- template <typename Attribute, typename Iterator, typename HasState>
+ template <typename Attribute, typename Iterator, typename HasState
+ , typename Idtype>
struct assign_to_attribute_from_value<
- Attribute, lex::lexertl::token<Iterator, mpl::vector0<>, HasState> >
+ Attribute, lex::lexertl::token<Iterator, mpl::vector0<>, HasState, Idtype> >
{
static void
- call(lex::lexertl::token<Iterator, mpl::vector0<>, HasState> const& t
+ call(lex::lexertl::token<Iterator, mpl::vector0<>, HasState, Idtype> const& t
, Attribute& attr)
{
// The default type returned by the token_def parser component (if
@@ -490,12 +497,13 @@
};
// same as above but using mpl::vector<> instead of mpl::vector0<>
- template <typename Attribute, typename Iterator, typename HasState>
+ template <typename Attribute, typename Iterator, typename HasState
+ , typename Idtype>
struct assign_to_attribute_from_value<
- Attribute, lex::lexertl::token<Iterator, mpl::vector<>, HasState> >
+ Attribute, lex::lexertl::token<Iterator, mpl::vector<>, HasState, Idtype> >
{
static void
- call(lex::lexertl::token<Iterator, mpl::vector<>, HasState> const& t
+ call(lex::lexertl::token<Iterator, mpl::vector<>, HasState, Idtype> const& t
, Attribute& attr)
{
// The default type returned by the token_def parser component (if
@@ -509,12 +517,13 @@
// has been explicitly omitted (i.e. no attribute value is used), which
// essentially means that every attribute gets initialized using default
// constructed values.
- template <typename Attribute, typename Iterator, typename HasState>
+ template <typename Attribute, typename Iterator, typename HasState
+ , typename Idtype>
struct assign_to_attribute_from_value<
- Attribute, lex::lexertl::token<Iterator, lex::omit, HasState> >
+ Attribute, lex::lexertl::token<Iterator, lex::omit, HasState, Idtype> >
{
static void
- call(lex::lexertl::token<Iterator, lex::omit, HasState> const& t
+ call(lex::lexertl::token<Iterator, lex::omit, HasState, Idtype> const& t
, Attribute& attr)
{
// do nothing
@@ -522,13 +531,14 @@
};
// This is called from the parse function of lexer_def_
- template <typename Iterator, typename AttributeTypes, typename HasState>
+ template <typename Iterator, typename AttributeTypes, typename HasState
+ , typename Idtype>
struct assign_to_attribute_from_value<
fusion::vector2<std::size_t, iterator_range<Iterator> >
- , lex::lexertl::token<Iterator, AttributeTypes, HasState> >
+ , lex::lexertl::token<Iterator, AttributeTypes, HasState, Idtype> >
{
static void
- call(lex::lexertl::token<Iterator, AttributeTypes, HasState> const& t
+ call(lex::lexertl::token<Iterator, AttributeTypes, HasState, Idtype> const& t
, fusion::vector2<std::size_t, iterator_range<Iterator> >& attr)
{
// The type returned by the lexer_def_ parser components is a
@@ -546,10 +556,12 @@
///////////////////////////////////////////////////////////////////////////
// Overload debug output for a single token, this integrates lexer tokens
// with Qi's simple_trace debug facilities
- template <typename Iterator, typename Attribute, typename HasState>
- struct token_printer_debug<lex::lexertl::token<Iterator, Attribute, HasState> >
+ template <typename Iterator, typename Attribute, typename HasState
+ , typename Idtype>
+ struct token_printer_debug<
+ lex::lexertl::token<Iterator, Attribute, HasState, Idtype> >
{
- typedef lex::lexertl::token<Iterator, Attribute, HasState> token_type;
+ typedef lex::lexertl::token<Iterator, Attribute, HasState, Idtype> token_type;
template <typename Out>
static void print(Out& out, token_type const& val)
@@ -559,7 +571,6 @@
out << ']';
}
};
-
}}}
#endif