Subject: [Boost-commit] svn:boost r72422 - in branches/release: . boost boost/spirit boost/spirit/home boost/spirit/home/lex boost/spirit/home/lex/lexer/lexertl boost/spirit/home/lex/qi boost/spirit/home/support boost/spirit/include libs libs/spirit libs/spirit/example libs/spirit/example/qi/compiler_tutorial libs/spirit/example/qi/compiler_tutorial/conjure libs/spirit/example/qi/compiler_tutorial/conjure_lexer
From: hartmut.kaiser_at_[hidden]
Date: 2011-06-05 14:28:44
Author: hkaiser
Date: 2011-06-05 14:28:43 EDT (Sun, 05 Jun 2011)
New Revision: 72422
URL: http://svn.boost.org/trac/boost/changeset/72422
Log:
Spirit: merging lexer fixes from trunk
Added:
branches/release/boost/spirit/home/lex/qi/plain_raw_token.hpp
- copied unchanged from r72410, /trunk/boost/spirit/home/lex/qi/plain_raw_token.hpp
branches/release/boost/spirit/home/lex/qi/plain_tokenid_mask.hpp
- copied unchanged from r72410, /trunk/boost/spirit/home/lex/qi/plain_tokenid_mask.hpp
branches/release/boost/spirit/include/lex_plain_token.hpp
- copied unchanged from r72410, /trunk/boost/spirit/include/lex_plain_token.hpp
branches/release/libs/spirit/example/qi/compiler_tutorial/conjure_lexer/ (props changed)
- copied from r72410, /trunk/libs/spirit/example/qi/compiler_tutorial/conjure_lexer/
Properties modified:
branches/release/ (props changed)
branches/release/boost/ (props changed)
branches/release/boost/spirit/ (props changed)
branches/release/boost/spirit/home/ (props changed)
branches/release/libs/ (props changed)
branches/release/libs/spirit/ (props changed)
branches/release/libs/spirit/example/ (props changed)
Text files modified:
branches/release/boost/spirit/home/lex/lexer/lexertl/position_token.hpp | 65 +++++++++++++++++++++++++++++++--------
branches/release/boost/spirit/home/lex/lexer/lexertl/token.hpp | 5 +-
branches/release/boost/spirit/home/lex/qi.hpp | 2 +
branches/release/boost/spirit/home/lex/qi/plain_token.hpp | 2
branches/release/boost/spirit/home/lex/qi/plain_tokenid.hpp | 2
branches/release/boost/spirit/home/lex/qi/state_switcher.hpp | 4 ++
branches/release/boost/spirit/home/support/common_terminals.hpp | 2 +
branches/release/libs/spirit/example/qi/compiler_tutorial/Jamfile | 10 ++++++
branches/release/libs/spirit/example/qi/compiler_tutorial/conjure/main.cpp | 2
9 files changed, 75 insertions(+), 19 deletions(-)
Modified: branches/release/boost/spirit/home/lex/lexer/lexertl/position_token.hpp
==============================================================================
--- branches/release/boost/spirit/home/lex/lexer/lexertl/position_token.hpp (original)
+++ branches/release/boost/spirit/home/lex/lexer/lexertl/position_token.hpp 2011-06-05 14:28:43 EDT (Sun, 05 Jun 2011)
@@ -184,7 +184,7 @@
if (this != &rhs)
{
id_ = rhs.id_;
- if (id_ != boost::lexer::npos && id_ != 0)
+ if (is_valid())
matched_ = rhs.matched_;
}
return *this;
@@ -349,7 +349,7 @@
///////////////////////////////////////////////////////////////////////////
// These specializations for an attribute list of length one cause all token
- // instances to expose as it attribute the specified type.
+ // instances to expose the specified type as its attribute.
///////////////////////////////////////////////////////////////////////////
template <typename Iterator, typename Attribute, typename HasState
, typename Idtype>
@@ -396,7 +396,7 @@
if (this != &rhs)
{
this->base_type::operator=(static_cast<base_type const&>(rhs));
- if (this->id_ != boost::lexer::npos && this->id_ != 0)
+ if (this->is_valid())
value_ = rhs.value_;
}
return *this;
@@ -452,7 +452,7 @@
if (this != &rhs)
{
this->base_type::operator=(static_cast<base_type const&>(rhs));
- if (this->id_ != boost::lexer::npos && this->id_ != 0)
+ if (this->is_valid())
value_ = rhs.value_;
}
return *this;
@@ -481,16 +481,45 @@
// iterators (see the first of the assign_to_attribute_from_value
// specializations below).
///////////////////////////////////////////////////////////////////////
- template <typename AttributeTypes>
- struct position_token_value
+ template <typename IteratorPair, typename AttributeTypes>
+ struct position_token_value_typesequence
{
typedef typename mpl::insert<
AttributeTypes
, typename mpl::begin<AttributeTypes>::type
- , unused_type
+ , IteratorPair
>::type sequence_type;
typedef typename make_variant_over<sequence_type>::type type;
};
+
+ ///////////////////////////////////////////////////////////////////////
+ // The type of the data item stored with a token instance is defined
+ // by the template parameter 'AttributeTypes' and may be:
+ //
+ // lex::omit: no data item is stored with the token
+ // instance (this is handled by the
+ // specializations of the token class
+ // below)
+ // mpl::vector0<>: each token instance stores a pair of
+ // iterators pointing to the matched input
+ // sequence
+ // mpl::vector<...>: each token instance stores a variant being
+ // able to store the pair of iterators pointing
+ // to the matched input sequence, or any of the
+ // types a specified in the mpl::vector<>
+ //
+ // All this is done to ensure the token type is as small (in terms
+ // of its byte-size) as possible.
+ ///////////////////////////////////////////////////////////////////////
+ template <typename IteratorPair, typename AttributeTypes>
+ struct position_token_value
+ : mpl::eval_if<
+ mpl::or_<
+ is_same<AttributeTypes, mpl::vector0<> >
+ , is_same<AttributeTypes, mpl::vector<> > >
+ , mpl::identity<IteratorPair>
+ , position_token_value_typesequence<IteratorPair, AttributeTypes> >
+ {};
}
template <typename Iterator, typename AttributeTypes, typename HasState
@@ -515,27 +544,35 @@
// the given data types as well. The conversion from the iterator pair
// to the required data type is done when it is accessed for the first
// time.
+ typedef iterator_range<Iterator> iterpair_type;
public:
typedef typename base_type::id_type id_type;
- typedef typename detail::position_token_value<AttributeTypes>::type
- token_value_type;
+ typedef typename detail::position_token_value<
+ iterpair_type, AttributeTypes>::type token_value_type;
typedef Iterator iterator_type;
// default constructed tokens correspond to EOI tokens
- position_token() : value_(unused) {}
+ position_token()
+ : value_(iterpair_type(iterator_type(), iterator_type()))
+ {}
// construct an invalid token
explicit position_token(int)
- : base_type(0), value_(unused) {}
+ : base_type(0)
+ , value_(iterpair_type(iterator_type(), iterator_type()))
+ {}
position_token(id_type id, std::size_t state, token_value_type const& value)
- : base_type(id, state, value), value_(value) {}
+ : base_type(id, state, value), value_(value)
+ {}
position_token(id_type id, std::size_t state, Iterator const& first
, Iterator const& last)
- : base_type(id, state, first, last), value_(unused) {}
+ : base_type(id, state, first, last)
+ , value_(iterpair_type(iterator_type(), iterator_type()))
+ {}
token_value_type& value() { return value_; }
token_value_type const& value() const { return value_; }
@@ -548,7 +585,7 @@
if (this != &rhs)
{
this->base_type::operator=(static_cast<base_type const&>(rhs));
- if (this->id_ != boost::lexer::npos && this->id_ != 0)
+ if (this->is_valid())
value_ = rhs.value_;
}
return *this;
Modified: branches/release/boost/spirit/home/lex/lexer/lexertl/token.hpp
==============================================================================
--- branches/release/boost/spirit/home/lex/lexer/lexertl/token.hpp (original)
+++ branches/release/boost/spirit/home/lex/lexer/lexertl/token.hpp 2011-06-05 14:28:43 EDT (Sun, 05 Jun 2011)
@@ -178,7 +178,8 @@
if (this != &rhs)
{
id_ = rhs.id_;
- matched_ = rhs.matched_;
+ if (is_valid())
+ matched_ = rhs.matched_;
}
return *this;
}
@@ -380,7 +381,7 @@
if (this != &rhs)
{
this->base_type::operator=(static_cast<base_type const&>(rhs));
- if (this->id_ != boost::lexer::npos && this->id_ != 0)
+ if (this->is_valid())
value_ = rhs.value_;
}
return *this;
Modified: branches/release/boost/spirit/home/lex/qi.hpp
==============================================================================
--- branches/release/boost/spirit/home/lex/qi.hpp (original)
+++ branches/release/boost/spirit/home/lex/qi.hpp 2011-06-05 14:28:43 EDT (Sun, 05 Jun 2011)
@@ -14,5 +14,7 @@
#include <boost/spirit/home/lex/qi/in_state.hpp>
#include <boost/spirit/home/lex/qi/plain_token.hpp>
#include <boost/spirit/home/lex/qi/plain_tokenid.hpp>
+#include <boost/spirit/home/lex/qi/plain_tokenid_mask.hpp>
+#include <boost/spirit/home/lex/qi/plain_raw_token.hpp>
#endif
Modified: branches/release/boost/spirit/home/lex/qi/plain_token.hpp
==============================================================================
--- branches/release/boost/spirit/home/lex/qi/plain_token.hpp (original)
+++ branches/release/boost/spirit/home/lex/qi/plain_token.hpp 2011-06-05 14:28:43 EDT (Sun, 05 Jun 2011)
@@ -91,7 +91,7 @@
typedef typename token_type::id_type id_type;
token_type const& t = *first;
- if (std::size_t(~0) == t.id() || id_type(id) == t.id()) {
+ if (std::size_t(~0) == id || id_type(id) == t.id()) {
spirit::traits::assign_to(t, attr);
++first;
return true;
Modified: branches/release/boost/spirit/home/lex/qi/plain_tokenid.hpp
==============================================================================
--- branches/release/boost/spirit/home/lex/qi/plain_tokenid.hpp (original)
+++ branches/release/boost/spirit/home/lex/qi/plain_tokenid.hpp 2011-06-05 14:28:43 EDT (Sun, 05 Jun 2011)
@@ -92,7 +92,7 @@
typedef typename token_type::id_type id_type;
token_type const& t = *first;
- if (std::size_t(~0) == t.id() || id_type(id) == t.id()) {
+ if (std::size_t(~0) == id || id_type(id) == t.id()) {
spirit::traits::assign_to(id, attr);
++first;
return true;
Modified: branches/release/boost/spirit/home/lex/qi/state_switcher.hpp
==============================================================================
--- branches/release/boost/spirit/home/lex/qi/state_switcher.hpp (original)
+++ branches/release/boost/spirit/home/lex/qi/state_switcher.hpp 2011-06-05 14:28:43 EDT (Sun, 05 Jun 2011)
@@ -57,8 +57,12 @@
namespace boost { namespace spirit { namespace qi
{
+#ifndef BOOST_SPIRIT_NO_PREDEFINED_TERMINALS
using spirit::set_state;
using spirit::in_state;
+#endif
+ using spirit::set_state_type;
+ using spirit::in_state_type;
///////////////////////////////////////////////////////////////////////////
namespace detail
Modified: branches/release/boost/spirit/home/support/common_terminals.hpp
==============================================================================
--- branches/release/boost/spirit/home/support/common_terminals.hpp (original)
+++ branches/release/boost/spirit/home/support/common_terminals.hpp 2011-06-05 14:28:43 EDT (Sun, 05 Jun 2011)
@@ -128,6 +128,8 @@
( in_state, in_state_type )
( token, token_type )
( tokenid, tokenid_type )
+ ( raw_token, raw_token_type )
+ ( tokenid_mask, tokenid_mask_type )
( attr, attr_type )
( columns, columns_type )
( auto_, auto_type )
Modified: branches/release/libs/spirit/example/qi/compiler_tutorial/Jamfile
==============================================================================
--- branches/release/libs/spirit/example/qi/compiler_tutorial/Jamfile (original)
+++ branches/release/libs/spirit/example/qi/compiler_tutorial/Jamfile 2011-06-05 14:28:43 EDT (Sun, 05 Jun 2011)
@@ -53,4 +53,14 @@
conjure/main.cpp
;
+exe conjure_lexer :
+ conjure_lexer/compiler.cpp
+ conjure_lexer/expression.cpp
+ conjure_lexer/function.cpp
+ conjure_lexer/lexer.cpp
+ conjure_lexer/main.cpp
+ conjure_lexer/statement.cpp
+ conjure_lexer/vm.cpp
+ ;
+
Modified: branches/release/libs/spirit/example/qi/compiler_tutorial/conjure/main.cpp
==============================================================================
--- branches/release/libs/spirit/example/qi/compiler_tutorial/conjure/main.cpp (original)
+++ branches/release/libs/spirit/example/qi/compiler_tutorial/conjure/main.cpp 2011-06-05 14:28:43 EDT (Sun, 05 Jun 2011)
@@ -90,7 +90,7 @@
if (p->nargs() != nargs)
{
std::cerr << "Error: main function requires " << p->nargs() << " arguments." << std::endl;
- std::cerr << nargs << "supplied." << std::endl;
+ std::cerr << nargs << " supplied." << std::endl;
return 1;
}