Subject: [Boost-commit] svn:boost r72201 - trunk/boost/spirit/home/lex/lexer/lexertl
From: hartmut.kaiser_at_[hidden]
Date: 2011-05-26 22:02:22
Author: hkaiser
Date: 2011-05-26 22:02:20 EDT (Thu, 26 May 2011)
New Revision: 72201
URL: http://svn.boost.org/trac/boost/changeset/72201
Log:
Spirit: fixed #5566: Spirit.Lex converting a token to its value in a lexer semantic action fails
Text files modified:
trunk/boost/spirit/home/lex/lexer/lexertl/functor.hpp | 3
trunk/boost/spirit/home/lex/lexer/lexertl/functor_data.hpp | 2
trunk/boost/spirit/home/lex/lexer/lexertl/static_functor_data.hpp | 163 +++++++++++++++++++++++++++++++++++++--
3 files changed, 158 insertions(+), 10 deletions(-)
Modified: trunk/boost/spirit/home/lex/lexer/lexertl/functor.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/lexertl/functor.hpp (original)
+++ trunk/boost/spirit/home/lex/lexer/lexertl/functor.hpp 2011-05-26 22:02:20 EDT (Thu, 26 May 2011)
@@ -196,6 +196,9 @@
// data.first_ back to the start of the previously matched token.
bool adjusted = data.adjust_start();
+ // set the end of the matched input sequence in the token data
+ data.set_end(end);
+
// invoke attached semantic actions, if defined, might change
// state, id, data.first_, and/or end
BOOST_SCOPED_ENUM(pass_flags) pass =
Modified: trunk/boost/spirit/home/lex/lexer/lexertl/functor_data.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/lexertl/functor_data.hpp (original)
+++ trunk/boost/spirit/home/lex/lexer/lexertl/functor_data.hpp 2011-05-26 22:02:20 EDT (Thu, 26 May 2011)
@@ -168,6 +168,8 @@
std::size_t get_state() const { return 0; }
void set_state(std::size_t) {}
+ void set_end(Iterator const& it) {}
+
Iterator& get_first() { return first_; }
Iterator const& get_first() const { return first_; }
Iterator const& get_last() const { return last_; }
Modified: trunk/boost/spirit/home/lex/lexer/lexertl/static_functor_data.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/lexertl/static_functor_data.hpp (original)
+++ trunk/boost/spirit/home/lex/lexer/lexertl/static_functor_data.hpp 2011-05-26 22:02:20 EDT (Thu, 26 May 2011)
@@ -112,7 +112,7 @@
//
// This function does nothing as long as no semantic actions are
// used.
- Iterator const& less(Iterator const& it, int n)
+ Iterator const& less(Iterator const& it, int)
{
// The following assertion fires most likely because you are
// using lexer semantic actions without using the actor_lexer
@@ -150,7 +150,7 @@
//
// This function does nothing as long as no semantic actions are
// used.
- bool lookahead(std::size_t id, std::size_t state = std::size_t(~0))
+ bool lookahead(std::size_t, std::size_t /*state*/ = std::size_t(~0))
{
// The following assertion fires most likely because you are
// using lexer semantic actions without using the actor_lexer
@@ -184,6 +184,8 @@
std::size_t get_state() const { return 0; }
void set_state(std::size_t) {}
+ void set_end(Iterator const& it) {}
+
Iterator& get_first() { return first_; }
Iterator const& get_first() const { return first_; }
Iterator const& get_last() const { return last_; }
@@ -204,7 +206,7 @@
next_token_functor next_token_;
get_state_name_type get_state_name_;
- bool bol_;
+ bool bol_; // helper storing whether last character was \n
private:
// silence MSVC warning C4512: assignment operator could not be generated
@@ -212,7 +214,7 @@
};
///////////////////////////////////////////////////////////////////////
- // doesn't support no actors, but does support states
+ // doesn't support lexer semantic actions, but supports state
template <typename Iterator, typename TokenValue>
class static_data<Iterator, mpl::false_, mpl::true_, TokenValue>
: public static_data<Iterator, mpl::false_, mpl::false_, TokenValue>
@@ -297,7 +299,8 @@
semantic_actions_type;
protected:
- typedef static_data<Iterator, mpl::false_, HasState, TokenValue> base_type;
+ typedef static_data<Iterator, mpl::false_, HasState, TokenValue>
+ base_type;
typedef typename base_type::char_type char_type;
typedef typename semantic_actions_type::functor_wrapper_type
functor_wrapper_type;
@@ -338,7 +341,7 @@
// token back to the input stream).
Iterator const& less(Iterator& it, int n)
{
- it = this->first_;
+ it = this->get_first();
std::advance(it, n);
return it;
}
@@ -350,7 +353,7 @@
// the current token value rather than replacing it.
void more()
{
- hold_ = this->first_;
+ hold_ = this->get_first();
has_hold_ = true;
}
@@ -368,7 +371,7 @@
state = this->state_;
return id == this->next_token_(
- state, bol, end, this->last_, unique_id);
+ state, bol, end, this->get_eoi(), unique_id);
}
// The adjust_start() and revert_adjust_start() are helper
@@ -379,14 +382,14 @@
if (!has_hold_)
return false;
- std::swap(this->first_, hold_);
+ std::swap(this->get_first(), hold_);
has_hold_ = false;
return true;
}
void revert_adjust_start()
{
// this will be called only if adjust_start above returned true
- std::swap(this->first_, hold_);
+ std::swap(this->get_first(), hold_);
has_hold_ = true;
}
@@ -423,6 +426,146 @@
// silence MSVC warning C4512: assignment operator could not be generated
static_data& operator= (static_data const&);
};
+
+ ///////////////////////////////////////////////////////////////////////
+ // does support lexer semantic actions, may support state, is used for
+ // position_token exposing exactly one type
+ template <typename Iterator, typename HasState, typename TokenValue>
+ class static_data<Iterator, mpl::true_, HasState, boost::optional<TokenValue> >
+ : public static_data<Iterator, mpl::false_, HasState, TokenValue>
+ {
+ public:
+ typedef semantic_actions<Iterator, HasState, static_data>
+ semantic_actions_type;
+
+ protected:
+ typedef static_data<Iterator, mpl::false_, HasState, TokenValue>
+ base_type;
+ typedef typename base_type::char_type char_type;
+ typedef typename semantic_actions_type::functor_wrapper_type
+ functor_wrapper_type;
+
+ public:
+ typedef Iterator base_iterator_type;
+ typedef boost::optional<TokenValue> token_value_type;
+ typedef boost::optional<TokenValue> const& get_value_type;
+ typedef typename base_type::state_type state_type;
+ typedef typename base_type::state_name_type state_name_type;
+
+ typedef detail::wrap_action<functor_wrapper_type
+ , Iterator, static_data, std::size_t> wrap_action_type;
+
+ template <typename IterData>
+ static_data (IterData const& data_, Iterator& first, Iterator const& last)
+ : base_type(data_, first, last)
+ , actions_(data_.actions_), hold_()
+ , has_value_(false), has_hold_(false)
+ {
+ spirit::traits::assign_to(first, last, value_);
+ has_value_ = true;
+ }
+
+ // invoke attached semantic actions, if defined
+ BOOST_SCOPED_ENUM(pass_flags) invoke_actions(std::size_t state
+ , std::size_t& id, std::size_t unique_id, Iterator& end)
+ {
+ return actions_.invoke_actions(state, id, unique_id, end, *this);
+ }
+
+ // The function less() is used by the implementation of the support
+ // function lex::less(). Its functionality is equivalent to flex'
+ // function yyless(): it returns an iterator positioned to the
+ // nth input character beyond the current start iterator (i.e. by
+ // assigning the return value to the placeholder '_end' it is
+ // possible to return all but the first n characters of the current
+ // token back to the input stream).
+ Iterator const& less(Iterator& it, int n)
+ {
+ it = this->get_first();
+ std::advance(it, n);
+ return it;
+ }
+
+ // The function more() is used by the implementation of the support
+ // function lex::more(). Its functionality is equivalent to flex'
+ // function yymore(): it tells the lexer that the next time it
+ // matches a rule, the corresponding token should be appended onto
+ // the current token value rather than replacing it.
+ void more()
+ {
+ hold_ = this->get_first();
+ has_hold_ = true;
+ }
+
+ // The function lookahead() is used by the implementation of the
+ // support function lex::lookahead. It can be used to implement
+ // lookahead for lexer engines not supporting constructs like flex'
+ // a/b (match a, but only when followed by b)
+ bool lookahead(std::size_t id, std::size_t state = std::size_t(~0))
+ {
+ Iterator end = end_;
+ std::size_t unique_id = boost::lexer::npos;
+ bool bol = this->bol_;
+
+ if (std::size_t(~0) == state)
+ state = this->state_;
+
+ return id == this->next_token_(
+ state, bol, end, this->get_eoi(), unique_id);
+ }
+
+ // The adjust_start() and revert_adjust_start() are helper
+ // functions needed to implement the functionality required for
+ // lex::more(). It is called from the functor body below.
+ bool adjust_start()
+ {
+ if (!has_hold_)
+ return false;
+
+ std::swap(this->get_first(), hold_);
+ has_hold_ = false;
+ return true;
+ }
+ void revert_adjust_start()
+ {
+ // this will be called only if adjust_start above returned true
+ std::swap(this->get_first(), hold_);
+ has_hold_ = true;
+ }
+
+ TokenValue const& get_value() const
+ {
+ if (!has_value_) {
+ spirit::traits::assign_to(this->get_first(), end_, value_);
+ has_value_ = true;
+ }
+ return value_;
+ }
+ template <typename Value>
+ void set_value(Value const& val)
+ {
+ value_ = val;
+ has_value_ = true;
+ }
+ void set_end(Iterator const& it)
+ {
+ end_ = it;
+ }
+ bool has_value() const { return has_value_; }
+ void reset_value() { has_value_ = false; }
+
+ protected:
+ semantic_actions_type const& actions_;
+ Iterator hold_; // iterator needed to support lex::more()
+ Iterator end_; // iterator pointing to end of matched token
+ mutable token_value_type value_; // token value to use
+ mutable bool has_value_; // 'true' if value_ is valid
+ bool has_hold_; // 'true' if hold_ is valid
+
+ private:
+ // silence MSVC warning C4512: assignment operator could not be generated
+ static_data& operator= (static_data const&);
+ };
}
}}}}