Subject: [Boost-commit] svn:boost r54676 - in trunk/boost/spirit/home/lex: . lexer lexer/lexertl
From: hartmut.kaiser_at_[hidden]
Date: 2009-07-05 14:50:31
Author: hkaiser
Date: 2009-07-05 14:50:29 EDT (Sun, 05 Jul 2009)
New Revision: 54676
URL: http://svn.boost.org/trac/boost/changeset/54676
Log:
Spirit: Added support for accessing the token value from a lexer semantic action
Text files modified:
trunk/boost/spirit/home/lex/argument.hpp | 41 ++++++++++++++++++++++++++++++++++++++
trunk/boost/spirit/home/lex/lexer/lexertl/functor.hpp | 22 +++++++++++++++----
trunk/boost/spirit/home/lex/lexer/lexertl/functor_data.hpp | 43 +++++++++++++++++++++++++++++----------
trunk/boost/spirit/home/lex/lexer/lexertl/lexer.hpp | 3 -
trunk/boost/spirit/home/lex/lexer/lexertl/token.hpp | 18 ++++++++++++++-
trunk/boost/spirit/home/lex/lexer/pass_flags.hpp | 7 +++--
trunk/boost/spirit/home/lex/tokenize_and_parse.hpp | 25 ++++++++++++++++++++++
7 files changed, 135 insertions(+), 24 deletions(-)
Modified: trunk/boost/spirit/home/lex/argument.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/argument.hpp (original)
+++ trunk/boost/spirit/home/lex/argument.hpp 2009-07-05 14:50:29 EDT (Sun, 05 Jul 2009)
@@ -133,6 +133,43 @@
};
///////////////////////////////////////////////////////////////////////////
+ // The value_getter is used to create the _value placeholder, which is a
+ // Phoenix actor used to access or change the value of the current token.
+ //
+ // This actor is invoked whenever the placeholder '_value' is used in a
+ // lexer semantic action:
+ //
+ // lex::token_def<> identifier = "[a-zA-Z_][a-zA-Z0-9_]*";
+ // this->self = identifier
+ // [ _value = construct_<std::string>(_start, _end) ];
+ //
+ // The example shows how to use _value to set the identifier name as the
+ // token value.
+ struct value_getter
+ {
+ typedef mpl::true_ no_nullary;
+
+ template <typename Env>
+ struct result
+ {
+ typedef typename
+ remove_const<
+ typename mpl::at_c<typename Env::args_type, 4>::type
+ >::type
+ context_type;
+
+ typedef typename context_type::token_value_type& type;
+ };
+
+ template <typename Env>
+ typename result<Env>::type
+ eval(Env const& env) const
+ {
+ return fusion::at_c<4>(env.args()).value();
+ }
+ };
+
+ ///////////////////////////////////////////////////////////////////////////
// The eoi_getter is used to create the _eoi placeholder, which is a
// Phoenix actor used to access the end of input iterator pointing to the
// end of the underlying input sequence.
@@ -184,6 +221,10 @@
// token
phoenix::actor<phoenix::argument<3> > const _tokenid = phoenix::argument<3>();
+ // '_value' may be used to access and change the token value of the current
+ // token
+ phoenix::actor<value_getter> const _value = value_getter();
+
// _state may be used to access and change the name of the current lexer
// state
phoenix::actor<state_context> const _state = state_context();
Modified: trunk/boost/spirit/home/lex/lexer/lexertl/functor.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/lexertl/functor.hpp (original)
+++ trunk/boost/spirit/home/lex/lexer/lexertl/functor.hpp 2009-07-05 14:50:29 EDT (Sun, 05 Jul 2009)
@@ -58,7 +58,7 @@
//
///////////////////////////////////////////////////////////////////////////
template <typename Token
- , template <typename, typename, typename> class FunctorData
+ , template <typename, typename, typename, typename> class FunctorData
, typename Iterator = typename Token::iterator_type
, typename SupportsActors = mpl::false_
, typename SupportsState = typename Token::has_state>
@@ -73,7 +73,9 @@
// Needed by compilers not implementing the resolution to DR45. For
// reference, see
// http://www.open-std.org/JTC1/SC22/WG21/docs/cwg_defects.html#45.
- friend class FunctorData<Iterator, SupportsActors, SupportsState>;
+ typedef typename Token::token_value_type token_value_type;
+ friend class FunctorData<Iterator, SupportsActors, SupportsState
+ , token_value_type>;
// Helper template allowing to assign a value on exit
template <typename T>
@@ -110,7 +112,8 @@
// interface to the iterator_policies::split_functor_input policy
typedef Token result_type;
typedef functor unique;
- typedef FunctorData<Iterator, SupportsActors, SupportsState> shared;
+ typedef FunctorData<Iterator, SupportsActors, SupportsState
+ , token_value_type> shared;
BOOST_SPIRIT_EOF_PREFIX result_type const eof;
@@ -190,6 +193,13 @@
assign_on_exit<Iterator> on_exit(data.get_first(), end);
return result = result_type(id, state, data.get_first(), end);
}
+ else if (pass_flags::pass_use_value == pass) {
+ // return matched token using the token value as set before
+ // using data.set_value(), advancing 'data.first_' past the
+ // matched sequence
+ assign_on_exit<Iterator> on_exit(data.get_first(), end);
+ return result = result_type(id, state, data.value());
+ }
else if (pass_flags::pass_fail == pass) {
// if the data.first_ got adjusted above, revert this adjustment
if (adjusted)
@@ -199,7 +209,9 @@
return result = result_type(0);
}
- // if this token needs to be ignored, just repeat the matching
+ // if this token needs to be ignored, just repeat the matching,
+ // while starting right after the current match
+ data.get_first() = end;
} while (true);
}
@@ -238,7 +250,7 @@
// eof token
///////////////////////////////////////////////////////////////////////////
template <typename Token
- , template <typename, typename, typename> class FunctorData
+ , template <typename, typename, typename, typename> class FunctorData
, typename Iterator, typename SupportsActors, typename SupportsState>
typename functor<Token, FunctorData, Iterator, SupportsActors, SupportsState>::result_type const
functor<Token, FunctorData, Iterator, SupportsActors, SupportsState>::eof =
Modified: trunk/boost/spirit/home/lex/lexer/lexertl/functor_data.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/lexertl/functor_data.hpp (original)
+++ trunk/boost/spirit/home/lex/lexer/lexertl/functor_data.hpp 2009-07-05 14:50:29 EDT (Sun, 05 Jul 2009)
@@ -23,13 +23,14 @@
namespace detail
{
///////////////////////////////////////////////////////////////////////
- template <typename Iterator, typename HasActors, typename HasState>
+ template <typename Iterator, typename HasActors, typename HasState
+ , typename TokenValue>
struct data; // no default specialization
///////////////////////////////////////////////////////////////////////
// neither supports state, nor actors
- template <typename Iterator>
- struct data<Iterator, mpl::false_, mpl::false_>
+ template <typename Iterator, typename TokenValue>
+ struct data<Iterator, mpl::false_, mpl::false_, TokenValue>
{
protected:
typedef typename
@@ -38,6 +39,7 @@
public:
typedef Iterator base_iterator_type;
+ typedef unused_type token_value_type;
typedef std::size_t state_type;
typedef char_type const* state_name_type;
typedef unused_type semantic_actions_type;
@@ -146,6 +148,8 @@
Iterator const& get_first() const { return first_; }
Iterator const& get_last() const { return last_; }
+ unused_type value() const { return unused; }
+
protected:
Iterator& first_;
Iterator last_;
@@ -156,16 +160,17 @@
///////////////////////////////////////////////////////////////////////
// doesn't support lexer semantic actions
- template <typename Iterator>
- struct data<Iterator, mpl::false_, mpl::true_>
- : data<Iterator, mpl::false_, mpl::false_>
+ template <typename Iterator, typename TokenValue>
+ struct data<Iterator, mpl::false_, mpl::true_, TokenValue>
+ : data<Iterator, mpl::false_, mpl::false_, TokenValue>
{
protected:
- typedef data<Iterator, mpl::false_, mpl::false_> base_type;
+ typedef data<Iterator, mpl::false_, mpl::false_, TokenValue> base_type;
typedef typename base_type::char_type char_type;
public:
typedef Iterator base_iterator_type;
+ typedef unused_type token_value_type;
typedef typename base_type::state_type state_type;
typedef typename base_type::state_name_type state_name_type;
typedef typename base_type::semantic_actions_type
@@ -221,22 +226,23 @@
///////////////////////////////////////////////////////////////////////
// does support lexer semantic actions, may support state
- template <typename Iterator, typename HasState>
- struct data<Iterator, mpl::true_, HasState>
- : data<Iterator, mpl::false_, HasState>
+ template <typename Iterator, typename HasState, typename TokenValue>
+ struct data<Iterator, mpl::true_, HasState, TokenValue>
+ : data<Iterator, mpl::false_, HasState, TokenValue>
{
public:
typedef semantic_actions<Iterator, HasState, data>
semantic_actions_type;
protected:
- typedef data<Iterator, mpl::false_, HasState> base_type;
+ typedef data<Iterator, mpl::false_, HasState, TokenValue> base_type;
typedef typename base_type::char_type char_type;
typedef typename semantic_actions_type::functor_wrapper_type
functor_wrapper_type;
public:
typedef Iterator base_iterator_type;
+ typedef TokenValue token_value_type;
typedef typename base_type::state_type state_type;
typedef typename base_type::state_name_type state_name_type;
@@ -310,10 +316,25 @@
has_hold_ = true;
}
+ TokenValue const& value() const
+ {
+ return value_;
+ }
+ TokenValue& value()
+ {
+ return value_;
+ }
+ template <typename Value>
+ void set_value(Value const& val)
+ {
+ value_ = val;
+ }
+
protected:
semantic_actions_type const& actions_;
Iterator hold_; // iterator needed to support lex::more()
bool has_hold_; // 'true' if hold_ is valid
+ TokenValue value_;
};
}
Modified: trunk/boost/spirit/home/lex/lexer/lexertl/lexer.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/lexertl/lexer.hpp (original)
+++ trunk/boost/spirit/home/lex/lexer/lexertl/lexer.hpp 2009-07-05 14:50:29 EDT (Sun, 05 Jul 2009)
@@ -205,7 +205,7 @@
///////////////////////////////////////////////////////////////////////////
template <typename Token = token<>
, typename Iterator = typename Token::iterator_type
- , typename Functor = functor<Token, lexertl::detail::data, Iterator, mpl::false_>
+ , typename Functor = functor<Token, lexertl::detail::data, Iterator>
, typename TokenSet = lex::token_set<token_set<Token, Iterator> > >
class lexer
{
@@ -362,7 +362,6 @@
private:
// lexertl specific data
mutable boost::lexer::basic_state_machine<char_type> state_machine_;
- std::size_t unique_ids_;
boost::lexer::regex_flags flags_;
boost::lexer::basic_rules<char_type> rules_;
Modified: trunk/boost/spirit/home/lex/lexer/lexertl/token.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/lexertl/token.hpp (original)
+++ trunk/boost/spirit/home/lex/lexer/lexertl/token.hpp 2009-07-05 14:50:29 EDT (Sun, 05 Jul 2009)
@@ -117,6 +117,7 @@
typedef Iterator iterator_type;
typedef mpl::false_ has_state;
typedef std::size_t id_type;
+ typedef unused_type token_value_type;
// default constructed tokens correspond to EOI tokens
token() : id_(boost::lexer::npos) {}
@@ -126,6 +127,9 @@
token(id_type id, std::size_t) : id_(id) {}
+ token(id_type id, std::size_t, token_value_type)
+ : id_(id) {}
+
token(id_type id, std::size_t, Iterator const& first
, Iterator const& last)
: id_(id)
@@ -204,6 +208,10 @@
token(id_type id, std::size_t state)
: base_type(id, boost::lexer::npos), state_(state) {}
+ token(id_type id, std::size_t state, token_value_type)
+ : base_type(id, boost::lexer::npos, unused)
+ , state_(state) {}
+
token(id_type id, std::size_t state
, Iterator const& first, Iterator const& last)
: base_type(id, boost::lexer::npos, first, last)
@@ -292,11 +300,13 @@
// from the iterator pair to the required data type is done when it is
// accessed for the first time.
typedef iterator_range<Iterator> iterpair_type;
+
+ public:
+ typedef typename base_type::id_type id_type;
typedef typename detail::token_value_type<
iterpair_type, AttributeTypes
>::type token_value_type;
- public:
typedef Iterator iterator_type;
// default constructed tokens correspond to EOI tokens
@@ -307,7 +317,11 @@
: base_type(0)
, value_(iterpair_type(iterator_type(), iterator_type())) {}
- token(std::size_t id, std::size_t state, Iterator const& first
+ token(id_type id, std::size_t state, token_value_type const& value)
+ : base_type(id, state, value)
+ , value_(value) {}
+
+ token(id_type id, std::size_t state, Iterator const& first
, Iterator const& last)
: base_type(id, state, first, last)
, value_(iterpair_type(first, last)) {}
Modified: trunk/boost/spirit/home/lex/lexer/pass_flags.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/pass_flags.hpp (original)
+++ trunk/boost/spirit/home/lex/lexer/pass_flags.hpp 2009-07-05 14:50:29 EDT (Sun, 05 Jul 2009)
@@ -17,9 +17,10 @@
///////////////////////////////////////////////////////////////////////////
BOOST_SCOPED_ENUM_START(pass_flags)
{
- pass_fail = 0, // make the current match fail in retrospective
- pass_normal = 1, // continue normal token matching, that's the default
- pass_ignore = 2 // ignore the current token and start matching the next
+ pass_fail = 0, // make the current match fail in retrospective
+ pass_normal = 1, // continue normal token matching, that's the default
+ pass_ignore = 2, // ignore the current token and start matching the next
+ pass_use_value = 3 // continue normal matching but use token value as set before
};
BOOST_SCOPED_ENUM_END
Modified: trunk/boost/spirit/home/lex/tokenize_and_parse.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/tokenize_and_parse.hpp (original)
+++ trunk/boost/spirit/home/lex/tokenize_and_parse.hpp 2009-07-05 14:50:29 EDT (Sun, 05 Jul 2009)
@@ -249,6 +249,29 @@
// in its 'INITIAL' state.
//
///////////////////////////////////////////////////////////////////////////
+ namespace detail
+ {
+ template <typename Token>
+ bool tokenize_callback(Token const& t, void (*f)(Token const&))
+ {
+ f(t);
+ return true;
+ }
+
+ template <typename Token, typename Eval>
+ bool tokenize_callback(Token const& t, phoenix::actor<Eval> const& f)
+ {
+ f(t);
+ return true;
+ }
+
+ template <typename Token>
+ bool tokenize_callback(Token const& t, bool (*f)(Token const&))
+ {
+ return f(t);
+ }
+ }
+
template <typename Iterator, typename Lexer, typename F>
inline bool
tokenize(Iterator& first, Iterator last, Lexer const& lex, F f
@@ -260,7 +283,7 @@
iterator_type end = lex.end();
for (/**/; iter != end; ++iter)
{
- if (!f(*iter))
+ if (!detail::tokenize_callback(*iter, f))
return false;
}
return (iter == end) ? true : false;