Subject: [Boost-commit] svn:boost r54691 - in trunk/boost/spirit/home/lex: . lexer lexer/lexertl
From: hartmut.kaiser_at_[hidden]
Date: 2009-07-05 17:29:55
Author: hkaiser
Date: 2009-07-05 17:29:54 EDT (Sun, 05 Jul 2009)
New Revision: 54691
URL: http://svn.boost.org/trac/boost/changeset/54691
Log:
Spirit: More work on lexer placeholder _value
Text files modified:
trunk/boost/spirit/home/lex/argument.hpp | 118 ++++++++++++++++++++++++++++++++++++---
trunk/boost/spirit/home/lex/lexer/lexertl/functor.hpp | 21 ++++--
trunk/boost/spirit/home/lex/lexer/lexertl/functor_data.hpp | 24 +++++--
trunk/boost/spirit/home/lex/lexer/lexertl/iterator.hpp | 6 ++
trunk/boost/spirit/home/lex/lexer/lexertl/static_functor_data.hpp | 24 +++++--
trunk/boost/spirit/home/lex/lexer/pass_flags.hpp | 3
trunk/boost/spirit/home/lex/tokenize_and_parse.hpp | 14 +++-
7 files changed, 172 insertions(+), 38 deletions(-)
Modified: trunk/boost/spirit/home/lex/argument.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/argument.hpp (original)
+++ trunk/boost/spirit/home/lex/argument.hpp 2009-07-05 17:29:54 EDT (Sun, 05 Jul 2009)
@@ -134,17 +134,16 @@
///////////////////////////////////////////////////////////////////////////
// The value_getter is used to create the _value placeholder, which is a
- // Phoenix actor used to access or change the value of the current token.
+ // Phoenix actor used to access the value of the current token.
//
- // This actor is invoked whenever the placeholder '_value' is used in a
- // lexer semantic action:
+ // This Phoenix actor is invoked whenever the placeholder '_value' is used
+ // as a rvalue inside a lexer semantic action:
//
// lex::token_def<> identifier = "[a-zA-Z_][a-zA-Z0-9_]*";
- // this->self = identifier
- // [ _value = construct_<std::string>(_start, _end) ];
+ // this->self = identifier [ std::cout << _value ];
//
- // The example shows how to use _value to set the identifier name as the
- // token value.
+ // The example shows how to use _value to print the identifier name (which
+ // is the initial token value).
struct value_getter
{
typedef mpl::true_ no_nullary;
@@ -165,7 +164,73 @@
typename result<Env>::type
eval(Env const& env) const
{
- return fusion::at_c<4>(env.args()).value();
+ return fusion::at_c<4>(env.args()).get_value();
+ }
+ };
+
+ ///////////////////////////////////////////////////////////////////////////
+ // The value_setter is a Phoenix actor used to change the name of the
+ // current lexer state by calling set_state_name() on the context (which
+ // is the 4th parameter to any lexer semantic actions).
+ //
+ // This Phoenix actor is invoked whenever the placeholder '_value' is used
+ // as a lvalue inside a lexer semantic action:
+ //
+ // lex::token_def<> identifier = "[a-zA-Z_][a-zA-Z0-9_]*";
+ // this->self = identifier [ _value = "identifier" ];
+ //
+ // The example shows how to change the token value after matching a token
+ // 'identifier'.
+ template <typename Actor>
+ struct value_setter
+ {
+ typedef mpl::true_ no_nullary;
+
+ template <typename Env>
+ struct result
+ {
+ typedef void type;
+ };
+
+ template <typename Env>
+ void eval(Env const& env) const
+ {
+ fusion::at_c<4>(env.args()).set_value(actor_());
+ }
+
+ value_setter(Actor const& actor)
+ : actor_(actor) {}
+
+ // see explanation for this constructor at the end of this file
+ value_setter(phoenix::actor<value_getter>, Actor const& actor)
+ : actor_(actor) {}
+
+ Actor actor_;
+ };
+
+ ///////////////////////////////////////////////////////////////////////////
+ // The value_context is used as a noop Phoenix actor to create the
+ // placeholder '_value' (see below). It is a noop actor because it is used
+ // as a placeholder only, while it is being converted either to a
+ // value_getter (if used as a rvalue) or to a value_setter (if used as a
+ // lvalue). The conversion is achieved by specializing and overloading a
+ // couple of the Phoenix templates from the Phoenix expression composition
+ // engine (see the end of this file).
+ struct value_context
+ {
+ typedef mpl::true_ no_nullary;
+
+ template <typename Env>
+ struct result
+ {
+ typedef unused_type type;
+ };
+
+ template <typename Env>
+ unused_type
+ eval(Env const& env) const
+ {
+ return unused;
}
};
@@ -223,7 +288,7 @@
// '_value' may be used to access and change the token value of the current
// token
- phoenix::actor<value_getter> const _value = value_getter();
+ phoenix::actor<value_context> const _value = value_context();
// _state may be used to access and change the name of the current lexer
// state
@@ -273,6 +338,41 @@
typedef spirit::state_setter<typename as_actor<RHS>::type> type;
};
+ ///////////////////////////////////////////////////////////////////////////
+ // The specialization of as_actor_base<> below is needed to convert all
+ // occurrences of _value in places where it's used as a rvalue into the
+ // proper Phoenix actor (spirit::value_getter) accessing the token value.
+ template<>
+ struct as_actor_base<actor<spirit::value_context> >
+ {
+ typedef spirit::value_getter type;
+
+ static spirit::value_getter
+ convert(actor<spirit::value_context>)
+ {
+ return spirit::value_getter();
+ }
+ };
+
+ ///////////////////////////////////////////////////////////////////////////
+ // The specialization of as_composite<> below is needed to convert all
+ // assignments to _value (places where it's used as a lvalue) into the
+ // proper Phoenix actor (spirit::value_setter) allowing to change the
+ // token value.
+ template <typename RHS>
+ struct as_composite<assign_eval, actor<spirit::value_context>, RHS>
+ {
+ // For an assignment to _value (a spirit::value_context actor), this
+ // specialization makes Phoenix's compose() function construct a
+ // spirit::value_setter actor from 1. the LHS, a spirit::value_getter
+ // actor (due to the specialization of as_actor_base<> above),
+ // and 2. the RHS actor.
+ // This is why spirit::value_setter needs a constructor which takes
+ // a dummy spirit::value_getter as its first argument in addition
+ // to its real, second argument (the RHS actor).
+ typedef spirit::value_setter<typename as_actor<RHS>::type> type;
+ };
+
}}
#undef SPIRIT_DECLARE_ARG
Modified: trunk/boost/spirit/home/lex/lexer/lexertl/functor.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/lexertl/functor.hpp (original)
+++ trunk/boost/spirit/home/lex/lexer/lexertl/functor.hpp 2009-07-05 17:29:54 EDT (Sun, 05 Jul 2009)
@@ -139,6 +139,7 @@
return result = mp.ftor.eof;
#endif
+ data.reset_value();
Iterator end = data.get_first();
std::size_t unique_id = boost::lexer::npos;
std::size_t id = data.next(end, unique_id);
@@ -187,18 +188,18 @@
BOOST_SCOPED_ENUM(pass_flags) pass =
data.invoke_actions(state, id, unique_id, end);
- if (pass_flags::pass_normal == pass) {
- // return matched token, advancing 'data.first_' past the
+ if (data.has_value()) {
+ // return matched token using the token value as set before
+ // using data.set_value(), advancing 'data.first_' past the
// matched sequence
assign_on_exit<Iterator> on_exit(data.get_first(), end);
- return result = result_type(id, state, data.get_first(), end);
+ return result = result_type(id, state, data.get_value());
}
- else if (pass_flags::pass_use_value == pass) {
- // return matched token using the token value as set before
- // using data.set_value(), advancing 'data.first_' past the
+ else if (pass_flags::pass_normal == pass) {
+ // return matched token, advancing 'data.first_' past the
// matched sequence
assign_on_exit<Iterator> on_exit(data.get_first(), end);
- return result = result_type(id, state, data.value());
+ return result = result_type(id, state, data.get_first(), end);
}
else if (pass_flags::pass_fail == pass) {
// if the data.first_ got adjusted above, revert this adjustment
@@ -234,6 +235,12 @@
}
template <typename MultiPass>
+ static std::size_t get_state(MultiPass& mp)
+ {
+ return mp.shared()->ftor.get_state();
+ }
+
+ template <typename MultiPass>
static std::size_t
map_state(MultiPass const& mp, char_type const* statename)
{
Modified: trunk/boost/spirit/home/lex/lexer/lexertl/functor_data.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/lexertl/functor_data.hpp (original)
+++ trunk/boost/spirit/home/lex/lexer/lexertl/functor_data.hpp 2009-07-05 17:29:54 EDT (Sun, 05 Jul 2009)
@@ -148,7 +148,12 @@
Iterator const& get_first() const { return first_; }
Iterator const& get_last() const { return last_; }
- unused_type value() const { return unused; }
+ iterator_range<Iterator> get_value() const
+ {
+ return iterator_range<Iterator>(first_, last_);
+ }
+ bool has_value() const { return false; }
+ void reset_value() {}
protected:
Iterator& first_;
@@ -252,7 +257,8 @@
template <typename IterData>
data (IterData const& data, Iterator& first, Iterator const& last)
: base_type(data, first, last)
- , actions_(data.actions_), hold_(), has_hold_(false) {}
+ , actions_(data.actions_), hold_(), has_hold_(false)
+ , has_value_(false) {}
// invoke attached semantic actions, if defined
BOOST_SCOPED_ENUM(pass_flags) invoke_actions(std::size_t state
@@ -316,25 +322,27 @@
has_hold_ = true;
}
- TokenValue const& value() const
- {
- return value_;
- }
- TokenValue& value()
+ TokenValue const& get_value() const
{
+ if (!has_value_)
+ value_ = base_type::get_value();
return value_;
}
template <typename Value>
void set_value(Value const& val)
{
value_ = val;
+ has_value_ = true;
}
+ bool has_value() const { return has_value_; }
+ void reset_value() { has_value_ = false; }
protected:
semantic_actions_type const& actions_;
Iterator hold_; // iterator needed to support lex::more()
+ mutable TokenValue value_; // token value to use
bool has_hold_; // 'true' if hold_ is valid
- TokenValue value_;
+ bool has_value_; // 'true' if value_ is valid
};
}
Modified: trunk/boost/spirit/home/lex/lexer/lexertl/iterator.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/lexertl/iterator.hpp (original)
+++ trunk/boost/spirit/home/lex/lexer/lexertl/iterator.hpp 2009-07-05 17:29:54 EDT (Sun, 05 Jul 2009)
@@ -95,6 +95,12 @@
return unique_functor_type::set_state(*this, state);
}
+ // get the curent state for the underlying lexer object
+ std::size_t get_state()
+ {
+ return unique_functor_type::get_state(*this);
+ }
+
// map the given state name to a corresponding state id as understood
// by the underlying lexer object
std::size_t map_state(char_type const* statename)
Modified: trunk/boost/spirit/home/lex/lexer/lexertl/static_functor_data.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/lexertl/static_functor_data.hpp (original)
+++ trunk/boost/spirit/home/lex/lexer/lexertl/static_functor_data.hpp 2009-07-05 17:29:54 EDT (Sun, 05 Jul 2009)
@@ -166,7 +166,12 @@
Iterator const& get_first() const { return first_; }
Iterator const& get_last() const { return last_; }
- unused_type value() const { return unused; }
+ iterator_range<Iterator> get_value() const
+ {
+ return iterator_range<Iterator>(first_, last_);
+ }
+ bool has_value() const { return false; }
+ void reset_value() {}
protected:
Iterator& first_;
@@ -274,7 +279,8 @@
static_data (IterData const& data, Iterator& first
, Iterator const& last)
: base_type(data, first, last)
- , actions_(data.actions_), hold_(), has_hold_(false) {}
+ , actions_(data.actions_), hold_(), has_hold_(false)
+ , has_value_(false) {}
// invoke attached semantic actions, if defined
BOOST_SCOPED_ENUM(pass_flags) invoke_actions(std::size_t state
@@ -338,25 +344,27 @@
has_hold_ = true;
}
- TokenValue const& value() const
- {
- return value_;
- }
- TokenValue& value()
+ TokenValue const& get_value() const
{
+ if (!has_value_)
+ value_ = base_type::get_value();
return value_;
}
template <typename Value>
void set_value(Value const& val)
{
value_ = val;
+ has_value_ = true;
}
+ bool has_value() const { return has_value_; }
+ void reset_value() { has_value_ = false; }
protected:
semantic_actions_type const& actions_;
Iterator hold_; // iterator needed to support lex::more()
+ mutable TokenValue value_; // token value to use
bool has_hold_; // 'true' if hold_ is valid
- TokenValue value_;
+ bool has_value_; // 'true' if value_ is valid
};
}
Modified: trunk/boost/spirit/home/lex/lexer/pass_flags.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/pass_flags.hpp (original)
+++ trunk/boost/spirit/home/lex/lexer/pass_flags.hpp 2009-07-05 17:29:54 EDT (Sun, 05 Jul 2009)
@@ -19,8 +19,7 @@
{
pass_fail = 0, // make the current match fail in retrospective
pass_normal = 1, // continue normal token matching, that's the default
- pass_ignore = 2, // ignore the current token and start matching the next
- pass_use_value = 3 // continue normal matching but use token value as set before
+ pass_ignore = 2 // ignore the current token and start matching the next
};
BOOST_SCOPED_ENUM_END
Modified: trunk/boost/spirit/home/lex/tokenize_and_parse.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/tokenize_and_parse.hpp (original)
+++ trunk/boost/spirit/home/lex/tokenize_and_parse.hpp 2009-07-05 17:29:54 EDT (Sun, 05 Jul 2009)
@@ -251,11 +251,10 @@
///////////////////////////////////////////////////////////////////////////
namespace detail
{
- template <typename Token>
- bool tokenize_callback(Token const& t, void (*f)(Token const&))
+ template <typename Token, typename F>
+ bool tokenize_callback(Token const& t, F f)
{
- f(t);
- return true;
+ return f(t);
}
template <typename Token, typename Eval>
@@ -266,6 +265,13 @@
}
template <typename Token>
+ bool tokenize_callback(Token const& t, void (*f)(Token const&))
+ {
+ f(t);
+ return true;
+ }
+
+ template <typename Token>
bool tokenize_callback(Token const& t, bool (*f)(Token const&))
{
return f(t);