Subject: [Boost-commit] svn:boost r53791 - in trunk: boost/spirit/home/lex boost/spirit/home/lex/lexer boost/spirit/home/lex/lexer/lexertl boost/spirit/home/qi/detail boost/spirit/home/support boost/spirit/home/support/detail libs/spirit/example/lex libs/spirit/example/lex/static_lexer libs/spirit/test libs/spirit/test/lex
From: hartmut.kaiser_at_[hidden]
Date: 2009-06-10 23:39:07
Author: hkaiser
Date: 2009-06-10 23:39:04 EDT (Wed, 10 Jun 2009)
New Revision: 53791
URL: http://svn.boost.org/trac/boost/changeset/53791
Log:
Spirit: Lot of work done in Lexer, fixed bugs, added support functions, refactored code
Added:
trunk/boost/spirit/home/lex/lexer/lexertl/functor_data.hpp (contents, props changed)
trunk/boost/spirit/home/lex/lexer/lexertl/semantic_action_data.hpp (contents, props changed)
trunk/boost/spirit/home/lex/lexer/lexertl/static_functor_data.hpp
- copied, changed from r53639, /trunk/boost/spirit/home/lex/lexer/lexertl/static_functor.hpp
trunk/boost/spirit/home/lex/lexer/support_functions.hpp (contents, props changed)
trunk/libs/spirit/test/lex/lexer_state_switcher.cpp (contents, props changed)
Removed:
trunk/boost/spirit/home/lex/lexer/lexertl/static_functor.hpp
Text files modified:
trunk/boost/spirit/home/lex/lexer/action.hpp | 8
trunk/boost/spirit/home/lex/lexer/lexer.hpp | 5
trunk/boost/spirit/home/lex/lexer/lexertl/functor.hpp | 302 +++++++-----------------
trunk/boost/spirit/home/lex/lexer/lexertl/iterator.hpp | 11
trunk/boost/spirit/home/lex/lexer/lexertl/lexer.hpp | 29 -
trunk/boost/spirit/home/lex/lexer/lexertl/static_functor_data.hpp | 494 ++++++++++++++++++---------------------
trunk/boost/spirit/home/lex/lexer/lexertl/static_lexer.hpp | 58 +---
trunk/boost/spirit/home/lex/lexer/lexertl/wrap_action.hpp | 149 ++++++++---
trunk/boost/spirit/home/lex/lexer/token_def.hpp | 4
trunk/boost/spirit/home/lex/tokenize_and_parse.hpp | 16
trunk/boost/spirit/home/qi/detail/pass_container.hpp | 10
trunk/boost/spirit/home/support/container.hpp | 26 ++
trunk/boost/spirit/home/support/detail/scoped_enum_emulation.hpp | 6
trunk/libs/spirit/example/lex/static_lexer/word_count_lexer_tokens.hpp | 22 +
trunk/libs/spirit/example/lex/strip_comments.cpp | 6
trunk/libs/spirit/example/lex/strip_comments_lexer.cpp | 24 +
trunk/libs/spirit/example/lex/word_count_lexer.cpp | 19 +
trunk/libs/spirit/test/CMakeLists.txt | 1
trunk/libs/spirit/test/Jamfile | 1
trunk/libs/spirit/test/lex/matlib.h | 20 +
20 files changed, 581 insertions(+), 630 deletions(-)
Modified: trunk/boost/spirit/home/lex/lexer/action.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/action.hpp (original)
+++ trunk/boost/spirit/home/lex/lexer/action.hpp 2009-06-10 23:39:04 EDT (Wed, 10 Jun 2009)
@@ -12,6 +12,8 @@
#include <boost/spirit/home/lex/meta_compiler.hpp>
#include <boost/spirit/home/lex/lexer_type.hpp>
+#include <boost/spirit/home/lex/argument.hpp>
+#include <boost/spirit/home/lex/lexer/support_functions.hpp>
#include <boost/mpl/if.hpp>
#include <boost/type_traits/remove_const.hpp>
#include <boost/type_traits/is_same.hpp>
@@ -19,6 +21,12 @@
///////////////////////////////////////////////////////////////////////////////
namespace boost { namespace spirit { namespace lex
{
+ using spirit::_start;
+ using spirit::_end;
+ using spirit::_pass;
+ using spirit::_tokenid;
+ using spirit::_state;
+
///////////////////////////////////////////////////////////////////////////
template <typename Subject, typename Action>
struct action : unary_lexer<action<Subject, Action> >
Modified: trunk/boost/spirit/home/lex/lexer/lexer.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/lexer.hpp (original)
+++ trunk/boost/spirit/home/lex/lexer/lexer.hpp 2009-06-10 23:39:04 EDT (Wed, 10 Jun 2009)
@@ -319,8 +319,9 @@
// access iterator interface
template <typename Iterator>
- iterator_type begin(Iterator& first, Iterator const& last) const
- { return this->lexer_type::begin(first, last); }
+ iterator_type begin(Iterator& first, Iterator const& last
+ , char_type const* initial_state = 0) const
+ { return this->lexer_type::begin(first, last, initial_state); }
iterator_type end() const
{ return this->lexer_type::end(); }
Modified: trunk/boost/spirit/home/lex/lexer/lexertl/functor.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/lexertl/functor.hpp (original)
+++ trunk/boost/spirit/home/lex/lexer/lexertl/functor.hpp 2009-06-10 23:39:04 EDT (Wed, 10 Jun 2009)
@@ -11,16 +11,10 @@
#endif
#include <boost/mpl/bool.hpp>
-#include <boost/function.hpp>
-#include <boost/range/iterator_range.hpp>
#include <boost/detail/iterator.hpp>
#include <boost/detail/workaround.hpp>
-#include <map>
-#include <boost/spirit/home/support/detail/lexer/generator.hpp>
-#include <boost/spirit/home/support/detail/lexer/rules.hpp>
-#include <boost/spirit/home/support/detail/lexer/state_machine.hpp>
-#include <boost/spirit/home/lex/lexer/lexertl/iterator_tokenizer.hpp>
-#include <boost/spirit/home/lex/lexer/lexertl/wrap_action.hpp>
+#include <boost/spirit/home/lex/lexer/pass_flags.hpp>
+#include <boost/assert.hpp>
#if 0 != __COMO_VERSION__ || !BOOST_WORKAROUND(BOOST_MSVC, <= 1310)
#define BOOST_SPIRIT_STATIC_EOF 1
@@ -31,151 +25,12 @@
namespace boost { namespace spirit { namespace lex { namespace lexertl
{
- namespace detail
- {
- ///////////////////////////////////////////////////////////////////////
- template <typename Iterator, typename HasActors, typename HasState>
- struct data; // no default specialization
-
- ///////////////////////////////////////////////////////////////////////
- // neither supports state, nor actors
- template <typename Iterator>
- struct data<Iterator, mpl::false_, mpl::false_>
- {
- typedef std::size_t state_type;
- typedef iterator_range<Iterator> iterpair_type;
- typedef typename boost::detail::iterator_traits<Iterator>::value_type
- char_type;
- typedef unused_type semantic_actions_type;
- typedef detail::wrap_action<unused_type, iterpair_type, data>
- wrap_action_type;
-
- // initialize the shared data
- template <typename IterData>
- data (IterData const& data_, Iterator& first_, Iterator const& last_)
- : state_machine(data_.state_machine_)
- , rules(data_.rules_)
- , first(first_), last(last_) {}
-
- std::size_t next(Iterator& end, std::size_t& unique_id)
- {
- typedef basic_iterator_tokeniser<Iterator> tokenizer;
- return tokenizer::next(state_machine, first, end, last, unique_id);
- }
-
- // nothing to invoke, so this is empty
- bool invoke_actions(std::size_t, std::size_t, std::size_t
- , Iterator const&)
- {
- return true; // always accept
- }
-
- std::size_t get_state() const { return 0; }
- void set_state_name (char_type const* state) {}
-
- boost::lexer::basic_state_machine<char_type> const& state_machine;
- boost::lexer::basic_rules<char_type> const& rules;
- Iterator& first;
- Iterator last;
- };
-
- ///////////////////////////////////////////////////////////////////////
- // doesn't support actors
- template <typename Iterator>
- struct data<Iterator, mpl::false_, mpl::true_>
- : data<Iterator, mpl::false_, mpl::false_>
- {
- typedef data<Iterator, mpl::false_, mpl::false_> base_type;
-
- typedef typename base_type::state_type state_type;
- typedef typename base_type::char_type char_type;
- typedef typename base_type::semantic_actions_type
- semantic_actions_type;
-
- // initialize the shared data
- template <typename IterData>
- data (IterData const& data_, Iterator& first_, Iterator const& last_)
- : base_type(data_, first_, last_), state(0) {}
-
- std::size_t next(Iterator& end, std::size_t& unique_id)
- {
- typedef basic_iterator_tokeniser<Iterator> tokenizer;
- return tokenizer::next(this->state_machine, state,
- this->first, end, this->last, unique_id);
- }
-
- std::size_t& get_state() { return state; }
-
- void set_state_name (char_type const* new_state)
- {
- std::size_t state_id = this->rules.state(new_state);
-
- // if the following assertion fires you've probably been using
- // a lexer state name which was not defined in your token
- // definition
- BOOST_ASSERT(state_id != boost::lexer::npos);
-
- if (state_id != boost::lexer::npos)
- state = state_id;
- }
-
- std::size_t state;
- };
-
- ///////////////////////////////////////////////////////////////////////
- // does support actors, but may have no state
- template <typename Iterator, typename HasState>
- struct data<Iterator, mpl::true_, HasState>
- : data<Iterator, mpl::false_, HasState>
- {
- typedef data<Iterator, mpl::false_, HasState> base_type;
-
- typedef iterator_range<Iterator> iterpair_type;
- typedef typename base_type::state_type state_type;
- typedef typename base_type::char_type char_type;
-
- typedef void functor_type(iterpair_type, std::size_t, bool&, data&);
- typedef boost::function<functor_type> functor_wrapper_type;
- typedef std::vector<std::vector<functor_wrapper_type> >
- semantic_actions_type;
-
- typedef detail::wrap_action<functor_wrapper_type
- , iterpair_type, data> wrap_action_type;
-
- template <typename IterData>
- data (IterData const& data_, Iterator& first_, Iterator const& last_)
- : base_type(data_, first_, last_)
- , actions_(data_.actions_) {}
-
- // invoke attached semantic actions, if defined
- bool invoke_actions(std::size_t state, std::size_t id
- , std::size_t unique_id, Iterator const& end)
- {
- if (state >= actions_.size())
- return true; // no action defined for this state
-
- std::vector<functor_wrapper_type> const& actions = actions_[state];
-
- if (unique_id >= actions.size() || !actions[unique_id])
- return true; // nothing to invoke, continue with 'match'
-
- iterpair_type itp(this->first, end);
- bool match = true;
- actions[unique_id](itp, id, match, *this);
- return match;
- }
-
- semantic_actions_type const& actions_;
- };
- }
-
///////////////////////////////////////////////////////////////////////////
//
// functor is a template usable as the functor object for the
// multi_pass iterator allowing to wrap a lexertl based dfa into a
// iterator based interface.
//
- // Iterator: the type of the underlying iterator
// Token: the type of the tokens produced by this functor
// this needs to expose a constructor with the following
// prototype:
@@ -187,6 +42,11 @@
// this token has been matched in, and 'first' and 'end'
// mark the start and the end of the token with respect
// to the underlying character stream.
+ // FunctorData:
+ // this is expected to encapsulate the shared part of the
+ // functor (see lex/lexer/lexertl/functor_data.hpp for an
+ // example and documentation).
+ // Iterator: the type of the underlying iterator
// SupportsActors:
// this is expected to be a mpl::bool_, if mpl::true_ the
// functor invokes functors which (optionally) have
@@ -198,6 +58,7 @@
//
///////////////////////////////////////////////////////////////////////////
template <typename Token
+ , template <typename, typename, typename> class FunctorData
, typename Iterator = typename Token::iterator_type
, typename SupportsActors = mpl::false_
, typename SupportsState = typename Token::has_state>
@@ -212,23 +73,22 @@
// Needed by compilers not implementing the resolution to DR45. For
// reference, see
// http://www.open-std.org/JTC1/SC22/WG21/docs/cwg_defects.html#45.
- template <typename Iterator_, typename HasActors, typename HasState>
- friend struct detail::data;
+ friend class FunctorData<Iterator, SupportsActors, SupportsState>;
// Helper template allowing to assign a value on exit
template <typename T>
struct assign_on_exit
{
- assign_on_exit(T& dst_, T const& src_)
- : dst(dst_), src(src_)
- {}
+ assign_on_exit(T& dst, T const& src)
+ : dst_(dst), src_(src) {}
+
~assign_on_exit()
{
- dst = src;
+ dst_ = src_;
}
- T& dst;
- T const& src;
+ T& dst_;
+ T const& src_;
};
public:
@@ -250,13 +110,15 @@
// interface to the iterator_policies::split_functor_input policy
typedef Token result_type;
typedef functor unique;
- typedef detail::data<Iterator, SupportsActors, SupportsState> shared;
+ typedef FunctorData<Iterator, SupportsActors, SupportsState> shared;
BOOST_SPIRIT_EOF_PREFIX result_type const eof;
///////////////////////////////////////////////////////////////////////
typedef Iterator iterator_type;
typedef typename shared::semantic_actions_type semantic_actions_type;
+ typedef typename shared::next_token_functor next_token_functor;
+ typedef typename shared::get_state_name_type get_state_name_type;
// this is needed to wrap the semantic actions in a proper way
typedef typename shared::wrap_action_type wrap_action_type;
@@ -266,78 +128,94 @@
static result_type& get_next(MultiPass& mp, result_type& result)
{
shared& data = mp.shared->ftor;
- if (data.first == data.last)
+ do {
+ if (data.first_ == data.last_)
#if defined(BOOST_SPIRIT_STATIC_EOF)
- return result = eof;
+ return result = eof;
#else
- return result = mp.ftor.eof;
+ return result = mp.ftor.eof;
#endif
- Iterator end = data.first;
- std::size_t unique_id = boost::lexer::npos;
- std::size_t id = data.next(end, unique_id);
+ Iterator end = data.first_;
+ std::size_t unique_id = boost::lexer::npos;
+ std::size_t id = data.next(end, unique_id);
- if (boost::lexer::npos == id) { // no match
+ if (boost::lexer::npos == id) { // no match
#if defined(BOOST_SPIRIT_LEXERTL_DEBUG)
- std::string next;
- Iterator it = data.first;
- for (std::size_t i = 0; i < 10 && it != data.last; ++it, ++i)
- next += *it;
-
- std::cerr << "Not matched, in state: " << data.get_state()
- << ", lookahead: >" << next << "<" << std::endl;
-#endif
- return result = result_type(0);
- }
- else if (0 == id) { // EOF reached
+ std::string next;
+ Iterator it = data.first_;
+ for (std::size_t i = 0; i < 10 && it != data.last_; ++it, ++i)
+ next += *it;
+
+ std::cerr << "Not matched, in state: " << data.get_state()
+ << ", lookahead: >" << next << "<" << std::endl;
+#endif
+ return result = result_type(0);
+ }
+ else if (0 == id) { // EOF reached
#if defined(BOOST_SPIRIT_STATIC_EOF)
- return result = eof;
+ return result = eof;
#else
- return result = mp.ftor.eof;
+ return result = mp.ftor.eof;
#endif
- }
+ }
#if defined(BOOST_SPIRIT_LEXERTL_DEBUG)
- {
- std::string next;
- Iterator it = end;
- for (std::size_t i = 0; i < 10 && it != data.last; ++it, ++i)
- next += *it;
-
- std::cerr << "Matched: " << id << ", in state: "
- << data.get_state() << ", string: >"
- << std::basic_string<char_type>(data.first, end) << "<"
- << ", lookahead: >" << next << "<" << std::endl;
- }
-#endif
- // invoke_actions might change state
- std::size_t state = data.get_state();
+ {
+ std::string next;
+ Iterator it = end;
+ for (std::size_t i = 0; i < 10 && it != data.last_; ++it, ++i)
+ next += *it;
+
+ std::cerr << "Matched: " << id << ", in state: "
+ << data.get_state() << ", string: >"
+ << std::basic_string<char_type>(data.first_, end) << "<"
+ << ", lookahead: >" << next << "<" << std::endl;
+ }
+#endif
+ // invoke_actions might change state, id, data.first_, and/or end
+ std::size_t state = data.get_state();
+
+ // account for a possibly pending lex::more(), i.e. moving
+ // data.first_ back to the start of the previously matched token.
+ bool adjusted = data.adjust_start();
+
+ // invoke attached semantic actions, if defined
+ BOOST_SCOPED_ENUM(pass_flags) pass =
+ data.invoke_actions(state, id, unique_id, end);
+
+ if (pass_flags::pass_normal == pass) {
+ // return matched token, advancing 'data.first_' past the
+ // matched sequence
+ assign_on_exit<Iterator> on_exit(data.first_, end);
+ return result = result_type(id, state, data.first_, end);
+ }
+ else if (pass_flags::pass_fail == pass) {
+ // if the data.first_ got adjusted above, revert this adjustment
+ if (adjusted)
+ data.revert_adjust_start();
+
+ // one of the semantic actions signaled no-match
+ return result = result_type(0);
+ }
- // invoke attached semantic actions, if defined
- if (!data.invoke_actions(state, id, unique_id, end))
- {
- // one of the semantic actions signaled no-match
- return result = result_type(0);
- }
+ // if this token needs to be ignored, just repeat the matching
- // return matched token, advancing 'data.first' past the
- // matched sequence
- assign_on_exit<Iterator> on_exit(data.first, end);
- return result = result_type(id, state, data.first, end);
+ } while (true);
}
// set_state are propagated up to the iterator interface, allowing to
// manipulate the current lexer state through any of the exposed
// iterators.
template <typename MultiPass>
- static std::size_t set_state(MultiPass& mp, std::size_t state_)
+ static std::size_t set_state(MultiPass& mp, std::size_t state)
{
- std::size_t oldstate = mp.shared->ftor.state;
- mp.shared->ftor.state = state_;
+ std::size_t oldstate = mp.shared->ftor.get_state();
+ mp.shared->ftor.set_state(state);
#if defined(BOOST_SPIRIT_LEXERTL_DEBUG)
std::cerr << "Switching state from: " << oldstate
- << " to: " << state_
+ << " to: " << state
<< std::endl;
#endif
return oldstate;
@@ -347,7 +225,7 @@
static std::size_t
map_state(MultiPass const& mp, char_type const* statename)
{
- return mp.shared->ftor.rules.state(statename);
+ return mp.shared->ftor.get_state_id(statename);
}
// we don't need this, but it must be there
@@ -359,11 +237,13 @@
///////////////////////////////////////////////////////////////////////////
// eof token
///////////////////////////////////////////////////////////////////////////
- template <typename Token, typename Iterator, typename SupportsActors
- , typename SupportsState>
- typename functor<Token, Iterator, SupportsActors, SupportsState>::result_type const
- functor<Token, Iterator, SupportsActors, SupportsState>::eof =
- typename functor<Token, Iterator, SupportsActors, SupportsState>::result_type();
+ template <typename Token
+ , template <typename, typename, typename> class FunctorData
+ , typename Iterator, typename SupportsActors, typename SupportsState>
+ typename functor<Token, FunctorData, Iterator, SupportsActors, SupportsState>::result_type const
+ functor<Token, FunctorData, Iterator, SupportsActors, SupportsState>::eof =
+ typename functor<Token, FunctorData, Iterator, SupportsActors
+ , SupportsState>::result_type();
#endif
}}}}
Added: trunk/boost/spirit/home/lex/lexer/lexertl/functor_data.hpp
==============================================================================
--- (empty file)
+++ trunk/boost/spirit/home/lex/lexer/lexertl/functor_data.hpp 2009-06-10 23:39:04 EDT (Wed, 10 Jun 2009)
@@ -0,0 +1,318 @@
+// Copyright (c) 2001-2009 Hartmut Kaiser
+//
+// Distributed under the Boost Software License, Version 1.0. (See accompanying
+// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+
+#if !defined(BOOST_SPIRIT_LEX_LEXER_FUNCTOR_DATA_JUN_10_2009_0954AM)
+#define BOOST_SPIRIT_LEX_LEXER_FUNCTOR_DATA_JUN_10_2009_0954AM
+
+#if defined(_MSC_VER) && (_MSC_VER >= 1020)
+#pragma once // MS compatible compilers support #pragma once
+#endif
+
+#include <boost/spirit/home/support/detail/lexer/generator.hpp>
+#include <boost/spirit/home/support/detail/lexer/rules.hpp>
+#include <boost/spirit/home/support/detail/lexer/state_machine.hpp>
+#include <boost/spirit/home/lex/lexer/lexertl/iterator_tokenizer.hpp>
+#include <boost/spirit/home/lex/lexer/lexertl/semantic_action_data.hpp>
+#include <boost/spirit/home/lex/lexer/lexertl/wrap_action.hpp>
+#include <boost/mpl/bool.hpp>
+
+namespace boost { namespace spirit { namespace lex { namespace lexertl
+{
+ namespace detail
+ {
+ ///////////////////////////////////////////////////////////////////////
+ template <typename Iterator, typename HasActors, typename HasState>
+ struct data; // no default specialization
+
+ ///////////////////////////////////////////////////////////////////////
+ // neither supports state, nor actors
+ template <typename Iterator>
+ struct data<Iterator, mpl::false_, mpl::false_>
+ {
+ protected:
+ typedef typename
+ boost::detail::iterator_traits<Iterator>::value_type
+ char_type;
+
+ public:
+ typedef Iterator base_iterator_type;
+ typedef std::size_t state_type;
+ typedef char_type const* state_name_type;
+ typedef unused_type semantic_actions_type;
+ typedef detail::wrap_action<unused_type, Iterator, data, std::size_t>
+ wrap_action_type;
+
+ typedef unused_type next_token_functor;
+ typedef unused_type get_state_name_type;
+
+ // initialize the shared data
+ template <typename IterData>
+ data (IterData const& data, Iterator& first, Iterator const& last)
+ : first_(first), last_(last)
+ , state_machine_(data.state_machine_)
+ , rules_(data.rules_) {}
+
+ // The following functions are used by the implementation of the
+ // placeholder '_state'.
+ void set_state_name (char_type const* state) {}
+ char_type const* get_state_name() const { return rules_.initial(); }
+ std::size_t get_state_id (char_type const* state) const
+ {
+ return 0;
+ }
+
+ // The function get_eoi() is used by the implementation of the
+ // placeholder '_eoi'.
+ Iterator const& get_eoi() const { return last_; }
+
+ // The function less() is used by the implementation of the support
+ // function lex::less(). Its functionality is equivalent to flex'
+ // function yyless(): it returns an iterator positioned to the
+ // nth input character beyond the current start iterator (i.e. by
+ // assigning the return value to the placeholder '_end' it is
+ // possible to return all but the first n characters of the current
+ // token back to the input stream.
+ //
+ // This function does nothing as long as no semantic actions are
+ // used.
+ Iterator const& less(Iterator const& it, int n)
+ {
+ // The following assertion fires most likely because you are
+ // using lexer semantic actions without using the actor_lexer
+ // as the base class for your token definition class.
+ BOOST_ASSERT(false);
+ return it;
+ }
+
+ // The function more() is used by the implemention of the support
+ // function lex::more(). Its functionality is equivalent to flex'
+ // function yymore(): it tells the lexer that the next time it
+ // matches a rule, the corresponding token should be appended onto
+ // the current token value rather than replacing it.
+ //
+ // These functions do nothing as long as no semantic actions are
+ // used.
+ void more()
+ {
+ // The following assertion fires most likely because you are
+ // using lexer semantic actions without using the actor_lexer
+ // as the base class for your token definition class.
+ BOOST_ASSERT(false);
+ }
+ bool adjust_start() { return false; }
+ void revert_adjust_start() {}
+
+ // The function lookahead() is used by the implementation of the
+ // support function lex::lookahead. It can be used to implement
+ // lookahead for lexer engines not supporting constructs like flex'
+ // a/b (match a, but only when followed by b):
+ //
+ // This function does nothing as long as no semantic actions are
+ // used.
+ bool lookahead(std::size_t id)
+ {
+ // The following assertion fires most likely because you are
+ // using lexer semantic actions without using the actor_lexer
+ // as the base class for your token definition class.
+ BOOST_ASSERT(false);
+ return false;
+ }
+
+ // the functions next, invoke_actions, and get_state are used by
+ // the functor implementation below
+
+ // The function next() tries to match the next token from the
+ // underlying input sequence.
+ std::size_t next(Iterator& end, std::size_t& unique_id)
+ {
+ typedef basic_iterator_tokeniser<Iterator> tokenizer;
+ return tokenizer::next(state_machine_, first_, end, last_
+ , unique_id);
+ }
+
+ // nothing to invoke, so this is empty
+ BOOST_SCOPED_ENUM(pass_flags) invoke_actions(std::size_t
+ , std::size_t, std::size_t, Iterator const&)
+ {
+ return pass_flags::pass_normal; // always accept
+ }
+
+ std::size_t get_state() const { return 0; }
+ void set_state(std::size_t state) {}
+
+ Iterator& first_;
+ Iterator last_;
+
+ protected:
+ boost::lexer::basic_state_machine<char_type> const& state_machine_;
+ boost::lexer::basic_rules<char_type> const& rules_;
+ };
+
+ ///////////////////////////////////////////////////////////////////////
+ // doesn't support lexer semantic actions
+ template <typename Iterator>
+ struct data<Iterator, mpl::false_, mpl::true_>
+ : data<Iterator, mpl::false_, mpl::false_>
+ {
+ protected:
+ typedef data<Iterator, mpl::false_, mpl::false_> base_type;
+ typedef typename base_type::char_type char_type;
+
+ public:
+ typedef Iterator base_iterator_type;
+ typedef typename base_type::state_type state_type;
+ typedef typename base_type::state_name_type state_name_type;
+ typedef typename base_type::semantic_actions_type
+ semantic_actions_type;
+
+ // initialize the shared data
+ template <typename IterData>
+ data (IterData const& data, Iterator& first, Iterator const& last)
+ : base_type(data, first, last)
+ , state_(0) {}
+
+ // The following functions are used by the implementation of the
+ // placeholder '_state'.
+ void set_state_name (char_type const* new_state)
+ {
+ std::size_t state_id = this->rules_.state(new_state);
+
+ // If the following assertion fires you've probably been using
+ // a lexer state name which was not defined in your token
+ // definition.
+ BOOST_ASSERT(state_id != boost::lexer::npos);
+
+ if (state_id != boost::lexer::npos)
+ state_ = state_id;
+ }
+ char_type const* get_state_name() const
+ {
+ return this->rules_.state(state_);
+ }
+ std::size_t get_state_id (char_type const* state) const
+ {
+ return this->rules_.state(state);
+ }
+
+ // the functions next() and get_state() are used by the functor
+ // implementation below
+
+ // The function next() tries to match the next token from the
+ // underlying input sequence.
+ std::size_t next(Iterator& end, std::size_t& unique_id)
+ {
+ typedef basic_iterator_tokeniser<Iterator> tokenizer;
+ return tokenizer::next(this->state_machine_, state_,
+ this->first_, end, this->last_, unique_id);
+ }
+
+ std::size_t& get_state() { return state_; }
+ void set_state(std::size_t state) { state_ = state; }
+
+ protected:
+ std::size_t state_;
+ };
+
+ ///////////////////////////////////////////////////////////////////////
+ // does support lexer semantic actions, may support state
+ template <typename Iterator, typename HasState>
+ struct data<Iterator, mpl::true_, HasState>
+ : data<Iterator, mpl::false_, HasState>
+ {
+ public:
+ typedef semantic_actions<Iterator, HasState, data>
+ semantic_actions_type;
+
+ protected:
+ typedef data<Iterator, mpl::false_, HasState> base_type;
+ typedef typename base_type::char_type char_type;
+ typedef typename semantic_actions_type::functor_wrapper_type
+ functor_wrapper_type;
+
+ public:
+ typedef Iterator base_iterator_type;
+ typedef typename base_type::state_type state_type;
+ typedef typename base_type::state_name_type state_name_type;
+
+ typedef detail::wrap_action<functor_wrapper_type
+ , Iterator, data, std::size_t> wrap_action_type;
+
+ template <typename IterData>
+ data (IterData const& data, Iterator& first, Iterator const& last)
+ : base_type(data, first, last)
+ , actions_(data.actions_), hold_(), has_hold_(false) {}
+
+ // invoke attached semantic actions, if defined
+ BOOST_SCOPED_ENUM(pass_flags) invoke_actions(std::size_t state
+ , std::size_t& id, std::size_t unique_id, Iterator& end)
+ {
+ return actions_.invoke_actions(state, id, unique_id, end, *this);
+ }
+
+ // The function less() is used by the implementation of the support
+ // function lex::less(). Its functionality is equivalent to flex'
+ // function yyless(): it returns an iterator positioned to the
+ // nth input character beyond the current start iterator (i.e. by
+ // assigning the return value to the placeholder '_end' it is
+ // possible to return all but the first n characters of the current
+ // token back to the input stream).
+ Iterator const& less(Iterator& it, int n)
+ {
+ it = this->first_;
+ std::advance(it, n);
+ return it;
+ }
+
+ // The function more() is used by the implemention of the support
+ // function lex::more(). Its functionality is equivalent to flex'
+ // function yymore(): it tells the lexer that the next time it
+ // matches a rule, the corresponding token should be appended onto
+ // the current token value rather than replacing it.
+ void more()
+ {
+ hold_ = this->first_;
+ has_hold_ = true;
+ }
+
+ // The function lookahead() is used by the implementation of the
+ // support function lex::lookahead. It can be used to implement
+ // lookahead for lexer engines not supporting constructs like flex'
+ // a/b (match a, but only when followed by b)
+ bool lookahead(std::size_t id)
+ {
+ Iterator end = this->first_;
+ std::size_t unique_id = boost::lexer::npos;
+ return id == next(end, unique_id);
+ }
+
+ // The adjust_start() and revert_adjust_start() are helper
+ // functions needed to implement the functionality required for
+ // lex::more(). It is called from the functor body below.
+ bool adjust_start()
+ {
+ if (!has_hold_)
+ return false;
+
+ std::swap(this->first_, hold_);
+ has_hold_ = false;
+ return true;
+ }
+ void revert_adjust_start()
+ {
+ // this will be called only if adjust_start above returned true
+ std::swap(this->first_, hold_);
+ has_hold_ = true;
+ }
+
+ protected:
+ semantic_actions_type const& actions_;
+ Iterator hold_; // iterator needed to support lex::more()
+ bool has_hold_; // 'true' if hold_ is valid
+ };
+ }
+
+}}}}
+
+#endif
Modified: trunk/boost/spirit/home/lex/lexer/lexertl/iterator.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/lexertl/iterator.hpp (original)
+++ trunk/boost/spirit/home/lex/lexer/lexertl/iterator.hpp 2009-06-10 23:39:04 EDT (Wed, 10 Jun 2009)
@@ -79,9 +79,12 @@
// for tokenization
template <typename IteratorData>
iterator(IteratorData const& iterdata_, base_iterator_type& first
- , base_iterator_type const& last)
+ , base_iterator_type const& last, char_type const* state = 0)
: base_type(functor_type(unique_functor_type()
- , shared_functor_type(iterdata_, first, last))) {}
+ , shared_functor_type(iterdata_, first, last)))
+ {
+ set_state(map_state(state));
+ }
// create an end iterator usable for end of range checking
iterator() {}
@@ -96,7 +99,9 @@
// by the underlying lexer object
std::size_t map_state(char_type const* statename)
{
- return unique_functor_type::map_state(*this, statename);
+ return (0 != statename)
+ ? unique_functor_type::map_state(*this, statename)
+ : 0;
}
};
Modified: trunk/boost/spirit/home/lex/lexer/lexertl/lexer.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/lexertl/lexer.hpp (original)
+++ trunk/boost/spirit/home/lex/lexer/lexertl/lexer.hpp 2009-06-10 23:39:04 EDT (Wed, 10 Jun 2009)
@@ -20,6 +20,7 @@
#include <boost/spirit/home/lex/lexer/lexertl/token.hpp>
#include <boost/spirit/home/lex/lexer/lexertl/functor.hpp>
+#include <boost/spirit/home/lex/lexer/lexertl/functor_data.hpp>
#include <boost/spirit/home/lex/lexer/lexertl/iterator.hpp>
#include <boost/spirit/home/lex/lexer/lexertl/unique_id.hpp>
#if defined(BOOST_SPIRIT_LEXERTL_DEBUG)
@@ -204,7 +205,7 @@
///////////////////////////////////////////////////////////////////////////
template <typename Token = token<>
, typename Iterator = typename Token::iterator_type
- , typename Functor = functor<Token, Iterator, mpl::false_>
+ , typename Functor = functor<Token, lexertl::detail::data, Iterator, mpl::false_>
, typename TokenSet = lex::token_set<token_set<Token, Iterator> > >
class lexer
{
@@ -238,13 +239,14 @@
public:
// Return the start iterator usable for iterating over the generated
// tokens.
- iterator_type begin(Iterator& first, Iterator const& last) const
+ iterator_type begin(Iterator& first, Iterator const& last
+ , char_type const* initial_state = 0) const
{
if (!init_dfa())
return iterator_type();
iterator_data_type iterator_data = { state_machine_, rules_, actions_ };
- return iterator_type(iterator_data, first, last);
+ return iterator_type(iterator_data, first, last, initial_state);
}
// Return the end iterator usable to stop iterating over the generated
@@ -287,7 +289,6 @@
add_state(state);
initialized_dfa_ = false;
rules_.add("*", tokset.get_rules(), state);
-// rules_.add(state, tokset.get_rules());
}
// Allow to associate a whole lexer instance with another lexer
@@ -301,7 +302,6 @@
add_state(state);
initialized_dfa_ = false;
rules_.add("*", lexer_def.get_rules(), state);
-// rules_.add(state, tokset.get_rules());
}
// interface for pattern definition management
@@ -340,23 +340,8 @@
template <typename F>
void add_action(id_type unique_id, std::size_t state, F act)
{
- // If you get compilation errors below stating value_type not being
- // a member of boost::fusion::unused_type, then you are probably
- // using semantic actions in your token definition without
- // the actor_lexer being specified as the base class of your lexer
- // (instead of the lexer class).
- typedef typename Functor::semantic_actions_type::value_type
- value_type;
typedef typename Functor::wrap_action_type wrapper_type;
-
- if (actions_.size() <= state)
- actions_.resize(state + 1);
-
- value_type& actions (actions_[state]);
- if (actions.size() <= unique_id)
- actions.resize(unique_id + 1);
-
- actions[unique_id] = wrapper_type::call(act);
+ actions_.add_action(unique_id, state, wrapper_type::call(act));
}
bool init_dfa() const
@@ -410,7 +395,7 @@
///////////////////////////////////////////////////////////////////////////
template <typename Token = token<>
, typename Iterator = typename Token::iterator_type
- , typename Functor = functor<Token, Iterator, mpl::true_>
+ , typename Functor = functor<Token, lexertl::detail::data, Iterator, mpl::true_>
, typename TokenSet = lex::token_set<token_set<Token, Iterator> > >
class actor_lexer : public lexer<Token, Iterator, Functor, TokenSet>
{
Added: trunk/boost/spirit/home/lex/lexer/lexertl/semantic_action_data.hpp
==============================================================================
--- (empty file)
+++ trunk/boost/spirit/home/lex/lexer/lexertl/semantic_action_data.hpp 2009-06-10 23:39:04 EDT (Wed, 10 Jun 2009)
@@ -0,0 +1,118 @@
+// Copyright (c) 2001-2009 Hartmut Kaiser
+//
+// Distributed under the Boost Software License, Version 1.0. (See accompanying
+// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+
+#if !defined(BOOST_SPIRIT_LEX_LEXER_SEMANTIC_ACTION_DATA_JUN_10_2009_0417PM)
+#define BOOST_SPIRIT_LEX_LEXER_SEMANTIC_ACTION_DATA_JUN_10_2009_0417PM
+
+#if defined(_MSC_VER) && (_MSC_VER >= 1020)
+#pragma once // MS compatible compilers support #pragma once
+#endif
+
+#include <boost/spirit/home/lex/lexer/pass_flags.hpp>
+#include <boost/mpl/bool.hpp>
+#include <boost/function.hpp>
+#include <vector>
+
+namespace boost { namespace spirit { namespace lex { namespace lexertl
+{
+ namespace detail
+ {
+ ///////////////////////////////////////////////////////////////////////
+ template <typename Iterator, typename SupportsState, typename Data>
+ struct semantic_actions;
+
+ // This specialization of semantic_actions will be used if the token
+ // type (lexer definition) does not support states, which simplifies
+ // the data structures used to store the semantic action function
+ // objects.
+ template <typename Iterator, typename Data>
+ struct semantic_actions<Iterator, mpl::false_, Data>
+ {
+ typedef void functor_type(Iterator&, Iterator&
+ , BOOST_SCOPED_ENUM(pass_flags)&, std::size_t&, Data&);
+ typedef boost::function<functor_type> functor_wrapper_type;
+
+ // add a semantic action function object
+ template <typename Idtype, typename F>
+ void add_action(Idtype unique_id, std::size_t, F act)
+ {
+ if (actions_.size() <= unique_id)
+ actions_.resize(unique_id + 1);
+
+ actions_[unique_id] = act;
+ }
+
+ // try to invoke a semantic action for the given token (unique_id)
+ BOOST_SCOPED_ENUM(pass_flags) invoke_actions(std::size_t state
+ , std::size_t& id, std::size_t unique_id, Iterator& end
+ , Data& data) const
+ {
+ // if there is nothing to invoke, continue with 'match'
+ if (unique_id >= actions_.size() || !actions_[unique_id])
+ return pass_flags::pass_normal;
+
+ // Note: all arguments might be changed by the invoked semantic
+ // action
+ BOOST_SCOPED_ENUM(pass_flags) match = pass_flags::pass_normal;
+ actions_[unique_id](data.first_, end, match, id, data);
+ return match;
+ }
+
+ std::vector<functor_wrapper_type> actions_;
+ };
+
+ // This specialization of semantic_actions will be used if the token
+ // type (lexer definition) needs to support states, resulting in a more
+ // complex data structure needed for storing the semantic action
+ // function objects.
+ template <typename Iterator, typename Data>
+ struct semantic_actions<Iterator, mpl::true_, Data>
+ {
+ typedef void functor_type(Iterator&, Iterator&
+ , BOOST_SCOPED_ENUM(pass_flags)&, std::size_t&, Data&);
+ typedef boost::function<functor_type> functor_wrapper_type;
+
+ // add a semantic action function object
+ template <typename Idtype, typename F>
+ void add_action(Idtype unique_id, std::size_t state, F act)
+ {
+ if (actions_.size() <= state)
+ actions_.resize(state + 1);
+
+ std::vector<functor_wrapper_type>& actions (actions_[state]);
+ if (actions.size() <= unique_id)
+ actions.resize(unique_id + 1);
+
+ actions[unique_id] = act;
+ }
+
+ // try to invoke a semantic action for the given token (unique_id)
+ BOOST_SCOPED_ENUM(pass_flags) invoke_actions(std::size_t state
+ , std::size_t& id, std::size_t unique_id, Iterator& end
+ , Data& data) const
+ {
+ // if there is no action defined for this state, return match
+ if (state >= actions_.size())
+ return pass_flags::pass_normal;
+
+ // if there is nothing to invoke, continue with 'match'
+ std::vector<functor_wrapper_type> const& actions = actions_[state];
+ if (unique_id >= actions.size() || !actions[unique_id])
+ return pass_flags::pass_normal;
+
+ // Note: all arguments might be changed by the invoked semantic
+ // action
+ BOOST_SCOPED_ENUM(pass_flags) match = pass_flags::pass_normal;
+ actions[unique_id](data.first_, end, match, id, data);
+ return match;
+ }
+
+ std::vector<std::vector<functor_wrapper_type> > actions_;
+ };
+ }
+
+}}}}
+
+#endif
Deleted: trunk/boost/spirit/home/lex/lexer/lexertl/static_functor.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/lexertl/static_functor.hpp 2009-06-10 23:39:04 EDT (Wed, 10 Jun 2009)
+++ (empty file)
@@ -1,382 +0,0 @@
-// Copyright (c) 2001-2009 Hartmut Kaiser
-//
-// Distributed under the Boost Software License, Version 1.0. (See accompanying
-// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
-
-#if !defined(BOOST_SPIRIT_LEX_LEXER_STATIC_FUNCTOR_FEB_10_2008_0755PM)
-#define BOOST_SPIRIT_LEX_LEXER_STATIC_FUNCTOR_FEB_10_2008_0755PM
-
-#if defined(_MSC_VER) && (_MSC_VER >= 1020)
-#pragma once // MS compatible compilers support #pragma once
-#endif
-
-#include <boost/mpl/bool.hpp>
-#include <boost/function.hpp>
-#include <boost/range/iterator_range.hpp>
-#include <boost/detail/iterator.hpp>
-#include <boost/detail/workaround.hpp>
-#include <map>
-#include <boost/spirit/home/support/detail/lexer/generator.hpp>
-#include <boost/spirit/home/support/detail/lexer/rules.hpp>
-#include <boost/spirit/home/support/detail/lexer/state_machine.hpp>
-#include <boost/spirit/home/lex/lexer/lexertl/iterator_tokenizer.hpp>
-#include <boost/spirit/home/lex/lexer/lexertl/wrap_action.hpp>
-
-#if 0 != __COMO_VERSION__ || !BOOST_WORKAROUND(BOOST_MSVC, <= 1310)
-#define BOOST_SPIRIT_STATIC_EOF 1
-#define BOOST_SPIRIT_EOF_PREFIX static
-#else
-#define BOOST_SPIRIT_EOF_PREFIX
-#endif
-
-namespace boost { namespace spirit { namespace lex { namespace lexertl
-{
- namespace detail
- {
- ///////////////////////////////////////////////////////////////////////
- template <typename Iterator, typename HasActors, typename HasState>
- struct static_data; // no default specialization
-
- ///////////////////////////////////////////////////////////////////////
- // doesn't support no state and no actors
- template <typename Iterator>
- struct static_data<Iterator, mpl::false_, mpl::false_>
- {
- typedef std::size_t state_type;
- typedef iterator_range<Iterator> iterpair_type;
- typedef typename
- boost::detail::iterator_traits<Iterator>::value_type
- char_type;
-
- typedef std::size_t (*next_token_functor)(std::size_t&,
- Iterator const&, Iterator&, Iterator const&, std::size_t&);
-
- typedef unused_type semantic_actions_type;
- typedef unused_type get_state_id_type;
-
- typedef detail::wrap_action<unused_type, iterpair_type, static_data>
- wrap_action_type;
-
- // initialize the shared data
- template <typename IterData>
- static_data (IterData const& data_, Iterator& first_, Iterator const& last_)
- : next_token(data_.next_), first(first_), last(last_)
- {}
-
- std::size_t next(Iterator& end, std::size_t& unique_id)
- {
- std::size_t state;
- return next_token(state, first, end, last, unique_id);
- }
-
- // nothing to invoke, so this is empty
- bool invoke_actions(std::size_t, std::size_t, std::size_t
- , Iterator const&)
- {
- return true; // always accept
- }
-
- std::size_t get_state() const { return 0; }
- void set_state_name (char const*) {}
-
- next_token_functor next_token;
- Iterator& first;
- Iterator last;
- };
-
- ///////////////////////////////////////////////////////////////////////
- // doesn't support no actors
- template <typename Iterator>
- struct static_data<Iterator, mpl::false_, mpl::true_>
- : static_data<Iterator, mpl::false_, mpl::false_>
- {
- typedef static_data<Iterator, mpl::false_, mpl::false_> base_type;
-
- typedef typename base_type::state_type state_type;
- typedef typename base_type::char_type char_type;
- typedef typename base_type::semantic_actions_type
- semantic_actions_type;
- typedef std::size_t (*get_state_id_type)(char const*);
-
- // initialize the shared data
- template <typename IterData>
- static_data (IterData const& data_, Iterator& first_, Iterator const& last_)
- : base_type(data_, first_, last_), state_(0)
- , get_state_id_(data_.get_state_id_)
- {}
-
- std::size_t next(Iterator& end, std::size_t& unique_id)
- {
- return this->next_token(state_, this->first, end, this->last
- , unique_id);
- }
-
- std::size_t& get_state() { return state_; }
- void set_state_name (char_type const* new_state)
- {
- std::size_t state_id = get_state_id_(new_state);
-
- // if the following assertion fires you've probably been using
- // a lexer state name which was not defined in your token
- // definition
- BOOST_ASSERT(state_id != boost::lexer::npos);
-
- if (state_id != boost::lexer::npos)
- state_ = state_id;
- }
-
- std::size_t state_;
- get_state_id_type get_state_id_;
- };
-
- ///////////////////////////////////////////////////////////////////////
- // does support actors, but may have no state
- template <typename Iterator, typename HasState>
- struct static_data<Iterator, mpl::true_, HasState>
- : static_data<Iterator, mpl::false_, HasState>
- {
- typedef static_data<Iterator, mpl::false_, HasState> base_type;
-
- typedef iterator_range<Iterator> iterpair_type;
- typedef typename base_type::state_type state_type;
- typedef typename base_type::char_type char_type;
-
- typedef void functor_type(iterpair_type, std::size_t, bool&, static_data&);
- typedef boost::function<functor_type> functor_wrapper_type;
- typedef std::vector<std::vector<functor_wrapper_type> >
- semantic_actions_type;
- typedef typename base_type::get_state_id_type get_state_id_type;
-
- typedef detail::wrap_action<functor_wrapper_type
- , iterpair_type, static_data> wrap_action_type;
-
- template <typename IterData>
- static_data (IterData const& data_, Iterator& first_, Iterator const& last_)
- : base_type(data_, first_, last_)
- , actions_(data_.actions_) {}
-
- // invoke attached semantic actions, if defined
- bool invoke_actions(std::size_t state, std::size_t id
- , std::size_t unique_id, Iterator const& end)
- {
- if (state >= actions_.size())
- return true; // no action defined for this state
-
- std::vector<functor_wrapper_type> const& actions = actions_[state];
-
- if (unique_id >= actions.size() || !actions[unique_id])
- return true; // nothing to invoke, continue with 'match'
-
- iterpair_type itp(this->first, end);
- bool match = true;
- actions[unique_id](itp, id, match, *this);
- return match;
- }
-
- semantic_actions_type const& actions_;
- };
- }
-
- ///////////////////////////////////////////////////////////////////////////
- //
- // static_functor is a template usable as the functor object for
- // the multi_pass iterator allowing to wrap a lexertl based dfa into a
- // iterator based interface.
- //
- // Iterator: the type of the underlying iterator
- // Token: the type of the tokens produced by this functor
- // this needs to expose a constructor with the following
- // prototype:
- //
- // Token(std::size_t id, std::size_t state,
- // Iterator start, Iterator end)
- //
- // where 'id' is the token id, state is the lexer state,
- // this token has been matched in, and 'first' and 'end'
- // mark the start and the end of the token with respect
- // to the underlying character stream.
- // SupportsActors:
- // this is expected to be a mpl::bool_, if mpl::true_ the
- // static_functor invokes functors which
- // (optionally) have been attached to the token definitions.
- // SupportState:
- // this is expected to be a mpl::bool_, if mpl::true_ the
- // functor supports different lexer states,
- // otherwise no lexer state is supported.
- //
- ///////////////////////////////////////////////////////////////////////////
- template <typename Token
- , typename Iterator = typename Token::iterator_type
- , typename SupportsActors = mpl::false_
- , typename SupportsState = typename Token::has_state>
- class static_functor
- {
- public:
- typedef typename boost::detail::iterator_traits<Iterator>::value_type
- char_type;
-
- private:
- // Needed by compilers not implementing the resolution to DR45. For
- // reference, see
- // http://www.open-std.org/JTC1/SC22/WG21/docs/cwg_defects.html#45.
- template <typename Iterator_, typename HasActors, typename HasState>
- friend struct detail::static_data;
-
- // Helper template allowing to assign a value on exit
- template <typename T>
- struct assign_on_exit
- {
- assign_on_exit(T& dst_, T const& src_)
- : dst(dst_), src(src_)
- {}
- ~assign_on_exit()
- {
- dst = src;
- }
-
- T& dst;
- T const& src;
- };
-
- public:
- static_functor()
-#if defined(__PGI)
- : eof()
-#endif
- {}
-
-#if BOOST_WORKAROUND(BOOST_MSVC, <= 1310)
- // somehow VC7.1 needs this (meaningless) assignment operator
- functor& operator=(functor const& rhs)
- {
- return *this;
- }
-#endif
-
- ///////////////////////////////////////////////////////////////////////
- // interface to the iterator_policies::split_functor_input policy
- typedef Token result_type;
- typedef static_functor unique;
- typedef detail::static_data<Iterator, SupportsActors, SupportsState> shared;
-
- BOOST_SPIRIT_EOF_PREFIX result_type const eof;
-
- ///////////////////////////////////////////////////////////////////////
- typedef Iterator iterator_type;
- typedef typename shared::semantic_actions_type semantic_actions_type;
- typedef typename shared::next_token_functor next_token_functor;
-
- // this is needed to wrap the semantic actions in a proper way
- typedef typename shared::wrap_action_type wrap_action_type;
-
- ///////////////////////////////////////////////////////////////////////
- template <typename MultiPass>
- static result_type& get_next(MultiPass& mp, result_type& result)
- {
- shared& data = mp.shared->ftor;
- if (data.first == data.last)
-#if defined(BOOST_SPIRIT_STATIC_EOF)
- return result = eof;
-#else
- return result = mp.ftor.eof;
-#endif
-
- Iterator end = data.first;
- std::size_t unique_id = boost::lexer::npos;
- std::size_t id = data.next(end, unique_id);
-
- if (boost::lexer::npos == id) { // no match
-#if defined(BOOST_SPIRIT_DEBUG)
- std::string next;
- Iterator it = data.first;
- for (std::size_t i = 0; i < 10 && it != data.last; ++it, ++i)
- next += *it;
-
- std::cerr << "Not matched, in state: " << data.get_state()
- << ", lookahead: >" << next << "<" << std::endl;
-#endif
- return result = result_type(0);
- }
- else if (0 == id) { // EOF reached
-#if defined(BOOST_SPIRIT_STATIC_EOF)
- return result = eof;
-#else
- return result = mp.ftor.eof;
-#endif
- }
-
-#if defined(BOOST_SPIRIT_DEBUG)
- {
- std::string next;
- Iterator it = end;
- for (std::size_t i = 0; i < 10 && it != data.last; ++it, ++i)
- next += *it;
-
- std::cerr << "Matched: " << id << ", in state: "
- << data.get_state() << ", string: >"
- << std::basic_string<char_type>(data.first, end) << "<"
- << ", lookahead: >" << next << "<" << std::endl;
- }
-#endif
- // invoke_actions might change state
- std::size_t state = data.get_state();
-
- // invoke attached semantic actions, if there are any defined
- if (!data.invoke_actions(state, id, unique_id, end))
- {
- // one of the semantic actions signaled no-match
- return result = result_type(0);
- }
-
- // return matched token, advancing 'data.first' past the matched
- // sequence
- assign_on_exit<Iterator> on_exit(data.first, end);
- return result = result_type(id, state, data.first, end);
- }
-
- // set_state is propagated up to the iterator interface, allowing to
- // manipulate the current lexer state through any of the exposed
- // iterators.
- template <typename MultiPass>
- static std::size_t set_state(MultiPass& mp, std::size_t state_)
- {
- std::size_t oldstate = mp.shared->ftor.state;
- mp.shared->ftor.state = state_;
-
-#if defined(BOOST_SPIRIT_DEBUG)
- std::cerr << "Switching state from: " << oldstate
- << " to: " << state_
- << std::endl;
-#endif
- return oldstate;
- }
-
- template <typename MultiPass>
- static std::size_t
- map_state(MultiPass const& mp, char_type const* statename)
- {
- return mp.shared->ftor.rules.state(statename);
- }
-
- // we don't need this, but it must be there
- template <typename MultiPass>
- static void destroy(MultiPass const&) {}
- };
-
-#if defined(BOOST_SPIRIT_STATIC_EOF)
- ///////////////////////////////////////////////////////////////////////////
- // eof token
- ///////////////////////////////////////////////////////////////////////////
- template <typename Token, typename Iterator, typename SupportsActors
- , typename SupportsState>
- typename static_functor<Token, Iterator, SupportsActors, SupportsState>::result_type const
- static_functor<Token, Iterator, SupportsActors, SupportsState>::eof =
- typename static_functor<Token, Iterator, SupportsActors, SupportsState>::result_type();
-#endif
-
-}}}}
-
-#undef BOOST_SPIRIT_EOF_PREFIX
-#undef BOOST_SPIRIT_STATIC_EOF
-
-#endif
Copied: trunk/boost/spirit/home/lex/lexer/lexertl/static_functor_data.hpp (from r53639, /trunk/boost/spirit/home/lex/lexer/lexertl/static_functor.hpp)
==============================================================================
--- /trunk/boost/spirit/home/lex/lexer/lexertl/static_functor.hpp (original)
+++ trunk/boost/spirit/home/lex/lexer/lexertl/static_functor_data.hpp 2009-06-10 23:39:04 EDT (Wed, 10 Jun 2009)
@@ -3,37 +3,40 @@
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
-#if !defined(BOOST_SPIRIT_LEX_LEXER_STATIC_FUNCTOR_FEB_10_2008_0755PM)
-#define BOOST_SPIRIT_LEX_LEXER_STATIC_FUNCTOR_FEB_10_2008_0755PM
+#if !defined(BOOST_SPIRIT_LEX_LEXER_STATIC_FUNCTOR_DATA_FEB_10_2008_0755PM)
+#define BOOST_SPIRIT_LEX_LEXER_STATIC_FUNCTOR_DATA_FEB_10_2008_0755PM
#if defined(_MSC_VER) && (_MSC_VER >= 1020)
#pragma once // MS compatible compilers support #pragma once
#endif
-#include <boost/mpl/bool.hpp>
-#include <boost/function.hpp>
-#include <boost/range/iterator_range.hpp>
-#include <boost/detail/iterator.hpp>
-#include <boost/detail/workaround.hpp>
-#include <map>
#include <boost/spirit/home/support/detail/lexer/generator.hpp>
#include <boost/spirit/home/support/detail/lexer/rules.hpp>
#include <boost/spirit/home/support/detail/lexer/state_machine.hpp>
#include <boost/spirit/home/lex/lexer/lexertl/iterator_tokenizer.hpp>
+#include <boost/spirit/home/lex/lexer/lexertl/semantic_action_data.hpp>
#include <boost/spirit/home/lex/lexer/lexertl/wrap_action.hpp>
-
-#if 0 != __COMO_VERSION__ || !BOOST_WORKAROUND(BOOST_MSVC, <= 1310)
-#define BOOST_SPIRIT_STATIC_EOF 1
-#define BOOST_SPIRIT_EOF_PREFIX static
-#else
-#define BOOST_SPIRIT_EOF_PREFIX
-#endif
+#include <boost/mpl/bool.hpp>
+#include <boost/algorithm/string/predicate.hpp>
namespace boost { namespace spirit { namespace lex { namespace lexertl
{
namespace detail
{
///////////////////////////////////////////////////////////////////////
+ template <typename Char, typename F>
+ inline std::size_t get_state_id(Char const* state, F f
+ , std::size_t numstates)
+ {
+ for (std::size_t i = 0; i < numstates; ++i)
+ {
+ if (boost::algorithm::equals(f(i), state))
+ return i;
+ }
+ return boost::lexer::npos;
+ }
+
+ ///////////////////////////////////////////////////////////////////////
template <typename Iterator, typename HasActors, typename HasState>
struct static_data; // no default specialization
@@ -42,46 +45,127 @@
template <typename Iterator>
struct static_data<Iterator, mpl::false_, mpl::false_>
{
- typedef std::size_t state_type;
- typedef iterator_range<Iterator> iterpair_type;
+ protected:
typedef typename
boost::detail::iterator_traits<Iterator>::value_type
char_type;
+ public:
+ typedef Iterator base_iterator_type;
+ typedef std::size_t state_type;
+ typedef char_type const* state_name_type;
+ typedef unused_type semantic_actions_type;
+ typedef detail::wrap_action<unused_type, Iterator, static_data
+ , std::size_t> wrap_action_type;
+
typedef std::size_t (*next_token_functor)(std::size_t&,
Iterator const&, Iterator&, Iterator const&, std::size_t&);
-
- typedef unused_type semantic_actions_type;
- typedef unused_type get_state_id_type;
-
- typedef detail::wrap_action<unused_type, iterpair_type, static_data>
- wrap_action_type;
+ typedef char_type const* const (*get_state_name_type)(std::size_t);
// initialize the shared data
template <typename IterData>
- static_data (IterData const& data_, Iterator& first_, Iterator const& last_)
- : next_token(data_.next_), first(first_), last(last_)
- {}
+ static_data (IterData const& data, Iterator& first
+ , Iterator const& last)
+ : first_(first), last_(last)
+ , next_token_(data.next_)
+ , get_state_name_(data.get_state_name_){}
+
+ // The following functions are used by the implementation of the
+ // placeholder '_state'.
+ void set_state_name (char_type const* state) {}
+ char_type const* get_state_name() const
+ {
+ return get_state_name_(0);
+ }
+ std::size_t get_state_id(char_type const* state) const
+ {
+ return 0;
+ }
+
+ // The function get_eoi() is used by the implementation of the
+ // placeholder '_eoi'.
+ Iterator const& get_eoi() const { return last_; }
+
+ // The function less() is used by the implementation of the support
+ // function lex::less(). Its functionality is equivalent to flex'
+ // function yyless(): it returns an iterator positioned to the
+ // nth input character beyond the current start iterator (i.e. by
+ // assigning the return value to the placeholder '_end' it is
+ // possible to return all but the first n characters of the current
+ // token back to the input stream.
+ //
+ // This function does nothing as long as no semantic actions are
+ // used.
+ Iterator const& less(Iterator const& it, int n)
+ {
+ // The following assertion fires most likely because you are
+ // using lexer semantic actions without using the actor_lexer
+ // as the base class for your token definition class.
+ BOOST_ASSERT(false);
+ return it;
+ }
+
+ // The function more() is used by the implemention of the support
+ // function lex::more(). Its functionality is equivalent to flex'
+ // function yymore(): it tells the lexer that the next time it
+ // matches a rule, the corresponding token should be appended onto
+ // the current token value rather than replacing it.
+ //
+ // These functions do nothing as long as no semantic actions are
+ // used.
+ void more()
+ {
+ // The following assertion fires most likely because you are
+ // using lexer semantic actions without using the actor_lexer
+ // as the base class for your token definition class.
+ BOOST_ASSERT(false);
+ }
+ bool adjust_start() { return false; }
+ void revert_adjust_start() {}
+ // The function lookahead() is used by the implementation of the
+ // support function lex::lookahead. It can be used to implement
+ // lookahead for lexer engines not supporting constructs like flex'
+ // a/b (match a, but only when followed by b):
+ //
+ // This function does nothing as long as no semantic actions are
+ // used.
+ bool lookahead(std::size_t id)
+ {
+ // The following assertion fires most likely because you are
+ // using lexer semantic actions without using the actor_lexer
+ // as the base class for your token definition class.
+ BOOST_ASSERT(false);
+ return false;
+ }
+
+ // the functions next, invoke_actions, and get_state are used by
+ // the functor implementation below
+
+ // The function next() tries to match the next token from the
+ // underlying input sequence.
std::size_t next(Iterator& end, std::size_t& unique_id)
{
std::size_t state;
- return next_token(state, first, end, last, unique_id);
+ return next_token_(state, first_, end, last_, unique_id);
}
// nothing to invoke, so this is empty
- bool invoke_actions(std::size_t, std::size_t, std::size_t
- , Iterator const&)
+ BOOST_SCOPED_ENUM(pass_flags) invoke_actions(std::size_t
+ , std::size_t, std::size_t, Iterator const&)
{
- return true; // always accept
+ return pass_flags::pass_normal; // always accept
}
std::size_t get_state() const { return 0; }
- void set_state_name (char const*) {}
+ void set_state(std::size_t state) {}
- next_token_functor next_token;
- Iterator& first;
- Iterator last;
+ Iterator& first_;
+ Iterator last_;
+
+ protected:
+ next_token_functor next_token_;
+ get_state_name_type get_state_name_;
};
///////////////////////////////////////////////////////////////////////
@@ -90,31 +174,30 @@
struct static_data<Iterator, mpl::false_, mpl::true_>
: static_data<Iterator, mpl::false_, mpl::false_>
{
+ protected:
typedef static_data<Iterator, mpl::false_, mpl::false_> base_type;
+ typedef typename base_type::char_type char_type;
+ public:
+ typedef Iterator base_iterator_type;
typedef typename base_type::state_type state_type;
- typedef typename base_type::char_type char_type;
+ typedef typename base_type::state_name_type state_name_type;
typedef typename base_type::semantic_actions_type
semantic_actions_type;
- typedef std::size_t (*get_state_id_type)(char const*);
// initialize the shared data
template <typename IterData>
- static_data (IterData const& data_, Iterator& first_, Iterator const& last_)
- : base_type(data_, first_, last_), state_(0)
- , get_state_id_(data_.get_state_id_)
- {}
+ static_data (IterData const& data, Iterator& first
+ , Iterator const& last)
+ : base_type(data, first, last), state_(0)
+ , num_states_(data.num_states_) {}
- std::size_t next(Iterator& end, std::size_t& unique_id)
- {
- return this->next_token(state_, this->first, end, this->last
- , unique_id);
- }
-
- std::size_t& get_state() { return state_; }
+ // The following functions are used by the implementation of the
+ // placeholder '_state'.
void set_state_name (char_type const* new_state)
{
- std::size_t state_id = get_state_id_(new_state);
+ std::size_t state_id = lexertl::detail::get_state_id(new_state
+ , this->get_state_name_, num_states_);
// if the following assertion fires you've probably been using
// a lexer state name which was not defined in your token
@@ -124,9 +207,33 @@
if (state_id != boost::lexer::npos)
state_ = state_id;
}
+ char_type const* get_state_name() const
+ {
+ return this->get_state_name_(state_);
+ }
+ std::size_t get_state_id(char_type const* state) const
+ {
+ return lexertl::detail::get_state_id(state, get_state_name_
+ , num_states_);
+ }
+ // the functions next() and get_state() are used by the functor
+ // implementation below
+
+ // The function next() tries to match the next token from the
+ // underlying input sequence.
+ std::size_t next(Iterator& end, std::size_t& unique_id)
+ {
+ return this->next_token_(state_, this->first_, end, this->last_
+ , unique_id);
+ }
+
+ std::size_t& get_state() { return state_; }
+ void set_state(std::size_t state) { state_ = state; }
+
+ protected:
std::size_t state_;
- get_state_id_type get_state_id_;
+ std::size_t num_states_;
};
///////////////////////////////////////////////////////////////////////
@@ -135,248 +242,99 @@
struct static_data<Iterator, mpl::true_, HasState>
: static_data<Iterator, mpl::false_, HasState>
{
- typedef static_data<Iterator, mpl::false_, HasState> base_type;
+ public:
+ typedef semantic_actions<Iterator, HasState, static_data>
+ semantic_actions_type;
- typedef iterator_range<Iterator> iterpair_type;
- typedef typename base_type::state_type state_type;
+ protected:
+ typedef static_data<Iterator, mpl::false_, HasState> base_type;
typedef typename base_type::char_type char_type;
+ typedef typename semantic_actions_type::functor_wrapper_type
+ functor_wrapper_type;
- typedef void functor_type(iterpair_type, std::size_t, bool&, static_data&);
- typedef boost::function<functor_type> functor_wrapper_type;
- typedef std::vector<std::vector<functor_wrapper_type> >
- semantic_actions_type;
- typedef typename base_type::get_state_id_type get_state_id_type;
+ public:
+ typedef Iterator base_iterator_type;
+ typedef typename base_type::state_type state_type;
+ typedef typename base_type::state_name_type state_name_type;
typedef detail::wrap_action<functor_wrapper_type
- , iterpair_type, static_data> wrap_action_type;
+ , Iterator, static_data, std::size_t> wrap_action_type;
template <typename IterData>
- static_data (IterData const& data_, Iterator& first_, Iterator const& last_)
- : base_type(data_, first_, last_)
- , actions_(data_.actions_) {}
+ static_data (IterData const& data, Iterator& first
+ , Iterator const& last)
+ : base_type(data, first, last)
+ , actions_(data.actions_), hold_(), has_hold_(false) {}
// invoke attached semantic actions, if defined
- bool invoke_actions(std::size_t state, std::size_t id
- , std::size_t unique_id, Iterator const& end)
+ BOOST_SCOPED_ENUM(pass_flags) invoke_actions(std::size_t state
+ , std::size_t& id, std::size_t unique_id, Iterator& end)
{
- if (state >= actions_.size())
- return true; // no action defined for this state
-
- std::vector<functor_wrapper_type> const& actions = actions_[state];
-
- if (unique_id >= actions.size() || !actions[unique_id])
- return true; // nothing to invoke, continue with 'match'
-
- iterpair_type itp(this->first, end);
- bool match = true;
- actions[unique_id](itp, id, match, *this);
- return match;
+ return actions_.invoke_actions(state, id, unique_id, end, *this);
}
- semantic_actions_type const& actions_;
- };
- }
-
- ///////////////////////////////////////////////////////////////////////////
- //
- // static_functor is a template usable as the functor object for
- // the multi_pass iterator allowing to wrap a lexertl based dfa into a
- // iterator based interface.
- //
- // Iterator: the type of the underlying iterator
- // Token: the type of the tokens produced by this functor
- // this needs to expose a constructor with the following
- // prototype:
- //
- // Token(std::size_t id, std::size_t state,
- // Iterator start, Iterator end)
- //
- // where 'id' is the token id, state is the lexer state,
- // this token has been matched in, and 'first' and 'end'
- // mark the start and the end of the token with respect
- // to the underlying character stream.
- // SupportsActors:
- // this is expected to be a mpl::bool_, if mpl::true_ the
- // static_functor invokes functors which
- // (optionally) have been attached to the token definitions.
- // SupportState:
- // this is expected to be a mpl::bool_, if mpl::true_ the
- // functor supports different lexer states,
- // otherwise no lexer state is supported.
- //
- ///////////////////////////////////////////////////////////////////////////
- template <typename Token
- , typename Iterator = typename Token::iterator_type
- , typename SupportsActors = mpl::false_
- , typename SupportsState = typename Token::has_state>
- class static_functor
- {
- public:
- typedef typename boost::detail::iterator_traits<Iterator>::value_type
- char_type;
-
- private:
- // Needed by compilers not implementing the resolution to DR45. For
- // reference, see
- // http://www.open-std.org/JTC1/SC22/WG21/docs/cwg_defects.html#45.
- template <typename Iterator_, typename HasActors, typename HasState>
- friend struct detail::static_data;
-
- // Helper template allowing to assign a value on exit
- template <typename T>
- struct assign_on_exit
- {
- assign_on_exit(T& dst_, T const& src_)
- : dst(dst_), src(src_)
- {}
- ~assign_on_exit()
- {
- dst = src;
- }
-
- T& dst;
- T const& src;
- };
-
- public:
- static_functor()
-#if defined(__PGI)
- : eof()
-#endif
- {}
-
-#if BOOST_WORKAROUND(BOOST_MSVC, <= 1310)
- // somehow VC7.1 needs this (meaningless) assignment operator
- functor& operator=(functor const& rhs)
- {
- return *this;
- }
-#endif
-
- ///////////////////////////////////////////////////////////////////////
- // interface to the iterator_policies::split_functor_input policy
- typedef Token result_type;
- typedef static_functor unique;
- typedef detail::static_data<Iterator, SupportsActors, SupportsState> shared;
-
- BOOST_SPIRIT_EOF_PREFIX result_type const eof;
-
- ///////////////////////////////////////////////////////////////////////
- typedef Iterator iterator_type;
- typedef typename shared::semantic_actions_type semantic_actions_type;
- typedef typename shared::next_token_functor next_token_functor;
-
- // this is needed to wrap the semantic actions in a proper way
- typedef typename shared::wrap_action_type wrap_action_type;
-
- ///////////////////////////////////////////////////////////////////////
- template <typename MultiPass>
- static result_type& get_next(MultiPass& mp, result_type& result)
- {
- shared& data = mp.shared->ftor;
- if (data.first == data.last)
-#if defined(BOOST_SPIRIT_STATIC_EOF)
- return result = eof;
-#else
- return result = mp.ftor.eof;
-#endif
-
- Iterator end = data.first;
- std::size_t unique_id = boost::lexer::npos;
- std::size_t id = data.next(end, unique_id);
-
- if (boost::lexer::npos == id) { // no match
-#if defined(BOOST_SPIRIT_DEBUG)
- std::string next;
- Iterator it = data.first;
- for (std::size_t i = 0; i < 10 && it != data.last; ++it, ++i)
- next += *it;
-
- std::cerr << "Not matched, in state: " << data.get_state()
- << ", lookahead: >" << next << "<" << std::endl;
-#endif
- return result = result_type(0);
- }
- else if (0 == id) { // EOF reached
-#if defined(BOOST_SPIRIT_STATIC_EOF)
- return result = eof;
-#else
- return result = mp.ftor.eof;
-#endif
+ // The function less() is used by the implementation of the support
+ // function lex::less(). Its functionality is equivalent to flex'
+ // function yyless(): it returns an iterator positioned to the
+ // nth input character beyond the current start iterator (i.e. by
+ // assigning the return value to the placeholder '_end' it is
+ // possible to return all but the first n characters of the current
+ // token back to the input stream).
+ Iterator const& less(Iterator& it, int n)
+ {
+ it = this->first_;
+ std::advance(it, n);
+ return it;
}
-#if defined(BOOST_SPIRIT_DEBUG)
+ // The function more() is used by the implemention of the support
+ // function lex::more(). Its functionality is equivalent to flex'
+ // function yymore(): it tells the lexer that the next time it
+ // matches a rule, the corresponding token should be appended onto
+ // the current token value rather than replacing it.
+ void more()
{
- std::string next;
- Iterator it = end;
- for (std::size_t i = 0; i < 10 && it != data.last; ++it, ++i)
- next += *it;
-
- std::cerr << "Matched: " << id << ", in state: "
- << data.get_state() << ", string: >"
- << std::basic_string<char_type>(data.first, end) << "<"
- << ", lookahead: >" << next << "<" << std::endl;
+ hold_ = this->first_;
+ has_hold_ = true;
}
-#endif
- // invoke_actions might change state
- std::size_t state = data.get_state();
- // invoke attached semantic actions, if there are any defined
- if (!data.invoke_actions(state, id, unique_id, end))
+ // The function lookahead() is used by the implementation of the
+ // support function lex::lookahead. It can be used to implement
+ // lookahead for lexer engines not supporting constructs like flex'
+ // a/b (match a, but only when followed by b)
+ bool lookahead(std::size_t id)
{
- // one of the semantic actions signaled no-match
- return result = result_type(0);
+ Iterator end = this->first_;
+ std::size_t unique_id = boost::lexer::npos;
+ return id == next(end, unique_id);
}
- // return matched token, advancing 'data.first' past the matched
- // sequence
- assign_on_exit<Iterator> on_exit(data.first, end);
- return result = result_type(id, state, data.first, end);
- }
-
- // set_state is propagated up to the iterator interface, allowing to
- // manipulate the current lexer state through any of the exposed
- // iterators.
- template <typename MultiPass>
- static std::size_t set_state(MultiPass& mp, std::size_t state_)
- {
- std::size_t oldstate = mp.shared->ftor.state;
- mp.shared->ftor.state = state_;
-
-#if defined(BOOST_SPIRIT_DEBUG)
- std::cerr << "Switching state from: " << oldstate
- << " to: " << state_
- << std::endl;
-#endif
- return oldstate;
- }
+ // The adjust_start() and revert_adjust_start() are helper
+ // functions needed to implement the functionality required for
+ // lex::more(). It is called from the functor body below.
+ bool adjust_start()
+ {
+ if (!has_hold_)
+ return false;
- template <typename MultiPass>
- static std::size_t
- map_state(MultiPass const& mp, char_type const* statename)
- {
- return mp.shared->ftor.rules.state(statename);
- }
+ std::swap(this->first_, hold_);
+ has_hold_ = false;
+ return true;
+ }
+ void revert_adjust_start()
+ {
+ // this will be called only if adjust_start above returned true
+ std::swap(this->first_, hold_);
+ has_hold_ = true;
+ }
- // we don't need this, but it must be there
- template <typename MultiPass>
- static void destroy(MultiPass const&) {}
- };
-
-#if defined(BOOST_SPIRIT_STATIC_EOF)
- ///////////////////////////////////////////////////////////////////////////
- // eof token
- ///////////////////////////////////////////////////////////////////////////
- template <typename Token, typename Iterator, typename SupportsActors
- , typename SupportsState>
- typename static_functor<Token, Iterator, SupportsActors, SupportsState>::result_type const
- static_functor<Token, Iterator, SupportsActors, SupportsState>::eof =
- typename static_functor<Token, Iterator, SupportsActors, SupportsState>::result_type();
-#endif
+ protected:
+ semantic_actions_type const& actions_;
+ Iterator hold_; // iterator needed to support lex::more()
+ bool has_hold_; // 'true' if hold_ is valid
+ };
+ }
}}}}
-#undef BOOST_SPIRIT_EOF_PREFIX
-#undef BOOST_SPIRIT_STATIC_EOF
-
#endif
Modified: trunk/boost/spirit/home/lex/lexer/lexertl/static_lexer.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/lexertl/static_lexer.hpp (original)
+++ trunk/boost/spirit/home/lex/lexer/lexertl/static_lexer.hpp 2009-06-10 23:39:04 EDT (Wed, 10 Jun 2009)
@@ -12,13 +12,13 @@
#include <boost/spirit/home/support/safe_bool.hpp>
#include <boost/spirit/home/lex/lexer/lexertl/token.hpp>
-#include <boost/spirit/home/lex/lexer/lexertl/static_functor.hpp>
+#include <boost/spirit/home/lex/lexer/lexertl/functor.hpp>
+#include <boost/spirit/home/lex/lexer/lexertl/static_functor_data.hpp>
#include <boost/spirit/home/lex/lexer/lexertl/iterator.hpp>
#include <boost/spirit/home/lex/lexer/lexertl/unique_id.hpp>
#if defined(BOOST_SPIRIT_DEBUG)
#include <boost/spirit/home/support/detail/lexer/debug.hpp>
#endif
-#include <boost/algorithm/string/predicate.hpp>
namespace boost { namespace spirit { namespace lex { namespace lexertl
{
@@ -167,9 +167,10 @@
template <typename Token = token<>
, typename LexerTables = static_::lexer
, typename Iterator = typename Token::iterator_type
- , typename Functor = static_functor<Token, Iterator, mpl::false_>
- , typename TokenSet =
- lex::token_set<static_token_set<Token, LexerTables, Iterator> > >
+ , typename Functor
+ = functor<Token, detail::static_data, Iterator, mpl::false_>
+ , typename TokenSet
+ = lex::token_set<static_token_set<Token, LexerTables, Iterator> > >
class static_lexer
{
public:
@@ -196,33 +197,25 @@
{
typename Functor::next_token_functor next_;
typename Functor::semantic_actions_type const& actions_;
- std::size_t (*get_state_id_)(char const*);
+ typename Functor::get_state_name_type get_state_name_;
+ std::size_t num_states_;
};
typedef LexerTables tables_type;
- static std::size_t get_state_id(char const* state)
- {
- for (std::size_t i = 0; i < tables_type::state_count(); ++i)
- {
- if (boost::equals(tables_type::state_name(i), state))
- return i;
- }
- return ~0;
- }
-
public:
// Return the start iterator usable for iterating over the generated
// tokens, the generated function next_token(...) is called to match
// the next token from the input.
template <typename Iterator_>
- iterator_type begin(Iterator_& first, Iterator_ const& last) const
+ iterator_type begin(Iterator_& first, Iterator_ const& last
+ , char_type const* initial_state = 0) const
{
iterator_data_type iterator_data = {
&tables_type::template next<Iterator_>, actions_,
- get_state_id
+ &tables_type::state_name, tables_type::state_count()
};
- return iterator_type(iterator_data, first, last);
+ return iterator_type(iterator_data, first, last, initial_state);
}
// Return the end iterator usable to stop iterating over the generated
@@ -258,7 +251,8 @@
std::size_t add_state(char_type const* state)
{
- return get_state_id(state);
+ return detail::get_state_id(state, &tables_type::state_name
+ , tables_type::state_count());
}
string_type initial_state() const
{
@@ -269,23 +263,8 @@
template <typename F>
void add_action(id_type unique_id, std::size_t state, F act)
{
- // If you get compilation errors below stating value_type not being
- // a member of boost::fusion::unused_type, then you are probably
- // using semantic actions in your token definition without
- // the static_actor_lexer being specified as the base class of your
- // lexer (instead of the static_lexer class).
- typedef typename Functor::semantic_actions_type::value_type
- value_type;
typedef typename Functor::wrap_action_type wrapper_type;
-
- if (actions_.size() <= state)
- actions_.resize(state + 1);
-
- value_type& actions (actions_[state]);
- if (actions.size() <= unique_id)
- actions.resize(unique_id + 1);
-
- actions[unique_id] = wrapper_type::call(act);
+ actions_.add_action(unique_id, state, wrapper_type::call(act));
}
bool init_dfa() const { return true; }
@@ -319,9 +298,10 @@
template <typename Token = token<>
, typename LexerTables = static_::lexer
, typename Iterator = typename Token::iterator_type
- , typename Functor = static_functor<Token, Iterator, mpl::true_>
- , typename TokenSet =
- lex::token_set<static_token_set<Token, LexerTables, Iterator> > >
+ , typename Functor
+ = functor<Token, detail::static_data, Iterator, mpl::true_>
+ , typename TokenSet
+ = lex::token_set<static_token_set<Token, LexerTables, Iterator> > >
class static_actor_lexer
: public static_lexer<Token, LexerTables, Iterator, Functor, TokenSet>
{
Modified: trunk/boost/spirit/home/lex/lexer/lexertl/wrap_action.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/lexertl/wrap_action.hpp (original)
+++ trunk/boost/spirit/home/lex/lexer/lexertl/wrap_action.hpp 2009-06-10 23:39:04 EDT (Wed, 10 Jun 2009)
@@ -14,16 +14,18 @@
#include <boost/spirit/home/phoenix/bind.hpp>
#include <boost/spirit/home/phoenix/scope.hpp>
#include <boost/spirit/home/support/attributes.hpp>
+#include <boost/spirit/home/lex/lexer/pass_flags.hpp>
///////////////////////////////////////////////////////////////////////////////
namespace boost { namespace spirit { namespace lex { namespace lexertl
{
namespace detail
{
- template <typename FunctionType, typename Attribute, typename Context>
+ template <typename FunctionType, typename Iterator, typename Context
+ , typename IdType>
struct wrap_action
{
- // plain functions with 4 arguments and function objects are not
+ // plain functions with 5 arguments and function objects are not
// touched at all
template <typename F>
static FunctionType call(F const& f)
@@ -31,24 +33,19 @@
return f;
}
- // wrap phoenix actor, make sure first argument is a fusion sequence
+ // wrap phoenix actor
struct phoenix_action
{
template <typename F, typename T1, typename T2, typename T3
- , typename T4>
+ , typename T4, typename T5>
struct result { typedef void type; };
template <typename Eval>
- void operator()(phoenix::actor<Eval> const& f
- , Attribute const& attr, std::size_t id, bool& pass
- , Context& ctx) const
+ void operator()(phoenix::actor<Eval> const& f, Iterator& start
+ , Iterator& end, BOOST_SCOPED_ENUM(pass_flags)& pass
+ , IdType& id, Context& ctx) const
{
- typedef typename
- traits::pass_attribute<unused_type, Attribute const>::type
- attribute_type;
-
- attribute_type attr_wrap(attr);
- f (attr_wrap, id, pass, ctx);
+ f (start, end, pass, id, ctx);
}
};
@@ -59,89 +56,145 @@
using phoenix::arg_names::_2;
using phoenix::arg_names::_3;
using phoenix::arg_names::_4;
+ using phoenix::arg_names::_5;
return phoenix::bind(phoenix_action(), phoenix::lambda[f],
- _1, _2, _3, _4);
+ _1, _2, _3, _4, _5);
}
- // semantic actions with 3 arguments
+ // semantic actions with 4 arguments
template <typename F>
- static void arg3_action(F* f, Attribute const& attr
- , std::size_t id, bool& pass, Context&)
+ static void arg4_action(F* f, Iterator& start, Iterator& end
+ , BOOST_SCOPED_ENUM(pass_flags)& pass, IdType& id
+ , Context const&)
{
- f(attr, id, pass);
+ f(start, end, pass, id);
}
- template <typename A0, typename A1, typename A2>
- static FunctionType call(void(*f)(A0, A1, A2))
+ template <typename F>
+ static void arg4_action_bool(F* f, Iterator& start, Iterator& end
+ , BOOST_SCOPED_ENUM(pass_flags)& pass, IdType& id, Context const&)
+ {
+ bool pass_bool = true;
+ f (start, end, pass_bool, id);
+ pass = pass_bool ? pass_fags::pass_normal : pass_fags::pass_fail;
+ }
+
+ template <typename A0, typename A1, typename A2, typename A3>
+ static FunctionType call(void(*f)(A0, A1, A2, A3))
{
- void (*pf)(void(*)(A0, A1, A2), Attribute const&, std::size_t
- , bool&, Context&) = &wrap_action::arg3_action;
+ void (*pf)(void(*)(A0, A1, A2, A3)
+ , Iterator&, Iterator&, BOOST_SCOPED_ENUM(pass_flags)&
+ , IdType&, Context const&) = &wrap_action::arg4_action;
using phoenix::arg_names::_1;
using phoenix::arg_names::_2;
using phoenix::arg_names::_3;
- return phoenix::bind(pf, f, _1, _2, _3);
+ using phoenix::arg_names::_4;
+ return phoenix::bind(pf, f, _1, _2, _3, _4);
}
- // semantic actions with 2 arguments
+ template <typename A0, typename A1, typename A3>
+ static FunctionType call(void(*f)(A0, A1, bool&, A3))
+ {
+ void (*pf)(void(*)(A0, A1, bool&, A3), Iterator&, Iterator&
+ , BOOST_SCOPED_ENUM(pass_flags)&, IdType&, Context const&)
+ = &wrap_action::arg4_action_bool;
+
+ using phoenix::arg_names::_1;
+ using phoenix::arg_names::_2;
+ using phoenix::arg_names::_3;
+ using phoenix::arg_names::_4;
+ return phoenix::bind(pf, f, _1, _2, _3, _4);
+ }
+
+ // semantic actions with 3 arguments
template <typename F>
- static void arg2_action(F* f, Attribute const& attr
- , std::size_t id, bool&, Context&)
+ static void arg3_action(F* f, Iterator& start, Iterator& end
+ , BOOST_SCOPED_ENUM(pass_flags)& pass, IdType
+ , Context const&)
{
- f(attr, id);
+ f(start, end, pass);
+ }
+
+ template <typename F>
+ static void arg3_action_bool(F* f, Iterator& start, Iterator& end
+ , BOOST_SCOPED_ENUM(pass_flags)& pass, IdType, Context const&)
+ {
+ bool pass_bool = true;
+ f (start, end, pass_bool);
+ pass = pass_bool ? pass_flags::pass_normal : pass_fags::pass_fail;
+ }
+
+ template <typename A0, typename A1, typename A2>
+ static FunctionType call(void(*f)(A0, A1, A2))
+ {
+ void (*pf)(void(*)(A0, A1, A2), Iterator&, Iterator&
+ , BOOST_SCOPED_ENUM(pass_flags)&, IdType
+ , Context const&) = &wrap_action::arg3_action;
+
+ using phoenix::arg_names::_1;
+ using phoenix::arg_names::_2;
+ using phoenix::arg_names::_3;
+ return phoenix::bind(pf, f, _1, _2, _3);
}
template <typename A0, typename A1>
- static FunctionType call(void(*f)(A0, A1))
+ static FunctionType call(void(*f)(A0, A1, bool&))
{
- void (*pf)(void(*)(A0, A1), Attribute const&, std::size_t
- , bool&, Context&) = &wrap_action::arg2_action;
+ void (*pf)(void(*)(A0, A1, bool&), Iterator&, Iterator&
+ , BOOST_SCOPED_ENUM(pass_flags)&, IdType, Context const&)
+ = &wrap_action::arg3_action_bool;
using phoenix::arg_names::_1;
using phoenix::arg_names::_2;
- return phoenix::bind(pf, f, _1, _2);
+ using phoenix::arg_names::_3;
+ return phoenix::bind(pf, f, _1, _2, _3);
}
- // semantic actions with 1 argument
+ // semantic actions with 2 arguments
template <typename F>
- static void arg1_action(F* f, Attribute const& attr
- , std::size_t, bool&, Context&)
+ static void arg2_action(F* f, Iterator& start, Iterator& end
+ , bool, IdType, Context const&)
{
- f(attr);
+ f(start, end);
}
- template <typename A0>
- static FunctionType call(void(*f)(A0))
+ template <typename A0, typename A1>
+ static FunctionType call(void(*f)(A0, A1))
{
- void (*pf)(void(*)(A0), Attribute const&, std::size_t
- , bool&, Context&) = &arg1_action;
+ void (*pf)(void(*)(A0, A1), Iterator&, Iterator&, bool
+ , IdType, Context const&) = &wrap_action::arg2_action;
using phoenix::arg_names::_1;
- return phoenix::bind(pf, f, _1);
+ using phoenix::arg_names::_2;
+ return phoenix::bind(pf, f, _1, _2);
}
+ // we assume that either both iterators are to be passed to the
+ // semantic action or none iterator at all (i.e. it's not possible
+ // to have a lexer semantic action function taking one arguments).
+
// semantic actions with 0 argument
template <typename F>
- static void arg0_action(F* f, Attribute const&
- , std::size_t, bool&, Context&)
+ static void arg0_action(F* f, Iterator&, Iterator&
+ , bool, IdType, Context const&)
{
f();
}
static FunctionType call(void(*f)())
{
- void (*pf)(void(*)(), Attribute const&, std::size_t
- , bool&, Context&) = &arg0_action;
+ void (*pf)(void(*)(), Iterator&, Iterator&, bool
+ , IdType, Context const&) = &arg0_action;
- using phoenix::arg_names::_1;
- return phoenix::bind(pf, f, _1);
+ return phoenix::bind(pf, f);
}
};
// specialization allowing to skip wrapping for lexer types not
// supporting semantic actions
- template <typename Attribute, typename Context>
- struct wrap_action<unused_type, Attribute, Context>
+ template <typename Iterator, typename Context, typename Idtype>
+ struct wrap_action<unused_type, Iterator, Context, Idtype>
{
// plain functors are not touched at all
template <typename F>
Added: trunk/boost/spirit/home/lex/lexer/support_functions.hpp
==============================================================================
--- (empty file)
+++ trunk/boost/spirit/home/lex/lexer/support_functions.hpp 2009-06-10 23:39:04 EDT (Wed, 10 Jun 2009)
@@ -0,0 +1,175 @@
+// Copyright (c) 2001-2009 Hartmut Kaiser
+//
+// Distributed under the Boost Software License, Version 1.0. (See accompanying
+// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+
+#if !defined(SPIRIT_LEX_SUPPORT_FUNCTIONS_JUN_08_2009_0211PM)
+#define SPIRIT_LEX_SUPPORT_FUNCTIONS_JUN_08_2009_0211PM
+
+#if defined(_MSC_VER)
+#pragma once
+#endif
+
+#include <boost/spirit/home/phoenix/core/actor.hpp>
+#include <boost/spirit/home/phoenix/core/argument.hpp>
+#include <boost/spirit/home/phoenix/core/compose.hpp>
+#include <boost/spirit/home/phoenix/core/value.hpp>
+#include <boost/spirit/home/phoenix/core/as_actor.hpp>
+#include <boost/spirit/home/support/detail/scoped_enum_emulation.hpp>
+#include <boost/spirit/home/lex/lexer/pass_flags.hpp>
+
+///////////////////////////////////////////////////////////////////////////////
+namespace boost { namespace spirit { namespace lex
+{
+ ///////////////////////////////////////////////////////////////////////////
+ // The function less() is used by the implementation of the support
+ // function lex::less(). Its functionality is equivalent to flex' function
+ // yyless(): it returns an iterator positioned to the nth input character
+ // beyond the current start iterator (i.e. by assigning the return value to
+ // the placeholder '_end' it is possible to return all but the first n
+ // characters of the current token back to the input stream.
+ //
+ // This Phoenix actor is invoked whenever the function lex::less(n) is
+ // used inside a lexer semantic action:
+ //
+ // lex::token_def<> identifier = "[a-zA-Z_][a-zA-Z0-9_]*";
+ // this->self = identifier [ _end = lex::less(4) ];
+ //
+ // The example shows how to limit the length of the matched identifier to
+ // four characters.
+ //
+ // Note: the function lex::less() has no effect if used on it's own, you
+ // need to use the returned result in order to make use of its
+ // functionality.
+ template <typename Actor>
+ struct less_type
+ {
+ typedef mpl::true_ no_nullary;
+
+ template <typename Env>
+ struct result
+ {
+ typedef typename
+ remove_const<
+ typename mpl::at_c<typename Env::args_type, 4>::type
+ >::type
+ context_type;
+ typedef typename context_type::base_iterator_type type;
+ };
+
+ template <typename Env>
+ typename result<Env>::type
+ eval(Env const& env) const
+ {
+ typename result<Env>::type it;
+ return fusion::at_c<4>(env.args()).less(it, actor_());
+ }
+
+ less_type(Actor const& actor)
+ : actor_(actor) {}
+
+ Actor actor_;
+ };
+
+ // The function lex::less() is used to create a Phoenix actor allowing to
+ // implement functionality similar to flex' function yyless().
+ template <typename T>
+ inline phoenix::actor<less_type<typename phoenix::as_actor<T>::type> >
+ less(T const& v)
+ {
+ typedef typename phoenix::as_actor<T>::type actor_type;
+ return less_type<actor_type>(phoenix::as_actor<T>::convert(v));
+ }
+
+ ///////////////////////////////////////////////////////////////////////////
+ // The function more() is used by the implemention of the support function
+ // lex::more(). Its functionality is equivalent to flex' function yymore():
+ // it tells the lexer that the next time it matches a rule, the
+ // corresponding token should be appended onto the current token value
+ // rather than replacing it.
+ //
+ // This Phoenix actor is invoked whenever the function lex::less(n) is
+ // used inside a lexer semantic action:
+ //
+ // lex::token_def<> identifier = "[a-zA-Z_][a-zA-Z0-9_]*";
+ // this->self = identifier [ lex::more() ];
+ //
+ // The example shows how prefix the next matched token with the matched
+ // identifier.
+ struct more_type
+ {
+ typedef mpl::true_ no_nullary;
+
+ template <typename Env>
+ struct result
+ {
+ typedef void type;
+ };
+
+ template <typename Env>
+ void eval(Env const& env) const
+ {
+ fusion::at_c<4>(env.args()).more();
+ }
+ };
+
+ // The function lex::more() is used to create a Phoenix actor allowing to
+ // implement functionality similar to flex' function yymore().
+ inline phoenix::actor<more_type>
+ more()
+ {
+ return more_type();
+ }
+
+ ///////////////////////////////////////////////////////////////////////////
+ template <typename Actor>
+ struct lookahead_type
+ {
+ typedef mpl::true_ no_nullary;
+
+ template <typename Env>
+ struct result
+ {
+ typedef bool type;
+ };
+
+ template <typename Env>
+ bool eval(Env const& env) const
+ {
+ return fusion::at_c<4>(env.args()).lookahead(actor_());
+ }
+
+ lookahead_type(Actor const& actor)
+ : actor_(actor) {}
+
+ Actor actor_;
+ };
+
+ template <typename T>
+ inline phoenix::actor<
+ lookahead_type<typename phoenix::as_actor<T>::type> >
+ lookahead(T const& id)
+ {
+ typedef typename phoenix::as_actor<T>::type actor_type;
+ return lookahead_type<actor_type>(phoenix::as_actor<T>::convert(id));
+ }
+
+ template <typename Attribute, typename Char, typename Idtype>
+ inline phoenix::actor<
+ lookahead_type<typename phoenix::as_actor<Idtype>::type> >
+ lookahead(token_def<Attribute, Char, Idtype> const& tok)
+ {
+ typedef typename phoenix::as_actor<Idtype>::type actor_type;
+ return lookahead_type<actor_type>(
+ phoenix::as_actor<Idtype>::convert(tok.id()));
+ }
+
+ ///////////////////////////////////////////////////////////////////////////
+ inline BOOST_SCOPED_ENUM(pass_flags) ignore()
+ {
+ return pass_flags::pass_ignore;
+ }
+
+}}}
+
+#endif
Modified: trunk/boost/spirit/home/lex/lexer/token_def.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/token_def.hpp (original)
+++ trunk/boost/spirit/home/lex/lexer/token_def.hpp 2009-06-10 23:39:04 EDT (Wed, 10 Jun 2009)
@@ -213,8 +213,8 @@
}
// general accessors
- Idtype id() const { return token_id_; }
- void id(Idtype id) { token_id_ = id; }
+ Idtype const& id() const { return token_id_; }
+ void id(Idtype const& id) { token_id_ = id; }
std::size_t unique_id() const { return unique_id_; }
string_type definition() const
Modified: trunk/boost/spirit/home/lex/tokenize_and_parse.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/tokenize_and_parse.hpp (original)
+++ trunk/boost/spirit/home/lex/tokenize_and_parse.hpp 2009-06-10 23:39:04 EDT (Wed, 10 Jun 2009)
@@ -244,31 +244,37 @@
// f: A functor (callable object) taking a single argument of
// the token type and returning a bool, indicating whether
// the tokenization should be canceled.
+ // initial_state: The name of the state the lexer should start matching.
+ // The default value is zero, causing the lexer to start
+ // in its 'INITIAL' state.
//
///////////////////////////////////////////////////////////////////////////
template <typename Iterator, typename Lexer, typename F>
inline bool
- tokenize(Iterator& first, Iterator last, Lexer const& lex, F f)
+ tokenize(Iterator& first, Iterator last, Lexer const& lex, F f
+ , typename Lexer::char_type const* initial_state = 0)
{
typedef typename Lexer::iterator_type iterator_type;
+ iterator_type iter = lex.begin(first, last, initial_state);
iterator_type end = lex.end();
- for (iterator_type iter = lex.begin(first, last); iter != end; ++iter)
+ for (/**/; iter != end; ++iter)
{
if (!f(*iter))
return false;
}
- return true;
+ return (iter == end) ? true : false;
}
///////////////////////////////////////////////////////////////////////////
template <typename Iterator, typename Lexer>
inline bool
- tokenize(Iterator& first, Iterator last, Lexer const& lex)
+ tokenize(Iterator& first, Iterator last, Lexer const& lex
+ , typename Lexer::char_type const* initial_state = 0)
{
typedef typename Lexer::iterator_type iterator_type;
- iterator_type iter = lex.begin(first, last);
+ iterator_type iter = lex.begin(first, last, initial_state);
iterator_type end = lex.end();
while (iter != end && token_is_valid(*iter))
Modified: trunk/boost/spirit/home/qi/detail/pass_container.hpp
==============================================================================
--- trunk/boost/spirit/home/qi/detail/pass_container.hpp (original)
+++ trunk/boost/spirit/home/qi/detail/pass_container.hpp 2009-06-10 23:39:04 EDT (Wed, 10 Jun 2009)
@@ -56,6 +56,7 @@
struct pass_container
{
typedef typename F::context_type context_type;
+ typedef typename F::iterator_type iterator_type;
pass_container(F const& f, Attr& attr)
: f(f), attr(attr) {}
@@ -92,7 +93,9 @@
bool dispatch_attribute(Component const& component, mpl::true_) const
{
typedef traits::is_container<
- typename traits::attribute_of<Component, context_type>::type
+ typename traits::attribute_of<
+ Component, context_type, iterator_type
+ >::type
> predicate;
return dispatch_attribute_element(component, predicate());
@@ -116,7 +119,9 @@
// of the current element (component). If this is has no attribute
// we shouldn't push an element into the container.
typedef traits::is_not_unused<
- typename traits::attribute_of<Component, context_type>::type
+ typename traits::attribute_of<
+ Component, context_type, iterator_type
+ >::type
> predicate;
return dispatch_attribute(component, predicate());
@@ -137,7 +142,6 @@
bool operator()(Component const& component) const
{
typedef typename traits::result_of::value<Attr>::type lhs;
- typedef typename F::iterator_type iterator_type;
typedef typename F::context_type context_type;
typedef typename traits::attribute_of<
Component, context_type, iterator_type>::type
Modified: trunk/boost/spirit/home/support/container.hpp
==============================================================================
--- trunk/boost/spirit/home/support/container.hpp (original)
+++ trunk/boost/spirit/home/support/container.hpp 2009-06-10 23:39:04 EDT (Wed, 10 Jun 2009)
@@ -66,12 +66,36 @@
///////////////////////////////////////////////////////////////////////////
namespace result_of
{
+ ///////////////////////////////////////////////////////////////////////
template <typename Container>
struct value
{
typedef typename Container::value_type type;
};
+ // this will be instantiated if the optional holds a container
+ template <typename T>
+ struct value<optional<T> > : value<T> {};
+
+ // this will be instantiated if the variant holds a container
+ template <BOOST_VARIANT_ENUM_PARAMS(typename T)>
+ struct value<boost::variant<BOOST_VARIANT_ENUM_PARAMS(T)> >
+ {
+ typedef typename
+ boost::variant<BOOST_VARIANT_ENUM_PARAMS(T)>::types
+ types;
+ typedef typename
+ mpl::find_if<types, is_container<mpl::_1> >::type
+ iter;
+
+ typedef typename value<
+ typename mpl::if_<
+ is_same<iter, typename mpl::end<types>::type>
+ , unused_type, typename mpl::deref<iter>::type
+ >::type
+ >::type type;
+ };
+
template <>
struct value<unused_type>
{
@@ -84,6 +108,7 @@
typedef unused_type type;
};
+ ///////////////////////////////////////////////////////////////////////
template <typename Container>
struct iterator
{
@@ -108,6 +133,7 @@
typedef unused_type const* type;
};
+ ///////////////////////////////////////////////////////////////////////
template <typename T>
struct optional_value
{
Modified: trunk/boost/spirit/home/support/detail/scoped_enum_emulation.hpp
==============================================================================
--- trunk/boost/spirit/home/support/detail/scoped_enum_emulation.hpp (original)
+++ trunk/boost/spirit/home/support/detail/scoped_enum_emulation.hpp 2009-06-10 23:39:04 EDT (Wed, 10 Jun 2009)
@@ -8,10 +8,14 @@
#define BOOST_SPIRIT_SCOPED_ENUM_EMULATION_HPP
#include <boost/version.hpp>
+#include <boost/config.hpp>
#if BOOST_VERSION >= 104000
-#include <boost/detail/scoped_enum_emulation.hpp>
+# include <boost/detail/scoped_enum_emulation.hpp>
#else
+# if !defined(BOOST_NO_SCOPED_ENUMS)
+# define BOOST_NO_SCOPED_ENUMS
+# endif
# define BOOST_SCOPED_ENUM_START(name) struct name { enum enum_t
# define BOOST_SCOPED_ENUM_END };
# define BOOST_SCOPED_ENUM(name) name::enum_t
Modified: trunk/libs/spirit/example/lex/static_lexer/word_count_lexer_tokens.hpp
==============================================================================
--- trunk/libs/spirit/example/lex/static_lexer/word_count_lexer_tokens.hpp (original)
+++ trunk/libs/spirit/example/lex/static_lexer/word_count_lexer_tokens.hpp 2009-06-10 23:39:04 EDT (Wed, 10 Jun 2009)
@@ -8,8 +8,8 @@
#include <boost/spirit/include/phoenix_operator.hpp>
#include <boost/spirit/include/phoenix_statement.hpp>
-#include <boost/spirit/include/phoenix_algorithm.hpp>
#include <boost/spirit/include/phoenix_core.hpp>
+#include <boost/iterator/iterator_traits.hpp>
///////////////////////////////////////////////////////////////////////////////
// Token definition: We use the lexertl based lexer engine as the underlying
@@ -18,6 +18,20 @@
// Note, the token definition type is derived from the 'lexertl_actor_lexer'
// template, which is a necessary to being able to use lexer semantic actions.
///////////////////////////////////////////////////////////////////////////////
+struct distance_func
+{
+ template <typename Iterator1, typename Iterator2>
+ struct result : boost::iterator_difference<Iterator1> {};
+
+ template <typename Iterator1, typename Iterator2>
+ typename result<Iterator1, Iterator2>::type
+ operator()(Iterator1& begin, Iterator2& end) const
+ {
+ return std::distance(begin, end);
+ }
+};
+boost::phoenix::function<distance_func> const distance = distance_func();
+
//[wcl_static_token_definition
template <typename Lexer>
struct word_count_lexer_tokens : boost::spirit::lex::lexer<Lexer>
@@ -28,13 +42,13 @@
, eol("\n")
, any(".")
{
- using boost::spirit::_1;
+ using boost::spirit::lex::_start;
+ using boost::spirit::lex::_end;
using boost::phoenix::ref;
- using boost::phoenix::distance;
// associate tokens with the lexer
this->self
- = word [++ref(w), ref(c) += distance(_1)]
+ = word [++ref(w), ref(c) += distance(_start, _end)]
| eol [++ref(c), ++ref(l)]
| any [++ref(c)]
;
Modified: trunk/libs/spirit/example/lex/strip_comments.cpp
==============================================================================
--- trunk/libs/spirit/example/lex/strip_comments.cpp (original)
+++ trunk/libs/spirit/example/lex/strip_comments.cpp 2009-06-10 23:39:04 EDT (Wed, 10 Jun 2009)
@@ -64,9 +64,9 @@
: strip_comments_tokens::base_type(match_flags::match_default)
{
// define tokens and associate them with the lexer
- cppcomment = "//[^\n]*";
- ccomment = "/\\*";
- endcomment = "\\*/";
+ cppcomment = "\\/\\/[^\n]*"; // '//[^\n]*'
+ ccomment = "\\/\\*"; // '/*'
+ endcomment = "\\*\\/"; // '*/'
// The following tokens are associated with the default lexer state
// (the "INITIAL" state). Specifying 'INITIAL' as a lexer state is
Modified: trunk/libs/spirit/example/lex/strip_comments_lexer.cpp
==============================================================================
--- trunk/libs/spirit/example/lex/strip_comments_lexer.cpp (original)
+++ trunk/libs/spirit/example/lex/strip_comments_lexer.cpp 2009-06-10 23:39:04 EDT (Wed, 10 Jun 2009)
@@ -67,10 +67,12 @@
: os(os_) {}
// This is called by the semantic action handling code during the lexing
- template <typename Range, typename LexerContext>
- void operator()(Range const& r, std::size_t, bool&, LexerContext&) const
+ template <typename Iterator, typename Context>
+ void operator()(Iterator const& b, Iterator const& e
+ , BOOST_SCOPED_ENUM(boost::spirit::lex::pass_flags)&
+ , std::size_t&, Context&) const
{
- os << r;
+ os << std::string(b, e);
}
std::basic_ostream<Char, Traits>& os;
@@ -92,8 +94,10 @@
: state(state_) {}
// This is called by the semantic action handling code during the lexing
- template <typename Range, typename LexerContext>
- void operator()(Range const&, std::size_t, bool&, LexerContext& ctx) const
+ template <typename Iterator, typename Context>
+ void operator()(Iterator const&, Iterator const&
+ , BOOST_SCOPED_ENUM(boost::spirit::lex::pass_flags)&
+ , std::size_t&, Context& ctx) const
{
ctx.set_state_name(state.c_str());
}
@@ -109,9 +113,9 @@
: strip_comments_tokens::base_type(match_flags::match_default)
{
// define tokens and associate them with the lexer
- cppcomment = "//[^\n]*";
- ccomment = "/\\*";
- endcomment = "\\*/";
+ cppcomment = "\\/\\/[^\n]*"; // '//[^\n]*'
+ ccomment = "\\/\\*"; // '/*'
+ endcomment = "\\*\\/"; // '*/'
any = ".";
eol = "\n";
@@ -128,8 +132,8 @@
// The following tokens are associated with the lexer state 'COMMENT'.
this->self("COMMENT")
= endcomment [ set_lexer_state("INITIAL") ]
- | eol
- | any
+ | "\n"
+ | "."
;
}
Modified: trunk/libs/spirit/example/lex/word_count_lexer.cpp
==============================================================================
--- trunk/libs/spirit/example/lex/word_count_lexer.cpp (original)
+++ trunk/libs/spirit/example/lex/word_count_lexer.cpp 2009-06-10 23:39:04 EDT (Wed, 10 Jun 2009)
@@ -58,6 +58,20 @@
// Note, the token definition type is derived from the 'lexertl_actor_lexer'
// template, which is a necessary to being able to use lexer semantic actions.
///////////////////////////////////////////////////////////////////////////////
+struct distance_func
+{
+ template <typename Iterator1, typename Iterator2>
+ struct result : boost::iterator_difference<Iterator1> {};
+
+ template <typename Iterator1, typename Iterator2>
+ typename result<Iterator1, Iterator2>::type
+ operator()(Iterator1& begin, Iterator2& end) const
+ {
+ return std::distance(begin, end);
+ }
+};
+boost::phoenix::function<distance_func> const distance = distance_func();
+
//[wcl_token_definition
template <typename Lexer>
struct word_count_tokens : lexer<Lexer>
@@ -68,12 +82,13 @@
, eol("\n")
, any(".")
{
+ using boost::spirit::lex::_start;
+ using boost::spirit::lex::_end;
using boost::phoenix::ref;
- using boost::phoenix::distance;
// associate tokens with the lexer
this->self
- = word [++ref(w), ref(c) += distance(_1)]
+ = word [++ref(w), ref(c) += distance(_start, _end)]
| eol [++ref(c), ++ref(l)]
| any [++ref(c)]
;
Modified: trunk/libs/spirit/test/CMakeLists.txt
==============================================================================
--- trunk/libs/spirit/test/CMakeLists.txt (original)
+++ trunk/libs/spirit/test/CMakeLists.txt 2009-06-10 23:39:04 EDT (Wed, 10 Jun 2009)
@@ -102,6 +102,7 @@
boost_test_run(lex_lexertl4 lex/lexertl4.cpp COMPILE_FLAGS ${test_compile_flags})
boost_test_run(lex_lexertl5 lex/lexertl5.cpp COMPILE_FLAGS ${test_compile_flags})
boost_test_run(lex_state_switcher_test lex/state_switcher_test.cpp COMPILE_FLAGS ${test_compile_flags})
+boost_test_run(lex_lexer_state_switcher_test lex/lexer_state_switcher_test.cpp COMPILE_FLAGS ${test_compile_flags})
boost_test_run(lex_regression001 lex/regression001.cpp COMPILE_FLAGS ${test_compile_flags})
boost_test_run(lex_regression002 lex/regression002.cpp COMPILE_FLAGS ${test_compile_flags})
Modified: trunk/libs/spirit/test/Jamfile
==============================================================================
--- trunk/libs/spirit/test/Jamfile (original)
+++ trunk/libs/spirit/test/Jamfile 2009-06-10 23:39:04 EDT (Wed, 10 Jun 2009)
@@ -108,6 +108,7 @@
[ run lex/lexertl4.cpp : : : : ]
[ run lex/lexertl5.cpp : : : : ]
[ run lex/state_switcher_test.cpp : : : : ]
+ [ run lex/lexer_state_switcher_test.cpp : : : : ]
[ run lex/regression001.cpp : : : : lex_regression001 ]
[ run lex/regression002.cpp : : : : lex_regression002 ]
Added: trunk/libs/spirit/test/lex/lexer_state_switcher.cpp
==============================================================================
--- (empty file)
+++ trunk/libs/spirit/test/lex/lexer_state_switcher.cpp 2009-06-10 23:39:04 EDT (Wed, 10 Jun 2009)
@@ -0,0 +1,72 @@
+// Copyright (c) 2001-2009 Hartmut Kaiser
+//
+// Distributed under the Boost Software License, Version 1.0. (See accompanying
+// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+
+#include <boost/mpl/print.hpp>
+#include <boost/detail/lightweight_test.hpp>
+#include <boost/spirit/include/phoenix_object.hpp>
+#include <boost/spirit/include/phoenix_operator.hpp>
+#include <boost/spirit/include/phoenix_statement.hpp>
+#include <boost/spirit/include/lex_lexertl.hpp>
+
+#include "test_parser.hpp"
+
+///////////////////////////////////////////////////////////////////////////////
+// Token definition
+///////////////////////////////////////////////////////////////////////////////
+template <typename Lexer>
+struct switch_state_tokens : boost::spirit::lex::lexer<Lexer>
+{
+ // define tokens and associate them with the lexer
+ switch_state_tokens()
+ {
+ using boost::phoenix::ref;
+ using boost::phoenix::val;
+ using boost::spirit::_state;
+
+ identifier = "[a-zA-Z_][a-zA-Z0-9_]*";
+ this->self = identifier [ ref(state_) = _state ];
+
+ integer = "[0-9]+";
+ this->self("INT") = integer [ _state = val("INITIAL") ];
+ }
+
+ std::string state_;
+ boost::spirit::lex::token_def<> identifier, integer;
+};
+
+///////////////////////////////////////////////////////////////////////////////
+int main()
+{
+ using namespace boost::spirit;
+ using namespace boost::spirit::qi;
+ using namespace boost::spirit::lex;
+ using namespace spirit_test;
+
+ typedef std::string::iterator base_iterator_type;
+ typedef boost::spirit::lex::lexertl::token<base_iterator_type> token_type;
+ typedef boost::spirit::lex::lexertl::actor_lexer<token_type> lexer_type;
+
+ {
+ switch_state_tokens<lexer_type> lex;
+
+ {
+ // verify whether using _state as an rvalue works
+ std::string input("abc123");
+ base_iterator_type first = input.begin();
+ BOOST_TEST(boost::spirit::lex::tokenize(first, input.end(), lex) &&
+ lex.state_ == "INITIAL");
+ }
+ {
+ // verify whether using _state as an lvalue works
+ std::string input("123abc123");
+ base_iterator_type first = input.begin();
+ BOOST_TEST(boost::spirit::lex::tokenize(first, input.end(), lex, "INT") &&
+ lex.state_ == "INITIAL");
+ }
+ }
+
+ return boost::report_errors();
+}
+
Modified: trunk/libs/spirit/test/lex/matlib.h
==============================================================================
--- trunk/libs/spirit/test/lex/matlib.h (original)
+++ trunk/libs/spirit/test/lex/matlib.h 2009-06-10 23:39:04 EDT (Wed, 10 Jun 2009)
@@ -14,8 +14,10 @@
{
std::string state;
set_lexer_state(const std::string &a):state(a){}
- template <class Range,class Context>
- void operator () (Range const &,std::size_t,bool &,Context &ctx) const
+ template <class Iterator,class Context>
+ void operator () (Iterator const&, Iterator const&
+ , BOOST_SCOPED_ENUM(boost::spirit::lex::pass_flags)&, std::size_t
+ , Context &ctx) const
{
ctx.set_state_name(state.c_str());
}
@@ -25,10 +27,12 @@
{
std::vector<double> &out;
store_double(std::vector<double> &a):out(a){}
- template <class Range,class LexerContext>
- void operator () (Range const & r,std::size_t,bool &,LexerContext &)const
+ template <class Iterator,class LexerContext>
+ void operator () (Iterator const& start, Iterator const& end
+ , BOOST_SCOPED_ENUM(boost::spirit::lex::pass_flags)&, std::size_t
+ , LexerContext &ctx) const
{
- std::string work(r.begin(),r.end());
+ std::string work(start, end);
out.push_back(std::atof(work.c_str()));
}
};
@@ -40,8 +44,10 @@
add_row(std::vector<std::vector<double> > &a,std::vector<double> &b)
:matrix(a),row(b) {}
- template <class Range,class Context>
- void operator () (Range const &,std::size_t ,bool &,Context &ctx) const
+ template <class Iterator,class Context>
+ void operator () (Iterator const&, Iterator const&
+ , BOOST_SCOPED_ENUM(boost::spirit::lex::pass_flags)&, std::size_t
+ , Context &ctx) const
{
matrix.push_back(std::vector<double>());
matrix.back().swap(row);