Subject: [Boost-commit] svn:boost r66795 - in trunk/boost/spirit/home: lex lex/detail lex/lexer lex/lexer/lexertl support/iterators/detail
From: hartmut.kaiser_at_[hidden]
Date: 2010-11-27 11:04:19
Author: hkaiser
Date: 2010-11-27 11:04:15 EST (Sat, 27 Nov 2010)
New Revision: 66795
URL: http://svn.boost.org/trac/boost/changeset/66795
Log:
Spirit: applying patch improving assert messages, fixing a couple of lexer problems caused by recent changes, added target state to token definition syntax, allowing semantic actions to be attached to lex::char_() and lex::string()
Text files modified:
trunk/boost/spirit/home/lex/detail/sequence_function.hpp | 8 +++++---
trunk/boost/spirit/home/lex/lexer/action.hpp | 5 +++--
trunk/boost/spirit/home/lex/lexer/char_token_def.hpp | 26 +++++++++++++++++++++++---
trunk/boost/spirit/home/lex/lexer/lexer.hpp | 25 ++++++++++++++++++-------
trunk/boost/spirit/home/lex/lexer/lexertl/functor.hpp | 31 +++++++++++++++++++++++--------
trunk/boost/spirit/home/lex/lexer/lexertl/functor_data.hpp | 12 +++++++++---
trunk/boost/spirit/home/lex/lexer/lexertl/lexer.hpp | 18 ++++++++++++++----
trunk/boost/spirit/home/lex/lexer/lexertl/static_functor_data.hpp | 12 +++++++++---
trunk/boost/spirit/home/lex/lexer/lexertl/static_lexer.hpp | 5 +++--
trunk/boost/spirit/home/lex/lexer/lexertl/token.hpp | 21 ++++++++++++++++++++-
trunk/boost/spirit/home/lex/lexer/sequence.hpp | 7 +++++--
trunk/boost/spirit/home/lex/lexer/string_token_def.hpp | 33 ++++++++++++++++++++++++++++-----
trunk/boost/spirit/home/lex/lexer/token_def.hpp | 17 ++++++++++++-----
trunk/boost/spirit/home/lex/lexer_type.hpp | 10 ++++++----
trunk/boost/spirit/home/lex/reference.hpp | 5 +++--
trunk/boost/spirit/home/support/iterators/detail/split_functor_input_policy.hpp | 4 ++--
16 files changed, 183 insertions(+), 56 deletions(-)
Modified: trunk/boost/spirit/home/lex/detail/sequence_function.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/detail/sequence_function.hpp (original)
+++ trunk/boost/spirit/home/lex/detail/sequence_function.hpp 2010-11-27 11:04:15 EST (Sat, 27 Nov 2010)
@@ -18,18 +18,20 @@
template <typename LexerDef, typename String>
struct sequence_collect_function
{
- sequence_collect_function(LexerDef& def_, String const& state_)
- : def(def_), state(state_) {}
+ sequence_collect_function(LexerDef& def_, String const& state_
+ , String const& targetstate_)
+ : def(def_), state(state_), targetstate(targetstate_) {}
template <typename Component>
bool operator()(Component const& component) const
{
- component.collect(def, state);
+ component.collect(def, state, targetstate);
return false; // execute for all sequence elements
}
LexerDef& def;
String const& state;
+ String const& targetstate;
private:
// silence MSVC warning C4512: assignment operator could not be generated
Modified: trunk/boost/spirit/home/lex/lexer/action.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/action.hpp (original)
+++ trunk/boost/spirit/home/lex/lexer/action.hpp 2010-11-27 11:04:15 EST (Sat, 27 Nov 2010)
@@ -29,11 +29,12 @@
: subject(subject), f(f) {}
template <typename LexerDef, typename String>
- void collect(LexerDef& lexdef, String const& state) const
+ void collect(LexerDef& lexdef, String const& state
+ , String const& targetstate) const
{
// collect the token definition information for the token_def
// this action is attached to
- subject.collect(lexdef, state);
+ subject.collect(lexdef, state, targetstate);
}
template <typename LexerDef>
Modified: trunk/boost/spirit/home/lex/lexer/char_token_def.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/char_token_def.hpp (original)
+++ trunk/boost/spirit/home/lex/lexer/char_token_def.hpp 2010-11-27 11:04:15 EST (Sat, 27 Nov 2010)
@@ -68,13 +68,31 @@
typedef typename CharEncoding::char_type char_type;
char_token_def(char_type ch)
- : ch(ch), unique_id_(std::size_t(~0)) {}
+ : ch(ch), unique_id_(std::size_t(~0)), token_state_(std::size_t(~0))
+ {}
template <typename LexerDef, typename String>
- void collect(LexerDef& lexdef, String const& state) const
+ void collect(LexerDef& lexdef, String const& state
+ , String const& targetstate) const
{
+ std::size_t state_id = lexdef.add_state(state.c_str());
+
+ // If the following assertion fires you are probably trying to use
+ // a single char_token_def instance in more than one lexer state.
+ // This is not possible. Please create a separate token_def instance
+ // from the same regular expression for each lexer state it needs
+ // to be associated with.
+ BOOST_ASSERT(
+ (std::size_t(~0) == token_state_ || state_id == token_state_) &&
+ "Can't use single char_token_def with more than one lexer state");
+
+ char_type const* target = targetstate.empty() ? 0 : targetstate.c_str();
+ if (target)
+ lexdef.add_state(target);
+
+ token_state_ = state_id;
unique_id_ = lexdef.add_token (state.c_str(), ch
- , static_cast<std::size_t>(ch));
+ , static_cast<std::size_t>(ch), target);
}
template <typename LexerDef>
@@ -82,9 +100,11 @@
std::size_t id() const { return static_cast<std::size_t>(ch); }
std::size_t unique_id() const { return unique_id_; }
+ std::size_t state() const { return token_state_; }
char_type ch;
mutable std::size_t unique_id_;
+ mutable std::size_t token_state_;
};
///////////////////////////////////////////////////////////////////////////
Modified: trunk/boost/spirit/home/lex/lexer/lexer.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/lexer.hpp (original)
+++ trunk/boost/spirit/home/lex/lexer/lexer.hpp 2010-11-27 11:04:15 EST (Sat, 27 Nov 2010)
@@ -121,7 +121,8 @@
{
if (id_type() == token_id)
token_id = static_cast<id_type>(c);
- def.def.add_token (def.state.c_str(), c, token_id);
+ def.def.add_token (def.state.c_str(), c, token_id
+ , def.targetstate.empty() ? 0 : def.targetstate.c_str());
return *this;
}
@@ -135,7 +136,8 @@
{
if (id_type() == token_id)
token_id = def.def.get_next_id();
- def.def.add_token (def.state.c_str(), s, token_id);
+ def.def.add_token (def.state.c_str(), s, token_id
+ , def.targetstate.empty() ? 0 : def.targetstate.c_str());
return *this;
}
@@ -213,7 +215,7 @@
template <typename TokenExpr>
void compile2pass(TokenExpr const& expr)
{
- expr.collect(def, state);
+ expr.collect(def, state, targetstate);
expr.add_actions(def);
}
@@ -225,9 +227,11 @@
compile2pass(compile<lex::domain>(expr));
}
- lexer_def_(LexerDef& def_, string_type const& state_)
+ lexer_def_(LexerDef& def_, string_type const& state_
+ , string_type const& targetstate_ = string_type())
: proto_base_type(terminal_type::make(alias()))
- , add(this_()), add_pattern(this_()), def(def_), state(state_)
+ , add(this_()), add_pattern(this_()), def(def_)
+ , state(state_), targetstate(targetstate_)
{}
// allow to switch states
@@ -235,9 +239,15 @@
{
return lexer_def_(def, state);
}
- lexer_def_ operator()(string_type const& state) const
+ lexer_def_ operator()(char_type const* state
+ , char_type const* targetstate) const
{
- return lexer_def_(def, state);
+ return lexer_def_(def, state, targetstate);
+ }
+ lexer_def_ operator()(string_type const& state
+ , string_type const& targetstate = string_type()) const
+ {
+ return lexer_def_(def, state, targetstate);
}
// allow to assign a token definition expression
@@ -261,6 +271,7 @@
private:
LexerDef& def;
string_type state;
+ string_type targetstate;
private:
// silence MSVC warning C4512: assignment operator could not be generated
Modified: trunk/boost/spirit/home/lex/lexer/lexertl/functor.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/lexertl/functor.hpp (original)
+++ trunk/boost/spirit/home/lex/lexer/lexertl/functor.hpp 2010-11-27 11:04:15 EST (Sat, 27 Nov 2010)
@@ -148,6 +148,9 @@
Iterator end = data.get_first();
std::size_t unique_id = boost::lexer::npos;
bool prev_bol = false;
+
+ // lexer matching might change state
+ std::size_t state = data.get_state();
std::size_t id = data.next(end, unique_id, prev_bol);
if (boost::lexer::npos == id) { // no match
@@ -157,7 +160,7 @@
for (std::size_t i = 0; i < 10 && it != data.get_last(); ++it, ++i)
next += *it;
- std::cerr << "Not matched, in state: " << data.get_state()
+ std::cerr << "Not matched, in state: " << state
<< ", lookahead: >" << next << "<" << std::endl;
#endif
return result = result_type(0);
@@ -178,19 +181,21 @@
next += *it;
std::cerr << "Matched: " << id << ", in state: "
- << data.get_state() << ", string: >"
+ << state << ", string: >"
<< std::basic_string<char_type>(data.get_first(), end) << "<"
<< ", lookahead: >" << next << "<" << std::endl;
+ if (data.get_state() != state) {
+ std::cerr << "Switched to state: "
+ << data.get_state() << std::endl;
+ }
}
#endif
- // invoke_actions might change state, id, data.first_, and/or end
- std::size_t state = data.get_state();
-
// account for a possibly pending lex::more(), i.e. moving
// data.first_ back to the start of the previously matched token.
bool adjusted = data.adjust_start();
- // invoke attached semantic actions, if defined
+ // invoke attached semantic actions, if defined, might change
+ // state, id, data.first_, and/or end
BOOST_SCOPED_ENUM(pass_flags) pass =
data.invoke_actions(state, id, unique_id, end);
@@ -208,16 +213,26 @@
return result = result_type(id, state, data.get_first(), end);
}
else if (pass_flags::pass_fail == pass) {
+#if defined(BOOST_SPIRIT_LEXERTL_DEBUG)
+ std::cerr << "Matching forced to fail" << std::endl;
+#endif
// if the data.first_ got adjusted above, revert this adjustment
if (adjusted)
data.revert_adjust_start();
// one of the semantic actions signaled no-match
data.reset_bol(prev_bol);
- continue; // retry matching
-// return result = result_type(0);
+ if (state != data.get_state())
+ continue; // retry matching if state has changed
+
+ // if the state is unchanged repeating the match wouldn't
+ // move the input forward, causing an infinite loop
+ return result = result_type(0);
}
+#if defined(BOOST_SPIRIT_LEXERTL_DEBUG)
+ std::cerr << "Token ignored, continuing matching" << std::endl;
+#endif
// if this token needs to be ignored, just repeat the matching,
// while starting right after the current match
data.get_first() = end;
Modified: trunk/boost/spirit/home/lex/lexer/lexertl/functor_data.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/lexertl/functor_data.hpp (original)
+++ trunk/boost/spirit/home/lex/lexer/lexertl/functor_data.hpp 2010-11-27 11:04:15 EST (Sat, 27 Nov 2010)
@@ -98,7 +98,9 @@
// The following assertion fires most likely because you are
// using lexer semantic actions without using the actor_lexer
// as the base class for your token definition class.
- BOOST_ASSERT(false);
+ BOOST_ASSERT(false &&
+ "Are you using lexer semantic actions without using the "
+ "actor_lexer base?");
return it;
}
@@ -115,7 +117,9 @@
// The following assertion fires most likely because you are
// using lexer semantic actions without using the actor_lexer
// as the base class for your token definition class.
- BOOST_ASSERT(false);
+ BOOST_ASSERT(false &&
+ "Are you using lexer semantic actions without using the "
+ "actor_lexer base?");
}
bool adjust_start() { return false; }
void revert_adjust_start() {}
@@ -132,7 +136,9 @@
// The following assertion fires most likely because you are
// using lexer semantic actions without using the actor_lexer
// as the base class for your token definition class.
- BOOST_ASSERT(false);
+ BOOST_ASSERT(false &&
+ "Are you using lexer semantic actions without using the "
+ "actor_lexer base?");
return false;
}
Modified: trunk/boost/spirit/home/lex/lexer/lexertl/lexer.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/lexertl/lexer.hpp (original)
+++ trunk/boost/spirit/home/lex/lexer/lexertl/lexer.hpp 2010-11-27 11:04:15 EST (Sat, 27 Nov 2010)
@@ -221,22 +221,32 @@
public:
// interface for token definition management
std::size_t add_token(char_type const* state, char_type tokendef,
- std::size_t token_id)
+ std::size_t token_id, char_type const* targetstate)
{
add_state(state);
initialized_dfa_ = false;
if (state == all_states())
return rules_.add(state, detail::escape(tokendef), token_id, rules_.dot());
- return rules_.add(state, detail::escape(tokendef), token_id, state);
+
+ if (0 == targetstate)
+ targetstate = state;
+ else
+ add_state(targetstate);
+ return rules_.add(state, detail::escape(tokendef), token_id, targetstate);
}
std::size_t add_token(char_type const* state, string_type const& tokendef,
- std::size_t token_id)
+ std::size_t token_id, char_type const* targetstate)
{
add_state(state);
initialized_dfa_ = false;
if (state == all_states())
return rules_.add(state, tokendef, token_id, rules_.dot());
- return rules_.add(state, tokendef, token_id, state);
+
+ if (0 == targetstate)
+ targetstate = state;
+ else
+ add_state(targetstate);
+ return rules_.add(state, tokendef, token_id, targetstate);
}
// interface for pattern definition management
Modified: trunk/boost/spirit/home/lex/lexer/lexertl/static_functor_data.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/lexertl/static_functor_data.hpp (original)
+++ trunk/boost/spirit/home/lex/lexer/lexertl/static_functor_data.hpp 2010-11-27 11:04:15 EST (Sat, 27 Nov 2010)
@@ -117,7 +117,9 @@
// The following assertion fires most likely because you are
// using lexer semantic actions without using the actor_lexer
// as the base class for your token definition class.
- BOOST_ASSERT(false);
+ BOOST_ASSERT(false &&
+ "Are you using lexer semantic actions without using the "
+ "actor_lexer base?");
return it;
}
@@ -134,7 +136,9 @@
// The following assertion fires most likely because you are
// using lexer semantic actions without using the actor_lexer
// as the base class for your token definition class.
- BOOST_ASSERT(false);
+ BOOST_ASSERT(false &&
+ "Are you using lexer semantic actions without using the "
+ "actor_lexer base?");
}
bool adjust_start() { return false; }
void revert_adjust_start() {}
@@ -151,7 +155,9 @@
// The following assertion fires most likely because you are
// using lexer semantic actions without using the actor_lexer
// as the base class for your token definition class.
- BOOST_ASSERT(false);
+ BOOST_ASSERT(false &&
+ "Are you using lexer semantic actions without using the "
+ "actor_lexer base?");
return false;
}
Modified: trunk/boost/spirit/home/lex/lexer/lexertl/static_lexer.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/lexertl/static_lexer.hpp (original)
+++ trunk/boost/spirit/home/lex/lexer/lexertl/static_lexer.hpp 2010-11-27 11:04:15 EST (Sat, 27 Nov 2010)
@@ -195,12 +195,13 @@
public:
// interface for token definition management
- std::size_t add_token (char_type const*, char_type, std::size_t)
+ std::size_t add_token (char_type const*, char_type, std::size_t
+ , char_type const*)
{
return unique_id_++;
}
std::size_t add_token (char_type const*, string_type const&
- , std::size_t)
+ , std::size_t, char_type const*)
{
return unique_id_++;
}
Modified: trunk/boost/spirit/home/lex/lexer/lexertl/token.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/lexertl/token.hpp (original)
+++ trunk/boost/spirit/home/lex/lexer/lexertl/token.hpp 2010-11-27 11:04:15 EST (Sat, 27 Nov 2010)
@@ -110,7 +110,7 @@
template <typename Iterator = char const*
, typename AttributeTypes = mpl::vector0<>
, typename HasState = mpl::true_
- , typename Idtyep = std::size_t>
+ , typename Idtype = std::size_t>
struct token;
///////////////////////////////////////////////////////////////////////////
@@ -170,6 +170,11 @@
// constructed iterator_range
token& operator= (token const& rhs)
{
+ if (this != &rhs)
+ {
+ id_ = rhs.id_;
+ matched_ = rhs.matched_;
+ }
return *this;
}
#endif
@@ -246,6 +251,20 @@
std::size_t state() const { return state_; }
+#if defined(BOOST_SPIRIT_DEBUG) && BOOST_WORKAROUND(BOOST_MSVC, == 1600)
+ // workaround for MSVC10 which has problems copying a default
+ // constructed iterator_range
+ token& operator= (token const& rhs)
+ {
+ if (this != &rhs)
+ {
+ this->base_type::operator=(static_cast<base_type const&>(rhs));
+ state_ = rhs.state_;
+ }
+ return *this;
+ }
+#endif
+
protected:
std::size_t state_; // lexer state this token was matched in
};
Modified: trunk/boost/spirit/home/lex/lexer/sequence.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/sequence.hpp (original)
+++ trunk/boost/spirit/home/lex/lexer/sequence.hpp 2010-11-27 11:04:15 EST (Sat, 27 Nov 2010)
@@ -40,9 +40,12 @@
: elements(elements) {}
template <typename LexerDef, typename String>
- void collect(LexerDef& lexdef, String const& state) const
+ void collect(LexerDef& lexdef, String const& state
+ , String const& targetstate) const
{
- detail::sequence_collect_function<LexerDef, String> f (lexdef, state);
+ typedef detail::sequence_collect_function<LexerDef, String>
+ collect_function_type;
+ collect_function_type f (lexdef, state, targetstate);
fusion::any(elements, f);
}
Modified: trunk/boost/spirit/home/lex/lexer/string_token_def.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/string_token_def.hpp (original)
+++ trunk/boost/spirit/home/lex/lexer/string_token_def.hpp 2010-11-27 11:04:15 EST (Sat, 27 Nov 2010)
@@ -62,15 +62,36 @@
typedef std::basic_string<char_type> string_type;
string_token_def(typename add_reference<String>::type str)
- : str_(str), id_(std::size_t(~0)) {}
-
- template <typename LexerDef, typename State>
- void collect(LexerDef& lexdef, State const& state) const
+ : str_(str), id_(std::size_t(~0)), unique_id_(std::size_t(~0))
+ , token_state_(std::size_t(~0))
+ {}
+
+ template <typename LexerDef, typename String>
+ void collect(LexerDef& lexdef, String const& state
+ , String const& targetstate) const
{
+ std::size_t state_id = lexdef.add_state(state.c_str());
+
+ // If the following assertion fires you are probably trying to use
+ // a single string_token_def instance in more than one lexer state.
+ // This is not possible. Please create a separate token_def instance
+ // from the same regular expression for each lexer state it needs
+ // to be associated with.
+ BOOST_ASSERT(
+ (std::size_t(~0) == token_state_ || state_id == token_state_) &&
+ "Can't use single string_token_def with more than one lexer state");
+
+ char_type const* target = targetstate.empty() ? 0 : targetstate.c_str();
+ if (target)
+ lexdef.add_state(target);
+
+ token_state_ = state_id;
+
typedef typename LexerDef::id_type id_type;
if (std::size_t(~0) == id_)
id_ = lexdef.get_next_id();
- unique_id_ = lexdef.add_token (state.c_str(), str_, id_);
+
+ unique_id_ = lexdef.add_token (state.c_str(), str_, id_, target);
}
template <typename LexerDef>
@@ -78,10 +99,12 @@
std::size_t id() const { return id_; }
std::size_t unique_id() const { return unique_id_; }
+ std::size_t state() const { return token_state_; }
string_type str_;
mutable std::size_t id_;
mutable std::size_t unique_id_;
+ mutable std::size_t token_state_;
};
///////////////////////////////////////////////////////////////////////////
Modified: trunk/boost/spirit/home/lex/lexer/token_def.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/token_def.hpp (original)
+++ trunk/boost/spirit/home/lex/lexer/token_def.hpp 2010-11-27 11:04:15 EST (Sat, 27 Nov 2010)
@@ -61,7 +61,7 @@
static std::size_t const all_states_id = static_cast<std::size_t>(-2);
public:
- // Qi interface: metafunction calculating parser return type
+ // Qi interface: meta-function calculating parser return type
template <typename Context, typename Iterator>
struct attribute
{
@@ -123,7 +123,8 @@
// Lex interface: collect token definitions and put it into the
// provided lexer def
template <typename LexerDef, typename String>
- void collect(LexerDef& lexdef, String const& state) const
+ void collect(LexerDef& lexdef, String const& state
+ , String const& targetstate) const
{
std::size_t state_id = lexdef.add_state(state.c_str());
@@ -132,7 +133,13 @@
// is not possible. Please create a separate token_def instance
// from the same regular expression for each lexer state it needs
// to be associated with.
- BOOST_ASSERT(std::size_t(~0) == token_state_ || state_id == token_state_);
+ BOOST_ASSERT(
+ (std::size_t(~0) == token_state_ || state_id == token_state_) &&
+ "Can't use single token_def with more than one lexer state");
+
+ char_type const* target = targetstate.empty() ? 0 : targetstate.c_str();
+ if (target)
+ lexdef.add_state(target);
token_state_ = state_id;
if (0 == token_id_)
@@ -140,11 +147,11 @@
if (0 == def_.which()) {
unique_id_ = lexdef.add_token(state.c_str()
- , get<string_type>(def_), token_id_);
+ , get<string_type>(def_), token_id_, target);
}
else {
unique_id_ = lexdef.add_token(state.c_str()
- , get<char_type>(def_), token_id_);
+ , get<char_type>(def_), token_id_, target);
}
}
Modified: trunk/boost/spirit/home/lex/lexer_type.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer_type.hpp (original)
+++ trunk/boost/spirit/home/lex/lexer_type.hpp 2010-11-27 11:04:15 EST (Sat, 27 Nov 2010)
@@ -23,11 +23,13 @@
typedef Derived derived_type;
typedef lex::domain domain;
- // Requirement: l.collect(def, state) -> void
+ // Requirement: l.collect(def, state, targetstate) -> void
//
- // l: a lexer component
- // def: token definition container
- // state: lexer state this token definition needs to be added to
+ // l: a lexer component
+ // def: token definition container
+ // state: lexer state this token definition needs to be added to
+ // targetstate: an optional lexer state the lexer should be switched
+ // into after matching this token
Derived const& derived() const
{
Modified: trunk/boost/spirit/home/lex/reference.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/reference.hpp (original)
+++ trunk/boost/spirit/home/lex/reference.hpp 2010-11-27 11:04:15 EST (Sat, 27 Nov 2010)
@@ -35,9 +35,10 @@
: qi::reference<Subject>(subject) {}
template <typename LexerDef, typename String>
- void collect(LexerDef& lexdef, String const& state) const
+ void collect(LexerDef& lexdef, String const& state
+ , String const& targetstate) const
{
- this->ref.get().collect(lexdef, state);
+ this->ref.get().collect(lexdef, state, targetstate);
}
template <typename LexerDef>
Modified: trunk/boost/spirit/home/support/iterators/detail/split_functor_input_policy.hpp
==============================================================================
--- trunk/boost/spirit/home/support/iterators/detail/split_functor_input_policy.hpp (original)
+++ trunk/boost/spirit/home/support/iterators/detail/split_functor_input_policy.hpp 2010-11-27 11:04:15 EST (Sat, 27 Nov 2010)
@@ -80,7 +80,7 @@
{
value_type& curtok = mp.shared()->curtok;
using namespace split_functor_input_is_valid_test_;
- while (!token_is_valid(curtok))
+ if (!token_is_valid(curtok))
functor_type::get_next(mp, curtok);
return curtok;
}
@@ -143,7 +143,7 @@
{
value_type& curtok = mp.shared()->curtok;
using namespace split_functor_input_is_valid_test_;
- while (!token_is_valid(curtok))
+ if (!token_is_valid(curtok))
functor_type::get_next(mp, curtok);
return curtok;
}