Subject: [Boost-commit] svn:boost r58577 - in branches/release/boost/spirit: . home home/karma home/karma/auto home/karma/detail home/lex/lexer/lexertl home/qi home/qi/auto home/qi/detail home/qi/directive home/qi/nonterminal home/qi/string home/support home/support/auto home/support/detail/lexer home/support/detail/lexer/parser home/support/detail/lexer/parser/tree home/support/nonterminal include repository/home/karma/string
From: hartmut.kaiser_at_[hidden]
Date: 2009-12-29 21:47:02
Author: hkaiser
Date: 2009-12-29 21:46:59 EST (Tue, 29 Dec 2009)
New Revision: 58577
URL: http://svn.boost.org/trac/boost/changeset/58577
Log:
Spirit: merging more changes from trunk
Added:
branches/release/boost/spirit/home/karma/phoenix_attributes.hpp (contents, props changed)
branches/release/boost/spirit/home/support/detail/lexer/generate_re2c.hpp (contents, props changed)
branches/release/boost/spirit/include/karma_phoenix_attributes.hpp (contents, props changed)
Removed:
branches/release/boost/spirit/repository/home/karma/string/
Properties modified:
branches/release/boost/spirit/ (props changed)
branches/release/boost/spirit/home/ (props changed)
branches/release/boost/spirit/home/karma/ (props changed)
branches/release/boost/spirit/home/support/attributes.hpp (contents, props changed)
Text files modified:
branches/release/boost/spirit/home/karma/auto/create_generator.hpp | 13
branches/release/boost/spirit/home/karma/detail/generate.hpp | 8
branches/release/boost/spirit/home/karma/detail/generate_auto.hpp | 4
branches/release/boost/spirit/home/karma/detail/pass_container.hpp | 9
branches/release/boost/spirit/home/karma/generate.hpp | 4
branches/release/boost/spirit/home/lex/lexer/lexertl/functor_data.hpp | 13
branches/release/boost/spirit/home/lex/lexer/lexertl/generate_static.hpp | 705 +++++++++++++++++++++++++++++----------
branches/release/boost/spirit/home/lex/lexer/lexertl/iterator_tokenizer.hpp | 56 ++
branches/release/boost/spirit/home/lex/lexer/lexertl/lexer.hpp | 17
branches/release/boost/spirit/home/lex/lexer/lexertl/static_functor_data.hpp | 15
branches/release/boost/spirit/home/lex/lexer/lexertl/static_lexer.hpp | 13
branches/release/boost/spirit/home/qi/auto/create_parser.hpp | 14
branches/release/boost/spirit/home/qi/auto/meta_create.hpp | 2
branches/release/boost/spirit/home/qi/detail/parse.hpp | 8
branches/release/boost/spirit/home/qi/detail/parse_auto.hpp | 4
branches/release/boost/spirit/home/qi/detail/pass_container.hpp | 13
branches/release/boost/spirit/home/qi/directive/repeat.hpp | 81 +++-
branches/release/boost/spirit/home/qi/nonterminal/error_handler.hpp | 4
branches/release/boost/spirit/home/qi/parse.hpp | 4
branches/release/boost/spirit/home/qi/string/symbols.hpp | 18
branches/release/boost/spirit/home/support/attributes.hpp | 2
branches/release/boost/spirit/home/support/attributes_fwd.hpp | 3
branches/release/boost/spirit/home/support/auto/meta_create.hpp | 18
branches/release/boost/spirit/home/support/detail/lexer/debug.hpp | 70 ---
branches/release/boost/spirit/home/support/detail/lexer/generate_cpp.hpp | 241 +++++++-----
branches/release/boost/spirit/home/support/detail/lexer/generator.hpp | 41 +-
branches/release/boost/spirit/home/support/detail/lexer/input.hpp | 19
branches/release/boost/spirit/home/support/detail/lexer/parser/parser.hpp | 20
branches/release/boost/spirit/home/support/detail/lexer/parser/tree/iteration_node.hpp | 2
branches/release/boost/spirit/home/support/detail/lexer/parser/tree/leaf_node.hpp | 2
branches/release/boost/spirit/home/support/detail/lexer/parser/tree/selection_node.hpp | 2
branches/release/boost/spirit/home/support/detail/lexer/parser/tree/sequence_node.hpp | 2
branches/release/boost/spirit/home/support/detail/lexer/state_machine.hpp | 2
branches/release/boost/spirit/home/support/detail/lexer/string_token.hpp | 76 ++++
branches/release/boost/spirit/home/support/meta_compiler.hpp | 2
branches/release/boost/spirit/home/support/nonterminal/expand_arg.hpp | 8
branches/release/boost/spirit/include/lex_generate_static_lexertl.hpp | 6
37 files changed, 1024 insertions(+), 497 deletions(-)
Modified: branches/release/boost/spirit/home/karma/auto/create_generator.hpp
==============================================================================
--- branches/release/boost/spirit/home/karma/auto/create_generator.hpp (original)
+++ branches/release/boost/spirit/home/karma/auto/create_generator.hpp 2009-12-29 21:46:59 EST (Tue, 29 Dec 2009)
@@ -12,6 +12,7 @@
#include <boost/spirit/home/karma/auto/meta_create.hpp>
+///////////////////////////////////////////////////////////////////////////////
namespace boost { namespace spirit { namespace result_of
{
///////////////////////////////////////////////////////////////////////////
@@ -20,9 +21,9 @@
: spirit::traits::meta_create<karma::domain, T> {};
}}}
+///////////////////////////////////////////////////////////////////////////////
namespace boost { namespace spirit { namespace karma
{
- ///////////////////////////////////////////////////////////////////////////
// Main API function for generator creation from data type
template <typename T>
typename result_of::create_generator<T>::type
@@ -32,4 +33,14 @@
}
}}}
+///////////////////////////////////////////////////////////////////////////////
+namespace boost { namespace spirit { namespace traits
+{
+ // Meta function returning true if create_generator does return a valid
+ // generator for the given type T.
+ template <typename T>
+ struct create_generator_exists
+ : meta_create_exists<karma::domain, T> {};
+}}}
+
#endif
Modified: branches/release/boost/spirit/home/karma/detail/generate.hpp
==============================================================================
--- branches/release/boost/spirit/home/karma/detail/generate.hpp (original)
+++ branches/release/boost/spirit/home/karma/detail/generate.hpp 2009-12-29 21:46:59 EST (Tue, 29 Dec 2009)
@@ -22,7 +22,7 @@
{
///////////////////////////////////////////////////////////////////////////
template <typename Expr, typename Enable = void>
- struct generate
+ struct generate_impl
{
// Report invalid expression error as early as possible.
// If you got an error_invalid_expression error message here,
@@ -33,7 +33,7 @@
};
template <typename Expr>
- struct generate<Expr
+ struct generate_impl<Expr
, typename enable_if<traits::matches<karma::domain, Expr> >::type>
{
template <typename OutputIterator>
@@ -64,7 +64,7 @@
///////////////////////////////////////////////////////////////////////////
template <typename Expr, typename Enable = void>
- struct generate_delimited
+ struct generate_delimited_impl
{
// Report invalid expression error as early as possible.
// If you got an error_invalid_expression error message here,
@@ -75,7 +75,7 @@
};
template <typename Expr>
- struct generate_delimited<Expr
+ struct generate_delimited_impl<Expr
, typename enable_if<traits::matches<karma::domain, Expr> >::type>
{
template <typename OutputIterator, typename Delimiter>
Modified: branches/release/boost/spirit/home/karma/detail/generate_auto.hpp
==============================================================================
--- branches/release/boost/spirit/home/karma/detail/generate_auto.hpp (original)
+++ branches/release/boost/spirit/home/karma/detail/generate_auto.hpp 2009-12-29 21:46:59 EST (Tue, 29 Dec 2009)
@@ -18,7 +18,7 @@
{
///////////////////////////////////////////////////////////////////////////
template <typename Expr>
- struct generate<Expr
+ struct generate_impl<Expr
, typename enable_if<traits::meta_create_exists<karma::domain, Expr> >::type>
{
template <typename OutputIterator>
@@ -32,7 +32,7 @@
///////////////////////////////////////////////////////////////////////////
template <typename Expr>
- struct generate_delimited<Expr
+ struct generate_delimited_impl<Expr
, typename enable_if<traits::meta_create_exists<karma::domain, Expr> >::type>
{
template <typename OutputIterator, typename Delimiter>
Modified: branches/release/boost/spirit/home/karma/detail/pass_container.hpp
==============================================================================
--- branches/release/boost/spirit/home/karma/detail/pass_container.hpp (original)
+++ branches/release/boost/spirit/home/karma/detail/pass_container.hpp 2009-12-29 21:46:59 EST (Tue, 29 Dec 2009)
@@ -108,9 +108,12 @@
template <typename Component>
bool dispatch_attribute(Component const& component, mpl::true_) const
{
- typedef traits::is_container<
- typename traits::attribute_of<Component, context_type>::type
- > predicate;
+ typedef typename traits::attribute_of<
+ Component, context_type>::type attribute_type;
+
+ typedef mpl::and_<
+ traits::is_container<attribute_type>
+ , is_convertible<Attr, attribute_type> > predicate;
return dispatch_attribute_element(component, predicate());
}
Modified: branches/release/boost/spirit/home/karma/generate.hpp
==============================================================================
--- branches/release/boost/spirit/home/karma/generate.hpp (original)
+++ branches/release/boost/spirit/home/karma/generate.hpp 2009-12-29 21:46:59 EST (Tue, 29 Dec 2009)
@@ -21,7 +21,7 @@
OutputIterator& sink
, Expr const& expr)
{
- return detail::generate<Expr>::call(sink, expr);
+ return detail::generate_impl<Expr>::call(sink, expr);
}
///////////////////////////////////////////////////////////////////////////
@@ -72,7 +72,7 @@
, BOOST_SCOPED_ENUM(delimit_flag) pre_delimit =
delimit_flag::dont_predelimit)
{
- return detail::generate_delimited<Expr>::call(
+ return detail::generate_delimited_impl<Expr>::call(
sink, expr, delimiter, pre_delimit);
}
Added: branches/release/boost/spirit/home/karma/phoenix_attributes.hpp
==============================================================================
--- (empty file)
+++ branches/release/boost/spirit/home/karma/phoenix_attributes.hpp 2009-12-29 21:46:59 EST (Tue, 29 Dec 2009)
@@ -0,0 +1,112 @@
+// Copyright (c) 2001-2009 Hartmut Kaiser
+//
+// Distributed under the Boost Software License, Version 1.0. (See accompanying
+// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+
+#if !defined(BOOST_SPIRIT_KARMA_PHOENIX_ATTRIBUTES_OCT_01_2009_1128AM)
+#define BOOST_SPIRIT_KARMA_PHOENIX_ATTRIBUTES_OCT_01_2009_1128AM
+
+#if defined(_MSC_VER)
+#pragma once
+#endif
+
+#include <boost/spirit/include/version.hpp>
+
+// we support Phoenix attributes only starting with V2.2
+#if SPIRIT_VERSION >= 0x2020
+
+#include <boost/spirit/home/support/attributes.hpp>
+#include <boost/spirit/home/support/container.hpp>
+
+#include <boost/spirit/include/phoenix_core.hpp>
+#include <boost/utility/result_of.hpp>
+
+///////////////////////////////////////////////////////////////////////////////
+namespace boost { namespace spirit { namespace traits
+{
+ ///////////////////////////////////////////////////////////////////////////
+ // Provide customization points allowing the use of phoenix expressions as
+ // generator functions in the context of generators expecting a container
+ // attribute (Kleene, plus, list, repeat, etc.)
+ ///////////////////////////////////////////////////////////////////////////
+ template <typename Eval>
+ struct is_container<phoenix::actor<Eval> const>
+ : is_container<typename boost::result_of<phoenix::actor<Eval>()>::type>
+ {};
+
+ template <typename Eval>
+ struct container_iterator<phoenix::actor<Eval> const>
+ {
+ typedef phoenix::actor<Eval> const& type;
+ };
+
+ template <typename Eval>
+ struct begin_container<phoenix::actor<Eval> const>
+ {
+ typedef phoenix::actor<Eval> const& type;
+ static type call(phoenix::actor<Eval> const& f)
+ {
+ return f;
+ }
+ };
+
+ template <typename Eval>
+ struct end_container<phoenix::actor<Eval> const>
+ {
+ typedef phoenix::actor<Eval> const& type;
+ static type call(phoenix::actor<Eval> const& f)
+ {
+ return f;
+ }
+ };
+
+ template <typename Eval>
+ struct deref_iterator<phoenix::actor<Eval> const>
+ {
+ typedef typename boost::result_of<phoenix::actor<Eval>()>::type type;
+ static type call(phoenix::actor<Eval> const& f)
+ {
+ return f();
+ }
+ };
+
+ template <typename Eval>
+ struct next_iterator<phoenix::actor<Eval> const>
+ {
+ typedef phoenix::actor<Eval> const& type;
+ static type call(phoenix::actor<Eval> const& f)
+ {
+ return f;
+ }
+ };
+
+ template <typename Eval>
+ struct compare_iterators<phoenix::actor<Eval> const>
+ {
+ static bool
+ call(phoenix::actor<Eval> const&, phoenix::actor<Eval> const&)
+ {
+ return false;
+ }
+ };
+
+ ///////////////////////////////////////////////////////////////////////////
+ // Handle Phoenix actors as attributes, just invoke the function object
+ // and deal with the result as the attribute.
+ ///////////////////////////////////////////////////////////////////////////
+ template <typename Eval>
+ struct extract_from_attribute<phoenix::actor<Eval> >
+ {
+ typedef typename boost::result_of<phoenix::actor<Eval>()>::type type;
+
+ template <typename Context>
+ static type call(phoenix::actor<Eval> const& f, Context& context)
+ {
+ return f(unused, context);
+ }
+ };
+
+}}}
+
+#endif
+#endif
Modified: branches/release/boost/spirit/home/lex/lexer/lexertl/functor_data.hpp
==============================================================================
--- branches/release/boost/spirit/home/lex/lexer/lexertl/functor_data.hpp (original)
+++ branches/release/boost/spirit/home/lex/lexer/lexertl/functor_data.hpp 2009-12-29 21:46:59 EST (Tue, 29 Dec 2009)
@@ -54,7 +54,8 @@
data (IterData const& data_, Iterator& first, Iterator const& last)
: first_(first), last_(last)
, state_machine_(data_.state_machine_)
- , rules_(data_.rules_) {}
+ , rules_(data_.rules_)
+ , bol_(data_.state_machine_.data()._seen_BOL_assertion) {}
// The following functions are used by the implementation of the
// placeholder '_state'.
@@ -88,7 +89,7 @@
return it;
}
- // The function more() is used by the implemention of the support
+ // The function more() is used by the implementation of the support
// function lex::more(). Its functionality is equivalent to flex'
// function yymore(): it tells the lexer that the next time it
// matches a rule, the corresponding token should be appended onto
@@ -130,7 +131,7 @@
std::size_t next(Iterator& end, std::size_t& unique_id)
{
typedef basic_iterator_tokeniser<Iterator> tokenizer;
- return tokenizer::next(state_machine_, first_, end, last_
+ return tokenizer::next(state_machine_, bol_, end, last_
, unique_id);
}
@@ -162,6 +163,8 @@
boost::lexer::basic_state_machine<char_type> const& state_machine_;
boost::lexer::basic_rules<char_type> const& rules_;
+ bool bol_; // helper storing whether last character was \n
+
private:
// silence MSVC warning C4512: assignment operator could not be generated
data& operator= (data const&);
@@ -223,7 +226,7 @@
{
typedef basic_iterator_tokeniser<Iterator> tokenizer;
return tokenizer::next(this->state_machine_, state_,
- this->get_first(), end, this->get_eoi(), unique_id);
+ this->bol_, end, this->get_eoi(), unique_id);
}
std::size_t& get_state() { return state_; }
@@ -290,7 +293,7 @@
return it;
}
- // The function more() is used by the implemention of the support
+ // The function more() is used by the implementation of the support
// function lex::more(). Its functionality is equivalent to flex'
// function yymore(): it tells the lexer that the next time it
// matches a rule, the corresponding token should be appended onto
Modified: branches/release/boost/spirit/home/lex/lexer/lexertl/generate_static.hpp
==============================================================================
--- branches/release/boost/spirit/home/lex/lexer/lexertl/generate_static.hpp (original)
+++ branches/release/boost/spirit/home/lex/lexer/lexertl/generate_static.hpp 2009-12-29 21:46:59 EST (Tue, 29 Dec 2009)
@@ -1,4 +1,4 @@
-// Copyright (c) 2008 Ben Hanson
+// Copyright (c) 2008-2009 Ben Hanson
// Copyright (c) 2008-2009 Hartmut Kaiser
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
@@ -16,6 +16,7 @@
#include <boost/spirit/home/support/detail/lexer/rules.hpp>
#include <boost/spirit/home/support/detail/lexer/size_t.hpp>
#include <boost/spirit/home/support/detail/lexer/state_machine.hpp>
+#include <boost/spirit/home/support/detail/lexer/debug.hpp>
#include <boost/spirit/home/lex/lexer/lexertl/static_version.hpp>
#include <boost/algorithm/string.hpp>
#include <boost/lexical_cast.hpp>
@@ -40,7 +41,7 @@
template <typename Char>
inline bool
generate_cpp_state_info (boost::lexer::basic_rules<Char> const& rules_
- , std::ostream &os_, char const* name_suffix = "")
+ , std::ostream &os_, char const* name_suffix)
{
// we need to re-sort the state names in ascending order of the state
// ids, filling possible gaps in between later
@@ -89,7 +90,8 @@
}
inline bool
- generate_cpp_state_table (std::ostream &os_, char const* name_suffix = "")
+ generate_cpp_state_table (std::ostream &os_, char const* name_suffix
+ , bool bol, bool eol)
{
std::string suffix(name_suffix[0] ? "_" : "");
suffix += name_suffix;
@@ -97,9 +99,13 @@
generate_delimiter(os_);
os_ << "// this defines a generic accessors for the information above\n";
os_ << "struct lexer" << suffix << "\n{\n";
- os_ << " // version number of compatible static lexer engine\n";
- os_ << " enum { static_version = "
- << boost::lexical_cast<std::string>(SPIRIT_STATIC_LEXER_VERSION) << " };\n\n";
+ os_ << " // version number and feature-set of compatible static lexer engine\n";
+ os_ << " enum\n";
+ os_ << " {\n static_version = "
+ << boost::lexical_cast<std::string>(SPIRIT_STATIC_LEXER_VERSION) << ",\n";
+ os_ << " supports_bol = " << std::boolalpha << bol << ",\n";
+ os_ << " supports_eol = " << std::boolalpha << eol << "\n";
+ os_ << " };\n\n";
os_ << " // return the number of lexer states\n";
os_ << " static std::size_t state_count()\n";
os_ << " {\n return lexer_state_count" << suffix << "; \n }\n\n";
@@ -108,102 +114,28 @@
os_ << " {\n return lexer_state_names" << suffix << "[idx]; \n }\n\n";
os_ << " // return the next matched token\n";
os_ << " template<typename Iterator>\n";
- os_ << " static std::size_t next(std::size_t &start_state_, Iterator const& start_\n";
+ os_ << " static std::size_t next(std::size_t &start_state_, bool& bol_\n";
os_ << " , Iterator &start_token_, Iterator const& end_, std::size_t& unique_id_)\n";
os_ << " {\n return next_token" << suffix
- << "(start_state_, start_, start_token_, end_, unique_id_);\n }\n";
+ << "(start_state_, bol_, start_token_, end_, unique_id_);\n }\n";
os_ << "};\n\n";
return os_.good();
}
///////////////////////////////////////////////////////////////////////////
- // Generate a tokenizer for the given state machine.
+ // generate function body based on traversing the DFA tables
template <typename Char>
- inline bool
- generate_cpp (boost::lexer::basic_state_machine<Char> const& sm_
- , boost::lexer::basic_rules<Char> const& rules_
- , std::ostream &os_, char const* name_suffix = ""
- , bool skip_on_nomatch = true, bool optimize_parameters = true)
+ bool generate_function_body_dfa(std::ostream & os_
+ , boost::lexer::basic_state_machine<Char> const &sm_)
{
- if (sm_.data()._lookup->empty())
- return false;
-
std::size_t const dfas_ = sm_.data()._dfa->size();
std::size_t const lookups_ = sm_.data()._lookup->front()->size();
- os_ << "// Copyright (c) 2008-2009 Ben Hanson\n";
- os_ << "// Copyright (c) 2008-2009 Hartmut Kaiser\n";
- os_ << "//\n";
- os_ << "// Distributed under the Boost Software License, "
- "Version 1.0. (See accompanying\n";
- os_ << "// file licence_1_0.txt or copy at "
- "http://www.boost.org/LICENSE_1_0.txt)\n\n";
- os_ << "// Auto-generated by boost::lexer, do not edit\n\n";
-
- std::string guard(name_suffix);
- guard += name_suffix[0] ? "_" : "";
- guard += __DATE__ "_" __TIME__;
- std::string::size_type p = guard.find_first_of(": ");
- while (std::string::npos != p)
- {
- guard.replace(p, 1, "_");
- p = guard.find_first_of(": ", p);
- }
- boost::to_upper(guard);
-
- os_ << "#if !defined(BOOST_SPIRIT_LEXER_NEXT_TOKEN_" << guard << ")\n";
- os_ << "#define BOOST_SPIRIT_LEXER_NEXT_TOKEN_" << guard << "\n\n";
-
- os_ << "#include <boost/detail/iterator.hpp>\n";
- os_ << "#include <boost/spirit/home/support/detail/lexer/char_traits.hpp>\n\n";
-
- generate_delimiter(os_);
- os_ << "// the generated table of state names and the tokenizer have to be\n"
- "// defined in the boost::spirit::lex::lexertl::static_ namespace\n";
- os_ << "namespace boost { namespace spirit { namespace lex { "
- "namespace lexertl { namespace static_ {\n\n";
-
- // generate the lexer state information variables
- if (!generate_cpp_state_info(rules_, os_, name_suffix))
- return false;
-
- generate_delimiter(os_);
- os_ << "// this function returns the next matched token\n";
- os_ << "template<typename Iterator>\n";
- os_ << "std::size_t next_token" << (name_suffix[0] ? "_" : "")
- << name_suffix << " (";
-
- if (dfas_ > 1)
- {
- os_ << "std::size_t& start_state_, ";
- }
- else if (!optimize_parameters)
- {
- os_ << "std::size_t& /*start_state_*/, ";
- }
-
- if (sm_.data()._seen_BOL_assertion)
- {
- os_ << "Iterator const& start_, ";
- }
- else if (!optimize_parameters)
- {
- os_ << "Iterator const& /*start_*/, ";
- }
-
- if (dfas_ > 1 || sm_.data()._seen_BOL_assertion || !optimize_parameters)
- {
- os_ << "\n ";
- }
-
- os_ << "Iterator &start_token_, Iterator const& end_, ";
- os_ << "std::size_t& unique_id_)\n";
- os_ << "{\n";
os_ << " enum {end_state_index, id_index, unique_id_index, "
- "state_index, bol_index,\n";
+ "state_index, bol_index,\n";
os_ << " eol_index, dead_state_index, dfa_offset};\n\n";
- os_ << " static const std::size_t npos = static_cast"
- "<std::size_t>(~0);\n";
+ os_ << " static std::size_t const npos = "
+ "static_cast<std::size_t>(~0);\n";
if (dfas_ > 1)
{
@@ -215,45 +147,36 @@
std::size_t const* lookup_ = &sm_.data()._lookup[state_]->front();
std::size_t const* dfa_ = &sm_.data()._dfa[state_]->front();
- os_ << " static const std::size_t lookup" << state_
+ os_ << " static std::size_t const lookup" << state_
<< "_[" << lookups_ << "] = {\n ";
-
for (/**/; i_ < count_; ++i_)
{
- const std::size_t index_ = i_ * 8;
-
+ std::size_t const index_ = i_ * 8;
os_ << lookup_[index_];
-
- for (; j_ < 8; ++j_)
+ for (/**/; j_ < 8; ++j_)
{
os_ << ", " << lookup_[index_ + j_];
}
-
if (i_ < count_ - 1)
{
os_ << ",\n ";
}
-
j_ = 1;
}
-
os_ << " };\n";
+
count_ = sm_.data()._dfa[state_]->size ();
- os_ << " static const std::size_t dfa" << state_ << "_[" <<
- count_ << "] = {\n ";
+ os_ << " static const std::size_t dfa" << state_ << "_["
+ << count_ << "] = {\n ";
count_ /= 8;
-
for (i_ = 0; i_ < count_; ++i_)
{
- const std::size_t index_ = i_ * 8;
-
+ std::size_t const index_ = i_ * 8;
os_ << dfa_[index_];
-
for (j_ = 1; j_ < 8; ++j_)
{
os_ << ", " << dfa_[index_ + j_];
}
-
if (i_ < count_ - 1)
{
os_ << ",\n ";
@@ -263,56 +186,47 @@
std::size_t const mod_ = sm_.data()._dfa[state_]->size () % 8;
if (mod_)
{
- const std::size_t index_ = count_ * 8;
-
+ std::size_t const index_ = count_ * 8;
if (count_)
{
os_ << ",\n ";
}
-
os_ << dfa_[index_];
-
for (j_ = 1; j_ < mod_; ++j_)
{
os_ << ", " << dfa_[index_ + j_];
}
}
-
os_ << " };\n";
}
std::size_t count_ = sm_.data()._dfa_alphabet.size();
std::size_t i_ = 1;
- os_ << " static const std::size_t *lookup_arr_[" << count_ <<
- "] = { lookup0_";
-
+ os_ << " static std::size_t const* lookup_arr_[" << count_
+ << "] = { lookup0_";
for (i_ = 1; i_ < count_; ++i_)
{
os_ << ", " << "lookup" << i_ << "_";
}
-
os_ << " };\n";
- os_ << " static const std::size_t dfa_alphabet_arr_[" <<
- count_ << "] = { ";
+ os_ << " static std::size_t const dfa_alphabet_arr_["
+ << count_ << "] = { ";
os_ << sm_.data()._dfa_alphabet.front ();
-
for (i_ = 1; i_ < count_; ++i_)
{
os_ << ", " << sm_.data()._dfa_alphabet[i_];
}
-
os_ << " };\n";
- os_ << " static const std::size_t *dfa_arr_[" << count_ <<
- "] = { ";
- os_ << "dfa0_";
+ os_ << " static std::size_t const* dfa_arr_[" << count_
+ << "] = { ";
+ os_ << "dfa0_";
for (i_ = 1; i_ < count_; ++i_)
{
os_ << ", " << "dfa" << i_ << "_";
}
-
os_ << " };\n";
}
else
@@ -323,46 +237,37 @@
std::size_t j_ = 1;
std::size_t count_ = lookups_ / 8;
- os_ << " static const std::size_t lookup_[";
+ os_ << " static std::size_t const lookup_[";
os_ << sm_.data()._lookup[0]->size() << "] = {\n ";
-
- for (; i_ < count_; ++i_)
+ for (/**/; i_ < count_; ++i_)
{
const std::size_t index_ = i_ * 8;
-
os_ << lookup_[index_];
-
- for (; j_ < 8; ++j_)
+ for (/**/; j_ < 8; ++j_)
{
os_ << ", " << lookup_[index_ + j_];
}
-
if (i_ < count_ - 1)
{
os_ << ",\n ";
}
-
j_ = 1;
}
-
os_ << " };\n";
- os_ << " static const std::size_t dfa_alphabet_ = " <<
- sm_.data()._dfa_alphabet.front () << ";\n";
- os_ << " static const std::size_t dfa_[" <<
- sm_.data()._dfa[0]->size () << "] = {\n ";
- count_ = sm_.data()._dfa[0]->size () / 8;
+ os_ << " static std::size_t const dfa_alphabet_ = "
+ << sm_.data()._dfa_alphabet.front () << ";\n";
+ os_ << " static std::size_t const dfa_["
+ << sm_.data()._dfa[0]->size () << "] = {\n ";
+ count_ = sm_.data()._dfa[0]->size () / 8;
for (i_ = 0; i_ < count_; ++i_)
{
const std::size_t index_ = i_ * 8;
-
os_ << dfa_[index_];
-
for (j_ = 1; j_ < 8; ++j_)
{
os_ << ", " << dfa_[index_ + j_];
}
-
if (i_ < count_ - 1)
{
os_ << ",\n ";
@@ -370,42 +275,53 @@
}
const std::size_t mod_ = sm_.data()._dfa[0]->size () % 8;
-
if (mod_)
{
const std::size_t index_ = count_ * 8;
-
if (count_)
{
os_ << ",\n ";
}
-
os_ << dfa_[index_];
-
for (j_ = 1; j_ < mod_; ++j_)
{
os_ << ", " << dfa_[index_ + j_];
}
}
-
os_ << " };\n";
}
- os_ << "\n if (start_token_ == end_) return 0;\n\n";
+ os_ << "\n if (start_token_ == end_)\n";
+ os_ << " {\n";
+ os_ << " unique_id_ = npos;\n";
+ os_ << " return 0;\n";
+ os_ << " }\n\n";
+ if (sm_.data()._seen_BOL_assertion)
+ {
+ os_ << " bool bol = bol_;\n\n";
+ }
if (dfas_ > 1)
{
os_ << "again:\n";
- os_ << " const std::size_t * lookup_ = lookup_arr_[start_state_];\n";
+ os_ << " std::size_t const* lookup_ = lookup_arr_[start_state_];\n";
os_ << " std::size_t dfa_alphabet_ = dfa_alphabet_arr_[start_state_];\n";
- os_ << " const std::size_t *dfa_ = dfa_arr_[start_state_];\n";
+ os_ << " std::size_t const*dfa_ = dfa_arr_[start_state_];\n";
}
- os_ << " const std::size_t *ptr_ = dfa_ + dfa_alphabet_;\n";
+ os_ << " std::size_t const* ptr_ = dfa_ + dfa_alphabet_;\n";
os_ << " Iterator curr_ = start_token_;\n";
os_ << " bool end_state_ = *ptr_ != 0;\n";
os_ << " std::size_t id_ = *(ptr_ + id_index);\n";
os_ << " std::size_t uid_ = *(ptr_ + unique_id_index);\n";
+ if (dfas_ > 1)
+ {
+ os_ << " std::size_t end_start_state_ = start_state_;\n";
+ }
+ if (sm_.data()._seen_BOL_assertion)
+ {
+ os_ << " bool end_bol_ = bol_;\n";
+ }
os_ << " Iterator end_token_ = start_token_;\n\n";
os_ << " while (curr_ != end_)\n";
@@ -423,8 +339,7 @@
if (sm_.data()._seen_BOL_assertion && sm_.data()._seen_EOL_assertion)
{
- os_ << " if (BOL_state_ && (start_token_ == start_ ||\n";
- os_ << " *(start_token_ - 1) == '\\n'))\n";
+ os_ << " if (BOL_state_ && bol)\n";
os_ << " {\n";
os_ << " ptr_ = &dfa_[BOL_state_ * dfa_alphabet_];\n";
os_ << " }\n";
@@ -434,17 +349,18 @@
os_ << " }\n";
os_ << " else\n";
os_ << " {\n";
- os_ << " std::size_t const state_ =\n";
-
if (lookups_ == 256)
{
- os_ << " ptr_[lookup_[<typename Traits::index_type>"
- "(*curr_++)]];\n";
+ os_ << " unsigned char index = \n";
+ os_ << " static_cast<unsigned char>(*curr_++);\n";
}
else
{
- os_ << " ptr_[lookup_[*curr_++]];\n";
+ os_ << " std::size_t index = *curr_++\n";
}
+ os_ << " bol = (index == '\n') ? true : false;\n";
+ os_ << " std::size_t const state_ = ptr_[\n";
+ os_ << " lookup_[static_cast<std::size_t>(index)]];\n";
os_ << '\n';
os_ << " if (state_ == 0) break;\n";
@@ -454,24 +370,24 @@
}
else if (sm_.data()._seen_BOL_assertion)
{
- os_ << " if (BOL_state_ && (start_token_ == start_ ||\n";
- os_ << " *(start_token_ - 1) == '\\n'))\n";
+ os_ << " if (BOL_state_ && bol)\n";
os_ << " {\n";
os_ << " ptr_ = &dfa_[BOL_state_ * dfa_alphabet_];\n";
os_ << " }\n";
os_ << " else\n";
os_ << " {\n";
- os_ << " std::size_t const state_ =\n";
-
if (lookups_ == 256)
{
- os_ << " ptr_[lookup_[static_cast<unsigned char>"
- "(*curr_++)]];\n";
+ os_ << " unsigned char index = \n";
+ os_ << " static_cast<unsigned char>(*curr_++);\n";
}
else
{
- os_ << " ptr_[lookup_[*curr_++]];\n";
+ os_ << " std::size_t index = *curr_++\n";
}
+ os_ << " bol = (index == '\n') ? true : false;\n";
+ os_ << " std::size_t const state_ = ptr_[\n";
+ os_ << " lookup_[static_cast<std::size_t>(index)]];\n";
os_ << '\n';
os_ << " if (state_ == 0) break;\n";
@@ -487,17 +403,18 @@
os_ << " }\n";
os_ << " else\n";
os_ << " {\n";
- os_ << " std::size_t const state_ =\n";
-
if (lookups_ == 256)
{
- os_ << " ptr_[lookup_[static_cast<unsigned char>"
- "(*curr_++)]];\n";
+ os_ << " unsigned char index = \n";
+ os_ << " static_cast<unsigned char>(*curr_++);\n";
}
else
{
- os_ << " ptr_[lookup_[*curr_++]];\n";
+ os_ << " std::size_t index = *curr_++\n";
}
+ os_ << " bol = (index == '\n') ? true : false;\n";
+ os_ << " std::size_t const state_ = ptr_[\n";
+ os_ << " lookup_[static_cast<std::size_t>(index)]];\n";
os_ << '\n';
os_ << " if (state_ == 0) break;\n";
@@ -511,8 +428,8 @@
if (lookups_ == 256)
{
- os_ << " ptr_[lookup_[static_cast<unsigned char>"
- "(*curr_++)]];\n";
+ os_ << " ptr_[lookup_["
+ "static_cast<unsigned char>(*curr_++)]];\n";
}
else
{
@@ -530,19 +447,21 @@
os_ << " end_state_ = true;\n";
os_ << " id_ = *(ptr_ + id_index);\n";
os_ << " uid_ = *(ptr_ + unique_id_index);\n";
-
if (dfas_ > 1)
{
- os_ << " start_state_ = *(ptr_ + state_index);\n";
+ os_ << " end_start_state_ = *(ptr_ + state_index);\n";
+ }
+ if (sm_.data()._seen_BOL_assertion)
+ {
+ os_ << " end_bol_ = bol;\n";
}
-
os_ << " end_token_ = curr_;\n";
os_ << " }\n";
os_ << " }\n\n";
if (sm_.data()._seen_EOL_assertion)
{
- os_ << " const std::size_t EOL_state_ = ptr_[eol_index];\n\n";
+ os_ << " std::size_t const EOL_state_ = ptr_[eol_index];\n\n";
os_ << " if (EOL_state_ && curr_ == end_)\n";
os_ << " {\n";
@@ -553,12 +472,14 @@
os_ << " end_state_ = true;\n";
os_ << " id_ = *(ptr_ + id_index);\n";
os_ << " uid_ = *(ptr_ + unique_id_index);\n";
-
if (dfas_ > 1)
{
- os_ << " start_state_ = *(ptr_ + state_index);\n";
+ os_ << " end_start_state_ = *(ptr_ + state_index);\n";
+ }
+ if (sm_.data()._seen_BOL_assertion)
+ {
+ os_ << " end_bol_ = bol;\n";
}
-
os_ << " end_token_ = curr_;\n";
os_ << " }\n";
os_ << " }\n\n";
@@ -571,29 +492,415 @@
if (dfas_ > 1)
{
- os_ << " if (id_ == 0) goto again;\n";
+ os_ << " start_state_ = end_start_state_;\n";
+ os_ << " if (id_ == 0)\n";
+ os_ << " {\n";
+ if (sm_.data()._seen_BOL_assertion)
+ {
+ os_ << " bol = end_bol_;\n";
+ }
+ os_ << " goto again;\n";
+ os_ << " }\n";
+ if (sm_.data()._seen_BOL_assertion)
+ {
+ os_ << " else\n";
+ os_ << " {\n";
+ os_ << " bol_ = end_bol_;\n";
+ os_ << " }\n";
+ }
+ }
+ else if (sm_.data()._seen_BOL_assertion)
+ {
+ os_ << " bol_ = end_bol_;\n";
}
os_ << " }\n";
os_ << " else\n";
os_ << " {\n";
- if (skip_on_nomatch)
+ if (sm_.data()._seen_BOL_assertion)
+ {
+ os_ << " bol_ = (*start_token_ == '\n') ? true : false;\n";
+ }
+
+ os_ << " id_ = npos;\n";
+ os_ << " uid_ = npos;\n";
+ os_ << " }\n\n";
+
+ os_ << " unique_id_ = uid_;\n";
+ os_ << " return id_;\n";
+ return os_.good();
+ }
+
+ ///////////////////////////////////////////////////////////////////////////
+ template <typename Char>
+ inline std::string get_charlit(Char ch)
+ {
+ std::basic_string<Char> result;
+ boost::lexer::basic_string_token<Char>::escape_char (ch, result);
+ return result;
+ }
+
+ // check whether state0_0 is referenced from any of the other states
+ template <typename Char>
+ bool need_label0_0(boost::lexer::basic_state_machine<Char> const &sm_)
+ {
+ typedef typename boost::lexer::basic_state_machine<Char>::iterator
+ iterator_type;
+ iterator_type iter_ = sm_.begin();
+ std::size_t states_ = iter_->states;
+
+ for (std::size_t state_ = 0; state_ < states_; ++state_)
+ {
+ if (0 == iter_->bol_index || 0 == iter_->eol_index)
+ {
+ return true;
+ }
+
+ std::size_t const transitions_ = iter_->transitions;
+ for (std::size_t t_ = 0; t_ < transitions_; ++t_)
+ {
+ if (0 == iter_->goto_state)
+ {
+ return true;
+ }
+ ++iter_;
+ }
+ if (transitions_ == 0) ++iter_;
+ }
+ return false;
+ }
+
+ ///////////////////////////////////////////////////////////////////////////
+ template <typename Char>
+ bool generate_function_body_switch(std::ostream & os_
+ , boost::lexer::basic_state_machine<Char> const &sm_)
+ {
+ typedef typename boost::lexer::basic_state_machine<Char>::iterator
+ iterator_type;
+
+ std::size_t const lookups_ = sm_.data()._lookup->front ()->size ();
+ iterator_type iter_ = sm_.begin();
+ iterator_type labeliter_ = iter_;
+ iterator_type end_ = sm_.end();
+ std::size_t const dfas_ = sm_.data()._dfa->size ();
+
+ os_ << " static std::size_t const npos = "
+ "static_cast<std::size_t>(~0);\n";
+
+ os_ << "\n if (start_token_ == end_)\n";
+ os_ << " {\n";
+ os_ << " unique_id_ = npos;\n";
+ os_ << " return 0;\n";
+ os_ << " }\n\n";
+
+ if (sm_.data()._seen_BOL_assertion)
+ {
+ os_ << " bool bol = bol_;\n";
+ }
+
+ if (dfas_ > 1)
+ {
+ os_ << "again:\n";
+ }
+
+ os_ << " Iterator curr_ = start_token_;\n";
+ os_ << " bool end_state_ = false;\n";
+ os_ << " std::size_t id_ = npos;\n";
+ os_ << " std::size_t uid_ = npos;\n";
+
+ if (dfas_ > 1)
+ {
+ os_ << " std::size_t end_start_state_ = start_state_;\n";
+ }
+
+ if (sm_.data()._seen_BOL_assertion)
+ {
+ os_ << " bool end_bol_ = bol_;\n";
+ }
+
+ os_ << " Iterator end_token_ = start_token_;\n";
+ os_ << '\n';
+
+ os_ << " " << ((lookups_ == 256) ? "char" : "wchar_t")
+ << " ch_ = 0;\n\n";
+
+ if (dfas_ > 1)
+ {
+ os_ << " switch (start_state_)\n";
+ os_ << " {\n";
+
+ for (std::size_t i_ = 0; i_ < dfas_; ++i_)
+ {
+ os_ << " case " << i_ << ":\n";
+ os_ << " goto state" << i_ << "_0;\n";
+ os_ << " break;\n";
+ }
+
+ os_ << " default:\n";
+ os_ << " goto end;\n";
+ os_ << " break;\n";
+ os_ << " }\n";
+ }
+
+ bool need_state0_0_label = need_label0_0(sm_);
+
+ for (std::size_t dfa_ = 0; dfa_ < dfas_; ++dfa_)
+ {
+ std::size_t const states_ = iter_->states;
+ for (std::size_t state_ = 0; state_ < states_; ++state_)
+ {
+ std::size_t const transitions_ = iter_->transitions;
+ std::size_t t_ = 0;
+
+ if (dfas_ > 1 || dfa_ != 0 || state_ != 0 || need_state0_0_label)
+ {
+ os_ << "\nstate" << dfa_ << '_' << state_ << ":\n";
+ }
+
+ if (iter_->end_state)
+ {
+ os_ << " end_state_ = true;\n";
+ os_ << " id_ = " << iter_->id << ";\n";
+ os_ << " uid_ = " << iter_->unique_id << ";\n";
+ os_ << " end_token_ = curr_;\n";
+
+ if (dfas_ > 1)
+ {
+ os_ << " end_start_state_ = " << iter_->goto_dfa <<
+ ";\n";
+ }
+
+ if (sm_.data()._seen_BOL_assertion)
+ {
+ os_ << " end_bol_ = bol;\n";
+ }
+
+ if (transitions_) os_ << '\n';
+ }
+
+ if (t_ < transitions_ ||
+ iter_->bol_index != boost::lexer::npos ||
+ iter_->eol_index != boost::lexer::npos)
+ {
+ os_ << " if (curr_ == end_) goto end;\n";
+ os_ << " ch_ = *curr_;\n";
+ if (iter_->bol_index != boost::lexer::npos)
+ {
+ os_ << "\n if (bol) goto state" << dfa_ << '_'
+ << iter_->bol_index << ";\n";
+ }
+ if (iter_->eol_index != boost::lexer::npos)
+ {
+ os_ << "\n if (ch_ == '\n') goto state" << dfa_
+ << '_' << iter_->eol_index << ";\n";
+ }
+ os_ << " ++curr_;\n";
+ }
+
+ for (/**/; t_ < transitions_; ++t_)
+ {
+ char const *ptr_ = iter_->token._charset.c_str();
+ char const *end_ = ptr_ + iter_->token._charset.size();
+ char start_char_ = 0;
+ char curr_char_ = 0;
+ bool range_ = false;
+ bool first_char_ = true;
+
+ os_ << "\n if (";
+
+ while (ptr_ != end_)
+ {
+ curr_char_ = *ptr_++;
+
+ if (*ptr_ == curr_char_ + 1)
+ {
+ if (!range_)
+ {
+ start_char_ = curr_char_;
+ }
+ range_ = true;
+ }
+ else
+ {
+ if (!first_char_)
+ {
+ os_ << ((iter_->token._negated) ? " && " : " || ");
+ }
+ else
+ {
+ first_char_ = false;
+ }
+ if (range_)
+ {
+ if (iter_->token._negated)
+ {
+ os_ << "!";
+ }
+ os_ << "(ch_ >= '" << get_charlit(start_char_)
+ << "' && ch_ <= '"
+ << get_charlit(curr_char_) << "')";
+ range_ = false;
+ }
+ else
+ {
+ os_ << "ch_ "
+ << ((iter_->token._negated) ? "!=" : "==")
+ << " '" << get_charlit(curr_char_) << "'";
+ }
+ }
+ }
+
+ os_ << ") goto state" << dfa_ << '_' << iter_->goto_state
+ << ";\n";
+ ++iter_;
+ }
+
+ if (!(dfa_ == dfas_ - 1 && state_ == states_ - 1))
+ {
+ os_ << " goto end;\n";
+ }
+
+ if (transitions_ == 0) ++iter_;
+ }
+ }
+
+ os_ << "\nend:\n";
+ os_ << " if (end_state_)\n";
+ os_ << " {\n";
+ os_ << " // return longest match\n";
+ os_ << " start_token_ = end_token_;\n";
+
+ if (dfas_ > 1)
+ {
+ os_ << " start_state_ = end_start_state_;\n";
+ os_ << "\n if (id_ == 0)\n";
+ os_ << " {\n";
+
+ if (sm_.data()._seen_BOL_assertion)
+ {
+ os_ << " bol = end_bol_;\n";
+ }
+
+ os_ << " goto again;\n";
+ os_ << " }\n";
+
+ if (sm_.data()._seen_BOL_assertion)
+ {
+ os_ << " else\n";
+ os_ << " {\n";
+ os_ << " bol_ = end_bol_;\n";
+ os_ << " }\n";
+ }
+ }
+ else if (sm_.data()._seen_BOL_assertion)
{
- os_ << " // No match causes char to be skipped\n";
- os_ << " ++start_token_;\n";
+ os_ << " bol_ = end_bol_;\n";
}
+ os_ << " }\n";
+ os_ << " else\n";
+ os_ << " {\n";
+
+ if (sm_.data()._seen_BOL_assertion)
+ {
+ os_ << " bol_ = (*start_token_ == '\\n') ? true : false;\n";
+ }
os_ << " id_ = npos;\n";
os_ << " uid_ = npos;\n";
os_ << " }\n\n";
os_ << " unique_id_ = uid_;\n";
os_ << " return id_;\n";
+ return os_.good();
+ }
+
+ ///////////////////////////////////////////////////////////////////////////
+ // Generate a tokenizer for the given state machine.
+ template <typename Char, typename F>
+ inline bool
+ generate_cpp (boost::lexer::basic_state_machine<Char> const& sm_
+ , boost::lexer::basic_rules<Char> const& rules_
+ , std::ostream &os_, char const* name_suffix, F generate_function_body)
+ {
+ if (sm_.data()._lookup->empty())
+ return false;
+
+ std::size_t const dfas_ = sm_.data()._dfa->size();
+ std::size_t const lookups_ = sm_.data()._lookup->front()->size();
+
+ os_ << "// Copyright (c) 2008-2009 Ben Hanson\n";
+ os_ << "// Copyright (c) 2008-2009 Hartmut Kaiser\n";
+ os_ << "//\n";
+ os_ << "// Distributed under the Boost Software License, "
+ "Version 1.0. (See accompanying\n";
+ os_ << "// file licence_1_0.txt or copy at "
+ "http://www.boost.org/LICENSE_1_0.txt)\n\n";
+ os_ << "// Auto-generated by boost::lexer, do not edit\n\n";
+
+ std::string guard(name_suffix);
+ guard += name_suffix[0] ? "_" : "";
+ guard += __DATE__ "_" __TIME__;
+ std::string::size_type p = guard.find_first_of(": ");
+ while (std::string::npos != p)
+ {
+ guard.replace(p, 1, "_");
+ p = guard.find_first_of(": ", p);
+ }
+ boost::to_upper(guard);
+
+ os_ << "#if !defined(BOOST_SPIRIT_LEXER_NEXT_TOKEN_" << guard << ")\n";
+ os_ << "#define BOOST_SPIRIT_LEXER_NEXT_TOKEN_" << guard << "\n\n";
+
+ os_ << "#include <boost/detail/iterator.hpp>\n";
+ os_ << "#include <boost/spirit/home/support/detail/lexer/char_traits.hpp>\n\n";
+
+ generate_delimiter(os_);
+ os_ << "// the generated table of state names and the tokenizer have to be\n"
+ "// defined in the boost::spirit::lex::lexertl::static_ namespace\n";
+ os_ << "namespace boost { namespace spirit { namespace lex { "
+ "namespace lexertl { namespace static_ {\n\n";
+
+ // generate the lexer state information variables
+ if (!generate_cpp_state_info(rules_, os_, name_suffix))
+ return false;
+
+ generate_delimiter(os_);
+ os_ << "// this function returns the next matched token\n";
+ os_ << "template<typename Iterator>\n";
+ os_ << "std::size_t next_token" << (name_suffix[0] ? "_" : "")
+ << name_suffix << " (";
+
+ if (dfas_ > 1)
+ {
+ os_ << "std::size_t& start_state_, ";
+ }
+ else
+ {
+ os_ << "std::size_t& /*start_state_*/, ";
+ }
+ if (sm_.data()._seen_BOL_assertion)
+ {
+ os_ << "bool& bol_, ";
+ }
+ else
+ {
+ os_ << "bool& /*bol_*/, ";
+ }
+ os_ << "\n ";
+
+ os_ << "Iterator &start_token_, Iterator const& end_, ";
+ os_ << "std::size_t& unique_id_)\n";
+ os_ << "{\n";
+ if (!generate_function_body(os_, sm_))
+ return false;
os_ << "}\n\n";
- if (!generate_cpp_state_table(os_, name_suffix))
+ if (!generate_cpp_state_table(os_, name_suffix
+ , sm_.data()._seen_BOL_assertion, sm_.data()._seen_EOL_assertion))
+ {
return false;
+ }
os_ << "}}}}} // namespace boost::spirit::lex::lexertl::static_\n\n";
@@ -605,15 +912,47 @@
} // namespace detail
///////////////////////////////////////////////////////////////////////////
- template <typename Lexer>
+ template <typename Lexer, typename F>
inline bool
generate_static(Lexer const& lexer, std::ostream& os
- , char const* name_suffix = "")
+ , char const* name_suffix, F f)
{
if (!lexer.init_dfa())
return false;
return detail::generate_cpp(lexer.state_machine_, lexer.rules_, os
- , name_suffix, false, false);
+ , name_suffix, f);
+ }
+
+ ///////////////////////////////////////////////////////////////////////////
+ // deprecated function, will be removed in the future (this has been
+ // replaced by the function generate_static_dfa - see below).
+ template <typename Lexer>
+ inline bool
+ generate_static(Lexer const& lexer, std::ostream& os
+ , char const* name_suffix = "")
+ {
+ return generate_static(lexer, os, name_suffix
+ , &detail::generate_function_body_dfa<typename Lexer::char_type>);
+ }
+
+ ///////////////////////////////////////////////////////////////////////////
+ template <typename Lexer>
+ inline bool
+ generate_static_dfa(Lexer const& lexer, std::ostream& os
+ , char const* name_suffix = "")
+ {
+ return generate_static(lexer, os, name_suffix
+ , &detail::generate_function_body_dfa<typename Lexer::char_type>);
+ }
+
+ ///////////////////////////////////////////////////////////////////////////
+ template <typename Lexer>
+ inline bool
+ generate_static_switch(Lexer const& lexer, std::ostream& os
+ , char const* name_suffix = "")
+ {
+ return generate_static(lexer, os, name_suffix
+ , &detail::generate_function_body_switch<typename Lexer::char_type>);
}
///////////////////////////////////////////////////////////////////////////////
Modified: branches/release/boost/spirit/home/lex/lexer/lexertl/iterator_tokenizer.hpp
==============================================================================
--- branches/release/boost/spirit/home/lex/lexer/lexertl/iterator_tokenizer.hpp (original)
+++ branches/release/boost/spirit/home/lex/lexer/lexertl/iterator_tokenizer.hpp 2009-12-29 21:46:59 EST (Tue, 29 Dec 2009)
@@ -30,9 +30,8 @@
static std::size_t next (
boost::lexer::basic_state_machine<char_type> const& state_machine_
- , std::size_t &dfa_state_, Iterator const& start_
- , Iterator &start_token_, Iterator const& end_
- , std::size_t& unique_id_)
+ , std::size_t &dfa_state_, bool& bol_, Iterator &start_token_
+ , Iterator const& end_, std::size_t& unique_id_)
{
if (start_token_ == end_)
{
@@ -40,16 +39,21 @@
return 0;
}
+ bool bol = bol_;
+
again:
std::size_t const* lookup_ = &state_machine_.data()._lookup[dfa_state_]->
front ();
std::size_t dfa_alphabet_ = state_machine_.data()._dfa_alphabet[dfa_state_];
std::size_t const* dfa_ = &state_machine_.data()._dfa[dfa_state_]->front ();
+
std::size_t const* ptr_ = dfa_ + dfa_alphabet_;
Iterator curr_ = start_token_;
bool end_state_ = *ptr_ != 0;
std::size_t id_ = *(ptr_ + boost::lexer::id_index);
std::size_t uid_ = *(ptr_ + boost::lexer::unique_id_index);
+ std::size_t end_start_state_ = dfa_state_;
+ bool end_bol_ = bol_;
Iterator end_token_ = start_token_;
while (curr_ != end_)
@@ -57,8 +61,7 @@
std::size_t const BOL_state_ = ptr_[boost::lexer::bol_index];
std::size_t const EOL_state_ = ptr_[boost::lexer::eol_index];
- if (BOL_state_ && (start_token_ == start_ ||
- *(start_token_ - 1) == '\n'))
+ if (BOL_state_ && bol)
{
ptr_ = &dfa_[BOL_state_ * dfa_alphabet_];
}
@@ -77,6 +80,7 @@
index_type index =
boost::lexer::char_traits<value_type>::call(*curr_++);
+ bol = (index == '\n') ? true : false;
std::size_t const state_ = ptr_[
lookup_[static_cast<std::size_t>(index)]];
@@ -93,7 +97,8 @@
end_state_ = true;
id_ = *(ptr_ + boost::lexer::id_index);
uid_ = *(ptr_ + boost::lexer::unique_id_index);
- dfa_state_ = *(ptr_ + boost::lexer::state_index);
+ end_start_state_ = *(ptr_ + boost::lexer::state_index);
+ end_bol_ = bol;
end_token_ = curr_;
}
}
@@ -109,19 +114,29 @@
end_state_ = true;
id_ = *(ptr_ + boost::lexer::id_index);
uid_ = *(ptr_ + boost::lexer::unique_id_index);
- dfa_state_ = *(ptr_ + boost::lexer::state_index);
+ end_start_state_ = *(ptr_ + boost::lexer::state_index);
+ end_bol_ = bol;
end_token_ = curr_;
}
}
if (end_state_) {
// return longest match
+ dfa_state_ = end_start_state_;
start_token_ = end_token_;
- if (id_ == 0)
+ if (id_ == 0)
+ {
+ bol = end_bol_;
goto again;
+ }
+ else
+ {
+ bol_ = end_bol_;
+ }
}
else {
+ bol_ = (*start_token_ == '\n') ? true : false;
id_ = boost::lexer::npos;
uid_ = boost::lexer::npos;
}
@@ -133,19 +148,26 @@
///////////////////////////////////////////////////////////////////////
static std::size_t next (
boost::lexer::basic_state_machine<char_type> const& state_machine_
- , Iterator const& start_, Iterator &start_token_, Iterator const& end_
+ , bool& bol_, Iterator &start_token_, Iterator const& end_
, std::size_t& unique_id_)
{
- if (start_token_ == end_) return 0;
+ if (start_token_ == end_)
+ {
+ unique_id_ = boost::lexer::npos;
+ return 0;
+ }
+ bool bol = bol_;
std::size_t const* lookup_ = &state_machine_.data()._lookup[0]->front();
std::size_t dfa_alphabet_ = state_machine_.data()._dfa_alphabet[0];
std::size_t const* dfa_ = &state_machine_.data()._dfa[0]->front ();
std::size_t const* ptr_ = dfa_ + dfa_alphabet_;
+
Iterator curr_ = start_token_;
bool end_state_ = *ptr_ != 0;
std::size_t id_ = *(ptr_ + boost::lexer::id_index);
std::size_t uid_ = *(ptr_ + boost::lexer::unique_id_index);
+ bool end_bol_ = bol_;
Iterator end_token_ = start_token_;
while (curr_ != end_)
@@ -153,8 +175,7 @@
std::size_t const BOL_state_ = ptr_[boost::lexer::bol_index];
std::size_t const EOL_state_ = ptr_[boost::lexer::eol_index];
- if (BOL_state_ && (start_token_ == start_ ||
- *(start_token_ - 1) == '\n'))
+ if (BOL_state_ && bol)
{
ptr_ = &dfa_[BOL_state_ * dfa_alphabet_];
}
@@ -170,9 +191,10 @@
typedef typename
boost::lexer::char_traits<value_type>::index_type
index_type;
-
+
index_type index =
boost::lexer::char_traits<value_type>::call(*curr_++);
+ bol = (index == '\n') ? true : false;
std::size_t const state_ = ptr_[
lookup_[static_cast<std::size_t>(index)]];
@@ -189,6 +211,7 @@
end_state_ = true;
id_ = *(ptr_ + boost::lexer::id_index);
uid_ = *(ptr_ + boost::lexer::unique_id_index);
+ end_bol_ = bol;
end_token_ = curr_;
}
}
@@ -204,15 +227,18 @@
end_state_ = true;
id_ = *(ptr_ + boost::lexer::id_index);
uid_ = *(ptr_ + boost::lexer::unique_id_index);
+ end_bol_ = bol;
end_token_ = curr_;
}
}
if (end_state_) {
// return longest match
+ bol_ = end_bol_;
start_token_ = end_token_;
}
else {
+ bol_ = *start_token_ == '\n';
id_ = boost::lexer::npos;
uid_ = boost::lexer::npos;
}
@@ -222,10 +248,6 @@
}
};
- ///////////////////////////////////////////////////////////////////////////
- typedef basic_iterator_tokeniser<char const *> tokeniser;
- typedef basic_iterator_tokeniser<wchar_t const *> wtokeniser;
-
}}}}
#endif
Modified: branches/release/boost/spirit/home/lex/lexer/lexertl/lexer.hpp
==============================================================================
--- branches/release/boost/spirit/home/lex/lexer/lexertl/lexer.hpp (original)
+++ branches/release/boost/spirit/home/lex/lexer/lexertl/lexer.hpp 2009-12-29 21:46:59 EST (Tue, 29 Dec 2009)
@@ -90,8 +90,8 @@
}
///////////////////////////////////////////////////////////////////////////
- template <typename Lexer>
- bool generate_static(Lexer const& lex, std::ostream& os, char const* name);
+ template <typename Lexer, typename F>
+ bool generate_static(Lexer const&, std::ostream&, char const*, F);
///////////////////////////////////////////////////////////////////////////
//
@@ -169,10 +169,10 @@
typedef typename Functor::semantic_actions_type semantic_actions_type;
iterator_data_type(
- boost::lexer::basic_state_machine<char_type> const& state_machine
- , boost::lexer::basic_rules<char_type> const& rules
- , semantic_actions_type const& actions)
- : state_machine_(state_machine), rules_(rules), actions_(actions)
+ boost::lexer::basic_state_machine<char_type> const& sm
+ , boost::lexer::basic_rules<char_type> const& rules
+ , semantic_actions_type const& actions)
+ : state_machine_(sm), rules_(rules), actions_(actions)
{}
boost::lexer::basic_state_machine<char_type> const& state_machine_;
@@ -294,8 +294,9 @@
typename Functor::semantic_actions_type actions_;
mutable bool initialized_dfa_;
- template <typename Lexer>
- friend bool generate_static(Lexer const&, std::ostream&, char const*);
+ // generator functions must be able to access members directly
+ template <typename Lexer, typename F>
+ friend bool generate_static(Lexer const&, std::ostream&, char const*, F);
};
///////////////////////////////////////////////////////////////////////////
Modified: branches/release/boost/spirit/home/lex/lexer/lexertl/static_functor_data.hpp
==============================================================================
--- branches/release/boost/spirit/home/lex/lexer/lexertl/static_functor_data.hpp (original)
+++ branches/release/boost/spirit/home/lex/lexer/lexertl/static_functor_data.hpp 2009-12-29 21:46:59 EST (Tue, 29 Dec 2009)
@@ -61,7 +61,7 @@
, std::size_t> wrap_action_type;
typedef std::size_t (*next_token_functor)(std::size_t&,
- Iterator const&, Iterator&, Iterator const&, std::size_t&);
+ bool&, Iterator&, Iterator const&, std::size_t&);
typedef char_type const* (*get_state_name_type)(std::size_t);
// initialize the shared data
@@ -70,7 +70,8 @@
, Iterator const& last)
: first_(first), last_(last)
, next_token_(data.next_)
- , get_state_name_(data.get_state_name_){}
+ , get_state_name_(data.get_state_name_)
+ , bol_(data.bol_) {}
// The following functions are used by the implementation of the
// placeholder '_state'.
@@ -107,7 +108,7 @@
return it;
}
- // The function more() is used by the implemention of the support
+ // The function more() is used by the implementation of the support
// function lex::more(). Its functionality is equivalent to flex'
// function yymore(): it tells the lexer that the next time it
// matches a rule, the corresponding token should be appended onto
@@ -149,7 +150,7 @@
std::size_t next(Iterator& end, std::size_t& unique_id)
{
std::size_t state;
- return next_token_(state, first_, end, last_, unique_id);
+ return next_token_(state, bol_, end, last_, unique_id);
}
// nothing to invoke, so this is empty
@@ -180,6 +181,8 @@
next_token_functor next_token_;
get_state_name_type get_state_name_;
+ bool bol_;
+
private:
// silence MSVC warning C4512: assignment operator could not be generated
static_data& operator= (static_data const&);
@@ -242,7 +245,7 @@
// underlying input sequence.
std::size_t next(Iterator& end, std::size_t& unique_id)
{
- return this->next_token_(state_, this->first_, end, this->last_
+ return this->next_token_(state_, this->bol_, end, this->last_
, unique_id);
}
@@ -312,7 +315,7 @@
return it;
}
- // The function more() is used by the implemention of the support
+ // The function more() is used by the implementation of the support
// function lex::more(). Its functionality is equivalent to flex'
// function yymore(): it tells the lexer that the next time it
// matches a rule, the corresponding token should be appended onto
Modified: branches/release/boost/spirit/home/lex/lexer/lexertl/static_lexer.hpp
==============================================================================
--- branches/release/boost/spirit/home/lex/lexer/lexertl/static_lexer.hpp (original)
+++ branches/release/boost/spirit/home/lex/lexer/lexertl/static_lexer.hpp 2009-12-29 21:46:59 EST (Tue, 29 Dec 2009)
@@ -138,16 +138,18 @@
typedef typename Functor::get_state_name_type get_state_name_type;
iterator_data_type(next_token_functor next
- , semantic_actions_type const& actions
- , get_state_name_type get_state_name, std::size_t num_states)
+ , semantic_actions_type const& actions
+ , get_state_name_type get_state_name, std::size_t num_states
+ , bool bol)
: next_(next), actions_(actions), get_state_name_(get_state_name)
- , num_states_(num_states)
+ , num_states_(num_states), bol_(bol)
{}
next_token_functor next_;
semantic_actions_type const& actions_;
get_state_name_type get_state_name_;
std::size_t num_states_;
+ bool bol_;
private:
// silence MSVC warning C4512: assignment operator could not be generated
@@ -173,8 +175,9 @@
, char_type const* initial_state = 0) const
{
iterator_data_type iterator_data(
- &tables_type::template next<Iterator_>, actions_,
- &tables_type::state_name, tables_type::state_count()
+ &tables_type::template next<Iterator_>, actions_
+ , &tables_type::state_name, tables_type::state_count()
+ , tables_type::supports_bol
);
return iterator_type(iterator_data, first, last, initial_state);
}
Modified: branches/release/boost/spirit/home/qi/auto/create_parser.hpp
==============================================================================
--- branches/release/boost/spirit/home/qi/auto/create_parser.hpp (original)
+++ branches/release/boost/spirit/home/qi/auto/create_parser.hpp 2009-12-29 21:46:59 EST (Tue, 29 Dec 2009)
@@ -12,17 +12,17 @@
#include <boost/spirit/home/qi/auto/meta_create.hpp>
+///////////////////////////////////////////////////////////////////////////////
namespace boost { namespace spirit { namespace result_of
{
- ///////////////////////////////////////////////////////////////////////////
template <typename T>
struct create_parser
: spirit::traits::meta_create<qi::domain, T> {};
}}}
+///////////////////////////////////////////////////////////////////////////////
namespace boost { namespace spirit { namespace qi
{
- ///////////////////////////////////////////////////////////////////////////
// Main API function for parser creation from data type
template <typename T>
typename result_of::create_parser<T>::type
@@ -32,4 +32,14 @@
}
}}}
+///////////////////////////////////////////////////////////////////////////////
+namespace boost { namespace spirit { namespace traits
+{
+ // Meta function returning true if create_parser does return a valid
+ // parser for the given type T.
+ template <typename T>
+ struct create_parser_exists
+ : meta_create_exists<qi::domain, T> {};
+}}}
+
#endif
Modified: branches/release/boost/spirit/home/qi/auto/meta_create.hpp
==============================================================================
--- branches/release/boost/spirit/home/qi/auto/meta_create.hpp (original)
+++ branches/release/boost/spirit/home/qi/auto/meta_create.hpp 2009-12-29 21:46:59 EST (Tue, 29 Dec 2009)
@@ -74,7 +74,7 @@
// STL container or a fusion sequence
// The default implementation will be chosen if no predefined mapping of
- // the data type T to a Karma component is defined.
+ // the data type T to a Qi component is defined.
struct no_auto_mapping_exists {};
template <typename T, typename Enable = void>
Modified: branches/release/boost/spirit/home/qi/detail/parse.hpp
==============================================================================
--- branches/release/boost/spirit/home/qi/detail/parse.hpp (original)
+++ branches/release/boost/spirit/home/qi/detail/parse.hpp 2009-12-29 21:46:59 EST (Tue, 29 Dec 2009)
@@ -22,7 +22,7 @@
{
///////////////////////////////////////////////////////////////////////////
template <typename Expr, typename Enable = void>
- struct parse
+ struct parse_impl
{
// Report invalid expression error as early as possible.
// If you got an error_invalid_expression error message here,
@@ -33,7 +33,7 @@
};
template <typename Expr>
- struct parse<Expr
+ struct parse_impl<Expr
, typename enable_if<traits::matches<qi::domain, Expr> >::type>
{
template <typename Iterator>
@@ -49,7 +49,7 @@
///////////////////////////////////////////////////////////////////////////
template <typename Expr, typename Enable = void>
- struct phrase_parse
+ struct phrase_parse_impl
{
// Report invalid expression error as early as possible.
// If you got an error_invalid_expression error message here,
@@ -60,7 +60,7 @@
};
template <typename Expr>
- struct phrase_parse<Expr
+ struct phrase_parse_impl<Expr
, typename enable_if<traits::matches<qi::domain, Expr> >::type>
{
template <typename Iterator, typename Skipper>
Modified: branches/release/boost/spirit/home/qi/detail/parse_auto.hpp
==============================================================================
--- branches/release/boost/spirit/home/qi/detail/parse_auto.hpp (original)
+++ branches/release/boost/spirit/home/qi/detail/parse_auto.hpp 2009-12-29 21:46:59 EST (Tue, 29 Dec 2009)
@@ -21,7 +21,7 @@
{
///////////////////////////////////////////////////////////////////////////
template <typename Expr>
- struct parse<Expr
+ struct parse_impl<Expr
, typename enable_if<traits::meta_create_exists<qi::domain, Expr> >::type>
{
template <typename Iterator>
@@ -37,7 +37,7 @@
///////////////////////////////////////////////////////////////////////////
template <typename Expr>
- struct phrase_parse<Expr
+ struct phrase_parse_impl<Expr
, typename enable_if<traits::meta_create_exists<qi::domain, Expr> >::type>
{
template <typename Iterator, typename Skipper>
Modified: branches/release/boost/spirit/home/qi/detail/pass_container.hpp
==============================================================================
--- branches/release/boost/spirit/home/qi/detail/pass_container.hpp (original)
+++ branches/release/boost/spirit/home/qi/detail/pass_container.hpp 2009-12-29 21:46:59 EST (Tue, 29 Dec 2009)
@@ -1,5 +1,6 @@
/*=============================================================================
Copyright (c) 2001-2009 Joel de Guzman
+ Copyright (c) 2001-2009 Hartmut Kaiser
Distributed under the Boost Software License, Version 1.0. (See accompanying
file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
@@ -16,6 +17,7 @@
#include <boost/type_traits/is_base_of.hpp>
#include <boost/type_traits/is_convertible.hpp>
#include <boost/mpl/bool.hpp>
+#include <boost/mpl/and.hpp>
namespace boost { namespace spirit { namespace qi { namespace detail
{
@@ -98,11 +100,12 @@
template <typename Component>
bool dispatch_attribute(Component const& component, mpl::true_) const
{
- typedef traits::is_container<
- typename traits::attribute_of<
- Component, context_type, iterator_type
- >::type
- > predicate;
+ typedef typename traits::attribute_of<
+ Component, context_type, iterator_type>::type attribute_type;
+
+ typedef mpl::and_<
+ traits::is_container<attribute_type>
+ , is_convertible<attribute_type, Attr> > predicate;
return dispatch_attribute_element(component, predicate());
}
Modified: branches/release/boost/spirit/home/qi/directive/repeat.hpp
==============================================================================
--- branches/release/boost/spirit/home/qi/directive/repeat.hpp (original)
+++ branches/release/boost/spirit/home/qi/directive/repeat.hpp 2009-12-29 21:46:59 EST (Tue, 29 Dec 2009)
@@ -20,6 +20,8 @@
#include <boost/spirit/home/support/attributes.hpp>
#include <boost/spirit/home/support/info.hpp>
#include <boost/fusion/include/at.hpp>
+#include <boost/foreach.hpp>
+#include <vector>
namespace boost { namespace spirit
{
@@ -149,6 +151,60 @@
: subject(subject), iter(iter) {}
template <typename Iterator, typename Context
+ , typename Skipper, typename ValueType, typename Attribute
+ , typename LoopVar>
+ bool parse_minimal(Iterator &first, Iterator const& last
+ , Context& context, Skipper const& skipper
+ , Attribute& attr, ValueType& val, LoopVar& i) const
+ {
+ // this scope allows save and required_attr to be reclaimed
+ // immediately after we're done with the required minimum
+ // iteration.
+ Iterator save = first;
+ std::vector<ValueType> required_attr;
+ for (; !iter.got_min(i); ++i)
+ {
+ if (!subject.parse(save, last, context, skipper, val) ||
+ !traits::push_back(required_attr, val))
+ {
+ return false;
+ }
+
+ first = save;
+ traits::clear(val);
+ }
+
+ // if we got the required number of items, these are copied
+ // over (appended) to the 'real' attribute
+ BOOST_FOREACH(ValueType const& v, required_attr)
+ {
+ traits::push_back(attr, v);
+ }
+ return true;
+ }
+
+ template <typename Iterator, typename Context
+ , typename Skipper, typename LoopVar>
+ bool parse_minimal(Iterator &first, Iterator const& last
+ , Context& context, Skipper const& skipper
+ , unused_type, unused_type, LoopVar& i) const
+ {
+ // this scope allows save and required_attr to be reclaimed
+ // immediately after we're done with the required minimum
+ // iteration.
+ Iterator save = first;
+ for (; !iter.got_min(i); ++i)
+ {
+ if (!subject.parse(save, last, context, skipper, unused))
+ {
+ return false;
+ }
+ first = save;
+ }
+ return true;
+ }
+
+ template <typename Iterator, typename Context
, typename Skipper, typename Attribute>
bool parse(Iterator& first, Iterator const& last
, Context& context, Skipper const& skipper
@@ -161,27 +217,10 @@
typename LoopIter::type i = iter.start();
// parse the minimum required
- if (!iter.got_min(i)) {
- // this scope allows save and save_attr to be reclaimed
- // immediately after we're done with the required minimum
- // iteration.
- Iterator save = first;
- Attribute save_attr; traits::swap_impl(save_attr, attr);
- for (; !iter.got_min(i); ++i)
- {
- if (!subject.parse(save, last, context, skipper, val) ||
- !traits::push_back(attr, val))
- {
- // if we fail before reaching the minimum iteration
- // required, restore the iterator and the attribute
- // then return false
- traits::swap_impl(save_attr, attr);
- return false;
- }
-
- first = save;
- traits::clear(val);
- }
+ if (!iter.got_min(i) &&
+ !parse_minimal(first, last, context, skipper, attr, val, i))
+ {
+ return false;
}
// parse some more up to the maximum specified
Modified: branches/release/boost/spirit/home/qi/nonterminal/error_handler.hpp
==============================================================================
--- branches/release/boost/spirit/home/qi/nonterminal/error_handler.hpp (original)
+++ branches/release/boost/spirit/home/qi/nonterminal/error_handler.hpp 2009-12-29 21:46:59 EST (Tue, 29 Dec 2009)
@@ -116,10 +116,10 @@
// The assertions below will fire if you are using a
// multi_pass as the underlying iterator, one of your error
- // handlers forced to 'fail' or 'retry' its guarded rule,
+ // handlers forced its guarded rule to 'fail' or 'retry',
// and the error handler has not been instantiated using
// either 'fail' or 'retry' in the first place. Please see
- // the mutli_pass docs for more information.
+ // the multi_pass docs for more information.
switch (r)
{
case fail:
Modified: branches/release/boost/spirit/home/qi/parse.hpp
==============================================================================
--- branches/release/boost/spirit/home/qi/parse.hpp (original)
+++ branches/release/boost/spirit/home/qi/parse.hpp 2009-12-29 21:46:59 EST (Tue, 29 Dec 2009)
@@ -24,7 +24,7 @@
, Iterator last
, Expr const& expr)
{
- return detail::parse<Expr>::call(first, last, expr);
+ return detail::parse_impl<Expr>::call(first, last, expr);
}
template <typename Iterator, typename Expr, typename Attr>
@@ -52,7 +52,7 @@
, Skipper const& skipper
, BOOST_SCOPED_ENUM(skip_flag) post_skip = skip_flag::postskip)
{
- return detail::phrase_parse<Expr>::call(
+ return detail::phrase_parse_impl<Expr>::call(
first, last, expr, skipper, post_skip);
}
Modified: branches/release/boost/spirit/home/qi/string/symbols.hpp
==============================================================================
--- branches/release/boost/spirit/home/qi/string/symbols.hpp (original)
+++ branches/release/boost/spirit/home/qi/string/symbols.hpp 2009-12-29 21:46:59 EST (Tue, 29 Dec 2009)
@@ -187,6 +187,18 @@
, traits::get_end<Char>(str), T());
}
+ template <typename Iterator>
+ value_type* prefix_find(Iterator& first, Iterator const& last)
+ {
+ return lookup->find(first, last, Filter());
+ }
+
+ template <typename Iterator>
+ value_type const* prefix_find(Iterator& first, Iterator const& last) const
+ {
+ return lookup->find(first, last, Filter());
+ }
+
template <typename Str>
value_type* find(Str const& str)
{
@@ -205,13 +217,15 @@
template <typename Iterator>
value_type* find_impl(Iterator begin, Iterator end)
{
- return lookup->find(begin, end, Filter());
+ value_type* r = lookup->find(begin, end, Filter());
+ return begin == end ? r : 0;
}
template <typename Iterator>
value_type const* find_impl(Iterator begin, Iterator end) const
{
- return lookup->find(begin, end, Filter());
+ value_type const* r = lookup->find(begin, end, Filter());
+ return begin == end ? r : 0;
}
public:
Modified: branches/release/boost/spirit/home/support/attributes.hpp
==============================================================================
--- branches/release/boost/spirit/home/support/attributes.hpp (original)
+++ branches/release/boost/spirit/home/support/attributes.hpp 2009-12-29 21:46:59 EST (Tue, 29 Dec 2009)
@@ -727,7 +727,7 @@
static void call(optional<T>& val)
{
if (val)
- val = none_t(); // leave optional uninitialized
+ val = none_t(); // leave optional uninitialized
}
};
Modified: branches/release/boost/spirit/home/support/attributes_fwd.hpp
==============================================================================
--- branches/release/boost/spirit/home/support/attributes_fwd.hpp (original)
+++ branches/release/boost/spirit/home/support/attributes_fwd.hpp 2009-12-29 21:46:59 EST (Tue, 29 Dec 2009)
@@ -109,9 +109,6 @@
struct end;
template <typename Iterator>
- struct next;
-
- template <typename Iterator>
struct deref;
}}}
Modified: branches/release/boost/spirit/home/support/auto/meta_create.hpp
==============================================================================
--- branches/release/boost/spirit/home/support/auto/meta_create.hpp (original)
+++ branches/release/boost/spirit/home/support/auto/meta_create.hpp 2009-12-29 21:46:59 EST (Tue, 29 Dec 2009)
@@ -20,15 +20,20 @@
#include <boost/type_traits/remove_reference.hpp>
#include <boost/fusion/include/fold.hpp>
+// needed for workaround below
+#if defined(__GNUC__) && ((__GNUC__ < 4) || (__GNUC__ == 4) && (__GNUC_MINOR__ < 1))
+#include <boost/type_traits/is_same.hpp>
+#endif
+
namespace boost { namespace spirit { namespace traits
{
///////////////////////////////////////////////////////////////////////////
- // This is the main dispatching point for meta_create to the correct domain
+ // This is the main dispatch point for meta_create to the correct domain
template <typename Domain, typename T, typename Enable = void>
struct meta_create;
///////////////////////////////////////////////////////////////////////////
- // This allows to query whether a valid mapping exits for the given data
+ // This allows to query whether a valid mapping exists for the given data
// type to a component in the given domain
template <typename Domain, typename T, typename Enable = void>
struct meta_create_exists : mpl::false_ {};
@@ -54,9 +59,18 @@
template <typename T>
struct result;
+// this is a workaround for older versions of g++ (< V4.1) which apparently have
+// problems with the following template specialization
+#if defined(__GNUC__) && ((__GNUC__ < 4) || (__GNUC__ == 4) && (__GNUC_MINOR__ < 2))
+ template <typename F, typename T1, typename T2>
+ struct result<F(T1, T2)>
+ {
+ BOOST_STATIC_ASSERT((is_same<F, nary_proto_expr_function>::value));
+#else
template <typename T1, typename T2>
struct result<nary_proto_expr_function(T1, T2)>
{
+#endif
typedef typename remove_const_ref<T1>::type left_type;
typedef typename
spirit::traits::meta_create<Domain, T2>::type
Modified: branches/release/boost/spirit/home/support/detail/lexer/debug.hpp
==============================================================================
--- branches/release/boost/spirit/home/support/detail/lexer/debug.hpp (original)
+++ branches/release/boost/spirit/home/support/detail/lexer/debug.hpp 2009-12-29 21:46:59 EST (Tue, 29 Dec 2009)
@@ -39,75 +39,7 @@
while (size_)
{
- switch (*ptr_)
- {
- case '\0':
- out_ += '\\';
- out_ += '0';
- break;
- case '\a':
- out_ += '\\';
- out_ += 'a';
- break;
- case '\b':
- out_ += '\\';
- out_ += 'b';
- break;
- case 27:
- out_ += '\\';
- out_ += 'x';
- out_ += '1';
- out_ += 'b';
- break;
- case '\f':
- out_ += '\\';
- out_ += 'f';
- break;
- case '\n':
- out_ += '\\';
- out_ += 'n';
- break;
- case '\r':
- out_ += '\\';
- out_ += 'r';
- break;
- case '\t':
- out_ += '\\';
- out_ += 't';
- break;
- case '\v':
- out_ += '\\';
- out_ += 'v';
- break;
- case '\\':
- out_ += '\\';
- out_ += '\\';
- break;
- case '"':
- out_ += '\\';
- out_ += '"';
- break;
- default:
- {
- if (*ptr_ < 32 && *ptr_ >= 0)
- {
- stringstream ss_;
-
- out_ += '\\';
- out_ += 'x';
- ss_ << std::hex <<
- static_cast<std::size_t> (*ptr_);
- out_ += ss_.str ();
- }
- else
- {
- out_ += *ptr_;
- }
-
- break;
- }
- }
-
+ basic_string_token<CharT>::escape_char (*ptr_, out_);
++ptr_;
--size_;
}
Modified: branches/release/boost/spirit/home/support/detail/lexer/generate_cpp.hpp
==============================================================================
--- branches/release/boost/spirit/home/support/detail/lexer/generate_cpp.hpp (original)
+++ branches/release/boost/spirit/home/support/detail/lexer/generate_cpp.hpp 2009-12-29 21:46:59 EST (Tue, 29 Dec 2009)
@@ -1,10 +1,10 @@
-// generate_cpp_code.hpp
+// generate_cpp.hpp
// Copyright (c) 2008-2009 Ben Hanson (http://www.benhanson.net/)
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file licence_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
-#ifndef BOOST_LEXER_GENERATE_CPP_CODE_HPP
-#define BOOST_LEXER_GENERATE_CPP_CODE_HPP
+#ifndef BOOST_LEXER_GENERATE_CPP_HPP
+#define BOOST_LEXER_GENERATE_CPP_HPP
#include "char_traits.hpp"
#include "consts.hpp"
@@ -79,7 +79,7 @@
upper_name_.begin (), ::toupper);
os_ << "#ifndef " << upper_name_ + '\n';
os_ << "#define " << upper_name_ + '\n';
- os_ << "// Copyright (c) 2008 Ben Hanson\n";
+ os_ << "// Copyright (c) 2008-2009 Ben Hanson\n";
os_ << "//\n";
os_ << "// Distributed under the Boost Software License, "
"Version 1.0. (See accompanying\n";
@@ -94,25 +94,6 @@
os_ << "std::size_t &start_state_, ";
}
- if (sm_._seen_BOL_assertion || !optimise_parameters_)
- {
- if (use_pointers_)
- {
- os_ << iterator_ << " const ";
- }
- else
- {
- os_ << "const " << iterator_;
- }
-
- os_ << "start_, ";
- }
-
- if (dfas_ > 1 || sm_._seen_BOL_assertion || !optimise_parameters_)
- {
- os_ << "\n ";
- }
-
if (use_pointers_)
{
os_ << iterator_ << " &";
@@ -133,11 +114,18 @@
os_ << "const " << iterator_;
}
- os_ << "end_)\n";
+ os_ << "end_, \n";
+ os_ << " std::size_t &unique_id_";
+
+ if (sm_._seen_BOL_assertion || !optimise_parameters_)
+ {
+ os_ << ", bool &beg_of_line_";
+ }
+
+ os_ << ")\n";
os_ << "{\n";
- os_ << " enum {end_state_index, id_index, state_index, bol_index, "
- "eol_index,\n";
- os_ << " dead_state_index, dfa_offset};\n";
+ os_ << " enum {end_state_index, id_index, unique_id_index, state_index, bol_index,\n";
+ os_ << " eol_index, dead_state_index, dfa_offset};\n";
os_ << " static const std::size_t npos = static_cast"
"<std::size_t>(~0);\n";
@@ -330,7 +318,11 @@
os_ << "};\n";
}
- os_ << "\n if (start_token_ == end_) return 0;\n\n";
+ os_ << "\n if (start_token_ == end_)\n";
+ os_ << " {\n";
+ os_ << " unique_id_ = npos;\n";
+ os_ << " return 0;\n";
+ os_ << " }\n\n";
if (dfas_ > 1)
{
@@ -346,6 +338,19 @@
os_ << " Iterator curr_ = start_token_;\n";
os_ << " bool end_state_ = *ptr_ != 0;\n";
os_ << " std::size_t id_ = *(ptr_ + id_index);\n";
+ os_ << " std::size_t uid_ = *(ptr_ + unique_id_index);\n";
+
+ if (dfas_ > 1)
+ {
+ os_ << " std::size_t end_start_state_ = start_state_;\n";
+ }
+
+ if (sm_._seen_BOL_assertion)
+ {
+ os_ << " bool bol_ = beg_of_line_;\n";
+ os_ << " bool end_bol_ = bol_;\n";
+ }
+
os_ << " Iterator end_token_ = start_token_;\n";
os_ << '\n';
os_ << " while (curr_ != end_)\n";
@@ -366,108 +371,89 @@
os_ << '\n';
}
- if (sm_._seen_BOL_assertion && sm_._seen_EOL_assertion)
+ if (sm_._seen_BOL_assertion)
{
- os_ << " if (BOL_state_ && (start_token_ == start_ ||\n";
- os_ << " *(start_token_ - 1) == '\\n'))\n";
+ os_ << " if (BOL_state_ && bol_)\n";
os_ << " {\n";
os_ << " ptr_ = &dfa_[BOL_state_ * dfa_alphabet_];\n";
os_ << " }\n";
- os_ << " else if (EOL_state_ && *curr_ == '\\n')\n";
- os_ << " {\n";
- os_ << " ptr_ = &dfa_[EOL_state_ * dfa_alphabet_];\n";
- os_ << " }\n";
- os_ << " else\n";
- os_ << " {\n";
- os_ << " const std::size_t state_ =\n";
+ }
- if (lookups_ == 256)
- {
- os_ << " ptr_[lookup_[static_cast<unsigned char>\n";
- os_ << " (*curr_++)]];\n";
- }
- else
+ if (sm_._seen_EOL_assertion)
+ {
+ os_ << " ";
+
+ if (sm_._seen_BOL_assertion)
{
- os_ << " ptr_[lookup_[*curr_++]];\n";
+ os_ << "else ";
}
- os_ << '\n';
- os_ << " if (state_ == 0) break;\n";
- os_ << '\n';
- os_ << " ptr_ = &dfa_[state_ * dfa_alphabet_];\n";
+ os_ << "if (EOL_state_ && *curr_ == '\\n')\n";
+ os_ << " {\n";
+ os_ << " ptr_ = &dfa_[EOL_state_ * dfa_alphabet_];\n";
os_ << " }\n";
}
- else if (sm_._seen_BOL_assertion)
+
+ std::string tab_ (sm_._seen_BOL_assertion || sm_._seen_EOL_assertion ? " " : "");
+
+ if (sm_._seen_BOL_assertion || sm_._seen_EOL_assertion)
{
- os_ << " if (BOL_state_ && (start_token_ == start_ ||\n";
- os_ << " *(start_token_ - 1) == '\\n'))\n";
- os_ << " {\n";
- os_ << " ptr_ = &dfa_[BOL_state_ * dfa_alphabet_];\n";
- os_ << " }\n";
os_ << " else\n";
os_ << " {\n";
- os_ << " const std::size_t state_ =\n";
+ }
+
+ if (sm_._seen_BOL_assertion)
+ {
+ os_ << " ";
if (lookups_ == 256)
{
- os_ << " ptr_[lookup_[static_cast<unsigned char>\n";
- os_ << " (*curr_++)]];\n";
+ os_ << "char";
}
else
{
- os_ << " ptr_[lookup_[*curr_++]];\n";
+ os_ << "wchar_t";
}
- os_ << '\n';
- os_ << " if (state_ == 0) break;\n";
- os_ << '\n';
- os_ << " ptr_ = &dfa_[state_ * dfa_alphabet_];\n";
- os_ << " }\n";
+ os_ << " prev_char_ = *curr_++;\n\n";
+ os_ << " bol_ = prev_char_ == '\\n';\n\n";
}
- else if (sm_._seen_EOL_assertion)
- {
- os_ << " if (EOL_state_ && *curr_ == '\\n')\n";
- os_ << " {\n";
- os_ << " ptr_ = &dfa_[EOL_state_ * dfa_alphabet_];\n";
- os_ << " }\n";
- os_ << " else\n";
- os_ << " {\n";
- os_ << " const std::size_t state_ =\n";
- if (lookups_ == 256)
- {
- os_ << " ptr_[lookup_[static_cast<unsigned char>\n";
- os_ << " (*curr_++)]];\n";
- }
- else
- {
- os_ << " ptr_[lookup_[*curr_++]];\n";
- }
+ os_ << tab_;
+ os_ << " const std::size_t state_ =\n";
+ os_ << tab_;
+ os_ << " ptr_[lookup_[";
- os_ << '\n';
- os_ << " if (state_ == 0) break;\n";
- os_ << '\n';
- os_ << " ptr_ = &dfa_[state_ * dfa_alphabet_];\n";
- os_ << " }\n";
+ if (lookups_ == 256)
+ {
+ os_ << "static_cast<unsigned char>(";
+ }
+
+ if (sm_._seen_BOL_assertion)
+ {
+ os_ << "prev_char";
}
else
{
- os_ << " const std::size_t state_ =\n";
+ os_ << "*curr_++";
+ }
- if (lookups_ == 256)
- {
- os_ << " ptr_[lookup_[static_cast<unsigned char>\n";
- os_ << " (*curr_++)]];\n";
- }
- else
- {
- os_ << " ptr_[lookup_[*curr_++]];\n";
- }
- os_ << '\n';
- os_ << " if (state_ == 0) break;\n";
- os_ << '\n';
- os_ << " ptr_ = &dfa_[state_ * dfa_alphabet_];\n";
+ if (lookups_ == 256)
+ {
+ os_ << ')';
+ }
+
+ os_ << "]];\n\n";
+
+ os_ << tab_;
+ os_ << " if (state_ == 0) break;\n\n";
+ os_ << tab_;
+ os_ << " ptr_ = &dfa_[state_ * dfa_alphabet_];\n";
+
+ if (sm_._seen_BOL_assertion || sm_._seen_EOL_assertion)
+ {
+ os_ << " }\n";
}
os_ << '\n';
@@ -475,10 +461,16 @@
os_ << " {\n";
os_ << " end_state_ = true;\n";
os_ << " id_ = *(ptr_ + id_index);\n";
+ os_ << " uid_ = *(ptr_ + unique_id_index);\n";
if (dfas_ > 1)
{
- os_ << " start_state_ = *(ptr_ + state_index);\n";
+ os_ << " end_start_state_ = *(ptr_ + state_index);\n";
+ }
+
+ if (sm_._seen_BOL_assertion)
+ {
+ os_ << " end_bol_ = bol_;\n";
}
os_ << " end_token_ = curr_;\n";
@@ -498,10 +490,16 @@
os_ << " {\n";
os_ << " end_state_ = true;\n";
os_ << " id_ = *(ptr_ + id_index);\n";
+ os_ << " uid_ = *(ptr_ + unique_id_index);\n";
if (dfas_ > 1)
{
- os_ << " start_state_ = *(ptr_ + state_index);\n";
+ os_ << " end_start_state_ = *(ptr_ + state_index);\n";
+ }
+
+ if (sm_._seen_BOL_assertion)
+ {
+ os_ << " end_bol_ = bol_;\n";
}
os_ << " end_token_ = curr_;\n";
@@ -513,18 +511,51 @@
os_ << " if (end_state_)\n";
os_ << " {\n";
os_ << " // return longest match\n";
+
+ if (dfas_ > 1)
+ {
+ os_ << " start_state_ = end_start_state_;\n";
+ }
+
+ if (sm_._seen_BOL_assertion && dfas_ < 2)
+ {
+ os_ << " beg_of_line_ = end_bol_;\n";
+ }
+
os_ << " start_token_ = end_token_;\n";
if (dfas_ > 1)
{
os_ << '\n';
- os_ << " if (id_ == 0) goto again;\n";
+ os_ << " if (id_ == 0)\n";
+ os_ << " {\n";
+
+ if (sm_._seen_BOL_assertion)
+ {
+ os_ << " bol_ = end_bol_;\n";
+ }
+
+ os_ << " goto again;\n";
+ os_ << " }\n";
+
+ if (sm_._seen_BOL_assertion)
+ {
+ os_ << " else\n";
+ os_ << " {\n";
+ os_ << " beg_of_line_ = end_bol_;\n";
+ os_ << " }\n";
+ }
}
os_ << " }\n";
os_ << " else\n";
os_ << " {\n";
+ if (sm_._seen_BOL_assertion)
+ {
+ os_ << " beg_of_line_ = *start_token_ == '\\n';\n";
+ }
+
if (skip_unknown_)
{
os_ << " // No match causes char to be skipped\n";
@@ -532,8 +563,10 @@
}
os_ << " id_ = npos;\n";
+ os_ << " uid_ = npos;\n";
os_ << " }\n";
os_ << '\n';
+ os_ << " unique_id_ = uid_;\n";
os_ << " return id_;\n";
os_ << "}\n";
os_ << "\n#endif\n";
Added: branches/release/boost/spirit/home/support/detail/lexer/generate_re2c.hpp
==============================================================================
--- (empty file)
+++ branches/release/boost/spirit/home/support/detail/lexer/generate_re2c.hpp 2009-12-29 21:46:59 EST (Tue, 29 Dec 2009)
@@ -0,0 +1,444 @@
+// generate_re2c.hpp
+// Copyright (c) 2009 Ben Hanson (http://www.benhanson.net/)
+//
+// Distributed under the Boost Software License, Version 1.0. (See accompanying
+// file licence_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+#ifndef LEXERTL_GENERATE_RE2C_HPP
+#define LEXERTL_GENERATE_RE2C_HPP
+
+#include "char_traits.hpp"
+#include "consts.hpp"
+#include "internals.hpp"
+#include <iostream>
+#include "runtime_error.hpp"
+#include "size_t.hpp"
+#include "state_machine.hpp"
+#include <vector>
+
+namespace boost
+{
+namespace lexer
+{
+// check whether state0_0 is referenced from any of the other states
+template <typename Char>
+bool need_label0_0(boost::lexer::basic_state_machine<Char> const &sm_)
+{
+ typedef typename boost::lexer::basic_state_machine<Char>::iterator
+ iterator_type;
+ iterator_type iter_ = sm_.begin();
+ std::size_t states_ = iter_->states;
+
+ for (std::size_t state_ = 0; state_ < states_; ++state_)
+ {
+ if (0 == iter_->bol_index || 0 == iter_->eol_index)
+ {
+ return true;
+ }
+
+ std::size_t const transitions_ = iter_->transitions;
+ for (std::size_t t_ = 0; t_ < transitions_; ++t_)
+ {
+ if (0 == iter_->goto_state)
+ {
+ return true;
+ }
+ ++iter_;
+ }
+ if (transitions_ == 0) ++iter_;
+ }
+ return false;
+}
+
+template<typename CharT>
+void generate_re2c (const basic_state_machine<CharT> &state_machine_,
+ std::ostream &os_, const bool use_pointers_ = false,
+ const bool skip_unknown_ = true, const bool optimise_parameters_ = true,
+ const char *name_ = "next_token")
+{
+ typedef typename boost::lexer::basic_string_token<CharT> string_token;
+ const detail::internals &sm_ = state_machine_.data ();
+
+ if (sm_._lookup->size () == 0)
+ {
+ throw runtime_error ("Cannot generate code from an empty "
+ "state machine");
+ }
+
+ std::string upper_name_ (__DATE__);
+ const std::size_t lookups_ = sm_._lookup->front ()->size ();
+ typename boost::lexer::basic_state_machine<CharT>::iterator iter_ =
+ state_machine_.begin();
+ typename boost::lexer::basic_state_machine<CharT>::iterator end_ =
+ state_machine_.end();
+ const std::size_t dfas_ = sm_._dfa->size ();
+ std::string::size_type pos_ = upper_name_.find (' ');
+ const char *iterator_ = 0;
+
+ if (use_pointers_)
+ {
+ if (lookups_ == 256)
+ {
+ iterator_ = "const char *";
+ }
+ else
+ {
+ iterator_ = "const wchar_t *";
+ }
+ }
+ else
+ {
+ iterator_ = "Iterator &";
+ }
+
+ while (pos_ != std::string::npos)
+ {
+ upper_name_.replace (pos_, 1, "_");
+ pos_ = upper_name_.find (' ', pos_);
+ }
+
+ upper_name_ += '_';
+ upper_name_ += __TIME__;
+
+ pos_ = upper_name_.find (':');
+
+ while (pos_ != std::string::npos)
+ {
+ upper_name_.erase (pos_, 1);
+ pos_ = upper_name_.find (':', pos_);
+ }
+
+ upper_name_ = '_' + upper_name_;
+ upper_name_ = name_ + upper_name_;
+ std::transform (upper_name_.begin (), upper_name_.end (),
+ upper_name_.begin (), ::toupper);
+ os_ << "#ifndef " << upper_name_ + '\n';
+ os_ << "#define " << upper_name_ + '\n';
+ os_ << "// Copyright (c) 2008-2009 Ben Hanson\n";
+ os_ << "//\n";
+ os_ << "// Distributed under the Boost Software License, "
+ "Version 1.0. (See accompanying\n";
+ os_ << "// file licence_1_0.txt or copy at "
+ "http://www.boost.org/LICENSE_1_0.txt)\n\n";
+ os_ << "// Auto-generated by boost::lexer\n";
+ os_ << "template<typename Iterator>\n";
+ os_ << "std::size_t " << name_ << " (";
+
+ if (dfas_ > 1 || !optimise_parameters_)
+ {
+ os_ << "std::size_t &start_state_, ";
+ }
+
+ if (use_pointers_)
+ {
+ os_ << iterator_ << " &";
+ }
+ else
+ {
+ os_ << iterator_;
+ }
+
+ os_ << "start_token_, ";
+
+ if (use_pointers_)
+ {
+ os_ << iterator_ << " const ";
+ }
+ else
+ {
+ os_ << "const " << iterator_;
+ }
+
+ os_ << "end_, \n";
+ os_ << " std::size_t &unique_id_";
+
+ if (sm_._seen_BOL_assertion || !optimise_parameters_)
+ {
+ os_ << ", bool &beg_of_line_";
+ }
+
+ os_ << ")\n";
+ os_ << "{\n";
+ os_ << " static const std::size_t npos = static_cast"
+ "<std::size_t>(~0);\n";
+ os_ << "\n if (start_token_ == end_)\n";
+ os_ << " {\n";
+ os_ << " unique_id_ = npos;\n";
+ os_ << " return 0;\n";
+ os_ << " }\n\n";
+
+ if (dfas_ > 1)
+ {
+ os_ << "again:\n";
+ }
+
+ os_ << " Iterator curr_ = start_token_;\n";
+ os_ << " bool end_state_ = false;\n";
+ os_ << " std::size_t id_ = npos;\n";
+ os_ << " std::size_t uid_ = npos;\n";
+
+ if (dfas_ > 1)
+ {
+ os_ << " std::size_t end_start_state_ = start_state_;\n";
+ }
+
+ if (sm_._seen_BOL_assertion)
+ {
+ os_ << " bool bol_ = beg_of_line_;\n";
+ os_ << " bool end_bol_ = bol_;\n";
+ }
+
+ os_ << " Iterator end_token_ = start_token_;\n";
+ os_ << '\n';
+
+ if (dfas_ > 1)
+ {
+ os_ << " switch (start_state_)\n";
+ os_ << " {\n";
+
+ for (std::size_t i_ = 0; i_ < dfas_; ++i_)
+ {
+ os_ << " case " << i_ << ":\n";
+ os_ << " goto " << i_ << "_0;\n";
+ os_ << " // Not needed, but to prevent warnings\n";
+ os_ << " break;\n";
+ }
+
+ os_ << " default:\n";
+ os_ << " throw std::runtime_error (\"Invalid start state!\")\n";
+ os_ << " break;\n";
+ os_ << " }\n\n";
+ }
+
+ os_ << " ";
+
+ if (lookups_ == 256)
+ {
+ os_ << "char";
+ }
+ else
+ {
+ os_ << "wchar_t";
+ }
+
+ os_ << " ch_ = 0;\n\n";
+
+ bool need_state0_0_label = need_label0_0(state_machine_);
+
+ for (std::size_t dfa_ = 0; dfa_ < dfas_; ++dfa_)
+ {
+ const std::size_t states_ = iter_->states;
+
+ for (std::size_t state_ = 0; state_ < states_; ++state_)
+ {
+ const std::size_t transitions_ = iter_->transitions;
+ std::size_t t_ = 0;
+
+ if (dfas_ > 1 || dfa_ != 0 || state_ != 0 || need_state0_0_label)
+ {
+ os_ << "state" << dfa_ << '_' << state_ << ":\n";
+ }
+
+ if (iter_->end_state)
+ {
+ os_ << " end_state_ = true;\n";
+ os_ << " id_ = " << iter_->id << ";\n";
+ os_ << " uid_ = " << iter_->unique_id << ";\n";
+ os_ << " end_token_ = curr_;\n";
+
+ if (dfas_ > 1)
+ {
+ os_ << " end_start_state_ = " << iter_->goto_dfa <<
+ ";\n";
+ }
+
+ if (sm_._seen_BOL_assertion)
+ {
+ os_ << " end_bol_ = bol_;\n";
+ }
+
+ if (transitions_) os_ << '\n';
+ }
+
+ if (t_ < transitions_ || iter_->bol_index != boost::lexer::npos ||
+ iter_->eol_index != boost::lexer::npos)
+ {
+ os_ << " if (curr_ == end_) goto end;\n\n";
+ os_ << " ch_ = *curr_;\n";
+
+ if (iter_->bol_index != boost::lexer::npos)
+ {
+ os_ << "\n if (bol_) goto state" << dfa_ << '_' <<
+ iter_->bol_index << ";\n\n";
+ }
+
+ if (iter_->eol_index != boost::lexer::npos)
+ {
+ os_ << "\n if (ch_ == '\n') goto state" << dfa_ << '_' <<
+ iter_->eol_index << ";\n\n";
+ }
+
+ os_ << " ++curr_;\n";
+ }
+
+ for (; t_ < transitions_; ++t_)
+ {
+ const char *ptr_ = iter_->token._charset.c_str();
+ const char *end_ = ptr_ + iter_->token._charset.size();
+ char start_char_ = 0;
+ char curr_char_ = 0;
+ bool range_ = false;
+ bool first_char_ = true;
+
+ os_ << "\n if (";
+
+ while (ptr_ != end_)
+ {
+ curr_char_ = *ptr_++;
+
+ if (*ptr_ == curr_char_ + 1)
+ {
+ if (!range_)
+ {
+ start_char_ = curr_char_;
+ }
+
+ range_ = true;
+ }
+ else
+ {
+ if (!first_char_)
+ {
+ if (iter_->token._negated)
+ {
+ os_ << " && ";
+ }
+ else
+ {
+ os_ << " || ";
+ }
+ }
+
+ first_char_ = false;
+
+ if (range_)
+ {
+ typename string_token::string temp_;
+
+ if (iter_->token._negated)
+ {
+ os_ << "!";
+ }
+
+ string_token::escape_char (start_char_, temp_);
+ os_ << "(ch_ >= '" << temp_;
+#if defined _MSC_VER && _MSC_VER <= 1200
+ temp_.erase ();
+#else
+ temp_.clear ();
+#endif
+ string_token::escape_char (curr_char_, temp_);
+ os_ << "' && ch_ <= '" << temp_ << "')";
+ range_ = false;
+ }
+ else
+ {
+ typename string_token::string temp_;
+
+ os_ << "ch_ ";
+
+ if (iter_->token._negated)
+ {
+ os_ << "!=";
+ }
+ else
+ {
+ os_ << "==";
+ }
+
+ string_token::escape_char (curr_char_, temp_);
+ os_ << " '" << temp_ << "'";
+ }
+ }
+ }
+
+ os_ << ") goto state" << dfa_ << '_' << iter_->goto_state <<
+ ";\n\n";
+ ++iter_;
+ }
+
+ if (!(dfa_ == dfas_ - 1 && state_ == states_ - 1))
+ {
+ os_ << " goto end;\n";
+ }
+
+ if (transitions_ == 0) ++iter_;
+ }
+ }
+
+ os_ << "end:\n";
+ os_ << " if (end_state_)\n";
+ os_ << " {\n";
+ os_ << " // return longest match\n";
+
+ if (dfas_ > 1)
+ {
+ os_ << " start_state_ = end_start_state_;\n";
+ }
+
+ if (sm_._seen_BOL_assertion && dfas_ < 2)
+ {
+ os_ << " beg_of_line_ = end_bol_;\n";
+ }
+
+ os_ << " start_token_ = end_token_;\n";
+
+ if (dfas_ > 1)
+ {
+ os_ << '\n';
+ os_ << " if (id_ == 0)\n";
+ os_ << " {\n";
+
+ if (sm_._seen_BOL_assertion)
+ {
+ os_ << " bol_ = end_bol_;\n";
+ }
+
+ os_ << " goto again;\n";
+ os_ << " }\n";
+
+ if (sm_._seen_BOL_assertion)
+ {
+ os_ << " else\n";
+ os_ << " {\n";
+ os_ << " beg_of_line_ = end_bol_;\n";
+ os_ << " }\n";
+ }
+ }
+
+ os_ << " }\n";
+ os_ << " else\n";
+ os_ << " {\n";
+
+ if (sm_._seen_BOL_assertion)
+ {
+ os_ << " beg_of_line_ = *start_token_ == '\\n';\n";
+ }
+
+ if (skip_unknown_)
+ {
+ os_ << " // No match causes char to be skipped\n";
+ os_ << " ++start_token_;\n";
+ }
+
+ os_ << " id_ = npos;\n";
+ os_ << " uid_ = npos;\n";
+ os_ << " }\n";
+ os_ << '\n';
+ os_ << " unique_id_ = uid_;\n";
+ os_ << " return id_;\n";
+ os_ << "}\n";
+ os_ << "\n#endif\n";
+}
+}
+}
+#endif
Modified: branches/release/boost/spirit/home/support/detail/lexer/generator.hpp
==============================================================================
--- branches/release/boost/spirit/home/support/detail/lexer/generator.hpp (original)
+++ branches/release/boost/spirit/home/support/detail/lexer/generator.hpp 2009-12-29 21:46:59 EST (Tue, 29 Dec 2009)
@@ -44,10 +44,10 @@
for (; index_ < size_; ++index_)
{
- internals_._lookup->push_back (0);
+ internals_._lookup->push_back (static_cast<size_t_vector *>(0));
internals_._lookup->back () = new size_t_vector;
internals_._dfa_alphabet.push_back (0);
- internals_._dfa->push_back (0);
+ internals_._dfa->push_back (static_cast<size_t_vector *>(0));
internals_._dfa->back () = new size_t_vector;
}
@@ -229,7 +229,7 @@
for (; iter_ != end_; ++iter_)
{
- node_ptr_vector_->push_back (0);
+ node_ptr_vector_->push_back (static_cast<detail::selection_node *>(0));
node_ptr_vector_->back () = new detail::selection_node
(root_, *iter_);
root_ = node_ptr_vector_->back ();
@@ -285,7 +285,8 @@
locale_, node_ptr_vector_, macromap_, token_map_,
seen_BOL_assertion_, seen_EOL_assertion_);
macro_iter_pair map_iter_ = macromap_.
- insert (macro_pair (name_, (detail::node const*)0));
+ insert (macro_pair (name_, static_cast<const detail::node *>
+ (0)));
map_iter_.first->second = node_;
}
@@ -404,9 +405,9 @@
if (!found_)
{
- seen_sets_->push_back (0);
+ seen_sets_->push_back (static_cast<node_set *>(0));
seen_sets_->back () = set_ptr_.release ();
- seen_vectors_->push_back (0);
+ seen_vectors_->push_back (static_cast<node_vector *>(0));
seen_vectors_->back () = vector_ptr_.release ();
hash_vector_.push_back (hash_);
// State 0 is the jam state...
@@ -465,7 +466,7 @@
typename charset_list::list::iterator end_;
charset_ptr overlap_ (new charset);
- lhs_->push_back (0);
+ lhs_->push_back (static_cast<charset *>(0));
lhs_->back () = rhs_->front ();
rhs_->pop_front ();
@@ -511,7 +512,8 @@
}
else
{
- iter_ = lhs_->insert (++iter_, (charset*)0);
+ iter_ = lhs_->insert (++iter_,
+ static_cast<charset *>(0));
*iter_ = overlap_.release ();
// VC++ 6 Hack:
@@ -525,7 +527,7 @@
if (!r_->empty ())
{
- lhs_->push_back (0);
+ lhs_->push_back (static_cast<charset *>(0));
lhs_->back () = r_.release ();
}
}
@@ -542,7 +544,7 @@
for (; iter_ != end_; ++iter_)
{
- list_->push_back (0);
+ list_->push_back (static_cast<charset *>(0));
list_->back () = new charset (iter_->first, iter_->second);
}
}
@@ -607,7 +609,7 @@
typename equivset_list::list::iterator end_;
equivset_ptr overlap_ (new equivset);
- lhs_->push_back (0);
+ lhs_->push_back (static_cast<equivset *>(0));
lhs_->back () = rhs_->front ();
rhs_->pop_front ();
@@ -653,7 +655,8 @@
}
else
{
- iter_ = lhs_->insert (++iter_, (equivset*)0);
+ iter_ = lhs_->insert (++iter_,
+ static_cast<equivset *>(0));
*iter_ = overlap_.release ();
// VC++ 6 Hack:
@@ -667,7 +670,7 @@
if (!r_->empty ())
{
- lhs_->push_back (0);
+ lhs_->push_back (static_cast<equivset *>(0));
lhs_->back () = r_.release ();
}
}
@@ -692,7 +695,7 @@
if (token_ != null_token)
{
- list_->push_back (0);
+ list_->push_back (static_cast<equivset *>(0));
if (token_ == bol_token || token_ == eol_token)
{
@@ -733,24 +736,26 @@
if (!found_)
{
- node_ptr_vector_->push_back (0);
+ node_ptr_vector_->push_back (static_cast<detail::leaf_node *>(0));
node_ptr_vector_->back () = new detail::leaf_node
(bol_token, true);
detail::node *lhs_ = node_ptr_vector_->back ();
- node_ptr_vector_->push_back (0);
+ node_ptr_vector_->push_back (static_cast<detail::leaf_node *>(0));
node_ptr_vector_->back () = new detail::leaf_node
(null_token, true);
detail::node *rhs_ = node_ptr_vector_->back ();
- node_ptr_vector_->push_back (0);
+ node_ptr_vector_->push_back
+ (static_cast<detail::selection_node *>(0));
node_ptr_vector_->back () =
new detail::selection_node (lhs_, rhs_);
lhs_ = node_ptr_vector_->back ();
- node_ptr_vector_->push_back (0);
+ node_ptr_vector_->push_back
+ (static_cast<detail::sequence_node *>(0));
node_ptr_vector_->back () =
new detail::sequence_node (lhs_, root_);
root_ = node_ptr_vector_->back ();
Modified: branches/release/boost/spirit/home/support/detail/lexer/input.hpp
==============================================================================
--- branches/release/boost/spirit/home/support/detail/lexer/input.hpp (original)
+++ branches/release/boost/spirit/home/support/detail/lexer/input.hpp 2009-12-29 21:46:59 EST (Tue, 29 Dec 2009)
@@ -172,6 +172,7 @@
bool end_state_ = *ptr_ != 0;
std::size_t id_ = *(ptr_ + id_index);
std::size_t uid_ = *(ptr_ + unique_id_index);
+ std::size_t end_start_state_ = start_state_;
bool end_bol_ = bol_;
FwdIter end_token_ = start_token_;
@@ -211,7 +212,7 @@
end_state_ = true;
id_ = *(ptr_ + id_index);
uid_ = *(ptr_ + unique_id_index);
- start_state_ = *(ptr_ + state_index);
+ end_start_state_ = *(ptr_ + state_index);
end_bol_ = bol_;
end_token_ = curr_;
}
@@ -228,7 +229,7 @@
end_state_ = true;
id_ = *(ptr_ + id_index);
uid_ = *(ptr_ + unique_id_index);
- start_state_ = *(ptr_ + state_index);
+ end_start_state_ = *(ptr_ + state_index);
end_bol_ = bol_;
end_token_ = curr_;
}
@@ -237,14 +238,18 @@
if (end_state_)
{
// return longest match
- _data.bol = end_bol_;
+ start_state_ = end_start_state_;
start_token_ = end_token_;
if (id_ == 0)
{
- bol_ = _data.bol;
+ bol_ = end_bol_;
goto again;
}
+ else
+ {
+ _data.bol = end_bol_;
+ }
}
else
{
@@ -279,6 +284,7 @@
bool end_state_ = *ptr_ != 0;
std::size_t id_ = *(ptr_ + id_index);
std::size_t uid_ = *(ptr_ + unique_id_index);
+ std::size_t end_start_state_ = start_state_;
FwdIter end_token_ = start_token_;
while (curr_ != end_)
@@ -298,7 +304,7 @@
end_state_ = true;
id_ = *(ptr_ + id_index);
uid_ = *(ptr_ + unique_id_index);
- start_state_ = *(ptr_ + state_index);
+ end_start_state_ = *(ptr_ + state_index);
end_token_ = curr_;
}
}
@@ -306,6 +312,7 @@
if (end_state_)
{
// return longest match
+ start_state_ = end_start_state_;
start_token_ = end_token_;
if (id_ == 0) goto again;
@@ -401,8 +408,8 @@
if (end_state_)
{
// return longest match
- start_token_ = end_token_;
_data.bol = end_bol_;
+ start_token_ = end_token_;
}
else
{
Modified: branches/release/boost/spirit/home/support/detail/lexer/parser/parser.hpp
==============================================================================
--- branches/release/boost/spirit/home/support/detail/lexer/parser/parser.hpp (original)
+++ branches/release/boost/spirit/home/support/detail/lexer/parser/parser.hpp 2009-12-29 21:46:59 EST (Tue, 29 Dec 2009)
@@ -114,12 +114,12 @@
}
else
{
- node_ptr_vector_->push_back (0);
+ node_ptr_vector_->push_back (static_cast<end_node *>(0));
node *rhs_node_ = new end_node (id_, unique_id_, dfa_state_);
node_ptr_vector_->back () = rhs_node_;
- node_ptr_vector_->push_back (0);
+ node_ptr_vector_->push_back (static_cast<sequence_node *>(0));
node_ptr_vector_->back () = new sequence_node
(lhs_node_, rhs_node_);
root_ = node_ptr_vector_->back ();
@@ -292,7 +292,7 @@
assert (handle_.top ()._type == token::CHARSET &&
handle_.size () == 1);
// store charset
- node_ptr_vector_->push_back (0);
+ node_ptr_vector_->push_back (static_cast<leaf_node *>(0));
const size_t id_ = handle_.top ()._id;
@@ -354,7 +354,7 @@
node *lhs_ = tree_node_stack_.top ();
- node_ptr_vector_->push_back (0);
+ node_ptr_vector_->push_back (static_cast<selection_node *>(0));
node_ptr_vector_->back () = new selection_node (lhs_, rhs_);
tree_node_stack_.top () = node_ptr_vector_->back ();
}
@@ -368,7 +368,7 @@
node *lhs_ = tree_node_stack_.top ();
- node_ptr_vector_->push_back (0);
+ node_ptr_vector_->push_back (static_cast<sequence_node *>(0));
node_ptr_vector_->back () = new sequence_node (lhs_, rhs_);
tree_node_stack_.top () = node_ptr_vector_->back ();
}
@@ -388,12 +388,12 @@
(*iter_)->greedy (greedy_);
}
- node_ptr_vector_->push_back (0);
+ node_ptr_vector_->push_back (static_cast<leaf_node *>(0));
node *rhs_ = new leaf_node (null_token, greedy_);
node_ptr_vector_->back () = rhs_;
- node_ptr_vector_->push_back (0);
+ node_ptr_vector_->push_back (static_cast<selection_node *>(0));
node_ptr_vector_->back () = new selection_node (lhs_, rhs_);
tree_node_stack_.top () = node_ptr_vector_->back ();
}
@@ -404,7 +404,7 @@
// perform *
node *ptr_ = tree_node_stack_.top ();
- node_ptr_vector_->push_back (0);
+ node_ptr_vector_->push_back (static_cast<iteration_node *>(0));
node_ptr_vector_->back () = new iteration_node (ptr_, greedy_);
tree_node_stack_.top () = node_ptr_vector_->back ();
}
@@ -416,12 +416,12 @@
node *lhs_ = tree_node_stack_.top ();
node *copy_ = lhs_->copy (node_ptr_vector_);
- node_ptr_vector_->push_back (0);
+ node_ptr_vector_->push_back (static_cast<iteration_node *>(0));
node *rhs_ = new iteration_node (copy_, greedy_);
node_ptr_vector_->back () = rhs_;
- node_ptr_vector_->push_back (0);
+ node_ptr_vector_->push_back (static_cast<sequence_node *>(0));
node_ptr_vector_->back () = new sequence_node (lhs_, rhs_);
tree_node_stack_.top () = node_ptr_vector_->back ();
}
Modified: branches/release/boost/spirit/home/support/detail/lexer/parser/tree/iteration_node.hpp
==============================================================================
--- branches/release/boost/spirit/home/support/detail/lexer/parser/tree/iteration_node.hpp (original)
+++ branches/release/boost/spirit/home/support/detail/lexer/parser/tree/iteration_node.hpp 2009-12-29 21:46:59 EST (Tue, 29 Dec 2009)
@@ -71,7 +71,7 @@
{
node *ptr_ = new_node_stack_.top ();
- node_ptr_vector_->push_back (0);
+ node_ptr_vector_->push_back (static_cast<iteration_node *>(0));
node_ptr_vector_->back () = new iteration_node (ptr_, _greedy);
new_node_stack_.top () = node_ptr_vector_->back ();
}
Modified: branches/release/boost/spirit/home/support/detail/lexer/parser/tree/leaf_node.hpp
==============================================================================
--- branches/release/boost/spirit/home/support/detail/lexer/parser/tree/leaf_node.hpp (original)
+++ branches/release/boost/spirit/home/support/detail/lexer/parser/tree/leaf_node.hpp 2009-12-29 21:46:59 EST (Tue, 29 Dec 2009)
@@ -95,7 +95,7 @@
node_stack &new_node_stack_, bool_stack &/*perform_op_stack_*/,
bool &/*down_*/) const
{
- node_ptr_vector_->push_back (0);
+ node_ptr_vector_->push_back (static_cast<leaf_node *>(0));
node_ptr_vector_->back () = new leaf_node (_token, _greedy);
new_node_stack_.push (node_ptr_vector_->back ());
}
Modified: branches/release/boost/spirit/home/support/detail/lexer/parser/tree/selection_node.hpp
==============================================================================
--- branches/release/boost/spirit/home/support/detail/lexer/parser/tree/selection_node.hpp (original)
+++ branches/release/boost/spirit/home/support/detail/lexer/parser/tree/selection_node.hpp 2009-12-29 21:46:59 EST (Tue, 29 Dec 2009)
@@ -75,7 +75,7 @@
node *lhs_ = new_node_stack_.top ();
- node_ptr_vector_->push_back (0);
+ node_ptr_vector_->push_back (static_cast<selection_node *>(0));
node_ptr_vector_->back () = new selection_node (lhs_, rhs_);
new_node_stack_.top () = node_ptr_vector_->back ();
}
Modified: branches/release/boost/spirit/home/support/detail/lexer/parser/tree/sequence_node.hpp
==============================================================================
--- branches/release/boost/spirit/home/support/detail/lexer/parser/tree/sequence_node.hpp (original)
+++ branches/release/boost/spirit/home/support/detail/lexer/parser/tree/sequence_node.hpp 2009-12-29 21:46:59 EST (Tue, 29 Dec 2009)
@@ -93,7 +93,7 @@
node *lhs_ = new_node_stack_.top ();
- node_ptr_vector_->push_back (0);
+ node_ptr_vector_->push_back (static_cast<sequence_node *>(0));
node_ptr_vector_->back () = new sequence_node (lhs_, rhs_);
new_node_stack_.top () = node_ptr_vector_->back ();
}
Modified: branches/release/boost/spirit/home/support/detail/lexer/state_machine.hpp
==============================================================================
--- branches/release/boost/spirit/home/support/detail/lexer/state_machine.hpp (original)
+++ branches/release/boost/spirit/home/support/detail/lexer/state_machine.hpp 2009-12-29 21:46:59 EST (Tue, 29 Dec 2009)
@@ -24,6 +24,8 @@
class basic_state_machine
{
public:
+ typedef CharT char_type;
+
class iterator
{
public:
Modified: branches/release/boost/spirit/home/support/detail/lexer/string_token.hpp
==============================================================================
--- branches/release/boost/spirit/home/support/detail/lexer/string_token.hpp (original)
+++ branches/release/boost/spirit/home/support/detail/lexer/string_token.hpp 2009-12-29 21:46:59 EST (Tue, 29 Dec 2009)
@@ -146,6 +146,82 @@
}
}
+ static void escape_char (const CharT ch_, string &out_)
+ {
+ switch (ch_)
+ {
+ case '\0':
+ out_ += '\\';
+ out_ += '0';
+ break;
+ case '\a':
+ out_ += '\\';
+ out_ += 'a';
+ break;
+ case '\b':
+ out_ += '\\';
+ out_ += 'b';
+ break;
+ case 27:
+ out_ += '\\';
+ out_ += 'x';
+ out_ += '1';
+ out_ += 'b';
+ break;
+ case '\f':
+ out_ += '\\';
+ out_ += 'f';
+ break;
+ case '\n':
+ out_ += '\\';
+ out_ += 'n';
+ break;
+ case '\r':
+ out_ += '\\';
+ out_ += 'r';
+ break;
+ case '\t':
+ out_ += '\\';
+ out_ += 't';
+ break;
+ case '\v':
+ out_ += '\\';
+ out_ += 'v';
+ break;
+ case '\\':
+ out_ += '\\';
+ out_ += '\\';
+ break;
+ case '"':
+ out_ += '\\';
+ out_ += '"';
+ break;
+ case '\'':
+ out_ += '\\';
+ out_ += '\'';
+ break;
+ default:
+ {
+ if (ch_ < 32 && ch_ >= 0)
+ {
+ std::basic_stringstream<CharT> ss_;
+
+ out_ += '\\';
+ out_ += 'x';
+ ss_ << std::hex <<
+ static_cast<std::size_t> (ch_);
+ out_ += ss_.str ();
+ }
+ else
+ {
+ out_ += ch_;
+ }
+
+ break;
+ }
+ }
+ }
+
private:
void intersect_same_types (basic_string_token &rhs_,
basic_string_token &overlap_)
Modified: branches/release/boost/spirit/home/support/meta_compiler.hpp
==============================================================================
--- branches/release/boost/spirit/home/support/meta_compiler.hpp (original)
+++ branches/release/boost/spirit/home/support/meta_compiler.hpp 2009-12-29 21:46:59 EST (Tue, 29 Dec 2009)
@@ -56,7 +56,7 @@
!use_operator<Domain, proto::tag::subscript>::value
), error_proto_tag_subscript_cannot_be_used, ());
-#if !BOOST_WORKAROUND(BOOST_MSVC, <= 1400)
+#if !BOOST_WORKAROUND(BOOST_MSVC, < 1400)
// this is the non-broken part for compilers properly supporting
// partial template specialization (VC7.1 does not)
struct cases
Modified: branches/release/boost/spirit/home/support/nonterminal/expand_arg.hpp
==============================================================================
--- branches/release/boost/spirit/home/support/nonterminal/expand_arg.hpp (original)
+++ branches/release/boost/spirit/home/support/nonterminal/expand_arg.hpp 2009-12-29 21:46:59 EST (Tue, 29 Dec 2009)
@@ -12,10 +12,12 @@
#endif
#include <boost/mpl/bool.hpp>
+#include <boost/mpl/or.hpp>
#include <boost/mpl/identity.hpp>
#include <boost/mpl/eval_if.hpp>
#include <boost/utility/result_of.hpp>
#include <boost/type_traits/is_scalar.hpp>
+#include <boost/spirit/home/support/string_traits.hpp>
namespace boost { namespace spirit { namespace detail
{
@@ -26,9 +28,11 @@
template <typename T>
struct result_type
{
+ // This is a temporary hack. The better way is to detect if T
+ // can be called given unused context.
typedef typename
mpl::eval_if<
- is_scalar<T>
+ mpl::or_<is_scalar<T>, traits::is_string<T> >
, mpl::identity<T const &>
, boost::result_of<T(unused_type, Context)>
>::type
@@ -69,7 +73,7 @@
typename result_type<T>::type
operator()(T const& x) const
{
- return call(x, is_scalar<T>());
+ return call(x, mpl::or_<is_scalar<T>, traits::is_string<T> >());
}
Context& context;
Added: branches/release/boost/spirit/include/karma_phoenix_attributes.hpp
==============================================================================
--- (empty file)
+++ branches/release/boost/spirit/include/karma_phoenix_attributes.hpp 2009-12-29 21:46:59 EST (Tue, 29 Dec 2009)
@@ -0,0 +1,18 @@
+/*=============================================================================
+ Copyright (c) 2001-2009 Joel de Guzman
+ Copyright (c) 2001-2009 Hartmut Kaiser
+ http://spirit.sourceforge.net/
+
+ Distributed under the Boost Software License, Version 1.0. (See accompanying
+ file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+=============================================================================*/
+#ifndef BOOST_SPIRIT_INCLUDE_KARMA_PHOENIX_ATTRIBUTES
+#define BOOST_SPIRIT_INCLUDE_KARMA_PHOENIX_ATTRIBUTES
+
+#if defined(_MSC_VER)
+#pragma once
+#endif
+
+#include <boost/spirit/home/karma/phoenix_attributes.hpp>
+
+#endif
Modified: branches/release/boost/spirit/include/lex_generate_static_lexertl.hpp
==============================================================================
--- branches/release/boost/spirit/include/lex_generate_static_lexertl.hpp (original)
+++ branches/release/boost/spirit/include/lex_generate_static_lexertl.hpp 2009-12-29 21:46:59 EST (Tue, 29 Dec 2009)
@@ -8,5 +8,11 @@
=============================================================================*/
#ifndef BOOST_SPIRIT_INCLUDE_LEX_LEXER_GENERATE_STATIC_LEXERTL
#define BOOST_SPIRIT_INCLUDE_LEX_LEXER_GENERATE_STATIC_LEXERTL
+
+#if defined(_MSC_VER)
+#pragma once
+#endif
+
#include <boost/spirit/home/lex/lexer/lexertl/generate_static.hpp>
+
#endif