Subject: [Boost-commit] svn:boost r75910 - in trunk: boost/spirit/home/lex/lexer/lexertl boost/spirit/home/support/detail/lexer libs/spirit/test libs/spirit/test/lex
From: hartmut.kaiser_at_[hidden]
Date: 2011-12-11 19:48:42
Author: hkaiser
Date: 2011-12-11 19:48:40 EST (Sun, 11 Dec 2011)
New Revision: 75910
URL: http://svn.boost.org/trac/boost/changeset/75910
Log:
Spirit: fixed #6253: lex::lexertl::generate_static_dfa compiler errors if lexer has wchar_t as underlying stream type
Added:
trunk/libs/spirit/test/lex/regression_static_wide_6253.cpp (contents, props changed)
Text files modified:
trunk/boost/spirit/home/lex/lexer/lexertl/generate_static.hpp | 208 ++++++++++++++++++++++++++-------------
trunk/boost/spirit/home/lex/lexer/lexertl/lexer.hpp | 110 ++++++++++----------
trunk/boost/spirit/home/support/detail/lexer/rules.hpp | 20 +++
trunk/libs/spirit/test/Jamfile | 1
4 files changed, 214 insertions(+), 125 deletions(-)
Modified: trunk/boost/spirit/home/lex/lexer/lexertl/generate_static.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/lexertl/generate_static.hpp (original)
+++ trunk/boost/spirit/home/lex/lexer/lexertl/generate_static.hpp 2011-12-11 19:48:40 EST (Sun, 11 Dec 2011)
@@ -1,7 +1,7 @@
// Copyright (c) 2008-2009 Ben Hanson
// Copyright (c) 2008-2011 Hartmut Kaiser
-//
-// Distributed under the Boost Software License, Version 1.0. (See accompanying
+//
+// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
#if !defined(BOOST_SPIRIT_LEX_LEXERTL_GENERATE_CPP_FEB_10_2008_0855PM)
@@ -22,33 +22,83 @@
#include <boost/lexical_cast.hpp>
///////////////////////////////////////////////////////////////////////////////
-namespace boost { namespace spirit { namespace lex { namespace lexertl
+namespace boost { namespace spirit { namespace lex { namespace lexertl
{
namespace detail
{
+ ///////////////////////////////////////////////////////////////////////////
+ template <typename CharT>
+ struct string_lit;
+
+ template <>
+ struct string_lit<char>
+ {
+ static char get(char c) { return c; }
+ static std::string get(char const* str = "") { return str; }
+ };
+
+ template <>
+ struct string_lit<wchar_t>
+ {
+ static wchar_t get(char c)
+ {
+ typedef std::ctype<wchar_t> ctype_t;
+ return std::use_facet<ctype_t>(std::locale()).widen(c);
+ }
+ static std::basic_string<wchar_t> get(char const* source = "")
+ {
+ using namespace std; // some systems have size_t in ns std
+ size_t len = strlen(source);
+ std::auto_ptr<wchar_t> result (new wchar_t[len+1]);
+ result.get()[len] = '\0';
+
+ // working with wide character streams is supported only if the
+ // platform provides the std::ctype<wchar_t> facet
+ BOOST_ASSERT(std::has_facet<std::ctype<wchar_t> >(std::locale()));
+
+ std::use_facet<std::ctype<wchar_t> >(std::locale())
+ .widen(source, source + len, result.get());
+ return result.get();
+ }
+ };
+
+ template <typename Char>
+ inline Char L(char c)
+ {
+ return string_lit<Char>::get(c);
+ }
+
+ template <typename Char>
+ inline std::basic_string<Char> L(char const* c = "")
+ {
+ return string_lit<Char>::get(c);
+ }
+
+ ///////////////////////////////////////////////////////////////////////////
+ template <typename Char>
inline bool
- generate_delimiter(std::ostream &os_)
+ generate_delimiter(std::basic_ostream<Char> &os_)
{
- os_ << std::string(80, '/') << "\n";
+ os_ << std::basic_string<Char>(80, '/') << "\n";
return os_.good();
}
///////////////////////////////////////////////////////////////////////////
- // Generate a table of the names of the used lexer states, which is a bit
- // tricky, because the table stored with the rules is sorted based on the
+ // Generate a table of the names of the used lexer states, which is a bit
+ // tricky, because the table stored with the rules is sorted based on the
// names, but we need it sorted using the state ids.
template <typename Char>
- inline bool
+ inline bool
generate_cpp_state_info (boost::lexer::basic_rules<Char> const& rules_
- , std::ostream &os_, char const* name_suffix)
+ , std::basic_ostream<Char> &os_, Char const* name_suffix)
{
- // we need to re-sort the state names in ascending order of the state
+ // we need to re-sort the state names in ascending order of the state
// ids, filling possible gaps in between later
- typedef typename
+ typedef typename
boost::lexer::basic_rules<Char>::string_size_t_map::const_iterator
state_iterator;
- typedef std::map<std::size_t, char const*> reverse_state_map_type;
+ typedef std::map<std::size_t, Char const*> reverse_state_map_type;
reverse_state_map_type reverse_state_map;
state_iterator send = rules_.statemap().end();
@@ -60,7 +110,8 @@
generate_delimiter(os_);
os_ << "// this table defines the names of the lexer states\n";
- os_ << "char const* const lexer_state_names"
+ os_ << boost::lexer::detail::strings<Char>::char_name()
+ << " const* const lexer_state_names"
<< (name_suffix[0] ? "_" : "") << name_suffix
<< "[" << rules_.statemap().size() << "] = \n{\n";
@@ -73,7 +124,9 @@
{
os_ << " 0, // \"<undefined state>\"\n";
}
- os_ << " \"" << (*rit).second << "\"";
+ os_ << " "
+ << boost::lexer::detail::strings<Char>::char_prefix()
+ << "\"" << (*rit).second << "\"";
if (++rit != rend)
os_ << ",\n";
else
@@ -83,17 +136,18 @@
generate_delimiter(os_);
os_ << "// this variable defines the number of lexer states\n";
- os_ << "std::size_t const lexer_state_count"
+ os_ << "std::size_t const lexer_state_count"
<< (name_suffix[0] ? "_" : "") << name_suffix
<< " = " << rules_.statemap().size() << ";\n\n";
return os_.good();
}
- inline bool
- generate_cpp_state_table (std::ostream &os_, char const* name_suffix
- , bool bol, bool eol)
+ template <typename Char>
+ inline bool
+ generate_cpp_state_table (std::basic_ostream<Char> &os_
+ , Char const* name_suffix, bool bol, bool eol)
{
- std::string suffix(name_suffix[0] ? "_" : "");
+ std::basic_string<Char> suffix(L<Char>(name_suffix[0] ? "_" : ""));
suffix += name_suffix;
generate_delimiter(os_);
@@ -102,7 +156,8 @@
os_ << " // version number and feature-set of compatible static lexer engine\n";
os_ << " enum\n";
os_ << " {\n static_version = "
- << boost::lexical_cast<std::string>(SPIRIT_STATIC_LEXER_VERSION) << ",\n";
+ << boost::lexical_cast<std::basic_string<Char> >(SPIRIT_STATIC_LEXER_VERSION)
+ << ",\n";
os_ << " supports_bol = " << std::boolalpha << bol << ",\n";
os_ << " supports_eol = " << std::boolalpha << eol << "\n";
os_ << " };\n\n";
@@ -110,13 +165,14 @@
os_ << " static std::size_t state_count()\n";
os_ << " {\n return lexer_state_count" << suffix << "; \n }\n\n";
os_ << " // return the name of the lexer state as given by 'idx'\n";
- os_ << " static char const* state_name(std::size_t idx)\n";
+ os_ << " static " << boost::lexer::detail::strings<Char>::char_name()
+ << " const* state_name(std::size_t idx)\n";
os_ << " {\n return lexer_state_names" << suffix << "[idx]; \n }\n\n";
os_ << " // return the next matched token\n";
os_ << " template<typename Iterator>\n";
os_ << " static std::size_t next(std::size_t &start_state_, bool& bol_\n";
os_ << " , Iterator &start_token_, Iterator const& end_, std::size_t& unique_id_)\n";
- os_ << " {\n return next_token" << suffix
+ os_ << " {\n return next_token" << suffix
<< "(start_state_, bol_, start_token_, end_, unique_id_);\n }\n";
os_ << "};\n\n";
return os_.good();
@@ -125,8 +181,8 @@
///////////////////////////////////////////////////////////////////////////
// generate function body based on traversing the DFA tables
template <typename Char>
- bool generate_function_body_dfa(std::ostream & os_
- , boost::lexer::basic_state_machine<Char> const &sm_)
+ bool generate_function_body_dfa(std::basic_ostream<Char>& os_
+ , boost::lexer::basic_state_machine<Char> const &sm_)
{
std::size_t const dfas_ = sm_.data()._dfa->size();
std::size_t const lookups_ = sm_.data()._lookup->front()->size();
@@ -147,7 +203,7 @@
std::size_t const* lookup_ = &sm_.data()._lookup[state_]->front();
std::size_t const* dfa_ = &sm_.data()._dfa[state_]->front();
- os_ << " static std::size_t const lookup" << state_
+ os_ << " static std::size_t const lookup" << state_
<< "_[" << lookups_ << "] = {\n ";
for (/**/; i_ < count_; ++i_)
{
@@ -203,7 +259,7 @@
std::size_t count_ = sm_.data()._dfa_alphabet.size();
std::size_t i_ = 1;
- os_ << " static std::size_t const* lookup_arr_[" << count_
+ os_ << " static std::size_t const* lookup_arr_[" << count_
<< "] = { lookup0_";
for (i_ = 1; i_ < count_; ++i_)
{
@@ -211,7 +267,7 @@
}
os_ << " };\n";
- os_ << " static std::size_t const dfa_alphabet_arr_["
+ os_ << " static std::size_t const dfa_alphabet_arr_["
<< count_ << "] = { ";
os_ << sm_.data()._dfa_alphabet.front ();
for (i_ = 1; i_ < count_; ++i_)
@@ -220,7 +276,7 @@
}
os_ << " };\n";
- os_ << " static std::size_t const* dfa_arr_[" << count_
+ os_ << " static std::size_t const* dfa_arr_[" << count_
<< "] = { ";
os_ << "dfa0_";
for (i_ = 1; i_ < count_; ++i_)
@@ -255,9 +311,9 @@
}
os_ << " };\n";
- os_ << " static std::size_t const dfa_alphabet_ = "
+ os_ << " static std::size_t const dfa_alphabet_ = "
<< sm_.data()._dfa_alphabet.front () << ";\n";
- os_ << " static std::size_t const dfa_["
+ os_ << " static std::size_t const dfa_["
<< sm_.data()._dfa[0]->size () << "] = {\n ";
count_ = sm_.data()._dfa[0]->size () / 8;
for (i_ = 0; i_ < count_; ++i_)
@@ -534,10 +590,10 @@
///////////////////////////////////////////////////////////////////////////
template <typename Char>
- inline std::string get_charlit(Char ch)
+ inline std::basic_string<Char> get_charlit(Char ch)
{
std::basic_string<Char> result;
- boost::lexer::basic_string_token<Char>::escape_char (ch, result);
+ boost::lexer::basic_string_token<Char>::escape_char(ch, result);
return result;
}
@@ -573,8 +629,8 @@
///////////////////////////////////////////////////////////////////////////
template <typename Char>
- bool generate_function_body_switch(std::ostream & os_
- , boost::lexer::basic_state_machine<Char> const &sm_)
+ bool generate_function_body_switch(std::basic_ostream<Char> & os_
+ , boost::lexer::basic_state_machine<Char> const &sm_)
{
typedef typename boost::lexer::basic_state_machine<Char>::iterator
iterator_type;
@@ -622,7 +678,7 @@
os_ << " Iterator end_token_ = start_token_;\n";
os_ << '\n';
- os_ << " " << ((lookups_ == 256) ? "char" : "wchar_t")
+ os_ << " " << ((lookups_ == 256) ? "char" : "wchar_t")
<< " ch_ = 0;\n\n";
if (dfas_ > 1)
@@ -679,7 +735,7 @@
if (transitions_) os_ << '\n';
}
- if (t_ < transitions_ ||
+ if (t_ < transitions_ ||
iter_->bol_index != boost::lexer::npos ||
iter_->eol_index != boost::lexer::npos)
{
@@ -687,12 +743,12 @@
os_ << " ch_ = *curr_;\n";
if (iter_->bol_index != boost::lexer::npos)
{
- os_ << "\n if (bol) goto state" << dfa_ << '_'
+ os_ << "\n if (bol) goto state" << dfa_ << '_'
<< iter_->bol_index << ";\n";
}
if (iter_->eol_index != boost::lexer::npos)
{
- os_ << "\n if (ch_ == '\n') goto state" << dfa_
+ os_ << "\n if (ch_ == '\n') goto state" << dfa_
<< '_' << iter_->eol_index << ";\n";
}
os_ << " ++curr_;\n";
@@ -700,10 +756,10 @@
for (/**/; t_ < transitions_; ++t_)
{
- char const *ptr_ = iter_->token._charset.c_str();
- char const *end_ = ptr_ + iter_->token._charset.size();
- char start_char_ = 0;
- char curr_char_ = 0;
+ Char const *ptr_ = iter_->token._charset.c_str();
+ Char const *end_ = ptr_ + iter_->token._charset.size();
+ Char start_char_ = 0;
+ Char curr_char_ = 0;
bool range_ = false;
bool first_char_ = true;
@@ -727,7 +783,7 @@
{
os_ << ((iter_->token._negated) ? " && " : " || ");
}
- else
+ else
{
first_char_ = false;
}
@@ -738,20 +794,20 @@
os_ << "!";
}
os_ << "(ch_ >= '" << get_charlit(start_char_)
- << "' && ch_ <= '"
+ << "' && ch_ <= '"
<< get_charlit(curr_char_) << "')";
range_ = false;
}
else
{
- os_ << "ch_ "
+ os_ << "ch_ "
<< ((iter_->token._negated) ? "!=" : "==")
<< " '" << get_charlit(curr_char_) << "'";
}
}
}
- os_ << ") goto state" << dfa_ << '_' << iter_->goto_state
+ os_ << ") goto state" << dfa_ << '_' << iter_->goto_state
<< ";\n";
++iter_;
}
@@ -818,10 +874,11 @@
///////////////////////////////////////////////////////////////////////////
// Generate a tokenizer for the given state machine.
template <typename Char, typename F>
- inline bool
+ inline bool
generate_cpp (boost::lexer::basic_state_machine<Char> const& sm_
, boost::lexer::basic_rules<Char> const& rules_
- , std::ostream &os_, char const* name_suffix, F generate_function_body)
+ , std::basic_ostream<Char> &os_, Char const* name_suffix
+ , F generate_function_body)
{
if (sm_.data()._lookup->empty())
return false;
@@ -838,14 +895,14 @@
"http://www.boost.org/LICENSE_1_0.txt)\n\n";
os_ << "// Auto-generated by boost::lexer, do not edit\n\n";
- std::string guard(name_suffix);
- guard += name_suffix[0] ? "_" : "";
- guard += __DATE__ "_" __TIME__;
- std::string::size_type p = guard.find_first_of(": ");
- while (std::string::npos != p)
+ std::basic_string<Char> guard(name_suffix);
+ guard += L<Char>(name_suffix[0] ? "_" : "");
+ guard += L<Char>(__DATE__ "_" __TIME__);
+ std::basic_string<Char>::size_type p = guard.find_first_of(L<Char>(": "));
+ while (std::string::npos != p)
{
- guard.replace(p, 1, "_");
- p = guard.find_first_of(": ", p);
+ guard.replace(p, 1, L<Char>("_"));
+ p = guard.find_first_of(L<Char>(": "), p);
}
boost::to_upper(guard);
@@ -868,14 +925,14 @@
generate_delimiter(os_);
os_ << "// this function returns the next matched token\n";
os_ << "template<typename Iterator>\n";
- os_ << "std::size_t next_token" << (name_suffix[0] ? "_" : "")
+ os_ << "std::size_t next_token" << (name_suffix[0] ? "_" : "")
<< name_suffix << " (";
if (dfas_ > 1)
{
os_ << "std::size_t& start_state_, ";
}
- else
+ else
{
os_ << "std::size_t& /*start_state_*/, ";
}
@@ -883,7 +940,7 @@
{
os_ << "bool& bol_, ";
}
- else
+ else
{
os_ << "bool& /*bol_*/, ";
}
@@ -896,7 +953,7 @@
return false;
os_ << "}\n\n";
- if (!generate_cpp_state_table(os_, name_suffix
+ if (!generate_cpp_state_table<Char>(os_, name_suffix
, sm_.data()._seen_BOL_assertion, sm_.data()._seen_EOL_assertion))
{
return false;
@@ -913,9 +970,10 @@
///////////////////////////////////////////////////////////////////////////
template <typename Lexer, typename F>
- inline bool
- generate_static(Lexer const& lexer, std::ostream& os
- , char const* name_suffix, F f)
+ inline bool
+ generate_static(Lexer const& lexer
+ , std::basic_ostream<typename Lexer::char_type>& os
+ , typename Lexer::char_type const* name_suffix, F f)
{
if (!lexer.init_dfa(true)) // always minimize DFA for static lexers
return false;
@@ -924,12 +982,14 @@
}
///////////////////////////////////////////////////////////////////////////
- // deprecated function, will be removed in the future (this has been
+ // deprecated function, will be removed in the future (this has been
// replaced by the function generate_static_dfa - see below).
template <typename Lexer>
- inline bool
- generate_static(Lexer const& lexer, std::ostream& os
- , char const* name_suffix = "")
+ inline bool
+ generate_static(Lexer const& lexer
+ , std::basic_ostream<typename Lexer::char_type>& os
+ , typename Lexer::char_type const* name_suffix =
+ detail::L<typename Lexer::char_type>())
{
return generate_static(lexer, os, name_suffix
, &detail::generate_function_body_dfa<typename Lexer::char_type>);
@@ -937,9 +997,11 @@
///////////////////////////////////////////////////////////////////////////
template <typename Lexer>
- inline bool
- generate_static_dfa(Lexer const& lexer, std::ostream& os
- , char const* name_suffix = "")
+ inline bool
+ generate_static_dfa(Lexer const& lexer
+ , std::basic_ostream<typename Lexer::char_type>& os
+ , typename Lexer::char_type const* name_suffix =
+ detail::L<typename Lexer::char_type>())
{
return generate_static(lexer, os, name_suffix
, &detail::generate_function_body_dfa<typename Lexer::char_type>);
@@ -947,9 +1009,11 @@
///////////////////////////////////////////////////////////////////////////
template <typename Lexer>
- inline bool
- generate_static_switch(Lexer const& lexer, std::ostream& os
- , char const* name_suffix = "")
+ inline bool
+ generate_static_switch(Lexer const& lexer
+ , std::basic_ostream<typename Lexer::char_type>& os
+ , typename Lexer::char_type const* name_suffix =
+ detail::L<typename Lexer::char_type>())
{
return generate_static(lexer, os, name_suffix
, &detail::generate_function_body_switch<typename Lexer::char_type>);
Modified: trunk/boost/spirit/home/lex/lexer/lexertl/lexer.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/lexertl/lexer.hpp (original)
+++ trunk/boost/spirit/home/lex/lexer/lexertl/lexer.hpp 2011-12-11 19:48:40 EST (Sun, 11 Dec 2011)
@@ -1,6 +1,6 @@
// Copyright (c) 2001-2011 Hartmut Kaiser
-//
-// Distributed under the Boost Software License, Version 1.0. (See accompanying
+//
+// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
#if !defined(BOOST_SPIRIT_LEX_LEXER_MAR_17_2007_0139PM)
@@ -28,13 +28,13 @@
#include <boost/foreach.hpp>
namespace boost { namespace spirit { namespace lex { namespace lexertl
-{
+{
///////////////////////////////////////////////////////////////////////////
namespace detail
{
///////////////////////////////////////////////////////////////////////
// The must_escape function checks if the given character value needs
- // to be preceded by a backslash character to disable its special
+ // to be preceded by a backslash character to disable its special
// meaning in the context of a regular expression
///////////////////////////////////////////////////////////////////////
template <typename Char>
@@ -60,15 +60,15 @@
}
///////////////////////////////////////////////////////////////////////
- // The escape function returns the string representation of the given
- // character value, possibly escaped with a backslash character, to
+ // The escape function returns the string representation of the given
+ // character value, possibly escaped with a backslash character, to
// allow it being safely used in a regular expression definition.
///////////////////////////////////////////////////////////////////////
template <typename Char>
- inline std::basic_string<Char> escape(Char ch)
- {
+ inline std::basic_string<Char> escape(Char ch)
+ {
std::basic_string<Char> result(1, ch);
- if (detail::must_escape(ch))
+ if (detail::must_escape(ch))
{
typedef typename std::basic_string<Char>::size_type size_type;
result.insert((size_type)0, 1, '\\');
@@ -77,7 +77,7 @@
}
///////////////////////////////////////////////////////////////////////
- //
+ //
///////////////////////////////////////////////////////////////////////
inline boost::lexer::regex_flags map_flags(unsigned int flags)
{
@@ -93,31 +93,33 @@
///////////////////////////////////////////////////////////////////////////
template <typename Lexer, typename F>
- bool generate_static(Lexer const&, std::ostream&, char const*, F);
+ bool generate_static(Lexer const&
+ , std::basic_ostream<typename Lexer::char_type>&
+ , typename Lexer::char_type const*, F);
///////////////////////////////////////////////////////////////////////////
//
- // Every lexer type to be used as a lexer for Spirit has to conform to
+ // Every lexer type to be used as a lexer for Spirit has to conform to
// the following public interface:
//
- // typedefs:
+ // typedefs:
// iterator_type The type of the iterator exposed by this lexer.
- // token_type The type of the tokens returned from the exposed
+ // token_type The type of the tokens returned from the exposed
// iterators.
//
// functions:
// default constructor
- // Since lexers are instantiated as base classes
- // only it might be a good idea to make this
+ // Since lexers are instantiated as base classes
+ // only it might be a good idea to make this
// constructor protected.
// begin, end Return a pair of iterators, when dereferenced
- // returning the sequence of tokens recognized in
- // the input stream given as the parameters to the
+ // returning the sequence of tokens recognized in
+ // the input stream given as the parameters to the
// begin() function.
- // add_token Should add the definition of a token to be
+ // add_token Should add the definition of a token to be
// recognized by this lexer.
// clear Should delete all current token definitions
- // associated with the given state of this lexer
+ // associated with the given state of this lexer
// object.
//
// template parameters:
@@ -126,25 +128,25 @@
// Token The type of the tokens to be returned from the
// exposed token iterator.
// Functor The type of the InputPolicy to use to instantiate
- // the multi_pass iterator type to be used as the
+ // the multi_pass iterator type to be used as the
// token iterator (returned from begin()/end()).
//
///////////////////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////
//
- // The lexer class is a implementation of a Spirit.Lex lexer on
- // top of Ben Hanson's lexertl library as outlined above (For more
+ // The lexer class is a implementation of a Spirit.Lex lexer on
+ // top of Ben Hanson's lexertl library as outlined above (For more
// information about lexertl go here: http://www.benhanson.net/lexertl.html).
//
- // This class is supposed to be used as the first and only template
+ // This class is supposed to be used as the first and only template
// parameter while instantiating instances of a lex::lexer class.
//
///////////////////////////////////////////////////////////////////////////
template <typename Token = token<>
, typename Iterator = typename Token::iterator_type
, typename Functor = functor<Token, lexertl::detail::data, Iterator> >
- class lexer
+ class lexer
{
private:
struct dummy { void true_() {} };
@@ -156,13 +158,13 @@
operator safe_bool() const
{ return initialized_dfa_ ? &dummy::true_ : 0; }
- typedef typename boost::detail::iterator_traits<Iterator>::value_type
+ typedef typename boost::detail::iterator_traits<Iterator>::value_type
char_type;
typedef std::basic_string<char_type> string_type;
typedef boost::lexer::basic_rules<char_type> basic_rules_type;
- // Every lexer type to be used as a lexer for Spirit has to conform to
+ // Every lexer type to be used as a lexer for Spirit has to conform to
// a public interface .
typedef Token token_type;
typedef typename Token::id_type id_type;
@@ -170,7 +172,7 @@
private:
// this type is purely used for the iterator_type construction below
- struct iterator_data_type
+ struct iterator_data_type
{
typedef typename Functor::semantic_actions_type semantic_actions_type;
@@ -195,7 +197,7 @@
// tokens.
iterator_type begin(Iterator& first, Iterator const& last
, char_type const* initial_state = 0) const
- {
+ {
if (!init_dfa()) // never minimize DFA for dynamic lexers
return iterator_type();
@@ -203,16 +205,16 @@
return iterator_type(iterator_data, first, last, initial_state);
}
- // Return the end iterator usable to stop iterating over the generated
+ // Return the end iterator usable to stop iterating over the generated
// tokens.
iterator_type end() const
- {
- return iterator_type();
+ {
+ return iterator_type();
}
protected:
// Lexer instances can be created by means of a derived class only.
- lexer(unsigned int flags)
+ lexer(unsigned int flags)
: flags_(detail::map_flags(flags))
, rules_(flags_)
, initialized_dfa_(false)
@@ -220,7 +222,7 @@
public:
// interface for token definition management
- std::size_t add_token(char_type const* state, char_type tokendef,
+ std::size_t add_token(char_type const* state, char_type tokendef,
std::size_t token_id, char_type const* targetstate)
{
add_state(state);
@@ -234,7 +236,7 @@
add_state(targetstate);
return rules_.add(state, detail::escape(tokendef), token_id, targetstate);
}
- std::size_t add_token(char_type const* state, string_type const& tokendef,
+ std::size_t add_token(char_type const* state, string_type const& tokendef,
std::size_t token_id, char_type const* targetstate)
{
add_state(state);
@@ -269,7 +271,7 @@
}
std::size_t add_state(char_type const* state)
{
- if (state == all_states())
+ if (state == all_states())
return all_states_id;
std::size_t stateid = rules_.state(state);
@@ -279,12 +281,12 @@
}
return stateid;
}
- string_type initial_state() const
- {
+ string_type initial_state() const
+ {
return string_type(rules_.initial());
}
- string_type all_states() const
- {
+ string_type all_states() const
+ {
return string_type(rules_.all_states());
}
@@ -293,14 +295,14 @@
void add_action(std::size_t unique_id, std::size_t state, F act)
{
// If you see an error here stating add_action is not a member of
- // fusion::unused_type then you are probably having semantic actions
+ // fusion::unused_type then you are probably having semantic actions
// attached to at least one token in the lexer definition without
// using the lex::lexertl::actor_lexer<> as its base class.
typedef typename Functor::wrap_action_type wrapper_type;
if (state == all_states_id) {
// add the action to all known states
- typedef typename
- basic_rules_type::string_size_t_map::value_type
+ typedef typename
+ basic_rules_type::string_size_t_map::value_type
state_type;
std::size_t states = rules_.statemap().size();
@@ -320,8 +322,8 @@
// actions_.add_action(unique_id, add_state(state), wrapper_type::call(act));
// }
- // We do not minimize the state machine by default anymore because
- // Ben said: "If you can afford to generate a lexer at runtime, there
+ // We do not minimize the state machine by default anymore because
+ // Ben said: "If you can afford to generate a lexer at runtime, there
// is little point in calling minimise."
// Go figure.
bool init_dfa(bool minimize = false) const
@@ -356,26 +358,28 @@
mutable bool initialized_dfa_;
// generator functions must be able to access members directly
- template <typename Lexer, typename F>
- friend bool generate_static(Lexer const&, std::ostream&, char const*, F);
+ template <typename Lexer, typename F>
+ friend bool generate_static(Lexer const&
+ , std::basic_ostream<typename Lexer::char_type>&
+ , typename Lexer::char_type const*, F);
};
///////////////////////////////////////////////////////////////////////////
//
- // The actor_lexer class is another implementation of a Spirit.Lex
- // lexer on top of Ben Hanson's lexertl library as outlined above (For
- // more information about lexertl go here:
+ // The actor_lexer class is another implementation of a Spirit.Lex
+ // lexer on top of Ben Hanson's lexertl library as outlined above (For
+ // more information about lexertl go here:
// http://www.benhanson.net/lexertl.html).
//
// The only difference to the lexer class above is that token_def
- // definitions may have semantic (lexer) actions attached while being
+ // definitions may have semantic (lexer) actions attached while being
// defined:
//
// int w;
// token_def word = "[^ \t\n]+";
// self = word[++ref(w)]; // see example: word_count_lexer
//
- // This class is supposed to be used as the first and only template
+ // This class is supposed to be used as the first and only template
// parameter while instantiating instances of a lex::lexer class.
//
///////////////////////////////////////////////////////////////////////////
@@ -386,7 +390,7 @@
{
protected:
// Lexer instances can be created by means of a derived class only.
- actor_lexer(unsigned int flags)
+ actor_lexer(unsigned int flags)
: lexer<Token, Iterator, Functor>(flags) {}
};
Modified: trunk/boost/spirit/home/support/detail/lexer/rules.hpp
==============================================================================
--- trunk/boost/spirit/home/support/detail/lexer/rules.hpp (original)
+++ trunk/boost/spirit/home/support/detail/lexer/rules.hpp 2011-12-11 19:48:40 EST (Sun, 11 Dec 2011)
@@ -44,6 +44,16 @@
{
return "*";
}
+
+ static const char *char_name ()
+ {
+ return "char";
+ }
+
+ static const char *char_prefix ()
+ {
+ return "";
+ }
};
template <>
@@ -63,6 +73,16 @@
{
return L"*";
}
+
+ static const char *char_name ()
+ {
+ return "wchar_t";
+ }
+
+ static const char *char_prefix ()
+ {
+ return "L";
+ }
};
}
Modified: trunk/libs/spirit/test/Jamfile
==============================================================================
--- trunk/libs/spirit/test/Jamfile (original)
+++ trunk/libs/spirit/test/Jamfile 2011-12-11 19:48:40 EST (Sun, 11 Dec 2011)
@@ -297,6 +297,7 @@
[ run lex/regression_file_iterator2.cpp : : : : lex_regression_file_iterator2 ]
[ run lex/regression_file_iterator3.cpp : : : : lex_regression_file_iterator3 ]
[ run lex/regression_file_iterator4.cpp : : : : lex_regression_file_iterator4 ]
+ [ run lex/regression_static_wide_6253.cpp : : : : regression_static_wide_6253 ]
;
Added: trunk/libs/spirit/test/lex/regression_static_wide_6253.cpp
==============================================================================
--- (empty file)
+++ trunk/libs/spirit/test/lex/regression_static_wide_6253.cpp 2011-12-11 19:48:40 EST (Sun, 11 Dec 2011)
@@ -0,0 +1,44 @@
+// Copyright (c) 2001-2011 Hartmut Kaiser
+// Copyright (c) 2011 Ryan Molden
+//
+// Distributed under the Boost Software License, Version 1.0. (See accompanying
+// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+
+#include <boost/detail/lightweight_test.hpp>
+#include <boost/config/warning_disable.hpp>
+
+#include <boost/spirit/include/lex_lexertl.hpp>
+#include <boost/spirit/include/lex_generate_static_lexertl.hpp>
+#include <boost/spirit/include/lex_static_lexertl.hpp>
+
+#include <fstream>
+
+using namespace std;
+using namespace boost::spirit;
+
+template <typename BaseLexer>
+struct my_lexer : boost::spirit::lex::lexer<BaseLexer>
+{
+ my_lexer()
+ {
+ token = L"Yay winning!";
+ this->self = token;
+ }
+
+ lex::token_def<lex::unused_type, wchar_t> token;
+};
+
+int main(int argc, char* argv[])
+{
+ typedef lex::lexertl::token<wchar_t const*> token_type;
+ typedef lex::lexertl::lexer<token_type> lexer_type;
+
+ my_lexer<lexer_type> lexer;
+
+ basic_ofstream<wchar_t> output_dfa("test_dfa.hpp");
+ BOOST_TEST(lex::lexertl::generate_static_dfa(lexer, output_dfa, L"test_dfa"));
+
+ basic_ofstream<wchar_t> output_switch("test_switch.hpp");
+ BOOST_TEST(lex::lexertl::generate_static_switch(lexer, output_switch, L"test_switch"));
+ return boost::report_errors();
+}