Subject: [Boost-commit] svn:boost r57561 - in trunk/boost/spirit/home: lex/lexer/lexertl support/detail/lexer
From: hartmut.kaiser_at_[hidden]
Date: 2009-11-10 22:55:35
Author: hkaiser
Date: 2009-11-10 22:55:34 EST (Tue, 10 Nov 2009)
New Revision: 57561
URL: http://svn.boost.org/trac/boost/changeset/57561
Log:
Spirit: started to integrate re2c style code generator for lexer
Added:
trunk/boost/spirit/home/support/detail/lexer/generate_re2c.hpp (contents, props changed)
Removed:
trunk/boost/spirit/home/lex/lexer/lexertl/generate_static.hpp
Text files modified:
trunk/boost/spirit/home/lex/lexer/lexertl/lexer.hpp | 17 +++++++++--------
trunk/boost/spirit/home/lex/lexer/lexertl/static_functor_data.hpp | 1 -
trunk/boost/spirit/home/lex/lexer/lexertl/static_lexer.hpp | 6 +++---
trunk/boost/spirit/home/support/detail/lexer/generate_cpp.hpp | 17 ++++++++---------
trunk/boost/spirit/home/support/detail/lexer/state_machine.hpp | 2 ++
5 files changed, 22 insertions(+), 21 deletions(-)
Deleted: trunk/boost/spirit/home/lex/lexer/lexertl/generate_static.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/lexertl/generate_static.hpp 2009-11-10 22:55:34 EST (Tue, 10 Nov 2009)
+++ (empty file)
@@ -1,670 +0,0 @@
-// Copyright (c) 2008 Ben Hanson
-// Copyright (c) 2008-2009 Hartmut Kaiser
-//
-// Distributed under the Boost Software License, Version 1.0. (See accompanying
-// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
-
-#if !defined(BOOST_SPIRIT_LEX_LEXERTL_GENERATE_CPP_FEB_10_2008_0855PM)
-#define BOOST_SPIRIT_LEX_LEXERTL_GENERATE_CPP_FEB_10_2008_0855PM
-
-#if defined(_MSC_VER)
-#pragma once
-#endif
-
-#include <boost/spirit/home/support/detail/lexer/char_traits.hpp>
-#include <boost/spirit/home/support/detail/lexer/consts.hpp>
-#include <boost/spirit/home/support/detail/lexer/rules.hpp>
-#include <boost/spirit/home/support/detail/lexer/size_t.hpp>
-#include <boost/spirit/home/support/detail/lexer/state_machine.hpp>
-#include <boost/spirit/home/lex/lexer/lexertl/static_version.hpp>
-#include <boost/algorithm/string.hpp>
-#include <boost/lexical_cast.hpp>
-
-///////////////////////////////////////////////////////////////////////////////
-namespace boost { namespace spirit { namespace lex { namespace lexertl
-{
- namespace detail
- {
-
- inline bool
- generate_delimiter(std::ostream &os_)
- {
- os_ << std::string(80, '/') << "\n";
- return os_.good();
- }
-
- ///////////////////////////////////////////////////////////////////////////
- // Generate a table of the names of the used lexer states, which is a bit
- // tricky, because the table stored with the rules is sorted based on the
- // names, but we need it sorted using the state ids.
- template <typename Char>
- inline bool
- generate_cpp_state_info (boost::lexer::basic_rules<Char> const& rules_
- , std::ostream &os_, char const* name_suffix = "")
- {
- // we need to re-sort the state names in ascending order of the state
- // ids, filling possible gaps in between later
- typedef typename
- boost::lexer::basic_rules<Char>::string_size_t_map::const_iterator
- state_iterator;
- typedef std::map<std::size_t, char const*> reverse_state_map_type;
-
- reverse_state_map_type reverse_state_map;
- state_iterator send = rules_.statemap().end();
- for (state_iterator sit = rules_.statemap().begin(); sit != send; ++sit)
- {
- typedef typename reverse_state_map_type::value_type value_type;
- reverse_state_map.insert(value_type((*sit).second, (*sit).first.c_str()));
- }
-
- generate_delimiter(os_);
- os_ << "// this table defines the names of the lexer states\n";
- os_ << "char const* const lexer_state_names"
- << (name_suffix[0] ? "_" : "") << name_suffix
- << "[" << rules_.statemap().size() << "] = \n{\n";
-
- typedef typename reverse_state_map_type::iterator iterator;
- iterator rend = reverse_state_map.end();
- std::size_t last_id = 0;
- for (iterator rit = reverse_state_map.begin(); rit != rend; ++last_id)
- {
- for (/**/; last_id < (*rit).first; ++last_id)
- {
- os_ << " 0, // \"<undefined state>\"\n";
- }
- os_ << " \"" << (*rit).second << "\"";
- if (++rit != rend)
- os_ << ",\n";
- else
- os_ << "\n"; // don't generate the final comma
- }
- os_ << "};\n\n";
-
- generate_delimiter(os_);
- os_ << "// this variable defines the number of lexer states\n";
- os_ << "std::size_t const lexer_state_count"
- << (name_suffix[0] ? "_" : "") << name_suffix
- << " = " << rules_.statemap().size() << ";\n\n";
- return os_.good();
- }
-
- inline bool
- generate_cpp_state_table (std::ostream &os_, char const* name_suffix
- , bool bol, bool eol)
- {
- std::string suffix(name_suffix[0] ? "_" : "");
- suffix += name_suffix;
-
- generate_delimiter(os_);
- os_ << "// this defines a generic accessors for the information above\n";
- os_ << "struct lexer" << suffix << "\n{\n";
- os_ << " // version number and feature-set of compatible static lexer engine\n";
- os_ << " enum\n";
- os_ << " {\n static_version = "
- << boost::lexical_cast<std::string>(SPIRIT_STATIC_LEXER_VERSION) << ",\n";
- os_ << " supports_bol = " << std::boolalpha << bol << ",\n";
- os_ << " supports_eol = " << std::boolalpha << eol << "\n";
- os_ << " };\n\n";
- os_ << " // return the number of lexer states\n";
- os_ << " static std::size_t const state_count()\n";
- os_ << " {\n return lexer_state_count" << suffix << "; \n }\n\n";
- os_ << " // return the name of the lexer state as given by 'idx'\n";
- os_ << " static char const* const state_name(std::size_t idx)\n";
- os_ << " {\n return lexer_state_names" << suffix << "[idx]; \n }\n\n";
- os_ << " // return the next matched token\n";
- os_ << " template<typename Iterator>\n";
- os_ << " static std::size_t next(std::size_t &start_state_, bool& bol_\n";
- os_ << " , Iterator &start_token_, Iterator const& end_, std::size_t& unique_id_)\n";
- os_ << " {\n return next_token" << suffix
- << "(start_state_, bol_, start_token_, end_, unique_id_);\n }\n";
- os_ << "};\n\n";
- return os_.good();
- }
-
- ///////////////////////////////////////////////////////////////////////////
- // Generate a tokenizer for the given state machine.
- template <typename Char>
- inline bool
- generate_cpp (boost::lexer::basic_state_machine<Char> const& sm_
- , boost::lexer::basic_rules<Char> const& rules_
- , std::ostream &os_, char const* name_suffix = ""
- , bool skip_on_nomatch = true, bool optimize_parameters = true)
- {
- if (sm_.data()._lookup->empty())
- return false;
-
- std::size_t const dfas_ = sm_.data()._dfa->size();
- std::size_t const lookups_ = sm_.data()._lookup->front()->size();
-
- os_ << "// Copyright (c) 2008-2009 Ben Hanson\n";
- os_ << "// Copyright (c) 2008-2009 Hartmut Kaiser\n";
- os_ << "//\n";
- os_ << "// Distributed under the Boost Software License, "
- "Version 1.0. (See accompanying\n";
- os_ << "// file licence_1_0.txt or copy at "
- "http://www.boost.org/LICENSE_1_0.txt)\n\n";
- os_ << "// Auto-generated by boost::lexer, do not edit\n\n";
-
- std::string guard(name_suffix);
- guard += name_suffix[0] ? "_" : "";
- guard += __DATE__ "_" __TIME__;
- std::string::size_type p = guard.find_first_of(": ");
- while (std::string::npos != p)
- {
- guard.replace(p, 1, "_");
- p = guard.find_first_of(": ", p);
- }
- boost::to_upper(guard);
-
- os_ << "#if !defined(BOOST_SPIRIT_LEXER_NEXT_TOKEN_" << guard << ")\n";
- os_ << "#define BOOST_SPIRIT_LEXER_NEXT_TOKEN_" << guard << "\n\n";
-
- os_ << "#include <boost/detail/iterator.hpp>\n";
- os_ << "#include <boost/spirit/home/support/detail/lexer/char_traits.hpp>\n\n";
-
- generate_delimiter(os_);
- os_ << "// the generated table of state names and the tokenizer have to be\n"
- "// defined in the boost::spirit::lex::lexertl::static_ namespace\n";
- os_ << "namespace boost { namespace spirit { namespace lex { "
- "namespace lexertl { namespace static_ {\n\n";
-
- // generate the lexer state information variables
- if (!generate_cpp_state_info(rules_, os_, name_suffix))
- return false;
-
- generate_delimiter(os_);
- os_ << "// this function returns the next matched token\n";
- os_ << "template<typename Iterator>\n";
- os_ << "std::size_t next_token" << (name_suffix[0] ? "_" : "")
- << name_suffix << " (";
-
- if (dfas_ > 1)
- {
- os_ << "std::size_t& start_state_, ";
- }
- else if (!optimize_parameters)
- {
- os_ << "std::size_t& /*start_state_*/, ";
- }
-
- if (sm_.data()._seen_BOL_assertion)
- {
- os_ << "bool& bol_, ";
- }
- else if (!optimize_parameters)
- {
- os_ << "bool& /*bol_*/, ";
- }
-
- if (dfas_ > 1 || sm_.data()._seen_BOL_assertion || !optimize_parameters)
- {
- os_ << "\n ";
- }
-
- os_ << "Iterator &start_token_, Iterator const& end_, ";
- os_ << "std::size_t& unique_id_)\n";
- os_ << "{\n";
- os_ << " enum {end_state_index, id_index, unique_id_index, "
- "state_index, bol_index,\n";
- os_ << " eol_index, dead_state_index, dfa_offset};\n\n";
- os_ << " static const std::size_t npos = static_cast"
- "<std::size_t>(~0);\n";
-
- if (dfas_ > 1)
- {
- for (std::size_t state_ = 0; state_ < dfas_; ++state_)
- {
- std::size_t i_ = 0;
- std::size_t j_ = 1;
- std::size_t count_ = lookups_ / 8;
- std::size_t const* lookup_ = &sm_.data()._lookup[state_]->front();
- std::size_t const* dfa_ = &sm_.data()._dfa[state_]->front();
-
- os_ << " static const std::size_t lookup" << state_
- << "_[" << lookups_ << "] = {\n ";
-
- for (/**/; i_ < count_; ++i_)
- {
- const std::size_t index_ = i_ * 8;
-
- os_ << lookup_[index_];
-
- for (; j_ < 8; ++j_)
- {
- os_ << ", " << lookup_[index_ + j_];
- }
-
- if (i_ < count_ - 1)
- {
- os_ << ",\n ";
- }
-
- j_ = 1;
- }
-
- os_ << " };\n";
- count_ = sm_.data()._dfa[state_]->size ();
- os_ << " static const std::size_t dfa" << state_ << "_[" <<
- count_ << "] = {\n ";
- count_ /= 8;
-
- for (i_ = 0; i_ < count_; ++i_)
- {
- const std::size_t index_ = i_ * 8;
-
- os_ << dfa_[index_];
-
- for (j_ = 1; j_ < 8; ++j_)
- {
- os_ << ", " << dfa_[index_ + j_];
- }
-
- if (i_ < count_ - 1)
- {
- os_ << ",\n ";
- }
- }
-
- std::size_t const mod_ = sm_.data()._dfa[state_]->size () % 8;
- if (mod_)
- {
- const std::size_t index_ = count_ * 8;
-
- if (count_)
- {
- os_ << ",\n ";
- }
-
- os_ << dfa_[index_];
-
- for (j_ = 1; j_ < mod_; ++j_)
- {
- os_ << ", " << dfa_[index_ + j_];
- }
- }
-
- os_ << " };\n";
- }
-
- std::size_t count_ = sm_.data()._dfa_alphabet.size();
- std::size_t i_ = 1;
-
- os_ << " static const std::size_t *lookup_arr_[" << count_ <<
- "] = { lookup0_";
-
- for (i_ = 1; i_ < count_; ++i_)
- {
- os_ << ", " << "lookup" << i_ << "_";
- }
-
- os_ << " };\n";
- os_ << " static const std::size_t dfa_alphabet_arr_[" <<
- count_ << "] = { ";
-
- os_ << sm_.data()._dfa_alphabet.front ();
-
- for (i_ = 1; i_ < count_; ++i_)
- {
- os_ << ", " << sm_.data()._dfa_alphabet[i_];
- }
-
- os_ << " };\n";
- os_ << " static const std::size_t *dfa_arr_[" << count_ <<
- "] = { ";
- os_ << "dfa0_";
-
- for (i_ = 1; i_ < count_; ++i_)
- {
- os_ << ", " << "dfa" << i_ << "_";
- }
-
- os_ << " };\n";
- }
- else
- {
- std::size_t const* lookup_ = &sm_.data()._lookup[0]->front();
- std::size_t const* dfa_ = &sm_.data()._dfa[0]->front();
- std::size_t i_ = 0;
- std::size_t j_ = 1;
- std::size_t count_ = lookups_ / 8;
-
- os_ << " static const std::size_t lookup_[";
- os_ << sm_.data()._lookup[0]->size() << "] = {\n ";
-
- for (; i_ < count_; ++i_)
- {
- const std::size_t index_ = i_ * 8;
-
- os_ << lookup_[index_];
-
- for (; j_ < 8; ++j_)
- {
- os_ << ", " << lookup_[index_ + j_];
- }
-
- if (i_ < count_ - 1)
- {
- os_ << ",\n ";
- }
-
- j_ = 1;
- }
-
- os_ << " };\n";
- os_ << " static const std::size_t dfa_alphabet_ = " <<
- sm_.data()._dfa_alphabet.front () << ";\n";
- os_ << " static const std::size_t dfa_[" <<
- sm_.data()._dfa[0]->size () << "] = {\n ";
- count_ = sm_.data()._dfa[0]->size () / 8;
-
- for (i_ = 0; i_ < count_; ++i_)
- {
- const std::size_t index_ = i_ * 8;
-
- os_ << dfa_[index_];
-
- for (j_ = 1; j_ < 8; ++j_)
- {
- os_ << ", " << dfa_[index_ + j_];
- }
-
- if (i_ < count_ - 1)
- {
- os_ << ",\n ";
- }
- }
-
- const std::size_t mod_ = sm_.data()._dfa[0]->size () % 8;
-
- if (mod_)
- {
- const std::size_t index_ = count_ * 8;
-
- if (count_)
- {
- os_ << ",\n ";
- }
-
- os_ << dfa_[index_];
-
- for (j_ = 1; j_ < mod_; ++j_)
- {
- os_ << ", " << dfa_[index_ + j_];
- }
- }
-
- os_ << " };\n";
- }
-
- os_ << "\n if (start_token_ == end_)\n";
- os_ << " {\n";
- os_ << " unique_id_ = boost::lexer::npos;\n";
- os_ << " return 0;\n";
- os_ << " }\n\n";
- if (sm_.data()._seen_BOL_assertion)
- {
- os_ << " bool bol = bol_;\n\n";
- }
-
- if (dfas_ > 1)
- {
- os_ << "again:\n";
- os_ << " const std::size_t * lookup_ = lookup_arr_[start_state_];\n";
- os_ << " std::size_t dfa_alphabet_ = dfa_alphabet_arr_[start_state_];\n";
- os_ << " const std::size_t *dfa_ = dfa_arr_[start_state_];\n";
- }
-
- os_ << " std::size_t const* ptr_ = dfa_ + dfa_alphabet_;\n";
- os_ << " Iterator curr_ = start_token_;\n";
- os_ << " bool end_state_ = *ptr_ != 0;\n";
- os_ << " std::size_t id_ = *(ptr_ + id_index);\n";
- os_ << " std::size_t uid_ = *(ptr_ + unique_id_index);\n";
- if (dfas_ > 1)
- {
- os_ << " std::size_t end_start_state_ = start_state_;\n";
- }
- if (sm_.data()._seen_BOL_assertion)
- {
- os_ << " bool end_bol_ = bol_;\n";
- }
- os_ << " Iterator end_token_ = start_token_;\n\n";
-
- os_ << " while (curr_ != end_)\n";
- os_ << " {\n";
-
- if (sm_.data()._seen_BOL_assertion)
- {
- os_ << " std::size_t const BOL_state_ = ptr_[bol_index];\n\n";
- }
-
- if (sm_.data()._seen_EOL_assertion)
- {
- os_ << " std::size_t const EOL_state_ = ptr_[eol_index];\n\n";
- }
-
- if (sm_.data()._seen_BOL_assertion && sm_.data()._seen_EOL_assertion)
- {
- os_ << " if (BOL_state_ && bol)\n";
- os_ << " {\n";
- os_ << " ptr_ = &dfa_[BOL_state_ * dfa_alphabet_];\n";
- os_ << " }\n";
- os_ << " else if (EOL_state_ && *curr_ == '\\n')\n";
- os_ << " {\n";
- os_ << " ptr_ = &dfa_[EOL_state_ * dfa_alphabet_];\n";
- os_ << " }\n";
- os_ << " else\n";
- os_ << " {\n";
- if (lookups_ == 256)
- {
- os_ << " unsigned char index = \n";
- os_ << " static_cast<unsigned char>(*curr_++);\n";
- }
- else
- {
- os_ << " std::size_t index = *curr_++\n";
- }
- os_ << " bol = (index == '\n') ? true : false;\n";
- os_ << " std::size_t const state_ = ptr_[\n";
- os_ << " lookup_[static_cast<std::size_t>(index)]];\n";
-
- os_ << '\n';
- os_ << " if (state_ == 0) break;\n";
- os_ << '\n';
- os_ << " ptr_ = &dfa_[state_ * dfa_alphabet_];\n";
- os_ << " }\n\n";
- }
- else if (sm_.data()._seen_BOL_assertion)
- {
- os_ << " if (BOL_state_ && bol)\n";
- os_ << " {\n";
- os_ << " ptr_ = &dfa_[BOL_state_ * dfa_alphabet_];\n";
- os_ << " }\n";
- os_ << " else\n";
- os_ << " {\n";
- if (lookups_ == 256)
- {
- os_ << " unsigned char index = \n";
- os_ << " static_cast<unsigned char>(*curr_++);\n";
- }
- else
- {
- os_ << " std::size_t index = *curr_++\n";
- }
- os_ << " bol = (index == '\n') ? true : false;\n";
- os_ << " std::size_t const state_ = ptr_[\n";
- os_ << " lookup_[static_cast<std::size_t>(index)]];\n";
-
- os_ << '\n';
- os_ << " if (state_ == 0) break;\n";
- os_ << '\n';
- os_ << " ptr_ = &dfa_[state_ * dfa_alphabet_];\n";
- os_ << " }\n\n";
- }
- else if (sm_.data()._seen_EOL_assertion)
- {
- os_ << " if (EOL_state_ && *curr_ == '\\n')\n";
- os_ << " {\n";
- os_ << " ptr_ = &dfa_[EOL_state_ * dfa_alphabet_];\n";
- os_ << " }\n";
- os_ << " else\n";
- os_ << " {\n";
- if (lookups_ == 256)
- {
- os_ << " unsigned char index = \n";
- os_ << " static_cast<unsigned char>(*curr_++);\n";
- }
- else
- {
- os_ << " std::size_t index = *curr_++\n";
- }
- os_ << " bol = (index == '\n') ? true : false;\n";
- os_ << " std::size_t const state_ = ptr_[\n";
- os_ << " lookup_[static_cast<std::size_t>(index)]];\n";
-
- os_ << '\n';
- os_ << " if (state_ == 0) break;\n";
- os_ << '\n';
- os_ << " ptr_ = &dfa_[state_ * dfa_alphabet_];\n";
- os_ << " }\n\n";
- }
- else
- {
- os_ << " std::size_t const state_ =\n";
-
- if (lookups_ == 256)
- {
- os_ << " ptr_[lookup_[static_cast<unsigned char>"
- "(*curr_++)]];\n";
- }
- else
- {
- os_ << " ptr_[lookup_[*curr_++]];\n";
- }
-
- os_ << '\n';
- os_ << " if (state_ == 0) break;\n";
- os_ << '\n';
- os_ << " ptr_ = &dfa_[state_ * dfa_alphabet_];\n\n";
- }
-
- os_ << " if (*ptr_)\n";
- os_ << " {\n";
- os_ << " end_state_ = true;\n";
- os_ << " id_ = *(ptr_ + id_index);\n";
- os_ << " uid_ = *(ptr_ + unique_id_index);\n";
- if (dfas_ > 1)
- {
- os_ << " end_start_state_ = *(ptr_ + state_index);\n";
- }
- if (sm_.data()._seen_BOL_assertion)
- {
- os_ << " end_bol_ = bol;\n";
- }
- os_ << " end_token_ = curr_;\n";
- os_ << " }\n";
- os_ << " }\n\n";
-
- if (sm_.data()._seen_EOL_assertion)
- {
- os_ << " const std::size_t EOL_state_ = ptr_[eol_index];\n\n";
-
- os_ << " if (EOL_state_ && curr_ == end_)\n";
- os_ << " {\n";
- os_ << " ptr_ = &dfa_[EOL_state_ * dfa_alphabet_];\n\n";
-
- os_ << " if (*ptr_)\n";
- os_ << " {\n";
- os_ << " end_state_ = true;\n";
- os_ << " id_ = *(ptr_ + id_index);\n";
- os_ << " uid_ = *(ptr_ + unique_id_index);\n";
- if (dfas_ > 1)
- {
- os_ << " end_start_state_ = *(ptr_ + state_index);\n";
- }
- if (sm_.data()._seen_BOL_assertion)
- {
- os_ << " end_bol_ = bol;\n";
- }
- os_ << " end_token_ = curr_;\n";
- os_ << " }\n";
- os_ << " }\n\n";
- }
-
- os_ << " if (end_state_)\n";
- os_ << " {\n";
- os_ << " // return longest match\n";
- os_ << " start_state_ = end_start_state_;\n";
- os_ << " start_token_ = end_token_;\n";
-
- if (dfas_ > 1)
- {
- os_ << " if (id_ == 0)\n";
- os_ << " {\n";
- if (sm_.data()._seen_BOL_assertion)
- {
- os_ << " bol_ = end_bol_;\n";
- }
- os_ << " goto again;\n";
- os_ << " }\n";
- if (sm_.data()._seen_BOL_assertion)
- {
- os_ << " else\n";
- os_ << " {\n";
- os_ << " bol_ = end_bol_;\n";
- os_ << " }\n";
- }
- }
-
- os_ << " }\n";
- os_ << " else\n";
- os_ << " {\n";
-
- if (sm_.data()._seen_BOL_assertion)
- {
- os_ << " bol_ = (*start_token_ == '\n') ? true : false;\n";
- }
- if (skip_on_nomatch)
- {
- os_ << " // No match causes char to be skipped\n";
- os_ << " ++start_token_;\n";
- }
-
- os_ << " id_ = npos;\n";
- os_ << " uid_ = npos;\n";
- os_ << " }\n\n";
-
- os_ << " unique_id_ = uid_;\n";
- os_ << " return id_;\n";
- os_ << "}\n\n";
-
- if (!generate_cpp_state_table(os_, name_suffix
- , sm_.data()._seen_BOL_assertion, sm_.data()._seen_EOL_assertion))
- {
- return false;
- }
-
- os_ << "}}}}} // namespace boost::spirit::lex::lexertl::static_\n\n";
-
- os_ << "#endif\n";
-
- return os_.good();
- }
-
- } // namespace detail
-
- ///////////////////////////////////////////////////////////////////////////
- template <typename Lexer>
- inline bool
- generate_static(Lexer const& lexer, std::ostream& os
- , char const* name_suffix = "")
- {
- if (!lexer.init_dfa())
- return false;
- return detail::generate_cpp(lexer.state_machine_, lexer.rules_, os
- , name_suffix, false, false);
- }
-
-///////////////////////////////////////////////////////////////////////////////
-}}}}
-
-#endif
Modified: trunk/boost/spirit/home/lex/lexer/lexertl/lexer.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/lexertl/lexer.hpp (original)
+++ trunk/boost/spirit/home/lex/lexer/lexertl/lexer.hpp 2009-11-10 22:55:34 EST (Tue, 10 Nov 2009)
@@ -90,8 +90,8 @@
}
///////////////////////////////////////////////////////////////////////////
- template <typename Lexer>
- bool generate_static(Lexer const& lex, std::ostream& os, char const* name);
+ template <typename Lexer, typename F>
+ bool generate_static(Lexer const&, std::ostream&, char const*, F);
///////////////////////////////////////////////////////////////////////////
//
@@ -169,10 +169,10 @@
typedef typename Functor::semantic_actions_type semantic_actions_type;
iterator_data_type(
- boost::lexer::basic_state_machine<char_type> const& state_machine
- , boost::lexer::basic_rules<char_type> const& rules
- , semantic_actions_type const& actions)
- : state_machine_(state_machine), rules_(rules), actions_(actions)
+ boost::lexer::basic_state_machine<char_type> const& sm
+ , boost::lexer::basic_rules<char_type> const& rules
+ , semantic_actions_type const& actions)
+ : state_machine_(sm), rules_(rules), actions_(actions)
{}
boost::lexer::basic_state_machine<char_type> const& state_machine_;
@@ -294,8 +294,9 @@
typename Functor::semantic_actions_type actions_;
mutable bool initialized_dfa_;
- template <typename Lexer>
- friend bool generate_static(Lexer const&, std::ostream&, char const*);
+ // generator functions must be able to access members directly
+ template <typename Lexer, typename F>
+ friend bool generate_static(Lexer const&, std::ostream&, char const*, F);
};
///////////////////////////////////////////////////////////////////////////
Modified: trunk/boost/spirit/home/lex/lexer/lexertl/static_functor_data.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/lexertl/static_functor_data.hpp (original)
+++ trunk/boost/spirit/home/lex/lexer/lexertl/static_functor_data.hpp 2009-11-10 22:55:34 EST (Tue, 10 Nov 2009)
@@ -61,7 +61,6 @@
, std::size_t> wrap_action_type;
typedef std::size_t (*next_token_functor)(std::size_t&,
-as Iterator const&, Iterator&, Iterator const&, std::size_t&);
bool&, Iterator&, Iterator const&, std::size_t&);
typedef char_type const* (*get_state_name_type)(std::size_t);
Modified: trunk/boost/spirit/home/lex/lexer/lexertl/static_lexer.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/lexertl/static_lexer.hpp (original)
+++ trunk/boost/spirit/home/lex/lexer/lexertl/static_lexer.hpp 2009-11-10 22:55:34 EST (Tue, 10 Nov 2009)
@@ -138,9 +138,9 @@
typedef typename Functor::get_state_name_type get_state_name_type;
iterator_data_type(next_token_functor next
- , semantic_actions_type const& actions
- , get_state_name_type get_state_name, std::size_t num_states
- , bool bol)
+ , semantic_actions_type const& actions
+ , get_state_name_type get_state_name, std::size_t num_states
+ , bool bol)
: next_(next), actions_(actions), get_state_name_(get_state_name)
, num_states_(num_states), bol_(bol)
{}
Modified: trunk/boost/spirit/home/support/detail/lexer/generate_cpp.hpp
==============================================================================
--- trunk/boost/spirit/home/support/detail/lexer/generate_cpp.hpp (original)
+++ trunk/boost/spirit/home/support/detail/lexer/generate_cpp.hpp 2009-11-10 22:55:34 EST (Tue, 10 Nov 2009)
@@ -1,10 +1,10 @@
-// generate_cpp_code.hpp
+// generate_cpp.hpp
// Copyright (c) 2008-2009 Ben Hanson (http://www.benhanson.net/)
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file licence_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
-#ifndef BOOST_LEXER_GENERATE_CPP_CODE_HPP
-#define BOOST_LEXER_GENERATE_CPP_CODE_HPP
+#ifndef BOOST_LEXER_GENERATE_CPP_HPP
+#define BOOST_LEXER_GENERATE_CPP_HPP
#include "char_traits.hpp"
#include "consts.hpp"
@@ -551,15 +551,14 @@
os_ << " else\n";
os_ << " {\n";
+ if (sm_._seen_BOL_assertion)
+ {
+ os_ << " beg_of_line_ = *start_token_ == '\\n';\n";
+ }
+
if (skip_unknown_)
{
os_ << " // No match causes char to be skipped\n";
-
- if (sm_._seen_BOL_assertion)
- {
- os_ << " beg_of_line_ = *start_token_ == '\\n';\n";
- }
-
os_ << " ++start_token_;\n";
}
Added: trunk/boost/spirit/home/support/detail/lexer/generate_re2c.hpp
==============================================================================
--- (empty file)
+++ trunk/boost/spirit/home/support/detail/lexer/generate_re2c.hpp 2009-11-10 22:55:34 EST (Tue, 10 Nov 2009)
@@ -0,0 +1,436 @@
+// generate_re2c.hpp
+// Copyright (c) 2009 Ben Hanson (http://www.benhanson.net/)
+//
+// Distributed under the Boost Software License, Version 1.0. (See accompanying
+// file licence_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+#ifndef LEXERTL_GENERATE_RE2C_HPP
+#define LEXERTL_GENERATE_RE2C_HPP
+
+#include "char_traits.hpp"
+#include "consts.hpp"
+#include "internals.hpp"
+#include <iostream>
+#include "runtime_error.hpp"
+#include "size_t.hpp"
+#include "state_machine.hpp"
+#include <vector>
+
+namespace boost
+{
+namespace lexer
+{
+inline std::string get_charcode(char ch)
+{
+ std::string result;
+ switch(ch) {
+ case '\t':
+ result = "\\t";
+ break;
+ case '\b':
+ result = "\\b";
+ break;
+ case '\r':
+ result = "\\r";
+ break;
+ case '\n':
+ result = "\\n";
+ break;
+ case '\f':
+ result = "\\f";
+ break;
+ case '\v':
+ result = "\\v";
+ break;
+ case '\\':
+ result = "\\\\";
+ break;
+ case '\'':
+ result = "\\'";
+ break;
+ default:
+ if (std::isprint(ch))
+ {
+ result = ch;
+ }
+ else
+ {
+ result = "\\x";
+ char buffer[3];
+ result += ltoa(ch, buffer, 16);
+ }
+ break;
+ }
+ return result;
+}
+
+inline std::basic_string<wchar_t> get_charcode(wchar_t ch)
+{
+ // not implemented yet
+ std::basic_string<wchar_t> result;
+ result = ch;
+ return result;
+}
+
+template<typename CharT>
+void generate_re2c (const basic_state_machine<CharT> &state_machine_,
+ std::ostream &os_, const bool use_pointers_ = false,
+ const bool skip_unknown_ = true, const bool optimise_parameters_ = true,
+ const char *name_ = "next_token")
+{
+ const detail::internals &sm_ = state_machine_.data ();
+
+ if (sm_._lookup->size () == 0)
+ {
+ throw runtime_error ("Cannot generate code from an empty "
+ "state machine");
+ }
+
+ std::string upper_name_ (__DATE__);
+ const std::size_t lookups_ = sm_._lookup->front ()->size ();
+ typename boost::lexer::basic_state_machine<CharT>::iterator iter_ =
+ state_machine_.begin();
+ typename boost::lexer::basic_state_machine<CharT>::iterator end_ =
+ state_machine_.end();
+ const std::size_t dfas_ = sm_._dfa->size ();
+ std::string::size_type pos_ = upper_name_.find (' ');
+ const char *iterator_ = 0;
+
+ if (use_pointers_)
+ {
+ if (lookups_ == 256)
+ {
+ iterator_ = "const char *";
+ }
+ else
+ {
+ iterator_ = "const wchar_t *";
+ }
+ }
+ else
+ {
+ iterator_ = "Iterator &";
+ }
+
+ while (pos_ != std::string::npos)
+ {
+ upper_name_.replace (pos_, 1, "_");
+ pos_ = upper_name_.find (' ', pos_);
+ }
+
+ upper_name_ += '_';
+ upper_name_ += __TIME__;
+
+ pos_ = upper_name_.find (':');
+
+ while (pos_ != std::string::npos)
+ {
+ upper_name_.erase (pos_, 1);
+ pos_ = upper_name_.find (':', pos_);
+ }
+
+ upper_name_ = '_' + upper_name_;
+ upper_name_ = name_ + upper_name_;
+ std::transform (upper_name_.begin (), upper_name_.end (),
+ upper_name_.begin (), ::toupper);
+ os_ << "#ifndef " << upper_name_ + '\n';
+ os_ << "#define " << upper_name_ + '\n';
+ os_ << "// Copyright (c) 2008-2009 Ben Hanson\n";
+ os_ << "//\n";
+ os_ << "// Distributed under the Boost Software License, "
+ "Version 1.0. (See accompanying\n";
+ os_ << "// file licence_1_0.txt or copy at "
+ "http://www.boost.org/LICENSE_1_0.txt)\n\n";
+ os_ << "// Auto-generated by boost::lexer\n";
+ os_ << "template<typename Iterator>\n";
+ os_ << "std::size_t " << name_ << " (";
+
+ if (dfas_ > 1 || !optimise_parameters_)
+ {
+ os_ << "std::size_t &start_state_, ";
+ }
+
+ if (use_pointers_)
+ {
+ os_ << iterator_ << " &";
+ }
+ else
+ {
+ os_ << iterator_;
+ }
+
+ os_ << "start_token_, ";
+
+ if (use_pointers_)
+ {
+ os_ << iterator_ << " const ";
+ }
+ else
+ {
+ os_ << "const " << iterator_;
+ }
+
+ os_ << "end_, \n";
+ os_ << " std::size_t &unique_id_";
+
+ if (sm_._seen_BOL_assertion || !optimise_parameters_)
+ {
+ os_ << ", bool &beg_of_line_";
+ }
+
+ os_ << ")\n";
+ os_ << "{\n";
+ os_ << " static const std::size_t npos = static_cast"
+ "<std::size_t>(~0);\n";
+ os_ << "\n if (start_token_ == end_)\n";
+ os_ << " {\n";
+ os_ << " unique_id_ = npos;\n";
+ os_ << " return 0;\n";
+ os_ << " }\n\n";
+
+ if (dfas_ > 1)
+ {
+ os_ << "again:\n";
+ }
+
+ os_ << " Iterator curr_ = start_token_;\n";
+ os_ << " bool end_state_ = false;\n";
+ os_ << " std::size_t id_ = npos;\n";
+ os_ << " std::size_t uid_ = npos;\n";
+
+ if (dfas_ > 1)
+ {
+ os_ << " std::size_t end_start_state_ = start_state_;\n";
+ }
+
+ if (sm_._seen_BOL_assertion)
+ {
+ os_ << " bool bol_ = beg_of_line_;\n";
+ os_ << " bool end_bol_ = bol_;\n";
+ }
+
+ os_ << " Iterator end_token_ = start_token_;\n";
+ os_ << '\n';
+
+ if (dfas_ > 1)
+ {
+ os_ << " switch (start_state_)\n";
+ os_ << " {\n";
+
+ for (std::size_t i_ = 0; i_ < dfas_; ++i_)
+ {
+ os_ << " case " << i_ << ":\n";
+ os_ << " goto " << i_ << "_0;\n";
+ os_ << " // Not needed, but to prevent warnings\n";
+ os_ << " break;\n";
+ }
+
+ os_ << " default:\n";
+ os_ << " break;\n";
+ os_ << " }\n\n";
+ }
+
+ os_ << " ";
+
+ if (lookups_ == 256)
+ {
+ os_ << "char";
+ }
+ else
+ {
+ os_ << "wchar_t";
+ }
+
+ os_ << " ch_ = 0;\n\n";
+
+ for (std::size_t dfa_ = 0; dfa_ < dfas_; ++dfa_)
+ {
+ const std::size_t states_ = iter_->states;
+
+ for (std::size_t state_ = 0; state_ < states_; ++state_)
+ {
+ const std::size_t transitions_ = iter_->transitions;
+ std::size_t t_ = 0;
+
+ if (dfas_ > 1 || dfa_ != 0 || state_ != 0)
+ {
+ os_ << "state" << dfa_ << '_' << state_ << ":\n";
+ }
+ if (iter_->end_state)
+ {
+ os_ << " end_state_ = true;\n";
+ os_ << " id_ = " << iter_->id << ";\n";
+ os_ << " uid_ = " << iter_->unique_id << ";\n";
+ os_ << " end_token_ = curr_;\n";
+
+ if (dfas_ > 1)
+ {
+ os_ << " end_start_state_ = " << iter_->goto_dfa <<
+ ";\n";
+ }
+
+ if (sm_._seen_BOL_assertion)
+ {
+ os_ << " end_bol_ = bol_;\n";
+ }
+ }
+
+ os_ << " if (curr_ == end_) goto end;\n";
+ os_ << " ch_ = *curr_;\n";
+
+ if (iter_->bol_index != boost::lexer::npos)
+ {
+ os_ << "\n if (bol_) goto state" << dfa_ << '_' <<
+ iter_->bol_index << ";\n\n";
+ }
+
+ if (iter_->eol_index != boost::lexer::npos)
+ {
+ os_ << "\n if (ch_ == '\n') goto state" << dfa_ << '_' <<
+ iter_->eol_index << ";\n\n";
+ }
+
+ os_ << " ++curr_;\n";
+
+ for (; t_ < transitions_; ++t_)
+ {
+ const char *ptr_ = iter_->token._charset.c_str();
+ const char *end_ = ptr_ + iter_->token._charset.size();
+ char start_char_ = 0;
+ char curr_char_ = 0;
+ bool range_ = false;
+ bool first_char_ = true;
+
+ os_ << "\n if (";
+
+ while (ptr_ != end_)
+ {
+ curr_char_ = *ptr_++;
+
+ if (*ptr_ == curr_char_ + 1)
+ {
+ if (!range_)
+ {
+ start_char_ = curr_char_;
+ }
+
+ range_ = true;
+ }
+ else
+ {
+ if (!first_char_)
+ {
+ os_ << " || ";
+ }
+
+ first_char_ = false;
+
+ if (range_)
+ {
+ if (iter_->token._negated)
+ {
+ os_ << "!";
+ }
+
+ os_ << "(ch_ >= '" << get_charcode(start_char_);
+ os_ << "' && ch_ <= '" << get_charcode(curr_char_) << "\')";
+ range_ = false;
+ }
+ else
+ {
+ os_ << "ch_ ";
+
+ if (iter_->token._negated)
+ {
+ os_ << "!=";
+ }
+ else
+ {
+ os_ << "==";
+ }
+
+ os_ << " '" << get_charcode(curr_char_) << "'";
+ }
+ }
+ }
+
+ os_ << ") goto state" << dfa_ << '_' << iter_->goto_state <<
+ ";\n";
+ ++iter_;
+ }
+
+ if (transitions_) os_ << '\n';
+
+ os_ << " goto end;\n";
+ os_ << '\n';
+
+ if (transitions_ == 0) ++iter_;
+ }
+ }
+
+ os_ << "end:\n";
+ os_ << " if (end_state_)\n";
+ os_ << " {\n";
+ os_ << " // return longest match\n";
+
+ if (dfas_ > 1)
+ {
+ os_ << " start_state_ = end_start_state_;\n";
+ }
+
+ if (sm_._seen_BOL_assertion && dfas_ < 2)
+ {
+ os_ << " beg_of_line_ = end_bol_;\n";
+ }
+
+ os_ << " start_token_ = end_token_;\n";
+
+ if (dfas_ > 1)
+ {
+ os_ << '\n';
+ os_ << " if (id_ == 0)\n";
+ os_ << " {\n";
+
+ if (sm_._seen_BOL_assertion)
+ {
+ os_ << " bol_ = end_bol_;\n";
+ }
+
+ os_ << " goto again;\n";
+ os_ << " }\n";
+
+ if (sm_._seen_BOL_assertion)
+ {
+ os_ << " else\n";
+ os_ << " {\n";
+ os_ << " beg_of_line_ = end_bol_;\n";
+ os_ << " }\n";
+ }
+ }
+
+ os_ << " }\n";
+ os_ << " else\n";
+ os_ << " {\n";
+
+ if (sm_._seen_BOL_assertion)
+ {
+ os_ << " beg_of_line_ = *start_token_ == '\\n';\n";
+ }
+
+ if (skip_unknown_)
+ {
+ os_ << " // No match causes char to be skipped\n";
+ os_ << " ++start_token_;\n";
+ }
+
+ os_ << " id_ = npos;\n";
+ os_ << " uid_ = npos;\n";
+ os_ << " }\n";
+ os_ << '\n';
+ os_ << " unique_id_ = uid_;\n";
+ os_ << " return id_;\n";
+ os_ << "}\n";
+ os_ << "\n#endif\n";
+}
+}
+}
+#endif
Modified: trunk/boost/spirit/home/support/detail/lexer/state_machine.hpp
==============================================================================
--- trunk/boost/spirit/home/support/detail/lexer/state_machine.hpp (original)
+++ trunk/boost/spirit/home/support/detail/lexer/state_machine.hpp 2009-11-10 22:55:34 EST (Tue, 10 Nov 2009)
@@ -24,6 +24,8 @@
class basic_state_machine
{
public:
+ typedef CharT char_type;
+
class iterator
{
public: