Subject: [Boost-commit] svn:boost r69833 - in trunk: boost/spirit/home/lex/lexer libs/spirit/doc libs/spirit/test libs/spirit/test/lex
From: hartmut.kaiser_at_[hidden]
Date: 2011-03-10 17:29:44
Author: hkaiser
Date: 2011-03-10 17:29:40 EST (Thu, 10 Mar 2011)
New Revision: 69833
URL: http://svn.boost.org/trac/boost/changeset/69833
Log:
Spirit: adding possibility to specify token id with lex::char_ and lex::string
Added:
trunk/libs/spirit/test/lex/string_token_id.cpp (contents, props changed)
Text files modified:
trunk/boost/spirit/home/lex/lexer/char_token_def.hpp | 89 ++++++++++++++++++++++++++++++++-------
trunk/boost/spirit/home/lex/lexer/string_token_def.hpp | 56 +++++++++++++++++++-----
trunk/libs/spirit/doc/what_s_new.qbk | 8 +++
trunk/libs/spirit/test/Jamfile | 1
4 files changed, 124 insertions(+), 30 deletions(-)
Modified: trunk/boost/spirit/home/lex/lexer/char_token_def.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/char_token_def.hpp (original)
+++ trunk/boost/spirit/home/lex/lexer/char_token_def.hpp 2011-03-10 17:29:40 EST (Thu, 10 Mar 2011)
@@ -21,29 +21,42 @@
///////////////////////////////////////////////////////////////////////////
// Enablers
///////////////////////////////////////////////////////////////////////////
+
+ // enables 'x'
template <>
- struct use_terminal<lex::domain, char> // enables 'x'
+ struct use_terminal<lex::domain, char>
: mpl::true_ {};
+ // enables "x"
template <>
- struct use_terminal<lex::domain, char[2]> // enables "x"
+ struct use_terminal<lex::domain, char[2]>
: mpl::true_ {};
+ // enables wchar_t
template <>
- struct use_terminal<lex::domain, wchar_t> // enables wchar_t
+ struct use_terminal<lex::domain, wchar_t>
: mpl::true_ {};
+ // enables L"x"
template <>
- struct use_terminal<lex::domain, wchar_t[2]> // enables L"x"
+ struct use_terminal<lex::domain, wchar_t[2]>
: mpl::true_ {};
+ // enables char_('x'), char_("x")
template <typename CharEncoding, typename A0>
struct use_terminal<lex::domain
, terminal_ex<
- tag::char_code<tag::char_, CharEncoding> // enables char_('x'), char_("x")
- , fusion::vector1<A0>
- >
- > : mpl::true_ {};
+ tag::char_code<tag::char_, CharEncoding>
+ , fusion::vector1<A0> > >
+ : mpl::true_ {};
+
+ // enables char_('x', ID), char_("x", ID)
+ template <typename CharEncoding, typename A0, typename A1>
+ struct use_terminal<lex::domain
+ , terminal_ex<
+ tag::char_code<tag::char_, CharEncoding>
+ , fusion::vector2<A0, A1> > >
+ : mpl::true_ {};
}}
namespace boost { namespace spirit { namespace lex
@@ -60,14 +73,16 @@
// represents a single character token definition
//
///////////////////////////////////////////////////////////////////////////
- template <typename CharEncoding = char_encoding::standard>
+ template <typename CharEncoding = char_encoding::standard
+ , typename IdType = std::size_t>
struct char_token_def
- : primitive_lexer<char_token_def<CharEncoding> >
+ : primitive_lexer<char_token_def<CharEncoding, IdType> >
{
typedef typename CharEncoding::char_type char_type;
- char_token_def(char_type ch)
- : ch(ch), unique_id_(std::size_t(~0)), token_state_(std::size_t(~0))
+ char_token_def(char_type ch, IdType const& id)
+ : ch(ch), id_(id), unique_id_(std::size_t(~0))
+ , token_state_(std::size_t(~0))
{}
template <typename LexerDef, typename String>
@@ -90,18 +105,18 @@
lexdef.add_state(target);
token_state_ = state_id;
- unique_id_ = lexdef.add_token (state.c_str(), ch
- , static_cast<std::size_t>(ch), target);
+ unique_id_ = lexdef.add_token (state.c_str(), ch, id_, target);
}
template <typename LexerDef>
void add_actions(LexerDef&) const {}
- std::size_t id() const { return static_cast<std::size_t>(ch); }
+ IdType id() const { return id_; }
std::size_t unique_id() const { return unique_id_; }
std::size_t state() const { return token_state_; }
char_type ch;
+ mutable IdType id_;
mutable std::size_t unique_id_;
mutable std::size_t token_state_;
};
@@ -162,7 +177,8 @@
template <typename Terminal>
result_type operator()(Terminal const& term, unused_type) const
{
- return result_type(fusion::at_c<0>(term.args));
+ Char ch = fusion::at_c<0>(term.args);
+ return result_type(ch, ch);
}
};
@@ -180,7 +196,46 @@
template <typename Terminal>
result_type operator()(Terminal const& term, unused_type) const
{
- return result_type(fusion::at_c<0>(term.args)[0]);
+ Char ch = fusion::at_c<0>(term.args)[0];
+ return result_type(ch, ch);
+ }
+ };
+
+ // handle char_('x', ID)
+ template <typename CharEncoding, typename Modifiers, typename A0, typename A1>
+ struct make_primitive<
+ terminal_ex<
+ tag::char_code<tag::char_, CharEncoding>
+ , fusion::vector2<A0, A1>
+ >
+ , Modifiers>
+ {
+ typedef char_token_def<CharEncoding> result_type;
+
+ template <typename Terminal>
+ result_type operator()(Terminal const& term, unused_type) const
+ {
+ return result_type(
+ fusion::at_c<0>(term.args), fusion::at_c<1>(term.args));
+ }
+ };
+
+ // handle char_("x", ID)
+ template <typename CharEncoding, typename Modifiers, typename Char, typename A1>
+ struct make_primitive<
+ terminal_ex<
+ tag::char_code<tag::char_, CharEncoding>
+ , fusion::vector2<Char(&)[2], A1> // single char strings
+ >
+ , Modifiers>
+ {
+ typedef char_token_def<CharEncoding> result_type;
+
+ template <typename Terminal>
+ result_type operator()(Terminal const& term, unused_type) const
+ {
+ return result_type(
+ fusion::at_c<0>(term.args)[0], fusion::at_c<1>(term.args));
}
};
}}} // namespace boost::spirit::lex
Modified: trunk/boost/spirit/home/lex/lexer/string_token_def.hpp
==============================================================================
--- trunk/boost/spirit/home/lex/lexer/string_token_def.hpp (original)
+++ trunk/boost/spirit/home/lex/lexer/string_token_def.hpp 2011-03-10 17:29:40 EST (Thu, 10 Mar 2011)
@@ -17,6 +17,7 @@
#include <boost/spirit/home/lex/meta_compiler.hpp>
#include <boost/type_traits/add_const.hpp>
#include <boost/type_traits/add_reference.hpp>
+#include <boost/type_traits/is_integral.hpp>
#include <boost/type_traits/remove_const.hpp>
#include <boost/fusion/include/vector.hpp>
#include <boost/fusion/include/at.hpp>
@@ -26,18 +27,28 @@
///////////////////////////////////////////////////////////////////////////
// Enablers
///////////////////////////////////////////////////////////////////////////
+
+ // enables strings
template <typename T>
struct use_terminal<lex::domain, T
- , typename enable_if<traits::is_string<T> >::type> // enables strings
+ , typename enable_if<traits::is_string<T> >::type>
: mpl::true_ {};
+ // enables string(str)
template <typename CharEncoding, typename A0>
struct use_terminal<lex::domain
, terminal_ex<
- tag::char_code<tag::string, CharEncoding> // enables string(str)
- , fusion::vector1<A0> >
- > : traits::is_string<A0> {};
+ tag::char_code<tag::string, CharEncoding>
+ , fusion::vector1<A0> > >
+ : traits::is_string<A0> {};
+ // enables string(str, ID)
+ template <typename CharEncoding, typename A0, typename A1>
+ struct use_terminal<lex::domain
+ , terminal_ex<
+ tag::char_code<tag::string, CharEncoding>
+ , fusion::vector2<A0, A1> > >
+ : traits::is_string<A0> {};
}}
namespace boost { namespace spirit { namespace lex
@@ -54,17 +65,18 @@
// represents a string based token definition
//
///////////////////////////////////////////////////////////////////////////
- template <typename String, typename CharEncoding = char_encoding::standard>
+ template <typename String, typename IdType = std::size_t
+ , typename CharEncoding = char_encoding::standard>
struct string_token_def
- : primitive_lexer<string_token_def<String, CharEncoding> >
+ : primitive_lexer<string_token_def<String, IdType, CharEncoding> >
{
typedef typename
remove_const<typename traits::char_type_of<String>::type>::type
char_type;
typedef std::basic_string<char_type> string_type;
- string_token_def(typename add_reference<String>::type str)
- : str_(str), id_(std::size_t(~0)), unique_id_(std::size_t(~0))
+ string_token_def(typename add_reference<String>::type str, IdType const& id)
+ : str_(str), id_(id), unique_id_(std::size_t(~0))
, token_state_(std::size_t(~0))
{}
@@ -90,8 +102,8 @@
token_state_ = state_id;
typedef typename LexerDef::id_type id_type;
- if (std::size_t(~0) == id_)
- id_ = lexdef.get_next_id();
+ if (IdType(~0) == id_)
+ id_ = IdType(lexdef.get_next_id());
unique_id_ = lexdef.add_token (state.c_str(), str_, id_, target);
}
@@ -104,7 +116,7 @@
std::size_t state() const { return token_state_; }
string_type str_;
- mutable std::size_t id_;
+ mutable IdType id_;
mutable std::size_t unique_id_;
mutable std::size_t token_state_;
};
@@ -134,15 +146,33 @@
, Modifiers>
{
typedef typename add_const<A0>::type const_string;
- typedef string_token_def<const_string, CharEncoding> result_type;
+ typedef string_token_def<const_string, std::size_t, CharEncoding>
+ result_type;
template <typename Terminal>
result_type operator()(Terminal const& term, unused_type) const
{
- return result_type(fusion::at_c<0>(term.args));
+ return result_type(fusion::at_c<0>(term.args), std::size_t(~0));
}
};
+ template <typename Modifiers, typename CharEncoding, typename A0, typename A1>
+ struct make_primitive<
+ terminal_ex<
+ tag::char_code<tag::string, CharEncoding>
+ , fusion::vector2<A0, A1> >
+ , Modifiers>
+ {
+ typedef typename add_const<A0>::type const_string;
+ typedef string_token_def<const_string, A1, CharEncoding> result_type;
+
+ template <typename Terminal>
+ result_type operator()(Terminal const& term, unused_type) const
+ {
+ return result_type(
+ fusion::at_c<0>(term.args), fusion::at_c<1>(term.args));
+ }
+ };
}}} // namespace boost::spirit::lex
#endif
Modified: trunk/libs/spirit/doc/what_s_new.qbk
==============================================================================
--- trunk/libs/spirit/doc/what_s_new.qbk (original)
+++ trunk/libs/spirit/doc/what_s_new.qbk 2011-03-10 17:29:40 EST (Thu, 10 Mar 2011)
@@ -61,6 +61,12 @@
instantiates the terminals that he needs. So, instead of writing `using qi::uint_`
one writes instead: `qi::uint_type uint_`.
+[heading New Features in Lex]
+
+* Added the possibility to speciy a token id while creating a token definition
+ using `lex::char_` and `lex::string`. Both primitives now accept a second
+ parameter which will be interpreted as the requested token id for any token
+ generated from this definition.
[heading Making Stuff Work]
@@ -81,6 +87,8 @@
in order for the new behavior to kick in. By default, the old behavior
is still in place.
* Alternatives now support attribute compatibility.
+* Fixed Boost ticket #5246: mmap_file_iterator Fails to initialize correctly.
+* Fixed Boost ticket #5246: mmap_file_iterator Fails to initialize correctly.
[endsect]
Modified: trunk/libs/spirit/test/Jamfile
==============================================================================
--- trunk/libs/spirit/test/Jamfile (original)
+++ trunk/libs/spirit/test/Jamfile 2011-03-10 17:29:40 EST (Thu, 10 Mar 2011)
@@ -218,6 +218,7 @@
[ run lex/semantic_actions.cpp : : : : lex_semantic_actions ]
[ run lex/auto_switch_lexerstate.cpp : : : : lex_auto_switch_lexerstate ]
[ run lex/id_type_enum.cpp : : : : lex_id_type_enum ]
+ [ run lex/string_token_id.cpp : : : : lex_string_token_id ]
;
Added: trunk/libs/spirit/test/lex/string_token_id.cpp
==============================================================================
--- (empty file)
+++ trunk/libs/spirit/test/lex/string_token_id.cpp 2011-03-10 17:29:40 EST (Thu, 10 Mar 2011)
@@ -0,0 +1,89 @@
+// Copyright (c) 2001-2010 Hartmut Kaiser
+//
+// Distributed under the Boost Software License, Version 1.0. (See accompanying
+// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+
+// #define BOOST_SPIRIT_LEXERTL_DEBUG
+#define BOOST_VARIANT_MINIMIZE_SIZE
+
+#include <boost/detail/lightweight_test.hpp>
+#include <boost/config/warning_disable.hpp>
+
+#include <boost/spirit/include/qi.hpp>
+#include <boost/spirit/include/lex_lexertl.hpp>
+#include <boost/spirit/include/phoenix_operator.hpp>
+#include <boost/spirit/include/phoenix_statement.hpp>
+#include <boost/spirit/include/phoenix_container.hpp>
+
+#include <iostream>
+#include <string>
+
+namespace qi = boost::spirit::qi;
+namespace lex = boost::spirit::lex;
+
+enum tokenids
+{
+ IDWORD = lex::min_token_id,
+ IDCHAR,
+ IDANY
+};
+
+template <typename Lexer>
+struct word_count_tokens : lex::lexer<Lexer>
+{
+ word_count_tokens()
+ {
+ this->self.add_pattern
+ ("TEST", "A")
+ ;
+
+ this->self =
+ lex::string("{TEST}", IDWORD)
+ | lex::char_('a', IDCHAR)
+ | lex::string(".", IDANY)
+ ;
+ }
+};
+
+template <typename Iterator>
+struct word_count_grammar : qi::grammar<Iterator>
+{
+ template <typename TokenDef>
+ word_count_grammar(TokenDef const& tok)
+ : word_count_grammar::base_type(start)
+ , w(0), c(0), a(0)
+ {
+ using boost::phoenix::ref;
+ using qi::token;
+
+ start = *( token(IDWORD) [++ref(w)]
+ | token(IDCHAR) [++ref(c)]
+ | token(IDANY) [++ref(a)]
+ )
+ ;
+ }
+ std::size_t w, c, a;
+ qi::rule<Iterator> start;
+};
+
+
+int main()
+{
+ typedef lex::lexertl::token<
+ const char*, boost::mpl::vector<std::string>
+ > token_type;
+
+ typedef lex::lexertl::lexer<token_type> lexer_type;
+ typedef word_count_tokens<lexer_type>::iterator_type iterator_type;
+ word_count_tokens<lexer_type> word_count; // Our lexer
+ word_count_grammar<iterator_type> g (word_count); // Our parser
+
+ std::string str ("AaBCD");
+ char const* first = str.c_str();
+ char const* last = &first[str.size()];
+
+ BOOST_TEST(lex::tokenize_and_parse(first, last, word_count, g));
+ BOOST_TEST(g.w == 1 && g.c == 1 && g.a == 3);
+
+ return boost::report_errors();
+}