Subject: [Boost-commit] svn:boost r60740 - in trunk/libs/spirit/example/scheme: . detail
From: joel_at_[hidden]
Date: 2010-03-20 22:59:41
Author: djowel
Date: 2010-03-20 22:59:40 EDT (Sat, 20 Mar 2010)
New Revision: 60740
URL: http://svn.boost.org/trac/boost/changeset/60740
Log:
Straight UTF-8 implementation
Text files modified:
trunk/libs/spirit/example/scheme/detail/utree_detail1.hpp | 3 ++
trunk/libs/spirit/example/scheme/detail/utree_detail2.hpp | 16 +++++++++++++-
trunk/libs/spirit/example/scheme/out.txt | 2
trunk/libs/spirit/example/scheme/sexpr.hpp | 42 ++++++++++++++++++---------------------
trunk/libs/spirit/example/scheme/sexpr_test.cpp | 2
trunk/libs/spirit/example/scheme/utree.hpp | 8 +++++++
6 files changed, 46 insertions(+), 27 deletions(-)
Modified: trunk/libs/spirit/example/scheme/detail/utree_detail1.hpp
==============================================================================
--- trunk/libs/spirit/example/scheme/detail/utree_detail1.hpp (original)
+++ trunk/libs/spirit/example/scheme/detail/utree_detail1.hpp 2010-03-20 22:59:40 EDT (Sat, 20 Mar 2010)
@@ -115,6 +115,9 @@
utree_type::info get_type() const;
void set_type(utree_type::info t);
+ int get_subtype() const;
+ void set_subtype(int t);
+
std::size_t size() const;
char const* str() const;
Modified: trunk/libs/spirit/example/scheme/detail/utree_detail2.hpp
==============================================================================
--- trunk/libs/spirit/example/scheme/detail/utree_detail2.hpp (original)
+++ trunk/libs/spirit/example/scheme/detail/utree_detail2.hpp 2010-03-20 22:59:40 EDT (Sat, 20 Mar 2010)
@@ -11,12 +11,24 @@
{
inline utree_type::info fast_string::get_type() const
{
- return static_cast<utree_type::info>(buff[small_string_size]);
+ return static_cast<utree_type::info>(buff[small_string_size] & 0x7);
}
inline void fast_string::set_type(utree_type::info t)
{
- buff[small_string_size] = static_cast<char>(t);
+ buff[small_string_size] = static_cast<char>(t)
+ | (buff[small_string_size] & 0xF8);
+ }
+
+ inline int fast_string::get_subtype() const
+ {
+ return buff[small_string_size] >> 3;
+ }
+
+ inline void fast_string::set_subtype(int t)
+ {
+ buff[small_string_size] = (t << 3)
+ | (buff[small_string_size] & 0x7);
}
inline std::size_t fast_string::size() const
Modified: trunk/libs/spirit/example/scheme/out.txt
==============================================================================
--- trunk/libs/spirit/example/scheme/out.txt (original)
+++ trunk/libs/spirit/example/scheme/out.txt 2010-03-20 22:59:40 EDT (Sat, 20 Mar 2010)
@@ -1,2 +1,2 @@
-success: (123.45 true false 255 63 "this is a € string" "Τη γλώσσα μου Îδωσαν ελληνική" b0123456789abcdef0123456789abcdef (92 ("another string" apple Sîne)))
+success: (123.45 true false 255 63 "this is a € string" "Τη γλώσσα μου Îδωσαν ελληνική" b0123456789abcdef123456789abcdef (92 ("another string" apple Sîne)))
Modified: trunk/libs/spirit/example/scheme/sexpr.hpp
==============================================================================
--- trunk/libs/spirit/example/scheme/sexpr.hpp (original)
+++ trunk/libs/spirit/example/scheme/sexpr.hpp 2010-03-20 22:59:40 EDT (Sat, 20 Mar 2010)
@@ -9,8 +9,6 @@
#include <string>
-#define BOOST_SPIRIT_UNICODE // We'll use unicode (UTF8) all throughout
-
#include <boost/config/warning_disable.hpp>
#include <boost/cstdint.hpp>
#include <boost/spirit/include/qi.hpp>
@@ -24,9 +22,8 @@
namespace scheme
{
- using boost::spirit::unicode::char_;
- using boost::spirit::unicode::space;
- using boost::spirit::unicode::print;
+ using boost::spirit::ascii::char_;
+ using boost::spirit::ascii::space;
using boost::spirit::qi::grammar;
using boost::spirit::qi::rule;
using boost::spirit::qi::eol;
@@ -46,11 +43,10 @@
using boost::spirit::qi::lit;
using boost::phoenix::function;
- typedef boost::spirit::char_encoding::unicode unicode;
typedef boost::uint32_t uchar; // a unicode code point
template <typename Iterator>
- struct white_space : grammar<Iterator, unicode>
+ struct white_space : grammar<Iterator>
{
white_space() : white_space::base_type(start)
{
@@ -60,7 +56,7 @@
;
}
- rule<Iterator, unicode> start;
+ rule<Iterator> start;
};
namespace detail
@@ -79,17 +75,17 @@
}
};
- struct push_symbol_utf8
+ struct push_symbol
{
template <typename S, typename C>
struct result { typedef void type; };
- void operator()(std::string& utf8, uchar code_point) const
+ void operator()(std::string& utf8, char ch) const
{
if (utf8.size() == 0)
utf8 += '\0'; // mark a symbol with prefix 0
// (a 0 byte at the beginning signifies a symbol)
- push_utf8()(utf8, code_point);
+ utf8 += ch;
}
};
@@ -129,7 +125,7 @@
}
template <typename Iterator>
- struct string : grammar<Iterator, unicode, std::string()>
+ struct string : grammar<Iterator, std::string()>
{
string() : string::base_type(start)
{
@@ -148,23 +144,23 @@
start
= '"'
- >> *(str_esc(_val) | (char_ - '"') [push_utf8(_val, _1)])
+ >> *(str_esc(_val) | (~char_('"')) [_val += _1])
>> '"'
;
}
- rule<Iterator, unicode, void(std::string&)> str_esc;
- rule<Iterator, unicode, std::string()> start;
+ rule<Iterator, void(std::string&)> str_esc;
+ rule<Iterator, std::string()> start;
};
template <typename Iterator>
- struct sexpr : grammar<Iterator, unicode, white_space<Iterator>, utree()>
+ struct sexpr : grammar<Iterator, white_space<Iterator>, utree()>
{
sexpr() : sexpr::base_type(start)
{
real_parser<double, strict_real_policies<double> > strict_double;
uint_parser<unsigned char, 16, 2, 2> hex2;
- function<detail::push_symbol_utf8> push_symbol_utf8;
+ function<detail::push_symbol> push_symbol;
function<detail::push_binary> push_binary;
start = atom | list;
@@ -178,8 +174,8 @@
| symbol [_val = _1]
;
- char const* exclude = " ();\"\n\r\t";
- symbol = +lexeme[print - char_(exclude)] [push_symbol_utf8(_val, _1)];
+ char const* exclude = " ();\"\0-\31\127";
+ symbol = +lexeme[~char_(exclude)] [push_symbol(_val, _1)];
number = strict_double [_val = _1]
| lexeme[no_case["0x"] >> hex] [_val = _1]
@@ -190,10 +186,10 @@
byte_str = lexeme[no_case['b'] >> +(hex2 [push_binary(_val, _1)])];
}
- rule<Iterator, unicode, white_space<Iterator>, utree()> start, list;
- rule<Iterator, unicode, utree()> atom, number;
- rule<Iterator, unicode, std::string()> symbol;
- rule<Iterator, unicode, std::string()> byte_str;
+ rule<Iterator, white_space<Iterator>, utree()> start, list;
+ rule<Iterator, utree()> atom, number;
+ rule<Iterator, std::string()> symbol;
+ rule<Iterator, std::string()> byte_str;
scheme::string<Iterator> string;
};
}
Modified: trunk/libs/spirit/example/scheme/sexpr_test.cpp
==============================================================================
--- trunk/libs/spirit/example/scheme/sexpr_test.cpp (original)
+++ trunk/libs/spirit/example/scheme/sexpr_test.cpp 2010-03-20 22:59:40 EDT (Sat, 20 Mar 2010)
@@ -58,7 +58,7 @@
std::istream_iterator<char>(),
std::back_inserter(source_code));
- typedef boost::u8_to_u32_iterator<std::string::const_iterator> iterator_type;
+ typedef std::string::const_iterator iterator_type;
iterator_type first(source_code.begin());
iterator_type last(source_code.end());
Modified: trunk/libs/spirit/example/scheme/utree.hpp
==============================================================================
--- trunk/libs/spirit/example/scheme/utree.hpp (original)
+++ trunk/libs/spirit/example/scheme/utree.hpp 2010-03-20 22:59:40 EDT (Sat, 20 Mar 2010)
@@ -50,6 +50,14 @@
struct nil {};
+ struct binary
+ {
+ binary(unsigned char bits[], std::size_t len)
+ : bits(bits), len(len) {}
+ unsigned char* bits;
+ std::size_t len;
+ };
+
utree();
explicit utree(bool b);
explicit utree(unsigned int i);