Subject: [Boost-commit] svn:boost r55578 - in sandbox/SOC/2009/unicode: boost/iterator boost/unicode libs/unicode/example
From: loufoque_at_[hidden]
Date: 2009-08-13 22:25:09
Author: mgaunard
Date: 2009-08-13 22:25:08 EDT (Thu, 13 Aug 2009)
New Revision: 55578
URL: http://svn.boost.org/trac/boost/changeset/55578
Log:
Normalization support
Added:
sandbox/SOC/2009/unicode/boost/unicode/cat.hpp (contents, props changed)
Text files modified:
sandbox/SOC/2009/unicode/boost/iterator/pipe_iterator.hpp | 77 ++++++++++++++++++++++++
sandbox/SOC/2009/unicode/boost/unicode/compose.hpp | 1
sandbox/SOC/2009/unicode/boost/unicode/compose_fwd.hpp | 127 ++++++++++++++++++++++++++++++++++-----
sandbox/SOC/2009/unicode/boost/unicode/hangul.hpp | 17 +++++
sandbox/SOC/2009/unicode/libs/unicode/example/compose.cpp | 8 ++
5 files changed, 210 insertions(+), 20 deletions(-)
Modified: sandbox/SOC/2009/unicode/boost/iterator/pipe_iterator.hpp
==============================================================================
--- sandbox/SOC/2009/unicode/boost/iterator/pipe_iterator.hpp (original)
+++ sandbox/SOC/2009/unicode/boost/iterator/pipe_iterator.hpp 2009-08-13 22:25:08 EDT (Thu, 13 Aug 2009)
@@ -6,6 +6,8 @@
#include <boost/range.hpp>
#include <boost/mpl/int.hpp>
+#include <boost/mpl/times.hpp>
+#include <boost/tuple/tuple.hpp>
#include <boost/concept/requires.hpp>
#include <boost/range/concepts.hpp>
@@ -58,6 +60,81 @@
return one_many_pipe<OneManyPipe>(p);
}
+/* TODO: make it work for pipes that don't expose max_output */
+/** Model of \c \xmlonly<conceptname>Pipe</conceptname>\endxmlonly
+ * constructed from two models of \c \xmlonly<conceptname>Pipe</conceptname>\endxmlonly
+ * and that applies one after the other. */
+template<typename P1, typename P2>
+struct multi_pipe
+{
+ BOOST_CONCEPT_ASSERT((PipeConcept<P1>));
+ BOOST_CONCEPT_ASSERT((PipeConcept<P2>));
+
+ BOOST_CONCEPT_ASSERT((Convertible<typename P1::output_type, typename P2::input_type>));
+
+ typedef typename P1::input_type input_type;
+ typedef typename P2::output_type output_type;
+
+ typedef typename mpl::times<
+ typename P1::max_output,
+ typename P2::max_output
+ >::type max_output;
+
+ multi_pipe() {}
+ multi_pipe(P1 p1_, P2 p2_ = P2()) : p1(p1_), p2(p2_) {}
+
+ template<typename In, typename Out>
+ std::pair<In, Out> ltr(In begin, In end, Out out)
+ {
+ typename P1::output_type buf[max_output::value];
+ typename P1::output_type* b = buf;
+
+ std::pair<In, typename P1::output_type*> pair = p1.ltr(begin, end, buf);
+ typename P1::output_type* e = pair.second;
+
+ do
+ {
+ tie(b, out) = p2.ltr(b, e, out);
+ }
+ while(b != e);
+
+ return std::make_pair(pair.first, out);
+ }
+
+ template<typename In, typename Out>
+ std::pair<In, Out> rtl(In begin, In end, Out out)
+ {
+ typename P1::output_type buf[max_output::value];
+ typename P1::output_type* b = buf;
+
+ std::pair<In, typename P1::output_type*> pair = p1.rtl(begin, end, buf);
+ typename P1::output_type* e = pair.second;
+
+ do
+ {
+ tie(b, out) = p2.ltr(b, e, out);
+ }
+ while(b != e);
+
+ return std::make_pair(pair.first, out);
+ }
+
+private:
+ P1 p1;
+ P2 p2;
+};
+
+template<typename P1, typename P2>
+BOOST_CONCEPT_REQUIRES(
+ ((PipeConcept<P1>))
+ ((PipeConcept<P2>))
+ ((Convertible<typename P1::output_type, typename P2::input_type>)),
+ (multi_pipe<P1, P2>)
+) make_multi_pipe(P1 p1, P2 p2)
+{
+ return multi_pipe<P1, P2>(p1, p2);
+}
+
/** Model of \c \xmlonly<conceptname>OneManyPipe</conceptname>\endxmlonly
* that casts its input to its template parameter and writes it to its output. */
template<typename T>
Added: sandbox/SOC/2009/unicode/boost/unicode/cat.hpp
==============================================================================
--- (empty file)
+++ sandbox/SOC/2009/unicode/boost/unicode/cat.hpp 2009-08-13 22:25:08 EDT (Thu, 13 Aug 2009)
@@ -0,0 +1,6 @@
+#ifndef BOOST_UNICODE_CAT_HPP
+#define BOOST_UNICODE_CAT_HPP
+
+
+
+#endif
Modified: sandbox/SOC/2009/unicode/boost/unicode/compose.hpp
==============================================================================
--- sandbox/SOC/2009/unicode/boost/unicode/compose.hpp (original)
+++ sandbox/SOC/2009/unicode/boost/unicode/compose.hpp 2009-08-13 22:25:08 EDT (Thu, 13 Aug 2009)
@@ -11,6 +11,7 @@
BOOST_UNICODE_PIPE_DEF(compose, 0)
BOOST_UNICODE_PIPE_DEF(decompose, 1)
+BOOST_UNICODE_PIPE_DEF(normalize, 1)
} // namespace unicode
} // namespace boost
Modified: sandbox/SOC/2009/unicode/boost/unicode/compose_fwd.hpp
==============================================================================
--- sandbox/SOC/2009/unicode/boost/unicode/compose_fwd.hpp (original)
+++ sandbox/SOC/2009/unicode/boost/unicode/compose_fwd.hpp 2009-08-13 22:25:08 EDT (Thu, 13 Aug 2009)
@@ -7,9 +7,18 @@
#include <boost/integer/static_pow.hpp>
#include <climits>
-#include <boost/iterator/pipe_iterator.hpp>
#include <vector>
+#include <boost/throw_exception.hpp>
+#include <stdexcept>
+#ifndef BOOST_NO_STD_LOCALE
+#include <sstream>
+#include <ios>
+#endif
+
+#include <boost/detail/unspecified.hpp>
+#include <boost/iterator/pipe_iterator.hpp>
+
namespace boost
{
namespace unicode
@@ -21,6 +30,30 @@
#undef BOOST_UNICODE_OPTION
#endif
+namespace detail
+{
+ struct combining_pred
+ {
+ bool operator()(char32 lft, char32 rgt) const
+ {
+ return ucd::get_combining_class(lft) < ucd::get_combining_class(rgt);
+ }
+ };
+
+ template<typename Size, typename Iterator, typename Comp>
+ void stable_sort_bounded(Iterator begin, Iterator end, Comp comp = std::less<typename std::iterator_traits<Iterator>::value_type>())
+ {
+#if defined(__GLIBCPP__) || defined(__GLIBCXX__) || defined(__SGI_STL_PORT) || defined(_STLPORT_VERSION)
+ typename std::iterator_traits<Iterator>::value_type buf[Size::value];
+ return std::__stable_sort_adaptive(begin, end, buf, Size::value, comp);
+#else
+ return std::stable_sort(begin, end, comp);
+#endif
+ }
+
+}
+
+/* TODO: special case the case when Out is a RandomAccessIterator */
/** Model of \c \xmlonly<conceptname>Pipe</conceptname>\endxmlonly
* that decomposes a combining character sequence, i.e. it transforms a combining
* character sequence into its canonically ordered decomposed equivalent.
@@ -31,53 +64,105 @@
typedef char32 input_type;
typedef char32 output_type;
+ typedef mpl::int_<31> max_output;
+
decomposer(unsigned mask_ = BOOST_UNICODE_OPTION(ucd::decomposition_type::canonical)) : mask(mask_)
{
}
- /** \post \c out is in Normalization Form D. */
+ /** Throws \c std::out_of_range if [<tt>begin</tt>, <tt>end</tt>[ is not stream-safe.
+ * \post \c out is in Normalization Form D. */
template<typename In, typename Out>
- std::pair<In, Out> ltr(In begin, In end, Out out)
+ std::pair<In, Out> ltr(In begin, In end, Out out, bool inverse = false)
{
+ In pos = begin;
+
+ char32 buf[max_output::value];
+ char32* out_pos = buf;
+
+ bool to_sort = false;
do
{
- char32 ch = *begin;
+ char32 ch = *pos;
if(ucd::get_combining_class(ch) != 0)
- {
- // canonical reorder, not handled yet
- }
+ to_sort = true;
iterator_range<const char32*> dec = ucd::get_decomposition(ch);
if(!empty(dec) && ((1 << ucd::get_decomposition_type(ch)) & mask))
{
- out = pipe(dec, *this, out); // we decompose recursively
+ for(const char32* p = boost::begin(dec); p != boost::end(dec); ++p)
+ out_pos = decompose_rec(*p, out_pos);
}
else if(BOOST_UNICODE_OPTION(ucd::decomposition_type::canonical) & mask)
{
- out = hangul_decomposer()(ch, out);
+ if((out_pos + hangul_decomposer::len(ch) - 1) != (buf + max_output::value))
+ out_pos = hangul_decomposer()(ch, out_pos);
+ else
+ not_stream_safe(begin, end);
+ }
+ else if(out_pos != (buf + max_output::value))
+ {
+ *out_pos++ = ch;
}
else
{
- *out++ = ch;
+ not_stream_safe(begin, end);
}
- ++begin;
+ ++pos;
}
- while(begin != end && ucd::get_combining_class(*begin) != 0);
+ while(pos != end && ((!inverse && ucd::get_combining_class(*pos) != 0) || (inverse && ucd::get_combining_class(*pos) == 0)));
- return std::make_pair(begin, out);
+ if(to_sort)
+ detail::stable_sort_bounded<max_output>(buf, out_pos, detail::combining_pred());
+
+ out = std::copy(buf, out_pos, out);
+ return std::make_pair(pos, out);
}
- /** \post \c out is in Normalization Form D. */
+ /** Throws \c std::out_of_range if [<tt>begin</tt>, <tt>end</tt>[ is not stream-safe.
+ * \post \c out is in Normalization Form D. */
template<typename In, typename Out>
std::pair<In, Out> rtl(In begin, In end, Out out)
{
- // NOT IMPLEMENTED
- *out++ = *--end;
- return std::make_pair(end, out);
+ std::pair<
+ reverse_iterator<In>,
+ Out
+ > p = ltr(make_reverse_iterator(end), make_reverse_iterator(begin), out, true);
+ return std::make_pair(p.first.base(), p.second);
}
private:
+ template<typename Iterator>
+ static void not_stream_safe(Iterator begin, Iterator end)
+ {
+#ifndef BOOST_NO_STD_LOCALE
+ std::stringstream ss;
+ ss << "Invalid Unicode stream-safe combining character sequence " << std::showbase << std::hex;
+ for(Iterator it = begin; it != end; ++it)
+ ss << *it << " ";
+ ss << "encountered while trying to decompose UTF-32 sequence";
+ std::out_of_range e(ss.str());
+#else
+ std::out_of_range e("Invalid Unicode stream-safe combining character sequence encountered while trying to decompose UTF-32 sequence");
+#endif
+ boost::throw_exception(e);
+ }
+
+ template<typename OutputIterator>
+ OutputIterator decompose_rec(char32 ch, OutputIterator out)
+ {
+ iterator_range<const char32*> dec = ucd::get_decomposition(ch);
+ if(!empty(dec) && ((1 << ucd::get_decomposition_type(ch)) & mask))
+ {
+ for(const char32* p = begin(dec); p != end(dec); ++p)
+ out = decompose_rec(*p, out);
+ return out;
+ }
+ *out++ = ch;
+ return out;
+ }
+
unsigned mask;
};
@@ -128,7 +213,8 @@
typedef char32 output_type;
typedef mpl::int_<1> max_output;
- /** \pre [<tt>begin</tt>, <tt>end</tt>[ is in Normalization Form D. */
+ /** \pre [<tt>begin</tt>, <tt>end</tt>[ is in Normalization Form D.
+ * \post \c out is in Normalization Form C. */
template<typename In, typename Out>
std::pair<In, Out> ltr(In begin, In end, Out out)
{
@@ -172,7 +258,8 @@
}
/* This could by made faster using a sorted table of reversed strings */
- /** \pre [<tt>begin</tt>, <tt>end</tt>[ is in Normalization Form D. */
+ /** \pre [<tt>begin</tt>, <tt>end</tt>[ is in Normalization Form D.
+ * \post \c out is in Normalization Form C. */
template<typename In, typename Out>
std::pair<In, Out> rtl(In begin, In end, Out out)
{
@@ -224,6 +311,8 @@
}
};
+typedef boost::detail::unspecified< multi_pipe<decomposer, composer> >::type normalizer;
+
} // namespace unicode
} // namespace boost
Modified: sandbox/SOC/2009/unicode/boost/unicode/hangul.hpp
==============================================================================
--- sandbox/SOC/2009/unicode/boost/unicode/hangul.hpp (original)
+++ sandbox/SOC/2009/unicode/boost/unicode/hangul.hpp 2009-08-13 22:25:08 EDT (Thu, 13 Aug 2009)
@@ -32,6 +32,7 @@
* Other code points are left unchanged. */
struct hangul_decomposer
{
+ typedef char32 input_type;
typedef char32 output_type;
typedef mpl::int_<3> max_output;
@@ -59,8 +60,23 @@
return out;
}
+ static int len(char32 ch)
+ {
+ using namespace detail;
+
+ char32 SIndex = ch - SBase;
+ char32 TIndex = SIndex % TCount;
+
+ if(SIndex < 0 || SIndex >= SCount)
+ return 1;
+ if(TIndex)
+ return 3;
+ return 2;
+ }
+
};
+/* TODO: implement it */
/** \c \xmlonly<conceptname>Pipe</conceptname>\endxmlonly that
* transforms <L, V>, <L, V, T> and <LV, T> Hangul code points sequences into the
* LV and LVT Hangul syllables, since those compositions are not part
@@ -68,6 +84,7 @@
* Other code points are left unchanged. */
struct hangul_composer
{
+ typedef char32 input_type;
typedef char32 output_type;
typedef mpl::int_<1> max_output;
Modified: sandbox/SOC/2009/unicode/libs/unicode/example/compose.cpp
==============================================================================
--- sandbox/SOC/2009/unicode/libs/unicode/example/compose.cpp (original)
+++ sandbox/SOC/2009/unicode/libs/unicode/example/compose.cpp 2009-08-13 22:25:08 EDT (Thu, 13 Aug 2009)
@@ -9,6 +9,8 @@
#include <iostream>
#include <iterator>
+#include <boost/range/adaptor/reversed.hpp>
+
namespace unicode = boost::unicode;
namespace ucd = unicode::ucd;
@@ -39,8 +41,12 @@
std::cout << std::endl;
std::cout << "Decomposition type: " << as_string(ucd::get_decomposition_type(cp)) << std::endl;
+ boost::char32 baz[] = { cp, 0x330 };
std::cout << "Canonical decomposition: ";
- unicode::decompose(boost::list_of(cp), std::ostream_iterator<boost::char32>(std::cout, " "));
+ std::cout << unicode::composed(unicode::decomposed(baz)) << std::endl;
+ std::cout << "reversed: " << boost::make_reversed_range(unicode::composed(unicode::decomposed(baz))) << std::endl;
+ std::cout << unicode::normalized(baz) << std::endl;
+ std::cout << "reversed: " << boost::make_reversed_range(unicode::normalized(baz));
std::cout << std::endl << std::endl;
std::cout << "Canonical decomposition of U+00A8: ";