Subject: [Boost-commit] svn:boost r84150 - in trunk/tools/quickbook: src test/unit
From: dnljms_at_[hidden]
Date: 2013-05-05 09:46:24
Author: danieljames
Date: 2013-05-05 09:46:23 EDT (Sun, 05 May 2013)
New Revision: 84150
URL: http://svn.boost.org/trac/boost/changeset/84150
Log:
Clean up mixed tabs/spaces when unindenting.
If the whitespace to be removed when unindenting a block is not
consistent, then replace all the indentation in that block with spaces
(i.e. expand tabs). This fixes some odd results that could happen, e.g.
....Line 1
--->Line 2
(dots = spaces, arrow = tab) would become:
...Line 1
Line 2
Because the first character was removed from each line. This now works
okay.
But I don't expand tabs when the whitespace that isn't removed is
inconsistent, which can have odd results, for example:
..Line 1
..->Line 2
......Line 3
Will just remove the first two spaces, resulting in:
Line 1
--->Line 2
....Line 3
This should be rare, as tabs after spaces are a bit odd. But maybe I
should convert to spaces more often. Perhaps when indentation is
inconsistent, or perhaps whenever a mixture of tabs and spaces are used,
or even always just do it.
Text files modified:
trunk/tools/quickbook/src/files.cpp | 147 ++++++++++++++++++++++++++++++---------
trunk/tools/quickbook/test/unit/source_map_test.cpp | 40 ++++++++++
2 files changed, 151 insertions(+), 36 deletions(-)
Modified: trunk/tools/quickbook/src/files.cpp
==============================================================================
--- trunk/tools/quickbook/src/files.cpp (original)
+++ trunk/tools/quickbook/src/files.cpp 2013-05-05 09:46:23 EDT (Sun, 05 May 2013)
@@ -468,68 +468,143 @@
}
}
+ boost::string_ref::size_type indentation_count(boost::string_ref x)
+ {
+ unsigned count = 0;
+
+ for(boost::string_ref::const_iterator begin = x.begin(), end = x.end();
+ begin != end; ++begin)
+ {
+ switch(*begin)
+ {
+ case ' ':
+ ++count;
+ break;
+ case '\t':
+ // hardcoded tab to 4 for now
+ count = count - (count % 4) + 4;
+ break;
+ default:
+ assert(false);
+ }
+ }
+
+ return count;
+ }
+
void mapped_file_builder::unindent_and_add(boost::string_ref x)
{
- std::string program(x.begin(), x.end());
+ // I wanted to do everything using a string_ref, but unfortunately
+ // they don't have all the overloads used in here. So...
+ std::string const program(x.begin(), x.end());
// Erase leading blank lines and newlines:
std::string::size_type start = program.find_first_not_of(" \t\r\n");
if (start == std::string::npos) return;
start = program.find_last_of("\r\n", start);
- if (start != std::string::npos)
- {
- ++start;
- program.erase(0, start);
- }
+ start = start == std::string::npos ? 0 : start + 1;
- assert(program.size() != 0);
+ assert(start < program.size());
- // Get the first line indent
- std::string::size_type indent = program.find_first_not_of(" \t");
- std::string::size_type pos = 0;
- if (std::string::npos == indent)
- {
- // Nothing left to do here. The code is empty (just spaces).
- // We clear the program to signal the caller that it is empty
- // and return early.
- program.clear();
- return;
- }
+ // Get the first line indentation
+ std::string::size_type indent = program.find_first_not_of(" \t", start) - start;
+ boost::string_ref::size_type full_indent = indentation_count(
+ boost::string_ref(&program[start], indent));
+
+ std::string::size_type pos = start;
// Calculate the minimum indent from the rest of the lines
- do
+ // Detecting a mix of spaces and tabs.
+ while (std::string::npos != (pos = program.find_first_of("\r\n", pos)))
{
pos = program.find_first_not_of("\r\n", pos);
- if (std::string::npos == pos)
- break;
+ if (std::string::npos == pos) break;
std::string::size_type n = program.find_first_not_of(" \t", pos);
- if (n != std::string::npos)
- {
- char ch = program[n];
- if (ch != '\r' && ch != '\n') // ignore empty lines
- indent = (std::min)(indent, n-pos);
- }
+ if (n == std::string::npos) break;
+
+ char ch = program[n];
+ if (ch == '\r' || ch == '\n') continue; // ignore empty lines
+
+ indent = (std::min)(indent, n-pos);
+ full_indent = (std::min)(full_indent, indentation_count(
+ boost::string_ref(&program[pos], n-pos)));
}
- while (std::string::npos != (pos = program.find_first_of("\r\n", pos)));
- // Trim white spaces from column 0..indent
- pos = 0;
- program.erase(0, indent);
+ // Detect if indentation is mixed.
+ bool mixed_indentation = false;
+ boost::string_ref first_indent(&program[start], indent);
+ pos = start;
+
while (std::string::npos != (pos = program.find_first_of("\r\n", pos)))
{
- if (std::string::npos == (pos = program.find_first_not_of("\r\n", pos)))
- {
+ pos = program.find_first_not_of("\r\n", pos);
+ if (std::string::npos == pos) break;
+
+ std::string::size_type n = program.find_first_not_of(" \t", pos);
+ if (n == std::string::npos || n-pos < indent) continue;
+
+ if (boost::string_ref(&program[pos], indent) != first_indent) {
+ mixed_indentation = true;
break;
}
+ }
+
+ std::string unindented_program;
+ std::string::size_type copied = start;
+
+ if (mixed_indentation)
+ {
+ pos = start;
+
+ do {
+ if (std::string::npos == (pos = program.find_first_not_of("\r\n", pos)))
+ break;
+
+ unindented_program.append(program.begin() + copied, program.begin() + pos);
+ copied = pos;
- std::string::size_type next = program.find_first_of("\r\n", pos);
- program.erase(pos, (std::min)(indent, next-pos));
+ std::string::size_type next = program.find_first_not_of(" \t", pos);
+
+ unsigned length = indentation_count(boost::string_ref(
+ &program[pos], next - pos));
+
+ if (length > full_indent) {
+ std::string new_indentation(length - full_indent, ' ');
+ unindented_program.append(new_indentation);
+ }
+
+ copied = next;
+ } while (std::string::npos !=
+ (pos = program.find_first_of("\r\n", pos)));
}
+ else
+ {
+ // Trim white spaces from column 0..indent
+ pos = start + indent;
+ copied = pos;
+
+ while (std::string::npos != (pos = program.find_first_of("\r\n", pos)))
+ {
+ if (std::string::npos == (pos = program.find_first_not_of("\r\n", pos)))
+ {
+ break;
+ }
+
+ unindented_program.append(program.begin() + copied, program.begin() + pos);
+ copied = pos;
+
+ std::string::size_type next = program.find_first_of("\r\n", pos);
+ copied = pos + (std::min)(indent, next-pos);
+ }
+ }
+
+ unindented_program.append(program.begin() + copied, program.end());
+ copied = program.size();
data->new_file->add_indented_mapped_file_section(x.begin());
- data->new_file->source_.append(program);
+ data->new_file->source_.append(unindented_program);
}
file_position mapped_file::position_of(boost::string_ref::const_iterator pos) const
Modified: trunk/tools/quickbook/test/unit/source_map_test.cpp
==============================================================================
--- trunk/tools/quickbook/test/unit/source_map_test.cpp (original)
+++ trunk/tools/quickbook/test/unit/source_map_test.cpp 2013-05-05 09:46:23 EDT (Sun, 05 May 2013)
@@ -306,11 +306,51 @@
}
+void indented_map_mixed_test()
+{
+ quickbook::mapped_file_builder builder;
+
+ {
+ boost::string_ref source("\tCode line 1\n Code line 2\n\t Code line 3\n \tCode line 4");
+ quickbook::file_ptr fake_file = new quickbook::file(
+ "(fake file)", source, 105u);
+ builder.start(fake_file);
+ builder.unindent_and_add(fake_file->source());
+ quickbook::file_ptr f1 = builder.release();
+ BOOST_TEST_EQ(f1->source(),
+ boost::string_ref("Code line 1\nCode line 2\n Code line 3\n Code line 4"));
+ }
+
+ {
+ boost::string_ref source(" Code line 1\n\tCode line 2");
+ quickbook::file_ptr fake_file = new quickbook::file(
+ "(fake file)", source, 105u);
+ builder.start(fake_file);
+ builder.unindent_and_add(fake_file->source());
+ quickbook::file_ptr f1 = builder.release();
+ BOOST_TEST_EQ(f1->source(),
+ boost::string_ref("Code line 1\n Code line 2"));
+ }
+
+ {
+ boost::string_ref source(" Code line 1\n \tCode line 2");
+ quickbook::file_ptr fake_file = new quickbook::file(
+ "(fake file)", source, 105u);
+ builder.start(fake_file);
+ builder.unindent_and_add(fake_file->source());
+ quickbook::file_ptr f1 = builder.release();
+ BOOST_TEST_EQ(f1->source(),
+ boost::string_ref("Code line 1\n\tCode line 2"));
+ }
+}
+
+
int main()
{
simple_map_tests();
indented_map_tests();
indented_map_tests2();
indented_map_leading_blanks_test();
+ indented_map_mixed_test();
return boost::report_errors();
}