Subject: [Boost-commit] svn:boost r75614 - in branches/quickbook-dev/tools/quickbook: src test/unit
From: dnljms_at_[hidden]
Date: 2011-11-22 18:45:39
Author: danieljames
Date: 2011-11-22 18:45:37 EST (Tue, 22 Nov 2011)
New Revision: 75614
URL: http://svn.boost.org/trac/boost/changeset/75614
Log:
Quickbook: Track position in code blocks and snippets.
Text files modified:
branches/quickbook-dev/tools/quickbook/src/actions.cpp | 27 +-
branches/quickbook-dev/tools/quickbook/src/code_snippet.cpp | 259 ++++++++++++++------------
branches/quickbook-dev/tools/quickbook/src/files.cpp | 385 +++++++++++++++++++++++++++++++++++++++
branches/quickbook-dev/tools/quickbook/src/files.hpp | 48 ++++
branches/quickbook-dev/tools/quickbook/src/input_path.cpp | 4
branches/quickbook-dev/tools/quickbook/src/iterator.hpp | 61 ------
branches/quickbook-dev/tools/quickbook/src/main_grammar.cpp | 2
branches/quickbook-dev/tools/quickbook/src/quickbook.cpp | 2
branches/quickbook-dev/tools/quickbook/src/syntax_highlight.cpp | 2
branches/quickbook-dev/tools/quickbook/src/utils.cpp | 60 ------
branches/quickbook-dev/tools/quickbook/src/utils.hpp | 3
branches/quickbook-dev/tools/quickbook/test/unit/Jamfile.v2 | 2
12 files changed, 582 insertions(+), 273 deletions(-)
Modified: branches/quickbook-dev/tools/quickbook/src/actions.cpp
==============================================================================
--- branches/quickbook-dev/tools/quickbook/src/actions.cpp (original)
+++ branches/quickbook-dev/tools/quickbook/src/actions.cpp 2011-11-22 18:45:37 EST (Tue, 22 Nov 2011)
@@ -203,7 +203,7 @@
void error_message_action::operator()(parse_iterator first, parse_iterator last) const
{
- file_position const pos = get_position(first, actions.current_file->source);
+ file_position const pos = actions.current_file->position_of(first.base());
std::string value(first, last);
std::string formatted_message = message;
@@ -218,7 +218,7 @@
void error_action::operator()(parse_iterator first, parse_iterator /*last*/) const
{
- file_position const pos = get_position(first, actions.current_file->source);
+ file_position const pos = actions.current_file->position_of(first.base());
detail::outerr(actions.current_file->path, pos.line)
<< "Syntax Error near column " << pos.column << ".\n";
@@ -589,20 +589,19 @@
write_anchors(actions, out);
// preprocess the code section to remove the initial indentation
- std::string program(first, last);
- detail::unindent(program);
- if (program.size() == 0)
- return; // Nothing left to do here. The program is empty.
+ mapped_file_builder mapped;
+ mapped.start(actions.current_file);
+ mapped.unindent_and_add(first.base(), last.base());
+
+ file const* file_ptr = mapped.release();
- file fake_file(
- actions.current_file->path,
- program,
- qbk_version_n);
+ if (file_ptr->source.empty())
+ return; // Nothing left to do here. The program is empty.
- parse_iterator first_(fake_file.source.begin());
- parse_iterator last_(fake_file.source.end());
+ parse_iterator first_(file_ptr->source.begin());
+ parse_iterator last_(file_ptr->source.end());
- file const* saved_file = &fake_file;
+ file const* saved_file = file_ptr;
boost::swap(actions.current_file, saved_file);
// print the code with syntax coloring
@@ -1588,7 +1587,7 @@
if (actions.ids.section_level() <= actions.min_section_level)
{
- file_position const pos = get_position(first, actions.current_file->source);
+ file_position const pos = actions.current_file->position_of(first);
detail::outerr(actions.current_file->path, pos.line)
<< "Mismatched [endsect] near column " << pos.column << ".\n";
Modified: branches/quickbook-dev/tools/quickbook/src/code_snippet.cpp
==============================================================================
--- branches/quickbook-dev/tools/quickbook/src/code_snippet.cpp (original)
+++ branches/quickbook-dev/tools/quickbook/src/code_snippet.cpp 2011-11-22 18:45:37 EST (Tue, 22 Nov 2011)
@@ -28,20 +28,25 @@
code_snippet_actions(std::vector<template_symbol>& storage,
file* source_file,
char const* source_type)
- : callout_id(0)
+ : last_code_pos(source_file->source.begin())
+ , in_code(false)
+ , callout_id(0)
+ , snippet_stack()
, storage(storage)
, source_file(source_file)
, source_type(source_type)
- {}
+ {
+ content.start(source_file);
+ }
- void pass_thru_char(char);
+ void mark(string_iterator first, string_iterator last);
void pass_thru(string_iterator first, string_iterator last);
void escaped_comment(string_iterator first, string_iterator last);
void start_snippet(string_iterator first, string_iterator last);
void end_snippet(string_iterator first, string_iterator last);
void callout(string_iterator first, string_iterator last);
- void append_code();
+ void append_code(string_iterator first, string_iterator last);
void close_code();
struct snippet_data
@@ -49,26 +54,28 @@
snippet_data(std::string const& id, int callout_base_id)
: id(id)
, callout_base_id(callout_base_id)
- , content()
, start_code(false)
- , end_code(false)
{}
std::string id;
int callout_base_id;
- std::string content;
bool start_code;
- bool end_code;
+ std::string::const_iterator source_pos;
+ mapped_file_builder::pos start_pos;
value_builder callouts;
boost::shared_ptr<snippet_data> next;
};
- void push_snippet_data(std::string const& id, int callout_base_id)
+ void push_snippet_data(std::string const& id, int callout_base_id,
+ std::string::const_iterator pos)
{
boost::shared_ptr<snippet_data> new_snippet(
new snippet_data(id, callout_base_id));
new_snippet->next = snippet_stack;
snippet_stack = new_snippet;
+ snippet_stack->start_code = in_code;
+ snippet_stack->source_pos = pos;
+ snippet_stack->start_pos = content.get_pos();
}
boost::shared_ptr<snippet_data> pop_snippet_data()
@@ -78,11 +85,13 @@
snippet->next.reset();
return snippet;
}
-
+
+ mapped_file_builder content;
+ std::string::const_iterator mark_begin, mark_end;
+ std::string::const_iterator last_code_pos;
+ bool in_code;
int callout_id;
boost::shared_ptr<snippet_data> snippet_stack;
- std::string code;
- std::string id;
std::vector<template_symbol>& storage;
file* source_file;
char const* const source_type;
@@ -116,10 +125,10 @@
code_elements =
start_snippet [boost::bind(&actions_type::start_snippet, &actions, _1, _2)]
| end_snippet [boost::bind(&actions_type::end_snippet, &actions, _1, _2)]
- | escaped_comment
- | pass_thru_comment
- | ignore
- | cl::anychar_p [boost::bind(&actions_type::pass_thru_char, &actions, _1)]
+ | escaped_comment [boost::bind(&actions_type::escaped_comment, &actions, _1, _2)]
+ | pass_thru_comment [boost::bind(&actions_type::pass_thru, &actions, _1, _2)]
+ | ignore [boost::bind(&actions_type::append_code, &actions, _1, _2)]
+ | cl::anychar_p
;
start_snippet =
@@ -127,7 +136,7 @@
>> !(cl::eol_p >> *cl::blank_p)
>> "#["
>> *cl::blank_p
- >> identifier [cl::assign_a(actions.id)]
+ >> identifier [boost::bind(&actions_type::mark, &actions, _1, _2)]
>> *(cl::anychar_p - cl::eol_p)
;
@@ -159,12 +168,12 @@
escaped_comment =
cl::confix_p(
*cl::space_p >> "#`",
- (*cl::anychar_p) [boost::bind(&actions_type::escaped_comment, &actions, _1, _2)],
+ (*cl::anychar_p) [boost::bind(&actions_type::mark, &actions, _1, _2)],
(cl::eol_p | cl::end_p)
)
| cl::confix_p(
*cl::space_p >> "\"\"\"`",
- (*cl::anychar_p) [boost::bind(&actions_type::escaped_comment, &actions, _1, _2)],
+ (*cl::anychar_p) [boost::bind(&actions_type::mark, &actions, _1, _2)],
"\"\"\""
)
;
@@ -175,10 +184,10 @@
= "#=" >> (cl::eps_p - '=')
>> ( *(cl::anychar_p - cl::eol_p)
>> (cl::eol_p | cl::end_p)
- ) [boost::bind(&actions_type::pass_thru, &actions, _1, _2)]
+ ) [boost::bind(&actions_type::mark, &actions, _1, _2)]
| cl::confix_p(
"\"\"\"=" >> (cl::eps_p - '='),
- (*cl::anychar_p) [boost::bind(&actions_type::pass_thru, &actions, _1, _2)],
+ (*cl::anychar_p) [boost::bind(&actions_type::mark, &actions, _1, _2)],
"\"\"\""
)
;
@@ -220,12 +229,12 @@
code_elements =
start_snippet [boost::bind(&actions_type::start_snippet, &actions, _1, _2)]
| end_snippet [boost::bind(&actions_type::end_snippet, &actions, _1, _2)]
- | escaped_comment
- | ignore
- | pass_thru_comment
- | line_callout
- | inline_callout
- | cl::anychar_p [boost::bind(&actions_type::pass_thru_char, &actions, _1)]
+ | escaped_comment [boost::bind(&actions_type::escaped_comment, &actions, _1, _2)]
+ | ignore [boost::bind(&actions_type::append_code, &actions, _1, _2)]
+ | pass_thru_comment [boost::bind(&actions_type::pass_thru, &actions, _1, _2)]
+ | line_callout [boost::bind(&actions_type::callout, &actions, _1, _2)]
+ | inline_callout [boost::bind(&actions_type::callout, &actions, _1, _2)]
+ | cl::anychar_p
;
start_snippet =
@@ -233,7 +242,7 @@
>> !(cl::eol_p >> *cl::blank_p)
>> "//["
>> *cl::blank_p
- >> identifier [cl::assign_a(actions.id)]
+ >> identifier [boost::bind(&actions_type::mark, &actions, _1, _2)]
>> *(cl::anychar_p - cl::eol_p)
|
*cl::blank_p
@@ -241,7 +250,7 @@
>> *cl::blank_p
>> "/*["
>> *cl::space_p
- >> identifier [cl::assign_a(actions.id)]
+ >> identifier [boost::bind(&actions_type::mark, &actions, _1, _2)]
>> *cl::space_p
>> "*/"
>> *cl::blank_p
@@ -249,7 +258,7 @@
|
"/*["
>> *cl::space_p
- >> identifier [cl::assign_a(actions.id)]
+ >> identifier [boost::bind(&actions_type::mark, &actions, _1, _2)]
>> *cl::space_p
>> "*/"
;
@@ -273,7 +282,7 @@
inline_callout
= cl::confix_p(
"/*<" >> *cl::space_p,
- (*cl::anychar_p) [boost::bind(&actions_type::callout, &actions, _1, _2)],
+ (*cl::anychar_p) [boost::bind(&actions_type::mark, &actions, _1, _2)],
">*/"
)
;
@@ -281,7 +290,7 @@
line_callout
= cl::confix_p(
"/*<<" >> *cl::space_p,
- (*cl::anychar_p) [boost::bind(&actions_type::callout, &actions, _1, _2)],
+ (*cl::anychar_p) [boost::bind(&actions_type::mark, &actions, _1, _2)],
">>*/"
)
>> *cl::space_p
@@ -310,12 +319,12 @@
escaped_comment
= cl::confix_p(
*cl::space_p >> "//`",
- (*cl::anychar_p) [boost::bind(&actions_type::escaped_comment, &actions, _1, _2)],
+ (*cl::anychar_p) [boost::bind(&actions_type::mark, &actions, _1, _2)],
(cl::eol_p | cl::end_p)
)
| cl::confix_p(
*cl::space_p >> "/*`",
- (*cl::anychar_p) [boost::bind(&actions_type::escaped_comment, &actions, _1, _2)],
+ (*cl::anychar_p) [boost::bind(&actions_type::mark, &actions, _1, _2)],
"*/"
)
;
@@ -326,10 +335,10 @@
= "//=" >> (cl::eps_p - '=')
>> ( *(cl::anychar_p - cl::eol_p)
>> (cl::eol_p | cl::end_p)
- ) [boost::bind(&actions_type::pass_thru, &actions, _1, _2)]
+ ) [boost::bind(&actions_type::mark, &actions, _1, _2)]
| cl::confix_p(
"/*=" >> (cl::eps_p - '='),
- (*cl::anychar_p) [boost::bind(&actions_type::pass_thru, &actions, _1, _2)],
+ (*cl::anychar_p) [boost::bind(&actions_type::mark, &actions, _1, _2)],
"*/"
)
;
@@ -374,116 +383,134 @@
return 0;
}
- void code_snippet_actions::append_code()
+ void code_snippet_actions::append_code(string_iterator first, string_iterator last)
{
- if(!snippet_stack) return;
- snippet_data& snippet = *snippet_stack;
-
- if (!code.empty())
- {
- if(snippet.content.empty())
- {
- snippet.start_code = true;
- }
- else if(!snippet.end_code)
- {
- snippet.content += "\n\n";
- snippet.content += source_type;
- snippet.content += "```\n";
- }
+ assert(last_code_pos <= first);
+
+ if(snippet_stack) {
+ if (last_code_pos != first) {
+ if (!in_code)
+ {
+ content.add("\n\n", last_code_pos);
+ content.add(source_type, last_code_pos);
+ content.add("```\n", last_code_pos);
- snippet.content += code;
- snippet.end_code = true;
+ in_code = true;
+ }
- code.clear();
+ content.add(last_code_pos, first);
+ }
}
+
+ last_code_pos = last;
}
void code_snippet_actions::close_code()
{
- if(!snippet_stack) return;
- snippet_data& snippet = *snippet_stack;
+ if (!snippet_stack) return;
- if(snippet.end_code)
+ if (in_code)
{
- snippet.content += "\n```\n\n";
- snippet.end_code = false;
+ content.add("\n```\n\n", last_code_pos);
+ in_code = false;
}
}
- void code_snippet_actions::pass_thru(string_iterator first, string_iterator last)
+ void code_snippet_actions::mark(string_iterator first, string_iterator last)
{
- if(!snippet_stack) return;
- code.append(first, last);
+ mark_begin = first;
+ mark_end = last;
}
- void code_snippet_actions::pass_thru_char(char c)
+ void code_snippet_actions::pass_thru(string_iterator first, string_iterator last)
{
if(!snippet_stack) return;
- code += c;
+ append_code(first, last);
+
+ if (!in_code)
+ {
+ content.add("\n\n", first);
+ content.add(source_type, first);
+ content.add("```\n", first);
+ in_code = true;
+ }
+
+ content.add(mark_begin, mark_end);
}
void code_snippet_actions::callout(string_iterator first, string_iterator last)
{
if(!snippet_stack) return;
- code += "``[[callout" + boost::lexical_cast<std::string>(callout_id) + "]]``";
+ append_code(first, last);
+
+ if (!in_code)
+ {
+ content.add("\n\n", first);
+ content.add(source_type, first);
+ content.add("```\n", first);
+ in_code = true;
+ }
+
+ content.add(
+ "``[[callout" + boost::lexical_cast<std::string>(callout_id) + "]]``",
+ first);
- snippet_stack->callouts.insert(qbk_value_ref(source_file, first, last, template_tags::block));
+ snippet_stack->callouts.insert(qbk_value_ref(source_file, mark_begin, mark_end, template_tags::block));
++callout_id;
}
void code_snippet_actions::escaped_comment(string_iterator first, string_iterator last)
{
- if (!snippet_stack)
- {
- id = "!";
- start_snippet(first,first);
- }
-
- snippet_data& snippet = *snippet_stack;
- append_code();
+ append_code(first, last);
close_code();
- std::string temp(first, last);
- detail::unindent(temp); // remove all indents
-
- if (temp.size() != 0)
+ if (mark_begin != mark_end)
{
- snippet.content += "\n" + temp; // add a linebreak to allow block markups
- }
+ if (!snippet_stack)
+ {
+ // TODO: This should be: start_snippet(first,first);
+ push_snippet_data("!", callout_id, first);
+ }
+
+ snippet_data& snippet = *snippet_stack;
- if (snippet.id == "!")
- {
- end_snippet(first,first);
+ content.add("\n", mark_begin);
+ content.unindent_and_add(mark_begin, mark_end);
+
+ if (snippet.id == "!")
+ {
+ end_snippet(last, last);
+ }
}
}
- void code_snippet_actions::start_snippet(string_iterator, string_iterator)
+ void code_snippet_actions::start_snippet(string_iterator first, string_iterator last)
{
- append_code();
- push_snippet_data(id, callout_id);
- id.clear();
+ append_code(first, last);
+ std::string id(mark_begin, mark_end);
+ push_snippet_data(id, callout_id, first);
}
- void code_snippet_actions::end_snippet(string_iterator first, string_iterator)
+ void code_snippet_actions::end_snippet(string_iterator first, string_iterator last)
{
// TODO: Error?
if(!snippet_stack) return;
- append_code();
+ append_code(first, last);
boost::shared_ptr<snippet_data> snippet = pop_snippet_data();
value callouts = snippet->callouts.release();
- std::string body;
- if(snippet->start_code) {
- body += "\n\n";
- body += source_type;
- body += "```\n";
- }
- body += snippet->content;
- if(snippet->end_code) {
- body += "\n```\n\n";
+ mapped_file_builder f;
+ f.start(source_file);
+ if (snippet->start_code) {
+ f.add("\n\n", snippet->source_pos);
+ f.add(source_type, snippet->source_pos);
+ f.add("```\n", snippet->source_pos);
+ }
+ f.add(content, snippet->start_pos, content.get_pos());
+ if (in_code) {
+ f.add("\n```\n\n", first);
}
std::vector<std::string> params;
@@ -493,37 +520,23 @@
params.push_back("[callout" + boost::lexical_cast<std::string>(snippet->callout_base_id + i) + "]");
++i;
}
-
- // TODO: Save position in start_snippet
- value_builder content;
- content.set_tag(template_tags::snippet);
- content.insert(qbk_value(body, qbk_version_n, template_tags::block));
- content.insert(callouts);
+ file* body = f.release();
- template_symbol symbol(snippet->id, params, content.release());
+ value_builder builder;
+ builder.set_tag(template_tags::snippet);
+ builder.insert(qbk_value_ref(body, body->source.begin(), body->source.end(),
+ template_tags::block));
+ builder.insert(callouts);
+
+ template_symbol symbol(snippet->id, params, builder.release());
storage.push_back(symbol);
- // Merge the snippet into its parent
+ // Copy the snippet's callouts to its parent
if(snippet_stack)
{
- snippet_data& next = *snippet_stack;
- if(!snippet->content.empty()) {
- if(!snippet->start_code) {
- close_code();
- }
- else if(!next.end_code) {
- next.content += "\n\n";
- next.content += source_type;
- next.content += "```\n";
- }
-
- next.content += snippet->content;
- next.end_code = snippet->end_code;
- }
-
- next.callouts.extend(callouts);
+ snippet_stack->callouts.extend(callouts);
}
}
}
Modified: branches/quickbook-dev/tools/quickbook/src/files.cpp
==============================================================================
--- branches/quickbook-dev/tools/quickbook/src/files.cpp (original)
+++ branches/quickbook-dev/tools/quickbook/src/files.cpp 2011-11-22 18:45:37 EST (Tue, 22 Nov 2011)
@@ -10,10 +10,19 @@
#include "files.hpp"
#include <boost/filesystem/v3/fstream.hpp>
#include <boost/unordered_map.hpp>
+#include <boost/range/algorithm/upper_bound.hpp>
+#include <boost/range/algorithm/transform.hpp>
#include <fstream>
+#include <list>
+#include <iterator>
namespace quickbook
{
+ namespace
+ {
+ boost::unordered_map<fs::path, file> files;
+ }
+
// Read the first few bytes in a file to see it starts with a byte order
// mark. If it doesn't, then write the characters we've already read in.
// Although, given how UTF-8 works, if we've read anything in, the files
@@ -91,11 +100,6 @@
}
}
- namespace
- {
- boost::unordered_map<fs::path, file> files;
- }
-
file* load(fs::path const& filename, unsigned qbk_version)
{
boost::unordered_map<fs::path, file>::iterator pos
@@ -130,4 +134,375 @@
return &pos->second;
}
+
+ file_position relative_position(
+ std::string::const_iterator begin,
+ std::string::const_iterator iterator)
+ {
+ file_position pos;
+ std::string::const_iterator line_begin = begin;
+
+ while (begin != iterator)
+ {
+ if (*begin == '\r')
+ {
+ ++begin;
+ ++pos.line;
+ line_begin = begin;
+ }
+ else if (*begin == '\n')
+ {
+ ++begin;
+ ++pos.line;
+ line_begin = begin;
+ if (begin == iterator) break;
+ if (*begin == '\r')
+ {
+ ++begin;
+ line_begin = begin;
+ }
+ }
+ else
+ {
+ ++begin;
+ }
+ }
+
+ pos.column = iterator - line_begin + 1;
+ return pos;
+ }
+
+ file_position file::position_of(std::string::const_iterator iterator) const
+ {
+ return relative_position(source.begin(), iterator);
+ }
+
+ // Mapped files.
+
+ struct mapped_file_section
+ {
+ enum section_types {
+ normal,
+ empty,
+ indented
+ };
+
+ std::string::size_type original_pos;
+ std::string::size_type our_pos;
+ section_types section_type;
+
+ mapped_file_section(
+ std::string::size_type original_pos,
+ std::string::size_type our_pos,
+ section_types section_type = normal) :
+ original_pos(original_pos), our_pos(our_pos), section_type(section_type) {}
+
+ std::string::size_type to_original_pos(std::string::size_type pos)
+ {
+ switch (section_type) {
+ case normal:
+ return pos - our_pos + original_pos;
+ case empty:
+ return original_pos;
+ case indented:
+ // Indented doesn't really work, but that's okay because we
+ // currently don't break up indented code.
+ assert(pos == our_pos);
+ return pos - our_pos + original_pos;
+ default:
+ assert(false);
+ return original_pos;
+ }
+ }
+
+ // If 'to_original_pos' worked for indented blocks, this wouldn't
+ // be necessary.
+ file_position calculate_position(
+ file_position const& original,
+ file_position const& relative) const
+ {
+ switch (section_type) {
+ case normal:
+ return file_position(
+ original.line + relative.line - 1,
+ relative.line == 1 ?
+ original.column + relative.column - 1 :
+ relative.column);
+ case empty:
+ return original;
+ case indented:
+ return file_position(
+ original.line + relative.line - 1,
+ original.column + relative.column - 1);
+ default:
+ assert(false);
+ return file_position();
+ }
+ }
+ };
+
+ struct mapped_section_original_cmp
+ {
+ bool operator()(mapped_file_section const& x,
+ mapped_file_section const& y)
+ {
+ return x.original_pos < y.original_pos;
+ }
+
+ bool operator()(mapped_file_section const& x,
+ std::string::size_type const& y)
+ {
+ return x.original_pos < y;
+ }
+
+ bool operator()(std::string::size_type const& x,
+ mapped_file_section const& y)
+ {
+ return x < y.original_pos;
+ }
+ };
+
+ struct mapped_section_pos_cmp
+ {
+ bool operator()(mapped_file_section const& x,
+ mapped_file_section const& y)
+ {
+ return x.our_pos < y.our_pos;
+ }
+
+ bool operator()(mapped_file_section const& x,
+ std::string::size_type const& y)
+ {
+ return x.our_pos < y;
+ }
+
+ bool operator()(std::string::size_type const& x,
+ mapped_file_section const& y)
+ {
+ return x < y.our_pos;
+ }
+ };
+
+ struct mapped_file : file
+ {
+ mapped_file(file const* original) :
+ file(original->path, std::string(), original->version()),
+ original(original), mapped_sections() {}
+
+ file const* original;
+ std::vector<mapped_file_section> mapped_sections;
+
+ void add_empty_mapped_file_section(std::string::const_iterator pos) {
+ std::string::size_type original_pos =
+ pos - original->source.begin();
+
+ if (mapped_sections.empty() ||
+ mapped_sections.back().section_type !=
+ mapped_file_section::empty ||
+ mapped_sections.back().original_pos != original_pos)
+ {
+ mapped_sections.push_back(mapped_file_section(
+ original_pos, source.size(),
+ mapped_file_section::empty));
+ }
+ }
+
+ void add_mapped_file_section(std::string::const_iterator pos) {
+ mapped_sections.push_back(mapped_file_section(
+ pos - original->source.begin(), source.size()));
+ }
+
+ void add_indented_mapped_file_section(std::string::const_iterator pos) {
+ mapped_sections.push_back(mapped_file_section(
+ pos - original->source.begin(), source.size(),
+ mapped_file_section::indented));
+ }
+
+ virtual file_position position_of(std::string::const_iterator) const;
+ };
+
+ namespace {
+ std::list<mapped_file> mapped_files;
+ }
+
+ struct mapped_file_builder_data
+ {
+ mapped_file_builder_data() { reset(); }
+ void reset() { new_file = mapped_files.end(); }
+
+ std::list<mapped_file>::iterator new_file;
+ };
+
+ mapped_file_builder::mapped_file_builder() : data(0) {}
+ mapped_file_builder::~mapped_file_builder() { delete data; }
+
+ void mapped_file_builder::start(file const* f)
+ {
+ if (!data) {
+ data = new mapped_file_builder_data;
+ }
+
+ assert(data->new_file == mapped_files.end());
+ mapped_files.push_back(mapped_file(f));
+ data->new_file = mapped_files.end();
+ --data->new_file;
+ }
+
+ file* mapped_file_builder::release()
+ {
+ file* r = &*data->new_file;
+ data->reset();
+ return r;
+ }
+
+ void mapped_file_builder::clear()
+ {
+ mapped_files.erase(data->new_file);
+ data->reset();
+ }
+
+ bool mapped_file_builder::empty() const
+ {
+ return data->new_file->source.empty();
+ }
+
+ mapped_file_builder::pos mapped_file_builder::get_pos() const
+ {
+ return data->new_file->source.size();
+ }
+
+ void mapped_file_builder::add(char const* x, iterator pos)
+ {
+ data->new_file->add_empty_mapped_file_section(pos);
+ data->new_file->source.append(x);
+ }
+
+ void mapped_file_builder::add(std::string const& x, iterator pos)
+ {
+ data->new_file->add_empty_mapped_file_section(pos);
+ data->new_file->source.append(x);
+ }
+
+ void mapped_file_builder::add(iterator begin, iterator end)
+ {
+ data->new_file->add_mapped_file_section(begin);
+ data->new_file->source.append(begin, end);
+ }
+
+ void mapped_file_builder::add(mapped_file_builder const& x)
+ {
+ add(x, 0, x.data->new_file->source.size());
+ }
+
+ void mapped_file_builder::add(mapped_file_builder const& x,
+ pos begin, pos end)
+ {
+ assert(data->new_file->original == x.data->new_file->original);
+ assert(begin <= x.data->new_file->source.size());
+ assert(end <= x.data->new_file->source.size());
+
+ std::vector<mapped_file_section>::iterator start =
+ boost::upper_bound(x.data->new_file->mapped_sections,
+ begin, mapped_section_pos_cmp());
+ assert(start != x.data->new_file->mapped_sections.begin());
+ --start;
+
+ std::string::size_type size = data->new_file->source.size();
+
+ data->new_file->mapped_sections.push_back(mapped_file_section(
+ start->to_original_pos(begin), size,
+ start->section_type));
+
+ for (++start; start != x.data->new_file->mapped_sections.end() &&
+ start->our_pos < end; ++start)
+ {
+ data->new_file->mapped_sections.push_back(mapped_file_section(
+ start->original_pos, start->our_pos - begin + size,
+ start->section_type));
+ }
+
+ data->new_file->source.append(
+ x.data->new_file->source.begin() + begin,
+ x.data->new_file->source.begin() + end);
+ }
+
+ void mapped_file_builder::unindent_and_add(iterator begin, iterator end)
+ {
+ std::string program(begin, end);
+
+ // Erase leading blank lines and newlines:
+ std::string::size_type start = program.find_first_not_of(" \t");
+ if (start != std::string::npos &&
+ (program[start] == '\r' || program[start] == '\n'))
+ {
+ program.erase(0, start);
+ }
+ start = program.find_first_not_of("\r\n");
+ program.erase(0, start);
+
+ if (program.size() == 0)
+ return; // nothing left to do
+
+ // Get the first line indent
+ std::string::size_type indent = program.find_first_not_of(" \t");
+ std::string::size_type pos = 0;
+ if (std::string::npos == indent)
+ {
+ // Nothing left to do here. The code is empty (just spaces).
+ // We clear the program to signal the caller that it is empty
+ // and return early.
+ program.clear();
+ return;
+ }
+
+ // Calculate the minimum indent from the rest of the lines
+ do
+ {
+ pos = program.find_first_not_of("\r\n", pos);
+ if (std::string::npos == pos)
+ break;
+
+ std::string::size_type n = program.find_first_not_of(" \t", pos);
+ if (n != std::string::npos)
+ {
+ char ch = program[n];
+ if (ch != '\r' && ch != '\n') // ignore empty lines
+ indent = (std::min)(indent, n-pos);
+ }
+ }
+ while (std::string::npos != (pos = program.find_first_of("\r\n", pos)));
+
+ // Trim white spaces from column 0..indent
+ pos = 0;
+ program.erase(0, indent);
+ while (std::string::npos != (pos = program.find_first_of("\r\n", pos)))
+ {
+ if (std::string::npos == (pos = program.find_first_not_of("\r\n", pos)))
+ {
+ break;
+ }
+
+ std::string::size_type next = program.find_first_of("\r\n", pos);
+ program.erase(pos, (std::min)(indent, next-pos));
+ }
+
+ data->new_file->add_indented_mapped_file_section(begin + indent);
+ data->new_file->source.append(program);
+ }
+
+ file_position mapped_file::position_of(std::string::const_iterator pos) const
+ {
+ std::vector<mapped_file_section>::const_iterator section =
+ boost::upper_bound(mapped_sections,
+ std::string::size_type(pos - source.begin()),
+ mapped_section_pos_cmp());
+ assert(section != mapped_sections.begin());
+ --section;
+
+ return section->calculate_position(
+ original->position_of(
+ original->source.begin() + section->original_pos),
+ relative_position(source.begin() + section->our_pos, pos)
+ );
+ }
}
Modified: branches/quickbook-dev/tools/quickbook/src/files.hpp
==============================================================================
--- branches/quickbook-dev/tools/quickbook/src/files.hpp (original)
+++ branches/quickbook-dev/tools/quickbook/src/files.hpp 2011-11-22 18:45:37 EST (Tue, 22 Nov 2011)
@@ -26,10 +26,19 @@
: std::runtime_error(arg) {}
};
+ struct file_position
+ {
+ file_position() : line(1), column(1) {}
+ file_position(int l, int c) : line(l), column(c) {}
+
+ int line;
+ int column;
+ };
+
struct file
{
fs::path const path;
- std::string const source;
+ std::string source;
private:
unsigned qbk_version;
public:
@@ -39,6 +48,8 @@
path(path), source(source), qbk_version(qbk_version)
{}
+ ~file() {}
+
unsigned version() const {
assert(qbk_version);
return qbk_version;
@@ -51,11 +62,46 @@
assert(!qbk_version || qbk_version == v);
qbk_version = v;
}
+
+ virtual file_position position_of(std::string::const_iterator) const;
};
// If version isn't supplied then it must be set later.
file* load(fs::path const& filename,
unsigned qbk_version = 0);
+
+ // Interface for creating fake files which are mapped to
+ // real files, so that the position can be found later.
+
+ struct mapped_file_builder_data;
+
+ struct mapped_file_builder
+ {
+ typedef std::string::const_iterator iterator;
+ typedef std::string::size_type pos;
+
+ mapped_file_builder();
+ ~mapped_file_builder();
+
+ void start(file const*);
+ file* release();
+ void clear();
+
+ bool empty() const;
+ pos get_pos() const;
+
+ void add(char const*, iterator);
+ void add(std::string const&, iterator);
+ void add(iterator, iterator);
+ void add(mapped_file_builder const&);
+ void add(mapped_file_builder const&, pos, pos);
+ void unindent_and_add(iterator, iterator);
+ private:
+ mapped_file_builder_data* data;
+
+ mapped_file_builder(mapped_file_builder const&);
+ mapped_file_builder& operator=(mapped_file_builder const&);
+ };
}
#endif // BOOST_QUICKBOOK_FILES_HPP
Modified: branches/quickbook-dev/tools/quickbook/src/input_path.cpp
==============================================================================
--- branches/quickbook-dev/tools/quickbook/src/input_path.cpp (original)
+++ branches/quickbook-dev/tools/quickbook/src/input_path.cpp 2011-11-22 18:45:37 EST (Tue, 22 Nov 2011)
@@ -232,7 +232,7 @@
ostream& outerr(file const* f, string_iterator pos)
{
- return outerr(f->path, get_position(pos, f->source).line);
+ return outerr(f->path, f->position_of(pos).line);
}
ostream& outwarn(fs::path const& file, int line)
@@ -252,6 +252,6 @@
ostream& outwarn(file const* f, string_iterator pos)
{
- return outwarn(f->path, get_position(pos, f->source).line);
+ return outwarn(f->path, f->position_of(pos).line);
}
}}
Modified: branches/quickbook-dev/tools/quickbook/src/iterator.hpp
==============================================================================
--- branches/quickbook-dev/tools/quickbook/src/iterator.hpp (original)
+++ branches/quickbook-dev/tools/quickbook/src/iterator.hpp 2011-11-22 18:45:37 EST (Tue, 22 Nov 2011)
@@ -16,15 +16,6 @@
namespace quickbook
{
- struct file_position
- {
- file_position() : line(1), column(1) {}
- file_position(int l, int c) : line(l), column(c) {}
-
- int line;
- int column;
- };
-
template <typename Iterator>
struct lookback_iterator
: boost::forward_iterator_helper<
@@ -38,8 +29,6 @@
lookback_iterator() {}
explicit lookback_iterator(Iterator base)
: original_(base), base_(base) {}
- explicit lookback_iterator(Iterator base, file_position const& position)
- : original_(base), base_(base) {}
friend bool operator==(
lookback_iterator const& x,
@@ -75,56 +64,6 @@
Iterator original_;
Iterator base_;
};
-
- template <typename String, typename Iterator>
- file_position get_position(
- Iterator iterator,
- String const& source)
- {
- file_position pos;
- Iterator line_begin = source.begin();
-
- Iterator begin = source.begin();
- while (begin != iterator)
- {
- assert(begin != source.end());
-
- if (*begin == '\r')
- {
- ++begin;
- ++pos.line;
- line_begin = begin;
- }
- else if (*begin == '\n')
- {
- ++begin;
- ++pos.line;
- line_begin = begin;
- if (begin == iterator) break;
- assert(begin != source.end());
- if (*begin == '\r')
- {
- ++begin;
- line_begin = begin;
- }
- }
- else
- {
- ++begin;
- }
- }
-
- pos.column = iterator - line_begin + 1;
- return pos;
- }
-
- template <typename String, typename Iterator>
- file_position get_position(
- lookback_iterator<Iterator> iterator,
- String const& source)
- {
- return get_position(iterator.base(), source);
- }
}
#endif
Modified: branches/quickbook-dev/tools/quickbook/src/main_grammar.cpp
==============================================================================
--- branches/quickbook-dev/tools/quickbook/src/main_grammar.cpp (original)
+++ branches/quickbook-dev/tools/quickbook/src/main_grammar.cpp 2011-11-22 18:45:37 EST (Tue, 22 Nov 2011)
@@ -391,7 +391,7 @@
[ ( qbk_since(106u)
>> *(line_comment | (cl::anychar_p - (cl::eol_p | '[' | ']')))
| qbk_before(106u)
- >> *(line_comment | (cl::anychar_p - (cl::eol_p | "[/")))
+ >> *(line_comment | (cl::anychar_p - (cl::eol_p | "[/")))
)
>> *eol
] [actions.element]
Modified: branches/quickbook-dev/tools/quickbook/src/quickbook.cpp
==============================================================================
--- branches/quickbook-dev/tools/quickbook/src/quickbook.cpp (original)
+++ branches/quickbook-dev/tools/quickbook/src/quickbook.cpp 2011-11-22 18:45:37 EST (Tue, 22 Nov 2011)
@@ -92,7 +92,7 @@
if (!info.full)
{
- file_position const& pos = get_position(info.stop, actor.current_file->source);
+ file_position const& pos = actor.current_file->position_of(info.stop.base());
detail::outerr(actor.current_file->path, pos.line)
<< "Syntax Error near column " << pos.column << ".\n";
++actor.error_count;
Modified: branches/quickbook-dev/tools/quickbook/src/syntax_highlight.cpp
==============================================================================
--- branches/quickbook-dev/tools/quickbook/src/syntax_highlight.cpp (original)
+++ branches/quickbook-dev/tools/quickbook/src/syntax_highlight.cpp 2011-11-22 18:45:37 EST (Tue, 22 Nov 2011)
@@ -141,7 +141,7 @@
void unexpected_char::operator()(parse_iterator first, parse_iterator last) const
{
- file_position const pos = get_position(first, escape_actions.current_file->source);
+ file_position const pos = escape_actions.current_file->position_of(first.base());
detail::outwarn(escape_actions.current_file->path, pos.line)
<< "in column:" << pos.column
Modified: branches/quickbook-dev/tools/quickbook/src/utils.cpp
==============================================================================
--- branches/quickbook-dev/tools/quickbook/src/utils.cpp (original)
+++ branches/quickbook-dev/tools/quickbook/src/utils.cpp 2011-11-22 18:45:37 EST (Tue, 22 Nov 2011)
@@ -45,66 +45,6 @@
return static_cast<char>(std::tolower(static_cast<unsigned char>(ch)));
}
- // un-indent a code segment
- void unindent(std::string& program)
- {
- // Erase leading blank lines and newlines:
- std::string::size_type start = program.find_first_not_of(" \t");
- if (start != std::string::npos &&
- (program[start] == '\r' || program[start] == '\n'))
- {
- program.erase(0, start);
- }
- start = program.find_first_not_of("\r\n");
- program.erase(0, start);
-
- if (program.size() == 0)
- return; // nothing left to do
-
- // Get the first line indent
- std::string::size_type indent = program.find_first_not_of(" \t");
- std::string::size_type pos = 0;
- if (std::string::npos == indent)
- {
- // Nothing left to do here. The code is empty (just spaces).
- // We clear the program to signal the caller that it is empty
- // and return early.
- program.clear();
- return;
- }
-
- // Calculate the minimum indent from the rest of the lines
- do
- {
- pos = program.find_first_not_of("\r\n", pos);
- if (std::string::npos == pos)
- break;
-
- std::string::size_type n = program.find_first_not_of(" \t", pos);
- if (n != std::string::npos)
- {
- char ch = program[n];
- if (ch != '\r' && ch != '\n') // ignore empty lines
- indent = (std::min)(indent, n-pos);
- }
- }
- while (std::string::npos != (pos = program.find_first_of("\r\n", pos)));
-
- // Trim white spaces from column 0..indent
- pos = 0;
- program.erase(0, indent);
- while (std::string::npos != (pos = program.find_first_of("\r\n", pos)))
- {
- if (std::string::npos == (pos = program.find_first_not_of("\r\n", pos)))
- {
- break;
- }
-
- std::string::size_type next = program.find_first_of("\r\n", pos);
- program.erase(pos, (std::min)(indent, next-pos));
- }
- }
-
std::string escape_uri(std::string uri)
{
for (std::string::size_type n = 0; n < uri.size(); ++n)
Modified: branches/quickbook-dev/tools/quickbook/src/utils.hpp
==============================================================================
--- branches/quickbook-dev/tools/quickbook/src/utils.hpp (original)
+++ branches/quickbook-dev/tools/quickbook/src/utils.hpp 2011-11-22 18:45:37 EST (Tue, 22 Nov 2011)
@@ -32,9 +32,6 @@
return out_name;
}
- // un-indent a code segment
- void unindent(std::string& program);
-
std::string escape_uri(std::string uri);
// given a file extension, return the type of the source file
Modified: branches/quickbook-dev/tools/quickbook/test/unit/Jamfile.v2
==============================================================================
--- branches/quickbook-dev/tools/quickbook/test/unit/Jamfile.v2 (original)
+++ branches/quickbook-dev/tools/quickbook/test/unit/Jamfile.v2 2011-11-22 18:45:37 EST (Tue, 22 Nov 2011)
@@ -15,7 +15,7 @@
<library>/boost//filesystem
;
-run values_test.cpp ../../src/values.cpp ../../src/string_ref.cpp ;
+run values_test.cpp ../../src/values.cpp ../../src/files.cpp ../../src/string_ref.cpp ;
run post_process_test.cpp ../../src/post_process.cpp ;
# Copied from spirit