Subject: [Boost-commit] svn:boost r83181 - in website/public_html/live: . site-tools/boost_site
From: dnljms_at_[hidden]
Date: 2013-02-27 04:05:42
Author: danieljames
Date: 2013-02-27 04:05:41 EST (Wed, 27 Feb 2013)
New Revision: 83181
URL: http://svn.boost.org/trac/boost/changeset/83181
Log:
Website: Merge initial part of rss change.
I have to do this in two parts, because the second part breaks the update code
by removing the old rss handling implementation.
Properties modified:
website/public_html/live/ (props changed)
Text files modified:
website/public_html/live/site-tools/boost_site/site_tools.py | 67 ++++++++++++---------
website/public_html/live/site-tools/boost_site/state.py | 121 +++++++++++++++++++--------------------
website/public_html/live/site-tools/boost_site/upgrade.py | 35 ++++++++++
3 files changed, 130 insertions(+), 93 deletions(-)
Modified: website/public_html/live/site-tools/boost_site/site_tools.py
==============================================================================
--- website/public_html/live/site-tools/boost_site/site_tools.py (original)
+++ website/public_html/live/site-tools/boost_site/site_tools.py 2013-02-27 04:05:41 EST (Wed, 27 Feb 2013)
@@ -4,7 +4,7 @@
# (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
import os, sys, subprocess, glob, re, time, xml.dom.minidom, codecs
-import boost_site.pages, boost_site.boostbook_parser, boost_site.util
+import boost_site.pages, boost_site.boostbook_parser, boost_site.util, boost_site.state
from boost_site.settings import settings
################################################################################
@@ -62,35 +62,38 @@
# Generate RSS feeds
if not refresh:
- old_rss_items_doc = xml.dom.minidom.parseString('''<items></items>''')
- old_rss_items = {}
- for feed_file in settings['feeds']:
- old_rss_items.update(pages.load_rss(feed_file, old_rss_items_doc))
+ rss_items = boost_site.state.load('generated/state/rss-items.txt')
for feed_file in settings['feeds']:
feed_data = settings['feeds'][feed_file]
- rss_feed = generate_rss_feed(feed_file, feed_data)
- rss_channel = rss_feed.getElementsByTagName('channel')[0]
+ rss_feed = rss_prefix(feed_file, feed_data)
feed_pages = pages.match_pages(feed_data['matches'])
if 'count' in feed_data:
feed_pages = feed_pages[:feed_data['count']]
for qbk_page in feed_pages:
+ item_xml = None
+
if qbk_page.loaded:
- item = generate_rss_item(rss_feed, qbk_page.qbk_file, qbk_page)
+ item = generate_rss_item(qbk_page.qbk_file, qbk_page)
pages.add_rss_item(item)
- rss_channel.appendChild(item['item'])
- elif qbk_page.qbk_file in old_rss_items:
- rss_channel.appendChild(
- rss_feed.importNode(
- old_rss_items[qbk_page.qbk_file]['item'], True))
+
+ item['item'] = item['item'].toxml('utf-8').decode('utf-8')
+ rss_items[qbk_page.qbk_file] = item
+ boost_site.state.save(rss_items, 'generated/state/rss-items.txt')
+
+ rss_feed += item['item']
+ elif qbk_page.qbk_file in rss_items:
+ rss_feed += rss_items[qbk_page.qbk_file]['item']
else:
print("Missing entry for %s" % qbk_page.qbk_file)
-
+
+ rss_feed += rss_postfix(feed_file, feed_data)
+
output_file = open(feed_file, 'wb')
try:
- output_file.write(rss_feed.toxml('utf-8'))
+ output_file.write(rss_feed.encode('utf-8'))
finally:
output_file.close()
@@ -108,8 +111,8 @@
################################################################################
-def generate_rss_feed(feed_file, details):
- rss = xml.dom.minidom.parseString('''<?xml version="1.0" encoding="UTF-8"?>
+def rss_prefix(feed_file, details):
+ return('''<?xml version="1.0" encoding="UTF-8"?>
<rss version="2.0" xmlns:boostbook="urn:boost.org:boostbook">
<channel>
<generator>Boost Website Site Tools</generator>
@@ -118,8 +121,6 @@
<description>%(description)s</description>
<language>%(language)s</language>
<copyright>%(copyright)s</copyright>
- </channel>
-</rss>
''' % {
'title' : encode_for_rss(details['title']),
'link' : encode_for_rss("http://www.boost.org/" + details['link']),
@@ -128,38 +129,46 @@
'copyright' : 'Distributed under the Boost Software License, Version 1.0. (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)'
} )
- return rss
+def rss_postfix(feed_file, details):
+ return '''
+ </channel>
+</rss>
+'''
-def generate_rss_item(rss_feed, qbk_file, page):
+def generate_rss_item(qbk_file, page):
assert page.loaded
+ rss_xml = xml.dom.minidom.parseString('''<?xml version="1.0" encoding="UTF-8"?>
+ <rss version="2.0" xmlns:boostbook="urn:boost.org:boostbook">
+ </rss>''')
+
page_link = 'http://www.boost.org/%s' % page.location
- item = rss_feed.createElement('item')
+ item = rss_xml.createElement('item')
node = xml.dom.minidom.parseString('<title>%s</title>'
% encode_for_rss(page.title_xml))
- item.appendChild(rss_feed.importNode(node.documentElement, True))
+ item.appendChild(rss_xml.importNode(node.documentElement, True))
node = xml.dom.minidom.parseString('<link>%s</link>'
% encode_for_rss(page_link))
- item.appendChild(rss_feed.importNode(node.documentElement, True))
+ item.appendChild(rss_xml.importNode(node.documentElement, True))
node = xml.dom.minidom.parseString('<guid>%s</guid>'
% encode_for_rss(page_link))
- item.appendChild(rss_feed.importNode(node.documentElement, True))
+ item.appendChild(rss_xml.importNode(node.documentElement, True))
# TODO: Convert date format?
- node = rss_feed.createElement('pubDate')
- node.appendChild(rss_feed.createTextNode(page.pub_date))
+ node = rss_xml.createElement('pubDate')
+ node.appendChild(rss_xml.createTextNode(page.pub_date))
item.appendChild(node)
- node = rss_feed.createElement('description')
+ node = rss_xml.createElement('description')
# Placing the description in a root element to make it well formed xml.
description = xml.dom.minidom.parseString(
'<x>%s</x>' % encode_for_rss(page.description_xml))
boost_site.util.base_links(description, page_link)
- node.appendChild(rss_feed.createTextNode(
+ node.appendChild(rss_xml.createTextNode(
boost_site.util.fragment_to_string(description.firstChild)))
item.appendChild(node)
Modified: website/public_html/live/site-tools/boost_site/state.py
==============================================================================
--- website/public_html/live/site-tools/boost_site/state.py (original)
+++ website/public_html/live/site-tools/boost_site/state.py 2013-02-27 04:05:41 EST (Wed, 27 Feb 2013)
@@ -11,78 +11,73 @@
def load(file_path):
state = {}
- if(file_path and os.path.isfile(file_path)):
+ if file_path and os.path.isfile(file_path):
file = open(file_path)
try:
while (True):
c = file.read(1)
- if(not c):
+ if not c:
break
- if(c == '#'):
+ if c == '#':
file.readline()
continue
- if(c != '('):
+ elif c == '(':
+ record_key = file.readline().rstrip()
+ if not record_key: raise StateParseError()
+ state[record_key] = read_record(file)
+ else:
raise StateParseError()
- record_key = file.readline().rstrip()
- if(not record_key): raise StateParseError()
- record = {}
- key = None
- value = None
- type = None
- while (True):
- c = file.read(1)
- if((c == ')' or c == '-') and key):
- if(not key):
- raise StateParseError()
-
- if(type == 'String'):
- value = value[:-1]
-
- record[key] = value
-
- if(c == ')'):
- if(file.readline() != '\n'): raise StateParseError()
- break
- elif(c == '-'):
- key = file.readline().rstrip()
- if(not key): raise StateParseError()
- type = 'None'
- value = None
- elif(c == '.'):
- if(not key or type != 'None'): raise StateParseError()
- type = 'Float'
- value = float(file.readline())
- elif(c == '!'):
- if(not key or type != 'None'): raise StateParseError()
- type = 'Bool'
- value = bool(file.readline())
- elif(c == '='):
- if(not key or type != 'None'): raise StateParseError()
- type = 'Int'
- value = int(file.readline())
- elif(c == '"'):
- if(not key): raise StateParseError()
- if(type == 'None'):
- type = 'String'
- if sys.version_info < (3, 0):
- value = file.readline().decode('utf-8')
- else:
- value = file.readline()
- elif(type == 'String'):
- if sys.version_info < (3, 0):
- value = value + file.readline().decode('utf-8')
- else:
- value = value + file.readline()
- else:
- raise StateParseError()
- else:
- raise StateParseError()
- state[record_key] = record
finally:
file.close()
return state
+def read_record(file):
+ record = {}
+
+ # This function sometimes needs to lookahead at the first character in a
+ # line, so always read it in advance.
+ c = file.read(1)
+
+ while (True):
+ if not c: raise StateParseError()
+
+ if c == ')':
+ if file.readline() != '\n': raise StateParseError()
+ return record
+
+ if c != '-': raise StateParseError()
+
+ key = file.readline().rstrip()
+ c = file.read(1)
+
+ if c == ')' or c == '-':
+ # The key has no value, so don't read anything. This 'c' will
+ # be dealt with in the next loop.
+ record[key] = None
+ elif c == '.':
+ record[key] = float(file.readline())
+ c = file.read(1)
+ elif c == '!':
+ record[key] = bool(file.readline())
+ c = file.read(1)
+ elif c == '=':
+ record[key] = int(file.readline())
+ c = file.read(1)
+ elif c == '"':
+ value = []
+ while c == '"':
+ if sys.version_info < (3, 0):
+ value.append(file.readline().decode('utf-8'))
+ else:
+ value.append(file.readline())
+
+ c = file.read(1)
+
+ record[key] = (''.join(value))[:-1]
+ else:
+ raise StateParseError()
+
def save(state, file_path):
file = open(file_path, "wb")
try:
@@ -108,7 +103,11 @@
write(file, '!')
write(file, str(record[key]))
write(file, "\n")
- elif isinstance(record[key], (int, float)):
+ elif isinstance(record[key], int):
+ write(file, '=')
+ write(file, str(record[key]))
+ write(file, "\n")
+ elif isinstance(record[key], float):
write(file, '.')
write(file, str(record[key]))
write(file, "\n")
@@ -121,4 +120,4 @@
file.close()
def write(file, str):
- file.write(str.encode('utf-8'))
\ No newline at end of file
+ file.write(str.encode('utf-8'))
Modified: website/public_html/live/site-tools/boost_site/upgrade.py
==============================================================================
--- website/public_html/live/site-tools/boost_site/upgrade.py (original)
+++ website/public_html/live/site-tools/boost_site/upgrade.py 2013-02-27 04:05:41 EST (Wed, 27 Feb 2013)
@@ -17,6 +17,7 @@
page.pub_date != 'In Progress':
page.flags.add('released')
pages.save()
+ return True
def upgrade2():
pages = boost_site.site_tools.load_pages()
@@ -28,6 +29,7 @@
else:
page.type = 'page'
pages.save()
+ return True
def upgrade3():
pages_raw = boost_site.state.load('generated/state/feed-pages.txt')
@@ -51,11 +53,37 @@
raise Exception("Unexpected flags: " + str(flags))
del page_details['flags']
boost_site.state.save(pages_raw, 'generated/state/feed-pages.txt')
+ return True
+
+def upgrade4():
+ """
+ Save all the rss entries to a state file.
+ Will remove the rss hashes soon, which should improve some the
+ rss handling a bit.
+ """
+ import xml.dom.minidom
+ from boost_site.settings import settings
+
+ pages = boost_site.site_tools.load_pages()
+
+ # Load RSS items from feeds.
+ old_rss_items_doc = xml.dom.minidom.parseString('''<items></items>''')
+ old_rss_items = {}
+ for feed_file in settings['feeds']:
+ old_rss_items.update(pages.load_rss(feed_file, old_rss_items_doc))
+
+ # Convert items to text (TODO: Should I support XML in state files?)
+ for file in old_rss_items:
+ old_rss_items[file]['item'] = old_rss_items[file]['item'].toxml('utf-8').decode('utf-8')
+
+ boost_site.state.save(old_rss_items, 'generated/state/rss-items.txt')
+ return True
versions = [
upgrade1,
upgrade2,
- upgrade3
+ upgrade3,
+ upgrade4
]
#
@@ -69,8 +97,9 @@
print("Upgrading to new version.")
for v in range(version.version, len(versions)):
- print("Upgrade " + (v + 1))
- versions[v]()
+ print("Upgrade " + str(v + 1))
+ if not versions[v]():
+ raise Exception("Error upgrading to version v")
version.version = v + 1
version.save()