filesystem: fixing utf-8 issues switching to jinja

Switching to work fully with utf-8 files also for the sitemap.txt
Switching to template engine jinja2 for better unicode support.
Templates has to be refreshed.

diff --git a/treecutter/directory.py b/treecutter/directory.py
index 19c4d8b4182cf4d4764d9ec77442a0473fd76839..7519a6b27bd4b044ccb57986eaf9961b10eed846 100644 (file)
--- a/treecutter/directory.py
+++ b/treecutter/directory.py
@@ -7,7 +7,7 @@ import treecutter.constants as const
 class Directory():
     """Class containing the state of the directory with articles"""
     def __init__(self):
-        self._cwd = '.'
+        self._cwd = u'.'
         self._tree = []
 
     def scan(self):

diff --git a/treecutter/page.py b/treecutter/page.py
index e8367bd8459f6072be89fb475c204acba06ca6a7..ae056523be9ed2b1cca774c37f8cc96c31c5d012 100644 (file)
--- a/treecutter/page.py
+++ b/treecutter/page.py
@@ -5,8 +5,10 @@ import tempfile
 import re
 import getpass
 import gnupg
+import codecs
 from lxml import etree
-from Cheetah.Template import Template
+#from jinja2 import Template
+import jinja2
 from pkg_resources import resource_filename, resource_listdir
 from time import time
 import treecutter.constants as const
@@ -129,7 +131,7 @@ class Page():
         retcode = subprocess.call(cmd)
         if retcode:
             print 'Error: '+' '.join(cmd)+' Returncode ['+str(retcode)+']'
-        tfo = open(outfile,'r')
+        tfo = codecs.open(outfile, "r", "utf-8")
         self._rendered_article = tfo.read()
         tfo.close()
         os.remove(infile)
@@ -140,14 +142,23 @@ class Page():
         htmlmenu =  sitemap.gen_menu(self._lang,None,"menu")
         levelmenu = sitemap.gen_menu(self._lang,self,"tree")
         langmenu = sitemap.lang_menu(self._lang,self._link)
-        template = Template(file=style+'index.'+self._lang+'.html.tmpl',
-                            searchList=[{'title':self._title},
-                                        {'menu':htmlmenu},
-                                        {'article':self._rendered_article},
-                                        {'levelmenu':levelmenu},
-                                        {'langmenu':langmenu}])
-        outfile = tdir+'html'.join(self._file.rsplit('xml',1))
+        article = self._rendered_article
+        templateLoader = jinja2.FileSystemLoader( searchpath="/" )
+        templateEnv = jinja2.Environment( loader=templateLoader )
+
+        templateVars = {'title':self._title,
+                        'menu':htmlmenu,
+                        'article':article,
+                        'levelmenu':levelmenu,
+                        'langmen':langmenu}
+
+        s = unicode(style, "utf-8")
+        t = s+'index.'+self._lang+'.xhtml.tmpl'
+        template = templateEnv.get_template( t )
+        templateout = template.render( templateVars )
+
+        outfile = tdir+'xhtml'.join(self._file.rsplit('xml',1))
         mkdir_p(os.path.dirname(outfile))
-        out = open(outfile, 'w')
-        out.write(str(template))
+        out = codecs.open(outfile, 'w', 'utf-8')
+        out.write(templateout)
         out.close()

diff --git a/treecutter/sitemap.py b/treecutter/sitemap.py
index 24b53fe5b4ab3bd84d65caca16cd1e1e07db54a6..0ecefe4278efae22c581d702bb7c513630e43700 100644 (file)
--- a/treecutter/sitemap.py
+++ b/treecutter/sitemap.py
@@ -1,5 +1,6 @@
 #!/usr/bin/python
 import os
+import codecs
 import re
 import shutil
 import gettext
@@ -30,17 +31,18 @@ class Sitemap():
     # representing the text on the site.
     # A link can have several pages in different languages.
     def add_link(self, link):
-        tokens = filter(None,re.split(r'(^/[\w-]*/|[\w-]*/)',link))
+        tokens = filter(None,re.split(r'(^/[\w:-]*$|^/[\w:-]*/|[\w:-]*/)',link,flags=re.UNICODE))
         self._tree.add(tokens,Link(link))
 
     def write_map(self):
-        f = open(self._file,'w')
-        f.write('\n'.join(link.link() for link in self._tree))
+        f = codecs.open(self._file,'w','utf-8')
+        s = '\n'.join(link.link() for link in self._tree)
+        f.write(s)
         f.close()
 
     def read_map(self):
         try:
-            f = open(self._file)
+            f = codecs.open(self._file, 'r', 'utf-8')
             sml = f.read().split()
             f.close()
             for line in sml:
@@ -113,10 +115,10 @@ class Sitemap():
             ln = self._isocode.xpath(isoxml)[0].get('name')
             if lang != 'en':
                 ln = self._tranlang[lang].gettext(ln)
-            p = link.link()
-            if p[-1] == '/':
-                p = p +'index'
-            p = p+'.'+l
+            p = unicode(link.link())
+            if p[-1] == u'/':
+                p = p +u'index'
+            p = p+u'.'+l
             li = html.li(html.a(ln.decode('utf-8'),
                                 href=self._subdir+p,hreflang=l))
             menu.append(li)