From 521ca183bb031c954d2f014f12d4f483e9d2cbbd Mon Sep 17 00:00:00 2001
From: Fredrik Unger <fred@tree.se>
Date: Fri, 7 Mar 2014 12:34:04 +0100
Subject: [PATCH] filesystem: fixing utf-8 issues switching to jinja

Switching to work fully with utf-8 files also for the sitemap.txt
Switching to template engine jinja2 for better unicode support.
Templates has to be refreshed.
---
 treecutter/directory.py |  2 +-
 treecutter/page.py      | 33 ++++++++++++++++++++++-----------
 treecutter/sitemap.py   | 18 ++++++++++--------
 3 files changed, 33 insertions(+), 20 deletions(-)

diff --git a/treecutter/directory.py b/treecutter/directory.py
index 19c4d8b..7519a6b 100644
--- a/treecutter/directory.py
+++ b/treecutter/directory.py
@@ -7,7 +7,7 @@ import treecutter.constants as const
 class Directory():
     """Class containing the state of the directory with articles"""
     def __init__(self):
-        self._cwd = '.'
+        self._cwd = u'.'
         self._tree = []
 
     def scan(self):
diff --git a/treecutter/page.py b/treecutter/page.py
index e8367bd..ae05652 100644
--- a/treecutter/page.py
+++ b/treecutter/page.py
@@ -5,8 +5,10 @@ import tempfile
 import re
 import getpass
 import gnupg
+import codecs
 from lxml import etree
-from Cheetah.Template import Template
+#from jinja2 import Template
+import jinja2
 from pkg_resources import resource_filename, resource_listdir
 from time import time
 import treecutter.constants as const
@@ -129,7 +131,7 @@ class Page():
         retcode = subprocess.call(cmd)
         if retcode:
             print 'Error: '+' '.join(cmd)+' Returncode ['+str(retcode)+']'
-        tfo = open(outfile,'r')
+        tfo = codecs.open(outfile, "r", "utf-8")
         self._rendered_article = tfo.read()
         tfo.close()
         os.remove(infile)
@@ -140,14 +142,23 @@ class Page():
         htmlmenu =  sitemap.gen_menu(self._lang,None,"menu")
         levelmenu = sitemap.gen_menu(self._lang,self,"tree")
         langmenu = sitemap.lang_menu(self._lang,self._link)
-        template = Template(file=style+'index.'+self._lang+'.html.tmpl',
-                            searchList=[{'title':self._title},
-                                        {'menu':htmlmenu},
-                                        {'article':self._rendered_article},
-                                        {'levelmenu':levelmenu},
-                                        {'langmenu':langmenu}])
-        outfile = tdir+'html'.join(self._file.rsplit('xml',1))
+        article = self._rendered_article
+        templateLoader = jinja2.FileSystemLoader( searchpath="/" )
+        templateEnv = jinja2.Environment( loader=templateLoader )
+
+        templateVars = {'title':self._title,
+                        'menu':htmlmenu,
+                        'article':article,
+                        'levelmenu':levelmenu,
+                        'langmen':langmenu}
+
+        s = unicode(style, "utf-8")
+        t = s+'index.'+self._lang+'.xhtml.tmpl'
+        template = templateEnv.get_template( t )
+        templateout = template.render( templateVars )
+
+        outfile = tdir+'xhtml'.join(self._file.rsplit('xml',1))
         mkdir_p(os.path.dirname(outfile))
-        out = open(outfile, 'w')
-        out.write(str(template))
+        out = codecs.open(outfile, 'w', 'utf-8')
+        out.write(templateout)
         out.close()
diff --git a/treecutter/sitemap.py b/treecutter/sitemap.py
index 24b53fe..0ecefe4 100644
--- a/treecutter/sitemap.py
+++ b/treecutter/sitemap.py
@@ -1,5 +1,6 @@
 #!/usr/bin/python
 import os
+import codecs
 import re
 import shutil
 import gettext
@@ -30,17 +31,18 @@ class Sitemap():
     # representing the text on the site.
     # A link can have several pages in different languages.
     def add_link(self, link):
-        tokens = filter(None,re.split(r'(^/[\w-]*/|[\w-]*/)',link))
+        tokens = filter(None,re.split(r'(^/[\w:-]*$|^/[\w:-]*/|[\w:-]*/)',link,flags=re.UNICODE))
         self._tree.add(tokens,Link(link))
 
     def write_map(self):
-        f = open(self._file,'w')
-        f.write('\n'.join(link.link() for link in self._tree))
+        f = codecs.open(self._file,'w','utf-8')
+        s = '\n'.join(link.link() for link in self._tree)
+        f.write(s)
         f.close()
 
     def read_map(self):
         try:
-            f = open(self._file)
+            f = codecs.open(self._file, 'r', 'utf-8')
             sml = f.read().split()
             f.close()
             for line in sml:
@@ -113,10 +115,10 @@ class Sitemap():
             ln = self._isocode.xpath(isoxml)[0].get('name')
             if lang != 'en':
                 ln = self._tranlang[lang].gettext(ln)
-            p = link.link()
-            if p[-1] == '/':
-                p = p +'index'
-            p = p+'.'+l
+            p = unicode(link.link())
+            if p[-1] == u'/':
+                p = p +u'index'
+            p = p+u'.'+l
             li = html.li(html.a(ln.decode('utf-8'),
                                 href=self._subdir+p,hreflang=l))
             menu.append(li)
-- 
2.30.2