src/tree-cutter.py

   1 #!/usr/bin/python
   2 import os
   3 import fnmatch
   4 import subprocess
   5 import amara
   6 import re
   7 import tempfile
   8 import errno
   9 import time
  10 import argparse
  11 import shutil
  12 import pygraphviz as pgv
  13 import glob
  14 import gettext
  15 from amara import bindery
  16 from amara.xslt import transform
  17 from Cheetah.Template import Template
  18
  19 parser = argparse.ArgumentParser(description='Process docbook article tree.')
  20 parser.add_argument('--style', nargs='?',
  21                     default=os.path.dirname(os.getcwd())+'/style/default/')
  22 parser.add_argument('--output', nargs='?',
  23                     default=os.path.dirname(os.getcwd())+'/htdocs/')
  24 args = parser.parse_args()
  25
  26 style_xslt = args.style+"docbook.xsl"
  27 style_tmpl = args.style+"index.en.html.tmpl"
  28 outputdir = args.output
  29
  30 tmptarget = tempfile.mkdtemp()+'/'
  31
  32 valid_scripts = ['.py','.pl']
  33 MAXLEVEL = 10000
  34
  35 def mkdir_p(path):
  36     try:
  37         os.makedirs(path)
  38     except OSError as exc: # Python >2.5
  39         if exc.errno == errno.EEXIST:
  40             pass
  41         else: raise
  42
  43 def publish(src,target):
  44     cmd = ["rsync","-a","--delete",src,target]
  45     retcode = subprocess.call(cmd)
  46     if retcode:
  47         print 'Error: '+' '.join(cmd)+' Returncode ['+str(retcode)+']'
  48
  49
  50 PREFIXES={u'db': u'http://docbook.org/ns/docbook',
  51           u'xi': u'http://www.w3.org/2001/XInclude',
  52           u'xl': u'http://www.w3.org/1999/xlink'}
  53
  54 class Directory():
  55     """Class containing the state of the directory with articles"""
  56     def __init__(self):
  57         self._cwd = '.'
  58         self._tree = []
  59
  60     def scan(self):
  61         for dirname, dirnames, filenames in os.walk(self._cwd):
  62             for filename in filenames:
  63                 if fnmatch.fnmatch(filename, '*.xml'):
  64                     file_ = os.path.join(dirname,filename)
  65                     doc = bindery.parse(file_, prefixes=PREFIXES)
  66                     title = doc.xml_select(u'/db:article/db:info/db:title')
  67                     menu  = doc.xml_select(u'/db:article/db:info/db:titleabbrev')
  68                     if title and menu:
  69                         base = file_.split('.')[1]
  70                         link = base.replace('index','')
  71                         self._tree.append(link)
  72
  73     def set(self):
  74         return set(self._tree)
  75
  76 class Page():
  77     """Class representing a version of a webpage"""
  78     def __init__(self,link,page):
  79         self._link = link
  80         self._file = page[1]
  81         self._lang = page[0]
  82         self._doc = None
  83         self._resources = []
  84         self._title = None
  85         self._menu = None
  86         self._rendered_article = None
  87
  88     def language(self):
  89         return self._lang
  90
  91     def resources(self):
  92         return set(self._resources)
  93
  94     def menu(self):
  95         return self._menu
  96
  97     def set_article(self,art):
  98         self._rendered_article = art
  99
 100     def prepare(self):
 101         self._doc = bindery.parse(self._file, prefixes=PREFIXES)
 102         if self._doc.xml_select(u'/db:article/db:info/db:title'):
 103             self._title = unicode(self._doc.article.info.title)
 104         if self._doc.xml_select(u'/db:article/db:info/db:titleabbrev'):
 105             self._menu = unicode(self._doc.article.info.titleabbrev)
 106
 107         dirname = os.path.dirname(self._file)
 108         code  = self._doc.xml_select(u"//xi:include[@parse='text']")
 109         if code:
 110             for c in code:
 111                 (p, ext) = os.path.splitext(c.href)
 112                 if ext in valid_scripts:
 113                     exe = os.path.join(os.path.abspath(dirname)+'/'+c.href)
 114                     xml = subprocess.Popen([exe],stdout=subprocess.PIPE)
 115                     xstr = bindery.parse(str(xml.stdout.read()))
 116                     idp = c.xml_index_on_parent
 117                     for x in xstr.xml_children:
 118                         c.xml_parent.xml_insert(idp,x)
 119                         c.xml_parent.xml_remove(c)
 120
 121         for r in self._doc.xml_select(u"//db:link[@xl:href]"):
 122             rf = os.path.join(dirname,r.href)
 123             if os.path.isfile(rf):
 124                 self._resources.append(rf)
 125         for i in self._doc.xml_select(u"//db:imagedata[@fileref]"):
 126             im = os.path.join(dirname,i.fileref)
 127             if os.path.isfile(im):
 128                 self._resources.append(im)
 129
 130     def render(self):
 131         #  amara can not handle the docbook stylesheets
 132         #  xmlarticle = transform(doc,style_xslt)
 133         cwd = os.getcwd()
 134         dirname = os.path.dirname(self._file)
 135         os.chdir(dirname)
 136         infile  = os.path.basename(tempfile.mktemp())
 137         outfile = tempfile.mktemp()
 138         tfi = open(infile,'w')
 139         tfi.write(self._doc.xml_encode())
 140         tfi.close()
 141 #  cmd = ["saxon-xslt-xinclude","-o",outfile,infile,style_xslt]
 142         cmd = ["xsltproc","--xinclude","--output",outfile,style_xslt,infile]
 143         retcode = subprocess.call(cmd)
 144         if retcode:
 145             print 'Error: '+' '.join(cmd)+' Returncode ['+str(retcode)+']'
 146         tfo = open(outfile,'r')
 147         self._rendered_article = tfo.read()
 148         tfo.close()
 149         os.remove(infile)
 150         os.remove(outfile)
 151         os.chdir(cwd)
 152
 153     def template(self,sitemap):
 154         htmlmenu =  sitemap.gen_menu(self._lang,None,"menu")
 155         levelmenu = sitemap.gen_menu(self._lang,self,"tree")
 156         template = Template(file=style_tmpl,
 157         langmenu = sitemap.lang_menu(self._lang,self._link)
 158                             searchList=[{'title':self._title},
 159                                         {'menu':htmlmenu},
 160                                         {'article':self._rendered_article},
 161                                         {'levelmenu':levelmenu},
 162                                         {'langmenu':langmenu}])
 163         outfile = tmptarget+'html'.join(self._file.rsplit('xml',1))
 164         mkdir_p(os.path.dirname(outfile))
 165         out = open(outfile, 'w')
 166         out.write(str(template))
 167         out.close()
 168
 169
 170 class Link():
 171     """Class representing a webpage on the site"""
 172     def __init__(self,link):
 173         self._link = link
 174         # find the representations of the link.
 175         self._pages = []
 176         path = link
 177         if self._link[-1] == '/':
 178             path = path+'index'
 179         lang = self._scan_languages(path)
 180         for l in lang:
 181             self._pages.append(Page(self,l))
 182
 183     def add_page(self,l):
 184         self._pages.append(Page(self,l))
 185
 186     def _scan_languages(self,path):
 187         lang = []
 188         for l in  glob.glob('.'+path+'*'):
 189             ls = l.split('.')
 190             if len(ls) > 3 and ls[3] == 'xml':
 191                 lang.append((ls[2],l))
 192         return lang
 193
 194     def link(self):
 195         return self._link
 196
 197     def prepare(self):
 198         for page in self._pages:
 199             page.prepare()
 200
 201     def languages(self):
 202         p = []
 203         for page in self._pages:
 204             p.append(page.language())
 205         return p
 206
 207     def render(self):
 208         for page in self._pages:
 209             page.render()
 210
 211     def template(self,sitemap):
 212         for page in self._pages:
 213             page.template(sitemap)
 214
 215     def page(self,lang):
 216         for page in self._pages:
 217             if page.language()==lang:
 218                 return page
 219         return None
 220
 221     def resources(self):
 222         res  = set()
 223         for page in self._pages:
 224             res = res.union(page.resources())
 225         return res
 226
 227
 228 class Node():
 229     def __init__(self,token,value):
 230         self._token = token
 231         self._value = value
 232         self._children = []
 233
 234     def token(self):
 235         return self._token
 236
 237     def value(self):
 238         return self._value
 239
 240     def children(self):
 241         return self._children
 242
 243 class Trie():
 244     def __init__(self):
 245         self._root = []
 246
 247     def __iter__(self):
 248         return self.inorder(self._root)
 249
 250     def inorder(self,t):
 251         for l in t:
 252             yield l.value()
 253             for x in self.inorder(l.children()):
 254                 yield x
 255
 256     def _add(self,trie, key, content):
 257         # is the key a leaf
 258         k = key.pop(0)
 259         if key == []:
 260             node = Node(k,content)
 261             trie.append(node)
 262         else:
 263             for ch in trie:
 264                 if ch.token() == k:
 265                     self._add(ch.children(), key, content)
 266
 267     def add(self,key, content):
 268         self._add(self._root, key, content)
 269
 270     def _graph(self, trie, G):
 271         for l in trie:
 272             G.add_node(l.token())
 273             for ch in l.children():
 274                 G.add_edge(l.token(),ch.token())
 275                 self._graph(l.children(), G)
 276
 277     def graph(self):
 278         G = pgv.AGraph(directed=True)
 279         G.add_node("sitemap")
 280         for ch in self._root:
 281             G.add_edge("sitemap",ch.token())
 282         self._graph(self._root, G)
 283 #        G.layout('dot')
 284 #        G.draw('g.png')
 285 #        print G.string()
 286
 287     def _menu(self, trie, lang, page, css):
 288         html = "<ul%s>\n" % css
 289         for l in trie:
 290             sel = ''
 291             p = l.value().page(lang)
 292             if p == page:
 293                 sel = ' class="selected"'
 294             if p != None:
 295                 html += '<li%s><a href="%s">%s</a>\n' \
 296                     % (sel,l.value().link(),p.menu())
 297             else:
 298                 html += '<li%s><a href="%s.en" hreflang="en">%s</a>*\n' \
 299                     % (sel,l.value().link(), l.value().page('en').menu())
 300             if l.children():
 301                 html += self._menu(l.children(), lang, page, "")
 302         html += "</ul>\n"
 303         return html
 304
 305     def menu(self,lang,page,cssclass):
 306         css = ''
 307         if cssclass:
 308             css = ' class="'+cssclass+'"'
 309         return self._menu(self._root, lang, page, css)
 310
 311 class Sitemap():
 312     """Class keeping the internal site structure"""
 313     def __init__(self):
 314         self._file = 'sitemap.txt'
 315         self._tree = Trie()
 316         self._sitelang = set()
 317         self._isocode = bindery.parse('/usr/share/xml/iso-codes/iso_639_3.xml')
 318         self._tranlang = {}
 319
 320     def add_link(self, link):
 321         tokens = filter(None,re.split(r'(^/\w*/|\w*/)',link))
 322         self._tree.add(tokens,Link(link))
 323
 324     def write_map(self):
 325         f = open(self._file,'w')
 326         f.write('\n'.join(link.link() for link in self._tree))
 327         f.close()
 328
 329     def read_map(self):
 330         try:
 331             f = open(self._file)
 332             sml = f.read().split()
 333             f.close()
 334             for line in sml:
 335                 self.add_link(line)
 336         except IOError, what_error:
 337             print 'INFO: Could not read sitemap.txt - one will be created'
 338
 339     def set(self):
 340         return set(link.link() for link in self._tree)
 341
 342     def process(self):
 343         t1 = time.time()
 344         for link in self._tree:
 345             link.prepare()
 346         t2 = time.time()
 347         print "Prepare  [%5.2f s]" % (round(t2-t1,2))
 348         for link in self._tree:
 349             self._sitelang = self._sitelang.union(set(link.languages()))
 350         for tran in self._sitelang:
 351             if tran != 'en':
 352                 self._tranlang[tran] = gettext.translation('iso_639_3', languages=[tran])
 353         t3 = time.time()
 354         print "Language [%5.2f s]" % (round(t3-t2,2))
 355         for link in self._tree:
 356             link.render()
 357         t4 = time.time()
 358         print "Render   [%5.2f s]" % (round(t4-t3,2))
 359         for link in self._tree:
 360             link.template(self)
 361         t5 = time.time()
 362         print "Template [%5.2f s]" % (round(t5-t4,2))
 363         t6 = time.time()
 364         res = set()
 365         cwd = os.getcwd()
 366         for link in self._tree:
 367             res = res.union(link.resources())
 368         for f in res:
 369             outfile = tmptarget+f
 370             mkdir_p(os.path.dirname(outfile))
 371             shutil.copyfile(f,outfile)
 372         print "Resources[%5.2f s]" % (round(t6-t5,2))
 373         sitmaplink = Link('/sitemap')
 374         for l in self._sitelang:
 375             sitmaplink.add_page((l,'/sitemap.'+l+'.xml'))
 376         for l in self._sitelang:
 377             sitmaplink.page(l).set_article(self.gen_menu(l,None,"tree sitemap"))
 378             sitmaplink.page(l).template(self)
 379         t7 = time.time()
 380         print "Sitemap  [%5.2f s]" % (round(t7-t6,2))
 381
 382     def graph(self):
 383         self._tree.graph()
 384
 385     def gen_menu(self,lang,page,cssclass):
 386         return self._tree.menu(lang,page,cssclass)
 387
 388     def lang_menu(self,lang,link):
 389         html = "<ul>"
 390         for l in link.languages():
 391             isoxml = u"//iso_639_3_entry[@*='"+l+"']"
 392             ln = self._isocode.xml_select(isoxml)[0].name
 393             if lang != 'en':
 394                 ln = self._tranlang[lang].gettext(ln)
 395             p = link.link()
 396             if p[-1] == '/':
 397                 p = p +'index'
 398             p = p+'.'+l
 399             html += '<li><a href="%s" hreflang="%s">%s</a></li>' % (p, l, ln)
 400         html += "</ul>"
 401         return html
 402
 403     def publish(self):
 404         publish(tmptarget, args.output)
 405         publish(args.style+"css", args.output)
 406         publish(args.style+"images",args.output)
 407
 408 def generateSitemap():
 409     sitemap = []
 410     try:
 411         sfile = open('sitemap.txt')
 412         flist = sfile.read().split()
 413         sfile.close()
 414         for f in flist:
 415             sitemap.append(dict(link=f))
 416     except IOError, what_error:
 417         print 'Sitemap missing - generating one.'
 418
 419     for dirname, dirnames, filenames in os.walk('.'):
 420         for filename in filenames:
 421             if fnmatch.fnmatch(filename, '*.xml'):
 422                 xfile = os.path.join(dirname,filename)
 423                 doc = bindery.parse(xfile,
 424                                     prefixes={u'db': u'http://docbook.org/ns/docbook',
 425                                               u'xi': u'http://www.w3.org/2001/XInclude',
 426                                               u'xl': u'http://www.w3.org/1999/xlink'})
 427                 title = doc.xml_select(u'/db:article/db:info/db:title')
 428                 menu  = doc.xml_select(u'/db:article/db:info/db:titleabbrev')
 429                 code  = doc.xml_select(u"//xi:include[@parse='text']")
 430                 resource = doc.xml_select(u"//db:link[@xl:href]")
 431                 image = doc.xml_select(u"//db:imagedata[@fileref]")
 432                 exe = 0
 433                 for c in code:
 434                     (p, ext) = os.path.splitext(c.href)
 435                     if ext in valid_scripts:
 436                         exe = 1
 437
 438                 if title and menu:
 439                     found = 0
 440                     base = xfile.split('.')[1]
 441                     link = base.replace('index','')
 442                     level = len(filter(None,re.split(r'(^/\w*/|\w*/)',link)))
 443                     res = []
 444                     for r in resource:
 445                         rf = os.path.join(dirname,r.href)
 446                         if os.path.isfile(rf):
 447                             res.append(rf)
 448                     for i in image:
 449                         im = os.path.join(dirname,i.fileref)
 450                         if os.path.isfile(im):
 451                             res.append(im)
 452                     page = dict(title=unicode(doc.article.info.title),
 453                                 menu=unicode(doc.article.info.titleabbrev),
 454                                 output=os.path.join(dirname,
 455                                                     filename.replace('xml','html')),
 456                                 exe=exe,
 457                                 file=xfile,
 458                                 res=res,
 459                                 level=level)
 460                     for l in sitemap:
 461                         if l['link'] == link:
 462                             found = 1
 463                             l.update(page)
 464                     if not found:
 465                         print "adding "+link+" to sitemap"
 466                         dd = dict(link=link)
 467                         dd.update(page)
 468                         sitemap.append(dd)
 469     sfile = open('sitemap.txt','w')
 470     for l in sitemap:
 471         sfile.write(l['link']+'\n')
 472     sfile.close()
 473     return sitemap
 474
 475 def expandXincludeTxt(page):
 476     doc = bindery.parse(page['file'],
 477                         prefixes={u'db': u'http://docbook.org/ns/docbook',
 478                                   u'xi': u'http://www.w3.org/2001/XInclude'})
 479     if page['exe']:
 480         code  = doc.xml_select(u"//xi:include[@parse='text']")
 481         for c in code:
 482             (p, ext) = os.path.splitext(c.href)
 483             if ext in valid_scripts:
 484                 exe = os.path.join(os.path.abspath(c.href))
 485                 xml = subprocess.Popen([exe],stdout=subprocess.PIPE)
 486                 xstr = bindery.parse(str(xml.stdout.read()))
 487                 id = c.xml_index_on_parent
 488                 for x in xstr.xml_children:
 489                     c.xml_parent.xml_insert(id,x)
 490                 c.xml_parent.xml_remove(c)
 491     return doc
 492
 493 def xsltConvert(doc):
 494 #  amara can not handle the docbook stylesheets
 495 #  xmlarticle = transform(doc,style_xslt)
 496     cwd = os.getcwd()
 497     rundir = os.path.dirname(page['file'])
 498     os.chdir(rundir)
 499     infile  = os.path.basename(tempfile.mktemp())
 500     outfile = tempfile.mktemp()
 501     tfi = open(infile,'w')
 502     tfi.write(doc.xml_encode())
 503     tfi.close()
 504 #  cmd = ["saxon-xslt-xinclude","-o",outfile,infile,style_xslt]
 505     cmd = ["xsltproc","--xinclude","--output",outfile,style_xslt,infile]
 506     retcode = subprocess.call(cmd)
 507     if retcode:
 508         print 'Error: '+' '.join(cmd)+' Returncode ['+str(retcode)+']'
 509     tfo = open(outfile,'r')
 510     result = tfo.read()
 511     tfo.close()
 512     os.remove(infile)
 513     os.remove(outfile)
 514     os.chdir(cwd)
 515     return result
 516
 517 def genMenu(page,sitemap,slevel,elevel):
 518     title = None
 519     sm = []
 520     if elevel == MAXLEVEL or elevel == 1 or page == None:
 521         html = '<ul>\n'
 522         sm = sitemap
 523     else:
 524         html = '<ul class="tree">\n'
 525         idx = sitemap.index(page)
 526         while (sitemap[idx]['level'] == page['level']):
 527             idx = idx-1
 528         title = sitemap[idx]['menu']
 529         idx = idx+1
 530         while (idx < len(sitemap) and sitemap[idx]['level'] == page['level']):
 531             sm.append(sitemap[idx])
 532             idx = idx+1
 533     oldlevel = slevel
 534
 535     for p in sm:
 536         if slevel > p['level'] or elevel < p['level']:
 537             continue
 538         if not title and p['link'] == '/':
 539             title = p['menu']
 540
 541         if oldlevel < p['level']:
 542             html+='<ul>\n'
 543         elif oldlevel > p['level']:
 544             if p['link'][-1] == '/':
 545                 html+='</li>\n'
 546             html+='</ul>\n</li>\n'
 547         if page != None and page == p:
 548             html+='<li class="selected"><a href="%s">%s</a>' % (p['link'],p['menu'])
 549         else:
 550             html+='<li><a href="%s">%s</a>' % (p['link'],p['menu'])
 551         if p['link'][-1] != '/' or p['link'] == '/':
 552             html+='</li>\n'
 553         oldlevel = p['level']
 554     html+='</ul>\n'
 555     return (html,title)
 556
 557 def writeToTemplate(page,doc,sitemap):
 558     (menu,menuname) = genMenu(page,sitemap,1,MAXLEVEL)
 559     (levelmenu,levelname) = genMenu(page,sitemap,page['level'],page['level'])
 560     template = Template(file=style_tmpl,
 561                         searchList=[{'title':page['title']},
 562                                     {'menu':menu},
 563                                     {'article':doc},
 564                                     {'levelmenu':levelmenu},
 565                                     {'levelname':levelname}])
 566     outfile = tmptarget+page['output']
 567     mkdir_p(os.path.dirname(outfile))
 568     out = open(outfile, 'w')
 569     out.write(str(template))
 570     out.close()
 571     for r in page['res']:
 572         mkdir_p(os.path.dirname(tmptarget+r))
 573         shutil.copyfile(r, tmptarget+r)
 574
 575 def createSitemap(sitemap):
 576     (menu,menuname) = genMenu(None,sitemap,1,MAXLEVEL)
 577     template = Template(file=style_tmpl,
 578                         searchList=[
 579             {'title':'Sitemap'},
 580             {'menu':menu},
 581             {'article':menu},
 582             {'levelmenu':''},
 583             {'levelname':''}])
 584     outfile = tmptarget+'sitemap.en.html'
 585     mkdir_p(os.path.dirname(outfile))
 586     out = open(outfile, 'w')
 587     out.write(str(template))
 588     out.close()
 589
 590 dir_ = Directory()
 591 sitemap = Sitemap()
 592
 593 dir_.scan()
 594 sitemap.read_map()
 595
 596 missing = dir_.set() - sitemap.set()
 597 removed = sitemap.set() - dir_.set()
 598 for page in removed:
 599     print page+' pages missing!!'
 600 for page in missing:
 601     print 'adding missing page '+page
 602     sitemap.add_link(page)
 603 if len(missing & removed) != 0:
 604     print 'writing new sitemap - please adjust if needed'
 605     sitemap.write_map()
 606 sitemap.graph()
 607
 608 sitemap.process()
 609
 610 t1 = time.time()
 611 sitemap.publish()
 612 t2 = time.time()
 613 print "Publish  [%5.2f s]" % (round(t2-t1,2))
 614
 615 sitemap = generateSitemap()
 616 tmptarget = tempfile.mkdtemp()+'/'
 617 tot = 0
 618 for page in sitemap:
 619     t1 = time.time()
 620     print "Page : %-30s %30s" % (page['link'],
 621                         time.ctime(os.stat(page['file']).st_mtime)),
 622     doc = expandXincludeTxt(page)
 623     pubdoc = xsltConvert(doc)
 624     writeToTemplate(page,pubdoc,sitemap)
 625     t2 = time.time()
 626     print "[%5.2f s]" % (round(t2-t1,2))
 627     tot = tot + (t2-t1)
 628
 629 print "Total time\t\t\t\t\t\t\t     [%5.2f s]" % (round(tot,2))
 630 createSitemap(sitemap)
 631 publish(tmptarget, args.output)
 632 publish(args.style+"css", args.output)
 633 publish(args.style+"images",args.output)