src/tree-cutter.py

   1 #!/usr/bin/python
   2 import os
   3 import fnmatch
   4 import subprocess
   5 import amara
   6 import re
   7 import tempfile
   8 import errno
   9 import time
  10 import argparse
  11 import shutil
  12 import pygraphviz as pgv
  13 import glob
  14 import gettext
  15 import shutil
  16 from amara import bindery
  17 from amara.xslt import transform
  18 from Cheetah.Template import Template
  19
  20 parser = argparse.ArgumentParser(description='Process docbook article tree.')
  21 parser.add_argument('--style', nargs='?',
  22                     default=os.path.dirname(os.getcwd())+'/style/default/')
  23 parser.add_argument('--output', nargs='?',
  24                     default=os.path.dirname(os.getcwd())+'/htdocs/')
  25 args = parser.parse_args()
  26
  27 style_xslt = args.style+"docbook.xsl"
  28 outputdir = args.output
  29
  30 tmptarget = tempfile.mkdtemp()+'/'
  31
  32 valid_scripts = ['.py','.pl']
  33 MAXLEVEL = 10000
  34
  35 def mkdir_p(path):
  36     try:
  37         os.makedirs(path)
  38     except OSError as exc: # Python >2.5
  39         if exc.errno == errno.EEXIST:
  40             pass
  41         else: raise
  42
  43 def publish(src,target):
  44     cmd = ["rsync","-a","--delete",src,target]
  45     retcode = subprocess.call(cmd)
  46     if retcode:
  47         print 'Error: '+' '.join(cmd)+' Returncode ['+str(retcode)+']'
  48
  49
  50 PREFIXES={u'db': u'http://docbook.org/ns/docbook',
  51           u'xi': u'http://www.w3.org/2001/XInclude',
  52           u'xl': u'http://www.w3.org/1999/xlink'}
  53
  54 class Directory():
  55     """Class containing the state of the directory with articles"""
  56     def __init__(self):
  57         self._cwd = '.'
  58         self._tree = []
  59
  60     def scan(self):
  61         for dirname, dirnames, filenames in os.walk(self._cwd):
  62             for filename in filenames:
  63                 if fnmatch.fnmatch(filename, '*.xml'):
  64                     file_ = os.path.join(dirname,filename)
  65                     doc = bindery.parse(file_, prefixes=PREFIXES)
  66                     title = doc.xml_select(u'/db:article/db:info/db:title')
  67                     menu  = doc.xml_select(u'/db:article/db:info/db:titleabbrev')
  68                     if title and menu:
  69                         base = file_.split('.')[1]
  70                         link = base.replace('index','')
  71                         self._tree.append(link)
  72
  73     def set(self):
  74         return set(self._tree)
  75
  76 class Page():
  77     """Class representing a version of a webpage"""
  78     def __init__(self,link,page):
  79         self._link = link
  80         self._file = page[1]
  81         self._lang = page[0]
  82         self._doc = None
  83         self._resources = []
  84         self._title = None
  85         self._menu = None
  86         self._rendered_article = None
  87
  88     def language(self):
  89         return self._lang
  90
  91     def resources(self):
  92         return set(self._resources)
  93
  94     def menu(self):
  95         return self._menu
  96
  97     def set_article(self,art):
  98         self._rendered_article = art
  99
 100     def prepare(self):
 101         self._doc = bindery.parse(self._file, prefixes=PREFIXES)
 102         if self._doc.xml_select(u'/db:article/db:info/db:title'):
 103             self._title = unicode(self._doc.article.info.title)
 104         if self._doc.xml_select(u'/db:article/db:info/db:titleabbrev'):
 105             self._menu = unicode(self._doc.article.info.titleabbrev)
 106
 107         dirname = os.path.dirname(self._file)
 108         code  = self._doc.xml_select(u"//xi:include[@parse='text']")
 109         if code:
 110             for c in code:
 111                 (p, ext) = os.path.splitext(c.href)
 112                 if ext in valid_scripts:
 113                     exe = os.path.join(os.path.abspath(dirname)+'/'+c.href)
 114                     xml = subprocess.Popen([exe],stdout=subprocess.PIPE)
 115                     xstr = bindery.parse(str(xml.stdout.read()))
 116                     idp = c.xml_index_on_parent
 117                     for x in xstr.xml_children:
 118                         c.xml_parent.xml_insert(idp,x)
 119                         c.xml_parent.xml_remove(c)
 120
 121         for r in self._doc.xml_select(u"//db:link[@xl:href]"):
 122             rf = os.path.join(dirname,r.href)
 123             if os.path.isfile(rf):
 124                 self._resources.append(rf)
 125         for i in self._doc.xml_select(u"//db:imagedata[@fileref]"):
 126             im = os.path.join(dirname,i.fileref)
 127             if os.path.isfile(im):
 128                 self._resources.append(im)
 129
 130     def render(self):
 131         #  amara can not handle the docbook stylesheets
 132         #  xmlarticle = transform(doc,style_xslt)
 133         cwd = os.getcwd()
 134         dirname = os.path.dirname(self._file)
 135         os.chdir(dirname)
 136         infile  = os.path.basename(tempfile.mktemp())
 137         outfile = tempfile.mktemp()
 138         tfi = open(infile,'w')
 139         tfi.write(self._doc.xml_encode())
 140         tfi.close()
 141 #  cmd = ["saxon-xslt-xinclude","-o",outfile,infile,style_xslt]
 142         cmd = ["xsltproc","--xinclude","--output",outfile,style_xslt,infile]
 143         retcode = subprocess.call(cmd)
 144         if retcode:
 145             print 'Error: '+' '.join(cmd)+' Returncode ['+str(retcode)+']'
 146         tfo = open(outfile,'r')
 147         self._rendered_article = tfo.read()
 148         tfo.close()
 149         os.remove(infile)
 150         os.remove(outfile)
 151         os.chdir(cwd)
 152
 153     def template(self,sitemap):
 154         htmlmenu =  sitemap.gen_menu(self._lang,None,"menu")
 155         levelmenu = sitemap.gen_menu(self._lang,self,"tree")
 156         langmenu = sitemap.lang_menu(self._lang,self._link)
 157         template = Template(file=args.style+'index.'+self._lang+'.html.tmpl',
 158                             searchList=[{'title':self._title},
 159                                         {'menu':htmlmenu},
 160                                         {'article':self._rendered_article},
 161                                         {'levelmenu':levelmenu},
 162                                         {'langmenu':langmenu}])
 163         outfile = tmptarget+'html'.join(self._file.rsplit('xml',1))
 164         mkdir_p(os.path.dirname(outfile))
 165         out = open(outfile, 'w')
 166         out.write(str(template))
 167         out.close()
 168
 169
 170 class Link():
 171     """Class representing a webpage on the site"""
 172     def __init__(self,link):
 173         self._link = link
 174         # find the representations of the link.
 175         self._pages = []
 176         path = link
 177         if self._link[-1] == '/':
 178             path = path+'index'
 179         lang = self._scan_languages(path)
 180         for l in lang:
 181             self._pages.append(Page(self,l))
 182
 183     def add_page(self,l):
 184         self._pages.append(Page(self,l))
 185
 186     def _scan_languages(self,path):
 187         lang = []
 188         for l in  glob.glob('.'+path+'*'):
 189             ls = l.split('.')
 190             if len(ls) > 3 and ls[3] == 'xml':
 191                 lang.append((ls[2],l))
 192         return lang
 193
 194     def link(self):
 195         return self._link
 196
 197     def prepare(self):
 198         for page in self._pages:
 199             page.prepare()
 200
 201     def languages(self):
 202         p = []
 203         for page in self._pages:
 204             p.append(page.language())
 205         return p
 206
 207     def render(self):
 208         for page in self._pages:
 209             page.render()
 210
 211     def template(self,sitemap):
 212         for page in self._pages:
 213             page.template(sitemap)
 214
 215     def page(self,lang):
 216         for page in self._pages:
 217             if page.language()==lang:
 218                 return page
 219         return None
 220
 221     def resources(self):
 222         res  = set()
 223         for page in self._pages:
 224             res = res.union(page.resources())
 225         return res
 226
 227
 228 class Node():
 229     def __init__(self,token,value):
 230         self._token = token
 231         self._value = value
 232         self._children = []
 233
 234     def token(self):
 235         return self._token
 236
 237     def value(self):
 238         return self._value
 239
 240     def children(self):
 241         return self._children
 242
 243 class Trie():
 244     def __init__(self):
 245         self._root = []
 246
 247     def __iter__(self):
 248         return self.inorder(self._root)
 249
 250     def inorder(self,t):
 251         for l in t:
 252             yield l.value()
 253             for x in self.inorder(l.children()):
 254                 yield x
 255
 256     def _add(self,trie, key, content):
 257         # is the key a leaf
 258         k = key.pop(0)
 259         if key == []:
 260             node = Node(k,content)
 261             trie.append(node)
 262         else:
 263             for ch in trie:
 264                 if ch.token() == k:
 265                     self._add(ch.children(), key, content)
 266
 267     def add(self,key, content):
 268         self._add(self._root, key, content)
 269
 270     def _graph(self, trie, G):
 271         for l in trie:
 272             G.add_node(l.token())
 273             for ch in l.children():
 274                 G.add_edge(l.token(),ch.token())
 275                 self._graph(l.children(), G)
 276
 277     def graph(self):
 278         G = pgv.AGraph(directed=True)
 279         G.add_node("sitemap")
 280         for ch in self._root:
 281             G.add_edge("sitemap",ch.token())
 282         self._graph(self._root, G)
 283 #        G.layout('dot')
 284 #        G.draw('g.png')
 285 #        print G.string()
 286
 287     def _menu(self, trie, lang, page, css):
 288         html = "<ul%s>\n" % css
 289         for l in trie:
 290             sel = ''
 291             p = l.value().page(lang)
 292             if p == page:
 293                 sel = ' class="selected"'
 294             if p != None:
 295                 html += '<li%s><a href="%s">%s</a>\n' \
 296                     % (sel,l.value().link(),p.menu())
 297             else:
 298                 html += '<li%s><a href="%s.en" hreflang="en">%s</a>*\n' \
 299                     % (sel,l.value().link(), l.value().page('en').menu())
 300             if l.children():
 301                 html += self._menu(l.children(), lang, page, "")
 302         html += "</ul>\n"
 303         return html
 304
 305     def menu(self,lang,page,cssclass):
 306         css = ''
 307         if cssclass:
 308             css = ' class="'+cssclass+'"'
 309         return self._menu(self._root, lang, page, css)
 310
 311 class Sitemap():
 312     """Class keeping the internal site structure"""
 313     def __init__(self):
 314         self._file = 'sitemap.txt'
 315         self._tree = Trie()
 316         self._sitelang = set()
 317         self._isocode = bindery.parse('/usr/share/xml/iso-codes/iso_639_3.xml')
 318         self._tranlang = {}
 319
 320     def add_link(self, link):
 321         tokens = filter(None,re.split(r'(^/[\w-]*/|[\w-]*/)',link))
 322         self._tree.add(tokens,Link(link))
 323
 324     def write_map(self):
 325         f = open(self._file,'w')
 326         f.write('\n'.join(link.link() for link in self._tree))
 327         f.close()
 328
 329     def read_map(self):
 330         try:
 331             f = open(self._file)
 332             sml = f.read().split()
 333             f.close()
 334             for line in sml:
 335                 self.add_link(line)
 336         except IOError, what_error:
 337             print 'INFO: Could not read sitemap.txt - one will be created'
 338
 339     def set(self):
 340         return set(link.link() for link in self._tree)
 341
 342     def process(self):
 343         t1 = time.time()
 344         for link in self._tree:
 345             link.prepare()
 346         t2 = time.time()
 347         print "Prepare  [%5.2f s]" % (round(t2-t1,2))
 348         for link in self._tree:
 349             self._sitelang = self._sitelang.union(set(link.languages()))
 350         for tran in self._sitelang:
 351             if tran != 'en':
 352                 self._tranlang[tran] = gettext.translation('iso_639_3',
 353                                                            languages=[tran])
 354         t3 = time.time()
 355         print "Language [%5.2f s]" % (round(t3-t2,2))
 356         for link in self._tree:
 357             link.render()
 358         t4 = time.time()
 359         print "Render   [%5.2f s]" % (round(t4-t3,2))
 360         for link in self._tree:
 361             link.template(self)
 362         t5 = time.time()
 363         print "Template [%5.2f s]" % (round(t5-t4,2))
 364         t6 = time.time()
 365         res = set()
 366         cwd = os.getcwd()
 367         for link in self._tree:
 368             res = res.union(link.resources())
 369         for f in res:
 370             outfile = tmptarget+f
 371             mkdir_p(os.path.dirname(outfile))
 372             shutil.copyfile(f,outfile)
 373         print "Resources[%5.2f s]" % (round(t6-t5,2))
 374         sitmaplink = Link('/sitemap')
 375         for l in self._sitelang:
 376             sitmaplink.add_page((l,'/sitemap.'+l+'.xml'))
 377         for l in self._sitelang:
 378             sitmaplink.page(l).set_article(self.gen_menu(l,None,"tree sitemap"))
 379             sitmaplink.page(l).template(self)
 380         t7 = time.time()
 381         print "Sitemap  [%5.2f s]" % (round(t7-t6,2))
 382
 383     def graph(self):
 384         self._tree.graph()
 385
 386     def gen_menu(self,lang,page,cssclass):
 387         return self._tree.menu(lang,page,cssclass)
 388
 389     def lang_menu(self,lang,link):
 390         html = "<ul>"
 391         for l in link.languages():
 392             isoxml = u"//iso_639_3_entry[@*='"+l+"']"
 393             ln = self._isocode.xml_select(isoxml)[0].name
 394             if lang != 'en':
 395                 ln = self._tranlang[lang].gettext(ln)
 396             p = link.link()
 397             if p[-1] == '/':
 398                 p = p +'index'
 399             p = p+'.'+l
 400             html += '<li><a href="%s" hreflang="%s">%s</a></li>' % (p, l, ln)
 401         html += "</ul>"
 402         return html
 403
 404     def publish(self):
 405         publish(tmptarget, args.output)
 406         publish(args.style+"css", args.output)
 407         publish(args.style+"images",args.output)
 408
 409 def generateSitemap():
 410     sitemap = []
 411     try:
 412         sfile = open('sitemap.txt')
 413         flist = sfile.read().split()
 414         sfile.close()
 415         for f in flist:
 416             sitemap.append(dict(link=f))
 417     except IOError, what_error:
 418         print 'Sitemap missing - generating one.'
 419
 420     for dirname, dirnames, filenames in os.walk('.'):
 421         for filename in filenames:
 422             if fnmatch.fnmatch(filename, '*.xml'):
 423                 xfile = os.path.join(dirname,filename)
 424                 doc = bindery.parse(xfile,
 425                                     prefixes={u'db': u'http://docbook.org/ns/docbook',
 426                                               u'xi': u'http://www.w3.org/2001/XInclude',
 427                                               u'xl': u'http://www.w3.org/1999/xlink'})
 428                 title = doc.xml_select(u'/db:article/db:info/db:title')
 429                 menu  = doc.xml_select(u'/db:article/db:info/db:titleabbrev')
 430                 code  = doc.xml_select(u"//xi:include[@parse='text']")
 431                 resource = doc.xml_select(u"//db:link[@xl:href]")
 432                 image = doc.xml_select(u"//db:imagedata[@fileref]")
 433                 exe = 0
 434                 for c in code:
 435                     (p, ext) = os.path.splitext(c.href)
 436                     if ext in valid_scripts:
 437                         exe = 1
 438
 439                 if title and menu:
 440                     found = 0
 441                     base = xfile.split('.')[1]
 442                     link = base.replace('index','')
 443                     level = len(filter(None,re.split(r'(^/\w*/|\w*/)',link)))
 444                     res = []
 445                     for r in resource:
 446                         rf = os.path.join(dirname,r.href)
 447                         if os.path.isfile(rf):
 448                             res.append(rf)
 449                     for i in image:
 450                         im = os.path.join(dirname,i.fileref)
 451                         if os.path.isfile(im):
 452                             res.append(im)
 453                     page = dict(title=unicode(doc.article.info.title),
 454                                 menu=unicode(doc.article.info.titleabbrev),
 455                                 output=os.path.join(dirname,
 456                                                     filename.replace('xml','html')),
 457                                 exe=exe,
 458                                 file=xfile,
 459                                 res=res,
 460                                 level=level)
 461                     for l in sitemap:
 462                         if l['link'] == link:
 463                             found = 1
 464                             l.update(page)
 465                     if not found:
 466                         print "adding "+link+" to sitemap"
 467                         dd = dict(link=link)
 468                         dd.update(page)
 469                         sitemap.append(dd)
 470     sfile = open('sitemap.txt','w')
 471     for l in sitemap:
 472         sfile.write(l['link']+'\n')
 473     sfile.close()
 474     return sitemap
 475
 476 def expandXincludeTxt(page):
 477     doc = bindery.parse(page['file'],
 478                         prefixes={u'db': u'http://docbook.org/ns/docbook',
 479                                   u'xi': u'http://www.w3.org/2001/XInclude'})
 480     if page['exe']:
 481         code  = doc.xml_select(u"//xi:include[@parse='text']")
 482         for c in code:
 483             (p, ext) = os.path.splitext(c.href)
 484             if ext in valid_scripts:
 485                 exe = os.path.join(os.path.abspath(c.href))
 486                 xml = subprocess.Popen([exe],stdout=subprocess.PIPE)
 487                 xstr = bindery.parse(str(xml.stdout.read()))
 488                 id = c.xml_index_on_parent
 489                 for x in xstr.xml_children:
 490                     c.xml_parent.xml_insert(id,x)
 491                 c.xml_parent.xml_remove(c)
 492     return doc
 493
 494 def xsltConvert(doc):
 495 #  amara can not handle the docbook stylesheets
 496 #  xmlarticle = transform(doc,style_xslt)
 497     cwd = os.getcwd()
 498     rundir = os.path.dirname(page['file'])
 499     os.chdir(rundir)
 500     infile  = os.path.basename(tempfile.mktemp())
 501     outfile = tempfile.mktemp()
 502     tfi = open(infile,'w')
 503     tfi.write(doc.xml_encode())
 504     tfi.close()
 505 #  cmd = ["saxon-xslt-xinclude","-o",outfile,infile,style_xslt]
 506     cmd = ["xsltproc","--xinclude","--output",outfile,style_xslt,infile]
 507     retcode = subprocess.call(cmd)
 508     if retcode:
 509         print 'Error: '+' '.join(cmd)+' Returncode ['+str(retcode)+']'
 510     tfo = open(outfile,'r')
 511     result = tfo.read()
 512     tfo.close()
 513     os.remove(infile)
 514     os.remove(outfile)
 515     os.chdir(cwd)
 516     return result
 517
 518 def genMenu(page,sitemap,slevel,elevel):
 519     title = None
 520     sm = []
 521     if elevel == MAXLEVEL or elevel == 1 or page == None:
 522         html = '<ul>\n'
 523         sm = sitemap
 524     else:
 525         html = '<ul class="tree">\n'
 526         idx = sitemap.index(page)
 527         while (sitemap[idx]['level'] == page['level']):
 528             idx = idx-1
 529         title = sitemap[idx]['menu']
 530         idx = idx+1
 531         while (idx < len(sitemap) and sitemap[idx]['level'] == page['level']):
 532             sm.append(sitemap[idx])
 533             idx = idx+1
 534     oldlevel = slevel
 535
 536     for p in sm:
 537         if slevel > p['level'] or elevel < p['level']:
 538             continue
 539         if not title and p['link'] == '/':
 540             title = p['menu']
 541
 542         if oldlevel < p['level']:
 543             html+='<ul>\n'
 544         elif oldlevel > p['level']:
 545             if p['link'][-1] == '/':
 546                 html+='</li>\n'
 547             html+='</ul>\n</li>\n'
 548         if page != None and page == p:
 549             html+='<li class="selected"><a href="%s">%s</a>' % (p['link'],p['menu'])
 550         else:
 551             html+='<li><a href="%s">%s</a>' % (p['link'],p['menu'])
 552         if p['link'][-1] != '/' or p['link'] == '/':
 553             html+='</li>\n'
 554         oldlevel = p['level']
 555     html+='</ul>\n'
 556     return (html,title)
 557
 558 def writeToTemplate(page,doc,sitemap):
 559     (menu,menuname) = genMenu(page,sitemap,1,MAXLEVEL)
 560     (levelmenu,levelname) = genMenu(page,sitemap,page['level'],page['level'])
 561     template = Template(file=style_tmpl,
 562                         searchList=[{'title':page['title']},
 563                                     {'menu':menu},
 564                                     {'article':doc},
 565                                     {'levelmenu':levelmenu},
 566                                     {'levelname':levelname}])
 567     outfile = tmptarget+page['output']
 568     mkdir_p(os.path.dirname(outfile))
 569     out = open(outfile, 'w')
 570     out.write(str(template))
 571     out.close()
 572     for r in page['res']:
 573         mkdir_p(os.path.dirname(tmptarget+r))
 574         shutil.copyfile(r, tmptarget+r)
 575
 576 def createSitemap(sitemap):
 577     (menu,menuname) = genMenu(None,sitemap,1,MAXLEVEL)
 578     template = Template(file=style_tmpl,
 579                         searchList=[
 580             {'title':'Sitemap'},
 581             {'menu':menu},
 582             {'article':menu},
 583             {'levelmenu':''},
 584             {'levelname':''}])
 585     outfile = tmptarget+'sitemap.en.html'
 586     mkdir_p(os.path.dirname(outfile))
 587     out = open(outfile, 'w')
 588     out.write(str(template))
 589     out.close()
 590
 591 dir_ = Directory()
 592 sitemap = Sitemap()
 593
 594 dir_.scan()
 595 sitemap.read_map()
 596
 597 missing = dir_.set() - sitemap.set()
 598 removed = sitemap.set() - dir_.set()
 599 for page in removed:
 600     print page+' pages missing!!'
 601 for page in missing:
 602     print 'adding missing page '+page
 603     sitemap.add_link(page)
 604 if len(missing)+len(removed) != 0:
 605     print 'writing new sitemap - please adjust if needed'
 606     sitemap.write_map()
 607 sitemap.graph()
 608
 609 sitemap.process()
 610
 611 t1 = time.time()
 612 sitemap.publish()
 613 t2 = time.time()
 614 print "Publish  [%5.2f s]" % (round(t2-t1,2))
 615
 616 sitemap = generateSitemap()
 617 tmptarget = tempfile.mkdtemp()+'/'
 618 tot = 0
 619 for page in sitemap:
 620     t1 = time.time()
 621     print "Page : %-30s %30s" % (page['link'],
 622                         time.ctime(os.stat(page['file']).st_mtime)),
 623     doc = expandXincludeTxt(page)
 624     pubdoc = xsltConvert(doc)
 625     writeToTemplate(page,pubdoc,sitemap)
 626     t2 = time.time()
 627     print "[%5.2f s]" % (round(t2-t1,2))
 628     tot = tot + (t2-t1)
 629
 630 print "Total time\t\t\t\t\t\t\t     [%5.2f s]" % (round(tot,2))
 631 createSitemap(sitemap)
 632 publish(tmptarget, args.output)
 633 publish(args.style+"css", args.output)
 634 publish(args.style+"images",args.output)