root/BADataMunger/trunk/dir2xml.py

"""
dir2xml.py

get all the batlas dirs from word-export html to xml once and for all
"""

import os
import re
import logging
import datetime as dt

import lxml.etree as etree

import placesaver
import batlaspipe as bp

from placesaver import periods, AWMC, ADLGAZ, DC, XML, TEI, refmagic, do_nscleanup
from bidmaker import DIAMOND_STOP_REGEX
from batlaspipe import SLASH_REGEX, NAMESPACE, XMLDECL, DASHNUM_END_REGEX, PREPARER

PLEIADES = 'http://atlantides.org/batlas/'

BAD_REGEX = re.compile('(........)config\.xml')


COMMENTFILE = r'./etc/dirxmlcomment.txt'

priorcitations = []

def htmldump(map, pipe, destdir):
    x = p['cleanxml']
    pcontent = etree.tostring(x).encode('utf-8')
    dest = ''.join((map, '.html'))
    dest = os.path.join(destdir, dest)
    g = open(dest,'w')
    g.write(pcontent)
    g.close()


def htmlconvert(configfile, sourcefile, destdir):
    p = bp.Pipe(configfile, sourcefile, destdir)
    p.cycle()
    return p


def htmlconvertall(configdir, sourcedir, destdir):

    maps = []
    configs = os.listdir(configdir)
    for config in configs:
        m = BAD_REGEX.match(config)
        if m:
            maps.append(m.group(1))
    for map in maps:
        config = ''.join((map, 'config', '.xml'))
        config = os.path.join(configdir, config)
        source = ''.join((map, '.htm'))
        source = os.path.join(sourcedir, source)
        p = htmlconvert(config, source, destdir)
        htmldump(p, destdir)
        
def xmlconvert(configfile, sourcefile, destdir):

    p = htmlconvert(configfile, sourcefile, destdir)
    p.idit()
    return p
    
    
def xmlconvertall(configdir, sourcedir, destdir):

    maps = []
    configs = os.listdir(configdir)
    for config in configs:
        m = BAD_REGEX.match(config)
        if m:
            maps.append(m.group(1))
    for map in maps:
        config = ''.join((map, 'config', '.xml'))
        config = os.path.join(configdir, config)
        source = ''.join((map, '.htm'))
        source = os.path.join(sourcedir, source)
        p = xmlconvert(config, source, destdir)
        xmldump(p, destdir)
        print 'wahoo'
        

    
def encodeids(place, parent):
    if len(place.batlasids) > 0:
        for i, bid in enumerate(place.batlasids):
            if i == 0 or bid != place.batlasids[0]:
                q = etree.SubElement(parent, '{%s}baid' % PLEIADES, id=bid)
        parent.xpath("*[local-name() = 'baid']")[0].attrib['primary'] = 'yes'
        
def encodetypes(place, parent):

    if place.dirtype == 'name':
        ptype = 'labeled feature'
    elif place.dirtype == 'numbered':
        ptype = 'numbered feature'
    elif place.dirtype == 'unlocated':
        ptype = 'unlocated toponym'
    elif place.dirtype == 'false':
        ptype = 'false toponym'
    else:
        ptype = place.dirtype
    q = etree.SubElement(parent, '{%s}type' % PLEIADES)
    q.text = ptype
    
    if 'group' in place.dirtype:
        q = etree.SubElement(parent, 'featurecount')
        q.text = "%s" % place.featurecount
    
    # additional feature types
    for i, t in enumerate(place.types):
        if i == 0:
            q = etree.SubElement(parent, 'subtype')
        elif t != place.types[i-1]:
            q = etree.SubElement(parent, 'subtype')
        q.text = t


def encodegrid(place, parent):

    if len(place.grid) > 0:
        q = etree.SubElement(parent, '{%s}gridsquare' % PLEIADES)
        q.text = place.grid
        

def encodemaplabels(place, parent):

    if len(place.namestring) > 0:
        if place.dirtype not in ['unlocated', 'false']:
            q = etree.SubElement(parent, '{%s}label' % PLEIADES, context='map')
            txt = SLASH_REGEX.sub('/', place.namestring)
            txt = DIAMOND_STOP_REGEX.sub('', txt)
            q.text = txt.strip()


def encodecitations(place, parent, mapnum):

    citcontent = u''
    if place.dirtype == 'unlocated': 
        citname =SLASH_REGEX.sub('/', place.namestring.strip())
        citcontent = "BAtlas %s unlocated %s" % (mapnum, citname)
    elif place.dirtype =='false':
        citname = SLASH_REGEX.sub('/',place.namestring.strip())
        citcontent = "BAtlas %s false name %s" % (mapnum, citname)
    elif place.dirtype == 'numbered' and len(place.placenames) == 0:
        citname = SLASH_REGEX.sub('/',place.locdesc.strip())
        citcontent = "BAtlas %s %s no. %s (%s)" % (mapnum, place.grid, place.namestring, citname)
    elif place.dirtype == 'numbered' and len(place.placenames) > 0:
        citname = SLASH_REGEX.sub('/',place.placenames[0].name.strip())
        citcontent = "BAtlas %s %s no. %s (%s)" % (mapnum, place.grid, place.namestring, citname)
    elif place.dirtype == 'name' and len(place.namestring) == 0:
        citname = SLASH_REGEX.sub('/',place.locdesc.strip())
        citcontent = "BAtlas %s %s %s" % (mapnum, place.grid, citname)
    elif len(place.namestring) == 0:
        citname = SLASH_REGEX.sub('/',place.locdesc.strip())
        citcontent = "BAtlas %s %s unnamed %s (%s)" % (mapnum, place.grid, place.dirtype.replace('-', ' '), citname)                    
    else:
        txt = SLASH_REGEX.sub('/', place.namestring)
        txt = DIAMOND_STOP_REGEX.sub('', txt)
        citname = txt.strip()
        citcontent = "BAtlas %s %s %s" % (mapnum, place.grid, citname)
    if len(citcontent) > 0:
        if len(place.batlasids) > 0:
            writecit(parent, citcontent, place.batlasids[0])
        else:
            writecit(parent, citcontent, '')
    return citname

def encodeplacenames(place, parent, mapnum, citname):

    for i, n in enumerate(place.placenames):
        q = etree.SubElement(parent, '{%s}geogname' % PLEIADES)
        q.text = n.name.strip()
       # if q.text not in citname:
       #     q.attrib['type'] = 'variant'
        if n.variant:
            q.attrib['type'] = 'variant'
        elif n.minorAlternative:
            q.attrib['type'] = 'minor-alternate'
        if n.completeness != 'complete':
            q.attrib['completeness'] = n.completeness
        if n.accuracy != 'accurate':
            q.attrib['accuracy'] = n.accuracy
        if n.inferred:
            q.attrib['inferred'] = 'yes'
        if n.certainty != 'certain':
            q.attrib['certainty'] = n.certainty
        if len(place.placenames) > 1 and citname != q.text:
            txt = q.text
            if n.completeness == 'reconstructable':
                txt = u"*%s" % txt
            if n.accuracy == 'inaccurate':
                txt = u"\u2018%s\u2019" % txt
            if n.inferred == True:
                txt = u"[%s]" % txt
            if n.certainty != 'certain':
                txt = u"%s?" % txt
            if n.minorAlternative:
                txt = u"\u00A7%s" % txt
            if place.dirtype == 'unlocated':
                txt = "BAtlas %s unlocated %s" % (mapnum, txt)
            elif place.dirtype == 'false':
                txt = "BAtlas %s false %s" % (mapnum, txt)
            elif len(place.placenames) > 1:
                txt = "BAtlas %s %s %s" % (mapnum, place.grid, txt)
            if len(txt) > 0:
                if 'island-group' in place.types:
                    txt = "%s Inss." % txt
                elif 'island' in place.types:
                    txt = "%s Ins." % txt
                elif 'river' in place.types:
                    txt = "%s fl." % txt
                # if appropriate, write a citation
                writecit(parent, txt, place.batlasids[0])
                
def encodelocdesc(place, parent):
    
    if len(place.locdesc) > 0:
        q = etree.SubElement(parent, 'location')
        q.text = SLASH_REGEX.sub('/',place.locdesc.strip())


def encodenotes(place, parent):
    if len(place.note) > 0:
        q = etree.SubElement(parent, 'note')
        q.text = place.note

def encodeitineraries(place, parent, mapnum):
    if len(place.itinraw) > 0:
        q = etree.SubElement(parent, 'itinerary')
        q.text = place.itinraw.strip()
        citcontent = "BATlas %s %s (%s)" % (mapnum, place.dirtype, place.itinraw.strip())
        if len(place.batlasids) > 0:
            writecit(parent, citcontent, place.batlasids[0])
        else:
            writecit(parent, citcontent, '')

def oldschoolFeatureID(place, parent, mapnum):
    if place.tablei == -1 and place.rowi == -1:
        placeid = "batlas-%s-anon-%s" % (mapnum, place.anonsequence)
    else:
        placeid = "batlas-%s-%s-%s" % (mapnum, place.tablei+1, place.rowi+1)
    tag_fid = etree.Element("{%s}featureID" % ADLGAZ)
    tag_fid.text = placeid
    parent.append(tag_fid)
    
def oldschoolTimePeriodNames(place, parent):
    for tp in place.periods:
        tag_tp = etree.Element("{%s}timePeriod" % ADLGAZ)
        tag_tpn = etree.Element("{%s}timePeriodName" % ADLGAZ)
        tpstring = periods[tp[0]]
        if tp[1] == 'less-confident':
            tpstring += "?"
        tag_tpn.text = tpstring
        tag_tp.append(tag_tpn)
        parent.append(tag_tp)

    
def oldschoolAttribution(pipe, parent):
    for c in pipe.creators:
        tag_c = etree.Element("{%s}creator" % DC)
        tag_c.text = c
        parent.append(tag_c)
    for c in pipe.contributors:
        tag_c = etree.Element("{%s}contributor" % DC)
        tag_c.text = c
        parent.append(tag_c)

def oldschoolFeatureNames(place, parent, magicrefs):
    for pn in place.placenames:
        
        tag_fn = etree.Element("{%s}featureName" % ADLGAZ)
        
        # transliteration
        tag_translit = etree.Element("{%s}transliteration" % AWMC)
        tag_translit.text = pn.name
        tag_fn.append(tag_translit)
        
        # classificationSection
        try:
            if place.types.index('people') != 0:
                nametype = 'ethnic'
        except ValueError:
            nametype = 'geographic'
        tag_cs = etree.Element("{%s}classificationSection" % ADLGAZ)
        tag_ct = etree.Element("{%s}classificationTerm" % ADLGAZ)
        tag_ct.text = nametype
        tag_cs.append(tag_ct)
        tag_css = etree.Element("{%s}classificationScheme" % ADLGAZ)
        tag_csn = etree.Element("{%s}schemeName" % ADLGAZ)
        tag_csn.text = "geoNameType"
        tag_css.append(tag_csn)
        tag_cs.append(tag_css)
        if pn.inferred:
            tag_naspect = etree.Element("{%s}nameAspect" % AWMC)
            tag_naspect.attrib['ref'] = 'na-inferred'
            tag_cs.append(tag_naspect)
        if pn.completeness != 'complete':
            tag_naspect = etree.Element("{%s}nameAspect" % AWMC)
            tag_naspect.attrib['ref'] = 'na-reconstructed'
            tag_cs.append(tag_naspect)
        if pn.accuracy != 'accurate':
            tag_naspect = etree.Element("{%s}nameAspect" % AWMC)
            tag_naspect.attrib['ref'] = 'na-inaccurate'
            tag_cs.append(tag_naspect)
        tag_nassoc = etree.Element("{%s}nameAssociation" % AWMC)
        tag_nassoc.attrib['ref'] = pn.certainty
        tag_cs.append(tag_nassoc)
        tag_fn.append(tag_cs)

        # timePeriods for the name
        for tp in pn.periods:
            tag_tp = etree.Element("{%s}timePeriod" % ADLGAZ)
            tag_tpn = etree.Element("{%s}timePeriodName" % ADLGAZ)
            tpstring = periods[tp[0]]
            if tp[1] == 'less-confident':
                tpstring += "?"
            tag_tpn.text = tpstring
            tag_tp.append(tag_tpn)
            tag_fn.append(tag_tp)
            
        # secondary references for the name
        if len(pn.references) > 0:
            tag_refs = etree.Element("{%s}secondaryReferences" % AWMC)
            for ref in pn.references:
                tag_bibl_xml = "<tei:bibl xmlns='%s' xmlns:tei='%s'>%s</tei:bibl>" % (TEI, TEI, magicrefs[ref])
                if "tei:title" in tag_bibl_xml:
                    pass
                else:
                    print place.namestring.encode('ascii', 'xmlcharrefreplace')
                    print ">>>> no title in: '%s'" % tag_bibl_xml.encode('ascii', 'xmlcharrefreplace')
                    
                tag_refs.append(etree.XML(tag_bibl_xml))
                
            tag_fn.append(tag_refs)

            
        parent.append(tag_fn)
        
def oldschoolReferences(place, parent, mapnum):

    # first, recall our origins in the barrington atlas
    tag_refs = etree.Element("{%s}secondaryReferences" % AWMC)
    try:
        thislabel = place.namestring.replace("/ ", "/")
    except:
        thislabel = ''
    if len(thislabel) > 0:
        findi = thislabel.find(u'\xA7')
        if findi > -1:
            thislabel = thislabel[:findi-1].strip()
        tag_bibl_xml = "<tei:bibl xmlns='%s' xmlns:tei='%s'><title>BAtlas</title> <biblScope>%s %s %s</biblScope></tei:bibl>" % (TEI, TEI, mapnum, place.grid, thislabel)
    else:
        tag_bibl_xml = "<tei:bibl xmlns='%s' xmlns:tei='%s'><title>BAtlas</title> <biblScope>%s %s</biblScope></tei:bibl>" % (TEI, TEI, mapnum, place.grid)
    
    tag_refs.append(etree.XML(tag_bibl_xml))
    
    # now, any other references 
    magicrefs = {}
    if len(place.placenames) == 1:
        for ref in place.references:
            magicrefs[ref] = refmagic(ref, place.placenames[0].name)
    elif len(place.placenames) > 1:
        for ref in place.references:
            magicrefs[ref] = refmagic(ref, place.namestring)
        for pn in place.placenames:
            for ref in pn.references:
                magicrefs[ref] = refmagic(ref, pn.name)
    
        
    if len(magicrefs) == 0 and len(place.references) > 0:
        for ref in place.references:
            magicrefs[ref] = refmagic(ref, "")
                        
    for ref in place.references:
        try:
            # need to fix this usage now that we have internal tagging
            tag_bibl_xml = "<tei:bibl xmlns='%s' xmlns:tei='%s'>%s</tei:bibl>" % (TEI, TEI, magicrefs[ref])
            if "tei:title" in tag_bibl_xml:
                pass
            else:
                print place.namestring.encode('ascii', 'xmlcharrefreplace')
                print ">>>> no title in: '%s'" % tag_bibl_xml.encode('ascii', 'xmlcharrefreplace')
        except KeyError:
            print 'KeyError %s' % ref.encode('ascii', 'backslashreplace')
            print magicrefs
        tag_refs.append(etree.XML(tag_bibl_xml))
    parent.append(tag_refs)
    return magicrefs

def encodeplace(place, parent, mapnum, pipe):
    e = etree.SubElement(parent, '{%s}place' % PLEIADES)
    oldschoolFeatureID(place, e, mapnum)
    encodeids(place, e)
    encodetypes(place, e)
    encodegrid(place, e)
    encodemaplabels(place, e)
    citname = encodecitations(place, e, mapnum)
    encodeplacenames(place, e, mapnum, citname)
    encodeitineraries(place, e, mapnum)
    encodelocdesc(place, e)
    encodenotes(place, e)
    oldschoolTimePeriodNames(place, e)
    oldschoolAttribution(pipe, e)
    refdict = oldschoolReferences(place, e, mapnum)
    oldschoolFeatureNames(place, e, refdict)
    

def xmldump(pipe, destdir):
    """Output an xml file containing the ids, along with some descriptive info
    """
    
    priorcitations = []
    # get all the places
    places = pipe['places']
    
    # serialize to xml
    d = etree.Element('{%s}featurelist' % PLEIADES)
    d.attrib['mapnum'] = pipe.map_number
    q = etree.SubElement(d, '{%s}uribase' % PLEIADES)
    q.text = "%s/" % NAMESPACE
    for p in places:
        encodeplace(p, d, pipe.map_number, pipe)        
        
    cleantree = do_nscleanup(pipe['contextpath'], d)
        
                    
     # write to file, prepending explanator comment text, date etc.
    cmntf = open(COMMENTFILE)
    cmnt = cmntf.read()
    cmntf.close()
    
    dtime = dt.datetime.utcnow()
    dtstamp = dtime.isoformat()
    dtyear = dtime.year
                
    fn = "map%s.xml" % pipe.map_number
    fpath = os.path.join(destdir, fn)
    f = open(fpath, 'w')
    f.write(XMLDECL)
    cmnt = cmnt % (fn, pipe.map_number, dtime.isoformat(), PREPARER)
    f.write(cmnt)
    etree.ElementTree(cleantree).write(f)
    f.close()
    logging.info("wrote output result xml file on %s" % fpath)
    
def writecit(parent, tcontent, primeid):
    ctcontent = u''.join(tcontent.split())
    if ctcontent in priorcitations:
        # if primeid ends in dash-number then postfix the number onto the citation
        m = DASHNUM_END_REGEX.search(primeid)
        n = DASHNUM_END_REGEX.search(tcontent)
        if m and not n:
            writecit(parent, "%s (%s)" % (tcontent, m.group(1)), primeid)
        else:
            logging.warning("Suppressed writing of citation '%s' for primary id '%s' because that citation is already in use" % (tcontent, primeid))
            
    else:
        q = etree.SubElement(parent, '{%s}citation' % PLEIADES)
        q.text = tcontent
        priorcitations.append(ctcontent)