root/BADataMunger/trunk/biblioextractor.py

# ===========================================================================
# Copyright (C) 2006-2008 Ancient World Mapping Center (UNC-CH) and the 
# Institute for the Study of the Ancient World (NYU)
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# About Pleiades
# --------------
#
# Pleiades is an international research network and associated web portal and
# content management system devoted to the study of ancient geography.
#
# See http://pleiades.stoa.org
#
# Funding for the creation of this software was provided by a grant from the
# U.S. National Endowment for the Humanities (http://www.neh.gov), and
# by the Institute for the Study of the Ancient World at New York University
# (http://www.nyu.edu/isaw)
# ===========================================================================

import re
import logging

import lxml.etree as etree

from etreehelps import getalltext
from texthelps import normalizetext

REANATITLE = u'\u201C(.*)\u201D'

def extract(source):
    """Extract bibliography from cleaned up wordxml. Finds the bibliography div and
    pulls out the titles and citations into a list. Also tries to find an abbreviations table
    in the directory listing div and appends any titles and citations there to the list.
    Expects as input an lxml etree. A python list of tuples like (shorttitle, fulltitle, fullcitation)
    is returned, where shorttitle and fulltitle are unicode strings and fullcitation is a
    lxml etree Element (an html p tag containing the full citation)."""
    
    logging.info("BEGIN extracting bibliography from cleaned-up word html")
    
    # determine which div contains the bibliography
    biblist = []
    divs = source.xpath("descendant::*[local-name()='div']")
    bibdiv = None
    dirlistdiv = None
    for i, div in enumerate(divs):
        text = getalltext(div).strip()
        if text == 'Bibliography':
            bibdiv = divs[i+1]  # yes, really, the next div
            logging.info("bibliography div is div %s" % (i+1))
        elif text.find('Abbreviation') > 0:
            dirlistdiv = divs[i]
            logging.info("abbreviation div is div %s" % i)
           
        
    # parse the bibliography div
    paras = bibdiv.xpath("*[local-name()='p']")
    logging.info ("the bibliography div contains %s paragraphs" % len(paras))
    for p in paras:
        text = normalizetext(p.text)
        alltext = normalizetext(getalltext(p))
        if text==alltext:
            # this paragraph contains a short title
            try:
                biblist.append((shorttitle, atitle, citation))
            except:
                pass
            shorttitle = text
        else:
            # this paragraph contains a citation of a work: try to extract title, first as from an article and, failing
            # that, as if from a book
            citation = p
            atitle = extract_title(citation)
                
    # parse the list of abbreviations out of the directory listing div
    if dirlistdiv:
        table = dirlistdiv.xpath("descendant::*[local-name()='p' and contains(., 'Abbreviation')]")[0]
        rows = table.xpath("following-sibling::*[local-name()='table'][1]/*[local-name()='tr']")
        logging.info("the abbreviations table has %s rows" % len(rows))
        for row in rows:
            shorttitle = normalizetext(getalltext(row.xpath("*[local-name()='td']")[0]))
            citation = row.xpath("*[local-name()='td'][2]/*[local-name()='p']")[0]
            #print etree.tostring(citation)
            atitle = extract_title(citation)
            biblist.append((shorttitle, atitle, citation))
        
    logging.info("DONE extracting bibliography from cleaned-up word html")
    
    return biblist
    
    
def extract_title(source):
    """Given an lxml etree Element, interpret its text content as a bibliographic citation 
    and attempt to parse from it an article title. If that fails, try to parse a book title."""
    
    
    reana = re.compile(REANATITLE)
    m = reana.search(normalizetext(getalltext(source)))
    
    try:
        atitle = m.groups()[0]
    except:
        atitle = normalizetext(getalltext(source.xpath("descendant::*[local-name()='i']")[0]))
    try:
        if atitle[-1] == ',':
            atitle = atitle[:-1]
    except:
        pass
    return atitle
    
    
def _test():
    import doctest
    doctest.testmod()
    doctest.testfile('tests/biblioextractor.txt')
    # invoke additional doctest files here

if __name__ == "__main__":
    _test()