root/BADataMunger/trunk/pipebiblio.py

# ===========================================================================
# Copyright (C) 2006-2008 Ancient World Mapping Center (UNC-CH) and the 
# Institute for the Study of the Ancient World (NYU)
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# About Pleiades
# --------------
#
# Pleiades is an international research network and associated web portal and
# content management system devoted to the study of ancient geography.
#
# See http://pleiades.stoa.org
#
# Funding for the creation of this software was provided by a grant from the
# U.S. National Endowment for the Humanities (http://www.neh.gov), and
# by the Institute for the Study of the Ancient World at New York University
# (http://www.nyu.edu/isaw)
# ===========================================================================
"""
python pipebiblio.py --directory=/path/to/directory-file --library=/path/to/bibliographic-library-file --destination=/path/to/destination/directory/

This script extracts bibliographic citations from an HTML version of a Barrington
Atlas Map-by-Map Directory and reformats them into MODS XML.
"""

from os.path import abspath, normcase, isdir, isfile, splitdrive, splitext, split, join, exists
import os
import logging
import sys
import getopt

import lxml.etree as etree

import wordhtml2xml
import wordstripper
import biblioextractor
import bibliosaver
from etreehelps import getalltext
from texthelps import normalizetext

class Pipe:
    """ A processing pipeline for bibliographic extraction and munging."""
    
    def __init__(self):
        logging.basicConfig(level=logging.DEBUG, format='%(levelname)-8s %(message)s')
        self.data = {}
        self.data['contextpath'] = os.getcwd()
        
    def cycle(self, dirfile, biblibfile, bibdestdir):
        self.loaddirfile(dirfile)
        self.fixdir()
        self.extractbiblio()
        self.savebiblio(biblibfile, bibdestdir)
        
    def loaddirfile(self, dirfile):
        f = open(dirfile)
        self.data['wordhtml'] = f.read()
        f.close()
        drive, dirpath = splitdrive(dirfile)
        filepath, filename = split(dirpath)
        self.data['filenameroot'], extension = splitext(filename)
        
    def fixdir(self):
        self.data['wordxml'] = wordhtml2xml.convert(self.data['wordhtml'])
        self.data['cleanxml'] = wordstripper.strip(self.data['contextpath'], self.data['wordxml'])
        
    def extractbiblio(self):
        self.data['bibliography'] = biblioextractor.extract(self.data['cleanxml'])
        
    def savebiblio(self,biblibfile,bibdestdir):
        self.data['bibdestdir'] = bibdestdir
        self.data['biblibfile'] = biblibfile
        bibliosaver.save_biblio_mods(self.data)
        
def main(argv):
    
    dirfile = None
    bibdestdir = None
    biblibfile = None
    
    try:                                
        opts, args = getopt.getopt(argv, "s:d:l:h", ["directory=", "destination=", "library=", "help"])
    except getopt.GetoptError:
        print __doc__
        sys.exit(2)                     

    for opt, arg in opts:
        if opt in ("-s", "--directory"):
            dirfile = abspath(normcase(arg))
        elif opt in ("-d", "--destination"):
            bibdestdir = abspath(normcase(arg))
        elif opt in ("-l", "--library"):
            biblibfile = abspath(normcase(arg))
        elif opt in ("-h", "--help"):
            print __doc__
            sys.exit(0)
            
    if not dirfile:
        print 'a source directory file must be specified with the -s (--directory) option'
        sys.exit(0)
    if not bibdestdir:
        print 'a destination directory must be specified with the -d (--destination) option'
        sys.exit(0)
    if not biblibfile:
        print 'a bibliographic library file must be specified with the -l (--library) option'
        sys.exit(0)
     
    if not isfile(dirfile):
        print "the specified source directory file (%s) is not an existing file" % dirfile
        sys.exit(0)
    if not isdir(bibdestdir):
        print "the specified destination directory (%s) is not an existing directory" % bibdestdir
        sys.exit(0)
    if not isfile(biblibfile):
        print "the specified bibliographic library file (%s) is not an existing file" % biblibfile
        sys.exit(0)
         
    p = Pipe()
    p.cycle(dirfile, biblibfile, bibdestdir)
            
if __name__ == "__main__":
    main(sys.argv[1:])