root/BADataMunger/trunk/pipeline.py

from os.path import normpath, normcase, isdir, isfile, splitdrive, splitext, split, join
import os
import logging

import lxml.etree as etree

import wordhtml2xml
import wordstripper
import biblioextractor
import bibliosaver
import tablegroker
import tableparser
import placesaver
import gismixer
from etreehelps import getalltext
from texthelps import normalizetext


class Pipe:
    """String together a series of transformations and operations to munge a 
    BAtlas directory file into Pleiades-ready data
        
    Use like:
    
import pipeline
p = pipeline.Pipe(r'/TomDocs/awmcwork/pleiadesact/svnbox/BADataMunger/config/BATL065_config.xml', r'/badigit/wordhtml/BATL065_.htm', r'/badigit/extraction/ba065points.xml', r'/badigit/bib/awmcbib-mods.xml', r'/badigit/xml4pleiades')
p.cycle()
        
            
    """

    def __init__(self, configfile, dirfile, gisfile, biblibfile, destdir):
        # source, source_config=None, source_gis=None):
        """Create a pipe object to manage the conversions and open the source 
        file
        """
        logging.basicConfig(level=logging.DEBUG, format='%(levelname)-8s %(message)s')
        logging.info("INITIALIZING: %s" % self.__class__)

        self.data = {}
        self['contextpath'] = os.getcwd()
        logging.info("contextpath: %s" % self['contextpath'])
        
        self['configfile'] = normpath(normcase(configfile))
        self['dirfile'] = normpath(normcase(dirfile))
        self['gisfile'] = normpath(normcase(gisfile))
        self['biblibfile'] = normpath(normcase(biblibfile))
        self['destdir'] = normpath(normcase(destdir))
        
        if not isfile(self['configfile']):
            raise Error, "No file found at path: %s" % self['configfile']
        elif not isfile(self['dirfile']):
            raise Error, "No file found at path: %s" % self['dirfile']
        elif not isfile(self['gisfile']):
            raise Error, "No file found at path: %s" % self['dirfile']
        elif not isdir(self['destdir']):
            raise Error, "No directory found at path: %s" % self['destdir']
        else:
            
            drive, dirpath = splitdrive(self['dirfile'])
            filepath, filename = split(dirpath)
            self['filenameroot'], extension = splitext(filename)  
            self['bibdestdir'] = join(self['destdir'], 'mods')      
            
            if not isdir(self['bibdestdir']):
                raise Error, "No directory found at path: %s" % self['bibdestdir']
            else:
                
                # read and store the directory file
                f = open(self['dirfile'])
                self['wordhtml'] = f.read()
                f.close()
                logging.info("read and stored directory file from %s" % self['dirfile'])
                
                # read and store the config file
                f = open(self['configfile'])
                config_text = f.read()
                f.close()
                logging.info("read and stored config file from %s" % self['configfile'])
                
                # get some essential info from the config file
                self['config'] = etree.XML(config_text)
                try:
                    self.map_number = self['config'].xpath("//map_number")[0].text
                except:
                    self.map_number = 'XYZ'
                logging.info("map number from config file: %s" % self.map_number)
                
                self.creators = []
                self.contributors = []
                self.rights = ''
                for c in self['config'].xpath("//creator"):
                    self.creators.append(normalizetext(getalltext(c)))
                for c in self['config'].xpath("//contributor"):
                    self.contributors.append(normalizetext(getalltext(c)))
                try:
                    self.rights = self['config'].xpath("//rights")[0].text
                except:
                    pass
                
                self.mixer = gismixer.gisMixer(self, self['gisfile'])

        logging.info("DONE INITIALIZING: %s\n" % self.__class__)
        
            
    def __getitem__(self, key): return self.data[key]
        
    def __setitem__(self, key, item): self.data[key] = item
        
    def cycle(self):
        """Cycle through all steps in the pipeline"""
        
        logging.info("CYCLING: %s" % self.__class__)

        self['wordxml'] = wordhtml2xml.convert(self['wordhtml'])
        self['cleanxml'] = wordstripper.strip(self['contextpath'], self['wordxml'])
        self['bibliography'] = biblioextractor.extract(self['cleanxml'])
        bibliosaver.save_biblio_mods(self)
        self['dirtables'] = tablegroker.grok(self['cleanxml'])
        self['places'] = tableparser.parse(self, self['dirtables'], self.map_number)
        self.mixer.mixall(self['config'], self['places'])
        placesaver.save_places_frank(self)
        
        logging.info("DONE CYCLING: %s\n" % self.__class__)
        
            

    def save(self, itemkey, encoding='utf-8'):
        """Basic save-with-encoding function for writing content to an arbitrary 
        file"""
        
        if itemkey == 'bibliography':
            save_bibliography(encoding)
        else:
            content = self[itemkey]
            try:
                pcontent = content.encode(encoding)
            except:
                pcontent = etree.tostring(content).encode(encoding)
            destfile = "%s-%s.xml" % (self['filenameroot'], itemkey)
            g = open(join(self['filepath'], destfile),'w')
            g.write(pcontent)
            g.close()
        
    def save_bibliography(self, encoding='utf-8'):
        """Save individual bibliography files."""
        
        bibs = self['bibliography']
        for i, bib in enumerate(bibs):
            div = etree.Element("div")
            head = etree.Element("head")
            head.text = bib[0]
            div.append(head)
            subhead = etree.Element("subhead")
            subhead.text = bib[1]
            div.append(subhead)
            div.append(bib[2])
            pcontent = etree.tostring(div).encode(encoding)
            destfile = "%s-biblio-%s.xml" % (self['filenameroot'], i)
            g = open(join(self['filepath'], destfile),'w')
            g.write(pcontent)
            g.close()