root/BADataMunger/trunk/gismixer.py

import logging

import lxml.etree as etree

from etreehelps import getalltext
from texthelps import normalizetext
import tableparser

ESRI = "http://www.esri.com/schemas/ArcGIS/9.0"

class gisMixer:
    """"""
    
    def __init__(self, parent, source):
        
        self.parent = parent
        # read in the source file
        f = open(source)
        gis_text = f.read()
        f.close()
        tree = etree.XML(gis_text)
        
        # get the field names
        e_fields = tree.xpath("//WorkspaceData/descendant::Fields/FieldArray/Field")
        self.fieldnames = [normalizetext(getalltext(e_field.xpath("Name")[0])) for e_field in e_fields]
        self.gridi = self.fieldnames.index('GridSquare')
        self.labeli = self.fieldnames.index('Label')
        self.disambiguatori = self.fieldnames.index('disambiguator')
        self.shapei = self.fieldnames.index('SHAPE')
        self.typei = self.fieldnames.index('Type')
        self.approxi = self.fieldnames.index('Approximate')
        
        # get the geometry type
        e_geofielddef = tree.xpath("//WorkspaceData/descendant::Fields/FieldArray/Field[%s]/GeometryDef/GeometryType" % (self.shapei + 1))
        self.geotype = normalizetext(getalltext(e_geofielddef[0]))
        logging.info("geotype=%s" % self.geotype)
        
        # get the data and store it for later use
        self.fieldvalues = []
        e_records = tree.xpath("//WorkspaceData/descendant::Records/Record")
        for i, e_record in enumerate(e_records):
            self.fieldvalues.append([])
            e_values = e_record.xpath("Values/Value")
            for j, e_value in enumerate(e_values):
                if j == self.shapei:
                    self.fieldvalues[i].append(self.parseshape(e_value))
                else:
                    self.fieldvalues[i].append(normalizetext(getalltext(e_value)))
            
        # create a match flag list, one item for each set of fielddata
        self.fieldmatches = [False for i in range(len(self.fieldvalues))]
            
        logging.info ("gisMixer got %s records from %s" % (len(self.fieldvalues), source))
        logging.debug("FIELD NAMES ARE:")
        for name in self.fieldnames:
            logging.debug("%s" % name )
        logging.debug("FIRST RECORD CONTENT:")
        for item in self.fieldvalues[0]:
            logging.debug("%s" % item)
            
    def parseshape(self, e_shape):
        coords = []
        if self.geotype == 'esriGeometryPoint':
            t_x = e_shape.findtext('X')
            t_y = e_shape.findtext('Y')
            cpair = (round(eval(t_x), 4), round(eval(t_y), 4))
            coords.append(cpair)
        else:
            logging.warning("gismixer.gisMixer.parseshape() has no support for geotype = %s" % self.geotype)
        return coords
            
    def mixall(self, config, places):

        logging.info("BEGIN attempting to match and combine %s directory places with %s map/gis places" \
            % (len(places),len(self.fieldvalues)))
        
        for place in places:
            if place.dirtype == 'unlocated' or place.dirtype == 'false':
                place.matched = True
            else:
                self.mixin(config, place)
                
        logging.info("DONE attempting to match and combine %s directory places with %s map/gis places" % \
            (len(places),len(self.fieldvalues)))
        
        logging.info("BEGIN attempting to deal with map/gis places for which there is no match in the directory")
        
        gisdata = self.fieldvalues
        matchedgisdata = [value for i, value in enumerate(self.fieldvalues) if self.fieldmatches[i]]
        
        
        if len(gisdata) == len(matchedgisdata):
            logging.info("all GIS places (%s) were matched" % len(gisdata))
        elif len(gisdata) > len(matchedgisdata):
            logging.info("%s of the %s GIS places were matched" % (len(matchedgisdata), len(gisdata)))
            unmatchedgisdata = [value for i, value in enumerate(self.fieldvalues) if self.fieldmatches[i] == False]
            suppressnodes = config.xpath("//suppress")
            lbl_suppressions = []
            full_suppressions = []
            for snode in suppressnodes:
                try:
                    supgrid = normalizetext(getalltext(snode.xpath("grid")[0]))
                except: 
                    supgrid = ''
                try: 
                    suplabel = normalizetext(getalltext(snode.xpath("label")[0]))
                except:
                    suplabel = ''
                try:
                    suptype = normalizetext(getalltext(snode.xpath("type")[0]))
                except:
                    suptype = ''
                try:
                    supdisamb = normalizetext(getalltext(snode.xpath("disambiguator")[0]))
                except:
                    supdisamb = ''
                if len(suplabel) > 0 and suptype == '' and supdisamb == '':
                    lbl_suppressions.append((supgrid, suplabel))
                else:
                    full_suppressions.append((supgrid, suplabel, suptype, supdisamb))
                
            #suppressions = [(normalizetext(getalltext(snode.xpath("grid")[0])), normalizetext(getalltext(snode.xpath("label")[0]))) for snode in suppressnodes]
            
            for i, value in enumerate(unmatchedgisdata):
                supmatches = [item for item in lbl_suppressions if item == (value[self.gridi], value[self.labeli])]
                if len(supmatches) == 1:
                    logging.info("suppressing item %s based on a grid-and-label suppression directive:" % i)
                    for j, item in enumerate(value):
                        logging.info("    %s = %s" % (self.fieldnames[j], item))
                elif len(supmatches) > 1:
                    logging.warning("multiple suppression directives match this item (%s, %s)" % (value[self.gridi], value[self.labeli]))
                else:
                    supmatches = [item for item in full_suppressions if item == (value[self.gridi], value[self.labeli], value[self.typei].lower(), value[self.disambiguatori])]
                    if len(supmatches) == 1:
                        logging.info("suppressing item %s based on a full suppression directive:" % i)
                        for j, item in enumerate(value):
                            logging.info("    %s = %s" % (self.fieldnames[j], item))
                    elif len(supmatches) > 1:
                        logging.warning("multiple suppression directives match this item (%s, %s, %s, %s)" % (value[self.gridi], value[self.labeli], value[self.typei].lower(), value[self.disambiguatori]))
                    else:
                        logging.warning("UNMATCHED map/gis place %s was NOT suppressed:" % i)
                        for j, item in enumerate(value):
                            logging.warning("    %s = %s" % (self.fieldnames[j], item))
                        places.append(self.addin(value, i))
        else:
            logging.warning("tallies absurdly claim that more GIS places were matched (%s) than are actually in the GIS data (%s)" \
                % (len(matchedgisdata), len(gisdata)))
        logging.info("DONE attempting to deal with map/gis places for which there is no match in the directory")
        
    def addin(self, gisdata, seq):
        p = tableparser.Place()
        p.anonsequence = seq
        p.map_number = self.parent.map_number
        p.tablei = -1
        p.rowi = -1
        p.matched = True
        p.shapes.append(gisdata[self.shapei])
        p.approximates.append(gisdata[self.approxi])
        p.types.append(self.get_type(gisdata[self.typei].lower()))
        p.grid = gisdata[self.gridi]
        return p

    def mixin(self, config, place):
        r = self.get_records(config, place)
        if r:

            if place.matched:
                logging.warning("place claims it's already matched")
            place.matched = True
                
            # mix the data
            
            for i, re in enumerate(r):
                place.shapes.append(re[self.shapei])
                place.approximates.append(re[self.approxi])
                place.types.append(self.get_type(re[self.typei].lower()))
        else:
            try:
                logging.warning( "failed to match %s: '%s' (%s - %s), rowi=%s, disambiguator: %s" % (place.dirtype, label.encode('ascii', 'backslashreplace'), place.locdesc.encode('ascii', 'backslashreplace'), place.itinraw.encode('ascii', 'backslashreplace'), place.rowi, disambiguator))
            except:
                logging.warning( "failed to match %s: '%s' (%s - %s), rowi=%s" % (place.dirtype, place.namestring.encode('ascii', 'backslashreplace'), place.locdesc.encode('ascii', 'backslashreplace'), place.itinraw.encode('ascii', 'backslashreplace'), place.rowi))
        
            
    def get_type(self, rawtype):
        if rawtype == 'road station':
            cleantype = 'station'
        elif rawtype == 'water wheel':
            cleantype = 'wheel'
        elif rawtype == 'mountain pass':
            cleantype = 'pass'
        elif rawtype == 'inland water':
            cleantype = 'water-inland'
        elif rawtype == 'open water':
            cleantype = 'water-open'
        elif rawtype == 'salt marsh':
            cleantype = 'salt-marsh'
        else:
            cleantype = rawtype
        return cleantype
                    
    def get_records(self, config, p):

        gridmatches = []
        typematches = []
        labelmatches = []
        matches = []
        
        candidates = [value for i, value in enumerate(self.fieldvalues) if self.fieldmatches[i] == False]
        label = self.clean_label(p.namestring)
        gridsquare = p.grid
        soughttype = p.dirtype.lower()
        
        try:
            e_dis = config.xpath("//disambiguator[@tabletype='%s' and @rowi='%s']" % (soughttype , p.rowi ))[0]
            disambiguator = normalizetext(getalltext(e_dis))
        except:
            disambiguator = 0
            
        logging.info("seeking a %s in gridsquare %s with disambiguator %s" % (soughttype, gridsquare, disambiguator))
        
        
        try:
            gridmatches = [value for value in candidates if value[self.gridi] == gridsquare]
        except ValueError:
            logging.warning( "gisMixer could not find any data for gridsquare = %s (seeking %s)" % (gridsquare, label))
        if len(gridmatches)>0 and soughttype != 'name' and soughttype != 'numbered':
            try:
                typematches = [value for value in gridmatches if value[self.typei].lower() == soughttype]
            except:
                logging.warning("gisMixer could not find any data for type = '%s' in grid '%s'" % (soughttype, gridsquare))
        elif len(gridmatches)>0:
            typematches = gridmatches
            
        if len(gridmatches) == 0:
            logging.warning("gisMixer found 0 gridmatches")
        elif len(typematches) == 0:
            logging.warning("gisMixer found 0 typematches in grid")
        
        if len(typematches) > 0:
            try: 
                labelmatches = [value for value in typematches if self.clean_label(value[self.labeli]) == label]
            except ValueError:
                logging.warning("gisMixer could not find any data for label = '%s' in grid '%s' with type '%s'" % (label, gridsquare, soughttype))
        if len(labelmatches) > 0 and disambiguator != 0:
            matches = [value for value in labelmatches if value[self.disambiguatori] == disambiguator]
            if len(matches) == 0:
                logging.warning("gisMaker could not find any data for label = '%s' and disambiguator = '%s' in grid '%s'" % (label, gridsquare, disambiguator))
        elif len(labelmatches) == 0:
            logging.warning("gisMaker could not find any data for label = '%s' in grid '%s'" % (label, gridsquare))
            matches = labelmatches
        else:
            matches = labelmatches
        if len(matches) == 1:
            thematch = matches[0]
            self.fieldmatches[self.fieldvalues.index(thematch)] = True
            return matches
        elif len(matches) > 1:
            try:
                multe = config.xpath("//multiple[@tabletype='%s' and @rowi='%s']" % (soughttype , p.rowi ))[0]
                #print multe
                multext = normalizetext(getalltext(multe))
                #print multext
                mult = eval(multext)
            except:
                mult = 0
            if len(matches) == mult:
                for thematch in matches:
                    self.fieldmatches[self.fieldvalues.index(thematch)] = True
                return matches
            else:
                logging.warning("gisMaker found %s apparent matches for label = '%s', in grid '%s', with disambiguator = '%s' and mult = '%s'" % (len(matches), label, gridsquare, disambiguator, mult))
                return None
        else:
            return None
            
    def clean_label(self, raw_label):
        clabel = raw_label.replace("/ ", "/")
        diamondi = clabel.find(u'\xa7')
        if diamondi > 0:
            clabel = clabel[:diamondi-1]
        clabel = clabel.replace(", Mon.", "")
        clabel = clabel.replace(", T.", "")
        clabel = clabel.replace(u"\u2018", u"'")
        clabel = clabel.replace(u"\u2019", u"'")
        #print "'%s'" % clabel.encode('ascii', 'backslashreplace')
        clabel = clabel.strip()
        return clabel