Changeset 1337

Timestamp:

07/18/08 17:08:16 (3 months ago)

Author:

thomase

Message:

refactor basic table parsing for extensibility (use a 'column map' dictionary for each type and pass it to common code instead of hardcoding handling for each column in each table); capture 'origstring' for later checking; add 'featurecount' as an attribute of place for 'group' features; better handling of serial italic tags in source names (pointless, annoying and time-wasting MSWord cruft); remove old testing code; stop garbling ellipses in namestrings; normalize capitalization in placenames (esp. dual caps in variant decompositions); catch quoted substrings in check for 'accuracy'

Files:

• BADataMunger/trunk/tableparser.py (modified) (9 diffs)

BADataMunger/trunk/tableparser.py

@@ -10 +10 @@
-from etreehelps import getalltext
-from texthelps import normalizetext
+
+CMAP_TMPL_PN_4 = {'grid':0, 'placenames':1, 'periods':2, 'references':3}
+CMAP_TMPL_UN_4 = { 'grid' : 0, 'locdesc' : 1, 'periods':2, 'references':3}
+CMAP_TMPL_MAT_DATELESS = {'grid':0,'placenames':1,'material':2,'references':3} 
+CMAP_TMPL_NAMED_5 = {'grid':0,'placenames':1,'namestring':1, 'periods':2,'locdesc':3,'references':4}
+
+GROUP_COUNT_REGEX = re.compile(r'\s+\((.)\)$')
-
-tabletypes = {
@@ -19 +26 @@
-    "False Toponyms":"false",
-    "False Toponym":"false",
+    "Levee":"levee",
+    "Levees":"levee",
-    "Name":"name",
-    "Names":"name",
@@ -28 +37 @@
-    "Unlocated Toponyms":"unlocated",
-    "Unlocated Toponym":"unlocated",
+    "Villas":"villa",
+    "Villa":"villa",
-    "Walls":"wall",
-    "Wall":"wall"
@@ -156 +167 @@
-        self.placenames=[]
-        self.namestring = u''
+        self.orignamestring = u''
-        self.number=''
-        self.periods=[]
@@ -163 +175 @@
-        self.dirtype=''
-        self.types=[]
+        self.featurecount = 1
+        self.material = ''
-        self.matched = False
-        
@@ -276 +290 @@
-            print "Untrapped tag = '%s'" % d.tag
-        
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-    def add_aqueduct(self, cells):
-        # grid | locdesc | periods | references
-        self.dirtype = 'aqueduct'
-
-
-
-
+
+
-
-    def add_bridge(self, cells):
-        # grid | locdesc | periods | references
-        self.dirtype = 'bridge'
-
-
-
-
+
+
-        
-    def add_earthworks(self, cells):
-        # grid | name/locdesc | periods | references
-        self.dirtype = 'earthworks'
-self.grid = normalizetext(getalltext(cells[0]))
-# parse_placenames tests for content inside italic or not; if inside italic, assumed to be locdesc, and added accordingly; otherwise
-
-    self.placenames = self.parse_placenames(cells[1], cells[2], cells[3])
-# self.locdesc = normalizetext(getalltext(cells[1]))
-self.periods = self.parse_periods(cells[2])
-references = self.parse_references(cells[3]
-
+cmap = CMAP_TMPL_PN_4
+self.addinator(cells, cmap)
+
+def add_levee(self, cells):
+self.dirtype = 'levee'
+cmap = CMAP_TMPL_UN_4
+addinator(cells, cmap
+
-    def add_mine(self, cells):
-        # grid | locdesc | material | references
-        self.dirtype = 'mine'
-
-
-
+
+
-        
-    def add_quarry(self, cells):
-        # grid | locdesc | material | references
-        self.dirtype = 'quarry'
-
-
-
-
+
+
-    
-    def add_name(self, cells):
-        # grid | placename(s) | periods | locdesc | references
-        self.dirtype = 'name'
-
-
-
-
-
-
+
+
-        
-    def add_false(self, cells):
-        # name | references | comment
-        self.dirtype = 'false'
+        cmap = {'namestring':0, 'references':1, 'comment':2}
+        self.addinator(cells, cmap)
-        self.placenames = self.parse_placenames(cells[0], None, cells[1])
-        self.namestring = normalizetext(getalltext(cells[0]))
-        self.references = self.parse_references(cells[1])
-        self.comment = normalizetext(getalltext(cells[2]))
-        
-    def add_numbered(self, cells):
-        # number | grid | location | period | reference
-        self.dirtype = 'numbered'
-
-
-
-
-
-
-
+
+
-        
-    def add_road(self, cells):
-        # itinerary | period | reference
-        self.dirtype = 'road'
-
-
-
-
-
+
+
-        
-    def add_unlocated(self, cells):
-        # name | period | probable location | reference
-        self.dirtype = 'unlocated'
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
-    def add_wall(self, cells):
-        # grid | locdesc | periods | references
-        self.dirtype = 'wall'
-
-
-
-
+
+
-        
-    def parse_periods(self, periodcell):
@@ -429 +458 @@
-            # test for italic tag
-            nameishmatches = namecell.xpath("descendant-or-self::*[contains(normalize-space(text()),'%s')]" % nameish.strip())
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-            nameishitalics = deepestnameishmatch.xpath("ancestor-or-self::*[local-name()='i']")
-            #if 'Brazda' in nameish:
-                #logging.debug("nameish = '%s'\n>>>> cellxml:\n%s\n" % (nameish, etree.tostring(namecell)))
-                #logging.debug("%s nameish matches" % len(nameishmatches))
-                #for nm in nameishmatches:
-                #    logging.debug("nameish match xml:\n%s\n" % etree.tostring(nm))
-                #logging.debug("deepestnameish match xml:\n%s\n" % deepestnameishmatch)
-                #logging.debug("%s nameishitalics" % len(nameishitalics))
-                
-            if len(nameishitalics) != 0:
-                # this is actually the modern location, not the placename
-                #logging.debug("interpreting italicized content in names column as location description for %s" % nameish.strip())
-                if len(self.locdesc) == 0:
-                    self.locdesc = nameish.strip()
-                else:
-                    self.locdesc = u"%s, %s" % (nameish.strip(), self.locdesc)
-
+replace(u'(...)', '').
-                # this nameish contains parentheses, which means that we need to generate and store multiple variants
-                variants = parse_name_variants(nameish.strip())
@@ -490 +528 @@
-                    pname.references = self.references
-
+        # normalize capitalization in placenames (this is mainly about dealing with generated variants beginning with parens
+        for pname in placenames:
+            if pname.variant:
+                pwords = pname.name.split()
+                qword = []
+                revariant = u''
+                for pword in pwords:
+                    if pword[0].isupper():
+                        qword = pword.capitalize()
+                    else:
+                        qword = pword
+                    revariant = u"%s %s" % (revariant, qword)
+                pname.name = revariant
+                
-        return placenames
-        
@@ -517 +569 @@
-        accuracy = 'accurate'
-        pattern = u'\u2018[^\u2019]+\u2019'
-mat
+sear
-        if m:
-            accuracy = 'inaccurate'