Changeset 1344

Timestamp:

07/22/08 13:36:47 (3 months ago)

Author:

thomase

Message:

suppress coastal change; develop more nuanced handling of bad markup inside names; add parsing support for many table types including: canal, causeway, cemetery, church 'cluster', clausurae, coastal change, dams, dikes, forts, lettered sites/places/water, levees, mines/quarries, monuments, numbered fort groups, road stations, springs, tumuli, wells

Files:

• BADataMunger/trunk/batlaspipe.py (modified) (1 diff)
• BADataMunger/trunk/bidmaker.py (modified) (3 diffs)
• BADataMunger/trunk/tableparser.py (modified) (11 diffs)

BADataMunger/trunk/batlaspipe.py

@@ -156 +156 @@
-                grid = place.grid
-
-
+,'coastal-change'
-                logging.info("batlaspipe.idit() deliberately suppressed ID creation for %s: (%s, %s, %s, %s)" % (place.dirtype, place.namestring, place.grid, place.locdesc, u' - '.join(place.itinerary)))
-            else:

BADataMunger/trunk/bidmaker.py

@@ -44 +44 @@
-        phrase = origstring.strip()
-        
-
+,'coastal-change'
-            # suppress certain types because they're not helpful
-            return u''
-            
-
+canal', '
-            # might be named or unnamed
-            if len(phrase) > 0:
@@ -56 +56 @@
-                phrase = "%s %s" % (dirtype, locdesc.strip())
-        
-
+,'lettered'
-            # the most basic case: features with names:
-            #     use the batlas directory namestring
@@ -66 +66 @@
-        
-            
-levee', 'quarry', 'villa', 'villa-group', 'wa
+bridge-group', 'causeway', 'cemetery', 'church-group', 'dam', 'fort', 'fort-group', 'levee', 'monument', 'quarry', 'quarry-group', 'road-station', 'spring', 'tumulus', 'villa', 'villa-group', 'wall', 'we
-            # second most common case: more-or-less normal features without names
-            phrase = "%s %s %s" % (dirtype, phrase, locdesc)

BADataMunger/trunk/tableparser.py

@@ -16 +16 @@
-CMAP_TMPL_NAMED_5 = {'grid':0,'placenames':1,'namestring':1, 'periods':2,'locdesc':3,'references':4}
-
-
+
+
+
+
+
-
-tabletypes = {
@@ -23 +27 @@
-    "Bridge":"bridge",
-    "Bridges":"bridge",
+    "Canals (unnamed)":"canal",
+    "Canal (unnamed)":"canal",
+    "Canals (named)":"canal_named",
+    "Causeway":"causeway",
+    "Causeways":"causeway",
+    "Cemeteries":"cemetery",
+    "Church Cluster":"church_group",
+    "Clausurae (linear barriers)":"clausura",
+    "Coastal Change":"coastal_change",
+    "Dam":"dam",
+    "Dikes / Levees":"levee",
-    "Earthworks":"earthworks",
-    "False Toponyms":"false",
-    "False Toponym":"false",
+    "Fort":"fort",
+    "Forts":"fort",
+    "Forts / Fortified Settlements":"fort_set",
+    "Lettered Sites":"lettered_site",
+    "Lettered Place Names":"lettered_places",
+    "Lettered Water Names":"lettered_water",
-    "Levee":"levee",
-    "Levees":"levee",
+    "Mines / Quarries":"quarry",
+    "Monuments":"monument",
-    "Name":"name",
-    "Names":"name",
-    "Numbered Sites":"numbered",
+    "Numbered Fort Group":"numbered_fort_group",
-    "Quarry":"quarry",
-    "Quarries":"quarry",
-    "Roads":"road",
-    "Road":"road",    
+    "Road Station":"road_station",
+    "Springs":"spring",
+    "Tumuli":"tumulus",
-    "Unlocated Toponyms":"unlocated",
-    "Unlocated Toponym":"unlocated",
@@ -40 +67 @@
-    "Villa":"villa",
-    "Walls":"wall",
-
+
+
-}
-
@@ -310 +338 @@
-                    print "cellmap:"
-                    print cellmap
-
+ or k == 'letter'
-                self.number = normalizetext(getalltext(cells[v]))
-            if k == 'itinerary':
@@ -346 +374 @@
-        self.dirtype = 'bridge'
-        cmap = CMAP_TMPL_UN_4
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-        
-    def add_earthworks(self, cells):
@@ -367 +436 @@
-        cmap = {'grid':0, 'locdesc':1, 'periods':2, 'material':3, 'references':4}
-        self.addinator(cells,cmap)
+        
+    def add_monument(self, cells):
+        self.dirtype = 'monument'
+        cmap = CMAP_TMPL_UN_4
+        self.addinator(cells, cmap)
-    
-    def add_name(self, cells):
@@ -379 +453 @@
-        self.placenames = self.parse_placenames(cells[0], None, cells[1])
-        
+    def add_fort(self, cells):
+        self.dirtype = 'fort'
+        cmap = CMAP_TMPL_UN_4
+        self.addinator(cells, cmap)
+        
+    def add_fort_set(self, cells):
+        self.dirtype = 'fort'
+        self.types.append('fortified-settlement')
+        cmap = CMAP_TMPL_UN_4
+        self.addinator(cells, cmap)
+        
-    def add_numbered(self, cells):
-        self.dirtype = 'numbered'
@@ -384 +469 @@
-        self.addinator(cells, cmap)
-        
+    def add_numbered_fort_group(self, cells):
+        self.dirtype = 'numbered'
+        self.types.append('fort-group')
+        cmap = {'number':0, 'namestring':0, 'grid':1, 'locdesc':2, 'periods':3, 'references':4} 
+        self.addinator(cells, cmap)
+        
+    def add_lettered_site(self, cells):
+        self.dirtype = 'lettered'
+        self.types.append('site')
+        cmap = {'letter':0, 'grid':1, 'placenames':2, 'namestring':2, 'periods':3, 'locdesc':4, 'references':5}
+        self.addinator(cells, cmap)
+        
+    def add_lettered_places(self, cells):
+        self.dirtype = 'lettered'
+        self.types.append('place')
+        cmap = {'letter':0, 'grid':1, 'placenames':2, 'namestring':2, 'periods':3, 'locdesc':4, 'references':5}
+        self.addinator(cells, cmap)
+        
+    def add_lettered_water(self, cells):
+        self.dirtype = 'lettered'
+        self.types.append('water')
+        cmap = {'letter':0, 'grid':1, 'placenames':2, 'namestring':2, 'periods':3, 'locdesc':4, 'references':5}
+        self.addinator(cells, cmap)
+
-    def add_road(self, cells):
-        self.dirtype = 'road'
@@ -389 +498 @@
-        self.addinator(cells,cmap)
-        
+    def add_road_station(self, cells):
+        self.dirtype = 'road-station'
+        cmap = CMAP_TMPL_UN_4
+        self.addinator(cells, cmap)
+        
+    def add_tumulus(self, cells):
+        self.dirtype = 'tumulus'
+        cmap = CMAP_TMPL_UN_4
+        self.addinator(cells, cmap)
+        
+    def add_spring(self, cells):
+        self.dirtype = 'spring'
+        cmap = CMAP_TMPL_UN_4
+        self.addinator(cells, cmap)
+        
-    def add_unlocated(self, cells):
-        self.dirtype = 'unlocated'
@@ -401 +525 @@
-    def add_wall(self, cells):
-        self.dirtype = 'wall'
+        cmap = CMAP_TMPL_UN_4
+        self.addinator(cells, cmap)
+        
+    def add_well(self, cells):
+        self.dirtype = 'well'
-        cmap = CMAP_TMPL_UN_4
-        self.addinator(cells, cmap)
@@ -449 +578 @@
-        for i, nameish in enumerate(nameishes):
-            # test to see if the nameish follows a "diamond" symbol (i.e., it is a minor alternative name)
+            diamond_regex = re.compile(u".+\u00A7.*%s([\s\/].*|$)" % CLEAN_NAMEISH_REGEX.sub('', nameish.strip()))
+            diamond_m = diamond_regex.search(celltext)
+            
+            # test for italic tag (and deal with MSWord formatting goofiness along the way)
-            try:
-diamond_regex = re.compile(u".+\u00A7.*%s" % nameish.strip().replace('*', ''
-sre_constants.e
-("error attempting to create a diamond_regex for nameish = '%s'; \n\terror detail
+nameishmatches = namecell.xpath('descendant-or-self::*[contains(normalize-space(text()),"%s")]' % nameish.strip(
+etree.XPathSyntaxE
+ ("incorporating nameish (%s) into an xpath expression caused a fatal error
-                raise
-            diamond_m = diamond_regex.match(celltext)
-            
-            # test for italic tag
-            nameishmatches = namecell.xpath("descendant-or-self::*[contains(normalize-space(text()),'%s')]" % nameish.strip())
-            try:
-                deepestnameishmatch = nameishmatches[-1]
-            except IndexError, detail:
-                cellxml = etree.tostring(namecell)
-#logging.warning("error attempting to use xpath to find '%s' in xml\n:%s\nattempting to trap for serial italic tags" % (nameish.strip(), cellxml
+m_postfix_numeral = POSTFIX_NUMERAL_REGEX.match(nameish.strip(
-                if '</i><i>' in cellxml:
-logging.info('found </i></i> in cell xml, attempting to remove and retry')
+ serial italic tags (pointless msword artifact); replace them and try again
-                    cellxml = cellxml.replace('</i><i>', '')
-                    celltree = etree.XML(cellxml)
-                    #logging.info("result of replacement attempt: '%s'" % etree.tostring(celltree))
-                    nameishmatches = celltree.xpath("descendant-or-self::*[contains(normalize-space(text()), '%s')]" % nameish.strip())
-                    if len(nameishmatches) > 0:
-                        logging.info("adjusted for serial italic tags in a name field for %s" % nameish.strip())
-                    deepestnameishmatch = nameishmatches[-1]
+                elif m_postfix_numeral:
+                    # there's a postfix numeral that may be causing trouble; try again without it
+                    nameishmatches = namecell.xpath('descendant-or-self::*[contains(normalize-space(text()), "%s")]' % m_postfix_numeral.groups(1)[0].strip())
+                    if len(nameishmatches) > 0:
+                        logging.info("adjusted for postfix numerial in a name field for %s" % nameish.strip())
+                    else:
+                        logging.critical("tried to adjust for a postfix numeral in a name field for %s but it didn't work\n\tcelltext: '%s'\n\tcelltxml:\n%s\n\tregex group: '%s'" % (nameish.strip(), celltext, cellxml, m_postfix_numeral.groups(1)[0].strip()))
+                    deepestnameishmatch = nameishmatches[-1]
-                else:
-serial italics do not seem to be the problem; error detail: %s" % detail
+index error: serial italics do not seem to be the problem\n\terror detail: %s\n\tnameish: %s\n\tcelltext: %s\n\tcelltxml:\n%s" % (detail, nameish.strip(), celltext, cellxml)
-                    raise
-