Changeset 1427 for BADataMunger

Timestamp:

09/12/08 15:45:31 (2 months ago)

Author:

thomase

Message:

sticking a fork in the aspect of BADataMunger that is the creation of batlas ids, or so I hope

Files:

• BADataMunger/trunk (modified) (1 prop)
• BADataMunger/trunk/batlaspipe.py (modified) (3 diffs)
• BADataMunger/trunk/bidmaker.py (modified) (1 diff)
• BADataMunger/trunk/config/BATL001_config.xml (added)
• BADataMunger/trunk/config/BATL001aconfig.xml (added)
• BADataMunger/trunk/config/BATL002_config.xml (added)
• BADataMunger/trunk/config/BATL003_config.xml (added)
• BADataMunger/trunk/config/BATL004_config.xml (added)
• BADataMunger/trunk/config/BATL005_config.xml (added)
• BADataMunger/trunk/config/BATL006_config.xml (added)
• BADataMunger/trunk/config/BATL064_config.xml (added)
• BADataMunger/trunk/config/BATL089_config.xml (added)
• BADataMunger/trunk/config/BATL090_config.xml (added)
• BADataMunger/trunk/config/BATL091_config.xml (added)
• BADataMunger/trunk/config/BATL092_config.xml (added)
• BADataMunger/trunk/config/BATL093_config.xml (added)
• BADataMunger/trunk/config/BATL094_config.xml (added)
• BADataMunger/trunk/config/BATL095_config.xml (added)
• BADataMunger/trunk/config/BATL096_config.xml (added)
• BADataMunger/trunk/config/BATL097_config.xml (added)
• BADataMunger/trunk/config/BATL098_config.xml (added)
• BADataMunger/trunk/config/BATL099_config.xml (added)
• BADataMunger/trunk/config/BATL100_config.xml (added)
• BADataMunger/trunk/config/BATL101_config.xml (added)
• BADataMunger/trunk/config/BATL102_config.xml (added)
• BADataMunger/trunk/etc/wordhtml (modified) (1 prop)
• BADataMunger/trunk/etc/wordhtml/BATL001A.htm (added)
• BADataMunger/trunk/etc/wordhtml/BATL001_.htm (added)
• BADataMunger/trunk/etc/wordhtml/BATL002_.htm (added)
• BADataMunger/trunk/etc/wordhtml/BATL003_.htm (added)
• BADataMunger/trunk/etc/wordhtml/BATL004_.htm (added)
• BADataMunger/trunk/etc/wordhtml/BATL005_.htm (added)
• BADataMunger/trunk/etc/wordhtml/BATL006_.htm (added)
• BADataMunger/trunk/etc/wordhtml/BATL064_.htm (added)
• BADataMunger/trunk/etc/wordhtml/BATL089_.htm (added)
• BADataMunger/trunk/etc/wordhtml/BATL090_.htm (added)
• BADataMunger/trunk/etc/wordhtml/BATL091_.htm (added)
• BADataMunger/trunk/etc/wordhtml/BATL092_.htm (added)
• BADataMunger/trunk/etc/wordhtml/BATL093_.htm (added)
• BADataMunger/trunk/etc/wordhtml/BATL094_.htm (added)
• BADataMunger/trunk/etc/wordhtml/BATL095_.htm (added)
• BADataMunger/trunk/etc/wordhtml/BATL096_.htm (added)
• BADataMunger/trunk/etc/wordhtml/BATL097_.htm (added)
• BADataMunger/trunk/etc/wordhtml/BATL098_.htm (added)
• BADataMunger/trunk/etc/wordhtml/BATL099_.htm (added)
• BADataMunger/trunk/etc/wordhtml/BATL100-orig.htm (added)
• BADataMunger/trunk/etc/wordhtml/BATL100-tidied.htm (added)
• BADataMunger/trunk/etc/wordhtml/BATL100_.htm (added)
• BADataMunger/trunk/etc/wordhtml/BATL101-orig.htm (added)
• BADataMunger/trunk/etc/wordhtml/BATL101-tidied.htm (added)
• BADataMunger/trunk/etc/wordhtml/BATL101_.htm (added)
• BADataMunger/trunk/etc/wordhtml/BATL102-orig.htm (added)
• BADataMunger/trunk/etc/wordhtml/BATL102-tidied-2_.htm (added)
• BADataMunger/trunk/etc/wordhtml/BATL102-tidied_.htm (added)
• BADataMunger/trunk/etc/wordhtml/BATL102_.htm (added)
• BADataMunger/trunk/tableparser.py (modified) (16 diffs)
• BADataMunger/trunk/wordnormalizer.py (modified) (1 diff)
• BADataMunger/trunk/wordstripper.xsl (modified) (2 diffs)

BADataMunger/trunk/batlaspipe.py

@@ -178 +178 @@
-                        if len(bid) > 0:
-                            place.batlasids.append(bid)
-
+
+
+
+
-                        bid = maker.makeID(altlabel, mapnum, grid, phraseprefix=label)
-                        bid = self.vetid(bid, priorids)
@@ -195 +198 @@
-                   
-                    if len(place.placenames) == 0:
-
+ and len(place.placenames) == 0
-                            names = place.locdesc.split('/')
-                            prefix = label
@@ -292 +295 @@
-                        citname = SLASH_REGEX.sub('/',p.namestring.strip())
-                        citcontent = "BAtlas %s false name %s" % (self.map_number, citname)
-
+ and len(p.placenames) == 0
-                        citname = SLASH_REGEX.sub('/',p.locdesc.strip())
+                        citcontent = "BAtlas %s %s no. %s (%s)" % (self.map_number, p.grid, p.namestring, citname)
+                    elif p.dirtype == 'numbered' and len(p.placenames) > 0:
+                        citname = SLASH_REGEX.sub('/',p.placenames[0].name.strip())
-                        citcontent = "BAtlas %s %s no. %s (%s)" % (self.map_number, p.grid, p.namestring, citname)
-                    elif p.dirtype == 'name' and len(p.namestring) == 0:

BADataMunger/trunk/bidmaker.py

@@ -66 +66 @@
-        
-            
-ole', 'monastery', 'monument', 'monument-group', 'pass', 'quarry', 'quarry-group
+ine-group', 'mole', 'monastery', 'monument', 'monument-group', 'pass', 'quarry', 'quarry-group', 'reservoir
-            # second most common case: more-or-less normal features without names
-            phrase = "%s %s %s" % (dirtype, phrase, locdesc)

BADataMunger/trunk/tableparser.py

@@ -26 +26 @@
-SERIAL_ITALICS_SPACES_REGEX = re.compile('</i>(\s+)<i>')
-SERIAL_ITALICS_COMMA_REGEX = re.compile('</i>(,\s+)<i>')
+EMPTY_ITALICS_REGEX = re.compile('<i> </i>')
-
-tabletypes = {
@@ -73 +74 @@
-    "Mines / Quarries":"quarry",
-    "Mines":"mine",
+    "Mines GLPMR":"mine_glpmr",
-    "Mine GLPMR":"mine_glpmr",
-    "Mines (Named)":"mine_glpmr_named",
@@ -83 +85 @@
-    "Names":"name",
-    "Names found only in Avienus":"names_avienus",
+    "Named Dioceses":"named_diocese",
+    "Named Provinces":"named_province",
+    "Named Provinces With Loc":"named_province_loc",
+    "Numbered Provinces":"numbered_province",
+    "Numbered Provinces With Loc":"numbered_province_loc",
-    "Numbered Bridges":"numbered_bridge",
-    "Numbered Features":"numbered",
@@ -91 +98 @@
-    "Numbered Villas":"numbered_villas",
-    "Pass":"pass",
+    "Pass (dated)":"pass_dated",
-    "Passes (dated)":"pass_dated",
-    "Quarry":"quarry",
-    "Quarries":"quarry",
-    "Quarry (dateless)":"quarry_dateless",
+    "Reservoir":"reservoir",
-    "Roads":"road",
-    "Road":"road",    
@@ -107 +116 @@
-    "Unlocated Toponyms":"unlocated",
-    "Unlocated Toponym":"unlocated",
+    "Unlocated Toponyms (dateless)":"unlocated_dateless",
+    "Unlocated Toponym (dateless)":"unlocated_dateless",
-    "Unlocated Coin Names":"unlocated_coin",
+    "Unlocated Provinces":"unlocated_province",
-    "Unnamed Sites":"unnamed_site",
-    "Villas":"villa",
@@ -113 +125 @@
-    "Walls":"wall",
-    "Wall":"wall",
+    "Walls (named)":"wall",
-    "Wall (Named No Loc)" : "wall_named_noloc",
-    "Wall (Unnamed)" : "wall_unnamed",
+    "Wall (unnamed)" : "wall_unnamed",
-    "Walls (Unnamed)" : "wall_unnamed",
+    "Walls (unnamed)" : "wall_unnamed",
-    "Walls (Dateless)":"wall_dateless",
-    "Walls / Fortifications":"wall",
@@ -165 +180 @@
-            # ignore empty rows
-            rows_blank+=1
- = 
+=
-            # ignore internal cross references
-            rows_xref+=1
@@ -176 +191 @@
-            p.rowi = ri
-            rowcells = row.xpath("*[local-name()='td']")
+            #rowlen = 0
+            #rowlens = [len(''.join(cell.text.split())) for cell in rowcells]
+            #for l in rowlens:
+             #   rowlen = rowlen + l
+            #if rowlen < 5:
+             #   logging.critical("There's no content in here!")
-            configs = pparser_config = context['config'].xpath('//%s' % tabletype)
-            if len(configs) == 1:
@@ -614 +635 @@
-    
-    def add_name(self, cells):
+        logging.debug("adding names for:\n\t%s\n\t%s\n\t%s\n\t%s\n\t%s" % tuple([etree.tostring(cell).replace('\n', ' ') for cell in cells]))
-        self.dirtype = 'name'
-        cmap = CMAP_TMPL_NAMED_5        
@@ -700 +722 @@
-        self.addinator(cells, cmap)
-
+    def add_reservoir(self, cells):
+        self.dirtype = 'reservoir'
+        cmap = CMAP_TMPL_UN_4
+        self.addinator(cells, cmap)
+        
-    def add_road(self, cells):
-        self.dirtype = 'road'
@@ -745 +772 @@
-        self.addinator(cells, cmap)
-        
+    def add_unlocated_dateless(self, cells):
+        self.dirtype = 'unlocated'
+        cmap = {'placenames':0, 'namestring':0, 'locdesc':1, 'references':2}
+        self.addinator(cells, cmap)
+        
-    def add_unlocated_coin(self, cells):
-        self.dirtype = 'unlocated_coin'
@@ -750 +782 @@
-        self.addinator(cells, cmap)
-        
+    def add_unlocated_province(self, cells):
+        self.dirtype = 'unlocated'
+        self.types.append('province')
+        cmap = {'placenames':0, 'namestring':0, 'locdesc':1}
+        self.addinator(cells, cmap)
+
-    def add_unnamed_site(self, cells):
-        self.dirtype = 'feature'
@@ -758 +796 @@
-        self.dirtype = 'unlocated'
-        cmap = {'placenames':0, 'namestring':0, 'avienus':1, 'locdesc':2, 'references':3}
+        self.addinator(cells, cmap)
+        
+    def add_named_diocese(self, cells):
+        self.dirtype = 'name'
+        self.types.append('diocese')
+        cmap = {'grid':0, 'namestring':1, 'placenames':1}
+        self.addinator(cells, cmap)
+        
+    def add_named_province(self, cells):
+        self.dirtype = 'name'
+        self.types.append('province')
+        cmap = {'grid':0, 'placenames':1, 'namestring':1}
+        self.addinator(cells, cmap)
+        
+    def add_named_province_loc(self, cells):
+        self.dirtype = 'name'
+        self.types.append('province')
+        cmap = {'grid':0, 'placenames':1, 'namestring':1, 'locdesc':2}
+        self.addinator(cells, cmap)
+        
+    def add_numbered_province_loc(self, cells):
+        self.dirtype = 'numbered'
+        self.types.append('province')
+        cmap = {'number':0, 'grid':1, 'namestring':0, 'placenames':2, 'locdesc':3}
+        self.addinator(cells, cmap)
+        
+    def add_numbered_province(self, cells):
+        self.dirtype = 'numbered'
+        self.types.append('province')
+        cmap = {'number':0, 'grid':1, 'namestring':0, 'placenames':2}
-        self.addinator(cells, cmap)
-        
@@ -846 +914 @@
-            # test to see if the nameish follows a "diamond" symbol (i.e., it is a minor alternative name)
-            try:
-
+
+
+
-                diamond_regex = re.compile(diamond_expr)
-            except sre_constants.error, detail:
@@ -852 +922 @@
-                raise
-            diamond_m = diamond_regex.search(celltext)
+            if diamond_m:
+                logging.debug("nameish (%s) follows a diamond: %s" % (nameish.strip(), diamond_m.group()))
-                
-            # test for italic tag (and deal with MSWord formatting goofiness along the way)
+            thexpath = 'descendant-or-self::*[contains(normalize-space(text()),"%s")]' % nameish.strip()
-            try:
-'descendant-or-self::*[contains(normalize-space(text()),"%s")]' % nameish.strip()
+thexpath
-            except etree.XPathSyntaxError, detail:
-                logging.critical ("incorporating nameish (%s) into an xpath expression caused a fatal error: %s" % (nameish.strip(), detail))
-                raise
-
+
+
+
+
+
-            try:
-                deepestnameishmatch = nameishmatches[-1]
@@ -866 +943 @@
-                m_postfix_numeral = POSTFIX_NUMERAL_REGEX.match(nameish.strip())
-                m_postfix_parenthetical = POSTFIX_PARENTHETICAL_REGEX.match(nameish.strip())
-
+
+
+
-                newcellxml = SERIAL_ITALICS_SPACES_REGEX.sub(' ', newcellxml)
-                newcellxml = SERIAL_ITALICS_COMMA_REGEX.sub(', ', newcellxml)

BADataMunger/trunk/wordnormalizer.py

@@ -36 +36 @@
-    (' ',' '),     # non-breaking space
-    (' ',' '),     # non-breaking space bis
-
+
+
-]
-

BADataMunger/trunk/wordstripper.xsl

@@ -49 +49 @@
-
-    <!-- suppress all style and br elements -->
-
+
+
+
-    
-    <!-- strip span tags, but pass through their sub nodes, note that line breaks may matter :| -->
@@ -58 +60 @@
-        </xsl:choose>
-    </xsl:template>
+    
+    <xsl:template match="*[local-name()='i' and normalize-space(text()) = ' ']"/>
-    
-    <xsl:template match="*"><xsl:call-template name="elepassthrough"/></xsl:template>