Changeset 1332

Timestamp:

07/11/08 15:52:30 (3 months ago)

Author:

thomase

Message:

this thing actually makes ids properly for all feature types (except roads) in maps 22, 38 and 65 and correctly saves them, plus alot of useful associated information to an xml file

Files:

• BADataMunger/trunk/batlaspipe.py (modified) (10 diffs)
• BADataMunger/trunk/bidmaker.py (modified) (5 diffs)
• BADataMunger/trunk/pipebiblio.py (modified) (2 diffs)

BADataMunger/trunk/batlaspipe.py

@@ -5 +5 @@
-import unicodedata
-
+import datetime as dt
+
-import lxml.etree as etree
-
@@ -11 +13 @@
-import tablegroker
-import tableparser
-import placesaver
-from etreehelps import getalltext
-from texthelps import normalizetext
@@ -17 +18 @@
-from bidmaker import BAtlasIDMaker, DIAMOND_STOP_REGEX
-
-
-
-
+
+
+
+
-
-class Pipe:
-
-
+
-        
-    Use like:
-    
-pipegeo
-pipegeo.Pipe(r'/TomDocs/awmcwork/pleiadesact/svnbox/BADataMunger/config/BATL065_config.xml', r'/badigit/wordhtml/BATL065_.htm', r'/badigit/scratchem
+batlaspipe as bp
+bp.Pipe(r'./config/BATL038_config.xml', r'/Users/tom/Sandbox/pleiades/wordhtml/BATL038_.htm', r'/Users/tom/Sandbox/scratch/batlasid38
-p.cycle()
-
+
+
-            
-    """
@@ -39 +41 @@
-        file
-        """
-DEBUG
+INFO
-        logging.info("INITIALIZING: %s" % self.__class__)
-
@@ -51 +53 @@
-        
-        if not isfile(self['configfile']):
-raise Error, "No file found at path: %s" % self['configfile']
+logging.critical( "No file found at path: %s" % self['configfile'])
-        elif not isfile(self['dirfile']):
-raise Error, "No file found at path: %s" % self['dirfile']
+logging.critical ("No file found at path: %s" % self['dirfile'])
-        elif not isdir(self['destdir']):
-raise Error, "No directory found at path: %s" % self['destdir']
+logging.critical( "No directory found at path: %s" % self['destdir'])
-        else:
-            
@@ -118 +120 @@
-    def idit(self):
-        """ Construct all appropriate id strings (primary and alternate) for each place
-        
-         the very first id should be the canonical id, composed from the entire label,
-         and then we support various proxies based on the individual slash-delimited
-         names within the label 
-         Note that we have to massage some minor directory convention differences
-         from label conventions:
-         - drop last part of label from any "diamond" (i.e., unicode section mark)
-            when making the primary id as these don't appear in the label (but do
-            treat them as proxy ids
-         - drop ", T." as this is included in dirs, but omitted in labels
-         - propagate final " fl." onto each prior name in a single label
-         - haven't decided what to do about numbered items and unnamed features
-         - unlocated features do get ids; substitute "unlocated" for gridsquare value
-         define and compile static regexes
-         """
-        
@@ -148 +136 @@
-                grid = place.grid
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-                else:
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
-                
-    def outit(self):
-
+
+
+
+
-        places = self['places']
-        d = etree.Element('featurelist')
-        d.attrib['mapnum'] = self.map_number
+        q = etree.SubElement(d, 'uribase')
+        q.text = "%s/" % NAMESPACE
-        for p in places:
-            if len(p.batlasids) > 0:
-
+
+
+
-                for i, bid in enumerate(p.batlasids):
-                    if i == 0 or bid != p.batlasids[0]:
-
-
-
-
-
+
+
+
+
+
+
+
-                if p.dirtype == 'name':
-                    ptype = 'labeled feature'
+                elif p.dirtype == 'numbered':
+                    ptype = 'numbered feature'
-                elif p.dirtype == 'unlocated':
-                    ptype = 'unlocated toponym'
@@ -199 +217 @@
-                q = etree.SubElement(e, 'type')
-                q.text = ptype
+                
+                # additional feature types
-                for i, t in enumerate(p.types):
-                    if i == 0:
@@ -205 +225 @@
-                        q = etree.SubElement(e, 'subtype')
-                    q.text = t
+                    
+                # gridsquare
-                if len(p.grid) > 0:
-                    q = etree.SubElement(e, 'gridsquare')
-                    q.text = p.grid
+                    
+                # labels actually appearing on the map
-                if len(p.namestring) > 0:
-
-
-
+
-                        q = etree.SubElement(e, 'label', context='map')
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-                        if p.dirtype == 'unlocated':
-n.name
+txt
-                        elif p.dirtype == 'false':
-n.name
+txt
-                        elif len(p.placenames) > 1:
-n.name
+txt
-                        if len(txt) > 0:
-                            q = etree.SubElement(e, 'citation')
-                            q.text = txt
+                            
+                # location description
-                if len(p.locdesc) > 0:
-                    q = etree.SubElement(e, 'location')
-
-
-
-
+
+
+
-                if len(p.itinraw) > 0:
-                    q = etree.SubElement(e, 'itinerary')
@@ -246 +311 @@
-                    q = etree.SubElement(e, 'citation')
-                    q.text = "BATlas %s %s (%s)" % (self.map_number, p.dirtype, p.itinraw.strip())
-
+
+
+
+
+
+
+
+
+
+
+
-        f = open(fpath, 'w')
+        f.write(XMLDECL)
+        cmnt = cmnt % (fn, self.map_number, dtime.isoformat(), PREPARER, dtime.year, self.map_number, fn)
+        f.write(cmnt)
-        etree.ElementTree(d).write(f)
-        f.close()
+        logging.info("wrote output result xml file on %s" % fpath)
+        

BADataMunger/trunk/bidmaker.py

@@ -12 +12 @@
-
-class BAtlasIDMaker:
+    """A class for making URIs for Barrington Atlas features"""
-    
-
+, phrasepostfix='', phraseprefix=''
-        """Make an ID from a properly prepared string."""
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-        return ident
+        
-        
-    def buildLabel (self, origstring, placenames, dirtype, locdesc = u'', itin = u''):
@@ -31 +46 @@
-            return u''
-            
-
+
+
+
+
+
+
+
+
+
-            # the most basic case: features with names:
-            #     use the batlas directory namestring
@@ -38 +61 @@
-            phrase = DIAMOND_STOP_REGEX.sub('', phrase)
-            phrase = SILENT_POSTFIX_REGEX.sub('', phrase)
+            
+        
-            
-        elif dirtype in ['aqueduct', 'bridge', 'wall']:
@@ -47 +72 @@
-                phrase = ITIN_IGNORE_REGEX.sub(u'', phrase)
-                
+       
+            
+            
-        # there are various missing classes
+        else:
+            logging.warning("bidmaker.buildLabel did nothing for origstring='%s', dirtype='%s', locdesc='%s'" % (origstring, dirtype, locdesc))
-            
-        if dirtype == 'false':
@@ -67 +97 @@
-            return phrase.strip()
-        
-     
-    def addSmartSuffix (self, name, placetypes):
-        """Sweet blackberry surprise."""

BADataMunger/trunk/pipebiblio.py

@@ -1 @@
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-import os
-import logging
+import sys
+import getopt
-
-import lxml.etree as etree
@@ -13 +53 @@
-
-class Pipe:
+    """ A processing pipeline for bibliographic extraction and munging."""
-    
-, dirfile, biblibfile, bibdestdir
+
-        logging.basicConfig(level=logging.DEBUG, format='%(levelname)-8s %(message)s')
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
-        f.close()
-        wordxml = wordhtml2xml.convert(wordhtml)
-        data['contextpath'] = os.getcwd()
-        cleanxml = wordstripper.strip(data['contextpath'], wordxml)
-        data['bibliography'] = biblioextractor.extract(cleanxml)
-        drive, dirpath = splitdrive(dirfile)
-        filepath, filename = split(dirpath)
-
-
-
-
+
-        
+    def fixdir(self):
+        self.data['wordxml'] = wordhtml2xml.convert(self.data['wordhtml'])
+        self.data['cleanxml'] = wordstripper.strip(self.data['contextpath'], self.data['wordxml'])
+        
+    def extractbiblio(self):
+        self.data['bibliography'] = biblioextractor.extract(self.data['cleanxml'])
+        
+    def savebiblio(self,biblibfile,bibdestdir):
+        self.data['bibdestdir'] = bibdestdir
+        self.data['biblibfile'] = biblibfile
+        bibliosaver.save_biblio_mods(self.data)
+        
+def main(argv):
+    
+    dirfile = None
+    bibdestdir = None
+    biblibfile = None
+    
+    try:                                
+        opts, args = getopt.getopt(argv, "s:d:l:h", ["directory=", "destination=", "library=", "help"])
+    except getopt.GetoptError:
+        print __doc__
+        sys.exit(2)                     
+
+    for opt, arg in opts:
+        if opt in ("-s", "--directory"):
+            dirfile = abspath(normcase(arg))
+        elif opt in ("-d", "--destination"):
+            bibdestdir = abspath(normcase(arg))
+        elif opt in ("-l", "--library"):
+            biblibfile = abspath(normcase(arg))
+        elif opt in ("-h", "--help"):
+            print __doc__
+            sys.exit(0)
+            
+    if not dirfile:
+        print 'a source directory file must be specified with the -s (--directory) option'
+        sys.exit(0)
+    if not bibdestdir:
+        print 'a destination directory must be specified with the -d (--destination) option'
+        sys.exit(0)
+    if not biblibfile:
+        print 'a bibliographic library file must be specified with the -l (--library) option'
+        sys.exit(0)
+     
+    if not isfile(dirfile):
+        print "the specified source directory file (%s) is not an existing file" % dirfile
+        sys.exit(0)
+    if not isdir(bibdestdir):
+        print "the specified destination directory (%s) is not an existing directory" % bibdestdir
+        sys.exit(0)
+    if not isfile(biblibfile):
+        print "the specified bibliographic library file (%s) is not an existing file" % biblibfile
+        sys.exit(0)
+         
+    p = Pipe()
+    p.cycle(dirfile, biblibfile, bibdestdir)
+            
+if __name__ == "__main__":
+    main(sys.argv[1:])
+