Changeset 845

Timestamp:

06/18/07 15:11:41 (2 years ago)

Author:

thomase

Message:

Fixed logic bug that prevented unlocated places from getting written to frankenfiles; incorporated uniform and more informative info and warning logging, using the python standard logging module.

Files:

• BADataMunger/trunk/biblioextractor.py (modified) (4 diffs)
• BADataMunger/trunk/gismixer.py (modified) (5 diffs)
• BADataMunger/trunk/modsmixer.py (modified) (13 diffs)
• BADataMunger/trunk/pipeline.py (modified) (8 diffs)
• BADataMunger/trunk/placesaver.py (modified) (3 diffs)
• BADataMunger/trunk/tableparser.py (modified) (4 diffs)
• BADataMunger/trunk/wordhtml2xml.py (modified) (3 diffs)
• BADataMunger/trunk/wordstripper.py (modified) (2 diffs)

BADataMunger/trunk/biblioextractor.py

@@ -1 +1 @@
-import re
+import logging
-
-import lxml.etree as etree
@@ -10 +11 @@
-    """Extract bibliography from cleaned up wordxml."""
-    
+    logging.info("BEGIN extracting bibliography from cleaned-up word html")
+    
-    # determine which div contains the bibliography
-    biblist = []
@@ -19 +22 @@
-        if text == 'Bibliography':
-            bibdiv = divs[i+1]  # yes, really, the next div
+            logging.info("bibliography div is div %s" % (i+1))
-        elif text.find('Abbreviation') > 0:
-            dirlistdiv = divs[i]
+            logging.info("abbreviation div is div %s" % i)
-           
-        
-    # parse the bibliography div
-    paras = bibdiv.xpath("*[local-name()='p']")
-
-
-
+
-    for p in paras:
-        text = p.text.replace(u'\n', u' ').strip()
@@ -48 +51 @@
-        table = dirlistdiv.xpath("descendant::*[local-name()='p' and contains(., 'Abbreviation')]")[0]
-        rows = table.xpath("following-sibling::*[local-name()='table'][1]/*[local-name()='tr']")
+        logging.info("the abbreviations table has %s rows" % len(rows))
-        for row in rows:
-            shorttitle = getalltext(row.xpath("*[local-name()='td']")[0]).replace('\n', ' ').strip()
-            citation = row.xpath("*[local-name()='td'][2]/*[local-name()='p']")[0]
-
+#
-            atitle = extract_title(citation)
-            biblist.append((shorttitle, atitle, citation))
-        
+    logging.info("DONE extracting bibliography from cleaned-up word html")
+    
-    return biblist
-    

BADataMunger/trunk/gismixer.py

@@ +1 @@
+import logging
+
-import lxml.etree as etree
-
@@ -34 +36 @@
-            self.fieldvalues.append([normalizetext(getalltext(e_value)) for e_value in e_values])
-            
-print "\ngisMixer got %s records from %s" % (len(self.fieldvalues), source
-print "field names are:"
+logging.info ("gisMixer got %s records from %s" % (len(self.fieldvalues), source)
+logging.debug("FIELD NAMES ARE:")
-        for name in self.fieldnames:
-print "%s" % name 
-print "first record content:"
+logging.debug("%s" % name )
+logging.debug("FIRST RECORD CONTENT:")
-        for item in self.fieldvalues[0]:
-print "%s" % item  
+logging.debug("%s" % item)
-            
-    def mixin(self, config, place):
@@ -77 +79 @@
-            
-        else:
-            print "mixin failed to match %s: '%s' (%s - %s), rowi=%s" % (place.type, label.encode('ascii', 'backslashreplace'), place.locdesc.encode('ascii', 'backslashreplace'), place.itinraw.encode('ascii', 'backslashreplace'), place.rowi)
-            try:
-print "\tdisambiguator=%s" % disambiguator
+logging.warning( "failed to match %s: '%s' (%s - %s), rowi=%s, disambiguator: %s" % (place.type, label.encode('ascii', 'backslashreplace'), place.locdesc.encode('ascii', 'backslashreplace'), place.itinraw.encode('ascii', 'backslashreplace'), place.rowi, disambiguator))
-            except:
-pass
+logging.warning( "failed to match %s: '%s' (%s - %s), rowi=%s" % (place.type, label.encode('ascii', 'backslashreplace'), place.locdesc.encode('ascii', 'backslashreplace'), place.itinraw.encode('ascii', 'backslashreplace'), place.rowi))
-        
-            
@@ -91 +92 @@
-            gridmatches = [value for value in self.fieldvalues if value[self.gridi] == gridsquare]
-        except ValueError:
-print "gisMixer could not find any data for gridsquare = %s (seeking %s)" % (gridsquare, label
+logging.warning( "gisMixer could not find any data for gridsquare = %s (seeking %s)" % (gridsquare, label)
-        if len(gridmatches)>0:
-            try: 
-                labelmatches = [value for value in gridmatches if value[self.labeli] == label]
-            except ValueError:
-print "gisMixer could not find any data for label = '%s' in grid '%s'" % (label, gridsquare
+logging.warning("gisMixer could not find any data for label = '%s' in grid '%s'" % (label, gridsquare)
-            if len(labelmatches) > 1 and disambiguator != None:
-                try:
-                    matches = [value for value in labelmatches if value[self.disambiguatori] == disambiguator]
-                except ValueError:
-print "gisMaker could not find any data for label = '%s' and disambiguator = '%s' in grid '%s'" % (label, gridsquare, disambiguator
+logging.warning("gisMaker could not find any data for label = '%s' and disambiguator = '%s' in grid '%s'" % (label, gridsquare, disambiguator)
-            else:
-                matches = labelmatches
@@ -108 +109 @@
-        elif len(matches) > 1:
-            if disambiguator != None:
-print "gisMaker found %s apparent matches for label = '%s' and disambiguator = '%s' in grid '%s'" % (label, gridsquare, disambiguator
+logging.warning("gisMaker found %s apparent matches for label = '%s' and disambiguator = '%s' in grid '%s'" % (label, gridsquare, disambiguator)
-            else:
-print "gisMaker found %s apparent matches for label = '%s' in grid '%s'" % (len(matches), label, gridsquare
+logging.warning("gisMaker found %s apparent matches for label = '%s' in grid '%s'" % (len(matches), label, gridsquare)
-            return None
-        else:

BADataMunger/trunk/modsmixer.py

@@ -1 +1 @@
-import re
+import logging
-
-import lxml.etree as etree
@@ -29 +30 @@
-    def __init__(self, student, library, destination):
-        
+        
+        logging.info("INITIALIZING: %s" % self.__class__)
+        
+        # read basic mods information from the "student" file: this is the stuff we want to enhance
-        f = open(student)
-        content = f.read()
@@ -34 +39 @@
-        f.close()
-        self.student = etree.XML(content)
+        logging.info("read mods information to enhance (aka 'student file') from: %s" % student)
+        logging.info("there are %s mods records in the 'student file'" % len(self.student.xpath(u"//mods:mods", NSD)))
+        
+        # read a library of mods information from the "library" file: this is where we will pull the enhancements from
-        f = open(library)
-        content = f.read()
@@ -39 +48 @@
-        content = content.replace('<modsCollection>', '<modsCollection xmlns="http://www.loc.gov/mods/v3" xmlns:xlink="http://www.w3.org/1999/xlink">')
-        self.library = etree.XML(content)
+        logging.info("read mods enhancement information (aka 'library file') from: %s" % library)
+        logging.info("there are %s mods records in the 'library file'" % len(self.library.xpath(u"//mods:mods", NSD)))
-        
-        snodes = self.student.xpath(u"//mods:mods", NSD)
@@ -48 +59 @@
-                s_title = u' '.join(s_title_node.text.replace('\n', ' ').strip().split())
-            except:
-print 'Something is wrong with the student title node for short_title = %s' % s_short_title
+logging.warning('Something is wrong with the student title node for short_title = %s' % s_short_title)
-            
-            lquery = u"//mods:mods/mods:titleInfo[@type='abbreviated']/mods:title[normalize-space(.) = '%s']/ancestor::mods:mods" % s_short_title
@@ -58 +69 @@
-                if len(lnodes) == 1:
-                    if len(lnodes[0].xpath(u"mods:titleInfo[@type='abbreviated']/mods:title[normalize-space(.) = '%s']/ancestor::mods:mods" % s_short_title, NSD)) != 1:
-print "found match in library on full title '%s', but short title did not match '%s'" % (s_title, s_short_title
+logging.warning("found match in library on full title '%s', but short title did not match '%s'" % (s_title, s_short_title)
-                        lnodes = self.library.xpath(lquery, NSD)
-                elif len(lnodes) > 1:
-print "found multiple matches on long title in library for '%s = %s" % (s_short_title.encode('latin', 'xmlcharrefreplace'), s_title.encode('latin', 'xmlcharrefreplace'
-
-print "found multiple matches in on short title library for '%s = %s" % (s_short_title.encode('latin', 'xmlcharrefreplace'), s_title.encode('latin', 'xmlcharrefreplace'
+logging.warning("found multiple matches on long title in library for '%s = %s" % (s_short_title.encode('latin', 'xmlcharrefreplace'), s_title.encode('latin', 'xmlcharrefreplace')
+
+logging.warning("found multiple matches in on short title library for '%s = %s" % (s_short_title.encode('latin', 'xmlcharrefreplace'), s_title.encode('latin', 'xmlcharrefreplace')
-            if len(lnodes) == 0:
-                squery = u"*[local-name()='abstract']/descendant::*[local-name()='i' and contains(normalize-space(.),'%s')]/ancestor::mods:mods" % s_title
-                if len(snode.xpath(squery)) == 0:
-print "    notice: unmatched article '%s'" % s_short_title.encode('latin', 'xmlcharrefreplace'
-
-print "WARNING: unmatched book '%s = %s" % (s_short_title.encode('latin', 'xmlcharrefreplace'), s_title.encode('latin', 'xmlcharrefreplace'
+logging.warning("unmatched article '%s'" % s_short_title.encode('latin', 'xmlcharrefreplace')
+
+logging.warning("unmatched book '%s = %s" % (s_short_title.encode('latin', 'xmlcharrefreplace'), s_title.encode('latin', 'xmlcharrefreplace')
-            if len(lnodes)==1:
-                # one match, so gather up all the data we will need
@@ -78 +89 @@
-                    l_title = u' '.join(l_title_node.text.replace('\n', ' ').strip().split())
-                except:
-print "no title node found in library match for %s = %s" % (s_short_title.encode('latin', 'backslashreplace'), s_title.encode('latin', 'backslashreplace'
+logging.warning("no title node found in library match for %s = %s" % (s_short_title.encode('latin', 'backslashreplace'), s_title.encode('latin', 'backslashreplace')
-                try:
-                    l_title_lang = l_title_node.xpath(u"../@*[local-name()='lang']")[0]
@@ -90 +101 @@
-                    snode.attrib['ID'] = lnode.attrib['ID']
-                else:
-print "no ID!!!!"
+logging.warning("no ID attribute on mods element %s of %s" % (len(lnode.xpath("previous::*[local-name()='mods']")), len(lnode.xpath("//*[local-name()='mods']"))))
-
-                # proper title type attributes
@@ -147 +158 @@
-                    s_title_node.xpath('..')[0].attrib['{http://www.w3.org/XML/1998/namespace}lang']=l_title_lang
-                else:
-print "No title language found for '%s'\n" % s_short_title.encode('latin1', 'xmlcharrefreplace'
+logging.warning("No title language found for '%s'\n" % s_short_title.encode('latin1', 'xmlcharrefreplace')
-                    
-                # copy over items verbatim from library record to student record
@@ -171 +182 @@
-        ricount = len(relatedItems)
-        richecked = 0
+        orphtot = 0
-        while orphans:
-            for relatedItem in relatedItems:
@@ -177 +189 @@
-                    lnodes = self.library.xpath("//mods:mods[@ID='%s']" % ri_id, NSD)
-                    if len(lnodes) != 1:
-print "failure seeking %s in library file; match count = %s" % (ri_id, len(lnodes
+logging.warning("failure seeking %s in library file; match count = %s" % (ri_id, len(lnodes)
-                        break
-                    else:
-print "adding %s" % ri_id
+orphtot += 1
-                        self.student.xpath("//mods:modsCollection", NSD)[0].append(lnodes[0])
-                richecked += 1
@@ -189 +201 @@
-            richecked = 0
-            
+        logging.info("added to 'student' an additional %s records from 'library' because they are related works" % orphtot)
+        
-        pcontent = etree.tostring(self.student).encode('utf-8')
-        pcontent = '<?xml version="1.0" encoding="UTF-8"?>\n' + pcontent
@@ -194 +208 @@
-        f.write(pcontent)
-        f.close()
+        
+        logging.info("saved enhanced records to %s" % destination)
+        
+        logging.info("DONE: %s\n" % self.__class__)
+        

BADataMunger/trunk/pipeline.py

@@ -1 +1 @@
-from os.path import normpath, normcase, isdir, isfile, splitdrive, splitext, split, join
-import os
+import logging
-
-import lxml.etree as etree
@@ -34 +35 @@
-        file
-        """
-
+
+
-
-        self.data = {}
-        self['contextpath'] = os.getcwd()
-print "contextpath: %s" % self['contextpath']
+logging.info("contextpath: %s" % self['contextpath'])
-        
-        self['configfile'] = normpath(normcase(configfile))
@@ -69 +71 @@
-                self['wordhtml'] = f.read()
-                f.close()
+                logging.info("read and stored directory file from %s" % self['dirfile'])
-                
-                # read and store the config file
@@ -74 +77 @@
-                config_text = f.read()
-                f.close()
+                logging.info("read and stored config file from %s" % self['configfile'])
-                
-                # get some essential info from the config file
@@ -81 +85 @@
-                except:
-                    self.map_number = 'XYZ'
+                logging.info("map number from config file: %s" % self.map_number)
+                
-                self.creators = []
-                self.contributors = []
@@ -94 +100 @@
-                
-                self.mixer = gismixer.gisMixer(self['gisfile'])
-
+
+
+
-            
-    def __getitem__(self, key): return self.data[key]
@@ -103 +111 @@
-        """Cycle through all steps in the pipeline"""
-        
+        logging.info("CYCLING: %s" % self.__class__)
+
-        self['wordxml'] = wordhtml2xml.convert(self['wordhtml'])
-        self['cleanxml'] = wordstripper.strip(self['contextpath'], self['wordxml'])
@@ -109 +119 @@
-        self['dirtables'] = tablegroker.grok(self['cleanxml'])
-        self['places'] = tableparser.parse(self, self['dirtables'], self.map_number)
+        logging.info("BEGIN attempting to match and combine %s directory places with %s map places" % (len(self['places']),len(self.mixer.fieldvalues)))
-        for place in self['places']:
-            if place.type == 'unlocated' or place.type == 'false':
-self
+place
-            else:
-                self.mixer.mixin(self['config'], place)
-
+
+
-        #placesaver.save_places_tei(self)
-        placesaver.save_places_frank(self)
-        
+        logging.info("DONE CYCLING: %s\n" % self.__class__)
+        
+            
+
-    def save(self, itemkey, encoding='utf-8'):
-        """Basic save-with-encoding function for writing content to an arbitrary 

BADataMunger/trunk/placesaver.py

@@ +1 @@
+import logging
-import re
-from os.path import join
@@ -33 +34 @@
-    """Save all the places using the Pleiades frankenformat."""
-    
+    
-    # iterate through the places list, creating a corresponding frankenfile for each place
-    places = self['places']
-
+
+
+
+
-        
-        
@@ -232 +237 @@
-        g.write(pcontent)
-        g.close()
+        
+    logging.info("DONE saving in frankenformat")
-    
-def save_places_tei(self):

BADataMunger/trunk/tableparser.py

@@ +1 @@
+import logging
-import operator
-import re
@@ -33 +34 @@
-    Return the info in a list."""
-    
+    logging.info("BEGIN attempt to parse the tables from the directory: tableparser.parse()")
-    places = []
-    for key in tabledict.keys():
@@ -42 +44 @@
-            places += parsetable(context, tabletype, tabledict[key], map_number)
-            
+    logging.info("DONE with tableparser.parse()")
-    return places
-    
-def parsetable(context, tabletype, table, map_number):
-    places = []
+    logging.info("BEGIN parsing a %s table: tableparser.parsetable()" % tabletype)
-    for ri, row in enumerate(table[1].xpath("descendant::*[local-name()='tr']")):
-        rowtext = normalizetext(getalltext(row))
@@ -85 +89 @@
-                    
-            places = places + [p]
-    
+logging.info("DONE with tableparser.parsetable(); found %s places of type %s" % (len(places), tabletype))
-    return places
-    

BADataMunger/trunk/wordhtml2xml.py

@@ +1 @@
+import logging
+
-from BeautifulSoup import BeautifulSoup
-import lxml.etree as etree
@@ -8 +10 @@
-def convert(source):
-    """Make html exported from Microsoft Word both well-formed and valid."""
+    
+    logging.info("BEGIN attempt to make MSWord-exported HTML both well-formed and valid: wordhtml2xml.convert()")
-    soup = BeautifulSoup(source)
-    html = soup.findAll('html')[0]
@@ -15 +19 @@
-    tree = etree.XML(XMLDOCTYPE + unicode(soup))
-    tree = wordnormalizer.normalize(tree)
+    logging.info("DONE with wordhtml2xml.convert()")
-    return tree
-    

BADataMunger/trunk/wordstripper.py

@@ +1 @@
+import logging
-from os.path import join
-
@@ -7 +8 @@
-def strip(contextpath, source):
-    """Strip unneeded formatting inherited from MSWord."""
+    logging.info("BEGIN attempt to strip unneeded formatting inherited from MSWord using %s: wordstripper.strip()" % XSLTFILE)
-    xslt_doc = etree.parse(join(contextpath, XSLTFILE))
-    transform = etree.XSLT(xslt_doc)
-    result = etree.XML(unicode(transform(source)))
+    logging.info("DONE with wordstripper.strip()")
-    return result
-