batlaspipe.py

Revision 1427, 18.4 kB (checked in by thomase, 4 months ago)
sticking a fork in the aspect of BADataMunger that is the creation of batlas ids, or so I hope

Line
1	from os.path import normpath, normcase, isdir, isfile, splitdrive, splitext, split, join
2	import os
3	import logging
4	import re
5	import unicodedata
6
7	import datetime as dt
8
9	import lxml.etree as etree
10
11	import wordhtml2xml
12	import wordstripper
13	import tablegroker
14	import tableparser
15	from etreehelps import getalltext
16	from texthelps import normalizetext
17
18	from bidmaker import BAtlasIDMaker, DIAMOND_STOP_REGEX
19
20	SLASH_REGEX = re.compile(r"\s/\s")
21	DASHNUM_END_REGEX = re.compile('-(\d+)$')
22	NAMESPACE = 'http://atlantides.org/batlas'
23	XMLDECL = '<?xml version="1.0" encoding="UTF-8"?>'
24	PREPARER = 'Tom Elliott'
25
26	priorcitations = []
27	def getpipe(mns):
28	mns = str(mns)
29
30	if mns[-1] not in ['_', 'a','I']:
31	mns += '_'
32	mns = mns.rjust(4, '0')
33	p = Pipe("config/BATL%sconfig.xml" % mns, "etc/wordhtml/BATL%s.htm" % mns, 'scratch')
34	return p
35
36	class Pipe:
37	"""String together a series of operations to create URIs for features from a Barrington Atlas map
38
39	Use like:
40
41	import batlaspipe as bp
42	p = batlaspipe.Pipe('config/BATL022_config.xml', 'etc/wordhtml/BATL022_.htm', 'scratch')p.cycle()
43	p.idit()
44	p.outit()
45
46	"""
47
48	def __init__(self, configfile, dirfile, destdir):
49	# source, source_config=None, source_gis=None):
50	"""Create a pipe object to manage the conversions and open the source
51	file
52	"""
53
54	priorcitations = []
55	logging.basicConfig(level=logging.INFO, format='%(levelname)-8s %(message)s')
56	logging.info("INITIALIZING: %s" % self.__class__)
57
58	self.data = {}
59	self['contextpath'] = os.getcwd()
60	logging.info("contextpath: %s" % self['contextpath'])
61
62	self['configfile'] = normpath(normcase(configfile))
63	self['dirfile'] = normpath(normcase(dirfile))
64	self['destdir'] = normpath(normcase(destdir))
65
66	if not isfile(self['configfile']):
67	logging.critical( "No file found at path: %s" % self['configfile'])
68	elif not isfile(self['dirfile']):
69	logging.critical ("No file found at path: %s" % self['dirfile'])
70	elif not isdir(self['destdir']):
71	logging.critical( "No directory found at path: %s" % self['destdir'])
72	else:
73
74	drive, dirpath = splitdrive(self['dirfile'])
75	filepath, filename = split(dirpath)
76	self['filenameroot'], extension = splitext(filename)
77
78
79	# read and store the directory file
80	f = open(self['dirfile'])
81	self['wordhtml'] = f.read()
82	f.close()
83	logging.info("read and stored directory file from %s" % self['dirfile'])
84
85	# read and store the config file
86	f = open(self['configfile'])
87	config_text = f.read()
88	f.close()
89	logging.info("read and stored config file from %s" % self['configfile'])
90
91	# get some essential info from the config file
92	self['config'] = etree.XML(config_text)
93	try:
94	self.map_number = self['config'].xpath("//map_number")[0].text
95	except:
96	self.map_number = 'XYZ'
97	logging.info("map number from config file: %s" % self.map_number)
98
99	self.creators = []
100	self.contributors = []
101	self.rights = ''
102	for c in self['config'].xpath("//creator"):
103	self.creators.append(normalizetext(getalltext(c)))
104	for c in self['config'].xpath("//contributor"):
105	self.contributors.append(normalizetext(getalltext(c)))
106	try:
107	self.rights = self['config'].xpath("//rights")[0].text
108	except:
109	pass
110
111	logging.info("DONE INITIALIZING: %s\n" % self.__class__)
112
113
114	def __getitem__(self, key): return self.data[key]
115
116	def __setitem__(self, key, item): self.data[key] = item
117
118	def cycle(self):
119	"""Cycle through all steps in the pipeline"""
120
121	logging.info("CYCLING: %s" % self.__class__)
122
123	self['wordxml'] = wordhtml2xml.convert(self['wordhtml'])
124	self['cleanxml'] = wordstripper.strip(self['contextpath'], self['wordxml'])
125	self['dirtables'] = tablegroker.grok(self['cleanxml'])
126	self['places'] = tableparser.parse(self, self['dirtables'], self.map_number)
127
128	logging.info("DONE CYCLING: %s\n" % self.__class__)
129
130
131	def vetid(self, ident, idents, variant=False, minor=False):
132	identregex = re.compile("%s(-\d)*" % ident)
133	them = [it for it in idents if identregex.match(it)]
134	newident = ident
135	if len(them) > 0:
136	if variant:
137	newident =''
138	logging.warning("id collision for %s; id suppressed because it's a variant" % ident)
139	elif minor:
140	newident = ''
141	logging.warning("id collision for %s; id suppressed because it's a minor alternative (i.e., diamond)" % ident)
142	else:
143	newident = "%s-%s" % (ident, len(them))
144	logging.info("id collision for %s; new id amended to %s" % (ident, newident))
145	return newident
146
147	def idit(self):
148	""" Construct all appropriate id strings (primary and alternate) for each place
149	"""
150
151	places = self['places']
152	mapnum = self.map_number
153
154	maker = BAtlasIDMaker()
155
156	priorids = []
157	for place in places:
158	place.batlasids = []
159	if place.dirtype == 'unlocated':
160	grid = u'unlocated'
161	elif place.dirtype == 'false':
162	grid = u'false'
163	else:
164	grid = place.grid
165
166	if place.dirtype in ['road','coastal-change', 'unlabeled']:
167	logging.info("batlaspipe.idit() deliberately suppressed ID creation for %s: (%s, %s, %s, %s)" % (place.dirtype, place.namestring, place.grid, place.locdesc, u' - '.join(place.itinerary)))
168	else:
169	label = maker.buildLabel(place.namestring, place.placenames, place.dirtype, place.locdesc, u' '.join(place.itinerary))
170
171	if len(label) > 0:
172	placetypes = [place.dirtype] + place.types
173
174	if place.dirtype == 'numbered':
175	label = "(%s)" % label
176	bid = maker.makeID('', mapnum, grid, label)
177	bid = self.vetid(bid, priorids)
178	if len(bid) > 0:
179	place.batlasids.append(bid)
180	if len(place.placenames) > 0:
181	altlabel = maker.buildAltLabel(label, place.placenames[0].name, placetypes)
182	else:
183	altlabel = maker.buildAltLabel(label, place.locdesc, placetypes)
184	bid = maker.makeID(altlabel, mapnum, grid, phraseprefix=label)
185	bid = self.vetid(bid, priorids)
186	if len(bid) > 0:
187	place.batlasids.append(bid)
188	else:
189	bid = maker.makeID(label, mapnum, grid)
190	bid = self.vetid(bid, priorids)
191	if len(bid) > 0:
192	place.batlasids.append(bid)
193
194
195	names = []
196	prefix = ''
197	postfix = ''
198
199	if len(place.placenames) == 0:
200	if place.dirtype == 'numbered' and len(place.placenames) == 0:
201	names = place.locdesc.split('/')
202	prefix = label
203	else:
204	names = place.namestring.split('/')
205	names = [(n, False, False) for n in names]
206	else:
207	names = [(p.name, p.variant, p.minorAlternative) for p in place.placenames]
208	if len(names) > 1:
209	for name in names:
210	label = maker.buildAltLabel(place.namestring, name[0], placetypes)
211	if len(label) > 0:
212	bid = maker.makeID(label, mapnum, grid, postfix, prefix)
213
214	if bid not in place.batlasids:
215	bid = self.vetid(bid, priorids, name[1], name[2])
216	if len(bid) > 0:
217	place.batlasids.append(bid)
218
219	if len(place.batlasids) == 0:
220	logging.critical("did not create id for %s: (%s, %s, %s, %s)" % (place.dirtype, place.namestring, place.grid, place.locdesc, u' - '.join(place.itinerary)))
221	else:
222	msg = "made %s id(s):" % len(place.batlasids)
223	for bid in place.batlasids:
224	msg += ("\n\t id: %s" % bid)
225	logging.debug(msg)
226	priorids.extend(place.batlasids)
227
228	def outit(self):
229	"""Output an xml file containing the ids, along with some descriptive info
230	"""
231
232	# get all the places
233	places = self['places']
234
235	d = etree.Element('featurelist')
236	d.attrib['mapnum'] = self.map_number
237	q = etree.SubElement(d, 'uribase')
238	q.text = "%s/" % NAMESPACE
239	for p in places:
240	try:
241	if len(p.batlasids) > 0:
242	e = etree.SubElement(d, 'feature')
243
244	# write all id/aliases
245	for i, bid in enumerate(p.batlasids):
246	if i == 0 or bid != p.batlasids[0]:
247	q = etree.SubElement(e, 'alias', id=bid)
248	e.xpath("*[local-name() = 'alias']")[0].attrib['primary'] = 'yes'
249
250	# feature type
251	if p.dirtype == 'name':
252	ptype = 'labeled feature'
253	elif p.dirtype == 'numbered':
254	ptype = 'numbered feature'
255	elif p.dirtype == 'unlocated':
256	ptype = 'unlocated toponym'
257	elif p.dirtype == 'false':
258	ptype = 'false toponym'
259	else:
260	ptype = p.dirtype
261	q = etree.SubElement(e, 'type')
262	q.text = ptype
263
264	if 'group' in p.dirtype:
265	q = etree.SubElement(e, 'featurecount')
266	q.text = "%s" % p.featurecount
267
268	# additional feature types
269	for i, t in enumerate(p.types):
270	if i == 0:
271	q = etree.SubElement(e, 'subtype')
272	elif t != p.types[i-1]:
273	q = etree.SubElement(e, 'subtype')
274	q.text = t
275
276	# gridsquare
277	if len(p.grid) > 0:
278	q = etree.SubElement(e, 'gridsquare')
279	q.text = p.grid
280
281	# labels actually appearing on the map
282	if len(p.namestring) > 0:
283	if p.dirtype not in ['unlocated', 'false']:
284	q = etree.SubElement(e, 'label', context='map')
285	txt = SLASH_REGEX.sub('/', p.namestring)
286	txt = DIAMOND_STOP_REGEX.sub('', txt)
287	q.text = txt.strip()
288
289	# citations
290	citcontent = u''
291	if p.dirtype == 'unlocated':
292	citname =SLASH_REGEX.sub('/', p.namestring.strip())
293	citcontent = "BAtlas %s unlocated %s" % (self.map_number, citname)
294	elif p.dirtype =='false':
295	citname = SLASH_REGEX.sub('/',p.namestring.strip())
296	citcontent = "BAtlas %s false name %s" % (self.map_number, citname)
297	elif p.dirtype == 'numbered' and len(p.placenames) == 0:
298	citname = SLASH_REGEX.sub('/',p.locdesc.strip())
299	citcontent = "BAtlas %s %s no. %s (%s)" % (self.map_number, p.grid, p.namestring, citname)
300	elif p.dirtype == 'numbered' and len(p.placenames) > 0:
301	citname = SLASH_REGEX.sub('/',p.placenames[0].name.strip())
302	citcontent = "BAtlas %s %s no. %s (%s)" % (self.map_number, p.grid, p.namestring, citname)
303	elif p.dirtype == 'name' and len(p.namestring) == 0:
304	citname = SLASH_REGEX.sub('/',p.locdesc.strip())
305	citcontent = "BAtlas %s %s %s" % (self.map_number, p.grid, citname)
306	elif len(p.namestring) == 0:
307	citname = SLASH_REGEX.sub('/',p.locdesc.strip())
308	citcontent = "BAtlas %s %s unnamed %s (%s)" % (self.map_number, p.grid, p.dirtype.replace('-', ' '), citname)
309	else:
310	citname = txt.strip()
311	citcontent = "BAtlas %s %s %s" % (self.map_number, p.grid, citname)
312	if len(citcontent) > 0:
313	writecit(e, citcontent, p.batlasids[0])
314
315	# enumerate placenames and if appropriate corresponding alternate citations
316	for i, n in enumerate(p.placenames):
317	q = etree.SubElement(e, 'geogname')
318	q.text = n.name.strip()
319	# if q.text not in citname:
320	# q.attrib['type'] = 'variant'
321	if n.variant:
322	q.attrib['type'] = 'variant'
323	elif n.minorAlternative:
324	q.attrib['type'] = 'minor-alternate'
325	if n.completeness != 'complete':
326	q.attrib['completeness'] = n.completeness
327	if n.accuracy != 'accurate':
328	q.attrib['accuracy'] = n.accuracy
329	if n.inferred:
330	q.attrib['inferred'] = 'yes'
331	if n.certainty != 'certain':
332	q.attrib['certainty'] = n.certainty
333	if len(p.placenames) > 1 and citname != q.text:
334	txt = q.text
335	if n.completeness == 'reconstructable':
336	txt = u"*%s" % txt
337	if n.accuracy == 'inaccurate':
338	txt = u"\u2018%s\u2019" % txt
339	if n.inferred == True:
340	txt = u"[%s]" % txt
341	if n.certainty != 'certain':
342	txt = u"%s?" % txt
343	if n.minorAlternative:
344	txt = u"\u00A7%s" % txt
345	if p.dirtype == 'unlocated':
346	txt = "BAtlas %s unlocated %s" % (self.map_number, txt)
347	elif p.dirtype == 'false':
348	txt = "BAtlas %s false %s" % (self.map_number, txt)
349	elif len(p.placenames) > 1:
350	txt = "BAtlas %s %s %s" % (self.map_number, p.grid, txt)
351	if len(txt) > 0:
352	if 'island-group' in p.types:
353	txt = "%s Inss." % txt
354	elif 'island' in p.types:
355	txt = "%s Ins." % txt
356	elif 'river' in p.types:
357	txt = "%s fl." % txt
358	# if appropriate, write a citation
359	writecit(e, txt, p.batlasids[0])
360
361	# location description
362	if len(p.locdesc) > 0:
363	q = etree.SubElement(e, 'location')
364	q.text = SLASH_REGEX.sub('/',p.locdesc.strip())
365
366	# unstructured notes
367	if len(p.note) > 0:
368	q = etree.SubElement(e, 'note')
369	q.text = p.note
370
371	# itineraries
372	if len(p.itinraw) > 0:
373	q = etree.SubElement(e, 'itinerary')
374	q.text = p.itinraw.strip()
375	citcontent = "BATlas %s %s (%s)" % (self.map_number, p.dirtype, p.itinraw.strip())
376	writecit(e, citcontent, p.batlasids[0])
377	except AttributeError, detail:
378	logging.critical('attribute error trying to count number of batlasids for place: %s' % detail)
379	print p
380
381	cmntf = open(r'./etc/batlasxmlcomment.txt')
382	cmnt = cmntf.read()
383	cmntf.close()
384
385	dtime = dt.datetime.utcnow()
386	dtstamp = dtime.isoformat()
387	dtyear = dtime.year
388
389	fn = "map%s.xml" % self.map_number
390	fpath = join(self['destdir'], fn)
391	f = open(fpath, 'w')
392	f.write(XMLDECL)
393	cmnt = cmnt % (fn, self.map_number, dtime.isoformat(), PREPARER, dtime.year, self.map_number, fn)
394	f.write(cmnt)
395	etree.ElementTree(d).write(f)
396	f.close()
397	logging.info("wrote output result xml file on %s" % fpath)
398
399	def writecit(parent, tcontent, primeid):
400	ctcontent = u''.join(tcontent.split())
401	if ctcontent in priorcitations:
402	# if primeid ends in dash-number then postfix the number onto the citation
403	m = DASHNUM_END_REGEX.search(primeid)
404	n = DASHNUM_END_REGEX.search(tcontent)
405	if m and not n:
406	writecit(parent, "%s (%s)" % (tcontent, m.group(1)), primeid)
407	else:
408	logging.warning("Suppressed writing of citation '%s' for primary id '%s' because that citation is already in use" % (tcontent, primeid))
409
410	else:
411	q = etree.SubElement(parent, 'citation')
412	q.text = tcontent
413	priorcitations.append(ctcontent)

Note: See TracBrowser for help on using the browser.

root/BADataMunger/trunk/batlaspipe.py

Download in other formats: