Navigation

← Previous Changeset
Next Changeset →

Changeset 1339

Timestamp:

07/18/08 17:10:56 (3 months ago)

Author:

thomase

Message:

prevent duplicate ids and citations; tweak citation rendering

Files:

BADataMunger/trunk/batlaspipe.py (modified) (10 diffs)

Legend:

: Unmodified
: Added
: Removed
: Modified
: Copied
: Moved

BADataMunger/trunk/batlaspipe.py

r1336	r1339
19	19
20	20	SLASH_REGEX = re.compile(r"\s/\s")
	21	DASHNUM_END_REGEX = re.compile('-(\d+)$')
21	22	NAMESPACE = 'http://atlantides.org/batlas'
22	23	XMLDECL = '<?xml version="1.0" encoding="UTF-8"?>'
23	24	PREPARER = 'Tom Elliott'
	25
	26	priorcitations = []
24	27
25	28	class Pipe:
…	…
40	43	file
41	44	"""
	45
	46	priorcitations = []
42	47	logging.basicConfig(level=logging.INFO, format='%(levelname)-8s %(message)s')
43	48	logging.info("INITIALIZING: %s" % self.__class__)
…	…
116	121
117	122
118
	123	def vetid(self, ident, idents, variant=False, minor=False):
	124	identregex = re.compile("%s(-\d)*" % ident)
	125	them = [it for it in idents if identregex.match(it)]
	126	newident = ident
	127	if len(them) > 0:
	128	if variant:
	129	newident =''
	130	logging.warning("id collision for %s; id suppressed because it's a variant" % ident)
	131	elif minor:
	132	newident = ''
	133	logging.warning("id collision for %s; id suppressed because it's a minor alternative (i.e., diamond)" % ident)
	134	else:
	135	newident = "%s-%s" % (ident, len(them))
	136	logging.info("id collision for %s; new id amended to %s" % (ident, newident))
	137	return newident
	138
119	139	def idit(self):
120	140	""" Construct all appropriate id strings (primary and alternate) for each place
…	…
125	145
126	146	maker = BAtlasIDMaker()
127
	147
	148	priorids = []
128	149	for place in places:
129	150	place.batlasids = []
…	…
145	166	if place.dirtype == 'numbered':
146	167	label = "(%s)" % label
147		place.batlasids.append(maker.makeID('', mapnum, grid, label))
	168	bid = maker.makeID('', mapnum, grid, label)
	169	bid = self.vetid(bid, priorids)
	170	if len(bid) > 0:
	171	place.batlasids.append(bid)
148	172	altlabel = maker.buildAltLabel(label, place.locdesc, placetypes)
149		place.batlasids.append(maker.makeID(altlabel, mapnum, grid, phraseprefix=label))
	173	bid = maker.makeID(altlabel, mapnum, grid, phraseprefix=label)
	174	bid = self.vetid(bid, priorids)
	175	if len(bid) > 0:
	176	place.batlasids.append(bid)
150	177	else:
151		place.batlasids.append(maker.makeID(label, mapnum, grid))
	178	bid = maker.makeID(label, mapnum, grid)
	179	bid = self.vetid(bid, priorids)
	180	if len(bid) > 0:
	181	place.batlasids.append(bid)
152	182
153	183
…	…
162	192	else:
163	193	names = place.namestring.split('/')
	194	names = [(n, False, False) for n in names]
164	195	else:
165		names = [~~p.name~~ for p in place.placenames]
	196	names = [(p.name, p.variant, p.minorAlternative) for p in place.placenames]
166	197	if len(names) > 1:
167	198	for name in names:
168		label = maker.buildAltLabel(place.namestring, name, placetypes)
	199	label = maker.buildAltLabel(place.namestring, name[0], placetypes)
169	200	if len(label) > 0:
170	201	bid = maker.makeID(label, mapnum, grid, postfix, prefix)
171		if bid != place.batlasids[0]:
172		place.batlasids.append(bid)
	202
	203	if bid not in place.batlasids:
	204	bid = self.vetid(bid, priorids, name[1], name[2])
	205	if len(bid) > 0:
	206	place.batlasids.append(bid)
173	207
174	208	if len(place.batlasids) == 0:
…	…
179	213	msg += ("\n\t id: %s" % bid)
180	214	logging.debug(msg)
	215	priorids.extend(place.batlasids)
181	216
182	217	def outit(self):
…	…
184	219	"""
185	220
186
	221	# get all the places
187	222	places = self['places']
	223
188	224	d = etree.Element('featurelist')
189	225	d.attrib['mapnum'] = self.map_number
…	…
191	227	q.text = "%s/" % NAMESPACE
192	228	for p in places:
193		if len(p.batlasids) > 0:
194		e = etree.SubElement(d, 'feature')
195
196		# write all id/aliases
197		for i, bid in enumerate(p.batlasids):
198		if i == 0 or bid != p.batlasids[0]:
199		q = etree.SubElement(e, 'alias', id=bid)
200		e.xpath("*[local-name() = 'alias']")[0].attrib['primary'] = 'yes'
201
202		# q = etree.SubElement(e, 'mapnumber')
203		# q.text = self.map_number
204
205		# feature type
206		if p.dirtype == 'name':
207		ptype = 'labeled feature'
208		elif p.dirtype == 'numbered':
209		ptype = 'numbered feature'
210		elif p.dirtype == 'unlocated':
211		ptype = 'unlocated toponym'
212		elif p.dirtype == 'false':
213		ptype = 'false toponym'
214		else:
215		ptype = p.dirtype
216		q = etree.SubElement(e, 'type')
217		q.text = ptype
218
219		# additional feature types
220		for i, t in enumerate(p.types):
221		if i == 0:
222		q = etree.SubElement(e, 'subtype')
223		elif t != p.types[i-1]:
224		q = etree.SubElement(e, 'subtype')
225		q.text = t
226
227		# gridsquare
228		if len(p.grid) > 0:
229		q = etree.SubElement(e, 'gridsquare')
230		q.text = p.grid
231
232		# labels actually appearing on the map
233		if len(p.namestring) > 0:
234		if p.dirtype not in ['unlocated', 'false']:
235		q = etree.SubElement(e, 'label', context='map')
236		txt = SLASH_REGEX.sub('/', p.namestring)
237		txt = DIAMOND_STOP_REGEX.sub('', txt)
238		q.text = txt.strip()
239
240		# citations
241		q = etree.SubElement(e, 'citation')
242		if p.dirtype == 'unlocated':
243		citname =SLASH_REGEX.sub('/', p.namestring.strip())
244		q.text = "BAtlas %s unlocated %s" % (self.map_number, citname)
245		elif p.dirtype =='false':
246		citname = SLASH_REGEX.sub('/',p.namestring.strip())
247		q.text = "BAtlas %s false name %s" % (self.map_number, citname)
248		elif p.dirtype == 'numbered':
249		citname = SLASH_REGEX.sub('/',p.locdesc.strip())
250		q.text = "BAtlas %s %s no. %s (%s)" % (self.map_number, p.grid, p.namestring, citname)
251		elif p.dirtype == 'name' and len(p.namestring) == 0:
252		citname = SLASH_REGEX.sub('/',p.locdesc.strip())
253		q.text = "BAtlas %s %s %s" % (self.map_number, p.grid, citname)
254		elif len(p.namestring) == 0:
255		citname = SLASH_REGEX.sub('/',p.locdesc.strip())
256		q.text = "BAtlas %s %s unnamed %s %s" % (self.map_number, p.grid, p.dirtype, citname)
257		else:
258		citname = txt.strip()
259		q.text = "BAtlas %s %s %s" % (self.map_number, p.grid, citname)
260
261		# enumerate placenames
262		for i, n in enumerate(p.placenames):
263		q = etree.SubElement(e, 'geogname')
264		q.text = n.name.strip()
265		# if q.text not in citname:
266		# q.attrib['type'] = 'variant'
267		if n.variant:
268		q.attrib['type'] = 'variant'
269		elif n.minorAlternative:
270		q.attrib['type'] = 'minor-alternate'
271		if n.completeness != 'complete':
272		q.attrib['completeness'] = n.completeness
273		if n.accuracy != 'accurate':
274		q.attrib['accuracy'] = n.accuracy
275		if n.inferred:
276		q.attrib['inferred'] = 'yes'
277		if n.certainty != 'certain':
278		q.attrib['certainty'] = n.certainty
279		if len(p.placenames) > 1 and citname != q.text:
280		txt = q.text
281		if n.completeness == 'reconstructable':
282		txt = u"*%s" % txt
283		if n.accuracy == 'inaccurate':
284		txt = u"\u2018%s\u2019" % txt
285		if n.inferred == True:
286		txt = u"[%s]" % txt
	229	try:
	230	if len(p.batlasids) > 0:
	231	e = etree.SubElement(d, 'feature')
	232
	233	# write all id/aliases
	234	for i, bid in enumerate(p.batlasids):
	235	if i == 0 or bid != p.batlasids[0]:
	236	q = etree.SubElement(e, 'alias', id=bid)
	237	e.xpath("*[local-name() = 'alias']")[0].attrib['primary'] = 'yes'
	238
	239	# feature type
	240	if p.dirtype == 'name':
	241	ptype = 'labeled feature'
	242	elif p.dirtype == 'numbered':
	243	ptype = 'numbered feature'
	244	elif p.dirtype == 'unlocated':
	245	ptype = 'unlocated toponym'
	246	elif p.dirtype == 'false':
	247	ptype = 'false toponym'
	248	else:
	249	ptype = p.dirtype
	250	q = etree.SubElement(e, 'type')
	251	q.text = ptype
	252
	253	if 'group' in p.dirtype:
	254	q = etree.SubElement(e, 'featurecount')
	255	q.text = "%s" % p.featurecount
	256
	257	# additional feature types
	258	for i, t in enumerate(p.types):
	259	if i == 0:
	260	q = etree.SubElement(e, 'subtype')
	261	elif t != p.types[i-1]:
	262	q = etree.SubElement(e, 'subtype')
	263	q.text = t
	264
	265	# gridsquare
	266	if len(p.grid) > 0:
	267	q = etree.SubElement(e, 'gridsquare')
	268	q.text = p.grid
	269
	270	# labels actually appearing on the map
	271	if len(p.namestring) > 0:
	272	if p.dirtype not in ['unlocated', 'false']:
	273	q = etree.SubElement(e, 'label', context='map')
	274	txt = SLASH_REGEX.sub('/', p.namestring)
	275	txt = DIAMOND_STOP_REGEX.sub('', txt)
	276	q.text = txt.strip()
	277
	278	# citations
	279	citcontent = u''
	280	if p.dirtype == 'unlocated':
	281	citname =SLASH_REGEX.sub('/', p.namestring.strip())
	282	citcontent = "BAtlas %s unlocated %s" % (self.map_number, citname)
	283	elif p.dirtype =='false':
	284	citname = SLASH_REGEX.sub('/',p.namestring.strip())
	285	citcontent = "BAtlas %s false name %s" % (self.map_number, citname)
	286	elif p.dirtype == 'numbered':
	287	citname = SLASH_REGEX.sub('/',p.locdesc.strip())
	288	citcontent = "BAtlas %s %s no. %s (%s)" % (self.map_number, p.grid, p.namestring, citname)
	289	elif p.dirtype == 'name' and len(p.namestring) == 0:
	290	citname = SLASH_REGEX.sub('/',p.locdesc.strip())
	291	citcontent = "BAtlas %s %s %s" % (self.map_number, p.grid, citname)
	292	elif len(p.namestring) == 0:
	293	citname = SLASH_REGEX.sub('/',p.locdesc.strip())
	294	citcontent = "BAtlas %s %s unnamed %s (%s)" % (self.map_number, p.grid, p.dirtype.replace('-', ' '), citname)
	295	else:
	296	citname = txt.strip()
	297	citcontent = "BAtlas %s %s %s" % (self.map_number, p.grid, citname)
	298	if len(citcontent) > 0:
	299	writecit(e, citcontent, p.batlasids[0])
	300
	301	# enumerate placenames and if appropriate corresponding alternate citations
	302	for i, n in enumerate(p.placenames):
	303	q = etree.SubElement(e, 'geogname')
	304	q.text = n.name.strip()
	305	# if q.text not in citname:
	306	# q.attrib['type'] = 'variant'
	307	if n.variant:
	308	q.attrib['type'] = 'variant'
	309	elif n.minorAlternative:
	310	q.attrib['type'] = 'minor-alternate'
	311	if n.completeness != 'complete':
	312	q.attrib['completeness'] = n.completeness
	313	if n.accuracy != 'accurate':
	314	q.attrib['accuracy'] = n.accuracy
	315	if n.inferred:
	316	q.attrib['inferred'] = 'yes'
287	317	if n.certainty != 'certain':
288		txt = u"%s?" % txt
289		if n.minorAlternative:
290		txt = u"\u00A7%s" % txt
291		if p.dirtype == 'unlocated':
292		txt = "BAtlas %s unlocated %s" % (self.map_number, txt)
293		elif p.dirtype == 'false':
294		txt = "BAtlas %s false %s" % (self.map_number, txt)
295		elif len(p.placenames) > 1:
296		txt = "BAtlas %s %s %s" % (self.map_number, p.grid, txt)
297		if len(txt) > 0:
298		if 'island-group' in p.types:
299		txt = "%s Inss." % txt
300		elif 'island' in p.types:
301		txt = "%s Ins." % txt
302		elif 'river' in p.types:
303		txt = "%s fl." % txt
304		q = etree.SubElement(e, 'citation')
305		q.text = txt
	318	q.attrib['certainty'] = n.certainty
	319	if len(p.placenames) > 1 and citname != q.text:
	320	txt = q.text
	321	if n.completeness == 'reconstructable':
	322	txt = u"*%s" % txt
	323	if n.accuracy == 'inaccurate':
	324	txt = u"\u2018%s\u2019" % txt
	325	if n.inferred == True:
	326	txt = u"[%s]" % txt
	327	if n.certainty != 'certain':
	328	txt = u"%s?" % txt
	329	if n.minorAlternative:
	330	txt = u"\u00A7%s" % txt
	331	if p.dirtype == 'unlocated':
	332	txt = "BAtlas %s unlocated %s" % (self.map_number, txt)
	333	elif p.dirtype == 'false':
	334	txt = "BAtlas %s false %s" % (self.map_number, txt)
	335	elif len(p.placenames) > 1:
	336	txt = "BAtlas %s %s %s" % (self.map_number, p.grid, txt)
	337	if len(txt) > 0:
	338	if 'island-group' in p.types:
	339	txt = "%s Inss." % txt
	340	elif 'island' in p.types:
	341	txt = "%s Ins." % txt
	342	elif 'river' in p.types:
	343	txt = "%s fl." % txt
	344	# if appropriate, write a citation
	345	writecit(e, txt, p.batlasids[0])
	346
	347	# location description
	348	if len(p.locdesc) > 0:
	349	q = etree.SubElement(e, 'location')
	350	q.text = SLASH_REGEX.sub('/',p.locdesc.strip())
306	351
307		# location description
308		if len(p.locdesc) > 0:
309		q = etree.SubElement(e, 'location')
310		q.text = SLASH_REGEX.sub('/',p.locdesc.strip())
311
312		# itineraries
313		if len(p.itinraw) > 0:
314		q = etree.SubElement(e, 'itinerary')
315		q.text = p.itinraw.strip()
316		q = etree.SubElement(e, 'citation')
317		q.text = "BATlas %s %s (%s)" % (self.map_number, p.dirtype, p.itinraw.strip())
318
	352	# itineraries
	353	if len(p.itinraw) > 0:
	354	q = etree.SubElement(e, 'itinerary')
	355	q.text = p.itinraw.strip()
	356	citcontent = "BATlas %s %s (%s)" % (self.map_number, p.dirtype, p.itinraw.strip())
	357	writecit(e, citcontent, p.batlasids[0])
	358	except AttributeError, detail:
	359	logging.critical('attribute error trying to count number of batlasids for place: %s' % detail)
	360	print p
	361
319	362	cmntf = open(r'./etc/batlasxmlcomment.txt')
320	363	cmnt = cmntf.read()
…	…
335	378	logging.info("wrote output result xml file on %s" % fpath)
336	379
	380	def writecit(parent, tcontent, primeid):
	381	ctcontent = u''.join(tcontent.split())
	382	if ctcontent in priorcitations:
	383	# if primeid ends in dash-number then postfix the number onto the citation
	384	m = DASHNUM_END_REGEX.search(primeid)
	385	n = DASHNUM_END_REGEX.search(tcontent)
	386	if m and not n:
	387	writecit(parent, "%s (%s)" % (tcontent, m.group(1)), primeid)
	388	else:
	389	logging.warning("Suppressed writing of citation '%s' for primary id '%s' because that citation is already in use" % (tcontent, primeid))
	390
	391	else:
	392	q = etree.SubElement(parent, 'citation')
	393	q.text = tcontent
	394	priorcitations.append(ctcontent)

Navigation

Changeset 1339

Legend:

BADataMunger/trunk/batlaspipe.py

Download in other formats: