root/BADataMunger/trunk/texthelps.py

# ===========================================================================
# Copyright (C) 2006-2008 Ancient World Mapping Center (UNC-CH) and the 
# Institute for the Study of the Ancient World (NYU)
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# About Pleiades
# --------------
#
# Pleiades is an international research network and associated web portal and
# content management system devoted to the study of ancient geography.
#
# See http://pleiades.stoa.org
#
# Funding for the creation of this software was provided by a grant from the
# U.S. National Endowment for the Humanities (http://www.neh.gov), and
# by the Institute for the Study of the Ancient World at New York University
# (http://www.nyu.edu/isaw)
# ===========================================================================

import re

SPACEPATTERN = u'\s+'

def normalizetext(source):
    """ Condense arbitrary spans of spaces and newlines in a unicode string down 
    to a single space. Returns a unicode string. """
    #return u' '.join(source.replace(u'\n', u' ').strip().split()).strip()
    rex = re.compile(SPACEPATTERN, re.UNICODE)
    return rex.sub(u' ', source).strip()
    
def _test():
    import doctest
    doctest.testmod()
    doctest.testfile('tests/texthelps.txt')
    # invoke additional doctest files here

if __name__ == "__main__":
    _test()