unicodeMagic.py

"""Copyright 2012-2013

Eindhoven University of Technology (Bogdan Vasilescu and Alexander Serebrenik) and

University of Mons (Tom Mens)



    This program is free software: you can redistribute it and/or modify

    it under the terms of the GNU Lesser General Public License as published by

    the Free Software Foundation, either version 3 of the License, or

    (at your option) any later version.



    This program is distributed in the hope that it will be useful,

    but WITHOUT ANY WARRANTY; without even the implied warranty of

    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

    GNU Lesser General Public License for more details.



    You should have received a copy of the GNU Lesser General Public License

    along with this program.  If not, see <http://www.gnu.org/licenses/>."""



"""Based on http://stackoverflow.com/questions/3085263/create-an-utf-8-csv-file-in-python"""



import csv, codecs, cStringIO



class UTF8Recoder:

	"""

	Iterator that reads an encoded stream and reencodes the input to UTF-8

	"""

	def __init__(self, f, encoding):

		self.reader = codecs.getreader(encoding)(f)



	def __iter__(self):

		return self



	def next(self):

		return self.reader.next().encode("utf-8")



class UnicodeReader:

	"""

	A CSV reader which will iterate over lines in the CSV file "f",

	which is encoded in the given encoding.

	"""



	def __init__(self, f, dialect=csv.excel, encoding="utf-8", **kwds):

		f = UTF8Recoder(f, encoding)

		self.reader = csv.reader(f, delimiter=';', dialect=dialect, **kwds)



	def next(self):

		row = self.reader.next()

		return [unicode(s, "utf-8") for s in row]



	def __iter__(self):

		return self


class UnicodeWriter:

	"""

	A CSV writer which will write rows to CSV file "f",

	which is encoded in the given encoding.

	"""



	def __init__(self, f, dialect=csv.excel, encoding="utf-8", **kwds):

		# Redirect output to a queue

		self.queue = cStringIO.StringIO()

		self.writer = csv.writer(self.queue, delimiter=';', dialect=dialect, **kwds)

		self.stream = f

		self.encoder = codecs.getincrementalencoder(encoding)()



	def writerow(self, row):

		self.writer.writerow([s.encode("utf-8") for s in row])

		# Fetch UTF-8 output from the queue ...

		data = self.queue.getvalue()

		data = data.decode("utf-8")

		# ... and reencode it into the target encoding

		data = self.encoder.encode(data)

		# write to the target stream

		self.stream.write(data)

		# empty queue

		self.queue.truncate(0)



	def writerows(self, rows):

		for row in rows:

			self.writerow(row)