Merge branch 'bug/csv-unicode' into develop

Closes #7

Conflicts:
	test_tablib.py

4 files changed, 176 insertions, 13 deletions

diff --git a/NOTICE b/NOTICE
index ca8ed11..15c0691 100644
--- a/NOTICE
+++ b/NOTICE
@@ -1,5 +1,5 @@
 Tablib includes some vendorized python libraries: ordereddict, pyyaml,
-simplejson, and xlwt.
+simplejson, unicodecsv, and xlwt.
 
 Markup License
 ==============
@@ -94,6 +94,37 @@ ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 POSSIBILITY OF SUCH DAMAGE.
 
 
+UnicodeCSV License
+==================
+
+Copyright 2010 Jeremy Dunck. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification, are
+permitted provided that the following conditions are met:
+
+   1. Redistributions of source code must retain the above copyright notice, this list of
+      conditions and the following disclaimer.
+
+   2. Redistributions in binary form must reproduce the above copyright notice, this list
+      of conditions and the following disclaimer in the documentation and/or other materials
+      provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY JEREMY DUNCK ``AS IS'' AND ANY EXPRESS OR IMPLIED
+WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL JEREMY DUNCK OR
+CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+The views and conclusions contained in the software and documentation are those of the
+authors and should not be interpreted as representing official policies, either expressed
+or implied, of Jeremy Dunck.
+
+
+
 
 XLWT License
 ============
@@ -105,15 +136,15 @@ Redistribution and use in source and binary forms, with or without
 modification, are permitted provided that the following conditions are met:
 
 1. Redistributions of source code must retain the above copyright notice,
-this list of conditions and the following disclaimer. 
+this list of conditions and the following disclaimer.
 
 2. Redistributions in binary form must reproduce the above copyright notice,
 this list of conditions and the following disclaimer in the documentation
-and/or other materials provided with the distribution. 
+and/or other materials provided with the distribution.
 
 3. None of the names of Stephen John Machin, Lingfo Pty Ltd and any
 contributors may be used to endorse or promote products derived from this
-software without specific prior written permission. 
+software without specific prior written permission.
 
 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
@@ -131,29 +162,29 @@ THE POSSIBILITY OF SUCH DAMAGE.
 """
   Copyright (C) 2005 Roman V. Kiseliov
   All rights reserved.
- 
+
   Redistribution and use in source and binary forms, with or without
   modification, are permitted provided that the following conditions
   are met:
- 
+
   1. Redistributions of source code must retain the above copyright
      notice, this list of conditions and the following disclaimer.
- 
+
   2. Redistributions in binary form must reproduce the above copyright
      notice, this list of conditions and the following disclaimer in
      the documentation and/or other materials provided with the
      distribution.
- 
+
   3. All advertising materials mentioning features or use of this
      software must display the following acknowledgment:
      "This product includes software developed by
       Roman V. Kiseliov <roman@kiseliov.ru>."
- 
+
   4. Redistributions of any form whatsoever must retain the following
      acknowledgment:
      "This product includes software developed by
       Roman V. Kiseliov <roman@kiseliov.ru>."
- 
+
   THIS SOFTWARE IS PROVIDED BY Roman V. Kiseliov ``AS IS'' AND ANY
   EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
diff --git a/tablib/formats/_csv.py b/tablib/formats/_csv.py
index 4b1dc02..bfe8b0f 100644
--- a/tablib/formats/_csv.py
+++ b/tablib/formats/_csv.py
@@ -5,12 +5,17 @@
 
 import sys
 if sys.version_info[0] > 2:
+    is_py3 = True
+
     from io import StringIO
+    import csv
 else:
+    is_py3 = False
     from cStringIO import StringIO
+    import tablib.packages.unicodecsv as csv
+
 
 
-import csv
 import os
 
 import tablib
@@ -20,11 +25,18 @@ title = 'csv'
 extentions = ('csv',)
 
 
+DEFAULT_ENCODING = 'utf-8'
+
+
 
 def export_set(dataset):
     """Returns CSV representation of Dataset."""
     stream = StringIO()
-    _csv = csv.writer(stream)
+
+    if is_py3:
+        _csv = csv.writer(stream)
+    else:
+        _csv = csv.writer(stream, encoding=DEFAULT_ENCODING)
 
     for row in dataset._package(dicts=False):
         _csv.writerow(row)
@@ -37,7 +49,10 @@ def import_set(dset, in_stream, headers=True):
 
     dset.wipe()
 
-    rows = csv.reader(in_stream.splitlines())
+    if is_py3:
+        rows = csv.reader(in_stream.splitlines())
+    else:
+        rows = csv.reader(in_stream.splitlines(), encoding=DEFAULT_ENCODING)
     for i, row in enumerate(rows):
 
         if (i == 0) and (headers):
diff --git a/tablib/packages/unicodecsv/__init__.py b/tablib/packages/unicodecsv/__init__.py
new file mode 100644
index 0000000..e640987
--- /dev/null
+++ b/tablib/packages/unicodecsv/__init__.py
@@ -0,0 +1,105 @@
+# -*- coding: utf-8 -*-
+import csv
+from csv import *
+
+#http://semver.org/
+VERSION = (0, 8, 0)
+__version__ = ".".join(map(str,VERSION))
+
+def _stringify(s, encoding):
+    if type(s)==unicode:
+        return s.encode(encoding)
+    elif isinstance(s, (int , float)):
+        pass #let csv.QUOTE_NONNUMERIC do its thing.
+    elif type(s) != str:
+        s=str(s)
+    return s
+
+def _stringify_list(l, encoding):
+    return [_stringify(s, encoding) for s in l]
+
+class UnicodeWriter(object):
+    """
+    >>> import unicodecsv
+    >>> from cStringIO import StringIO
+    >>> f = StringIO()
+    >>> w = unicodecsv.writer(f, encoding='utf-8')
+    >>> w.writerow((u'é', u'ñ'))
+    >>> f.seek(0)
+    >>> r = unicodecsv.reader(f, encoding='utf-8')
+    >>> row = r.next()
+    >>> print row[0], row[1]
+    é ñ
+    """
+    def __init__(self, f, dialect=csv.excel, encoding="utf-8", **kwds):
+        self.writer = csv.writer(f)
+        self.dialect = dialect
+        self.encoding = encoding
+        self.writer = csv.writer(f, dialect=dialect, **kwds)
+
+    def writerow(self, row):
+        self.writer.writerow(_stringify_list(row, self.encoding))
+
+    def writerows(self, rows):
+        for row in rows:
+          self.writerow(row)
+writer = UnicodeWriter
+
+class UnicodeReader(object):
+    def __init__(self, f, dialect=csv.excel, encoding="utf-8", **kwds):
+        self.reader = csv.reader(f, dialect=dialect, **kwds)
+        self.encoding = encoding
+
+    def next(self):
+        row = self.reader.next()
+        return [unicode(s, self.encoding) for s in row]
+
+    def __iter__(self):
+        return self
+reader = UnicodeReader
+
+class DictWriter(csv.DictWriter):
+    """
+    >>> from cStringIO import StringIO
+    >>> f = StringIO()
+    >>> w = DictWriter(f, ['a', 'b'], restval=u'î')
+    >>> w.writerow({'a':'1'})
+    >>> w.writerow({'a':'1', 'b':u'ø'})
+    >>> w.writerow({'a':u'é'})
+    >>> f.seek(0)
+    >>> r = DictReader(f, fieldnames=['a'], restkey='r')
+    >>> r.next() == {'a':u'1', 'r':[u"î"]}
+    True
+    >>> r.next() == {'a':u'1', 'r':[u"ø"]}
+    True
+    >>> r.next() == {'a':u'é', 'r':[u"î"]}
+    """
+    def __init__(self, csvfile, fieldnames, restval='', extrasaction='raise', dialect='excel', encoding='utf-8', *args, **kwds):
+        self.fieldnames = fieldnames
+        self.encoding = encoding
+        self.restval = restval
+        self.writer = csv.DictWriter(csvfile, fieldnames, restval, extrasaction, dialect, *args, **kwds)
+    def writerow(self, d):
+        for fieldname in self.fieldnames:
+            if fieldname in d:
+                d[fieldname] = _stringify(d[fieldname], self.encoding)
+            else:
+                d[fieldname] = _stringify(self.restval, self.encoding)
+        self.writer.writerow(d)
+
+class DictReader(csv.DictReader):
+    def __init__(self, csvfile, fieldnames=None, restkey=None, restval=None, dialect='excel', encoding='utf-8', *args, **kwds):
+        self.restkey = restkey
+        self.encoding = encoding
+        self.reader = csv.DictReader(csvfile, fieldnames, restkey, restval, dialect, *args, **kwds)
+    def next(self):
+        d = self.reader.next()
+        for k, v in d.items():
+            if k == self.restkey:
+                rest = v
+                if rest:
+                    d[self.restkey] = [unicode(v, self.encoding) for v in rest]
+            else:
+                if v is not None:
+                    d[k] = unicode(v, self.encoding)
+        return d
diff --git a/test_tablib.py b/test_tablib.py
index 320c759..7791935 100755
--- a/test_tablib.py
+++ b/test_tablib.py
@@ -499,6 +499,18 @@ class TablibTestCase(unittest.TestCase):
         for name in [r['last_name'] for r in self.founders.dict]:
             self.assertTrue(name.isupper())
 
+    def test_unicode_csv(self):
+        """Check if unicode in csv export doesn't raise."""
+        
+        data = tablib.Dataset()
+        
+        if sys.version_info[0] > 2:
+            data.append(['\xfc', '\xfd'])
+        else:
+            exec("data.append([u'\xfc', u'\xfd'])")
+            
+        
+        data.csv
 
 if __name__ == '__main__':
     unittest.main()