Merge branch 'utility-script-2'

4 files changed, 154 insertions, 9 deletions

diff --git a/README.rst b/README.rst
index 8d7bfd4..9ae08e7 100644
--- a/README.rst
+++ b/README.rst
@@ -10,10 +10,10 @@ human-readable Unicode strings that should still be somewhat intelligeble
 title). 
 
 In most of these examples you could represent Unicode characters as
-"???" or "\\15BA\\15A0\\1610", to mention two extreme cases. But that's
+`???` or `\\15BA\\15A0\\1610`, to mention two extreme cases. But that's
 nearly useless to someone who actually wants to read what the text says.
 
-What Unidecode provides is a middle road: function unidecode() takes
+What Unidecode provides is a middle road: function `unidecode()` takes
 Unicode data and tries to represent it in ASCII characters (i.e., the
 universally displayable characters between 0x00 and 0x7F), where the
 compromises taken when mapping between two character sets are chosen to be
@@ -35,7 +35,7 @@ built-in functions). It is based on hand-tuned character mappings that for
 example also contain ASCII approximations for symbols and non-Latin
 alphabets.
 
-This is a Python port of Text::Unidecode Perl module by
+This is a Python port of `Text::Unidecode` Perl module by
 Sean M. Burke <sburke@cpan.org>.
 
 
@@ -54,6 +54,15 @@ ASCII bytes in Python 3.x)::
     >>> unidecode(u"\u5317\u4EB0")
     'Bei Jing '
 
+A command line utility is also included (see `unidecode --help` for a full list
+of available options)::
+
+    $ echo hello | unidecode
+    hello
+    $ unidecode -c hello
+    hello
+    $ unidecode hello.txt
+    hello
 
 Requirements
 ------------
@@ -80,11 +89,11 @@ Python.
 Installation
 ------------
 
-You install Unidecode, as you would install any Python module, by running
-these commands::
+To install Unidecode from the source distribution and run unit tests, use these
+commands::
 
-    python setup.py install
-    python setup.py test
+    $ python setup.py install
+    $ python setup.py test
 
 
 Source
@@ -92,7 +101,7 @@ Source
 
 You can get the latest development version of Unidecode with::
 
-    git clone https://www.tablix.org/~avian/git/unidecode.git
+    $ git clone https://www.tablix.org/~avian/git/unidecode.git
 
 
 Support
@@ -111,7 +120,7 @@ Copyright 2001, Sean M. Burke <sburke@cpan.org>, all rights reserved.
 
 Python code and later additions:
 
-Copyright 2014, Tomaz Solc <tomaz.solc@tablix.org>
+Copyright 2015, Tomaz Solc <tomaz.solc@tablix.org>
 
 This program is free software; you can redistribute it and/or modify it
 under the terms of the GNU General Public License as published by the Free
diff --git a/setup.py b/setup.py
index 32ed1c8..5c37aed 100644
--- a/setup.py
+++ b/setup.py
@@ -14,6 +14,12 @@ setup(name='Unidecode',
 
       test_suite = 'tests',
 
+      entry_points = {
+		'console_scripts': [
+			'unidecode = unidecode.util:main'
+		]
+      },
+
       classifiers = [
 	"License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+)",
 	"Programming Language :: Python",
diff --git a/tests/test_utility.py b/tests/test_utility.py
new file mode 100644
index 0000000..96aba07
--- /dev/null
+++ b/tests/test_utility.py
@@ -0,0 +1,72 @@
+# vim:ts=4 sw=4 expandtab softtabstop=4
+import os
+import locale
+import unittest
+import subprocess
+import sys
+import tempfile
+
+PY3 = sys.version_info[0] >= 3
+
+here = os.path.dirname(__file__)
+
+if PY3:
+    def _u(x):
+        return x
+else:
+    def _u(x):
+        return x.decode('unicode-escape')
+
+def get_cmd():
+    sys_path = os.path.join(here, "..")
+
+    return [sys.executable, "-c",
+            "import sys; sys.path.insert(0, '%s'); from unidecode.util import main; main()" % (sys_path,)]
+
+def run(argv):
+    cmd = get_cmd()
+    p = subprocess.Popen(cmd + argv, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
+
+    out, err = p.communicate()
+
+    return out.decode('ascii'), err.decode('ascii')
+
+def temp(content):
+    f = tempfile.NamedTemporaryFile()
+    f.write(content)
+    f.flush()
+    return f
+
+class TestUnidecodeUtility(unittest.TestCase):
+
+    TEST_UNICODE = _u('\u9769')
+    TEST_ASCII = 'Ge '
+
+    def test_encoding_error(self):
+        f = temp(self.TEST_UNICODE.encode('sjis'))
+        out, err = run(['-e', 'utf8', f.name])
+
+        expected = 'Unable to decode input: invalid start byte, start: 0, end: 1\n'
+        self.assertEqual(err, expected)
+
+    def test_file_specified_encoding(self):
+        f = temp(self.TEST_UNICODE.encode('sjis'))
+
+        out, err = run(['-e', 'sjis', f.name])
+        self.assertEqual(out, self.TEST_ASCII)
+
+    def test_file_default_encoding(self):
+        f = temp(self.TEST_UNICODE.encode(locale.getpreferredencoding()))
+        out, err = run([f.name])
+        self.assertEqual(out, self.TEST_ASCII)
+
+    def test_file_stdin(self):
+        cmd = get_cmd()
+        p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+
+        out, err = p.communicate(self.TEST_UNICODE.encode(locale.getpreferredencoding()))
+        self.assertEqual(out.decode('ascii'), self.TEST_ASCII)
+
+    def test_commandline(self):
+        out = run(['-e', 'sjis', '-c', self.TEST_UNICODE.encode('sjis')])[0]
+        self.assertEqual(out, self.TEST_ASCII + '\n')
diff --git a/unidecode/util.py b/unidecode/util.py
new file mode 100644
index 0000000..477280d
--- /dev/null
+++ b/unidecode/util.py
@@ -0,0 +1,58 @@
+# vim:ts=4 sw=4 expandtab softtabstop=4
+from __future__ import print_function
+import optparse
+import locale
+import os
+import sys
+import warnings
+
+from unidecode import unidecode
+
+PY3 = sys.version_info[0] >= 3
+
+def fatal(msg):
+    sys.stderr.write(msg + "\n")
+    sys.exit(1)
+
+def main():
+    default_encoding = locale.getpreferredencoding()
+
+    parser = optparse.OptionParser('%prog [options] [FILE]',
+            description="Transliterate Unicode text into ASCII. FILE is path to file to transliterate. "
+            "Standard input is used if FILE is omitted and -c is not specified.")
+    parser.add_option('-e', '--encoding', metavar='ENCODING', default=default_encoding,
+            help='Specify an encoding (default is %s)' % (default_encoding,))
+    parser.add_option('-c', metavar='TEXT', dest='text',
+            help='Transliterate TEXT instead of FILE')
+
+    options, args = parser.parse_args()
+
+    encoding = options.encoding
+
+    if args:
+        if options.text:
+            fatal("Can't use both FILE and -c option")
+        else:
+            with open(args[0], 'rb') as f:
+                stream = f.read()
+    elif options.text:
+        if PY3:
+            stream = os.fsencode(options.text)
+        else:
+            stream = options.text
+        # add a newline to the string if it comes from the
+        # command line so that the result is printed nicely
+        # on the console.
+        stream += '\n'.encode('ascii')
+    else:
+        if PY3:
+            stream = sys.stdin.buffer.read()
+        else:
+            stream = sys.stdin.read()
+
+    try:
+        stream = stream.decode(encoding)
+    except UnicodeDecodeError as e:
+        fatal('Unable to decode input: %s, start: %d, end: %d' % (e.reason, e.start, e.end))
+
+    sys.stdout.write(unidecode(stream))