From aa6bb0cfe3bda491aea7293be2ab78ccc40bd061 Mon Sep 17 00:00:00 2001 From: Doug Hellmann Date: Wed, 31 Jul 2013 12:02:28 -0400 Subject: Fix default encoding issue with python 2.6 This change addresses issue #38: "fix unicode handling issues". The issue was originally reported against neutron client (https://bugs.launchpad.net/python-neutronclient/+bug/1189112) but was tracked down to the fact that python 2.6 does not set the default encoding for sys.stdout properly. A change to python 2.7 fixes the problem there and later (http://hg.python.org/cpython/rev/e60ef17561dc/), but since cliff supports python 2.6 it needs to handle the case explicitly. Change-Id: Id06507d78c7c82b25f39366ea4a6dfa4ef3a3a97 --- cliff/app.py | 25 ++++++++- cliff/formatters/table.py | 5 +- cliff/tests/__init__.py | 0 cliff/tests/test_app.py | 124 +++++++++++++++++++++++++++++++++++++++++- demoapp/cliffdemo/encoding.py | 23 ++++++++ demoapp/setup.py | 1 + 6 files changed, 172 insertions(+), 6 deletions(-) create mode 100644 cliff/tests/__init__.py create mode 100644 demoapp/cliffdemo/encoding.py diff --git a/cliff/app.py b/cliff/app.py index 7713df1..261c5cc 100644 --- a/cliff/app.py +++ b/cliff/app.py @@ -2,6 +2,8 @@ """ import argparse +import codecs +import locale import logging import logging.handlers import os @@ -67,14 +69,31 @@ class App(object): """ self.command_manager = command_manager self.command_manager.add_command('help', HelpCommand) - self.stdin = stdin or sys.stdin - self.stdout = stdout or sys.stdout - self.stderr = stderr or sys.stderr + self._set_streams(stdin, stdout, stderr) self.interactive_app_factory = interactive_app_factory self.parser = self.build_option_parser(description, version) self.interactive_mode = False self.interpreter = None + def _set_streams(self, stdin, stdout, stderr): + locale.setlocale(locale.LC_ALL, '') + if sys.version_info[:2] == (2, 6): + # Configure the input and output streams. If a stream is + # provided, it must be configured correctly by the + # caller. If not, make sure the versions of the standard + # streams used by default are wrapped with encodings. This + # works around a problem with Python 2.6 fixed in 2.7 and + # later (http://hg.python.org/cpython/rev/e60ef17561dc/). + lang, encoding = locale.getdefaultlocale() + encoding = getattr(sys.stdout, 'encoding', None) or encoding + self.stdin = stdin or codecs.getreader(encoding)(sys.stdin) + self.stdout = stdout or codecs.getwriter(encoding)(sys.stdout) + self.stderr = stderr or codecs.getwriter(encoding)(sys.stderr) + else: + self.stdin = stdin or sys.stdin + self.stdout = stdout or sys.stdout + self.stderr = stderr or sys.stderr + def build_option_parser(self, description, version, argparse_kwargs=None): """Return an argparse option parser for this application. diff --git a/cliff/formatters/table.py b/cliff/formatters/table.py index b13b364..e625a25 100644 --- a/cliff/formatters/table.py +++ b/cliff/formatters/table.py @@ -22,7 +22,10 @@ class TableFormatter(ListFormatter, SingleFormatter): pass def emit_list(self, column_names, data, stdout, parsed_args): - x = prettytable.PrettyTable(column_names, print_empty=False) + x = prettytable.PrettyTable( + column_names, + print_empty=False, + ) x.padding_width = 1 # Figure out the types of the columns in the # first row and set the alignment of the diff --git a/cliff/tests/__init__.py b/cliff/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cliff/tests/test_app.py b/cliff/tests/test_app.py index db80bbc..57256ee 100644 --- a/cliff/tests/test_app.py +++ b/cliff/tests/test_app.py @@ -1,11 +1,19 @@ +# -*- encoding: utf-8 -*- from argparse import ArgumentError +try: + from StringIO import StringIO +except ImportError: + # Probably python 3, that test won't be run so ignore the error + pass +import sys + +import nose +import mock from cliff.app import App from cliff.command import Command from cliff.commandmanager import CommandManager -import mock - def make_app(): cmd_mgr = CommandManager('cliff.tests') @@ -227,3 +235,115 @@ def test_option_parser_conflicting_option_custom_arguments_should_not_throw(): ) MyApp() + + +def test_output_encoding_default(): + # The encoding should come from getdefaultlocale() because + # stdout has no encoding set. + if sys.version_info[:2] != (2, 6): + raise nose.SkipTest('only needed for python 2.6') + data = '\xc3\xa9' + u_data = data.decode('utf-8') + + class MyApp(App): + def __init__(self): + super(MyApp, self).__init__( + description='testing', + version='0.1', + command_manager=CommandManager('tests'), + ) + + stdout = StringIO() + getdefaultlocale = lambda: ('ignored', 'utf-8') + + with mock.patch('sys.stdout', stdout): + with mock.patch('locale.getdefaultlocale', getdefaultlocale): + app = MyApp() + app.stdout.write(u_data) + actual = stdout.getvalue() + assert data == actual + + +def test_output_encoding_sys(): + # The encoding should come from sys.stdout because it is set + # there. + if sys.version_info[:2] != (2, 6): + raise nose.SkipTest('only needed for python 2.6') + data = '\xc3\xa9' + u_data = data.decode('utf-8') + + class MyApp(App): + def __init__(self): + super(MyApp, self).__init__( + description='testing', + version='0.1', + command_manager=CommandManager('tests'), + ) + + stdout = StringIO() + stdout.encoding = 'utf-8' + getdefaultlocale = lambda: ('ignored', 'utf-16') + + with mock.patch('sys.stdout', stdout): + with mock.patch('locale.getdefaultlocale', getdefaultlocale): + app = MyApp() + app.stdout.write(u_data) + actual = stdout.getvalue() + assert data == actual + + +def test_error_encoding_default(): + # The encoding should come from getdefaultlocale() because + # stdout has no encoding set. + if sys.version_info[:2] != (2, 6): + raise nose.SkipTest('only needed for python 2.6') + data = '\xc3\xa9' + u_data = data.decode('utf-8') + + class MyApp(App): + def __init__(self): + super(MyApp, self).__init__( + description='testing', + version='0.1', + command_manager=CommandManager('tests'), + ) + + stderr = StringIO() + getdefaultlocale = lambda: ('ignored', 'utf-8') + + with mock.patch('sys.stderr', stderr): + with mock.patch('locale.getdefaultlocale', getdefaultlocale): + app = MyApp() + app.stderr.write(u_data) + actual = stderr.getvalue() + assert data == actual + + +def test_error_encoding_sys(): + # The encoding should come from sys.stdout (not sys.stderr) + # because it is set there. + if sys.version_info[:2] != (2, 6): + raise nose.SkipTest('only needed for python 2.6') + data = '\xc3\xa9' + u_data = data.decode('utf-8') + + class MyApp(App): + def __init__(self): + super(MyApp, self).__init__( + description='testing', + version='0.1', + command_manager=CommandManager('tests'), + ) + + stdout = StringIO() + stdout.encoding = 'utf-8' + stderr = StringIO() + getdefaultlocale = lambda: ('ignored', 'utf-16') + + with mock.patch('sys.stdout', stdout): + with mock.patch('sys.stderr', stderr): + with mock.patch('locale.getdefaultlocale', getdefaultlocale): + app = MyApp() + app.stderr.write(u_data) + actual = stderr.getvalue() + assert data == actual diff --git a/demoapp/cliffdemo/encoding.py b/demoapp/cliffdemo/encoding.py new file mode 100644 index 0000000..6c6c751 --- /dev/null +++ b/demoapp/cliffdemo/encoding.py @@ -0,0 +1,23 @@ +# -*- encoding: utf-8 -*- + +import logging + +from cliff.lister import Lister + + +class Encoding(Lister): + """Show some unicode text + """ + + log = logging.getLogger(__name__) + + def take_action(self, parsed_args): + messages = [ + u'pi: π', + u'GB18030:鼀丅㐀ٸཌྷᠧꌢ€', + ] + return ( + ('UTF-8', 'Unicode'), + [(repr(t.encode('utf-8')), t) + for t in messages], + ) diff --git a/demoapp/setup.py b/demoapp/setup.py index 33dd73b..330e03e 100644 --- a/demoapp/setup.py +++ b/demoapp/setup.py @@ -68,6 +68,7 @@ setup( 'files = cliffdemo.list:Files', 'file = cliffdemo.show:File', 'show file = cliffdemo.show:File', + 'unicode = cliffdemo.encoding:Encoding', ], }, -- cgit v1.2.1