From 0af9398d27e323bf52d4ead678e3d7c69c383848 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Tue, 23 Mar 2010 03:22:05 +0000 Subject: Merged revisions 79306,79311,79325 via svnmerge from svn+ssh://pythondev@svn.python.org/python/trunk ................ r79306 | benjamin.peterson | 2010-03-22 17:40:06 -0500 (Mon, 22 Mar 2010) | 21 lines Merged revisions 79077,79137,79304-79305 via svnmerge from svn+ssh://pythondev@svn.python.org/sandbox/trunk/2to3/lib2to3 ........ r79077 | benjamin.peterson | 2010-03-18 18:05:29 -0500 (Thu, 18 Mar 2010) | 1 line port detect_encoding improvements from py3k ........ r79137 | benjamin.peterson | 2010-03-20 11:12:53 -0500 (Sat, 20 Mar 2010) | 1 line add a fixer for setting sys.exitfunc #2356 ........ r79304 | benjamin.peterson | 2010-03-22 17:20:22 -0500 (Mon, 22 Mar 2010) | 1 line fix test_parser when it's run in a path with spaces #7666 ........ r79305 | benjamin.peterson | 2010-03-22 17:27:07 -0500 (Mon, 22 Mar 2010) | 1 line normalize whitespace ........ ................ r79311 | benjamin.peterson | 2010-03-22 17:54:42 -0500 (Mon, 22 Mar 2010) | 9 lines Merged revisions 79309 via svnmerge from svn+ssh://pythondev@svn.python.org/sandbox/trunk/2to3/lib2to3 ........ r79309 | benjamin.peterson | 2010-03-22 17:50:47 -0500 (Mon, 22 Mar 2010) | 1 line pass correct symbol in ........ ................ r79325 | benjamin.peterson | 2010-03-22 22:03:55 -0500 (Mon, 22 Mar 2010) | 13 lines Merged revisions 79313,79324 via svnmerge from svn+ssh://pythondev@svn.python.org/sandbox/trunk/2to3/lib2to3 ........ r79313 | benjamin.peterson | 2010-03-22 17:59:57 -0500 (Mon, 22 Mar 2010) | 1 line another case where a symbol is needed ........ r79324 | benjamin.peterson | 2010-03-22 21:59:47 -0500 (Mon, 22 Mar 2010) | 1 line use unicode literals ........ ................ --- Lib/lib2to3/pgen2/tokenize.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) (limited to 'Lib/lib2to3/pgen2/tokenize.py') diff --git a/Lib/lib2to3/pgen2/tokenize.py b/Lib/lib2to3/pgen2/tokenize.py index 7ae02801ea..701daf8a8d 100644 --- a/Lib/lib2to3/pgen2/tokenize.py +++ b/Lib/lib2to3/pgen2/tokenize.py @@ -253,14 +253,16 @@ def detect_encoding(readline): in. It detects the encoding from the presence of a utf-8 bom or an encoding - cookie as specified in pep-0263. If both a bom and a cookie are present, - but disagree, a SyntaxError will be raised. If the encoding cookie is an - invalid charset, raise a SyntaxError. + cookie as specified in pep-0263. If both a bom and a cookie are present, but + disagree, a SyntaxError will be raised. If the encoding cookie is an invalid + charset, raise a SyntaxError. Note that if a utf-8 bom is found, + 'utf-8-sig' is returned. If no encoding is specified, then the default of 'utf-8' will be returned. """ bom_found = False encoding = None + default = 'utf-8' def read_or_stop(): try: return readline() @@ -287,17 +289,16 @@ def detect_encoding(readline): if codec.name != 'utf-8': # This behaviour mimics the Python interpreter raise SyntaxError('encoding problem: utf-8') - else: - # Allow it to be properly encoded and decoded. - encoding = 'utf-8-sig' + encoding += '-sig' return encoding first = read_or_stop() if first.startswith(BOM_UTF8): bom_found = True first = first[3:] + default = 'utf-8-sig' if not first: - return 'utf-8', [] + return default, [] encoding = find_cookie(first) if encoding: @@ -305,13 +306,13 @@ def detect_encoding(readline): second = read_or_stop() if not second: - return 'utf-8', [first] + return default, [first] encoding = find_cookie(second) if encoding: return encoding, [first, second] - return 'utf-8', [first, second] + return default, [first, second] def untokenize(iterable): """Transform tokens back into Python source code. -- cgit v1.2.1