From 1718b94353733bb79043a7c6d80efeba8bd0c8d1 Mon Sep 17 00:00:00 2001 From: Philip Thiem Date: Sat, 17 May 2014 04:14:19 -0500 Subject: Starting a unicode_utils module. --HG-- extra : source : 2e47fa11a272ed61f7c1bbf88aae27e81040fe93 --- setuptools/unicode_utils.py | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 setuptools/unicode_utils.py (limited to 'setuptools') diff --git a/setuptools/unicode_utils.py b/setuptools/unicode_utils.py new file mode 100644 index 00000000..d2de941a --- /dev/null +++ b/setuptools/unicode_utils.py @@ -0,0 +1,41 @@ +import unicodedata +import sys +from setuptools.compat import unicode as decoded_string + + +# HFS Plus uses decomposed UTF-8 +def decompose(path): + if isinstance(path, decoded_string): + return unicodedata.normalize('NFD', path) + try: + path = path.decode('utf-8') + path = unicodedata.normalize('NFD', path) + path = path.encode('utf-8') + except UnicodeError: + pass # Not UTF-8 + return path + + +def filesys_decode(path): + """ + Ensure that the given path is decoded, + NONE when no expected encoding works + """ + + fs_enc = sys.getfilesystemencoding() + if isinstance(path, decoded_string): + return path + + for enc in (fs_enc, "utf-8"): + try: + return path.decode(enc) + except UnicodeDecodeError: + continue + + +def try_encode(string, enc): + "turn unicode encoding into a functional routine" + try: + return string.encode(enc) + except UnicodeEncodeError: + return None -- cgit v1.2.1