Fix handling of non-ascii bytes tokens

2 files changed, 10 insertions, 1 deletions

diff --git a/src/zope/schema/tests/test_vocabulary.py b/src/zope/schema/tests/test_vocabulary.py
index 63904b6..8dc17ea 100644
--- a/src/zope/schema/tests/test_vocabulary.py
+++ b/src/zope/schema/tests/test_vocabulary.py
@@ -58,6 +58,13 @@ class SimpleTermTests(unittest.TestCase):
         self.assertEqual(term.token, 'term')
         self.assertFalse(ITitledTokenizedTerm.providedBy(term))
 
+    def test_bytes_non_ascii_value(self):
+        from zope.schema.interfaces import ITitledTokenizedTerm
+        term = self._makeOne(b'Snowman \xe2\x98\x83')
+        self.assertEqual(term.value, b'Snowman \xe2\x98\x83')
+        self.assertEqual(term.token, 'Snowman \\xe2\\x98\\x83')
+        self.assertFalse(ITitledTokenizedTerm.providedBy(term))
+
     def test_unicode_non_ascii_value(self):
         from zope.schema.interfaces import ITitledTokenizedTerm
         term = self._makeOne(u'Snowman \u2603')
diff --git a/src/zope/schema/vocabulary.py b/src/zope/schema/vocabulary.py
index a7bbc63..eba0d4b 100644
--- a/src/zope/schema/vocabulary.py
+++ b/src/zope/schema/vocabulary.py
@@ -55,7 +55,9 @@ class SimpleTerm(object):
         # we want here. On the other hand, we want to try to keep the token as
         # readable as possible. On both 2 and 3, self.token should be a native
         # string (ASCIILine).
-        if not isinstance(token, (str, bytes, text_type)):
+        if isinstance(token, bytes):
+            token = token.decode('raw_unicode_escape')
+        elif not isinstance(token, (str, text_type)):
             # Nothing we recognize as intended to be textual data.
             # Get its str() as promised
             token = str(token)