Properly read py2 object metadata on py3

Replicated, unencrypted metadata is written down differently on py2
vs py3, and has been since we started supporting py3. Fortunately,
we can inspect the raw xattr bytes to determine whether the pickle
was written using py2 or py3, so we can properly read legacy py2 meta
under py3 rather than hitting a unicode error.

Closes-Bug: #2012531
Change-Id: I5876e3b88f0bb1224299b57541788f590f64ddd4

4 files changed, 256 insertions, 33 deletions

diff --git a/swift/cli/info.py b/swift/cli/info.py
index 7826a17b8..d99fb3b19 100644
--- a/swift/cli/info.py
+++ b/swift/cli/info.py
@@ -30,6 +30,7 @@ from swift.container.backend import ContainerBroker, DATADIR as CBDATADIR
 from swift.obj.diskfile import get_data_dir, read_metadata, DATADIR_BASE, \
     extract_policy
 from swift.common.storage_policy import POLICIES
+from swift.common.swob import wsgi_to_str
 from swift.common.middleware.crypto.crypto_utils import load_crypto_meta
 from swift.common.utils import md5
 
@@ -537,6 +538,8 @@ def print_obj(datafile, check_etag=True, swift_dir='/etc/swift',
         except EOFError:
             print("Invalid metadata")
             raise InfoSystemExit()
+        metadata = {wsgi_to_str(k): v if k == 'name' else wsgi_to_str(v)
+                    for k, v in metadata.items()}
 
         etag = metadata.pop('ETag', '')
         length = metadata.pop('Content-Length', '')
diff --git a/swift/obj/diskfile.py b/swift/obj/diskfile.py
index efd897907..8b6b7d01b 100644
--- a/swift/obj/diskfile.py
+++ b/swift/obj/diskfile.py
@@ -167,24 +167,36 @@ def _encode_metadata(metadata):
     return dict(((encode_str(k), encode_str(v)) for k, v in metadata.items()))
 
 
-def _decode_metadata(metadata):
+def _decode_metadata(metadata, metadata_written_by_py3):
     """
     Given a metadata dict from disk, convert keys and values to native strings.
 
     :param metadata: a dict
+    :param metadata_written_by_py3:
     """
     if six.PY2:
-        def to_str(item):
+        def to_str(item, is_name=False):
+            # For years, py2 and py3 handled non-ascii metadata differently;
+            # see https://bugs.launchpad.net/swift/+bug/2012531
+            if metadata_written_by_py3 and not is_name:
+                # do our best to read new-style data replicated from a py3 node
+                item = item.decode('utf8').encode('latin1')
             if isinstance(item, six.text_type):
                 return item.encode('utf8')
             return item
     else:
-        def to_str(item):
+        def to_str(item, is_name=False):
+            # For years, py2 and py3 handled non-ascii metadata differently;
+            # see https://bugs.launchpad.net/swift/+bug/2012531
+            if not metadata_written_by_py3 and isinstance(item, bytes) \
+                    and not is_name:
+                # do our best to read old py2 data
+                item = item.decode('latin1')
             if isinstance(item, six.binary_type):
                 return item.decode('utf8', 'surrogateescape')
             return item
 
-    return dict(((to_str(k), to_str(v)) for k, v in metadata.items()))
+    return {to_str(k): to_str(v, k == b'name') for k, v in metadata.items()}
 
 
 def read_metadata(fd, add_missing_checksum=False):
@@ -238,6 +250,7 @@ def read_metadata(fd, add_missing_checksum=False):
                 "stored checksum='%s', computed='%s'" % (
                     fd, metadata_checksum, computed_checksum))
 
+    metadata_written_by_py3 = (b'_codecs\nencode' in metadata[:32])
     # strings are utf-8 encoded when written, but have not always been
     # (see https://bugs.launchpad.net/swift/+bug/1678018) so encode them again
     # when read
@@ -245,7 +258,7 @@ def read_metadata(fd, add_missing_checksum=False):
         metadata = pickle.loads(metadata)
     else:
         metadata = pickle.loads(metadata, encoding='bytes')
-    return _decode_metadata(metadata)
+    return _decode_metadata(metadata, metadata_written_by_py3)
 
 
 def write_metadata(fd, metadata, xattr_size=65536):
diff --git a/test/unit/cli/test_info.py b/test/unit/cli/test_info.py
index 80813db87..85fb79819 100644
--- a/test/unit/cli/test_info.py
+++ b/test/unit/cli/test_info.py
@@ -1219,7 +1219,7 @@ class TestPrintObjFullMeta(TestCliInfoBase):
             md = {'name': '/AUTH_admin/c/obj',
                   'Content-Type': 'application/octet-stream',
                   'ETag': 'd41d8cd98f00b204e9800998ecf8427e',
-                  'Content-Length': 0}
+                  'Content-Length': '0'}
             write_metadata(fp, md)
 
     def test_print_obj(self):
@@ -1256,7 +1256,7 @@ class TestPrintObjFullMeta(TestCliInfoBase):
             md = {'name': '/AUTH_admin/c/obj',
                   'Content-Type': 'application/octet-stream',
                   'ETag': 'd41d8cd98f00b204e9800998ecf8427e',
-                  'Content-Length': 0}
+                  'Content-Length': '0'}
             write_metadata(fp, md)
 
         object_ring = ring.Ring(self.testdir, ring_name='object-2')
@@ -1297,7 +1297,7 @@ class TestPrintObjFullMeta(TestCliInfoBase):
             md = {'name': '/AUTH_admin/c/obj',
                   'Content-Type': 'application/octet-stream',
                   'ETag': 'd41d8cd98f00b204e9800998ecf8427e',
-                  'Content-Length': 0}
+                  'Content-Length': '0'}
             write_metadata(fp, md)
 
         object_ring = ring.Ring(self.testdir, ring_name='object-3')
@@ -1368,7 +1368,7 @@ class TestPrintObjFullMeta(TestCliInfoBase):
             md = {'name': '/AUTH_admin/c/obj',
                   'Content-Type': 'application/octet-stream',
                   'ETag': 'badetag',
-                  'Content-Length': 0}
+                  'Content-Length': '0'}
             write_metadata(fp, md)
 
         out = StringIO()
@@ -1703,5 +1703,5 @@ class TestPrintObjWeirdPath(TestPrintObjFullMeta):
             md = {'name': '/AUTH_admin/c/obj',
                   'Content-Type': 'application/octet-stream',
                   'ETag': 'd41d8cd98f00b204e9800998ecf8427e',
-                  'Content-Length': 0}
+                  'Content-Length': '0'}
             write_metadata(fp, md)
diff --git a/test/unit/obj/test_diskfile.py b/test/unit/obj/test_diskfile.py
index 327d860e5..8ee4edb45 100644
--- a/test/unit/obj/test_diskfile.py
+++ b/test/unit/obj/test_diskfile.py
@@ -15,6 +15,8 @@
 
 """Tests for swift.obj.diskfile"""
 
+from __future__ import print_function
+import binascii
 import six.moves.cPickle as pickle
 import os
 import errno
@@ -28,6 +30,7 @@ import uuid
 import xattr
 import re
 import six
+import sys
 from collections import defaultdict
 from random import shuffle, randint
 from shutil import rmtree
@@ -554,6 +557,233 @@ class TestDiskFileModuleMethods(unittest.TestCase):
         exp_dir = '/srv/node/sdb5/objects-1/123'
         self.assertEqual(part_dir, exp_dir)
 
+    def test_can_read_old_meta(self):
+        # outputs taken from `xattr -l <diskfile>`
+        cases = {
+            'python_2.7.18_swift_2.13_replicated': '''
+0000   80 02 7D 71 01 28 55 0E 43 6F 6E 74 65 6E 74 2D    ..}q.(U.Content-
+0010   4C 65 6E 67 74 68 71 02 55 02 31 33 55 04 6E 61    Lengthq.U.13U.na
+0020   6D 65 71 03 55 12 2F 41 55 54 48 5F 74 65 73 74    meq.U./AUTH_test
+0030   2F E2 98 83 2F E2 98 83 71 04 55 13 58 2D 4F 62    /.../...q.U.X-Ob
+0040   6A 65 63 74 2D 4D 65 74 61 2D 4D 74 69 6D 65 55    ject-Meta-MtimeU
+0050   11 31 36 38 32 39 35 39 38 37 34 2E 37 35 36 32    .1682959874.7562
+0060   30 35 71 05 55 04 45 54 61 67 71 06 55 20 36 62    05q.U.ETagq.U 6b
+0070   37 64 39 61 31 63 35 64 31 36 37 63 63 35 30 30    7d9a1c5d167cc500
+0080   33 37 66 32 39 66 32 39 30 62 62 33 37 35 71 07    37f29f290bb375q.
+0090   55 0B 58 2D 54 69 6D 65 73 74 61 6D 70 71 08 55    U.X-Timestampq.U
+00A0   10 31 36 38 32 39 36 32 36 35 31 2E 39 37 34 39    .1682962651.9749
+00B0   34 55 11 58 2D 4F 62 6A 65 63 74 2D 4D 65 74 61    4U.X-Object-Meta
+00C0   2D E2 98 83 55 03 E2 98 83 71 09 55 0C 43 6F 6E    -...U....q.U.Con
+00D0   74 65 6E 74 2D 54 79 70 65 71 0A 55 18 61 70 70    tent-Typeq.U.app
+00E0   6C 69 63 61 74 69 6F 6E 2F 6F 63 74 65 74 2D 73    lication/octet-s
+00F0   74 72 65 61 6D 71 0B 75 2E                         treamq.u.
+            ''',
+            'python_2.7.18_swift_2.13_ec': '''
+0000   80 02 7D 71 01 28 55 0E 43 6F 6E 74 65 6E 74 2D    ..}q.(U.Content-
+0010   4C 65 6E 67 74 68 71 02 55 02 38 34 55 04 6E 61    Lengthq.U.84U.na
+0020   6D 65 71 03 55 12 2F 41 55 54 48 5F 74 65 73 74    meq.U./AUTH_test
+0030   2F E2 98 83 2F E2 98 83 71 04 58 1E 00 00 00 58    /.../...q.X....X
+0040   2D 4F 62 6A 65 63 74 2D 53 79 73 6D 65 74 61 2D    -Object-Sysmeta-
+0050   45 63 2D 46 72 61 67 2D 49 6E 64 65 78 71 05 55    Ec-Frag-Indexq.U
+0060   01 35 55 13 58 2D 4F 62 6A 65 63 74 2D 4D 65 74    .5U.X-Object-Met
+0070   61 2D 4D 74 69 6D 65 55 11 31 36 38 32 39 35 39    a-MtimeU.1682959
+0080   38 37 34 2E 37 35 36 32 30 35 71 06 58 22 00 00    874.756205q.X"..
+0090   00 58 2D 4F 62 6A 65 63 74 2D 53 79 73 6D 65 74    .X-Object-Sysmet
+00A0   61 2D 45 63 2D 43 6F 6E 74 65 6E 74 2D 4C 65 6E    a-Ec-Content-Len
+00B0   67 74 68 71 07 55 02 31 33 71 08 58 18 00 00 00    gthq.U.13q.X....
+00C0   58 2D 4F 62 6A 65 63 74 2D 53 79 73 6D 65 74 61    X-Object-Sysmeta
+00D0   2D 45 63 2D 45 74 61 67 71 09 55 20 36 62 37 64    -Ec-Etagq.U 6b7d
+00E0   39 61 31 63 35 64 31 36 37 63 63 35 30 30 33 37    9a1c5d167cc50037
+00F0   66 32 39 66 32 39 30 62 62 33 37 35 71 0A 55 04    f29f290bb375q.U.
+0100   45 54 61 67 71 0B 55 20 65 32 66 64 34 33 30 65    ETagq.U e2fd430e
+0110   61 66 37 32 32 33 63 32 35 30 33 63 34 65 38 33    af7223c2503c4e83
+0120   30 31 63 66 66 33 37 63 71 0C 55 0B 58 2D 54 69    01cff37cq.U.X-Ti
+0130   6D 65 73 74 61 6D 70 71 0D 55 10 31 36 38 32 39    mestampq.U.16829
+0140   36 32 32 36 32 2E 31 36 31 39 39 55 11 58 2D 4F    62262.16199U.X-O
+0150   62 6A 65 63 74 2D 4D 65 74 61 2D E2 98 83 55 03    bject-Meta-...U.
+0160   E2 98 83 71 0E 58 1A 00 00 00 58 2D 4F 62 6A 65    ...q.X....X-Obje
+0170   63 74 2D 53 79 73 6D 65 74 61 2D 45 63 2D 53 63    ct-Sysmeta-Ec-Sc
+0180   68 65 6D 65 71 0F 55 1A 6C 69 62 65 72 61 73 75    hemeq.U.liberasu
+0190   72 65 63 6F 64 65 5F 72 73 5F 76 61 6E 64 20 34    recode_rs_vand 4
+01A0   2B 32 71 10 55 0C 43 6F 6E 74 65 6E 74 2D 54 79    +2q.U.Content-Ty
+01B0   70 65 71 11 55 18 61 70 70 6C 69 63 61 74 69 6F    peq.U.applicatio
+01C0   6E 2F 6F 63 74 65 74 2D 73 74 72 65 61 6D 71 12    n/octet-streamq.
+01D0   58 20 00 00 00 58 2D 4F 62 6A 65 63 74 2D 53 79    X ...X-Object-Sy
+01E0   73 6D 65 74 61 2D 45 63 2D 53 65 67 6D 65 6E 74    smeta-Ec-Segment
+01F0   2D 53 69 7A 65 71 13 55 07 31 30 34 38 35 37 36    -Sizeq.U.1048576
+0200   71 14 75 2E                                        q.u.
+            ''',
+            'python_2.7.18_swift_2.23_replicated': '''
+0000   80 02 7D 71 01 28 55 0E 43 6F 6E 74 65 6E 74 2D    ..}q.(U.Content-
+0010   4C 65 6E 67 74 68 71 02 55 02 31 33 71 03 55 04    Lengthq.U.13q.U.
+0020   6E 61 6D 65 71 04 55 12 2F 41 55 54 48 5F 74 65    nameq.U./AUTH_te
+0030   73 74 2F E2 98 83 2F E2 98 83 71 05 55 0C 43 6F    st/.../...q.U.Co
+0040   6E 74 65 6E 74 2D 54 79 70 65 71 06 55 18 61 70    ntent-Typeq.U.ap
+0050   70 6C 69 63 61 74 69 6F 6E 2F 6F 63 74 65 74 2D    plication/octet-
+0060   73 74 72 65 61 6D 71 07 55 04 45 54 61 67 71 08    streamq.U.ETagq.
+0070   55 20 36 62 37 64 39 61 31 63 35 64 31 36 37 63    U 6b7d9a1c5d167c
+0080   63 35 30 30 33 37 66 32 39 66 32 39 30 62 62 33    c50037f29f290bb3
+0090   37 35 71 09 55 0B 58 2D 54 69 6D 65 73 74 61 6D    75q.U.X-Timestam
+00A0   70 71 0A 55 10 31 36 38 32 39 36 33 32 30 39 2E    pq.U.1682963209.
+00B0   38 32 32 37 32 71 0B 55 11 58 2D 4F 62 6A 65 63    82272q.U.X-Objec
+00C0   74 2D 4D 65 74 61 2D E2 98 83 71 0C 55 03 E2 98    t-Meta-...q.U...
+00D0   83 71 0D 55 13 58 2D 4F 62 6A 65 63 74 2D 4D 65    .q.U.X-Object-Me
+00E0   74 61 2D 4D 74 69 6D 65 71 0E 55 11 31 36 38 32    ta-Mtimeq.U.1682
+00F0   39 35 39 38 37 34 2E 37 35 36 32 30 35 71 0F 75    959874.756205q.u
+0100   2E                                                 .
+            ''',
+            'python_3.10.6_swift_2.23_replicated': '''
+0000   80 02 7D 71 00 28 63 5F 63 6F 64 65 63 73 0A 65    ..}q.(c_codecs.e
+0010   6E 63 6F 64 65 0A 71 01 58 0B 00 00 00 58 2D 54    ncode.q.X....X-T
+0020   69 6D 65 73 74 61 6D 70 71 02 58 06 00 00 00 6C    imestampq.X....l
+0030   61 74 69 6E 31 71 03 86 71 04 52 71 05 68 01 58    atin1q..q.Rq.h.X
+0040   10 00 00 00 31 36 38 32 39 36 33 30 31 37 2E 31    ....1682963017.1
+0050   30 34 37 32 71 06 68 03 86 71 07 52 71 08 68 01    0472q.h..q.Rq.h.
+0060   58 0C 00 00 00 43 6F 6E 74 65 6E 74 2D 54 79 70    X....Content-Typ
+0070   65 71 09 68 03 86 71 0A 52 71 0B 68 01 58 18 00    eq.h..q.Rq.h.X..
+0080   00 00 61 70 70 6C 69 63 61 74 69 6F 6E 2F 6F 63    ..application/oc
+0090   74 65 74 2D 73 74 72 65 61 6D 71 0C 68 03 86 71    tet-streamq.h..q
+00A0   0D 52 71 0E 68 01 58 0E 00 00 00 43 6F 6E 74 65    .Rq.h.X....Conte
+00B0   6E 74 2D 4C 65 6E 67 74 68 71 0F 68 03 86 71 10    nt-Lengthq.h..q.
+00C0   52 71 11 68 01 58 02 00 00 00 31 33 71 12 68 03    Rq.h.X....13q.h.
+00D0   86 71 13 52 71 14 68 01 58 04 00 00 00 45 54 61    .q.Rq.h.X....ETa
+00E0   67 71 15 68 03 86 71 16 52 71 17 68 01 58 20 00    gq.h..q.Rq.h.X .
+00F0   00 00 36 62 37 64 39 61 31 63 35 64 31 36 37 63    ..6b7d9a1c5d167c
+0100   63 35 30 30 33 37 66 32 39 66 32 39 30 62 62 33    c50037f29f290bb3
+0110   37 35 71 18 68 03 86 71 19 52 71 1A 68 01 58 13    75q.h..q.Rq.h.X.
+0120   00 00 00 58 2D 4F 62 6A 65 63 74 2D 4D 65 74 61    ...X-Object-Meta
+0130   2D 4D 74 69 6D 65 71 1B 68 03 86 71 1C 52 71 1D    -Mtimeq.h..q.Rq.
+0140   68 01 58 11 00 00 00 31 36 38 32 39 35 39 38 37    h.X....168295987
+0150   34 2E 37 35 36 32 30 35 71 1E 68 03 86 71 1F 52    4.756205q.h..q.R
+0160   71 20 68 01 58 1A 00 00 00 58 2D 4F 62 6A 65 63    q h.X....X-Objec
+0170   74 2D 4D 65 74 61 2D C3 83 C2 A2 C3 82 C2 98 C3    t-Meta-.........
+0180   82 C2 83 71 21 68 03 86 71 22 52 71 23 68 01 58    ...q!h..q"Rq#h.X
+0190   0C 00 00 00 C3 83 C2 A2 C3 82 C2 98 C3 82 C2 83    ................
+01A0   71 24 68 03 86 71 25 52 71 26 68 01 58 04 00 00    q$h..q%Rq&h.X...
+01B0   00 6E 61 6D 65 71 27 68 03 86 71 28 52 71 29 68    .nameq'h..q(Rq)h
+01C0   01 58 18 00 00 00 2F 41 55 54 48 5F 74 65 73 74    .X..../AUTH_test
+01D0   2F C3 A2 C2 98 C2 83 2F C3 A2 C2 98 C2 83 71 2A    /....../......q*
+01E0   68 03 86 71 2B 52 71 2C 75 2E                      h..q+Rq,u.
+            ''',
+            'python_2.7.18_swift_2.23_ec': '''
+0000   80 02 7D 71 01 28 55 0E 43 6F 6E 74 65 6E 74 2D    ..}q.(U.Content-
+0010   4C 65 6E 67 74 68 71 02 55 02 38 34 71 03 55 04    Lengthq.U.84q.U.
+0020   6E 61 6D 65 71 04 55 12 2F 41 55 54 48 5F 74 65    nameq.U./AUTH_te
+0030   73 74 2F E2 98 83 2F E2 98 83 71 05 55 1E 58 2D    st/.../...q.U.X-
+0040   4F 62 6A 65 63 74 2D 53 79 73 6D 65 74 61 2D 45    Object-Sysmeta-E
+0050   63 2D 46 72 61 67 2D 49 6E 64 65 78 55 01 35 55    c-Frag-IndexU.5U
+0060   0C 43 6F 6E 74 65 6E 74 2D 54 79 70 65 71 06 55    .Content-Typeq.U
+0070   18 61 70 70 6C 69 63 61 74 69 6F 6E 2F 6F 63 74    .application/oct
+0080   65 74 2D 73 74 72 65 61 6D 71 07 55 22 58 2D 4F    et-streamq.U"X-O
+0090   62 6A 65 63 74 2D 53 79 73 6D 65 74 61 2D 45 63    bject-Sysmeta-Ec
+00A0   2D 43 6F 6E 74 65 6E 74 2D 4C 65 6E 67 74 68 55    -Content-LengthU
+00B0   02 31 33 71 08 55 18 58 2D 4F 62 6A 65 63 74 2D    .13q.U.X-Object-
+00C0   53 79 73 6D 65 74 61 2D 45 63 2D 45 74 61 67 55    Sysmeta-Ec-EtagU
+00D0   20 36 62 37 64 39 61 31 63 35 64 31 36 37 63 63     6b7d9a1c5d167cc
+00E0   35 30 30 33 37 66 32 39 66 32 39 30 62 62 33 37    50037f29f290bb37
+00F0   35 71 09 55 04 45 54 61 67 71 0A 55 20 65 32 66    5q.U.ETagq.U e2f
+0100   64 34 33 30 65 61 66 37 32 32 33 63 32 35 30 33    d430eaf7223c2503
+0110   63 34 65 38 33 30 31 63 66 66 33 37 63 71 0B 55    c4e8301cff37cq.U
+0120   0B 58 2D 54 69 6D 65 73 74 61 6D 70 71 0C 55 10    .X-Timestampq.U.
+0130   31 36 38 32 39 36 33 31 33 30 2E 33 35 39 38 36    1682963130.35986
+0140   71 0D 55 11 58 2D 4F 62 6A 65 63 74 2D 4D 65 74    q.U.X-Object-Met
+0150   61 2D E2 98 83 71 0E 55 03 E2 98 83 71 0F 55 1A    a-...q.U....q.U.
+0160   58 2D 4F 62 6A 65 63 74 2D 53 79 73 6D 65 74 61    X-Object-Sysmeta
+0170   2D 45 63 2D 53 63 68 65 6D 65 55 1A 6C 69 62 65    -Ec-SchemeU.libe
+0180   72 61 73 75 72 65 63 6F 64 65 5F 72 73 5F 76 61    rasurecode_rs_va
+0190   6E 64 20 34 2B 32 71 10 55 13 58 2D 4F 62 6A 65    nd 4+2q.U.X-Obje
+01A0   63 74 2D 4D 65 74 61 2D 4D 74 69 6D 65 71 11 55    ct-Meta-Mtimeq.U
+01B0   11 31 36 38 32 39 35 39 38 37 34 2E 37 35 36 32    .1682959874.7562
+01C0   30 35 71 12 55 20 58 2D 4F 62 6A 65 63 74 2D 53    05q.U X-Object-S
+01D0   79 73 6D 65 74 61 2D 45 63 2D 53 65 67 6D 65 6E    ysmeta-Ec-Segmen
+01E0   74 2D 53 69 7A 65 55 07 31 30 34 38 35 37 36 71    t-SizeU.1048576q
+01F0   13 75 2E                                           .u.
+            ''',
+            'python_3.10.6_swift_2.23_ec': '''
+0000   80 02 7D 71 00 28 63 5F 63 6F 64 65 63 73 0A 65    ..}q.(c_codecs.e
+0010   6E 63 6F 64 65 0A 71 01 58 0B 00 00 00 58 2D 54    ncode.q.X....X-T
+0020   69 6D 65 73 74 61 6D 70 71 02 58 06 00 00 00 6C    imestampq.X....l
+0030   61 74 69 6E 31 71 03 86 71 04 52 71 05 68 01 58    atin1q..q.Rq.h.X
+0040   10 00 00 00 31 36 38 32 39 36 32 39 35 35 2E 33    ....1682962955.3
+0050   37 35 34 36 71 06 68 03 86 71 07 52 71 08 68 01    7546q.h..q.Rq.h.
+0060   58 0C 00 00 00 43 6F 6E 74 65 6E 74 2D 54 79 70    X....Content-Typ
+0070   65 71 09 68 03 86 71 0A 52 71 0B 68 01 58 18 00    eq.h..q.Rq.h.X..
+0080   00 00 61 70 70 6C 69 63 61 74 69 6F 6E 2F 6F 63    ..application/oc
+0090   74 65 74 2D 73 74 72 65 61 6D 71 0C 68 03 86 71    tet-streamq.h..q
+00A0   0D 52 71 0E 68 01 58 0E 00 00 00 43 6F 6E 74 65    .Rq.h.X....Conte
+00B0   6E 74 2D 4C 65 6E 67 74 68 71 0F 68 03 86 71 10    nt-Lengthq.h..q.
+00C0   52 71 11 68 01 58 02 00 00 00 38 34 71 12 68 03    Rq.h.X....84q.h.
+00D0   86 71 13 52 71 14 68 01 58 04 00 00 00 45 54 61    .q.Rq.h.X....ETa
+00E0   67 71 15 68 03 86 71 16 52 71 17 68 01 58 20 00    gq.h..q.Rq.h.X .
+00F0   00 00 65 32 66 64 34 33 30 65 61 66 37 32 32 33    ..e2fd430eaf7223
+0100   63 32 35 30 33 63 34 65 38 33 30 31 63 66 66 33    c2503c4e8301cff3
+0110   37 63 71 18 68 03 86 71 19 52 71 1A 68 01 58 13    7cq.h..q.Rq.h.X.
+0120   00 00 00 58 2D 4F 62 6A 65 63 74 2D 4D 65 74 61    ...X-Object-Meta
+0130   2D 4D 74 69 6D 65 71 1B 68 03 86 71 1C 52 71 1D    -Mtimeq.h..q.Rq.
+0140   68 01 58 11 00 00 00 31 36 38 32 39 35 39 38 37    h.X....168295987
+0150   34 2E 37 35 36 32 30 35 71 1E 68 03 86 71 1F 52    4.756205q.h..q.R
+0160   71 20 68 01 58 1A 00 00 00 58 2D 4F 62 6A 65 63    q h.X....X-Objec
+0170   74 2D 4D 65 74 61 2D C3 83 C2 A2 C3 82 C2 98 C3    t-Meta-.........
+0180   82 C2 83 71 21 68 03 86 71 22 52 71 23 68 01 58    ...q!h..q"Rq#h.X
+0190   0C 00 00 00 C3 83 C2 A2 C3 82 C2 98 C3 82 C2 83    ................
+01A0   71 24 68 03 86 71 25 52 71 26 68 01 58 18 00 00    q$h..q%Rq&h.X...
+01B0   00 58 2D 4F 62 6A 65 63 74 2D 53 79 73 6D 65 74    .X-Object-Sysmet
+01C0   61 2D 45 63 2D 45 74 61 67 71 27 68 03 86 71 28    a-Ec-Etagq'h..q(
+01D0   52 71 29 68 01 58 20 00 00 00 36 62 37 64 39 61    Rq)h.X ...6b7d9a
+01E0   31 63 35 64 31 36 37 63 63 35 30 30 33 37 66 32    1c5d167cc50037f2
+01F0   39 66 32 39 30 62 62 33 37 35 71 2A 68 03 86 71    9f290bb375q*h..q
+0200   2B 52 71 2C 68 01 58 22 00 00 00 58 2D 4F 62 6A    +Rq,h.X"...X-Obj
+0210   65 63 74 2D 53 79 73 6D 65 74 61 2D 45 63 2D 43    ect-Sysmeta-Ec-C
+0220   6F 6E 74 65 6E 74 2D 4C 65 6E 67 74 68 71 2D 68    ontent-Lengthq-h
+0230   03 86 71 2E 52 71 2F 68 01 58 02 00 00 00 31 33    ..q.Rq/h.X....13
+0240   71 30 68 03 86 71 31 52 71 32 68 01 58 1E 00 00    q0h..q1Rq2h.X...
+0250   00 58 2D 4F 62 6A 65 63 74 2D 53 79 73 6D 65 74    .X-Object-Sysmet
+0260   61 2D 45 63 2D 46 72 61 67 2D 49 6E 64 65 78 71    a-Ec-Frag-Indexq
+0270   33 68 03 86 71 34 52 71 35 68 01 58 01 00 00 00    3h..q4Rq5h.X....
+0280   35 71 36 68 03 86 71 37 52 71 38 68 01 58 1A 00    5q6h..q7Rq8h.X..
+0290   00 00 58 2D 4F 62 6A 65 63 74 2D 53 79 73 6D 65    ..X-Object-Sysme
+02A0   74 61 2D 45 63 2D 53 63 68 65 6D 65 71 39 68 03    ta-Ec-Schemeq9h.
+02B0   86 71 3A 52 71 3B 68 01 58 1A 00 00 00 6C 69 62    .q:Rq;h.X....lib
+02C0   65 72 61 73 75 72 65 63 6F 64 65 5F 72 73 5F 76    erasurecode_rs_v
+02D0   61 6E 64 20 34 2B 32 71 3C 68 03 86 71 3D 52 71    and 4+2q<h..q=Rq
+02E0   3E 68 01 58 20 00 00 00 58 2D 4F 62 6A 65 63 74    >h.X ...X-Object
+02F0   2D 53 79 73 6D 65 74 61 2D 45 63 2D 53 65 67 6D    -Sysmeta-Ec-Segm
+0300   65 6E 74 2D 53 69 7A 65 71 3F 68 03 86 71 40 52    ent-Sizeq?h..q@R
+0310   71 41 68 01 58 07 00 00 00 31 30 34 38 35 37 36    qAh.X....1048576
+0320   71 42 68 03 86 71 43 52 71 44 68 01 58 04 00 00    qBh..qCRqDh.X...
+0330   00 6E 61 6D 65 71 45 68 03 86 71 46 52 71 47 68    .nameqEh..qFRqGh
+0340   01 58 18 00 00 00 2F 41 55 54 48 5F 74 65 73 74    .X..../AUTH_test
+0350   2F C3 A2 C2 98 C2 83 2F C3 A2 C2 98 C2 83 71 48    /....../......qH
+0360   68 03 86 71 49 52 71 4A 75 2E                      h..qIRqJu.
+            ''',
+        }
+
+        def raw_xattr(output):
+            return binascii.unhexlify(''.join(
+                line[7:55] for line in output.split('\n')
+            ).replace(' ', ''))
+
+        path = os.path.join(self.testdir, str(uuid.uuid4()))
+        for case, xattr_output in cases.items():
+            try:
+                to_write = raw_xattr(xattr_output)
+                with open(path, 'wb') as fp:
+                    xattr.setxattr(
+                        fp.fileno(), 'user.swift.metadata', to_write)
+                with open(path, 'rb') as fd:
+                    actual = diskfile.read_metadata(fd)
+                # name should come out as native strings
+                expected_name = b'/AUTH_test/\xe2\x98\x83/\xe2\x98\x83'
+                if not six.PY2:
+                    expected_name = expected_name.decode('utf8')
+                self.assertEqual(actual['name'], expected_name)
+                # other meta will be WSGI strings, though
+                self.assertEqual(
+                    actual['X-Object-Meta-\xe2\x98\x83'], '\xe2\x98\x83')
+            except Exception:
+                print('Failure in %s' % case, file=sys.stderr)
+                raise
+
     def test_write_read_metadata(self):
         path = os.path.join(self.testdir, str(uuid.uuid4()))
         metadata = {'name': '/a/c/o',
@@ -597,29 +827,6 @@ class TestDiskFileModuleMethods(unittest.TestCase):
             diskfile.write_metadata(fd, metadata)
         check_metadata(as_native, str)
 
-        # mock the read path to check the write path encoded persisted metadata
-        with mock.patch.object(diskfile, '_decode_metadata', lambda x: x):
-            check_metadata(as_bytes, bytes)
-
-        # simulate a legacy diskfile that might have persisted
-        # (some) unicode metadata
-        with mock.patch.object(diskfile, '_encode_metadata', lambda x: x):
-            with open(path, 'wb') as fd:
-                diskfile.write_metadata(fd, metadata)
-        # sanity check: mock read path again to see that we did persist unicode
-        with mock.patch.object(diskfile, '_decode_metadata', lambda x: x):
-            with open(path, 'rb') as fd:
-                actual = diskfile.read_metadata(fd)
-                for k, v in actual.items():
-                    if isinstance(k, six.text_type) and \
-                            k == u'X-Object-Meta-Strange':
-                        self.assertIsInstance(v, six.text_type)
-                        break
-                else:
-                    self.fail('Did not find X-Object-Meta-Strange')
-        # check that read_metadata converts binary_type
-        check_metadata(as_native, str)
-
 
 @patch_policies
 class TestObjectAuditLocationGenerator(unittest.TestCase):