path: root/morphlib
diff options
authorRichard Maw <>2014-07-11 15:09:14 (GMT)
committerRichard Maw <>2014-07-11 15:09:14 (GMT)
commit1a5e3e748a5ea4f48e3e88fa3859db4c186d6ba5 (patch)
treea3748d413eaee6a24dffd6246aa4ec106fc090bf /morphlib
parentce1fedb4e5ab82105853c4f3a8e05fb83f62c18e (diff)
Make our use of json binary path safebaserock/richardmaw/bugfix/unicode-safe-json
json only accepts unicode. Various APIs such as file paths and environment variables allow binary data, so we need to support this properly. This patch changes every[1] use of json.load or json.dump to escape non-unicode data strings. This appears exactly as it used to if the input was valid unicode, if it isn't it will insert \xabcd escapes in the place of non-unicode data. When loading back in, if json.load is told to unescape it with `encoding='unicode-escape'` then it will convert it back correctly. This change was primarily to support file paths that weren't valid unicode, where this would choke and die. Now it works, but any tools that parsed the metadata need to unescape the paths. [1]: The interface to the remote repo cache uses json data, but I haven't changes its json.load calls to unescape the data, since the repo caches haven't been made to escape the data.
Diffstat (limited to 'morphlib')
4 files changed, 22 insertions, 14 deletions
diff --git a/morphlib/ b/morphlib/
index 3c0d9e0..4bb435d 100644
--- a/morphlib/
+++ b/morphlib/
@@ -154,7 +154,8 @@ def get_chunk_files(f): # pragma: no cover
def get_stratum_files(f, lac): # pragma: no cover
- for ca in (ArtifactCacheReference(a) for a in json.load(f)):
+ for ca in (ArtifactCacheReference(a)
+ for a in json.load(f, encoding='unicode-escape')):
cf = lac.get(ca)
for filename in get_chunk_files(cf):
yield filename
@@ -197,7 +198,7 @@ def write_overlap_metadata(artifact, overlaps, lac): # pragma: no cover
[ for a in afs], list(files)
] for afs, files in overlaps.iteritems()
- ], f, indent=4)
+ ], f, indent=4, encoding='unicode-escape')
@@ -234,7 +235,8 @@ class BuilderBase(object):
with self.local_artifact_cache.put_source_metadata(
self.artifact.source, self.artifact.cache_key,
'meta') as f:
- json.dump(meta, f, indent=4, sort_keys=True)
+ json.dump(meta, f, indent=4, sort_keys=True,
+ encoding='unicode-escape')
def create_metadata(self, artifact_name, contents=[]):
@@ -294,7 +296,7 @@ class BuilderBase(object):
# Unit tests use StringIO, which in Python 2.6 isn't usable with
# the "with" statement. So we don't do it with "with".
f = self._open(filename, 'w')
- f.write(json.dumps(meta, indent=4, sort_keys=True))
+ json.dump(meta, f, indent=4, sort_keys=True, encoding='unicode-escape')
def new_artifact(self, artifact_name):
@@ -580,9 +582,11 @@ class StratumBuilder(BuilderBase):
meta = self.create_metadata(,
[ for x in constituents])
with lac.put_artifact_metadata(self.artifact, 'meta') as f:
- json.dump(meta, f, indent=4, sort_keys=True)
+ json.dump(meta, f, indent=4, sort_keys=True,
+ encoding='unicode-escape')
with self.local_artifact_cache.put(self.artifact) as f:
- json.dump([c.basename() for c in constituents], f)
+ json.dump([c.basename() for c in constituents], f,
+ encoding='unicode-escape')
return [self.artifact]
@@ -643,7 +647,7 @@ class SystemBuilder(BuilderBase): # pragma: no cover
cache = self.local_artifact_cache
with cache.get(stratum_artifact) as stratum_file:
- artifact_list = json.load(stratum_file)
+ artifact_list = json.load(stratum_file, encoding='unicode-escape')
for chunk in (ArtifactCacheReference(a) for a in artifact_list):'Unpacking chunk %(basename)s',
basename=chunk.basename(), chatty=True)
@@ -671,7 +675,8 @@ class SystemBuilder(BuilderBase): # pragma: no cover
# download the chunk artifacts if necessary
for stratum_artifact in self.artifact.dependencies:
f = self.local_artifact_cache.get(stratum_artifact)
- chunks = [ArtifactCacheReference(a) for a in json.load(f)]
+ chunks = [ArtifactCacheReference(a)
+ for a in json.load(f, encoding='unicode-escape')]
diff --git a/morphlib/ b/morphlib/
index cc6ce92..83971bb 100644
--- a/morphlib/
+++ b/morphlib/
@@ -66,11 +66,12 @@ class Morphology(object):
def _load_json(text):
- return json.loads(text, object_pairs_hook=OrderedDict)
+ return json.loads(text, object_pairs_hook=OrderedDict,
+ encoding='unicode-escape')
def _dump_json(obj, f):
- text = json.dumps(obj, indent=4)
+ text = json.dumps(obj, indent=4, encoding='unicode-escape')
text = re.sub(" \n", "\n", text)
diff --git a/morphlib/plugins/ b/morphlib/plugins/
index 9384c42..30e356e 100644
--- a/morphlib/plugins/
+++ b/morphlib/plugins/
@@ -497,7 +497,8 @@ class DeployPlugin(cliapp.Plugin):
metadata_path = os.path.join(
system_tree, 'baserock', 'deployment.meta')
with morphlib.savefile.SaveFile(metadata_path, 'w') as f:
- f.write(json.dumps(metadata, indent=4, sort_keys=True))
+ json.dump(metadata, f, indent=4,
+ sort_keys=True, encoding='unicode-escape')
return system_tree
except Exception:
diff --git a/morphlib/ b/morphlib/
index eac5b44..7e89142 100644
--- a/morphlib/
+++ b/morphlib/
@@ -1,4 +1,4 @@
-# Copyright (C) 2013 Codethink Limited
+# Copyright (C) 2013-2014 Codethink Limited
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -66,14 +66,15 @@ class SystemMetadataDir(collections.MutableMapping):
with open(self._join_path('%s.meta' % key), 'r') as f:
- return json.load(f)
+ return json.load(f, encoding='unicode-escape')
except IOError:
raise KeyError(key)
def __setitem__(self, key, value):
with open(self._join_path('%s.meta' % key), 'w') as f:
- json.dump(value, f, indent=4, sort_keys=True)
+ json.dump(value, f, indent=4, sort_keys=True,
+ encoding='unicode-escape')
def __delitem__(self, key):