diff options
author | Lars Wirzenius <lars.wirzenius@codethink.co.uk> | 2014-07-15 12:23:49 +0000 |
---|---|---|
committer | Lars Wirzenius <lars.wirzenius@codethink.co.uk> | 2014-07-15 12:23:49 +0000 |
commit | e8adedb8f3f27d9212caf277b8e8f7c6792a20c2 (patch) | |
tree | f8460c9ff73cea780d944f18d1bab49f2653b1cf /morphlib | |
parent | 3f686a074fdb66e75e67e276d21bdc604b34920c (diff) | |
parent | 40a5c752de62dd1b3ef3ec0850f1f24e7af67052 (diff) | |
download | morph-e8adedb8f3f27d9212caf277b8e8f7c6792a20c2.tar.gz |
Fix JSON file handling to be binary filename safe
Reviewed-by: Lars Wirzenius
Reviewed-by: Pedro Alvarez
Diffstat (limited to 'morphlib')
-rw-r--r-- | morphlib/artifact.py | 10 | ||||
-rw-r--r-- | morphlib/builder2.py | 21 | ||||
-rw-r--r-- | morphlib/localartifactcache.py | 15 | ||||
-rw-r--r-- | morphlib/morph2.py | 5 | ||||
-rw-r--r-- | morphlib/plugins/deploy_plugin.py | 3 | ||||
-rw-r--r-- | morphlib/systemmetadatadir.py | 7 |
6 files changed, 39 insertions, 22 deletions
diff --git a/morphlib/artifact.py b/morphlib/artifact.py index 20fdb185..da6d3763 100644 --- a/morphlib/artifact.py +++ b/morphlib/artifact.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012, 2013 Codethink Limited +# Copyright (C) 2012, 2013, 2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -56,13 +56,13 @@ class Artifact(object): def basename(self): # pragma: no cover return '%s.%s.%s' % (self.cache_key, - self.source.morphology['kind'], - self.name) + str(self.source.morphology['kind']), + str(self.name)) def metadata_basename(self, metadata_name): # pragma: no cover return '%s.%s.%s.%s' % (self.cache_key, - self.source.morphology['kind'], - self.name, + str(self.source.morphology['kind']), + str(self.name), metadata_name) def get_dependency_prefix_set(self): diff --git a/morphlib/builder2.py b/morphlib/builder2.py index 3c0d9e02..4bb435d9 100644 --- a/morphlib/builder2.py +++ b/morphlib/builder2.py @@ -154,7 +154,8 @@ def get_chunk_files(f): # pragma: no cover def get_stratum_files(f, lac): # pragma: no cover - for ca in (ArtifactCacheReference(a) for a in json.load(f)): + for ca in (ArtifactCacheReference(a) + for a in json.load(f, encoding='unicode-escape')): cf = lac.get(ca) for filename in get_chunk_files(cf): yield filename @@ -197,7 +198,7 @@ def write_overlap_metadata(artifact, overlaps, lac): # pragma: no cover [ [a.name for a in afs], list(files) ] for afs, files in overlaps.iteritems() - ], f, indent=4) + ], f, indent=4, encoding='unicode-escape') f.close() @@ -234,7 +235,8 @@ class BuilderBase(object): with self.local_artifact_cache.put_source_metadata( self.artifact.source, self.artifact.cache_key, 'meta') as f: - json.dump(meta, f, indent=4, sort_keys=True) + json.dump(meta, f, indent=4, sort_keys=True, + encoding='unicode-escape') f.write('\n') def create_metadata(self, artifact_name, contents=[]): @@ -294,7 +296,7 @@ class BuilderBase(object): # Unit tests use StringIO, which in Python 2.6 isn't usable with # the "with" statement. So we don't do it with "with". f = self._open(filename, 'w') - f.write(json.dumps(meta, indent=4, sort_keys=True)) + json.dump(meta, f, indent=4, sort_keys=True, encoding='unicode-escape') f.close() def new_artifact(self, artifact_name): @@ -580,9 +582,11 @@ class StratumBuilder(BuilderBase): meta = self.create_metadata(self.artifact.name, [x.name for x in constituents]) with lac.put_artifact_metadata(self.artifact, 'meta') as f: - json.dump(meta, f, indent=4, sort_keys=True) + json.dump(meta, f, indent=4, sort_keys=True, + encoding='unicode-escape') with self.local_artifact_cache.put(self.artifact) as f: - json.dump([c.basename() for c in constituents], f) + json.dump([c.basename() for c in constituents], f, + encoding='unicode-escape') self.save_build_times() return [self.artifact] @@ -643,7 +647,7 @@ class SystemBuilder(BuilderBase): # pragma: no cover cache = self.local_artifact_cache with cache.get(stratum_artifact) as stratum_file: - artifact_list = json.load(stratum_file) + artifact_list = json.load(stratum_file, encoding='unicode-escape') for chunk in (ArtifactCacheReference(a) for a in artifact_list): self.app.status(msg='Unpacking chunk %(basename)s', basename=chunk.basename(), chatty=True) @@ -671,7 +675,8 @@ class SystemBuilder(BuilderBase): # pragma: no cover # download the chunk artifacts if necessary for stratum_artifact in self.artifact.dependencies: f = self.local_artifact_cache.get(stratum_artifact) - chunks = [ArtifactCacheReference(a) for a in json.load(f)] + chunks = [ArtifactCacheReference(a) + for a in json.load(f, encoding='unicode-escape')] download_depends(chunks, self.local_artifact_cache, self.remote_artifact_cache) diff --git a/morphlib/localartifactcache.py b/morphlib/localartifactcache.py index 4c7f7832..955ee97f 100644 --- a/morphlib/localartifactcache.py +++ b/morphlib/localartifactcache.py @@ -96,14 +96,23 @@ class LocalArtifactCache(object): os.utime(filename, None) return open(filename) + def _join(self, basename): + '''Wrapper for pyfilesystem's getsyspath. + + This is required because its API throws us a garbage unicode + string, when file paths are binary data. + ''' + return str(self.cachefs.getsyspath(basename)) + def artifact_filename(self, artifact): - return self.cachefs.getsyspath(artifact.basename()) + basename = artifact.basename() + return self._join(basename) def _artifact_metadata_filename(self, artifact, name): - return self.cachefs.getsyspath(artifact.metadata_basename(name)) + return self._join(artifact.metadata_basename(name)) def _source_metadata_filename(self, source, cachekey, name): - return self.cachefs.getsyspath('%s.%s' % (cachekey, name)) + return self._join('%s.%s' % (cachekey, name)) def clear(self): '''Clear everything from the artifact cache directory. diff --git a/morphlib/morph2.py b/morphlib/morph2.py index cc6ce926..83971bb8 100644 --- a/morphlib/morph2.py +++ b/morphlib/morph2.py @@ -66,11 +66,12 @@ class Morphology(object): @staticmethod def _load_json(text): - return json.loads(text, object_pairs_hook=OrderedDict) + return json.loads(text, object_pairs_hook=OrderedDict, + encoding='unicode-escape') @staticmethod def _dump_json(obj, f): - text = json.dumps(obj, indent=4) + text = json.dumps(obj, indent=4, encoding='unicode-escape') text = re.sub(" \n", "\n", text) f.write(text) f.write('\n') diff --git a/morphlib/plugins/deploy_plugin.py b/morphlib/plugins/deploy_plugin.py index 9384c422..30e356e8 100644 --- a/morphlib/plugins/deploy_plugin.py +++ b/morphlib/plugins/deploy_plugin.py @@ -497,7 +497,8 @@ class DeployPlugin(cliapp.Plugin): metadata_path = os.path.join( system_tree, 'baserock', 'deployment.meta') with morphlib.savefile.SaveFile(metadata_path, 'w') as f: - f.write(json.dumps(metadata, indent=4, sort_keys=True)) + json.dump(metadata, f, indent=4, + sort_keys=True, encoding='unicode-escape') return system_tree except Exception: shutil.rmtree(system_tree) diff --git a/morphlib/systemmetadatadir.py b/morphlib/systemmetadatadir.py index eac5b446..7e89142c 100644 --- a/morphlib/systemmetadatadir.py +++ b/morphlib/systemmetadatadir.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -66,14 +66,15 @@ class SystemMetadataDir(collections.MutableMapping): self._check_key(key) try: with open(self._join_path('%s.meta' % key), 'r') as f: - return json.load(f) + return json.load(f, encoding='unicode-escape') except IOError: raise KeyError(key) def __setitem__(self, key, value): self._check_key(key) with open(self._join_path('%s.meta' % key), 'w') as f: - json.dump(value, f, indent=4, sort_keys=True) + json.dump(value, f, indent=4, sort_keys=True, + encoding='unicode-escape') def __delitem__(self, key): self._check_key(key) |