diff options
Diffstat (limited to 'distbuild/serialise.py')
| -rw-r--r-- | distbuild/serialise.py | 258 |
1 files changed, 258 insertions, 0 deletions
diff --git a/distbuild/serialise.py b/distbuild/serialise.py new file mode 100644 index 00000000..cd871042 --- /dev/null +++ b/distbuild/serialise.py @@ -0,0 +1,258 @@ +# distbuild/serialise.py -- (de)serialise Artifact object graphs +# +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.. + + +import json + +import morphlib +import logging + + +morphology_attributes = [ + 'needs_artifact_metadata_cached', +] + + +def serialise_artifact(artifact): + '''Serialise an Artifact object and its dependencies into string form.''' + + def encode_morphology(morphology): + result = {} + for key in morphology.keys(): + result[key] = morphology[key] + for x in morphology_attributes: + result['__%s' % x] = getattr(morphology, x) + return result + + def encode_artifact(artifact): + return { + 'name': artifact.name, + 'cache_id': artifact.cache_id, + 'cache_key': artifact.cache_key, + 'dependencies': artifact.dependencies, + 'dependents': artifact.dependents, + 'metadata_version': artifact.metadata_version, + } + + def encode_source(source): + source_dic = { + 'repo': None, + 'repo_name': source.repo_name, + 'original_ref': source.original_ref, + 'sha1': source.sha1, + 'tree': source.tree, + 'morphology': encode_morphology(source.morphology), + 'filename': source.filename, + + # dict keys are converted to strings by json + # so we encode the artifact ids as strings + 'artifact_ids': [str(id(artifact)) for (_, artifact) + in source.artifacts.iteritems()], + } + + if source.morphology['kind'] == 'chunk': + source_dic['build_mode'] = source.build_mode + source_dic['prefix'] = source.prefix + return source_dic + + def encode_single_artifact(a, artifacts, source_id): + if artifact.source.morphology['kind'] == 'system': + arch = artifact.source.morphology['arch'] + else: + arch = artifact.arch + + logging.debug('encode_single_artifact dependencies: %s' + % str([('id: %s' % str(id(d)), d.name) for d in a.dependencies])) + + return { + 'source_id': source_id, + 'name': a.name, + 'cache_id': a.cache_id, + 'cache_key': a.cache_key, + 'dependencies': [str(id(artifacts[id(d)])) + for d in a.dependencies], + 'arch': arch + } + + visited = set() + def traverse(a): + visited.add(a) + for dep in a.dependencies: + if dep in visited: + continue + for ret in traverse(dep): + yield ret + yield a + + + artifacts = {} + encoded_artifacts = {} + encoded_sources = {} + + for a in traverse(artifact): + logging.debug('traversing artifacts at %s' % a.name) + + if id(a.source) not in encoded_sources: + if a.source.morphology['kind'] == 'chunk': + for (_, sa) in a.source.artifacts.iteritems(): + if id(sa) not in artifacts: + logging.debug('encoding source artifact %s' % sa.name) + artifacts[id(sa)] = sa + encoded_artifacts[id(sa)] = encode_single_artifact(sa, + artifacts, id(a.source)) + else: + # We create separate sources for strata and systems, + # this is a bit of a hack, but needed to allow + # us to build strata and systems independently + + s = a.source + t = morphlib.source.Source(s.repo_name, s.original_ref, + s.sha1, s.tree, s.morphology, s.filename) + + t.artifacts = {a.name: a} + a.source = t + + encoded_sources[id(a.source)] = encode_source(a.source) + + if id(a) not in artifacts: + artifacts[id(a)] = a + logging.debug('encoding artifact %s' % a.name) + encoded_artifacts[id(a)] = encode_single_artifact(a, artifacts, + id(a.source)) + + encoded_artifacts['_root'] = str(id(artifact)) + + return json.dumps({'sources': encoded_sources, + 'artifacts': encoded_artifacts}) + + +def deserialise_artifact(encoded): + '''Re-construct the Artifact object (and dependencies). + + The argument should be a string returned by ``serialise_artifact``. + The reconstructed Artifact objects will be sufficiently like the + originals that they can be used as a build graph, and other such + purposes, by Morph. + + ''' + + def unserialise_morphology(le_dict): + '''Convert a dict into something that kinda acts like a Morphology. + + As it happens, we don't need the full Morphology so we cheat. + Cheating is good. + + ''' + + class FakeMorphology(dict): + + def get_commands(self, which): + '''Get commands to run from a morphology or build system''' + if self[which] is None: + attr = '_'.join(which.split('-')) + bs = morphlib.buildsystem.lookup_build_system( + self['build-system']) + return getattr(bs, attr) + else: + return self[which] + + morphology = FakeMorphology(le_dict) + for x in morphology_attributes: + setattr(morphology, x, le_dict['__%s' % x]) + del morphology['__%s' % x] + return morphology + + def unserialise_source_artifacts(source, artifacts_dict): + '''Convert this dict into a list of artifacts''' + return {a['name']: Artifact(source, + a['name'], + a['cache_id'], + a['cache_key'], + a['dependencies'], + a['dependents'], + a['metadata_version']) for a in artifacts_dict} + + def unserialise_source(le_dict): + '''Convert a dict into a Source object.''' + + morphology = unserialise_morphology(le_dict['morphology']) + source = morphlib.source.Source(le_dict['repo_name'], + le_dict['original_ref'], + le_dict['sha1'], + le_dict['tree'], + morphology, + le_dict['filename']) + + if morphology['kind'] == 'chunk': + source.build_mode = le_dict['build_mode'] + source.prefix = le_dict['prefix'] + return source + + def unserialise_single_artifact(artifact_dict, source): + '''Convert dict into an Artifact object. + + Do not set dependencies, that will be dealt with later. + + ''' + + artifact = morphlib.artifact.Artifact(source, artifact_dict['name']) + artifact.cache_id = artifact_dict['cache_id'] + artifact.cache_key = artifact_dict['cache_key'] + artifact.arch = artifact_dict['arch'] + artifact.source = source + + return artifact + + le_dicts = json.loads(encoded) + artifacts_dict = le_dicts['artifacts'] + sources_dict = le_dicts['sources'] + + artifact_ids = ([artifacts_dict['_root']] + + filter(lambda k: k != '_root', artifacts_dict.keys())) + + source_ids = [sid for sid in sources_dict.keys()] + + artifacts = {} + sources = {} + + for source_id in source_ids: + source_dict = sources_dict[source_id] + sources[source_id] = unserialise_source(source_dict) + + # clear the source artifacts that get automatically generated + # we want to add the ones that were sent to us + sources[source_id].artifacts = {} + source_artifacts = source_dict['artifact_ids'] + + for artifact_id in source_artifacts: + if artifact_id not in artifacts: + artifact_dict = artifacts_dict[artifact_id] + artifact = unserialise_single_artifact(artifact_dict, + sources[source_id]) + + artifacts[artifact_id] = artifact + + key = artifacts[artifact_id].name + sources[source_id].artifacts[key] = artifacts[artifact_id] + + # now add the dependencies + for artifact_id in artifact_ids: + artifact = artifacts[artifact_id] + artifact.dependencies = [artifacts[aid] for aid in + artifacts_dict[artifact_id]['dependencies']] + + return artifacts[artifacts_dict['_root']] |