diff options
| -rw-r--r-- | distbuild/serialise.py | 144 |
1 files changed, 118 insertions, 26 deletions
diff --git a/distbuild/serialise.py b/distbuild/serialise.py index 060833b1..cd871042 100644 --- a/distbuild/serialise.py +++ b/distbuild/serialise.py @@ -19,6 +19,7 @@ import json import morphlib +import logging morphology_attributes = [ @@ -36,6 +37,16 @@ def serialise_artifact(artifact): for x in morphology_attributes: result['__%s' % x] = getattr(morphology, x) return result + + def encode_artifact(artifact): + return { + 'name': artifact.name, + 'cache_id': artifact.cache_id, + 'cache_key': artifact.cache_key, + 'dependencies': artifact.dependencies, + 'dependents': artifact.dependents, + 'metadata_version': artifact.metadata_version, + } def encode_source(source): source_dic = { @@ -46,25 +57,35 @@ def serialise_artifact(artifact): 'tree': source.tree, 'morphology': encode_morphology(source.morphology), 'filename': source.filename, + + # dict keys are converted to strings by json + # so we encode the artifact ids as strings + 'artifact_ids': [str(id(artifact)) for (_, artifact) + in source.artifacts.iteritems()], } + if source.morphology['kind'] == 'chunk': source_dic['build_mode'] = source.build_mode source_dic['prefix'] = source.prefix return source_dic - def encode_single_artifact(a, encoded): + def encode_single_artifact(a, artifacts, source_id): if artifact.source.morphology['kind'] == 'system': arch = artifact.source.morphology['arch'] else: arch = artifact.arch + + logging.debug('encode_single_artifact dependencies: %s' + % str([('id: %s' % str(id(d)), d.name) for d in a.dependencies])) + return { - 'source': encode_source(a.source), + 'source_id': source_id, 'name': a.name, 'cache_id': a.cache_id, 'cache_key': a.cache_key, - 'dependencies': [encoded[d.cache_key]['cache_key'] - for d in a.dependencies], - 'arch': arch, + 'dependencies': [str(id(artifacts[id(d)])) + for d in a.dependencies], + 'arch': arch } visited = set() @@ -77,13 +98,46 @@ def serialise_artifact(artifact): yield ret yield a - encoded = {} + + artifacts = {} + encoded_artifacts = {} + encoded_sources = {} + for a in traverse(artifact): - if a.cache_key not in encoded: - encoded[a.cache_key] = encode_single_artifact(a, encoded) + logging.debug('traversing artifacts at %s' % a.name) + + if id(a.source) not in encoded_sources: + if a.source.morphology['kind'] == 'chunk': + for (_, sa) in a.source.artifacts.iteritems(): + if id(sa) not in artifacts: + logging.debug('encoding source artifact %s' % sa.name) + artifacts[id(sa)] = sa + encoded_artifacts[id(sa)] = encode_single_artifact(sa, + artifacts, id(a.source)) + else: + # We create separate sources for strata and systems, + # this is a bit of a hack, but needed to allow + # us to build strata and systems independently + + s = a.source + t = morphlib.source.Source(s.repo_name, s.original_ref, + s.sha1, s.tree, s.morphology, s.filename) + + t.artifacts = {a.name: a} + a.source = t + + encoded_sources[id(a.source)] = encode_source(a.source) + + if id(a) not in artifacts: + artifacts[id(a)] = a + logging.debug('encoding artifact %s' % a.name) + encoded_artifacts[id(a)] = encode_single_artifact(a, artifacts, + id(a.source)) - encoded['_root'] = artifact.cache_key - return json.dumps(encoded) + encoded_artifacts['_root'] = str(id(artifact)) + + return json.dumps({'sources': encoded_sources, + 'artifacts': encoded_artifacts}) def deserialise_artifact(encoded): @@ -121,7 +175,17 @@ def deserialise_artifact(encoded): setattr(morphology, x, le_dict['__%s' % x]) del morphology['__%s' % x] return morphology - + + def unserialise_source_artifacts(source, artifacts_dict): + '''Convert this dict into a list of artifacts''' + return {a['name']: Artifact(source, + a['name'], + a['cache_id'], + a['cache_key'], + a['dependencies'], + a['dependents'], + a['metadata_version']) for a in artifacts_dict} + def unserialise_source(le_dict): '''Convert a dict into a Source object.''' @@ -132,35 +196,63 @@ def deserialise_artifact(encoded): le_dict['tree'], morphology, le_dict['filename']) + if morphology['kind'] == 'chunk': source.build_mode = le_dict['build_mode'] source.prefix = le_dict['prefix'] return source - def unserialise_single_artifact(le_dict): + def unserialise_single_artifact(artifact_dict, source): '''Convert dict into an Artifact object. Do not set dependencies, that will be dealt with later. ''' - source = unserialise_source(le_dict['source']) - artifact = morphlib.artifact.Artifact(source, le_dict['name']) - artifact.cache_id = le_dict['cache_id'] - artifact.cache_key = le_dict['cache_key'] - artifact.arch = le_dict['arch'] + artifact = morphlib.artifact.Artifact(source, artifact_dict['name']) + artifact.cache_id = artifact_dict['cache_id'] + artifact.cache_key = artifact_dict['cache_key'] + artifact.arch = artifact_dict['arch'] + artifact.source = source + return artifact le_dicts = json.loads(encoded) - cache_keys = [k for k in le_dicts.keys() if k != '_root'] + artifacts_dict = le_dicts['artifacts'] + sources_dict = le_dicts['sources'] + + artifact_ids = ([artifacts_dict['_root']] + + filter(lambda k: k != '_root', artifacts_dict.keys())) + + source_ids = [sid for sid in sources_dict.keys()] + artifacts = {} - for cache_key in cache_keys: - le_dict = le_dicts[cache_key] - artifacts[cache_key] = unserialise_single_artifact(le_dict) - for cache_key in cache_keys: - le_dict = le_dicts[cache_key] - artifact = artifacts[cache_key] - artifact.dependencies = [artifacts[k] for k in le_dict['dependencies']] + sources = {} + + for source_id in source_ids: + source_dict = sources_dict[source_id] + sources[source_id] = unserialise_source(source_dict) + + # clear the source artifacts that get automatically generated + # we want to add the ones that were sent to us + sources[source_id].artifacts = {} + source_artifacts = source_dict['artifact_ids'] + + for artifact_id in source_artifacts: + if artifact_id not in artifacts: + artifact_dict = artifacts_dict[artifact_id] + artifact = unserialise_single_artifact(artifact_dict, + sources[source_id]) + + artifacts[artifact_id] = artifact + + key = artifacts[artifact_id].name + sources[source_id].artifacts[key] = artifacts[artifact_id] - return artifacts[le_dicts['_root']] + # now add the dependencies + for artifact_id in artifact_ids: + artifact = artifacts[artifact_id] + artifact.dependencies = [artifacts[aid] for aid in + artifacts_dict[artifact_id]['dependencies']] + return artifacts[artifacts_dict['_root']] |