diff options
Diffstat (limited to 'distbuild/serialise.py')
| -rw-r--r-- | distbuild/serialise.py | 223 |
1 files changed, 105 insertions, 118 deletions
diff --git a/distbuild/serialise.py b/distbuild/serialise.py index d410b6cf..a7c6c4b9 100644 --- a/distbuild/serialise.py +++ b/distbuild/serialise.py @@ -17,16 +17,12 @@ import json +import yaml import morphlib import logging -morphology_attributes = [ - 'needs_artifact_metadata_cached', -] - - def serialise_artifact(artifact): '''Serialise an Artifact object and its dependencies into string form.''' @@ -34,93 +30,88 @@ def serialise_artifact(artifact): result = {} for key in morphology.keys(): result[key] = morphology[key] - for x in morphology_attributes: - result['__%s' % x] = getattr(morphology, x) return result - def encode_source(source): + def encode_source(source, prune_leaf=False): source_dic = { + 'name': source.name, 'repo': None, 'repo_name': source.repo_name, 'original_ref': source.original_ref, 'sha1': source.sha1, 'tree': source.tree, - 'morphology': encode_morphology(source.morphology), + 'morphology': id(source.morphology), 'filename': source.filename, - - # dict keys are converted to strings by json - # so we encode the artifact ids as strings - 'artifact_ids': [str(id(artifact)) for (_, artifact) - in source.artifacts.iteritems()], + 'artifact_ids': [], + 'cache_id': source.cache_id, + 'cache_key': source.cache_key, + 'dependencies': [], } + if not prune_leaf: + source_dic['artifact_ids'].extend(id(artifact) for (_, artifact) + in source.artifacts.iteritems()) + source_dic['dependencies'].extend(id(d) + for d in source.dependencies) if source.morphology['kind'] == 'chunk': source_dic['build_mode'] = source.build_mode source_dic['prefix'] = source.prefix return source_dic - def encode_artifact(a, artifacts, source_id): - if artifact.source.morphology['kind'] == 'system': + def encode_artifact(a): + if artifact.source.morphology['kind'] == 'system': # pragma: no cover arch = artifact.source.morphology['arch'] else: arch = artifact.arch return { - 'source_id': source_id, + 'source_id': id(a.source), 'name': a.name, - 'cache_id': a.cache_id, - 'cache_key': a.cache_key, - 'dependencies': [str(id(artifacts[id(d)])) - for d in a.dependencies], - 'arch': arch + 'arch': arch, + 'dependents': [id(d) + for d in a.dependents], } - visited = set() - def traverse(a): - visited.add(a) - for dep in a.dependencies: - if dep in visited: - continue - for ret in traverse(dep): - yield ret - yield a - - - artifacts = {} encoded_artifacts = {} encoded_sources = {} + encoded_morphologies = {} + visited_artifacts = {} - for a in traverse(artifact): + for a in artifact.walk(): if id(a.source) not in encoded_sources: - if a.source.morphology['kind'] == 'chunk': - for (_, sa) in a.source.artifacts.iteritems(): - if id(sa) not in artifacts: - artifacts[id(sa)] = sa - encoded_artifacts[id(sa)] = encode_artifact(sa, - artifacts, id(a.source)) - else: - # We create separate sources for strata and systems, - # this is a bit of a hack, but needed to allow - # us to build strata and systems independently - - s = a.source - t = morphlib.source.Source(s.repo_name, s.original_ref, - s.sha1, s.tree, s.morphology, s.filename) - - t.artifacts = {a.name: a} - a.source = t - + for sa in a.source.artifacts.itervalues(): + if id(sa) not in encoded_artifacts: + visited_artifacts[id(sa)] = sa + encoded_artifacts[id(sa)] = encode_artifact(sa) + encoded_morphologies[id(a.source.morphology)] = \ + encode_morphology(a.source.morphology) encoded_sources[id(a.source)] = encode_source(a.source) - if id(a) not in artifacts: - artifacts[id(a)] = a - encoded_artifacts[id(a)] = encode_artifact(a, artifacts, - id(a.source)) - - encoded_artifacts['_root'] = str(id(artifact)) - - return json.dumps({'sources': encoded_sources, - 'artifacts': encoded_artifacts}) + if id(a) not in encoded_artifacts: # pragma: no cover + visited_artifacts[id(a)] = a + encoded_artifacts[id(a)] = encode_artifact(a) + + # Include one level of dependents above encoded artifacts, as we need + # them to be able to tell whether two sources are in the same stratum. + for a in visited_artifacts.itervalues(): + for source in a.dependents: # pragma: no cover + if id(source) not in encoded_sources: + encoded_morphologies[id(source.morphology)] = \ + encode_morphology(source.morphology) + encoded_sources[id(source)] = \ + encode_source(source, prune_leaf=True) + + content = { + 'sources': encoded_sources, + 'artifacts': encoded_artifacts, + 'morphologies': encoded_morphologies, + 'root_artifact': id(artifact), + 'default_split_rules': { + 'chunk': morphlib.artifactsplitrule.DEFAULT_CHUNK_RULES, + 'stratum': morphlib.artifactsplitrule.DEFAULT_STRATUM_RULES, + }, + } + return json.dumps(yaml.dump(content)) def deserialise_artifact(encoded): @@ -141,38 +132,25 @@ def deserialise_artifact(encoded): ''' - class FakeMorphology(dict): - - def get_commands(self, which): - '''Get commands to run from a morphology or build system''' - if self[which] is None: - attr = '_'.join(which.split('-')) - bs = morphlib.buildsystem.lookup_build_system( - self['build-system']) - return getattr(bs, attr) - else: - return self[which] - - morphology = FakeMorphology(le_dict) - for x in morphology_attributes: - setattr(morphology, x, le_dict['__%s' % x]) - del morphology['__%s' % x] - return morphology - - def decode_source(le_dict): + return morphlib.morphology.Morphology(le_dict) + + def decode_source(le_dict, morphology, split_rules): '''Convert a dict into a Source object.''' - morphology = decode_morphology(le_dict['morphology']) - source = morphlib.source.Source(le_dict['repo_name'], + source = morphlib.source.Source(le_dict['name'], + le_dict['repo_name'], le_dict['original_ref'], le_dict['sha1'], le_dict['tree'], morphology, - le_dict['filename']) + le_dict['filename'], + split_rules) if morphology['kind'] == 'chunk': source.build_mode = le_dict['build_mode'] source.prefix = le_dict['prefix'] + source.cache_id = le_dict['cache_id'] + source.cache_key = le_dict['cache_key'] return source def decode_artifact(artifact_dict, source): @@ -183,48 +161,57 @@ def deserialise_artifact(encoded): ''' artifact = morphlib.artifact.Artifact(source, artifact_dict['name']) - artifact.cache_id = artifact_dict['cache_id'] - artifact.cache_key = artifact_dict['cache_key'] artifact.arch = artifact_dict['arch'] artifact.source = source return artifact - le_dicts = json.loads(encoded) + le_dicts = yaml.load(json.loads(encoded)) artifacts_dict = le_dicts['artifacts'] sources_dict = le_dicts['sources'] - - artifact_ids = ([artifacts_dict['_root']] + - filter(lambda k: k != '_root', artifacts_dict.keys())) - - source_ids = [sid for sid in sources_dict.keys()] + morphologies_dict = le_dicts['morphologies'] + root_artifact = le_dicts['root_artifact'] + assert root_artifact in artifacts_dict artifacts = {} sources = {} - - for source_id in source_ids: + morphologies = {id: decode_morphology(d) + for (id, d) in morphologies_dict.iteritems()} + + # Decode sources + for source_id, source_dict in sources_dict.iteritems(): + morphology = morphologies[source_dict['morphology']] + kind = morphology['kind'] + ruler = getattr(morphlib.artifactsplitrule, 'unify_%s_matches' % kind) + if kind in ('chunk', 'stratum'): + rules = ruler(morphology, le_dicts['default_split_rules'][kind]) + else: # pragma: no cover + rules = ruler(morphology) + sources[source_id] = decode_source(source_dict, morphology, rules) + + # decode artifacts + for artifact_id, artifact_dict in artifacts_dict.iteritems(): + source_id = artifact_dict['source_id'] + source = sources[source_id] + artifact = decode_artifact(artifact_dict, source) + artifacts[artifact_id] = artifact + + # add source artifacts reference + for source_id, source in sources.iteritems(): source_dict = sources_dict[source_id] - sources[source_id] = decode_source(source_dict) - - # clear the source artifacts that get automatically generated - # we want to add the ones that were sent to us - sources[source_id].artifacts = {} - source_artifacts = source_dict['artifact_ids'] - - for artifact_id in source_artifacts: - if artifact_id not in artifacts: - artifact_dict = artifacts_dict[artifact_id] - artifact = decode_artifact(artifact_dict, sources[source_id]) - - artifacts[artifact_id] = artifact - - key = artifacts[artifact_id].name - sources[source_id].artifacts[key] = artifacts[artifact_id] - - # now add the dependencies - for artifact_id in artifact_ids: - artifact = artifacts[artifact_id] - artifact.dependencies = [artifacts[aid] for aid in - artifacts_dict[artifact_id]['dependencies']] - - return artifacts[artifacts_dict['_root']] + source.artifacts = {artifacts[a].name: artifacts[a] + for a in source_dict['artifact_ids']} + + # add source dependencies + for source_id, source_dict in sources_dict.iteritems(): + source = sources[source_id] + source.dependencies = [artifacts[aid] + for aid in source_dict['dependencies']] + + # add artifact dependents + for artifact_id, artifact in artifacts.iteritems(): + artifact_dict = artifacts_dict[artifact_id] + artifact.dependents = [sources[sid] + for sid in artifact_dict['dependents']] + + return artifacts[root_artifact] |