diff options
Diffstat (limited to 'morphlib')
-rw-r--r-- | morphlib/app.py | 85 | ||||
-rw-r--r-- | morphlib/artifactresolver.py | 4 | ||||
-rw-r--r-- | morphlib/buildcommand.py | 5 | ||||
-rw-r--r-- | morphlib/builder2.py | 2 | ||||
-rw-r--r-- | morphlib/cachekeycomputer.py | 28 | ||||
-rw-r--r-- | morphlib/cachekeycomputer_tests.py | 9 | ||||
-rw-r--r-- | morphlib/morph2.py | 4 | ||||
-rw-r--r-- | morphlib/morph2_tests.py | 1 | ||||
-rw-r--r-- | morphlib/morphloader.py | 31 | ||||
-rw-r--r-- | morphlib/morphloader_tests.py | 30 | ||||
-rw-r--r-- | morphlib/morphologyfactory.py | 2 | ||||
-rw-r--r-- | morphlib/plugins/deploy_plugin.py | 10 | ||||
-rw-r--r-- | morphlib/plugins/distbuild_plugin.py | 2 | ||||
-rw-r--r-- | morphlib/plugins/list_artifacts_plugin.py | 2 | ||||
-rw-r--r-- | morphlib/plugins/update_gits_plugin.py | 83 | ||||
-rw-r--r-- | morphlib/yamlparse.py | 2 |
16 files changed, 154 insertions, 146 deletions
diff --git a/morphlib/app.py b/morphlib/app.py index e0874317..a543443e 100644 --- a/morphlib/app.py +++ b/morphlib/app.py @@ -286,7 +286,7 @@ class Morph(cliapp.Application): morphlib.util.sanitise_morphology_path(args[2])) args = args[3:] - def create_source_pool(self, lrc, rrc, triplet): + def create_source_pool(self, lrc, rrc, repo, ref, filename): pool = morphlib.sourcepool.SourcePool() def add_to_pool(reponame, ref, filename, absref, tree, morphology): @@ -294,7 +294,7 @@ class Morph(cliapp.Application): morphology, filename) pool.add(source) - self.traverse_morphs([triplet], lrc, rrc, + self.traverse_morphs(repo, ref, [filename], lrc, rrc, update=not self.settings['no-git-update'], visit=add_to_pool) return pool @@ -338,53 +338,74 @@ class Morph(cliapp.Application): absref, tree = repo.resolve_ref(ref) return absref, tree - def traverse_morphs(self, triplets, lrc, rrc, update=True, + def traverse_morphs(self, definitions_repo, definitions_ref, + system_filenames, lrc, rrc, update=True, visit=lambda rn, rf, fn, arf, m: None): morph_factory = morphlib.morphologyfactory.MorphologyFactory(lrc, rrc, self) - queue = collections.deque(triplets) + definitions_queue = collections.deque(system_filenames) + chunk_in_definitions_repo_queue = [] + chunk_in_source_repo_queue = [] resolved_refs = {} resolved_morphologies = {} - while queue: - reponame, ref, filename = queue.popleft() + # Resolve the (repo, ref) pair for the definitions repo, cache result. + definitions_absref, definitions_tree = self.resolve_ref( + lrc, rrc, definitions_repo, definitions_ref, update) - # Resolve the (repo, ref) reference, cache result. - reference = (reponame, ref) - if not reference in resolved_refs: - resolved_refs[reference] = self.resolve_ref( - lrc, rrc, reponame, ref, update) - absref, tree = resolved_refs[reference] + while definitions_queue: + filename = definitions_queue.popleft() - # Fetch the (repo, ref, filename) morphology, cache result. - reference = (reponame, absref, filename) - if not reference in resolved_morphologies: - resolved_morphologies[reference] = \ - morph_factory.get_morphology(reponame, absref, filename) - morphology = resolved_morphologies[reference] + key = (definitions_repo, definitions_absref, filename) + if not key in resolved_morphologies: + resolved_morphologies[key] = morph_factory.get_morphology(*key) + morphology = resolved_morphologies[key] - visit(reponame, ref, filename, absref, tree, morphology) + visit(definitions_repo, definitions_ref, filename, + definitions_absref, definitions_tree, morphology) if morphology['kind'] == 'cluster': raise cliapp.AppException( "Cannot build a morphology of type 'cluster'.") elif morphology['kind'] == 'system': - queue.extend( - (s.get('repo') or reponame, - s.get('ref') or ref, - morphlib.util.sanitise_morphology_path(s['morph'])) + definitions_queue.extend( + morphlib.util.sanitise_morphology_path(s['morph']) for s in morphology['strata']) elif morphology['kind'] == 'stratum': if morphology['build-depends']: - queue.extend( - (s.get('repo') or reponame, - s.get('ref') or ref, - morphlib.util.sanitise_morphology_path(s['morph'])) + definitions_queue.extend( + morphlib.util.sanitise_morphology_path(s['morph']) for s in morphology['build-depends']) - queue.extend( - (c['repo'], - c['ref'], - morphlib.util.sanitise_morphology_path(c['morph'])) - for c in morphology['chunks']) + for c in morphology['chunks']: + if 'morph' not in c: + path = morphlib.util.sanitise_morphology_path( + c.get('morph', c['name'])) + chunk_in_source_repo_queue.append( + (c['repo'], c['ref'], path)) + continue + chunk_in_definitions_repo_queue.append( + (c['repo'], c['ref'], c['morph'])) + + for repo, ref, filename in chunk_in_definitions_repo_queue: + if (repo, ref) not in resolved_refs: + resolved_refs[repo, ref] = self.resolve_ref( + lrc, rrc, repo, ref, update) + absref, tree = resolved_refs[repo, ref] + key = (definitions_repo, definitions_absref, filename) + if not key in resolved_morphologies: + resolved_morphologies[key] = morph_factory.get_morphology(*key) + morphology = resolved_morphologies[key] + visit(repo, ref, filename, absref, tree, morphology) + + for repo, ref, filename in chunk_in_source_repo_queue: + if (repo, ref) not in resolved_refs: + resolved_refs[repo, ref] = self.resolve_ref( + lrc, rrc, repo, ref, update) + absref, tree = resolved_refs[repo, ref] + key = (repo, absref, filename) + if key not in resolved_morphologies: + resolved_morphologies[key] = morph_factory.get_morphology(*key) + morphology = resolved_morphologies[key] + visit(repo, ref, filename, absref, tree, morphology) def cache_repo_and_submodules(self, cache, url, ref, done): subs_to_process = set() diff --git a/morphlib/artifactresolver.py b/morphlib/artifactresolver.py index c18042a3..9e3cea48 100644 --- a/morphlib/artifactresolver.py +++ b/morphlib/artifactresolver.py @@ -191,10 +191,12 @@ class ArtifactResolver(object): name_to_processed_artifacts = {} for info in source.morphology['chunks']: + filename = morphlib.util.sanitise_morphology_path( + info.get('morph', info['name'])) chunk_source = self._source_pool.lookup( info['repo'], info['ref'], - morphlib.util.sanitise_morphology_path(info['morph'])) + filename) chunk_name = chunk_source.morphology['name'] diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py index 45c6ef00..1cb0c4d9 100644 --- a/morphlib/buildcommand.py +++ b/morphlib/buildcommand.py @@ -90,7 +90,7 @@ class BuildCommand(object): ''' self.app.status(msg='Creating source pool', chatty=True) srcpool = self.app.create_source_pool( - self.lrc, self.rrc, (repo_name, ref, filename)) + self.lrc, self.rrc, repo_name, ref, filename) return srcpool @@ -231,7 +231,8 @@ class BuildCommand(object): for spec in specs: repo_name = spec.get('repo') or src.repo_name ref = spec.get('ref') or src.original_ref - filename = morphlib.util.sanitise_morphology_path(spec['morph']) + filename = morphlib.util.sanitise_morphology_path( + spec.get('morph', spec.get('name'))) logging.debug( 'Validating cross ref to %s:%s:%s' % (repo_name, ref, filename)) diff --git a/morphlib/builder2.py b/morphlib/builder2.py index 4bb435d9..d739dc13 100644 --- a/morphlib/builder2.py +++ b/morphlib/builder2.py @@ -488,7 +488,7 @@ class ChunkBuilder(BuilderBase): def filepaths(destdir): for dirname, subdirs, basenames in os.walk(destdir): subdirsymlinks = [os.path.join(dirname, x) for x in subdirs - if os.path.islink(x)] + if os.path.islink(os.path.join(dirname, x))] filenames = [os.path.join(dirname, x) for x in basenames] for relpath in (os.path.relpath(x, destdir) for x in [dirname] + subdirsymlinks + filenames): diff --git a/morphlib/cachekeycomputer.py b/morphlib/cachekeycomputer.py index ca374436..292047c8 100644 --- a/morphlib/cachekeycomputer.py +++ b/morphlib/cachekeycomputer.py @@ -93,7 +93,6 @@ class CacheKeyComputer(object): def _calculate(self, artifact): keys = { 'env': self._filterenv(self._build_env.env), - 'filename': artifact.source.filename, 'kids': [{'artifact': a.name, 'cache-key': self.compute_key(a)} for a in artifact.dependencies], 'metadata-version': artifact.metadata_version @@ -106,9 +105,24 @@ class CacheKeyComputer(object): keys['tree'] = artifact.source.tree keys['split-rules'] = [(a, [rgx.pattern for rgx in r._regexes]) for (a, r) in artifact.source.split_rules] + + # Include morphology contents, since it doesn't always come + # from the source tree + morphology = artifact.source.morphology + # include {pre-,,post-}{configure,build,test,install}-commands + # in morphology key + for prefix in ('pre-', '', 'post-'): + for cmdtype in ('configure', 'build', 'test', 'install'): + cmd_field = prefix + cmdtype + '-commands' + keys[cmd_field] = morphology.get_commands(cmd_field) + keys['devices'] = morphology.get('devices') + keys['max-jobs'] = morphology.get('max-jobs') + keys['system-integration'] = morphology.get('system-integration', + {}) + # products is omitted as they are part of the split-rules elif kind in ('system', 'stratum'): morphology = artifact.source.morphology - le_dict = dict((k, morphology[k]) for k in morphology.keys()) + morph_dict = dict((k, morphology[k]) for k in morphology.keys()) # Disregard all fields of a morphology that aren't important ignored_fields = ( @@ -117,13 +131,9 @@ class CacheKeyComputer(object): # so are already handled by the 'kids' field. 'strata', 'build-depends', 'chunks', 'products') - for ignored_field in ignored_fields: - if ignored_field in le_dict: - del le_dict[ignored_field] - - checksum = hashlib.sha1() - self._hash_thing(checksum, le_dict) - keys['morphology-sha1'] = checksum.hexdigest() + for key in morph_dict: + if key not in ignored_fields: + keys[key] = morph_dict[key] if kind == 'stratum': keys['stratum-format-version'] = 1 elif kind == 'system': diff --git a/morphlib/cachekeycomputer_tests.py b/morphlib/cachekeycomputer_tests.py index dd10307f..9e18b19d 100644 --- a/morphlib/cachekeycomputer_tests.py +++ b/morphlib/cachekeycomputer_tests.py @@ -36,15 +36,18 @@ class CacheKeyComputerTests(unittest.TestCase): for name, text in { 'chunk.morph': '''{ "name": "chunk", - "kind": "chunk" + "kind": "chunk", + "description": "A test chunk" }''', 'chunk2.morph': '''{ "name": "chunk2", - "kind": "chunk" + "kind": "chunk", + "description": "A test chunk" }''', 'chunk3.morph': '''{ "name": "chunk3", - "kind": "chunk" + "kind": "chunk", + "description": "A test chunk" }''', 'stratum.morph': '''{ "name": "stratum", diff --git a/morphlib/morph2.py b/morphlib/morph2.py index 83971bb8..b49c0f73 100644 --- a/morphlib/morph2.py +++ b/morphlib/morph2.py @@ -191,8 +191,6 @@ class Morphology(object): for source in self['chunks']: if 'repo' not in source: self._set_default_value(source, 'repo', source['name']) - if 'morph' not in source: - self._set_default_value(source, 'morph', source['name']) if 'build-depends' not in source: self._set_default_value(source, 'build-depends', None) if 'build-mode' not in source: @@ -226,7 +224,7 @@ class Morphology(object): return info elif self['kind'] == 'stratum': for info in self['chunks']: - source_name = info.get('alias', info['morph']) + source_name = info.get('alias', info['name']) if source_name == name: return info raise KeyError('"%s" not found' % name) diff --git a/morphlib/morph2_tests.py b/morphlib/morph2_tests.py index d02aab1c..c9957ad5 100644 --- a/morphlib/morph2_tests.py +++ b/morphlib/morph2_tests.py @@ -95,7 +95,6 @@ class MorphologyTests(unittest.TestCase): ''') self.assertEqual(m['chunks'][0]['repo'], 'le-chunk') - self.assertEqual(m['chunks'][0]['morph'], 'le-chunk') self.assertEqual(m['chunks'][0]['build-depends'], None) def test_returns_dict_keys(self): diff --git a/morphlib/morphloader.py b/morphlib/morphloader.py index 45416a19..21e10827 100644 --- a/morphlib/morphloader.py +++ b/morphlib/morphloader.py @@ -224,7 +224,7 @@ class DuplicateDeploymentNameError(MorphologyValidationError): % (cluster_filename, '\n ' + '\n '.join(duplicates))) -class OrderedDumper(yaml.SafeDumper): +class MorphologyDumper(yaml.SafeDumper): keyorder = ( 'name', 'kind', @@ -274,9 +274,36 @@ class OrderedDumper(yaml.SafeDumper): return dumper.represent_mapping('tag:yaml.org,2002:map', cls._iter_in_global_order(mapping)) + @classmethod + def _represent_str(cls, dumper, orig_data): + fallback_representer = yaml.representer.SafeRepresenter.represent_str + try: + data = unicode(orig_data, 'ascii') + if data.count('\n') == 0: + return fallback_representer(dumper, orig_data) + except UnicodeDecodeError: + try: + data = unicode(orig_data, 'utf-8') + if data.count('\n') == 0: + return fallback_representer(dumper, orig_data) + except UnicodeDecodeError: + return fallback_representer(dumper, orig_data) + return dumper.represent_scalar(u'tag:yaml.org,2002:str', + data, style='|') + + @classmethod + def _represent_unicode(cls, dumper, data): + if data.count('\n') == 0: + return yaml.representer.SafeRepresenter.represent_unicode(dumper, + data) + return dumper.represent_scalar(u'tag:yaml.org,2002:str', + data, style='|') + def __init__(self, *args, **kwargs): yaml.SafeDumper.__init__(self, *args, **kwargs) self.add_representer(dict, self._represent_dict) + self.add_representer(str, self._represent_str) + self.add_representer(unicode, self._represent_unicode) class MorphologyLoader(object): @@ -394,7 +421,7 @@ class MorphologyLoader(object): def save_to_string(self, morphology): '''Return normalised textual form of morphology.''' - return yaml.dump(morphology.data, Dumper=OrderedDumper, + return yaml.dump(morphology.data, Dumper=MorphologyDumper, default_flow_style=False) def save_to_file(self, filename, morphology): diff --git a/morphlib/morphloader_tests.py b/morphlib/morphloader_tests.py index 82663298..f4d2f9b6 100644 --- a/morphlib/morphloader_tests.py +++ b/morphlib/morphloader_tests.py @@ -912,3 +912,33 @@ build-system: dummy # deployment keys field order self.assertLess(s.find('type'), s.find('location')) self.assertLess(s.find('location'), s.find('HOSTNAME')) + + def test_multi_line_round_trip(self): + s = ('name: foo\n' + 'kind: bar\n' + 'description: |\n' + ' 1 2 3\n' + ' 4 5 6\n' + ' 7 8 9\n') + m = self.loader.parse_morphology_text(s, 'string') + self.assertEqual(s, self.loader.save_to_string(m)) + + def test_smoketest_multi_line_unicode(self): + m = morphlib.morph3.Morphology( + name=u'foo', + description=u'1 2 3\n4 5 6\n7 8 9\n', + ) + s = self.loader.save_to_string(m) + + def test_smoketest_multi_line_unicode_encoded(self): + m = morphlib.morph3.Morphology( + name=u'foo \u263A'.encode('utf-8'), + description=u'1 \u263A\n2 \u263A\n3 \u263A\n'.encode('utf-8'), + ) + s = self.loader.save_to_string(m) + + def test_smoketest_binary_garbage(self): + m = morphlib.morph3.Morphology( + description='\x92', + ) + s = self.loader.save_to_string(m) diff --git a/morphlib/morphologyfactory.py b/morphlib/morphologyfactory.py index 1cde2c77..cd1972be 100644 --- a/morphlib/morphologyfactory.py +++ b/morphlib/morphologyfactory.py @@ -88,7 +88,7 @@ class MorphologyFactory(object): file_list = repo.ls_tree(sha1) elif self._rrc is not None: self.status(msg="Retrieving %(reponame)s %(sha1)s %(filename)s" - " from the remote artifact cache.", + " from the remote git cache.", reponame=reponame, sha1=sha1, filename=filename, chatty=True) try: diff --git a/morphlib/plugins/deploy_plugin.py b/morphlib/plugins/deploy_plugin.py index 30e356e8..38c17bc2 100644 --- a/morphlib/plugins/deploy_plugin.py +++ b/morphlib/plugins/deploy_plugin.py @@ -308,7 +308,7 @@ class DeployPlugin(cliapp.Plugin): all_deployments = set() deployments = set() for system in cluster_morphology['systems']: - all_deployments.update([sys_id for sys_id in system['deploy']]) + all_deployments.update(system['deploy'].iterkeys()) if 'subsystems' in system: all_subsystems.update(loader._get_subsystem_names(system)) for item in args[1:]: @@ -367,9 +367,9 @@ class DeployPlugin(cliapp.Plugin): 'Cannot directly deploy subsystems. Create a top ' 'level deployment for the subsystem %s instead.' % subsystem) - if not any(deployment in var - for deployment in all_deployments) \ - and not subsystem in var: + if (not any(deployment in var + for deployment in all_deployments) + and not subsystem in var): raise cliapp.AppException( 'Variable referenced a non-existent deployment ' 'name: %s' % var) @@ -377,7 +377,7 @@ class DeployPlugin(cliapp.Plugin): def deploy_system(self, build_command, deploy_tempdir, root_repo_dir, build_repo, ref, system, env_vars, deployment_filter, parent_location): - sys_ids = set(sys_id for sys_id, _ in system['deploy'].iteritems()) + sys_ids = set(system['deploy'].iterkeys()) if deployment_filter and not \ any(sys_id in deployment_filter for sys_id in sys_ids): return diff --git a/morphlib/plugins/distbuild_plugin.py b/morphlib/plugins/distbuild_plugin.py index c60dee6e..50ab7eeb 100644 --- a/morphlib/plugins/distbuild_plugin.py +++ b/morphlib/plugins/distbuild_plugin.py @@ -59,7 +59,7 @@ class SerialiseArtifactPlugin(cliapp.Plugin): raise cliapp.AppException('Must get triplet') repo_name, ref, morph_name = args - filename = '%s.morph' % morph_name + filename = morphlib.util.sanitise_morphology_path(morph_name) build_command = morphlib.buildcommand.BuildCommand(self.app) srcpool = build_command.create_source_pool(repo_name, ref, filename) artifact = build_command.resolve_artifacts(srcpool) diff --git a/morphlib/plugins/list_artifacts_plugin.py b/morphlib/plugins/list_artifacts_plugin.py index 872c0754..8074206b 100644 --- a/morphlib/plugins/list_artifacts_plugin.py +++ b/morphlib/plugins/list_artifacts_plugin.py @@ -85,7 +85,7 @@ class ListArtifactsPlugin(cliapp.Plugin): self.app.status( msg='Creating source pool for %s' % system_filename, chatty=True) source_pool = self.app.create_source_pool( - self.lrc, self.rrc, (repo, ref, system_filename)) + self.lrc, self.rrc, repo, ref, system_filename) self.app.status( msg='Resolving artifacts for %s' % system_filename, chatty=True) diff --git a/morphlib/plugins/update_gits_plugin.py b/morphlib/plugins/update_gits_plugin.py deleted file mode 100644 index 46686391..00000000 --- a/morphlib/plugins/update_gits_plugin.py +++ /dev/null @@ -1,83 +0,0 @@ -# Copyright (C) 2012-2013 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -import cliapp -import os - -import morphlib - - -class UpdateGitsPlugin(cliapp.Plugin): - - def enable(self): - self.app.add_subcommand('update-gits', - self.update_gits, - arg_synopsis='(REPO REF MORPHOLOGY)...') - - def disable(self): - pass - - def update_gits(self, args): - '''Manually update cached git repositories for the given morphology - - Command line arguments: - - * `REPO` is a git repository URL. - * `REF` is a git commit ref (sha1, branch, tag). - * `MORPHOLOGY` is a morphology filename. - - This updates the local cached copy of a git repository, and any - git repositories of components in the morphology (for system - and stratum morphologies). - - You do not normally need to do this. Morph updates the cached - repositories automatically anyway. - - ''' - - app = self.app - if not os.path.exists(app.settings['cachedir']): - os.mkdir(app.settings['cachedir']) - cachedir = os.path.join(app.settings['cachedir'], 'gits') - repo_resolver = morphlib.repoaliasresolver.RepoAliasResolver( - app.settings['repo-alias']) - tarball_base_url = app.settings['tarball-server'] - cache = morphlib.localrepocache.LocalRepoCache( - app, cachedir, repo_resolver, tarball_base_url) - - subs_to_process = set() - - def visit(reponame, ref, filename, absref, tree, morphology): - app.status(msg='Updating %(repo_name)s %(ref)s %(filename)s', - repo_name=reponame, ref=ref, filename=filename) - assert cache.has_repo(reponame) - cached_repo = cache.get_repo(reponame) - try: - submodules = morphlib.git.Submodules(app, cached_repo.path, - absref) - submodules.load() - except morphlib.git.NoModulesFileError: - pass - else: - for submod in submodules: - subs_to_process.add((submod.url, submod.commit)) - - app.traverse_morphs(app.itertriplets(args), cache, None, - update=True, visit=visit) - - done = set() - for url, ref in subs_to_process: - app.cache_repo_and_submodules(cache, url, ref, done) diff --git a/morphlib/yamlparse.py b/morphlib/yamlparse.py index 726b4181..6f139304 100644 --- a/morphlib/yamlparse.py +++ b/morphlib/yamlparse.py @@ -29,7 +29,7 @@ if morphlib.got_yaml: # pragma: no cover def dump(*args, **kwargs): if 'default_flow_style' not in kwargs: kwargs['default_flow_style'] = False - return yaml.dump(Dumper=morphlib.morphloader.OrderedDumper, + return yaml.dump(Dumper=morphlib.morphloader.MorphologyDumper, *args, **kwargs) else: # pragma: no cover |