diff options
Diffstat (limited to 'morphlib')
| -rw-r--r-- | morphlib/app.py | 85 | ||||
| -rw-r--r-- | morphlib/artifactresolver.py | 4 | ||||
| -rw-r--r-- | morphlib/buildcommand.py | 5 | ||||
| -rw-r--r-- | morphlib/cachekeycomputer.py | 28 | ||||
| -rw-r--r-- | morphlib/cachekeycomputer_tests.py | 9 | ||||
| -rw-r--r-- | morphlib/morph2.py | 4 | ||||
| -rw-r--r-- | morphlib/morph2_tests.py | 1 | ||||
| -rw-r--r-- | morphlib/plugins/deploy_plugin.py | 10 | ||||
| -rw-r--r-- | morphlib/plugins/update_gits_plugin.py | 83 |
9 files changed, 90 insertions, 139 deletions
diff --git a/morphlib/app.py b/morphlib/app.py index e0874317..a543443e 100644 --- a/morphlib/app.py +++ b/morphlib/app.py @@ -286,7 +286,7 @@ class Morph(cliapp.Application): morphlib.util.sanitise_morphology_path(args[2])) args = args[3:] - def create_source_pool(self, lrc, rrc, triplet): + def create_source_pool(self, lrc, rrc, repo, ref, filename): pool = morphlib.sourcepool.SourcePool() def add_to_pool(reponame, ref, filename, absref, tree, morphology): @@ -294,7 +294,7 @@ class Morph(cliapp.Application): morphology, filename) pool.add(source) - self.traverse_morphs([triplet], lrc, rrc, + self.traverse_morphs(repo, ref, [filename], lrc, rrc, update=not self.settings['no-git-update'], visit=add_to_pool) return pool @@ -338,53 +338,74 @@ class Morph(cliapp.Application): absref, tree = repo.resolve_ref(ref) return absref, tree - def traverse_morphs(self, triplets, lrc, rrc, update=True, + def traverse_morphs(self, definitions_repo, definitions_ref, + system_filenames, lrc, rrc, update=True, visit=lambda rn, rf, fn, arf, m: None): morph_factory = morphlib.morphologyfactory.MorphologyFactory(lrc, rrc, self) - queue = collections.deque(triplets) + definitions_queue = collections.deque(system_filenames) + chunk_in_definitions_repo_queue = [] + chunk_in_source_repo_queue = [] resolved_refs = {} resolved_morphologies = {} - while queue: - reponame, ref, filename = queue.popleft() + # Resolve the (repo, ref) pair for the definitions repo, cache result. + definitions_absref, definitions_tree = self.resolve_ref( + lrc, rrc, definitions_repo, definitions_ref, update) - # Resolve the (repo, ref) reference, cache result. - reference = (reponame, ref) - if not reference in resolved_refs: - resolved_refs[reference] = self.resolve_ref( - lrc, rrc, reponame, ref, update) - absref, tree = resolved_refs[reference] + while definitions_queue: + filename = definitions_queue.popleft() - # Fetch the (repo, ref, filename) morphology, cache result. - reference = (reponame, absref, filename) - if not reference in resolved_morphologies: - resolved_morphologies[reference] = \ - morph_factory.get_morphology(reponame, absref, filename) - morphology = resolved_morphologies[reference] + key = (definitions_repo, definitions_absref, filename) + if not key in resolved_morphologies: + resolved_morphologies[key] = morph_factory.get_morphology(*key) + morphology = resolved_morphologies[key] - visit(reponame, ref, filename, absref, tree, morphology) + visit(definitions_repo, definitions_ref, filename, + definitions_absref, definitions_tree, morphology) if morphology['kind'] == 'cluster': raise cliapp.AppException( "Cannot build a morphology of type 'cluster'.") elif morphology['kind'] == 'system': - queue.extend( - (s.get('repo') or reponame, - s.get('ref') or ref, - morphlib.util.sanitise_morphology_path(s['morph'])) + definitions_queue.extend( + morphlib.util.sanitise_morphology_path(s['morph']) for s in morphology['strata']) elif morphology['kind'] == 'stratum': if morphology['build-depends']: - queue.extend( - (s.get('repo') or reponame, - s.get('ref') or ref, - morphlib.util.sanitise_morphology_path(s['morph'])) + definitions_queue.extend( + morphlib.util.sanitise_morphology_path(s['morph']) for s in morphology['build-depends']) - queue.extend( - (c['repo'], - c['ref'], - morphlib.util.sanitise_morphology_path(c['morph'])) - for c in morphology['chunks']) + for c in morphology['chunks']: + if 'morph' not in c: + path = morphlib.util.sanitise_morphology_path( + c.get('morph', c['name'])) + chunk_in_source_repo_queue.append( + (c['repo'], c['ref'], path)) + continue + chunk_in_definitions_repo_queue.append( + (c['repo'], c['ref'], c['morph'])) + + for repo, ref, filename in chunk_in_definitions_repo_queue: + if (repo, ref) not in resolved_refs: + resolved_refs[repo, ref] = self.resolve_ref( + lrc, rrc, repo, ref, update) + absref, tree = resolved_refs[repo, ref] + key = (definitions_repo, definitions_absref, filename) + if not key in resolved_morphologies: + resolved_morphologies[key] = morph_factory.get_morphology(*key) + morphology = resolved_morphologies[key] + visit(repo, ref, filename, absref, tree, morphology) + + for repo, ref, filename in chunk_in_source_repo_queue: + if (repo, ref) not in resolved_refs: + resolved_refs[repo, ref] = self.resolve_ref( + lrc, rrc, repo, ref, update) + absref, tree = resolved_refs[repo, ref] + key = (repo, absref, filename) + if key not in resolved_morphologies: + resolved_morphologies[key] = morph_factory.get_morphology(*key) + morphology = resolved_morphologies[key] + visit(repo, ref, filename, absref, tree, morphology) def cache_repo_and_submodules(self, cache, url, ref, done): subs_to_process = set() diff --git a/morphlib/artifactresolver.py b/morphlib/artifactresolver.py index c18042a3..9e3cea48 100644 --- a/morphlib/artifactresolver.py +++ b/morphlib/artifactresolver.py @@ -191,10 +191,12 @@ class ArtifactResolver(object): name_to_processed_artifacts = {} for info in source.morphology['chunks']: + filename = morphlib.util.sanitise_morphology_path( + info.get('morph', info['name'])) chunk_source = self._source_pool.lookup( info['repo'], info['ref'], - morphlib.util.sanitise_morphology_path(info['morph'])) + filename) chunk_name = chunk_source.morphology['name'] diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py index 45c6ef00..1cb0c4d9 100644 --- a/morphlib/buildcommand.py +++ b/morphlib/buildcommand.py @@ -90,7 +90,7 @@ class BuildCommand(object): ''' self.app.status(msg='Creating source pool', chatty=True) srcpool = self.app.create_source_pool( - self.lrc, self.rrc, (repo_name, ref, filename)) + self.lrc, self.rrc, repo_name, ref, filename) return srcpool @@ -231,7 +231,8 @@ class BuildCommand(object): for spec in specs: repo_name = spec.get('repo') or src.repo_name ref = spec.get('ref') or src.original_ref - filename = morphlib.util.sanitise_morphology_path(spec['morph']) + filename = morphlib.util.sanitise_morphology_path( + spec.get('morph', spec.get('name'))) logging.debug( 'Validating cross ref to %s:%s:%s' % (repo_name, ref, filename)) diff --git a/morphlib/cachekeycomputer.py b/morphlib/cachekeycomputer.py index ca374436..292047c8 100644 --- a/morphlib/cachekeycomputer.py +++ b/morphlib/cachekeycomputer.py @@ -93,7 +93,6 @@ class CacheKeyComputer(object): def _calculate(self, artifact): keys = { 'env': self._filterenv(self._build_env.env), - 'filename': artifact.source.filename, 'kids': [{'artifact': a.name, 'cache-key': self.compute_key(a)} for a in artifact.dependencies], 'metadata-version': artifact.metadata_version @@ -106,9 +105,24 @@ class CacheKeyComputer(object): keys['tree'] = artifact.source.tree keys['split-rules'] = [(a, [rgx.pattern for rgx in r._regexes]) for (a, r) in artifact.source.split_rules] + + # Include morphology contents, since it doesn't always come + # from the source tree + morphology = artifact.source.morphology + # include {pre-,,post-}{configure,build,test,install}-commands + # in morphology key + for prefix in ('pre-', '', 'post-'): + for cmdtype in ('configure', 'build', 'test', 'install'): + cmd_field = prefix + cmdtype + '-commands' + keys[cmd_field] = morphology.get_commands(cmd_field) + keys['devices'] = morphology.get('devices') + keys['max-jobs'] = morphology.get('max-jobs') + keys['system-integration'] = morphology.get('system-integration', + {}) + # products is omitted as they are part of the split-rules elif kind in ('system', 'stratum'): morphology = artifact.source.morphology - le_dict = dict((k, morphology[k]) for k in morphology.keys()) + morph_dict = dict((k, morphology[k]) for k in morphology.keys()) # Disregard all fields of a morphology that aren't important ignored_fields = ( @@ -117,13 +131,9 @@ class CacheKeyComputer(object): # so are already handled by the 'kids' field. 'strata', 'build-depends', 'chunks', 'products') - for ignored_field in ignored_fields: - if ignored_field in le_dict: - del le_dict[ignored_field] - - checksum = hashlib.sha1() - self._hash_thing(checksum, le_dict) - keys['morphology-sha1'] = checksum.hexdigest() + for key in morph_dict: + if key not in ignored_fields: + keys[key] = morph_dict[key] if kind == 'stratum': keys['stratum-format-version'] = 1 elif kind == 'system': diff --git a/morphlib/cachekeycomputer_tests.py b/morphlib/cachekeycomputer_tests.py index dd10307f..9e18b19d 100644 --- a/morphlib/cachekeycomputer_tests.py +++ b/morphlib/cachekeycomputer_tests.py @@ -36,15 +36,18 @@ class CacheKeyComputerTests(unittest.TestCase): for name, text in { 'chunk.morph': '''{ "name": "chunk", - "kind": "chunk" + "kind": "chunk", + "description": "A test chunk" }''', 'chunk2.morph': '''{ "name": "chunk2", - "kind": "chunk" + "kind": "chunk", + "description": "A test chunk" }''', 'chunk3.morph': '''{ "name": "chunk3", - "kind": "chunk" + "kind": "chunk", + "description": "A test chunk" }''', 'stratum.morph': '''{ "name": "stratum", diff --git a/morphlib/morph2.py b/morphlib/morph2.py index 83971bb8..b49c0f73 100644 --- a/morphlib/morph2.py +++ b/morphlib/morph2.py @@ -191,8 +191,6 @@ class Morphology(object): for source in self['chunks']: if 'repo' not in source: self._set_default_value(source, 'repo', source['name']) - if 'morph' not in source: - self._set_default_value(source, 'morph', source['name']) if 'build-depends' not in source: self._set_default_value(source, 'build-depends', None) if 'build-mode' not in source: @@ -226,7 +224,7 @@ class Morphology(object): return info elif self['kind'] == 'stratum': for info in self['chunks']: - source_name = info.get('alias', info['morph']) + source_name = info.get('alias', info['name']) if source_name == name: return info raise KeyError('"%s" not found' % name) diff --git a/morphlib/morph2_tests.py b/morphlib/morph2_tests.py index d02aab1c..c9957ad5 100644 --- a/morphlib/morph2_tests.py +++ b/morphlib/morph2_tests.py @@ -95,7 +95,6 @@ class MorphologyTests(unittest.TestCase): ''') self.assertEqual(m['chunks'][0]['repo'], 'le-chunk') - self.assertEqual(m['chunks'][0]['morph'], 'le-chunk') self.assertEqual(m['chunks'][0]['build-depends'], None) def test_returns_dict_keys(self): diff --git a/morphlib/plugins/deploy_plugin.py b/morphlib/plugins/deploy_plugin.py index 30e356e8..38c17bc2 100644 --- a/morphlib/plugins/deploy_plugin.py +++ b/morphlib/plugins/deploy_plugin.py @@ -308,7 +308,7 @@ class DeployPlugin(cliapp.Plugin): all_deployments = set() deployments = set() for system in cluster_morphology['systems']: - all_deployments.update([sys_id for sys_id in system['deploy']]) + all_deployments.update(system['deploy'].iterkeys()) if 'subsystems' in system: all_subsystems.update(loader._get_subsystem_names(system)) for item in args[1:]: @@ -367,9 +367,9 @@ class DeployPlugin(cliapp.Plugin): 'Cannot directly deploy subsystems. Create a top ' 'level deployment for the subsystem %s instead.' % subsystem) - if not any(deployment in var - for deployment in all_deployments) \ - and not subsystem in var: + if (not any(deployment in var + for deployment in all_deployments) + and not subsystem in var): raise cliapp.AppException( 'Variable referenced a non-existent deployment ' 'name: %s' % var) @@ -377,7 +377,7 @@ class DeployPlugin(cliapp.Plugin): def deploy_system(self, build_command, deploy_tempdir, root_repo_dir, build_repo, ref, system, env_vars, deployment_filter, parent_location): - sys_ids = set(sys_id for sys_id, _ in system['deploy'].iteritems()) + sys_ids = set(system['deploy'].iterkeys()) if deployment_filter and not \ any(sys_id in deployment_filter for sys_id in sys_ids): return diff --git a/morphlib/plugins/update_gits_plugin.py b/morphlib/plugins/update_gits_plugin.py deleted file mode 100644 index 46686391..00000000 --- a/morphlib/plugins/update_gits_plugin.py +++ /dev/null @@ -1,83 +0,0 @@ -# Copyright (C) 2012-2013 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -import cliapp -import os - -import morphlib - - -class UpdateGitsPlugin(cliapp.Plugin): - - def enable(self): - self.app.add_subcommand('update-gits', - self.update_gits, - arg_synopsis='(REPO REF MORPHOLOGY)...') - - def disable(self): - pass - - def update_gits(self, args): - '''Manually update cached git repositories for the given morphology - - Command line arguments: - - * `REPO` is a git repository URL. - * `REF` is a git commit ref (sha1, branch, tag). - * `MORPHOLOGY` is a morphology filename. - - This updates the local cached copy of a git repository, and any - git repositories of components in the morphology (for system - and stratum morphologies). - - You do not normally need to do this. Morph updates the cached - repositories automatically anyway. - - ''' - - app = self.app - if not os.path.exists(app.settings['cachedir']): - os.mkdir(app.settings['cachedir']) - cachedir = os.path.join(app.settings['cachedir'], 'gits') - repo_resolver = morphlib.repoaliasresolver.RepoAliasResolver( - app.settings['repo-alias']) - tarball_base_url = app.settings['tarball-server'] - cache = morphlib.localrepocache.LocalRepoCache( - app, cachedir, repo_resolver, tarball_base_url) - - subs_to_process = set() - - def visit(reponame, ref, filename, absref, tree, morphology): - app.status(msg='Updating %(repo_name)s %(ref)s %(filename)s', - repo_name=reponame, ref=ref, filename=filename) - assert cache.has_repo(reponame) - cached_repo = cache.get_repo(reponame) - try: - submodules = morphlib.git.Submodules(app, cached_repo.path, - absref) - submodules.load() - except morphlib.git.NoModulesFileError: - pass - else: - for submod in submodules: - subs_to_process.add((submod.url, submod.commit)) - - app.traverse_morphs(app.itertriplets(args), cache, None, - update=True, visit=visit) - - done = set() - for url, ref in subs_to_process: - app.cache_repo_and_submodules(cache, url, ref, done) |