From 63aa14fc7fd988b03f2206cc9ed7503afa7039a0 Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Fri, 18 Jul 2014 16:10:17 +0100 Subject: Tidy deploy_plugin a little --- morphlib/plugins/deploy_plugin.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'morphlib') diff --git a/morphlib/plugins/deploy_plugin.py b/morphlib/plugins/deploy_plugin.py index 30e356e8..38c17bc2 100644 --- a/morphlib/plugins/deploy_plugin.py +++ b/morphlib/plugins/deploy_plugin.py @@ -308,7 +308,7 @@ class DeployPlugin(cliapp.Plugin): all_deployments = set() deployments = set() for system in cluster_morphology['systems']: - all_deployments.update([sys_id for sys_id in system['deploy']]) + all_deployments.update(system['deploy'].iterkeys()) if 'subsystems' in system: all_subsystems.update(loader._get_subsystem_names(system)) for item in args[1:]: @@ -367,9 +367,9 @@ class DeployPlugin(cliapp.Plugin): 'Cannot directly deploy subsystems. Create a top ' 'level deployment for the subsystem %s instead.' % subsystem) - if not any(deployment in var - for deployment in all_deployments) \ - and not subsystem in var: + if (not any(deployment in var + for deployment in all_deployments) + and not subsystem in var): raise cliapp.AppException( 'Variable referenced a non-existent deployment ' 'name: %s' % var) @@ -377,7 +377,7 @@ class DeployPlugin(cliapp.Plugin): def deploy_system(self, build_command, deploy_tempdir, root_repo_dir, build_repo, ref, system, env_vars, deployment_filter, parent_location): - sys_ids = set(sys_id for sys_id, _ in system['deploy'].iteritems()) + sys_ids = set(system['deploy'].iterkeys()) if deployment_filter and not \ any(sys_id in deployment_filter for sys_id in sys_ids): return -- cgit v1.2.1 From d0496731276c7b5314e7201117ca8536be4b5308 Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Fri, 18 Jul 2014 16:10:19 +0100 Subject: Remove update-gits command This hasn't been used in a long time to my knowledge, its API completely misses the point, and its implementation relies on old APIs that need to change. Until we discover we need it, and work out what it should look like, I think the best thing to do is get rid of it. --- morphlib/plugins/update_gits_plugin.py | 83 ---------------------------------- 1 file changed, 83 deletions(-) delete mode 100644 morphlib/plugins/update_gits_plugin.py (limited to 'morphlib') diff --git a/morphlib/plugins/update_gits_plugin.py b/morphlib/plugins/update_gits_plugin.py deleted file mode 100644 index 46686391..00000000 --- a/morphlib/plugins/update_gits_plugin.py +++ /dev/null @@ -1,83 +0,0 @@ -# Copyright (C) 2012-2013 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -import cliapp -import os - -import morphlib - - -class UpdateGitsPlugin(cliapp.Plugin): - - def enable(self): - self.app.add_subcommand('update-gits', - self.update_gits, - arg_synopsis='(REPO REF MORPHOLOGY)...') - - def disable(self): - pass - - def update_gits(self, args): - '''Manually update cached git repositories for the given morphology - - Command line arguments: - - * `REPO` is a git repository URL. - * `REF` is a git commit ref (sha1, branch, tag). - * `MORPHOLOGY` is a morphology filename. - - This updates the local cached copy of a git repository, and any - git repositories of components in the morphology (for system - and stratum morphologies). - - You do not normally need to do this. Morph updates the cached - repositories automatically anyway. - - ''' - - app = self.app - if not os.path.exists(app.settings['cachedir']): - os.mkdir(app.settings['cachedir']) - cachedir = os.path.join(app.settings['cachedir'], 'gits') - repo_resolver = morphlib.repoaliasresolver.RepoAliasResolver( - app.settings['repo-alias']) - tarball_base_url = app.settings['tarball-server'] - cache = morphlib.localrepocache.LocalRepoCache( - app, cachedir, repo_resolver, tarball_base_url) - - subs_to_process = set() - - def visit(reponame, ref, filename, absref, tree, morphology): - app.status(msg='Updating %(repo_name)s %(ref)s %(filename)s', - repo_name=reponame, ref=ref, filename=filename) - assert cache.has_repo(reponame) - cached_repo = cache.get_repo(reponame) - try: - submodules = morphlib.git.Submodules(app, cached_repo.path, - absref) - submodules.load() - except morphlib.git.NoModulesFileError: - pass - else: - for submod in submodules: - subs_to_process.add((submod.url, submod.commit)) - - app.traverse_morphs(app.itertriplets(args), cache, None, - update=True, visit=visit) - - done = set() - for url, ref in subs_to_process: - app.cache_repo_and_submodules(cache, url, ref, done) -- cgit v1.2.1 From 416594f21f672f651a383f37dd244381af61b10b Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Fri, 18 Jul 2014 16:10:20 +0100 Subject: Use chunk morpholgy contents in cache keys Previously the contents of the morphology would be included by virtue of the fact that it came from the source repository, so would be included in the "tree" field. Now that chunk morphologies can come from the definitions repository, it is not always included in the "tree" field, so the logical contents of the morphology need to be included in the cache key computation. Build commands are included after looking them up in the build-system, so that in future, we don't need to change the chunk morphology compatibility version when we change how build-systems work. Since we may be moving the morphologies about in the definitions repository, it would suck if we had to do a full rebuild after we move things, so I dropped the filename from the cache key. This also tweaks the system and stratum cache keys to include the contents directly, rather than hashed in the "morphology-sha1" field. --- morphlib/cachekeycomputer.py | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) (limited to 'morphlib') diff --git a/morphlib/cachekeycomputer.py b/morphlib/cachekeycomputer.py index ca374436..292047c8 100644 --- a/morphlib/cachekeycomputer.py +++ b/morphlib/cachekeycomputer.py @@ -93,7 +93,6 @@ class CacheKeyComputer(object): def _calculate(self, artifact): keys = { 'env': self._filterenv(self._build_env.env), - 'filename': artifact.source.filename, 'kids': [{'artifact': a.name, 'cache-key': self.compute_key(a)} for a in artifact.dependencies], 'metadata-version': artifact.metadata_version @@ -106,9 +105,24 @@ class CacheKeyComputer(object): keys['tree'] = artifact.source.tree keys['split-rules'] = [(a, [rgx.pattern for rgx in r._regexes]) for (a, r) in artifact.source.split_rules] + + # Include morphology contents, since it doesn't always come + # from the source tree + morphology = artifact.source.morphology + # include {pre-,,post-}{configure,build,test,install}-commands + # in morphology key + for prefix in ('pre-', '', 'post-'): + for cmdtype in ('configure', 'build', 'test', 'install'): + cmd_field = prefix + cmdtype + '-commands' + keys[cmd_field] = morphology.get_commands(cmd_field) + keys['devices'] = morphology.get('devices') + keys['max-jobs'] = morphology.get('max-jobs') + keys['system-integration'] = morphology.get('system-integration', + {}) + # products is omitted as they are part of the split-rules elif kind in ('system', 'stratum'): morphology = artifact.source.morphology - le_dict = dict((k, morphology[k]) for k in morphology.keys()) + morph_dict = dict((k, morphology[k]) for k in morphology.keys()) # Disregard all fields of a morphology that aren't important ignored_fields = ( @@ -117,13 +131,9 @@ class CacheKeyComputer(object): # so are already handled by the 'kids' field. 'strata', 'build-depends', 'chunks', 'products') - for ignored_field in ignored_fields: - if ignored_field in le_dict: - del le_dict[ignored_field] - - checksum = hashlib.sha1() - self._hash_thing(checksum, le_dict) - keys['morphology-sha1'] = checksum.hexdigest() + for key in morph_dict: + if key not in ignored_fields: + keys[key] = morph_dict[key] if kind == 'stratum': keys['stratum-format-version'] = 1 elif kind == 'system': -- cgit v1.2.1 From 11926f251856e48955c986de9653cdfb829d1c0f Mon Sep 17 00:00:00 2001 From: Adam Coldrick Date: Fri, 18 Jul 2014 16:10:21 +0100 Subject: unittests: Test that the description field is ignored for chunks --- morphlib/cachekeycomputer_tests.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'morphlib') diff --git a/morphlib/cachekeycomputer_tests.py b/morphlib/cachekeycomputer_tests.py index dd10307f..9e18b19d 100644 --- a/morphlib/cachekeycomputer_tests.py +++ b/morphlib/cachekeycomputer_tests.py @@ -36,15 +36,18 @@ class CacheKeyComputerTests(unittest.TestCase): for name, text in { 'chunk.morph': '''{ "name": "chunk", - "kind": "chunk" + "kind": "chunk", + "description": "A test chunk" }''', 'chunk2.morph': '''{ "name": "chunk2", - "kind": "chunk" + "kind": "chunk", + "description": "A test chunk" }''', 'chunk3.morph': '''{ "name": "chunk3", - "kind": "chunk" + "kind": "chunk", + "description": "A test chunk" }''', 'stratum.morph': '''{ "name": "stratum", -- cgit v1.2.1 From abb93656aa0b8de1e086c8f142bf468d3b50f26e Mon Sep 17 00:00:00 2001 From: Adam Coldrick Date: Fri, 18 Jul 2014 16:10:22 +0100 Subject: Don't set 'morph' field by default, and don't expect to find it --- morphlib/artifactresolver.py | 4 +++- morphlib/buildcommand.py | 3 ++- morphlib/morph2.py | 4 +--- morphlib/morph2_tests.py | 1 - 4 files changed, 6 insertions(+), 6 deletions(-) (limited to 'morphlib') diff --git a/morphlib/artifactresolver.py b/morphlib/artifactresolver.py index c18042a3..9e3cea48 100644 --- a/morphlib/artifactresolver.py +++ b/morphlib/artifactresolver.py @@ -191,10 +191,12 @@ class ArtifactResolver(object): name_to_processed_artifacts = {} for info in source.morphology['chunks']: + filename = morphlib.util.sanitise_morphology_path( + info.get('morph', info['name'])) chunk_source = self._source_pool.lookup( info['repo'], info['ref'], - morphlib.util.sanitise_morphology_path(info['morph'])) + filename) chunk_name = chunk_source.morphology['name'] diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py index 45c6ef00..4dea7b68 100644 --- a/morphlib/buildcommand.py +++ b/morphlib/buildcommand.py @@ -231,7 +231,8 @@ class BuildCommand(object): for spec in specs: repo_name = spec.get('repo') or src.repo_name ref = spec.get('ref') or src.original_ref - filename = morphlib.util.sanitise_morphology_path(spec['morph']) + filename = morphlib.util.sanitise_morphology_path( + spec.get('morph', spec.get('name'))) logging.debug( 'Validating cross ref to %s:%s:%s' % (repo_name, ref, filename)) diff --git a/morphlib/morph2.py b/morphlib/morph2.py index 83971bb8..b49c0f73 100644 --- a/morphlib/morph2.py +++ b/morphlib/morph2.py @@ -191,8 +191,6 @@ class Morphology(object): for source in self['chunks']: if 'repo' not in source: self._set_default_value(source, 'repo', source['name']) - if 'morph' not in source: - self._set_default_value(source, 'morph', source['name']) if 'build-depends' not in source: self._set_default_value(source, 'build-depends', None) if 'build-mode' not in source: @@ -226,7 +224,7 @@ class Morphology(object): return info elif self['kind'] == 'stratum': for info in self['chunks']: - source_name = info.get('alias', info['morph']) + source_name = info.get('alias', info['name']) if source_name == name: return info raise KeyError('"%s" not found' % name) diff --git a/morphlib/morph2_tests.py b/morphlib/morph2_tests.py index d02aab1c..c9957ad5 100644 --- a/morphlib/morph2_tests.py +++ b/morphlib/morph2_tests.py @@ -95,7 +95,6 @@ class MorphologyTests(unittest.TestCase): ''') self.assertEqual(m['chunks'][0]['repo'], 'le-chunk') - self.assertEqual(m['chunks'][0]['morph'], 'le-chunk') self.assertEqual(m['chunks'][0]['build-depends'], None) def test_returns_dict_keys(self): -- cgit v1.2.1 From e4c105f914ebb0df18cedbb8111f9d93f79b79dc Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Fri, 18 Jul 2014 16:10:23 +0100 Subject: Rewrite traverse_morphs to be able to load chunk morphologies from definitions It will now load the morphology from the definitions repository if the "morph" field is present in the chunk spec. Rather than adapting the loop to fit yet-more changing circumstances, it has been partially rewritten, so there is one loop for loading systems and strata from definitions.git, another for chunks from definitions.git, and a third for chunks in the source repository. This is tidier than attempting to fit the logic in the main loop, as it makes it easier to remove afterwards when we no longer need to load chunk morphologies from the source repository. --- morphlib/app.py | 85 ++++++++++++++++++++++++++++++------------------ morphlib/buildcommand.py | 2 +- 2 files changed, 54 insertions(+), 33 deletions(-) (limited to 'morphlib') diff --git a/morphlib/app.py b/morphlib/app.py index e0874317..a543443e 100644 --- a/morphlib/app.py +++ b/morphlib/app.py @@ -286,7 +286,7 @@ class Morph(cliapp.Application): morphlib.util.sanitise_morphology_path(args[2])) args = args[3:] - def create_source_pool(self, lrc, rrc, triplet): + def create_source_pool(self, lrc, rrc, repo, ref, filename): pool = morphlib.sourcepool.SourcePool() def add_to_pool(reponame, ref, filename, absref, tree, morphology): @@ -294,7 +294,7 @@ class Morph(cliapp.Application): morphology, filename) pool.add(source) - self.traverse_morphs([triplet], lrc, rrc, + self.traverse_morphs(repo, ref, [filename], lrc, rrc, update=not self.settings['no-git-update'], visit=add_to_pool) return pool @@ -338,53 +338,74 @@ class Morph(cliapp.Application): absref, tree = repo.resolve_ref(ref) return absref, tree - def traverse_morphs(self, triplets, lrc, rrc, update=True, + def traverse_morphs(self, definitions_repo, definitions_ref, + system_filenames, lrc, rrc, update=True, visit=lambda rn, rf, fn, arf, m: None): morph_factory = morphlib.morphologyfactory.MorphologyFactory(lrc, rrc, self) - queue = collections.deque(triplets) + definitions_queue = collections.deque(system_filenames) + chunk_in_definitions_repo_queue = [] + chunk_in_source_repo_queue = [] resolved_refs = {} resolved_morphologies = {} - while queue: - reponame, ref, filename = queue.popleft() + # Resolve the (repo, ref) pair for the definitions repo, cache result. + definitions_absref, definitions_tree = self.resolve_ref( + lrc, rrc, definitions_repo, definitions_ref, update) - # Resolve the (repo, ref) reference, cache result. - reference = (reponame, ref) - if not reference in resolved_refs: - resolved_refs[reference] = self.resolve_ref( - lrc, rrc, reponame, ref, update) - absref, tree = resolved_refs[reference] + while definitions_queue: + filename = definitions_queue.popleft() - # Fetch the (repo, ref, filename) morphology, cache result. - reference = (reponame, absref, filename) - if not reference in resolved_morphologies: - resolved_morphologies[reference] = \ - morph_factory.get_morphology(reponame, absref, filename) - morphology = resolved_morphologies[reference] + key = (definitions_repo, definitions_absref, filename) + if not key in resolved_morphologies: + resolved_morphologies[key] = morph_factory.get_morphology(*key) + morphology = resolved_morphologies[key] - visit(reponame, ref, filename, absref, tree, morphology) + visit(definitions_repo, definitions_ref, filename, + definitions_absref, definitions_tree, morphology) if morphology['kind'] == 'cluster': raise cliapp.AppException( "Cannot build a morphology of type 'cluster'.") elif morphology['kind'] == 'system': - queue.extend( - (s.get('repo') or reponame, - s.get('ref') or ref, - morphlib.util.sanitise_morphology_path(s['morph'])) + definitions_queue.extend( + morphlib.util.sanitise_morphology_path(s['morph']) for s in morphology['strata']) elif morphology['kind'] == 'stratum': if morphology['build-depends']: - queue.extend( - (s.get('repo') or reponame, - s.get('ref') or ref, - morphlib.util.sanitise_morphology_path(s['morph'])) + definitions_queue.extend( + morphlib.util.sanitise_morphology_path(s['morph']) for s in morphology['build-depends']) - queue.extend( - (c['repo'], - c['ref'], - morphlib.util.sanitise_morphology_path(c['morph'])) - for c in morphology['chunks']) + for c in morphology['chunks']: + if 'morph' not in c: + path = morphlib.util.sanitise_morphology_path( + c.get('morph', c['name'])) + chunk_in_source_repo_queue.append( + (c['repo'], c['ref'], path)) + continue + chunk_in_definitions_repo_queue.append( + (c['repo'], c['ref'], c['morph'])) + + for repo, ref, filename in chunk_in_definitions_repo_queue: + if (repo, ref) not in resolved_refs: + resolved_refs[repo, ref] = self.resolve_ref( + lrc, rrc, repo, ref, update) + absref, tree = resolved_refs[repo, ref] + key = (definitions_repo, definitions_absref, filename) + if not key in resolved_morphologies: + resolved_morphologies[key] = morph_factory.get_morphology(*key) + morphology = resolved_morphologies[key] + visit(repo, ref, filename, absref, tree, morphology) + + for repo, ref, filename in chunk_in_source_repo_queue: + if (repo, ref) not in resolved_refs: + resolved_refs[repo, ref] = self.resolve_ref( + lrc, rrc, repo, ref, update) + absref, tree = resolved_refs[repo, ref] + key = (repo, absref, filename) + if key not in resolved_morphologies: + resolved_morphologies[key] = morph_factory.get_morphology(*key) + morphology = resolved_morphologies[key] + visit(repo, ref, filename, absref, tree, morphology) def cache_repo_and_submodules(self, cache, url, ref, done): subs_to_process = set() diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py index 4dea7b68..1cb0c4d9 100644 --- a/morphlib/buildcommand.py +++ b/morphlib/buildcommand.py @@ -90,7 +90,7 @@ class BuildCommand(object): ''' self.app.status(msg='Creating source pool', chatty=True) srcpool = self.app.create_source_pool( - self.lrc, self.rrc, (repo_name, ref, filename)) + self.lrc, self.rrc, repo_name, ref, filename) return srcpool -- cgit v1.2.1