From 14313b1c68a1c068ec841b4ac8aa074f0784267a Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Fri, 18 Jul 2014 16:10:16 +0100 Subject: Use run_morph in yarns This is required to ensure the right version of morph is used. I have a .bashrc that causes `morph` to be "$HOME/morph/morph", so it fails to find morph, because HOME is set to a directory inside DATADIR. --- yarns/implementations.yarn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yarns/implementations.yarn b/yarns/implementations.yarn index d32bfd61..f0b5c80a 100644 --- a/yarns/implementations.yarn +++ b/yarns/implementations.yarn @@ -441,7 +441,7 @@ architecture as the machine doing the testing. This uses `morph print-architecture` to get a value appropriate for morph. IMPLEMENTS WHEN the user creates an uncommitted system morphology called (\S+) for our architecture in system branch (\S+) - arch=$(morph print-architecture) + arch=$(run_morph print-architecture) name="$(basename "${MATCH_1%.*}")" install -m644 -D /dev/stdin << EOF "$DATADIR/workspace/$MATCH_2/test/morphs/$MATCH_1" arch: $arch -- cgit v1.2.1 From 63aa14fc7fd988b03f2206cc9ed7503afa7039a0 Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Fri, 18 Jul 2014 16:10:17 +0100 Subject: Tidy deploy_plugin a little --- morphlib/plugins/deploy_plugin.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/morphlib/plugins/deploy_plugin.py b/morphlib/plugins/deploy_plugin.py index 30e356e8..38c17bc2 100644 --- a/morphlib/plugins/deploy_plugin.py +++ b/morphlib/plugins/deploy_plugin.py @@ -308,7 +308,7 @@ class DeployPlugin(cliapp.Plugin): all_deployments = set() deployments = set() for system in cluster_morphology['systems']: - all_deployments.update([sys_id for sys_id in system['deploy']]) + all_deployments.update(system['deploy'].iterkeys()) if 'subsystems' in system: all_subsystems.update(loader._get_subsystem_names(system)) for item in args[1:]: @@ -367,9 +367,9 @@ class DeployPlugin(cliapp.Plugin): 'Cannot directly deploy subsystems. Create a top ' 'level deployment for the subsystem %s instead.' % subsystem) - if not any(deployment in var - for deployment in all_deployments) \ - and not subsystem in var: + if (not any(deployment in var + for deployment in all_deployments) + and not subsystem in var): raise cliapp.AppException( 'Variable referenced a non-existent deployment ' 'name: %s' % var) @@ -377,7 +377,7 @@ class DeployPlugin(cliapp.Plugin): def deploy_system(self, build_command, deploy_tempdir, root_repo_dir, build_repo, ref, system, env_vars, deployment_filter, parent_location): - sys_ids = set(sys_id for sys_id, _ in system['deploy'].iteritems()) + sys_ids = set(system['deploy'].iterkeys()) if deployment_filter and not \ any(sys_id in deployment_filter for sys_id in sys_ids): return -- cgit v1.2.1 From a464a9e4f1e20ce98308470ac8adf87fec415407 Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Fri, 18 Jul 2014 16:10:18 +0100 Subject: Remove use of "morph" field in chunk specs in yarns We don't use this in definitions.git, and we're going to change its semantics, so the test suite would break until we introduced the new semantics, unless we remove its use of the old semantics first. --- yarns/implementations.yarn | 3 --- yarns/regression.yarn | 7 ++++++- yarns/splitting.yarn | 3 --- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/yarns/implementations.yarn b/yarns/implementations.yarn index f0b5c80a..2c325985 100644 --- a/yarns/implementations.yarn +++ b/yarns/implementations.yarn @@ -88,7 +88,6 @@ another to hold a chunk. - name: test-chunk repo: test:test-chunk ref: master - morph: test-chunk build-mode: test build-depends: [] EOF @@ -281,7 +280,6 @@ have a morphology using the test architecture. - name: stage1-chunk repo: test:test-chunk ref: master - morph: stage1-chunk build-mode: bootstrap build-depends: [] EOF @@ -295,7 +293,6 @@ have a morphology using the test architecture. - name: test-chunk repo: test:test-chunk ref: master - morph: test-chunk build-mode: test build-depends: [] EOF diff --git a/yarns/regression.yarn b/yarns/regression.yarn index 97e187f5..05e2e70c 100644 --- a/yarns/regression.yarn +++ b/yarns/regression.yarn @@ -90,10 +90,15 @@ Implementations - name: bootstrap-chunk repo: test:test-chunk ref: master - morph: test-chunk build-mode: bootstrap build-depends: [] EOF run_in "$DATADIR/gits/morphs" git add . run_in "$DATADIR/gits/morphs" git commit -m "Add bootstrap-system" + + sed -e 's/test-chunk/bootstrap-chunk/g' \ + "$DATADIR/gits/test-chunk/test-chunk.morph" \ + > "$DATADIR/gits/test-chunk/bootstrap-chunk.morph" + run_in "$DATADIR/gits/test-chunk" git add . + run_in "$DATADIR/gits/test-chunk" git commit -m "Add bootstrap-chunk" diff --git a/yarns/splitting.yarn b/yarns/splitting.yarn index dbe87a5c..d35153e4 100644 --- a/yarns/splitting.yarn +++ b/yarns/splitting.yarn @@ -142,7 +142,6 @@ Implementations - name: test-chunk repo: test:test-chunk ref: master - morph: test-chunk build-mode: test build-depends: [] artifacts: @@ -190,13 +189,11 @@ Implementations - name: test-chunk repo: test:test-chunk ref: master - morph: test-chunk build-mode: test build-depends: [] - name: unbuildable-chunk repo: test:test-chunk ref: refs/heads/master - morph: unbuildable-chunk build-mode: test build-depends: - test-chunk -- cgit v1.2.1 From d0496731276c7b5314e7201117ca8536be4b5308 Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Fri, 18 Jul 2014 16:10:19 +0100 Subject: Remove update-gits command This hasn't been used in a long time to my knowledge, its API completely misses the point, and its implementation relies on old APIs that need to change. Until we discover we need it, and work out what it should look like, I think the best thing to do is get rid of it. --- morphlib/plugins/update_gits_plugin.py | 83 ---------------------------------- tests/update-gits-chunk.script | 43 ------------------ tests/update-gits-stratum.script | 43 ------------------ tests/update-gits-submodules.script | 49 -------------------- tests/update-gits-submodules.stdout | 3 -- without-test-modules | 1 - 6 files changed, 222 deletions(-) delete mode 100644 morphlib/plugins/update_gits_plugin.py delete mode 100755 tests/update-gits-chunk.script delete mode 100755 tests/update-gits-stratum.script delete mode 100755 tests/update-gits-submodules.script delete mode 100644 tests/update-gits-submodules.stdout diff --git a/morphlib/plugins/update_gits_plugin.py b/morphlib/plugins/update_gits_plugin.py deleted file mode 100644 index 46686391..00000000 --- a/morphlib/plugins/update_gits_plugin.py +++ /dev/null @@ -1,83 +0,0 @@ -# Copyright (C) 2012-2013 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -import cliapp -import os - -import morphlib - - -class UpdateGitsPlugin(cliapp.Plugin): - - def enable(self): - self.app.add_subcommand('update-gits', - self.update_gits, - arg_synopsis='(REPO REF MORPHOLOGY)...') - - def disable(self): - pass - - def update_gits(self, args): - '''Manually update cached git repositories for the given morphology - - Command line arguments: - - * `REPO` is a git repository URL. - * `REF` is a git commit ref (sha1, branch, tag). - * `MORPHOLOGY` is a morphology filename. - - This updates the local cached copy of a git repository, and any - git repositories of components in the morphology (for system - and stratum morphologies). - - You do not normally need to do this. Morph updates the cached - repositories automatically anyway. - - ''' - - app = self.app - if not os.path.exists(app.settings['cachedir']): - os.mkdir(app.settings['cachedir']) - cachedir = os.path.join(app.settings['cachedir'], 'gits') - repo_resolver = morphlib.repoaliasresolver.RepoAliasResolver( - app.settings['repo-alias']) - tarball_base_url = app.settings['tarball-server'] - cache = morphlib.localrepocache.LocalRepoCache( - app, cachedir, repo_resolver, tarball_base_url) - - subs_to_process = set() - - def visit(reponame, ref, filename, absref, tree, morphology): - app.status(msg='Updating %(repo_name)s %(ref)s %(filename)s', - repo_name=reponame, ref=ref, filename=filename) - assert cache.has_repo(reponame) - cached_repo = cache.get_repo(reponame) - try: - submodules = morphlib.git.Submodules(app, cached_repo.path, - absref) - submodules.load() - except morphlib.git.NoModulesFileError: - pass - else: - for submod in submodules: - subs_to_process.add((submod.url, submod.commit)) - - app.traverse_morphs(app.itertriplets(args), cache, None, - update=True, visit=visit) - - done = set() - for url, ref in subs_to_process: - app.cache_repo_and_submodules(cache, url, ref, done) diff --git a/tests/update-gits-chunk.script b/tests/update-gits-chunk.script deleted file mode 100755 index 49072419..00000000 --- a/tests/update-gits-chunk.script +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2012-2013 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -## Test that 'morph update-gits' updates every chunk listed on the -## command line. - -set -eu - -# Local repos do not get cached, so we need to fake some remote repositories -git daemon --port=9419 --pid-file="$DATADIR/git.pid" --reuseaddr \ - --export-all --base-path="$DATADIR" 2> /dev/null & -trap 'kill $(cat "$DATADIR/git.pid")' INT TERM ERR -REPO_ALIAS='test=git://127.0.0.1:9419/%s#git://127.0.0.1:9419/%s' - -# Create an empty commit in a chunk repository -cd "$DATADIR/chunk-repo" -git checkout --quiet farrokh -git commit --quiet --allow-empty --allow-empty-message -m "" -NEWREF="$(git rev-parse --verify farrokh)" - -"$SRCDIR/scripts/test-morph" --repo-alias=$REPO_ALIAS update-gits \ - test:chunk-repo farrokh hello - -kill $(cat "$DATADIR/git.pid") - -# Check the top commit of the cached repo's farrokh branch -cd "$DATADIR/cache/gits/"*chunk?repo* -test "$(git rev-parse --verify refs/heads/farrokh)" = "$NEWREF" diff --git a/tests/update-gits-stratum.script b/tests/update-gits-stratum.script deleted file mode 100755 index e4c79289..00000000 --- a/tests/update-gits-stratum.script +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2012-2013 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -## Test that 'morph update-gits' updates every chunk that depends on the -## strata listed on the command line. - -set -eu - -# Local repos do not get cached, so we need to fake some remote repositories -git daemon --port=9419 --pid-file="$DATADIR/git.pid" --reuseaddr \ - --export-all --base-path="$DATADIR" 2> /dev/null & -trap 'kill $(cat "$DATADIR/git.pid")' INT TERM ERR -REPO_ALIAS='test=git://127.0.0.1:9419/%s#git://127.0.0.1:9419/%s' - -# Create an empty commit in a chunk repository -cd "$DATADIR/chunk-repo" -git checkout --quiet farrokh -git commit --quiet --allow-empty --allow-empty-message -m "" -NEWREF="$(git show-ref --hash farrokh)" - -"$SRCDIR/scripts/test-morph" --repo-alias=$REPO_ALIAS update-gits \ - test:morphs-repo master hello-stratum - -kill $(cat "$DATADIR/git.pid") - -# Check the top commit of the cached repo's farrokh branch -cd "$DATADIR/cache/gits/"*chunk?repo* -test "$(git show-ref --hash refs/heads/farrokh)" = "$NEWREF" diff --git a/tests/update-gits-submodules.script b/tests/update-gits-submodules.script deleted file mode 100755 index 65958919..00000000 --- a/tests/update-gits-submodules.script +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash -# -# Copyright (C) 2012-2013 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -## Test that 'morph update-gits' caches all the submodules needed by -## a chunk. - -set -eu - -# Local repos do not get cached, so we need to fake some remote repositories -git daemon --port=9419 --pid-file="$DATADIR/git.pid" --reuseaddr \ - --export-all --base-path="$DATADIR" 2> /dev/null & -trap 'kill $(cat "$DATADIR/git.pid")' INT TERM ERR -REPO_ALIAS='test=git://127.0.0.1:9419/%s#git://127.0.0.1:9419/%s' - -# Create a repo to use as a submodule -submodule="$DATADIR/submodule-repo" -"$SRCDIR/scripts/cmd-filter" git init --quiet "$submodule" -"$SRCDIR/scripts/run-git-in" "$submodule" commit --quiet --allow-empty \ - --allow-empty-message -m "" -# Create an empty commit in a chunk repository -chunkrepo="$DATADIR/chunk-repo" -"$SRCDIR/scripts/run-git-in" "$chunkrepo" checkout --quiet farrokh -"$SRCDIR/scripts/run-git-in" "$chunkrepo" submodule --quiet add "$submodule" \ - > /dev/null -"$SRCDIR/scripts/run-git-in" "$chunkrepo" commit --quiet -m "add submodule" - -"$SRCDIR/scripts/test-morph" --repo-alias=$REPO_ALIAS update-gits \ - test:morphs-repo master hello-stratum - -kill $(cat "$DATADIR/git.pid") - -test -d "$DATADIR/cache/gits/"*chunk?repo* && echo chunk-repo cached -test -d "$DATADIR/cache/gits/"*morphs?repo* && echo morphs-repo cached -test -d "$DATADIR/cache/gits/"*submodule?repo* && echo submodule-repo cached diff --git a/tests/update-gits-submodules.stdout b/tests/update-gits-submodules.stdout deleted file mode 100644 index 5b20a81e..00000000 --- a/tests/update-gits-submodules.stdout +++ /dev/null @@ -1,3 +0,0 @@ -chunk-repo cached -morphs-repo cached -submodule-repo cached diff --git a/without-test-modules b/without-test-modules index 61a97877..a3aedc45 100644 --- a/without-test-modules +++ b/without-test-modules @@ -16,7 +16,6 @@ morphlib/plugins/syslinux-disk-systembuilder_plugin.py morphlib/plugins/disk-systembuilder_plugin.py morphlib/plugins/tarball-systembuilder_plugin.py morphlib/plugins/show_dependencies_plugin.py -morphlib/plugins/update_gits_plugin.py morphlib/plugins/branch_and_merge_plugin.py morphlib/buildcommand.py morphlib/plugins/build_plugin.py -- cgit v1.2.1 From 416594f21f672f651a383f37dd244381af61b10b Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Fri, 18 Jul 2014 16:10:20 +0100 Subject: Use chunk morpholgy contents in cache keys Previously the contents of the morphology would be included by virtue of the fact that it came from the source repository, so would be included in the "tree" field. Now that chunk morphologies can come from the definitions repository, it is not always included in the "tree" field, so the logical contents of the morphology need to be included in the cache key computation. Build commands are included after looking them up in the build-system, so that in future, we don't need to change the chunk morphology compatibility version when we change how build-systems work. Since we may be moving the morphologies about in the definitions repository, it would suck if we had to do a full rebuild after we move things, so I dropped the filename from the cache key. This also tweaks the system and stratum cache keys to include the contents directly, rather than hashed in the "morphology-sha1" field. --- morphlib/cachekeycomputer.py | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/morphlib/cachekeycomputer.py b/morphlib/cachekeycomputer.py index ca374436..292047c8 100644 --- a/morphlib/cachekeycomputer.py +++ b/morphlib/cachekeycomputer.py @@ -93,7 +93,6 @@ class CacheKeyComputer(object): def _calculate(self, artifact): keys = { 'env': self._filterenv(self._build_env.env), - 'filename': artifact.source.filename, 'kids': [{'artifact': a.name, 'cache-key': self.compute_key(a)} for a in artifact.dependencies], 'metadata-version': artifact.metadata_version @@ -106,9 +105,24 @@ class CacheKeyComputer(object): keys['tree'] = artifact.source.tree keys['split-rules'] = [(a, [rgx.pattern for rgx in r._regexes]) for (a, r) in artifact.source.split_rules] + + # Include morphology contents, since it doesn't always come + # from the source tree + morphology = artifact.source.morphology + # include {pre-,,post-}{configure,build,test,install}-commands + # in morphology key + for prefix in ('pre-', '', 'post-'): + for cmdtype in ('configure', 'build', 'test', 'install'): + cmd_field = prefix + cmdtype + '-commands' + keys[cmd_field] = morphology.get_commands(cmd_field) + keys['devices'] = morphology.get('devices') + keys['max-jobs'] = morphology.get('max-jobs') + keys['system-integration'] = morphology.get('system-integration', + {}) + # products is omitted as they are part of the split-rules elif kind in ('system', 'stratum'): morphology = artifact.source.morphology - le_dict = dict((k, morphology[k]) for k in morphology.keys()) + morph_dict = dict((k, morphology[k]) for k in morphology.keys()) # Disregard all fields of a morphology that aren't important ignored_fields = ( @@ -117,13 +131,9 @@ class CacheKeyComputer(object): # so are already handled by the 'kids' field. 'strata', 'build-depends', 'chunks', 'products') - for ignored_field in ignored_fields: - if ignored_field in le_dict: - del le_dict[ignored_field] - - checksum = hashlib.sha1() - self._hash_thing(checksum, le_dict) - keys['morphology-sha1'] = checksum.hexdigest() + for key in morph_dict: + if key not in ignored_fields: + keys[key] = morph_dict[key] if kind == 'stratum': keys['stratum-format-version'] = 1 elif kind == 'system': -- cgit v1.2.1 From 11926f251856e48955c986de9653cdfb829d1c0f Mon Sep 17 00:00:00 2001 From: Adam Coldrick Date: Fri, 18 Jul 2014 16:10:21 +0100 Subject: unittests: Test that the description field is ignored for chunks --- morphlib/cachekeycomputer_tests.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/morphlib/cachekeycomputer_tests.py b/morphlib/cachekeycomputer_tests.py index dd10307f..9e18b19d 100644 --- a/morphlib/cachekeycomputer_tests.py +++ b/morphlib/cachekeycomputer_tests.py @@ -36,15 +36,18 @@ class CacheKeyComputerTests(unittest.TestCase): for name, text in { 'chunk.morph': '''{ "name": "chunk", - "kind": "chunk" + "kind": "chunk", + "description": "A test chunk" }''', 'chunk2.morph': '''{ "name": "chunk2", - "kind": "chunk" + "kind": "chunk", + "description": "A test chunk" }''', 'chunk3.morph': '''{ "name": "chunk3", - "kind": "chunk" + "kind": "chunk", + "description": "A test chunk" }''', 'stratum.morph': '''{ "name": "stratum", -- cgit v1.2.1 From abb93656aa0b8de1e086c8f142bf468d3b50f26e Mon Sep 17 00:00:00 2001 From: Adam Coldrick Date: Fri, 18 Jul 2014 16:10:22 +0100 Subject: Don't set 'morph' field by default, and don't expect to find it --- morphlib/artifactresolver.py | 4 +++- morphlib/buildcommand.py | 3 ++- morphlib/morph2.py | 4 +--- morphlib/morph2_tests.py | 1 - 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/morphlib/artifactresolver.py b/morphlib/artifactresolver.py index c18042a3..9e3cea48 100644 --- a/morphlib/artifactresolver.py +++ b/morphlib/artifactresolver.py @@ -191,10 +191,12 @@ class ArtifactResolver(object): name_to_processed_artifacts = {} for info in source.morphology['chunks']: + filename = morphlib.util.sanitise_morphology_path( + info.get('morph', info['name'])) chunk_source = self._source_pool.lookup( info['repo'], info['ref'], - morphlib.util.sanitise_morphology_path(info['morph'])) + filename) chunk_name = chunk_source.morphology['name'] diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py index 45c6ef00..4dea7b68 100644 --- a/morphlib/buildcommand.py +++ b/morphlib/buildcommand.py @@ -231,7 +231,8 @@ class BuildCommand(object): for spec in specs: repo_name = spec.get('repo') or src.repo_name ref = spec.get('ref') or src.original_ref - filename = morphlib.util.sanitise_morphology_path(spec['morph']) + filename = morphlib.util.sanitise_morphology_path( + spec.get('morph', spec.get('name'))) logging.debug( 'Validating cross ref to %s:%s:%s' % (repo_name, ref, filename)) diff --git a/morphlib/morph2.py b/morphlib/morph2.py index 83971bb8..b49c0f73 100644 --- a/morphlib/morph2.py +++ b/morphlib/morph2.py @@ -191,8 +191,6 @@ class Morphology(object): for source in self['chunks']: if 'repo' not in source: self._set_default_value(source, 'repo', source['name']) - if 'morph' not in source: - self._set_default_value(source, 'morph', source['name']) if 'build-depends' not in source: self._set_default_value(source, 'build-depends', None) if 'build-mode' not in source: @@ -226,7 +224,7 @@ class Morphology(object): return info elif self['kind'] == 'stratum': for info in self['chunks']: - source_name = info.get('alias', info['morph']) + source_name = info.get('alias', info['name']) if source_name == name: return info raise KeyError('"%s" not found' % name) diff --git a/morphlib/morph2_tests.py b/morphlib/morph2_tests.py index d02aab1c..c9957ad5 100644 --- a/morphlib/morph2_tests.py +++ b/morphlib/morph2_tests.py @@ -95,7 +95,6 @@ class MorphologyTests(unittest.TestCase): ''') self.assertEqual(m['chunks'][0]['repo'], 'le-chunk') - self.assertEqual(m['chunks'][0]['morph'], 'le-chunk') self.assertEqual(m['chunks'][0]['build-depends'], None) def test_returns_dict_keys(self): -- cgit v1.2.1 From e4c105f914ebb0df18cedbb8111f9d93f79b79dc Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Fri, 18 Jul 2014 16:10:23 +0100 Subject: Rewrite traverse_morphs to be able to load chunk morphologies from definitions It will now load the morphology from the definitions repository if the "morph" field is present in the chunk spec. Rather than adapting the loop to fit yet-more changing circumstances, it has been partially rewritten, so there is one loop for loading systems and strata from definitions.git, another for chunks from definitions.git, and a third for chunks in the source repository. This is tidier than attempting to fit the logic in the main loop, as it makes it easier to remove afterwards when we no longer need to load chunk morphologies from the source repository. --- morphlib/app.py | 85 ++++++++++++++++++++++++++++++------------------ morphlib/buildcommand.py | 2 +- 2 files changed, 54 insertions(+), 33 deletions(-) diff --git a/morphlib/app.py b/morphlib/app.py index e0874317..a543443e 100644 --- a/morphlib/app.py +++ b/morphlib/app.py @@ -286,7 +286,7 @@ class Morph(cliapp.Application): morphlib.util.sanitise_morphology_path(args[2])) args = args[3:] - def create_source_pool(self, lrc, rrc, triplet): + def create_source_pool(self, lrc, rrc, repo, ref, filename): pool = morphlib.sourcepool.SourcePool() def add_to_pool(reponame, ref, filename, absref, tree, morphology): @@ -294,7 +294,7 @@ class Morph(cliapp.Application): morphology, filename) pool.add(source) - self.traverse_morphs([triplet], lrc, rrc, + self.traverse_morphs(repo, ref, [filename], lrc, rrc, update=not self.settings['no-git-update'], visit=add_to_pool) return pool @@ -338,53 +338,74 @@ class Morph(cliapp.Application): absref, tree = repo.resolve_ref(ref) return absref, tree - def traverse_morphs(self, triplets, lrc, rrc, update=True, + def traverse_morphs(self, definitions_repo, definitions_ref, + system_filenames, lrc, rrc, update=True, visit=lambda rn, rf, fn, arf, m: None): morph_factory = morphlib.morphologyfactory.MorphologyFactory(lrc, rrc, self) - queue = collections.deque(triplets) + definitions_queue = collections.deque(system_filenames) + chunk_in_definitions_repo_queue = [] + chunk_in_source_repo_queue = [] resolved_refs = {} resolved_morphologies = {} - while queue: - reponame, ref, filename = queue.popleft() + # Resolve the (repo, ref) pair for the definitions repo, cache result. + definitions_absref, definitions_tree = self.resolve_ref( + lrc, rrc, definitions_repo, definitions_ref, update) - # Resolve the (repo, ref) reference, cache result. - reference = (reponame, ref) - if not reference in resolved_refs: - resolved_refs[reference] = self.resolve_ref( - lrc, rrc, reponame, ref, update) - absref, tree = resolved_refs[reference] + while definitions_queue: + filename = definitions_queue.popleft() - # Fetch the (repo, ref, filename) morphology, cache result. - reference = (reponame, absref, filename) - if not reference in resolved_morphologies: - resolved_morphologies[reference] = \ - morph_factory.get_morphology(reponame, absref, filename) - morphology = resolved_morphologies[reference] + key = (definitions_repo, definitions_absref, filename) + if not key in resolved_morphologies: + resolved_morphologies[key] = morph_factory.get_morphology(*key) + morphology = resolved_morphologies[key] - visit(reponame, ref, filename, absref, tree, morphology) + visit(definitions_repo, definitions_ref, filename, + definitions_absref, definitions_tree, morphology) if morphology['kind'] == 'cluster': raise cliapp.AppException( "Cannot build a morphology of type 'cluster'.") elif morphology['kind'] == 'system': - queue.extend( - (s.get('repo') or reponame, - s.get('ref') or ref, - morphlib.util.sanitise_morphology_path(s['morph'])) + definitions_queue.extend( + morphlib.util.sanitise_morphology_path(s['morph']) for s in morphology['strata']) elif morphology['kind'] == 'stratum': if morphology['build-depends']: - queue.extend( - (s.get('repo') or reponame, - s.get('ref') or ref, - morphlib.util.sanitise_morphology_path(s['morph'])) + definitions_queue.extend( + morphlib.util.sanitise_morphology_path(s['morph']) for s in morphology['build-depends']) - queue.extend( - (c['repo'], - c['ref'], - morphlib.util.sanitise_morphology_path(c['morph'])) - for c in morphology['chunks']) + for c in morphology['chunks']: + if 'morph' not in c: + path = morphlib.util.sanitise_morphology_path( + c.get('morph', c['name'])) + chunk_in_source_repo_queue.append( + (c['repo'], c['ref'], path)) + continue + chunk_in_definitions_repo_queue.append( + (c['repo'], c['ref'], c['morph'])) + + for repo, ref, filename in chunk_in_definitions_repo_queue: + if (repo, ref) not in resolved_refs: + resolved_refs[repo, ref] = self.resolve_ref( + lrc, rrc, repo, ref, update) + absref, tree = resolved_refs[repo, ref] + key = (definitions_repo, definitions_absref, filename) + if not key in resolved_morphologies: + resolved_morphologies[key] = morph_factory.get_morphology(*key) + morphology = resolved_morphologies[key] + visit(repo, ref, filename, absref, tree, morphology) + + for repo, ref, filename in chunk_in_source_repo_queue: + if (repo, ref) not in resolved_refs: + resolved_refs[repo, ref] = self.resolve_ref( + lrc, rrc, repo, ref, update) + absref, tree = resolved_refs[repo, ref] + key = (repo, absref, filename) + if key not in resolved_morphologies: + resolved_morphologies[key] = morph_factory.get_morphology(*key) + morphology = resolved_morphologies[key] + visit(repo, ref, filename, absref, tree, morphology) def cache_repo_and_submodules(self, cache, url, ref, done): subs_to_process = set() diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py index 4dea7b68..1cb0c4d9 100644 --- a/morphlib/buildcommand.py +++ b/morphlib/buildcommand.py @@ -90,7 +90,7 @@ class BuildCommand(object): ''' self.app.status(msg='Creating source pool', chatty=True) srcpool = self.app.create_source_pool( - self.lrc, self.rrc, (repo_name, ref, filename)) + self.lrc, self.rrc, repo_name, ref, filename) return srcpool -- cgit v1.2.1 From e86eae318a83a429a0f27954866217ebf2efa6a5 Mon Sep 17 00:00:00 2001 From: Adam Coldrick Date: Fri, 18 Jul 2014 16:10:24 +0100 Subject: yarns: Test building and deployment of a system with a chunk in definitions --- yarns/implementations.yarn | 24 +++++++++++++----------- yarns/regression.yarn | 10 ++++------ yarns/splitting.yarn | 15 +++++++++------ 3 files changed, 26 insertions(+), 23 deletions(-) diff --git a/yarns/implementations.yarn b/yarns/implementations.yarn index 2c325985..ce229d6c 100644 --- a/yarns/implementations.yarn +++ b/yarns/implementations.yarn @@ -87,16 +87,12 @@ another to hold a chunk. chunks: - name: test-chunk repo: test:test-chunk + morph: test-chunk.morph ref: master build-mode: test build-depends: [] EOF - run_in "$DATADIR/gits/morphs" git init . - run_in "$DATADIR/gits/morphs" git add . - run_in "$DATADIR/gits/morphs" git commit -m Initial. - run_in "$DATADIR/gits/morphs" git tag -a "test-tag" -m "Tagging test-tag" - # Create the chunk repository. mkdir "$DATADIR/gits/test-chunk" @@ -105,7 +101,7 @@ another to hold a chunk. # dummy files in all the places that different kinds of files are # usually installed. e.g. executables in `/bin` and `/usr/bin` - install -m644 -D /dev/stdin << 'EOF' "$DATADIR/gits/test-chunk/test-chunk.morph" + install -m644 -D /dev/stdin << 'EOF' "$DATADIR/gits/morphs/test-chunk.morph" name: test-chunk kind: chunk build-system: manual @@ -239,9 +235,14 @@ another to hold a chunk. done EOF + run_in "$DATADIR/gits/morphs" git init . + run_in "$DATADIR/gits/morphs" git add . + run_in "$DATADIR/gits/morphs" git commit -m Initial. + run_in "$DATADIR/gits/morphs" git tag -a "test-tag" -m "Tagging test-tag" + run_in "$DATADIR/gits/test-chunk" git init . run_in "$DATADIR/gits/test-chunk" git add . - run_in "$DATADIR/gits/test-chunk" git commit -m Initial. + run_in "$DATADIR/gits/test-chunk" git commit --allow-empty -m Initial. # Create the Morph configuration file so we can access the repos # using test:foo URL aliases. @@ -262,15 +263,14 @@ have a morphology using the test architecture. IMPLEMENTS GIVEN a system called (\S+) for the test architecture in the git server - cat << EOF > "$DATADIR/gits/test-chunk/stage1-chunk.morph" + cat << EOF > "$DATADIR/gits/morphs/stage1-chunk.morph" name: stage1-chunk kind: chunk build-system: dummy EOF - run_in "$DATADIR/gits/test-chunk" git init . - run_in "$DATADIR/gits/test-chunk" git add . - run_in "$DATADIR/gits/test-chunk" git commit -m Initial. + run_in "$DATADIR/gits/morphs" git add . + run_in "$DATADIR/gits/morphs" git commit -m "Add chunk for $MATCH_1" install -m644 -D /dev/stdin << EOF "$DATADIR/gits/morphs/strata/build-essential.morph" @@ -278,6 +278,7 @@ have a morphology using the test architecture. kind: stratum chunks: - name: stage1-chunk + morph: stage1-chunk.morph repo: test:test-chunk ref: master build-mode: bootstrap @@ -291,6 +292,7 @@ have a morphology using the test architecture. - morph: strata/build-essential.morph chunks: - name: test-chunk + morph: test-chunk.morph repo: test:test-chunk ref: master build-mode: test diff --git a/yarns/regression.yarn b/yarns/regression.yarn index 05e2e70c..e5b3d875 100644 --- a/yarns/regression.yarn +++ b/yarns/regression.yarn @@ -88,17 +88,15 @@ Implementations kind: stratum chunks: - name: bootstrap-chunk + morph: bootstrap-chunk.morph repo: test:test-chunk ref: master build-mode: bootstrap build-depends: [] EOF + sed -e 's/name: test-chunk/name: bootstrap-chunk/g' \ + "$DATADIR/gits/morphs/test-chunk.morph" \ + > "$DATADIR/gits/morphs/bootstrap-chunk.morph" run_in "$DATADIR/gits/morphs" git add . run_in "$DATADIR/gits/morphs" git commit -m "Add bootstrap-system" - - sed -e 's/test-chunk/bootstrap-chunk/g' \ - "$DATADIR/gits/test-chunk/test-chunk.morph" \ - > "$DATADIR/gits/test-chunk/bootstrap-chunk.morph" - run_in "$DATADIR/gits/test-chunk" git add . - run_in "$DATADIR/gits/test-chunk" git commit -m "Add bootstrap-chunk" diff --git a/yarns/splitting.yarn b/yarns/splitting.yarn index d35153e4..1b67a881 100644 --- a/yarns/splitting.yarn +++ b/yarns/splitting.yarn @@ -90,7 +90,7 @@ Implementations IMPLEMENTS GIVEN chunk (\S+) includes the default splitting rules # Append default products rules name="$(basename "${MATCH_1%.*}")" - cat <>"$DATADIR/gits/$name/$MATCH_1" + cat <>"$DATADIR/gits/morphs/$MATCH_1" products: - artifact: $name-bins include: [ "(usr/)?s?bin/.*" ] @@ -117,8 +117,8 @@ Implementations - artifact: $name-misc include: [ .* ] EOF - run_in "$DATADIR/gits/$name" git add "$MATCH_1" - run_in "$DATADIR/gits/$name" git commit -m 'Add default splitting rules' + run_in "$DATADIR/gits/morphs" git add "$MATCH_1" + run_in "$DATADIR/gits/morphs" git commit -m 'Add default splitting rules' IMPLEMENTS GIVEN stratum (\S+) includes the default splitting rules name=$(basename "${MATCH_1%.*}") @@ -142,6 +142,7 @@ Implementations - name: test-chunk repo: test:test-chunk ref: master + morph: test-chunk.morph build-mode: test build-depends: [] artifacts: @@ -169,14 +170,14 @@ Implementations IMPLEMENTS GIVEN stratum (\S+) has chunks that aren't used in (\S+) # Create an extra chunk that will never successfully build - cat >"$DATADIR/gits/test-chunk/unbuildable-chunk.morph" <"$DATADIR/gits/morphs/unbuildable-chunk.morph" < Date: Fri, 18 Jul 2014 16:10:25 +0100 Subject: yarns: Make test-chunk use files from its source repo It is entirely possible that we could accidentally give chunks that use morphologies from the definitions repository, the definitions repository to build from, rather than the source repository. --- yarns/implementations.yarn | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/yarns/implementations.yarn b/yarns/implementations.yarn index ce229d6c..5b5b1724 100644 --- a/yarns/implementations.yarn +++ b/yarns/implementations.yarn @@ -156,7 +156,7 @@ another to hold a chunk. # autotools configuration, they go into /libexec. - | - install -D /dev/null "$DESTDIR/$PREFIX/libexec/test-bin" + install -D test-bin "$DESTDIR/$PREFIX/libexec/test-bin" # As well as run-time libraries, there's development files. For C # this is headers, which describe the API of the libraries, which @@ -169,7 +169,7 @@ another to hold a chunk. # `-m` option. - | - install -D -m 644 /dev/null "$DESTDIR/$PREFIX/include/test.h" + install -D -m 644 test.h "$DESTDIR/$PREFIX/include/test.h" # `pkg-config` is a standard way to locate libraries and get the # compiler flags needed to build with the library. It's also used @@ -179,7 +179,7 @@ another to hold a chunk. - | for pkgdir in lib lib32 lib64 share; do - install -D -m 644 /dev/null \ + install -D -m 644 test.pc \ "$DESTDIR/$PREFIX/$pkgdir/pkgconfig/test.pc" done @@ -241,6 +241,19 @@ another to hold a chunk. run_in "$DATADIR/gits/morphs" git tag -a "test-tag" -m "Tagging test-tag" run_in "$DATADIR/gits/test-chunk" git init . + cat > "$DATADIR/gits/test-chunk/test-bin" <<'EOF' + #!/bin/sh + echo Hello World + EOF + cat > "$DATADIR/gits/test-chunk/test.h" <<'EOF' + int foo(void); + EOF + cat > "$DATADIR/gits/test-chunk/test.pc" <<'EOF' + prefix=/usr + includedir=${prefix}/include + Name: test + Cflags: -I{includedir} + EOF run_in "$DATADIR/gits/test-chunk" git add . run_in "$DATADIR/gits/test-chunk" git commit --allow-empty -m Initial. -- cgit v1.2.1