From 60c378c55d5d0ef89184b49ae95e445f8de422e3 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Thu, 23 Jul 2015 18:02:00 +0000 Subject: Add support for Baserock definitions version 6 Change-Id: I891d1b13ed0581b293fe6b09b3cc73af8fd81d67 --- morphlib/morphloader.py | 77 +++++++++++++++++------- morphlib/morphloader_tests.py | 124 +++++++++++++++++---------------------- morphlib/sourceresolver.py | 44 ++++++++++---- morphlib/sourceresolver_tests.py | 2 + yarns/implementations.yarn | 3 +- 5 files changed, 150 insertions(+), 100 deletions(-) diff --git a/morphlib/morphloader.py b/morphlib/morphloader.py index 171954f9..479bc8fb 100644 --- a/morphlib/morphloader.py +++ b/morphlib/morphloader.py @@ -158,6 +158,30 @@ class ChunkSpecRefNotStringError(MorphologyValidationError): 'in stratum %(stratum_name)s is not a string' % locals()) +class ChunkSpecConflictingFieldsError(MorphologyValidationError): + + def __init__(self, fields, chunk_name, stratum_name): + self.chunk_name = chunk_name + self.stratum_name = stratum_name + self.fields = fields + MorphologyValidationError.__init__( + self, 'Conflicting fields "%s" for %s in stratum %s.' % ( + ', and '.join(fields), chunk_name, stratum_name)) + + +class ChunkSpecNoBuildInstructionsError(MorphologyValidationError): + + def __init__(self, chunk_name, stratum_name): + self.chunk_name = chunk_name + self.stratum_name = stratum_name + self.msg = ( + 'Chunk %(chunk_name)s in stratum %(stratum_name)s has no ' + 'build-system defined, and no chunk .morph file referenced ' + 'either. Please specify how to build the chunk, either by setting ' + '"build-system: " in the stratum, or adding a chunk .morph file ' + 'and setting "morph: " in the stratum.' % locals()) + + class SystemStrataNotListError(MorphologyValidationError): def __init__(self, system_name, strata_type): @@ -386,6 +410,8 @@ class MorphologyLoader(object): 'pre-strip-commands': None, 'strip-commands': None, 'post-strip-commands': None}) + + self._definitions_version = definitions_version self._lookup_build_system = lookup_build_system def parse_morphology_text(self, text, morph_filename): @@ -533,25 +559,6 @@ class MorphologyLoader(object): if len(morph.get('chunks', [])) == 0: raise EmptyStratumError(morph['name'], morph.filename) - # All chunk names must be unique within a stratum. - names = set() - for spec in morph['chunks']: - name = spec.get('alias', spec['name']) - if name in names: - raise DuplicateChunkError(morph['name'], name) - names.add(name) - - # All chunk refs must be strings. - for spec in morph['chunks']: - if 'ref' in spec: - ref = spec['ref'] - if ref == None: - raise EmptyRefError( - spec.get('alias', spec['name']), morph.filename) - elif not isinstance(ref, basestring): - raise ChunkSpecRefNotStringError( - ref, spec.get('alias', spec['name']), morph.filename) - # Require build-dependencies for the stratum itself, unless # it has chunks built in bootstrap mode. if 'build-depends' in morph: @@ -573,15 +580,45 @@ class MorphologyLoader(object): # Validate build-dependencies if specified self._validate_stratum_specs_fields(morph, 'build-depends') - # Check build-dependencies for each chunk. + # All chunk names must be unique within a stratum. + names = set() + for spec in morph['chunks']: + name = spec.get('alias', spec['name']) + if name in names: + raise DuplicateChunkError(morph['name'], name) + names.add(name) + + # Check each reference to a chunk. for spec in morph['chunks']: chunk_name = spec.get('alias', spec['name']) + + # All chunk refs must be strings. + if 'ref' in spec: + ref = spec['ref'] + if ref == None: + raise EmptyRefError( + spec.get('alias', spec['name']), morph.filename) + elif not isinstance(ref, basestring): + raise ChunkSpecRefNotStringError( + ref, spec.get('alias', spec['name']), morph.filename) + + # The build-depends field must be a list. if 'build-depends' in spec: if not isinstance(spec['build-depends'], list): raise InvalidTypeError( '%s.build-depends' % chunk_name, list, type(spec['build-depends']), morph['name']) + if self._definitions_version >= 6: + # Either 'morph' or 'build-system' must be specified. + if 'morph' in spec and 'build-system' in spec: + raise ChunkSpecConflictingFieldsError( + ['morph', 'build-system'], chunk_name, morph.filename) + if 'morph' not in spec and 'build-system' not in spec: + raise ChunkSpecNoBuildInstructionsError( + chunk_name, morph.filename) + + @classmethod def _validate_chunk(cls, morphology): errors = [] diff --git a/morphlib/morphloader_tests.py b/morphlib/morphloader_tests.py index f11bf5c1..6cb93094 100644 --- a/morphlib/morphloader_tests.py +++ b/morphlib/morphloader_tests.py @@ -26,10 +26,31 @@ import morphlib from morphlib.morphloader import MorphologyObsoleteFieldWarning +def stratum_template(name): + '''Returns a valid example stratum, with one chunk reference.''' + m = morphlib.morphology.Morphology({ + "name": name, + "kind": "stratum", + "build-depends": [ + { "morph": "foo" }, + ], + "chunks": [ + { + "name": "chunk", + "repo": "test:repo", + "ref": "sha1", + "build-system": "manual", + } + ] + }) + return m + + class MorphologyLoaderTests(unittest.TestCase): def setUp(self): - self.loader = morphlib.morphloader.MorphologyLoader() + self.loader = morphlib.morphloader.MorphologyLoader( + definitions_version=6) self.tempdir = tempfile.mkdtemp() self.filename = os.path.join(self.tempdir, 'foo.morph') @@ -318,6 +339,9 @@ chunks: { "kind": "stratum", "name": "foo", + "build-depends": [ + {"morph": "bar"}, + ], "chunks": [ { "name": "chunk", @@ -358,93 +382,49 @@ chunks: self.assertEqual(m['arch'], 'armv7l') def test_validate_requires_build_deps_or_bootstrap_mode_for_strata(self): - m = morphlib.morphology.Morphology( - { - "name": "stratum-no-bdeps-no-bootstrap", - "kind": "stratum", - "chunks": [ - { - "name": "chunk", - "repo": "test:repo", - "ref": "sha1", - "build-depends": [] - } - ] - }) + m = stratum_template("stratum-no-bdeps-no-bootstrap") + + self.loader.validate(m) + del m['build-depends'] self.assertRaises( morphlib.morphloader.NoStratumBuildDependenciesError, self.loader.validate, m) - m['build-depends'] = [ - { - "morph": "foo", - }, - ] - self.loader.validate(m) - - del m['build-depends'] m['chunks'][0]['build-mode'] = 'bootstrap' self.loader.validate(m) def test_validate_stratum_build_deps_are_list(self): - m = morphlib.morphology.Morphology( - { - "name": "stratum-invalid-bdeps", - "kind": "stratum", - "build-depends": 0.1, - "chunks": [ - { - "name": "chunk", - "repo": "test:repo", - "ref": "sha1", - "build-depends": [] - } - ] - }) - + m = stratum_template("stratum-invalid-bdeps") + m['build-depends'] = 0.1 self.assertRaises( morphlib.morphloader.InvalidTypeError, self.loader.validate, m) def test_validate_chunk_build_deps_are_list(self): - m = morphlib.morphology.Morphology( - { - "name": "stratum-invalid-bdeps", - "kind": "stratum", - "build-depends": [ - { "morph": "foo" }, - ], - "chunks": [ - { - "name": "chunk", - "repo": "test:repo", - "ref": "sha1", - "build-depends": 0.1 - } - ] - }) - + m = stratum_template("stratum-invalid-bdeps") + m['chunks'][0]['build-depends'] = 0.1 self.assertRaises( morphlib.morphloader.InvalidTypeError, self.loader.validate, m) - def test_validate_requires_chunks_in_strata(self): - m = morphlib.morphology.Morphology( - { - "name": "stratum", - "kind": "stratum", - "chunks": [ - ], - "build-depends": [ - { - "repo": "foo", - "ref": "foo", - "morph": "foo", - }, - ], - }) + def test_validate_chunk_has_build_instructions(self): + m = stratum_template("stratum-no-build-instructions") + del m['chunks'][0]['build-system'] + self.assertRaises( + morphlib.morphloader.ChunkSpecNoBuildInstructionsError, + self.loader.validate, m) + + def test_validate_chunk_conflicting_build_instructions(self): + m = stratum_template("stratum-conflicting-build-instructions") + m['chunks'][0]['morph'] = 'conflicting-information' + self.assertRaises( + morphlib.morphloader.ChunkSpecConflictingFieldsError, + self.loader.validate, m) + def test_validate_requires_chunks_in_strata(self): + m = stratum_template("stratum-no-chunks") + del m['chunks'] self.assertRaises( morphlib.morphloader.EmptyStratumError, self.loader.validate, m) @@ -622,6 +602,10 @@ build-system: dummy 'pre-install-commands': None, 'post-install-commands': None, + 'strip-commands': None, + 'pre-strip-commands': None, + 'post-strip-commands': None, + 'products': [], 'system-integration': [], 'devices': [], @@ -803,6 +787,7 @@ build-system: dummy { "name": "le-chunk", "ref": "ref", + "build-system": "manual", "build-depends": [], } ] @@ -823,6 +808,7 @@ build-system: dummy "repo": "le-chunk", "morph": "le-chunk", "ref": "ref", + "build-system": "manual", "build-depends": [], } ] diff --git a/morphlib/sourceresolver.py b/morphlib/sourceresolver.py index af704945..cd83f0ea 100644 --- a/morphlib/sourceresolver.py +++ b/morphlib/sourceresolver.py @@ -33,7 +33,7 @@ tree_cache_filename = 'trees.cache.pickle' buildsystem_cache_size = 10000 buildsystem_cache_filename = 'detected-chunk-buildsystems.cache.pickle' -supported_versions = [3, 4, 5] +supported_versions = [3, 4, 5, 6] class PickleCacheManager(object): # pragma: no cover '''Cache manager for PyLRU that reads and writes to Pickle files. @@ -399,6 +399,9 @@ class SourceResolver(object): sanitise_morphology_path(s['morph']) for s in morphology['build-depends']) for c in morphology['chunks']: + # This field is only valid in strata from definitions + # version 6 onwards. Validation is done in morphloader.py. + buildsystem = c.get('build-system') if 'morph' not in c: # Autodetect a path if one is not given. This is to # support the deprecated approach of putting the chunk @@ -412,7 +415,8 @@ class SourceResolver(object): path = sanitise_morphology_path( c.get('morph', c['name'])) - chunk_queue.add((c['repo'], c['ref'], path)) + chunk_queue.add((c['repo'], c['ref'], path, + buildsystem)) else: # Now, does this path actually exist? path = c['morph'] @@ -424,7 +428,8 @@ class SourceResolver(object): raise MorphologyReferenceNotFoundError( path, filename) - chunk_queue.add((c['repo'], c['ref'], path)) + chunk_queue.add((c['repo'], c['ref'], path, + buildsystem)) return chunk_queue @@ -498,9 +503,9 @@ class SourceResolver(object): def process_chunk(self, resolved_morphologies, resolved_trees, resolved_buildsystems, definitions_checkout_dir, - definitions_repo, definitions_absref, morph_loader, - chunk_repo, chunk_ref, filename, - visit): # pragma: no cover + definitions_repo, definitions_absref, + definitions_version, morph_loader, chunk_repo, chunk_ref, + filename, chunk_buildsystem, visit): # pragma: no cover absref = None tree = None chunk_key = None @@ -532,7 +537,26 @@ class SourceResolver(object): resolved_morphologies, resolved_buildsystems, morph_loader, definition_key, chunk_key, buildsystem, morph_name) - if chunk_key in resolved_buildsystems: + if definitions_version >= 6: + # All build-system information is specified in the definitions from + # version 6 onwards. Either 'morph' or 'build-system' should be + # specified for each chunk. + if chunk_buildsystem is None: + # The validation done in 'morphloader' should mean that this + # never happens. + raise SourceResolverError( + 'Please specify either "build-system" or "morph" for %s.' % + chunk_key) + + buildsystem = morphlib.buildsystem.lookup_build_system( + chunk_buildsystem) + + if definition_key in resolved_morphologies: + morphology = resolved_morphologies[definition_key] + else: + morphology = generate_morph_and_cache_buildsystem(buildsystem) + + elif chunk_key in resolved_buildsystems: logging.debug('Build system for %s is cached', str(chunk_key)) self.status(msg='Build system for %(chunk)s is cached', chunk=str(chunk_key), @@ -615,13 +639,13 @@ class SourceResolver(object): system_filenames, visit) # Now process all the chunks involved in the build. - for repo, ref, filename in chunk_queue: + for repo, ref, filename, buildsystem in chunk_queue: self.process_chunk(resolved_morphologies, resolved_trees, resolved_buildsystems, definitions_checkout_dir, definitions_repo, definitions_absref, - morph_loader, repo, ref, filename, - visit) + definitions_version, morph_loader, repo, + ref, filename, buildsystem, visit) def create_source_pool(lrc, rrc, repo, ref, filenames, cachedir, diff --git a/morphlib/sourceresolver_tests.py b/morphlib/sourceresolver_tests.py index 5985579c..6fe1dc54 100644 --- a/morphlib/sourceresolver_tests.py +++ b/morphlib/sourceresolver_tests.py @@ -353,3 +353,5 @@ class SourceResolverTests(unittest.TestCase): morphlib.morphloader.MorphologyLoader(), 'reponame', 'sha1', 'stratum-empty.morph') + def test_parses_version_file(self): + self.assertEqual(self.sr._parse_version_file('version: 6\n'), 6) diff --git a/yarns/implementations.yarn b/yarns/implementations.yarn index 52fbd0bb..3bef0374 100644 --- a/yarns/implementations.yarn +++ b/yarns/implementations.yarn @@ -238,7 +238,7 @@ another to hold a chunk. mkdir "$DATADIR/gits/morphs" cd "$DATADIR/gits/morphs" git init . - echo 'version: 5' > VERSION + echo 'version: 6' > VERSION arch=$(run_morph print-architecture) install -m644 -D /dev/stdin << EOF "systems/test-system.morph" @@ -274,6 +274,7 @@ another to hold a chunk. ref: $(run_in "$DATADIR/gits/bootstrap-chunk" git rev-parse bootstrap) unpetrify-ref: nootstrap build-mode: bootstrap + build-system: autotools build-depends: [] - name: stage2-chunk morph: stage2-chunk.morph -- cgit v1.2.1