summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorSam Thursfield <sam.thursfield@codethink.co.uk>2015-07-23 18:02:00 +0000
committerSam Thursfield <sam.thursfield@codethink.co.uk>2015-07-29 11:21:40 +0000
commit60c378c55d5d0ef89184b49ae95e445f8de422e3 (patch)
tree6ad4d0cdfd509ba252ab4339ebce3c4522dfe5dc
parent1ae4b3d77bdf7f666c7817346baf08129a6247f3 (diff)
downloadmorph-60c378c55d5d0ef89184b49ae95e445f8de422e3.tar.gz
Add support for Baserock definitions version 6
Change-Id: I891d1b13ed0581b293fe6b09b3cc73af8fd81d67
-rw-r--r--morphlib/morphloader.py77
-rw-r--r--morphlib/morphloader_tests.py124
-rw-r--r--morphlib/sourceresolver.py44
-rw-r--r--morphlib/sourceresolver_tests.py2
-rw-r--r--yarns/implementations.yarn3
5 files changed, 150 insertions, 100 deletions
diff --git a/morphlib/morphloader.py b/morphlib/morphloader.py
index 171954f9..479bc8fb 100644
--- a/morphlib/morphloader.py
+++ b/morphlib/morphloader.py
@@ -158,6 +158,30 @@ class ChunkSpecRefNotStringError(MorphologyValidationError):
'in stratum %(stratum_name)s is not a string' % locals())
+class ChunkSpecConflictingFieldsError(MorphologyValidationError):
+
+ def __init__(self, fields, chunk_name, stratum_name):
+ self.chunk_name = chunk_name
+ self.stratum_name = stratum_name
+ self.fields = fields
+ MorphologyValidationError.__init__(
+ self, 'Conflicting fields "%s" for %s in stratum %s.' % (
+ ', and '.join(fields), chunk_name, stratum_name))
+
+
+class ChunkSpecNoBuildInstructionsError(MorphologyValidationError):
+
+ def __init__(self, chunk_name, stratum_name):
+ self.chunk_name = chunk_name
+ self.stratum_name = stratum_name
+ self.msg = (
+ 'Chunk %(chunk_name)s in stratum %(stratum_name)s has no '
+ 'build-system defined, and no chunk .morph file referenced '
+ 'either. Please specify how to build the chunk, either by setting '
+ '"build-system: " in the stratum, or adding a chunk .morph file '
+ 'and setting "morph: " in the stratum.' % locals())
+
+
class SystemStrataNotListError(MorphologyValidationError):
def __init__(self, system_name, strata_type):
@@ -386,6 +410,8 @@ class MorphologyLoader(object):
'pre-strip-commands': None,
'strip-commands': None,
'post-strip-commands': None})
+
+ self._definitions_version = definitions_version
self._lookup_build_system = lookup_build_system
def parse_morphology_text(self, text, morph_filename):
@@ -533,25 +559,6 @@ class MorphologyLoader(object):
if len(morph.get('chunks', [])) == 0:
raise EmptyStratumError(morph['name'], morph.filename)
- # All chunk names must be unique within a stratum.
- names = set()
- for spec in morph['chunks']:
- name = spec.get('alias', spec['name'])
- if name in names:
- raise DuplicateChunkError(morph['name'], name)
- names.add(name)
-
- # All chunk refs must be strings.
- for spec in morph['chunks']:
- if 'ref' in spec:
- ref = spec['ref']
- if ref == None:
- raise EmptyRefError(
- spec.get('alias', spec['name']), morph.filename)
- elif not isinstance(ref, basestring):
- raise ChunkSpecRefNotStringError(
- ref, spec.get('alias', spec['name']), morph.filename)
-
# Require build-dependencies for the stratum itself, unless
# it has chunks built in bootstrap mode.
if 'build-depends' in morph:
@@ -573,15 +580,45 @@ class MorphologyLoader(object):
# Validate build-dependencies if specified
self._validate_stratum_specs_fields(morph, 'build-depends')
- # Check build-dependencies for each chunk.
+ # All chunk names must be unique within a stratum.
+ names = set()
+ for spec in morph['chunks']:
+ name = spec.get('alias', spec['name'])
+ if name in names:
+ raise DuplicateChunkError(morph['name'], name)
+ names.add(name)
+
+ # Check each reference to a chunk.
for spec in morph['chunks']:
chunk_name = spec.get('alias', spec['name'])
+
+ # All chunk refs must be strings.
+ if 'ref' in spec:
+ ref = spec['ref']
+ if ref == None:
+ raise EmptyRefError(
+ spec.get('alias', spec['name']), morph.filename)
+ elif not isinstance(ref, basestring):
+ raise ChunkSpecRefNotStringError(
+ ref, spec.get('alias', spec['name']), morph.filename)
+
+ # The build-depends field must be a list.
if 'build-depends' in spec:
if not isinstance(spec['build-depends'], list):
raise InvalidTypeError(
'%s.build-depends' % chunk_name, list,
type(spec['build-depends']), morph['name'])
+ if self._definitions_version >= 6:
+ # Either 'morph' or 'build-system' must be specified.
+ if 'morph' in spec and 'build-system' in spec:
+ raise ChunkSpecConflictingFieldsError(
+ ['morph', 'build-system'], chunk_name, morph.filename)
+ if 'morph' not in spec and 'build-system' not in spec:
+ raise ChunkSpecNoBuildInstructionsError(
+ chunk_name, morph.filename)
+
+
@classmethod
def _validate_chunk(cls, morphology):
errors = []
diff --git a/morphlib/morphloader_tests.py b/morphlib/morphloader_tests.py
index f11bf5c1..6cb93094 100644
--- a/morphlib/morphloader_tests.py
+++ b/morphlib/morphloader_tests.py
@@ -26,10 +26,31 @@ import morphlib
from morphlib.morphloader import MorphologyObsoleteFieldWarning
+def stratum_template(name):
+ '''Returns a valid example stratum, with one chunk reference.'''
+ m = morphlib.morphology.Morphology({
+ "name": name,
+ "kind": "stratum",
+ "build-depends": [
+ { "morph": "foo" },
+ ],
+ "chunks": [
+ {
+ "name": "chunk",
+ "repo": "test:repo",
+ "ref": "sha1",
+ "build-system": "manual",
+ }
+ ]
+ })
+ return m
+
+
class MorphologyLoaderTests(unittest.TestCase):
def setUp(self):
- self.loader = morphlib.morphloader.MorphologyLoader()
+ self.loader = morphlib.morphloader.MorphologyLoader(
+ definitions_version=6)
self.tempdir = tempfile.mkdtemp()
self.filename = os.path.join(self.tempdir, 'foo.morph')
@@ -318,6 +339,9 @@ chunks:
{
"kind": "stratum",
"name": "foo",
+ "build-depends": [
+ {"morph": "bar"},
+ ],
"chunks": [
{
"name": "chunk",
@@ -358,93 +382,49 @@ chunks:
self.assertEqual(m['arch'], 'armv7l')
def test_validate_requires_build_deps_or_bootstrap_mode_for_strata(self):
- m = morphlib.morphology.Morphology(
- {
- "name": "stratum-no-bdeps-no-bootstrap",
- "kind": "stratum",
- "chunks": [
- {
- "name": "chunk",
- "repo": "test:repo",
- "ref": "sha1",
- "build-depends": []
- }
- ]
- })
+ m = stratum_template("stratum-no-bdeps-no-bootstrap")
+
+ self.loader.validate(m)
+ del m['build-depends']
self.assertRaises(
morphlib.morphloader.NoStratumBuildDependenciesError,
self.loader.validate, m)
- m['build-depends'] = [
- {
- "morph": "foo",
- },
- ]
- self.loader.validate(m)
-
- del m['build-depends']
m['chunks'][0]['build-mode'] = 'bootstrap'
self.loader.validate(m)
def test_validate_stratum_build_deps_are_list(self):
- m = morphlib.morphology.Morphology(
- {
- "name": "stratum-invalid-bdeps",
- "kind": "stratum",
- "build-depends": 0.1,
- "chunks": [
- {
- "name": "chunk",
- "repo": "test:repo",
- "ref": "sha1",
- "build-depends": []
- }
- ]
- })
-
+ m = stratum_template("stratum-invalid-bdeps")
+ m['build-depends'] = 0.1
self.assertRaises(
morphlib.morphloader.InvalidTypeError,
self.loader.validate, m)
def test_validate_chunk_build_deps_are_list(self):
- m = morphlib.morphology.Morphology(
- {
- "name": "stratum-invalid-bdeps",
- "kind": "stratum",
- "build-depends": [
- { "morph": "foo" },
- ],
- "chunks": [
- {
- "name": "chunk",
- "repo": "test:repo",
- "ref": "sha1",
- "build-depends": 0.1
- }
- ]
- })
-
+ m = stratum_template("stratum-invalid-bdeps")
+ m['chunks'][0]['build-depends'] = 0.1
self.assertRaises(
morphlib.morphloader.InvalidTypeError,
self.loader.validate, m)
- def test_validate_requires_chunks_in_strata(self):
- m = morphlib.morphology.Morphology(
- {
- "name": "stratum",
- "kind": "stratum",
- "chunks": [
- ],
- "build-depends": [
- {
- "repo": "foo",
- "ref": "foo",
- "morph": "foo",
- },
- ],
- })
+ def test_validate_chunk_has_build_instructions(self):
+ m = stratum_template("stratum-no-build-instructions")
+ del m['chunks'][0]['build-system']
+ self.assertRaises(
+ morphlib.morphloader.ChunkSpecNoBuildInstructionsError,
+ self.loader.validate, m)
+
+ def test_validate_chunk_conflicting_build_instructions(self):
+ m = stratum_template("stratum-conflicting-build-instructions")
+ m['chunks'][0]['morph'] = 'conflicting-information'
+ self.assertRaises(
+ morphlib.morphloader.ChunkSpecConflictingFieldsError,
+ self.loader.validate, m)
+ def test_validate_requires_chunks_in_strata(self):
+ m = stratum_template("stratum-no-chunks")
+ del m['chunks']
self.assertRaises(
morphlib.morphloader.EmptyStratumError,
self.loader.validate, m)
@@ -622,6 +602,10 @@ build-system: dummy
'pre-install-commands': None,
'post-install-commands': None,
+ 'strip-commands': None,
+ 'pre-strip-commands': None,
+ 'post-strip-commands': None,
+
'products': [],
'system-integration': [],
'devices': [],
@@ -803,6 +787,7 @@ build-system: dummy
{
"name": "le-chunk",
"ref": "ref",
+ "build-system": "manual",
"build-depends": [],
}
]
@@ -823,6 +808,7 @@ build-system: dummy
"repo": "le-chunk",
"morph": "le-chunk",
"ref": "ref",
+ "build-system": "manual",
"build-depends": [],
}
]
diff --git a/morphlib/sourceresolver.py b/morphlib/sourceresolver.py
index af704945..cd83f0ea 100644
--- a/morphlib/sourceresolver.py
+++ b/morphlib/sourceresolver.py
@@ -33,7 +33,7 @@ tree_cache_filename = 'trees.cache.pickle'
buildsystem_cache_size = 10000
buildsystem_cache_filename = 'detected-chunk-buildsystems.cache.pickle'
-supported_versions = [3, 4, 5]
+supported_versions = [3, 4, 5, 6]
class PickleCacheManager(object): # pragma: no cover
'''Cache manager for PyLRU that reads and writes to Pickle files.
@@ -399,6 +399,9 @@ class SourceResolver(object):
sanitise_morphology_path(s['morph'])
for s in morphology['build-depends'])
for c in morphology['chunks']:
+ # This field is only valid in strata from definitions
+ # version 6 onwards. Validation is done in morphloader.py.
+ buildsystem = c.get('build-system')
if 'morph' not in c:
# Autodetect a path if one is not given. This is to
# support the deprecated approach of putting the chunk
@@ -412,7 +415,8 @@ class SourceResolver(object):
path = sanitise_morphology_path(
c.get('morph', c['name']))
- chunk_queue.add((c['repo'], c['ref'], path))
+ chunk_queue.add((c['repo'], c['ref'], path,
+ buildsystem))
else:
# Now, does this path actually exist?
path = c['morph']
@@ -424,7 +428,8 @@ class SourceResolver(object):
raise MorphologyReferenceNotFoundError(
path, filename)
- chunk_queue.add((c['repo'], c['ref'], path))
+ chunk_queue.add((c['repo'], c['ref'], path,
+ buildsystem))
return chunk_queue
@@ -498,9 +503,9 @@ class SourceResolver(object):
def process_chunk(self, resolved_morphologies, resolved_trees,
resolved_buildsystems, definitions_checkout_dir,
- definitions_repo, definitions_absref, morph_loader,
- chunk_repo, chunk_ref, filename,
- visit): # pragma: no cover
+ definitions_repo, definitions_absref,
+ definitions_version, morph_loader, chunk_repo, chunk_ref,
+ filename, chunk_buildsystem, visit): # pragma: no cover
absref = None
tree = None
chunk_key = None
@@ -532,7 +537,26 @@ class SourceResolver(object):
resolved_morphologies, resolved_buildsystems, morph_loader,
definition_key, chunk_key, buildsystem, morph_name)
- if chunk_key in resolved_buildsystems:
+ if definitions_version >= 6:
+ # All build-system information is specified in the definitions from
+ # version 6 onwards. Either 'morph' or 'build-system' should be
+ # specified for each chunk.
+ if chunk_buildsystem is None:
+ # The validation done in 'morphloader' should mean that this
+ # never happens.
+ raise SourceResolverError(
+ 'Please specify either "build-system" or "morph" for %s.' %
+ chunk_key)
+
+ buildsystem = morphlib.buildsystem.lookup_build_system(
+ chunk_buildsystem)
+
+ if definition_key in resolved_morphologies:
+ morphology = resolved_morphologies[definition_key]
+ else:
+ morphology = generate_morph_and_cache_buildsystem(buildsystem)
+
+ elif chunk_key in resolved_buildsystems:
logging.debug('Build system for %s is cached', str(chunk_key))
self.status(msg='Build system for %(chunk)s is cached',
chunk=str(chunk_key),
@@ -615,13 +639,13 @@ class SourceResolver(object):
system_filenames, visit)
# Now process all the chunks involved in the build.
- for repo, ref, filename in chunk_queue:
+ for repo, ref, filename, buildsystem in chunk_queue:
self.process_chunk(resolved_morphologies, resolved_trees,
resolved_buildsystems,
definitions_checkout_dir,
definitions_repo, definitions_absref,
- morph_loader, repo, ref, filename,
- visit)
+ definitions_version, morph_loader, repo,
+ ref, filename, buildsystem, visit)
def create_source_pool(lrc, rrc, repo, ref, filenames, cachedir,
diff --git a/morphlib/sourceresolver_tests.py b/morphlib/sourceresolver_tests.py
index 5985579c..6fe1dc54 100644
--- a/morphlib/sourceresolver_tests.py
+++ b/morphlib/sourceresolver_tests.py
@@ -353,3 +353,5 @@ class SourceResolverTests(unittest.TestCase):
morphlib.morphloader.MorphologyLoader(), 'reponame', 'sha1',
'stratum-empty.morph')
+ def test_parses_version_file(self):
+ self.assertEqual(self.sr._parse_version_file('version: 6\n'), 6)
diff --git a/yarns/implementations.yarn b/yarns/implementations.yarn
index 52fbd0bb..3bef0374 100644
--- a/yarns/implementations.yarn
+++ b/yarns/implementations.yarn
@@ -238,7 +238,7 @@ another to hold a chunk.
mkdir "$DATADIR/gits/morphs"
cd "$DATADIR/gits/morphs"
git init .
- echo 'version: 5' > VERSION
+ echo 'version: 6' > VERSION
arch=$(run_morph print-architecture)
install -m644 -D /dev/stdin << EOF "systems/test-system.morph"
@@ -274,6 +274,7 @@ another to hold a chunk.
ref: $(run_in "$DATADIR/gits/bootstrap-chunk" git rev-parse bootstrap)
unpetrify-ref: nootstrap
build-mode: bootstrap
+ build-system: autotools
build-depends: []
- name: stage2-chunk
morph: stage2-chunk.morph