summaryrefslogtreecommitdiff
path: root/morphlib
diff options
context:
space:
mode:
authorPedro Alvarez <pedro.alvarez@codethink.co.uk>2016-03-26 18:59:01 +0000
committerPedro Alvarez <pedro.alvarez@codethink.co.uk>2016-03-29 16:55:50 +0000
commita7f12476d4e7b2025a60be58027b67b9e551f31b (patch)
tree8a5de760ab7281368d11a25c659780d3aea3539f /morphlib
parent9404317020ff0455cbfd3ca7976d546af823759b (diff)
downloadmorph-a7f12476d4e7b2025a60be58027b67b9e551f31b.tar.gz
Add support for definitions version 8
This code is a rework from changes done by: - Tiago Gomes <tiago.gomes@codethink.co.uk> https://storyboard.baserock.org/#!/story/86 Change-Id: I3475c2bcb648a272fee33bc878a521f79d4e6581
Diffstat (limited to 'morphlib')
-rw-r--r--morphlib/artifactresolver.py3
-rw-r--r--morphlib/buildcommand.py33
-rw-r--r--morphlib/builder.py42
-rw-r--r--morphlib/builder_tests.py3
-rw-r--r--morphlib/definitions_version.py4
-rw-r--r--morphlib/git.py47
-rw-r--r--morphlib/gitdir.py22
-rw-r--r--morphlib/gitdir_tests.py6
-rw-r--r--morphlib/morphloader.py76
-rw-r--r--morphlib/morphloader_tests.py98
-rw-r--r--morphlib/plugins/build_plugin.py6
-rw-r--r--morphlib/plugins/cross-bootstrap_plugin.py13
-rw-r--r--morphlib/plugins/distbuild_plugin.py6
-rw-r--r--morphlib/repocache.py9
-rw-r--r--morphlib/source.py2
-rw-r--r--morphlib/sourceresolver.py65
16 files changed, 287 insertions, 148 deletions
diff --git a/morphlib/artifactresolver.py b/morphlib/artifactresolver.py
index f3936df1..c1418924 100644
--- a/morphlib/artifactresolver.py
+++ b/morphlib/artifactresolver.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2012-2015 Codethink Limited
+# Copyright (C) 2012-2016 Codethink Limited
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -189,6 +189,7 @@ class ArtifactResolver(object):
# Resolve now to avoid a search for the parent morphology later
chunk_source.build_mode = info['build-mode']
chunk_source.prefix = info['prefix']
+ chunk_source.submodules = info['submodules']
# Add these chunks to the processed artifacts, so other
# chunks may refer to them.
diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py
index e185a808..35f55caf 100644
--- a/morphlib/buildcommand.py
+++ b/morphlib/buildcommand.py
@@ -59,9 +59,10 @@ class BuildCommand(object):
self.app.status(msg='Deciding on task order')
srcpool = self.create_source_pool(
repo_name, ref, [filename], original_ref)
+ definitions_version = srcpool.definitions_version
self.validate_sources(srcpool)
root_artifact = self.resolve_artifacts(srcpool)
- self.build_in_order(root_artifact)
+ self.build_in_order(root_artifact, definitions_version)
self.app.status(
msg='Build of %(repo_name)s %(ref)s %(filename)s ended '
@@ -273,7 +274,7 @@ class BuildCommand(object):
known_sources.add(artifact.source)
yield artifact.source
- def build_in_order(self, root_artifact):
+ def build_in_order(self, root_artifact, definitions_version):
'''Build everything specified in a build order.'''
self.app.status(msg='Starting build of %(name)s',
@@ -289,11 +290,11 @@ class BuildCommand(object):
'name': s.name,
})
- self.cache_or_build_source(s, build_env)
+ self.cache_or_build_source(s, build_env, definitions_version)
self.app.status_prefix = old_prefix
- def cache_or_build_source(self, source, build_env):
+ def cache_or_build_source(self, source, build_env, definitions_version):
'''Make artifacts of the built source available in the local cache.
This can be done by retrieving from a remote artifact cache, or if
@@ -309,7 +310,7 @@ class BuildCommand(object):
pass
if any(not self.lac.has(artifact) for artifact in artifacts):
- self.build_source(source, build_env)
+ self.build_source(source, build_env, definitions_version)
for a in artifacts:
self.app.status(msg='%(kind)s %(name)s is cached at %(cachepath)s',
@@ -317,7 +318,7 @@ class BuildCommand(object):
cachepath=self.lac.artifact_filename(a),
chatty=(source.morphology['kind'] != "system"))
- def build_source(self, source, build_env):
+ def build_source(self, source, build_env, definitions_version):
'''Build all artifacts for one source.
All the dependencies are assumed to be built and available
@@ -329,7 +330,7 @@ class BuildCommand(object):
name=source.name,
kind=source.morphology['kind'])
- self.fetch_sources(source)
+ self.fetch_sources(source, definitions_version)
# TODO: Make an artifact.walk() that takes multiple root artifacts.
# as this does a walk for every artifact. This was the status
# quo before build logic was made to work per-source, but we can
@@ -366,7 +367,8 @@ class BuildCommand(object):
else:
staging_area = self.create_staging_area(source, build_env, False)
- self.build_and_cache(staging_area, source, setup_mounts)
+ self.build_and_cache(staging_area, source, setup_mounts,
+ definitions_version)
self.remove_staging_area(staging_area)
td = datetime.datetime.now() - starttime
@@ -385,13 +387,18 @@ class BuildCommand(object):
ordered_deps.append(dep)
return ordered_deps
- def fetch_sources(self, source):
+ def fetch_sources(self, source, definitions_version):
'''Update the local git repository cache with the sources.'''
repo_name = source.repo_name
source.repo = self.repo_cache.get_updated_repo(repo_name,
ref=source.sha1)
- self.repo_cache.ensure_submodules(source.repo, source.sha1)
+ if source.morphology['kind'] == 'chunk':
+ if definitions_version >= 8:
+ self.repo_cache.ensure_submodules(
+ source.repo, source.sha1, source.submodules)
+ else:
+ self.repo_cache.ensure_submodules(source.repo, source.sha1)
def cache_artifacts_locally(self, artifacts):
'''Get artifacts missing from local cache from remote cache.'''
@@ -529,7 +536,8 @@ class BuildCommand(object):
if target_source.build_mode == 'staging':
morphlib.builder.ldconfig(self.app, staging_area.dirname)
- def build_and_cache(self, staging_area, source, setup_mounts):
+ def build_and_cache(self, staging_area, source, setup_mounts,
+ definitions_version):
'''Build a source and put its artifacts into the local cache.'''
self.app.status(msg='Starting actual build: %(name)s '
@@ -537,7 +545,8 @@ class BuildCommand(object):
name=source.name, sha1=source.sha1[:7])
builder = morphlib.builder.Builder(
self.app, staging_area, self.lac, self.rac, self.repo_cache,
- self.app.settings['max-jobs'], setup_mounts)
+ self.app.settings['max-jobs'], setup_mounts,
+ definitions_version)
return builder.build_and_cache(source)
class InitiatorBuildCommand(BuildCommand):
diff --git a/morphlib/builder.py b/morphlib/builder.py
index c980a276..166b790b 100644
--- a/morphlib/builder.py
+++ b/morphlib/builder.py
@@ -35,10 +35,11 @@ import morphlib.gitversion
SYSTEM_INTEGRATION_PATH = os.path.join('baserock', 'system-integration')
-def extract_sources(app, repo_cache, repo, sha1, srcdir): #pragma: no cover
+def extract_sources(app, definitions_version, repo_cache, repo, sha1,
+ destdir, source): #pragma: no cover
'''Get sources from git to a source directory, including submodules'''
- def extract_repo(repo, sha1, destdir):
+ def extract_repo(repo, sha1, destdir, submodules_map=None):
app.status(msg='Extracting %(source)s into %(target)s',
source=repo.original_name,
target=destdir)
@@ -53,16 +54,27 @@ def extract_sources(app, repo_cache, repo, sha1, srcdir): #pragma: no cover
else:
tuples = []
for sub in submodules:
- cached_repo = repo_cache.get_updated_repo(sub.url, sub.commit)
+ if submodules_map and sub.name in submodules_map:
+ url = submodules_map[sub.name]['url']
+ else:
+ url = sub.url
+ cached_repo = repo_cache.get_updated_repo(url, sub.commit)
sub_dir = os.path.join(destdir, sub.path)
tuples.append((cached_repo, sub.commit, sub_dir))
return tuples
- todo = [(repo, sha1, srcdir)]
- while todo:
- repo, sha1, srcdir = todo.pop()
- todo += extract_repo(repo, sha1, srcdir)
- set_mtime_recursively(srcdir)
+ if definitions_version >= 8:
+ todo = [(repo, sha1, destdir)]
+ while todo:
+ repo, sha1, destdir = todo.pop()
+ todo += extract_repo(repo, sha1, destdir, source.submodules)
+ else:
+ todo = [(repo, sha1, destdir)]
+ while todo:
+ repo, sha1, destdir = todo.pop()
+ todo += extract_repo(repo, sha1, destdir)
+
+ set_mtime_recursively(destdir)
def set_mtime_recursively(root): # pragma: no cover
'''Set the mtime for every file in a directory tree to the same.
@@ -147,7 +159,7 @@ class BuilderBase(object):
def __init__(self, app, staging_area, local_artifact_cache,
remote_artifact_cache, source, repo_cache, max_jobs,
- setup_mounts):
+ setup_mounts, definitions_version):
self.app = app
self.staging_area = staging_area
self.local_artifact_cache = local_artifact_cache
@@ -157,6 +169,7 @@ class BuilderBase(object):
self.max_jobs = max_jobs
self.build_watch = morphlib.stopwatch.Stopwatch()
self.setup_mounts = setup_mounts
+ self.definitions_version = definitions_version
def save_build_times(self):
'''Write the times captured by the stopwatch'''
@@ -374,7 +387,6 @@ class ChunkBuilder(BuilderBase):
stderr=subprocess.STDOUT,
logfile=logfilepath,
ccache_dir=ccache_dir)
-
if stdout:
stdout.flush()
@@ -490,7 +502,8 @@ class ChunkBuilder(BuilderBase):
def get_sources(self, srcdir): # pragma: no cover
s = self.source
- extract_sources(self.app, self.repo_cache, s.repo, s.sha1, srcdir)
+ extract_sources(self.app, self.definitions_version, self.repo_cache,
+ s.repo, s.sha1, srcdir, s)
class StratumBuilder(BuilderBase):
@@ -725,7 +738,8 @@ class Builder(object): # pragma: no cover
}
def __init__(self, app, staging_area, local_artifact_cache,
- remote_artifact_cache, repo_cache, max_jobs, setup_mounts):
+ remote_artifact_cache, repo_cache, max_jobs, setup_mounts,
+ definitions_version):
self.app = app
self.staging_area = staging_area
self.local_artifact_cache = local_artifact_cache
@@ -733,6 +747,7 @@ class Builder(object): # pragma: no cover
self.repo_cache = repo_cache
self.max_jobs = max_jobs
self.setup_mounts = setup_mounts
+ self.definitions_version = definitions_version
def build_and_cache(self, source):
kind = source.morphology['kind']
@@ -740,7 +755,8 @@ class Builder(object): # pragma: no cover
self.local_artifact_cache,
self.remote_artifact_cache, source,
self.repo_cache, self.max_jobs,
- self.setup_mounts)
+ self.setup_mounts,
+ self.definitions_version)
self.app.status(msg='Builder.build: artifact %s with %s' %
(source.name, repr(o)),
chatty=True)
diff --git a/morphlib/builder_tests.py b/morphlib/builder_tests.py
index 54bc4a8f..6d88a472 100644
--- a/morphlib/builder_tests.py
+++ b/morphlib/builder_tests.py
@@ -156,7 +156,8 @@ class BuilderBaseTests(unittest.TestCase):
self.artifact,
self.repo_cache,
self.max_jobs,
- False)
+ False,
+ 8)
def test_runs_desired_command(self):
self.builder.runcmd(['foo', 'bar'])
diff --git a/morphlib/definitions_version.py b/morphlib/definitions_version.py
index b531a021..e887ddd8 100644
--- a/morphlib/definitions_version.py
+++ b/morphlib/definitions_version.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2015 Codethink Limited
+# Copyright (C) 2015-2016 Codethink Limited
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -24,7 +24,7 @@ import yaml
import morphlib
-SUPPORTED_VERSIONS = [7]
+SUPPORTED_VERSIONS = [7, 8]
class DefinitionsVersionError(cliapp.AppException):
diff --git a/morphlib/git.py b/morphlib/git.py
index cab551ef..12c24b0e 100644
--- a/morphlib/git.py
+++ b/morphlib/git.py
@@ -34,11 +34,17 @@ class NoModulesFileError(cliapp.AppException):
class Submodule(object):
- def __init__(self, name, url, path):
+ def __init__(self, name, url, sha1, path):
self.name = name
self.url = url
+ self.commit = sha1
self.path = path
+ def __str__(self):
+ return "{name}|{url}|{path}".format(name=self.name,
+ url=self.url,
+ path=self.path)
+
class InvalidSectionError(cliapp.AppException):
@@ -48,14 +54,6 @@ class InvalidSectionError(cliapp.AppException):
'title: [%s]' % (repo, ref, section))
-class MissingSubmoduleCommitError(cliapp.AppException):
-
- def __init__(self, repo, ref, submodule):
- Exception.__init__(self,
- '%s:%s:.gitmodules: No commit object found for '
- 'submodule "%s"' % (repo, ref, submodule))
-
-
class Submodules(object):
def __init__(self, repo, ref, runcmd_cb=cliapp.runcmd):
@@ -87,6 +85,7 @@ class Submodules(object):
raise NoModulesFileError(self.repo, self.ref)
def _validate_and_read_entries(self, parser):
+ gd = morphlib.gitdir.GitDirectory(self.repo)
for section in parser.sections():
# validate section name against the 'section "foo"' pattern
section_pattern = r'submodule "(.*)"'
@@ -97,33 +96,9 @@ class Submodules(object):
path = parser.get(section, 'path')
# create a submodule object
- submodule = Submodule(name, url, path)
- try:
- # list objects in the parent repo tree to find the commit
- # object that corresponds to the submodule
- commit = gitcmd(self.runcmd_cb, 'ls-tree', self.ref,
- submodule.path, cwd=self.repo)
-
- # read the commit hash from the output
- fields = commit.split()
- if len(fields) >= 2 and fields[1] == 'commit':
- submodule.commit = commit.split()[2]
-
- # fail if the commit hash is invalid
- if len(submodule.commit) != 40:
- raise MissingSubmoduleCommitError(self.repo,
- self.ref,
- submodule.name)
-
- # add a submodule object to the list
- self.submodules.append(submodule)
- else:
- logging.warning('Skipping submodule "%s" as %s:%s has '
- 'a non-commit object for it' %
- (submodule.name, self.repo, self.ref))
- except cliapp.AppException:
- raise MissingSubmoduleCommitError(self.repo, self.ref,
- submodule.name)
+ sha1 = gd.get_submodule_commit(self.ref, path)
+ submodule = Submodule(name, url, sha1, path)
+ self.submodules.append(submodule)
else:
raise InvalidSectionError(self.repo, self.ref, section)
diff --git a/morphlib/gitdir.py b/morphlib/gitdir.py
index ca4a4c76..24dd9ed7 100644
--- a/morphlib/gitdir.py
+++ b/morphlib/gitdir.py
@@ -54,6 +54,14 @@ class ExpectedSha1Error(cliapp.AppException):
self, 'SHA1 expected, got %s' % ref)
+class MissingSubmoduleCommitError(cliapp.AppException):
+
+ def __init__(self, repo, ref, submodule):
+ cliapp.AppException.__init__(self, # pragma
+ '%s:%s:.gitmodules: No commit object found for '
+ 'submodule "%s"' % (repo, ref, submodule))
+
+
class RefChangeError(cliapp.AppException):
pass
@@ -822,6 +830,20 @@ class GitDirectory(object):
except Exception as e:
raise RefDeleteError(self, ref, old_sha1, e)
+ def get_submodule_commit(self, parent_ref,
+ submodule_path): # pragma: no cover
+ try:
+ lstree_output = morphlib.git.gitcmd(self._runcmd, 'ls-tree',
+ parent_ref, submodule_path)
+ except cliapp.AppException:
+ raise MissingSubmoduleCommitError(self.dirname, parent_ref,
+ submodule_path)
+ m = re.match("160000 commit (?P<sha1>\w{40})", lstree_output)
+ if not m:
+ raise MissingSubmoduleCommitError(self.dirname, parent_ref,
+ submodule_path)
+ return m.group('sha1')
+
def describe(self):
version = morphlib.git.gitcmd(self._runcmd, 'describe',
'--always', '--dirty=-unreproducible')
diff --git a/morphlib/gitdir_tests.py b/morphlib/gitdir_tests.py
index 4da98bbc..d74b4a13 100644
--- a/morphlib/gitdir_tests.py
+++ b/morphlib/gitdir_tests.py
@@ -104,6 +104,12 @@ class GitDirectoryTests(unittest.TestCase):
gitdir = self.empty_git_directory()
self.assertIsInstance(gitdir.get_index(), morphlib.gitindex.GitIndex)
+ def test_non_existent_submodule_path_raises_error(self):
+ gitdir = self.empty_git_directory()
+ self.assertRaises(
+ morphlib.gitdir.MissingSubmoduleCommitError,
+ gitdir.get_submodule_commit, 'master', 'somepath')
+
class GitDirectoryAnchoredRefTests(unittest.TestCase):
diff --git a/morphlib/morphloader.py b/morphlib/morphloader.py
index 66f4763a..31123e95 100644
--- a/morphlib/morphloader.py
+++ b/morphlib/morphloader.py
@@ -15,6 +15,7 @@
# =*= License: GPL-2 =*=
+import os
import collections
import warnings
import yaml
@@ -34,8 +35,10 @@ class MorphologyNotYamlError(MorphologySyntaxError):
class NotADictionaryError(MorphologySyntaxError):
- def __init__(self, morph_filename):
+ def __init__(self, morph_filename, errmsg=None):
self.msg = 'Not a dictionary: morphology %s' % morph_filename
+ if errmsg:
+ self.msg += "\n%s" % (errmsg)
class MorphologyValidationError(morphlib.Error):
@@ -58,6 +61,17 @@ class MissingFieldError(MorphologyValidationError):
'Missing field %s from morphology %s' % (field, morphology_name))
+class InvalidStringError(MorphologyValidationError):
+
+ def __init__(self, field, spec, morph_filename):
+ self.field = field
+ self.spec = spec
+ self.morph_filename = morph_filename
+ MorphologyValidationError.__init__(
+ self, "Field '%(field)s' must be a non-empty string in %(spec)s"\
+ " for morphology %(morph_filename)s" % locals())
+
+
class InvalidFieldError(MorphologyValidationError):
def __init__(self, field, morphology_name):
@@ -119,27 +133,6 @@ class DuplicateChunkError(MorphologyValidationError):
'in stratum %(stratum_name)s' % locals())
-class EmptyRefError(MorphologyValidationError):
-
- def __init__(self, ref_location, morph_filename):
- self.ref_location = ref_location
- self.morph_filename = morph_filename
- MorphologyValidationError.__init__(
- self, 'Empty ref found for %(ref_location)s '\
- 'in %(morph_filename)s' % locals())
-
-
-class ChunkSpecRefNotStringError(MorphologyValidationError):
-
- def __init__(self, ref_value, chunk_name, stratum_name):
- self.ref_value = ref_value
- self.chunk_name = chunk_name
- self.stratum_name = stratum_name
- MorphologyValidationError.__init__(
- self, 'Ref %(ref_value)s for %(chunk_name)s '\
- 'in stratum %(stratum_name)s is not a string' % locals())
-
-
class ChunkSpecConflictingFieldsError(MorphologyValidationError):
def __init__(self, fields, chunk_name, stratum_name):
@@ -246,6 +239,7 @@ class MorphologyDumper(yaml.SafeDumper):
'build-mode',
'artifacts',
'max-jobs',
+ 'submodules',
'products',
'chunks',
'build-system',
@@ -357,6 +351,7 @@ class MorphologyLoader(object):
'strip-commands': None,
'post-strip-commands': None,
'devices': [],
+ 'submodules': {},
'products': [],
'max-jobs': None,
'build-system': 'manual',
@@ -537,14 +532,20 @@ class MorphologyLoader(object):
for spec in morph['chunks']:
chunk_name = spec['name']
- # All chunk refs must be strings.
- if 'ref' in spec:
- ref = spec['ref']
- if ref == None:
- raise EmptyRefError(spec['name'], morph.filename)
- elif not isinstance(ref, basestring):
- raise ChunkSpecRefNotStringError(
- ref, spec['name'], morph.filename)
+ # All chunks repos and refs must be strings
+
+ def validate_chunk_str_field(field, spec, morph_filename):
+ if field not in spec:
+ raise MissingFieldError('%s in %s' % (field, spec),
+ morph.filename)
+ val = spec[field]
+ if not val or not isinstance(val, basestring) or (
+ not val.strip()):
+ raise InvalidStringError(
+ field, spec, morph_filename)
+
+ validate_chunk_str_field('repo', spec, morph.filename)
+ validate_chunk_str_field('ref', spec, morph.filename)
# The build-depends field must be a list.
if 'build-depends' in spec:
@@ -561,6 +562,18 @@ class MorphologyLoader(object):
raise ChunkSpecNoBuildInstructionsError(
chunk_name, morph.filename)
+ def validate_submodules(submodules, morph_filename):
+ for sub_name in submodules:
+ validate_chunk_str_field('url', submodules[sub_name],
+ morph_filename)
+
+ if 'submodules' in spec:
+ if not isinstance(spec['submodules'], dict):
+ raise NotADictionaryError(
+ morph.filename, "The 'submodules' in chunk '%s' have "
+ "to be a dict" % (chunk_name))
+ validate_submodules(spec['submodules'], morph.filename)
+
@classmethod
def _validate_chunk(cls, morphology):
errors = []
@@ -704,6 +717,9 @@ class MorphologyLoader(object):
if 'prefix' not in spec:
spec['prefix'] = \
self._static_defaults['chunk']['prefix']
+ if 'submodules' not in spec:
+ spec['submodules'] = \
+ self._static_defaults['chunk']['submodules']
def _set_chunk_defaults(self, morph):
if morph['max-jobs'] is not None:
diff --git a/morphlib/morphloader_tests.py b/morphlib/morphloader_tests.py
index db22264f..b5d3f568 100644
--- a/morphlib/morphloader_tests.py
+++ b/morphlib/morphloader_tests.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2013-2015 Codethink Limited
+# Copyright (C) 2013-2016 Codethink Limited
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -208,7 +208,42 @@ build-system: manual
self.assertRaises(
morphlib.morphloader.InvalidFieldError, self.loader.validate, m)
- def test_validate_requires_chunk_refs_in_stratum_to_be_strings(self):
+ def test_validate_requires_chunk_repos_to_exist(self):
+ m = morphlib.morphology.Morphology({
+ 'kind': 'stratum',
+ 'name': 'foo',
+ 'build-depends': [],
+ 'chunks': [
+ {
+ 'name': 'chunk',
+ 'ref': 'master',
+ 'build-depends': []
+ }
+ ]
+ })
+ with self.assertRaises(
+ morphlib.morphloader.MissingFieldError):
+ self.loader.validate(m)
+
+ def test_validate_requires_chunk_repos_in_stratum_to_be_strings(self):
+ m = morphlib.morphology.Morphology({
+ 'kind': 'stratum',
+ 'name': 'foo',
+ 'build-depends': [],
+ 'chunks': [
+ {
+ 'name': 'chunk',
+ 'repo': 1,
+ 'ref': 'master',
+ 'build-depends': []
+ }
+ ]
+ })
+ with self.assertRaises(
+ morphlib.morphloader.InvalidStringError):
+ self.loader.validate(m)
+
+ def test_validate_requires_chunk_refs_to_exist(self):
m = morphlib.morphology.Morphology({
'kind': 'stratum',
'name': 'foo',
@@ -217,33 +252,78 @@ build-system: manual
{
'name': 'chunk',
'repo': 'test:repo',
- 'ref': 1,
'build-depends': []
}
]
})
with self.assertRaises(
- morphlib.morphloader.ChunkSpecRefNotStringError):
+ morphlib.morphloader.MissingFieldError):
self.loader.validate(m)
- def test_fails_to_validate_stratum_with_empty_refs_for_a_chunk(self):
+ def test_validate_requires_chunk_refs_in_stratum_to_be_strings(self):
m = morphlib.morphology.Morphology({
'kind': 'stratum',
'name': 'foo',
'build-depends': [],
- 'chunks' : [
+ 'chunks': [
{
'name': 'chunk',
'repo': 'test:repo',
- 'ref': None,
+ 'ref': 1,
'build-depends': []
}
]
})
with self.assertRaises(
- morphlib.morphloader.EmptyRefError):
+ morphlib.morphloader.InvalidStringError):
self.loader.validate(m)
+ def test_fails_to_validate_stratum_with_a_missing_url(self):
+ m = morphlib.morphology.Morphology({
+ 'kind': 'stratum',
+ 'name': 'foo',
+ 'build-depends': [],
+ 'chunks': [
+ {
+ 'name': 'chunk',
+ 'repo': 'test:repo',
+ 'ref': 'master',
+ 'build-system': 'manual',
+ 'build-depends': [],
+ 'submodules':
+ {
+ 'foolib': {}
+ }
+ }
+ ]
+ })
+ self.assertRaises(
+ morphlib.morphloader.MissingFieldError, self.loader.validate, m)
+
+ def test_fails_to_validate_stratum_with_no_dict_submodules(self):
+ m = morphlib.morphology.Morphology({
+ 'kind': 'stratum',
+ 'name': 'foo',
+ 'build-depends': [],
+ 'chunks': [
+ {
+ 'name': 'chunk',
+ 'repo': 'test:repo',
+ 'ref': 'master',
+ 'build-system': 'manual',
+ 'build-depends': [],
+ 'submodules':
+ [
+ {
+ 'foolib': {}
+ }
+ ]
+ }
+ ]
+ })
+ self.assertRaises(
+ morphlib.morphloader.NotADictionaryError, self.loader.validate, m)
+
def test_fails_to_validate_stratum_which_build_depends_on_self(self):
text = '''\
name: bad-stratum
@@ -567,6 +647,7 @@ build-system: manual
'pre-strip-commands': None,
'post-strip-commands': None,
+ 'submodules': {},
'products': [],
'system-integration': [],
'devices': [],
@@ -606,6 +687,7 @@ build-system: manual
"morph": "bar",
'build-mode': 'bootstrap',
'build-depends': [],
+ 'submodules': {},
'prefix': '/usr',
},
],
diff --git a/morphlib/plugins/build_plugin.py b/morphlib/plugins/build_plugin.py
index 0aab2d51..69848b04 100644
--- a/morphlib/plugins/build_plugin.py
+++ b/morphlib/plugins/build_plugin.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2012-2015 Codethink Limited
+# Copyright (C) 2012-2016 Codethink Limited
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -226,6 +226,8 @@ class BuildPlugin(cliapp.Plugin):
'''
bc = morphlib.buildcommand.BuildCommand(self.app)
bc.validate_sources(source_pool)
+ bc.source_pool = source_pool
+ definitions_version = source_pool.definitions_version
root = bc.resolve_artifacts(source_pool)
if not component_names:
component_names = [root.source.name]
@@ -234,7 +236,7 @@ class BuildPlugin(cliapp.Plugin):
raise ComponentNotInSystemError(not_found, filename)
for name, component in components.iteritems():
component.build_env = root.build_env
- bc.build_in_order(component)
+ bc.build_in_order(component, definitions_version)
self.app.status(msg='%(kind)s %(name)s is cached at %(path)s',
kind=component.source.morphology['kind'],
name=name,
diff --git a/morphlib/plugins/cross-bootstrap_plugin.py b/morphlib/plugins/cross-bootstrap_plugin.py
index c6ab8017..7c1793f5 100644
--- a/morphlib/plugins/cross-bootstrap_plugin.py
+++ b/morphlib/plugins/cross-bootstrap_plugin.py
@@ -104,7 +104,8 @@ class BootstrapSystemBuilder(morphlib.builder.BuilderBase):
if not os.path.exists(source_dir):
os.makedirs(source_dir)
morphlib.builder.extract_sources(
- self.app, self.repo_cache, s.repo, s.sha1, source_dir)
+ self.app, self.definitions_version, self.repo_cache,
+ s.repo, s.sha1, source_dir, s)
name = s.name
chunk_script = os.path.join(path, 'src', 'build-%s' % name)
@@ -256,6 +257,8 @@ class CrossBootstrapPlugin(cliapp.Plugin):
srcpool = build_command.create_source_pool(
root_repo, ref, [morph_name])
+ definitions_version = srcpool.definitions_version
+
# FIXME: this is a quick fix in order to get it working for
# Baserock 13 release, it is not a reasonable fix
def validate(self, root_artifact):
@@ -289,10 +292,11 @@ class CrossBootstrapPlugin(cliapp.Plugin):
'mode can be cross-compiled.')
for s in cross_sources:
- build_command.cache_or_build_source(s, build_env)
+ build_command.cache_or_build_source(s, build_env,
+ definitions_version)
for s in native_sources:
- build_command.fetch_sources(s)
+ build_command.fetch_sources(s, definitions_version)
# Install those to the output tarball ...
self.app.status(msg='Building final bootstrap system image')
@@ -302,7 +306,8 @@ class CrossBootstrapPlugin(cliapp.Plugin):
system_artifact.source, build_env, use_chroot=False)
builder = BootstrapSystemBuilder(
self.app, staging_area, build_command.lac, build_command.rac,
- system_artifact.source, build_command.repo_cache, 1, False)
+ system_artifact.source, build_command.repo_cache, 1, False,
+ definitions_version)
builder.build_and_cache()
self.app.status(
diff --git a/morphlib/plugins/distbuild_plugin.py b/morphlib/plugins/distbuild_plugin.py
index 2da089f3..50abfc71 100644
--- a/morphlib/plugins/distbuild_plugin.py
+++ b/morphlib/plugins/distbuild_plugin.py
@@ -1,6 +1,6 @@
# distbuild_plugin.py -- Morph distributed build plugin
#
-# Copyright (C) 2014-2015 Codethink Limited
+# Copyright (C) 2014-2016 Codethink Limited
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -248,6 +248,8 @@ class WorkerBuild(cliapp.Plugin):
artifact_reference.ref,
[artifact_reference.root_filename])
+ definitions_version = source_pool.definitions_version
+
root = bc.resolve_artifacts(source_pool)
# Now, before we start the build, we garbage collect the caches
@@ -263,7 +265,7 @@ class WorkerBuild(cliapp.Plugin):
source = self.find_source(source_pool, artifact_reference)
build_env = bc.new_build_env(artifact_reference.arch)
- bc.build_source(source, build_env)
+ bc.build_source(source, build_env, definitions_version)
def find_source(self, source_pool, artifact_reference):
for s in source_pool.lookup(artifact_reference.source_repo,
diff --git a/morphlib/repocache.py b/morphlib/repocache.py
index f6978ec4..941d958a 100644
--- a/morphlib/repocache.py
+++ b/morphlib/repocache.py
@@ -369,7 +369,7 @@ class RepoCache(object):
return self._get_repo(repo_name)
def ensure_submodules(self, toplevel_repo,
- toplevel_ref): # pragma: no cover
+ toplevel_ref, submodules={}): # pragma: no cover
'''Ensure any submodules of a given repo are cached and up to date.'''
def submodules_for_repo(repo_path, ref):
@@ -377,7 +377,8 @@ class RepoCache(object):
submodules = morphlib.git.Submodules(repo_path, ref,
runcmd_cb=self.runcmd_cb)
submodules.load()
- return [(submod.url, submod.commit) for submod in submodules]
+ return [(submod.name, submod.url, submod.commit)
+ for submod in submodules]
except morphlib.git.NoModulesFileError:
return []
@@ -385,8 +386,10 @@ class RepoCache(object):
subs_to_process = submodules_for_repo(toplevel_repo.dirname,
toplevel_ref)
while subs_to_process:
- url, ref = subs_to_process.pop()
+ name, url, ref = subs_to_process.pop()
done.add((url, ref))
+ if name in submodules:
+ url = submodules[name]['url']
cached_repo = self.get_updated_repo(url, ref=ref)
diff --git a/morphlib/source.py b/morphlib/source.py
index 135c14cc..3f9d606c 100644
--- a/morphlib/source.py
+++ b/morphlib/source.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2012-2015 Codethink Limited
+# Copyright (C) 2012-2016 Codethink Limited
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
diff --git a/morphlib/sourceresolver.py b/morphlib/sourceresolver.py
index 5d04ece9..5af789c0 100644
--- a/morphlib/sourceresolver.py
+++ b/morphlib/sourceresolver.py
@@ -329,7 +329,7 @@ class SourceResolver(object):
# Morph code doesn't need to know about the predefined
# build instructions.
chunk_filename = c['name'] + '.morph'
- chunk_queue.add((c["name"], c['repo'], c['ref'],
+ chunk_queue.add((c['name'], c['repo'], c['ref'],
chunk_filename, c['build-system']))
return chunk_queue
@@ -375,10 +375,29 @@ class SourceResolver(object):
visit(chunk_repo, chunk_ref, filename, absref, tree, morphology,
predefined_split_rules)
- def traverse_morphs(self, definitions_repo, definitions_ref,
- system_filenames,
- visit=lambda rn, rf, fn, arf, m: None,
- definitions_original_ref=None):
+ def add_morphs_to_source_pool(self, definitions_repo, definitions_ref,
+ system_filenames, pool,
+ definitions_original_ref=None):
+
+ def add_to_pool(reponame, ref, filename, absref, tree, morphology,
+ predefined_split_rules):
+ # If there are duplicate chunks which have the same 'name' and the
+ # same build instructions, we might cause a stack overflow in
+ # cachekeycomputer.py when trying to hash the build graph. The
+ # _find_duplicate_chunks() function doesn't handle this case, it
+ # is checking for duplicates with the same name but different build
+ # instructions.
+ if morphology['kind'] != 'stratum':
+ if pool.lookup(reponame, ref, filename):
+ raise morphlib.Error(
+ "There are multiple versions of component '%s'" %
+ morphology['name'])
+
+ sources = morphlib.source.make_sources(
+ reponame, ref, filename, absref, tree, morphology,
+ predefined_split_rules)
+ for source in sources:
+ pool.add(source)
resolved_morphologies = {}
@@ -401,12 +420,12 @@ class SourceResolver(object):
definitions_cached_repo.extract_commit(
definitions_absref, definitions_checkout_dir)
- definitions_version = self._check_version_file(
+ pool.definitions_version = self._check_version_file(
definitions_checkout_dir)
predefined_build_systems, predefined_split_rules = \
self._get_defaults(
- definitions_checkout_dir, definitions_version)
+ definitions_checkout_dir, pool.definitions_version)
morph_loader = morphlib.morphloader.MorphologyLoader(
predefined_build_systems=predefined_build_systems)
@@ -417,17 +436,18 @@ class SourceResolver(object):
chunk_queue = self._process_definitions_with_children(
resolved_morphologies, definitions_checkout_dir,
definitions_repo, definitions_ref, definitions_absref,
- definitions_tree, morph_loader, system_filenames, visit,
- predefined_split_rules)
+ definitions_tree, morph_loader, system_filenames,
+ add_to_pool, predefined_split_rules)
# Now process all the chunks involved in the build.
for name, repo, ref, filename, buildsystem in chunk_queue:
self.process_chunk(resolved_morphologies, resolved_trees,
definitions_checkout_dir, morph_loader,
name, repo, ref, filename, buildsystem,
- visit, predefined_build_systems,
+ add_to_pool, predefined_build_systems,
predefined_split_rules)
+
class DuplicateChunkError(morphlib.Error):
def _make_msg(self, (name, sources)): # pragma: no cover
@@ -477,34 +497,13 @@ def create_source_pool(repo_cache, repo, ref, filenames,
'''
pool = morphlib.sourcepool.SourcePool()
- def add_to_pool(reponame, ref, filename, absref, tree, morphology,
- predefined_split_rules):
- # If there are duplicate chunks which have the same 'name' and the
- # same build instructions, we might cause a stack overflow in
- # cachekeycomputer.py when trying to hash the build graph. The
- # _find_duplicate_chunks() function doesn't handle this case, it
- # is checking for duplicates with the same name but different build
- # instructions.
- if morphology['kind'] != 'stratum':
- if pool.lookup(reponame, ref, filename):
- raise morphlib.Error(
- "There are multiple versions of component '%s'" %
- morphology['name'])
-
- sources = morphlib.source.make_sources(
- reponame, ref, filename, absref, tree, morphology,
- predefined_split_rules)
- for source in sources:
- pool.add(source)
-
tree_cache_manager = PickleCacheManager(
os.path.join(repo_cache.cachedir, tree_cache_filename),
tree_cache_size)
resolver = SourceResolver(repo_cache, tree_cache_manager, status_cb)
- resolver.traverse_morphs(repo, ref, filenames,
- visit=add_to_pool,
- definitions_original_ref=original_ref)
+ resolver.add_morphs_to_source_pool(repo, ref, filenames, pool,
+ definitions_original_ref=original_ref)
# No two chunks may have the same name
duplicate_chunks = _find_duplicate_chunks(pool)