summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorPedro Alvarez <pedro.alvarez@codethink.co.uk>2016-03-26 18:59:01 +0000
committerPedro Alvarez <pedro.alvarez@codethink.co.uk>2016-03-26 19:27:19 +0000
commit45e39e81698df91251c7fac5a642e211e98d834b (patch)
treef4d2fa2774dd29f302617002bd8da417928a1408
parent9404317020ff0455cbfd3ca7976d546af823759b (diff)
downloadmorph-baserock/pedroalvarez/defv8-submodules-squashed.tar.gz
Add support for definitions version 8baserock/pedroalvarez/defv8-submodules-squashed
This code is a rework from changes done by: - Tiago Gomes <tiago.gomes@codethink.co.uk> https://storyboard.baserock.org/#!/story/86 Change-Id: I3475c2bcb648a272fee33bc878a521f79d4e6581
-rw-r--r--morphlib/artifactresolver.py3
-rw-r--r--morphlib/buildcommand.py11
-rw-r--r--morphlib/builder.py42
-rw-r--r--morphlib/builder_tests.py3
-rw-r--r--morphlib/definitions_version.py4
-rw-r--r--morphlib/git.py47
-rw-r--r--morphlib/gitdir.py22
-rw-r--r--morphlib/gitdir_tests.py6
-rw-r--r--morphlib/morphloader.py76
-rw-r--r--morphlib/morphloader_tests.py98
-rw-r--r--morphlib/plugins/build_plugin.py3
-rw-r--r--morphlib/plugins/cross-bootstrap_plugin.py8
-rw-r--r--morphlib/repocache.py9
-rw-r--r--morphlib/source.py2
-rw-r--r--morphlib/sourceresolver.py8
-rw-r--r--yarns/building.yarn8
-rw-r--r--yarns/implementations.yarn56
17 files changed, 302 insertions, 104 deletions
diff --git a/morphlib/artifactresolver.py b/morphlib/artifactresolver.py
index f3936df1..c1418924 100644
--- a/morphlib/artifactresolver.py
+++ b/morphlib/artifactresolver.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2012-2015 Codethink Limited
+# Copyright (C) 2012-2016 Codethink Limited
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -189,6 +189,7 @@ class ArtifactResolver(object):
# Resolve now to avoid a search for the parent morphology later
chunk_source.build_mode = info['build-mode']
chunk_source.prefix = info['prefix']
+ chunk_source.submodules = info['submodules']
# Add these chunks to the processed artifacts, so other
# chunks may refer to them.
diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py
index e185a808..346fdc8a 100644
--- a/morphlib/buildcommand.py
+++ b/morphlib/buildcommand.py
@@ -95,6 +95,7 @@ class BuildCommand(object):
self.repo_cache, repo_name, ref, filenames,
original_ref=original_ref,
status_cb=self.app.status)
+ self.source_pool = srcpool
return srcpool
def validate_sources(self, srcpool):
@@ -391,7 +392,12 @@ class BuildCommand(object):
repo_name = source.repo_name
source.repo = self.repo_cache.get_updated_repo(repo_name,
ref=source.sha1)
- self.repo_cache.ensure_submodules(source.repo, source.sha1)
+ if source.morphology['kind'] == 'chunk':
+ if self.source_pool.definitions_version >= 8:
+ self.repo_cache.ensure_submodules(
+ source.repo, source.sha1, source.submodules)
+ else:
+ self.repo_cache.ensure_submodules(source.repo, source.sha1)
def cache_artifacts_locally(self, artifacts):
'''Get artifacts missing from local cache from remote cache.'''
@@ -537,7 +543,8 @@ class BuildCommand(object):
name=source.name, sha1=source.sha1[:7])
builder = morphlib.builder.Builder(
self.app, staging_area, self.lac, self.rac, self.repo_cache,
- self.app.settings['max-jobs'], setup_mounts)
+ self.app.settings['max-jobs'], setup_mounts,
+ self.source_pool.definitions_version)
return builder.build_and_cache(source)
class InitiatorBuildCommand(BuildCommand):
diff --git a/morphlib/builder.py b/morphlib/builder.py
index c980a276..166b790b 100644
--- a/morphlib/builder.py
+++ b/morphlib/builder.py
@@ -35,10 +35,11 @@ import morphlib.gitversion
SYSTEM_INTEGRATION_PATH = os.path.join('baserock', 'system-integration')
-def extract_sources(app, repo_cache, repo, sha1, srcdir): #pragma: no cover
+def extract_sources(app, definitions_version, repo_cache, repo, sha1,
+ destdir, source): #pragma: no cover
'''Get sources from git to a source directory, including submodules'''
- def extract_repo(repo, sha1, destdir):
+ def extract_repo(repo, sha1, destdir, submodules_map=None):
app.status(msg='Extracting %(source)s into %(target)s',
source=repo.original_name,
target=destdir)
@@ -53,16 +54,27 @@ def extract_sources(app, repo_cache, repo, sha1, srcdir): #pragma: no cover
else:
tuples = []
for sub in submodules:
- cached_repo = repo_cache.get_updated_repo(sub.url, sub.commit)
+ if submodules_map and sub.name in submodules_map:
+ url = submodules_map[sub.name]['url']
+ else:
+ url = sub.url
+ cached_repo = repo_cache.get_updated_repo(url, sub.commit)
sub_dir = os.path.join(destdir, sub.path)
tuples.append((cached_repo, sub.commit, sub_dir))
return tuples
- todo = [(repo, sha1, srcdir)]
- while todo:
- repo, sha1, srcdir = todo.pop()
- todo += extract_repo(repo, sha1, srcdir)
- set_mtime_recursively(srcdir)
+ if definitions_version >= 8:
+ todo = [(repo, sha1, destdir)]
+ while todo:
+ repo, sha1, destdir = todo.pop()
+ todo += extract_repo(repo, sha1, destdir, source.submodules)
+ else:
+ todo = [(repo, sha1, destdir)]
+ while todo:
+ repo, sha1, destdir = todo.pop()
+ todo += extract_repo(repo, sha1, destdir)
+
+ set_mtime_recursively(destdir)
def set_mtime_recursively(root): # pragma: no cover
'''Set the mtime for every file in a directory tree to the same.
@@ -147,7 +159,7 @@ class BuilderBase(object):
def __init__(self, app, staging_area, local_artifact_cache,
remote_artifact_cache, source, repo_cache, max_jobs,
- setup_mounts):
+ setup_mounts, definitions_version):
self.app = app
self.staging_area = staging_area
self.local_artifact_cache = local_artifact_cache
@@ -157,6 +169,7 @@ class BuilderBase(object):
self.max_jobs = max_jobs
self.build_watch = morphlib.stopwatch.Stopwatch()
self.setup_mounts = setup_mounts
+ self.definitions_version = definitions_version
def save_build_times(self):
'''Write the times captured by the stopwatch'''
@@ -374,7 +387,6 @@ class ChunkBuilder(BuilderBase):
stderr=subprocess.STDOUT,
logfile=logfilepath,
ccache_dir=ccache_dir)
-
if stdout:
stdout.flush()
@@ -490,7 +502,8 @@ class ChunkBuilder(BuilderBase):
def get_sources(self, srcdir): # pragma: no cover
s = self.source
- extract_sources(self.app, self.repo_cache, s.repo, s.sha1, srcdir)
+ extract_sources(self.app, self.definitions_version, self.repo_cache,
+ s.repo, s.sha1, srcdir, s)
class StratumBuilder(BuilderBase):
@@ -725,7 +738,8 @@ class Builder(object): # pragma: no cover
}
def __init__(self, app, staging_area, local_artifact_cache,
- remote_artifact_cache, repo_cache, max_jobs, setup_mounts):
+ remote_artifact_cache, repo_cache, max_jobs, setup_mounts,
+ definitions_version):
self.app = app
self.staging_area = staging_area
self.local_artifact_cache = local_artifact_cache
@@ -733,6 +747,7 @@ class Builder(object): # pragma: no cover
self.repo_cache = repo_cache
self.max_jobs = max_jobs
self.setup_mounts = setup_mounts
+ self.definitions_version = definitions_version
def build_and_cache(self, source):
kind = source.morphology['kind']
@@ -740,7 +755,8 @@ class Builder(object): # pragma: no cover
self.local_artifact_cache,
self.remote_artifact_cache, source,
self.repo_cache, self.max_jobs,
- self.setup_mounts)
+ self.setup_mounts,
+ self.definitions_version)
self.app.status(msg='Builder.build: artifact %s with %s' %
(source.name, repr(o)),
chatty=True)
diff --git a/morphlib/builder_tests.py b/morphlib/builder_tests.py
index 54bc4a8f..6d88a472 100644
--- a/morphlib/builder_tests.py
+++ b/morphlib/builder_tests.py
@@ -156,7 +156,8 @@ class BuilderBaseTests(unittest.TestCase):
self.artifact,
self.repo_cache,
self.max_jobs,
- False)
+ False,
+ 8)
def test_runs_desired_command(self):
self.builder.runcmd(['foo', 'bar'])
diff --git a/morphlib/definitions_version.py b/morphlib/definitions_version.py
index b531a021..e887ddd8 100644
--- a/morphlib/definitions_version.py
+++ b/morphlib/definitions_version.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2015 Codethink Limited
+# Copyright (C) 2015-2016 Codethink Limited
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -24,7 +24,7 @@ import yaml
import morphlib
-SUPPORTED_VERSIONS = [7]
+SUPPORTED_VERSIONS = [7, 8]
class DefinitionsVersionError(cliapp.AppException):
diff --git a/morphlib/git.py b/morphlib/git.py
index cab551ef..12c24b0e 100644
--- a/morphlib/git.py
+++ b/morphlib/git.py
@@ -34,11 +34,17 @@ class NoModulesFileError(cliapp.AppException):
class Submodule(object):
- def __init__(self, name, url, path):
+ def __init__(self, name, url, sha1, path):
self.name = name
self.url = url
+ self.commit = sha1
self.path = path
+ def __str__(self):
+ return "{name}|{url}|{path}".format(name=self.name,
+ url=self.url,
+ path=self.path)
+
class InvalidSectionError(cliapp.AppException):
@@ -48,14 +54,6 @@ class InvalidSectionError(cliapp.AppException):
'title: [%s]' % (repo, ref, section))
-class MissingSubmoduleCommitError(cliapp.AppException):
-
- def __init__(self, repo, ref, submodule):
- Exception.__init__(self,
- '%s:%s:.gitmodules: No commit object found for '
- 'submodule "%s"' % (repo, ref, submodule))
-
-
class Submodules(object):
def __init__(self, repo, ref, runcmd_cb=cliapp.runcmd):
@@ -87,6 +85,7 @@ class Submodules(object):
raise NoModulesFileError(self.repo, self.ref)
def _validate_and_read_entries(self, parser):
+ gd = morphlib.gitdir.GitDirectory(self.repo)
for section in parser.sections():
# validate section name against the 'section "foo"' pattern
section_pattern = r'submodule "(.*)"'
@@ -97,33 +96,9 @@ class Submodules(object):
path = parser.get(section, 'path')
# create a submodule object
- submodule = Submodule(name, url, path)
- try:
- # list objects in the parent repo tree to find the commit
- # object that corresponds to the submodule
- commit = gitcmd(self.runcmd_cb, 'ls-tree', self.ref,
- submodule.path, cwd=self.repo)
-
- # read the commit hash from the output
- fields = commit.split()
- if len(fields) >= 2 and fields[1] == 'commit':
- submodule.commit = commit.split()[2]
-
- # fail if the commit hash is invalid
- if len(submodule.commit) != 40:
- raise MissingSubmoduleCommitError(self.repo,
- self.ref,
- submodule.name)
-
- # add a submodule object to the list
- self.submodules.append(submodule)
- else:
- logging.warning('Skipping submodule "%s" as %s:%s has '
- 'a non-commit object for it' %
- (submodule.name, self.repo, self.ref))
- except cliapp.AppException:
- raise MissingSubmoduleCommitError(self.repo, self.ref,
- submodule.name)
+ sha1 = gd.get_submodule_commit(self.ref, path)
+ submodule = Submodule(name, url, sha1, path)
+ self.submodules.append(submodule)
else:
raise InvalidSectionError(self.repo, self.ref, section)
diff --git a/morphlib/gitdir.py b/morphlib/gitdir.py
index ca4a4c76..24dd9ed7 100644
--- a/morphlib/gitdir.py
+++ b/morphlib/gitdir.py
@@ -54,6 +54,14 @@ class ExpectedSha1Error(cliapp.AppException):
self, 'SHA1 expected, got %s' % ref)
+class MissingSubmoduleCommitError(cliapp.AppException):
+
+ def __init__(self, repo, ref, submodule):
+ cliapp.AppException.__init__(self, # pragma
+ '%s:%s:.gitmodules: No commit object found for '
+ 'submodule "%s"' % (repo, ref, submodule))
+
+
class RefChangeError(cliapp.AppException):
pass
@@ -822,6 +830,20 @@ class GitDirectory(object):
except Exception as e:
raise RefDeleteError(self, ref, old_sha1, e)
+ def get_submodule_commit(self, parent_ref,
+ submodule_path): # pragma: no cover
+ try:
+ lstree_output = morphlib.git.gitcmd(self._runcmd, 'ls-tree',
+ parent_ref, submodule_path)
+ except cliapp.AppException:
+ raise MissingSubmoduleCommitError(self.dirname, parent_ref,
+ submodule_path)
+ m = re.match("160000 commit (?P<sha1>\w{40})", lstree_output)
+ if not m:
+ raise MissingSubmoduleCommitError(self.dirname, parent_ref,
+ submodule_path)
+ return m.group('sha1')
+
def describe(self):
version = morphlib.git.gitcmd(self._runcmd, 'describe',
'--always', '--dirty=-unreproducible')
diff --git a/morphlib/gitdir_tests.py b/morphlib/gitdir_tests.py
index 4da98bbc..d74b4a13 100644
--- a/morphlib/gitdir_tests.py
+++ b/morphlib/gitdir_tests.py
@@ -104,6 +104,12 @@ class GitDirectoryTests(unittest.TestCase):
gitdir = self.empty_git_directory()
self.assertIsInstance(gitdir.get_index(), morphlib.gitindex.GitIndex)
+ def test_non_existent_submodule_path_raises_error(self):
+ gitdir = self.empty_git_directory()
+ self.assertRaises(
+ morphlib.gitdir.MissingSubmoduleCommitError,
+ gitdir.get_submodule_commit, 'master', 'somepath')
+
class GitDirectoryAnchoredRefTests(unittest.TestCase):
diff --git a/morphlib/morphloader.py b/morphlib/morphloader.py
index 66f4763a..31123e95 100644
--- a/morphlib/morphloader.py
+++ b/morphlib/morphloader.py
@@ -15,6 +15,7 @@
# =*= License: GPL-2 =*=
+import os
import collections
import warnings
import yaml
@@ -34,8 +35,10 @@ class MorphologyNotYamlError(MorphologySyntaxError):
class NotADictionaryError(MorphologySyntaxError):
- def __init__(self, morph_filename):
+ def __init__(self, morph_filename, errmsg=None):
self.msg = 'Not a dictionary: morphology %s' % morph_filename
+ if errmsg:
+ self.msg += "\n%s" % (errmsg)
class MorphologyValidationError(morphlib.Error):
@@ -58,6 +61,17 @@ class MissingFieldError(MorphologyValidationError):
'Missing field %s from morphology %s' % (field, morphology_name))
+class InvalidStringError(MorphologyValidationError):
+
+ def __init__(self, field, spec, morph_filename):
+ self.field = field
+ self.spec = spec
+ self.morph_filename = morph_filename
+ MorphologyValidationError.__init__(
+ self, "Field '%(field)s' must be a non-empty string in %(spec)s"\
+ " for morphology %(morph_filename)s" % locals())
+
+
class InvalidFieldError(MorphologyValidationError):
def __init__(self, field, morphology_name):
@@ -119,27 +133,6 @@ class DuplicateChunkError(MorphologyValidationError):
'in stratum %(stratum_name)s' % locals())
-class EmptyRefError(MorphologyValidationError):
-
- def __init__(self, ref_location, morph_filename):
- self.ref_location = ref_location
- self.morph_filename = morph_filename
- MorphologyValidationError.__init__(
- self, 'Empty ref found for %(ref_location)s '\
- 'in %(morph_filename)s' % locals())
-
-
-class ChunkSpecRefNotStringError(MorphologyValidationError):
-
- def __init__(self, ref_value, chunk_name, stratum_name):
- self.ref_value = ref_value
- self.chunk_name = chunk_name
- self.stratum_name = stratum_name
- MorphologyValidationError.__init__(
- self, 'Ref %(ref_value)s for %(chunk_name)s '\
- 'in stratum %(stratum_name)s is not a string' % locals())
-
-
class ChunkSpecConflictingFieldsError(MorphologyValidationError):
def __init__(self, fields, chunk_name, stratum_name):
@@ -246,6 +239,7 @@ class MorphologyDumper(yaml.SafeDumper):
'build-mode',
'artifacts',
'max-jobs',
+ 'submodules',
'products',
'chunks',
'build-system',
@@ -357,6 +351,7 @@ class MorphologyLoader(object):
'strip-commands': None,
'post-strip-commands': None,
'devices': [],
+ 'submodules': {},
'products': [],
'max-jobs': None,
'build-system': 'manual',
@@ -537,14 +532,20 @@ class MorphologyLoader(object):
for spec in morph['chunks']:
chunk_name = spec['name']
- # All chunk refs must be strings.
- if 'ref' in spec:
- ref = spec['ref']
- if ref == None:
- raise EmptyRefError(spec['name'], morph.filename)
- elif not isinstance(ref, basestring):
- raise ChunkSpecRefNotStringError(
- ref, spec['name'], morph.filename)
+ # All chunks repos and refs must be strings
+
+ def validate_chunk_str_field(field, spec, morph_filename):
+ if field not in spec:
+ raise MissingFieldError('%s in %s' % (field, spec),
+ morph.filename)
+ val = spec[field]
+ if not val or not isinstance(val, basestring) or (
+ not val.strip()):
+ raise InvalidStringError(
+ field, spec, morph_filename)
+
+ validate_chunk_str_field('repo', spec, morph.filename)
+ validate_chunk_str_field('ref', spec, morph.filename)
# The build-depends field must be a list.
if 'build-depends' in spec:
@@ -561,6 +562,18 @@ class MorphologyLoader(object):
raise ChunkSpecNoBuildInstructionsError(
chunk_name, morph.filename)
+ def validate_submodules(submodules, morph_filename):
+ for sub_name in submodules:
+ validate_chunk_str_field('url', submodules[sub_name],
+ morph_filename)
+
+ if 'submodules' in spec:
+ if not isinstance(spec['submodules'], dict):
+ raise NotADictionaryError(
+ morph.filename, "The 'submodules' in chunk '%s' have "
+ "to be a dict" % (chunk_name))
+ validate_submodules(spec['submodules'], morph.filename)
+
@classmethod
def _validate_chunk(cls, morphology):
errors = []
@@ -704,6 +717,9 @@ class MorphologyLoader(object):
if 'prefix' not in spec:
spec['prefix'] = \
self._static_defaults['chunk']['prefix']
+ if 'submodules' not in spec:
+ spec['submodules'] = \
+ self._static_defaults['chunk']['submodules']
def _set_chunk_defaults(self, morph):
if morph['max-jobs'] is not None:
diff --git a/morphlib/morphloader_tests.py b/morphlib/morphloader_tests.py
index db22264f..b5d3f568 100644
--- a/morphlib/morphloader_tests.py
+++ b/morphlib/morphloader_tests.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2013-2015 Codethink Limited
+# Copyright (C) 2013-2016 Codethink Limited
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -208,7 +208,42 @@ build-system: manual
self.assertRaises(
morphlib.morphloader.InvalidFieldError, self.loader.validate, m)
- def test_validate_requires_chunk_refs_in_stratum_to_be_strings(self):
+ def test_validate_requires_chunk_repos_to_exist(self):
+ m = morphlib.morphology.Morphology({
+ 'kind': 'stratum',
+ 'name': 'foo',
+ 'build-depends': [],
+ 'chunks': [
+ {
+ 'name': 'chunk',
+ 'ref': 'master',
+ 'build-depends': []
+ }
+ ]
+ })
+ with self.assertRaises(
+ morphlib.morphloader.MissingFieldError):
+ self.loader.validate(m)
+
+ def test_validate_requires_chunk_repos_in_stratum_to_be_strings(self):
+ m = morphlib.morphology.Morphology({
+ 'kind': 'stratum',
+ 'name': 'foo',
+ 'build-depends': [],
+ 'chunks': [
+ {
+ 'name': 'chunk',
+ 'repo': 1,
+ 'ref': 'master',
+ 'build-depends': []
+ }
+ ]
+ })
+ with self.assertRaises(
+ morphlib.morphloader.InvalidStringError):
+ self.loader.validate(m)
+
+ def test_validate_requires_chunk_refs_to_exist(self):
m = morphlib.morphology.Morphology({
'kind': 'stratum',
'name': 'foo',
@@ -217,33 +252,78 @@ build-system: manual
{
'name': 'chunk',
'repo': 'test:repo',
- 'ref': 1,
'build-depends': []
}
]
})
with self.assertRaises(
- morphlib.morphloader.ChunkSpecRefNotStringError):
+ morphlib.morphloader.MissingFieldError):
self.loader.validate(m)
- def test_fails_to_validate_stratum_with_empty_refs_for_a_chunk(self):
+ def test_validate_requires_chunk_refs_in_stratum_to_be_strings(self):
m = morphlib.morphology.Morphology({
'kind': 'stratum',
'name': 'foo',
'build-depends': [],
- 'chunks' : [
+ 'chunks': [
{
'name': 'chunk',
'repo': 'test:repo',
- 'ref': None,
+ 'ref': 1,
'build-depends': []
}
]
})
with self.assertRaises(
- morphlib.morphloader.EmptyRefError):
+ morphlib.morphloader.InvalidStringError):
self.loader.validate(m)
+ def test_fails_to_validate_stratum_with_a_missing_url(self):
+ m = morphlib.morphology.Morphology({
+ 'kind': 'stratum',
+ 'name': 'foo',
+ 'build-depends': [],
+ 'chunks': [
+ {
+ 'name': 'chunk',
+ 'repo': 'test:repo',
+ 'ref': 'master',
+ 'build-system': 'manual',
+ 'build-depends': [],
+ 'submodules':
+ {
+ 'foolib': {}
+ }
+ }
+ ]
+ })
+ self.assertRaises(
+ morphlib.morphloader.MissingFieldError, self.loader.validate, m)
+
+ def test_fails_to_validate_stratum_with_no_dict_submodules(self):
+ m = morphlib.morphology.Morphology({
+ 'kind': 'stratum',
+ 'name': 'foo',
+ 'build-depends': [],
+ 'chunks': [
+ {
+ 'name': 'chunk',
+ 'repo': 'test:repo',
+ 'ref': 'master',
+ 'build-system': 'manual',
+ 'build-depends': [],
+ 'submodules':
+ [
+ {
+ 'foolib': {}
+ }
+ ]
+ }
+ ]
+ })
+ self.assertRaises(
+ morphlib.morphloader.NotADictionaryError, self.loader.validate, m)
+
def test_fails_to_validate_stratum_which_build_depends_on_self(self):
text = '''\
name: bad-stratum
@@ -567,6 +647,7 @@ build-system: manual
'pre-strip-commands': None,
'post-strip-commands': None,
+ 'submodules': {},
'products': [],
'system-integration': [],
'devices': [],
@@ -606,6 +687,7 @@ build-system: manual
"morph": "bar",
'build-mode': 'bootstrap',
'build-depends': [],
+ 'submodules': {},
'prefix': '/usr',
},
],
diff --git a/morphlib/plugins/build_plugin.py b/morphlib/plugins/build_plugin.py
index 0aab2d51..be6d9602 100644
--- a/morphlib/plugins/build_plugin.py
+++ b/morphlib/plugins/build_plugin.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2012-2015 Codethink Limited
+# Copyright (C) 2012-2016 Codethink Limited
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -226,6 +226,7 @@ class BuildPlugin(cliapp.Plugin):
'''
bc = morphlib.buildcommand.BuildCommand(self.app)
bc.validate_sources(source_pool)
+ bc.source_pool = source_pool
root = bc.resolve_artifacts(source_pool)
if not component_names:
component_names = [root.source.name]
diff --git a/morphlib/plugins/cross-bootstrap_plugin.py b/morphlib/plugins/cross-bootstrap_plugin.py
index c6ab8017..daed260e 100644
--- a/morphlib/plugins/cross-bootstrap_plugin.py
+++ b/morphlib/plugins/cross-bootstrap_plugin.py
@@ -104,7 +104,8 @@ class BootstrapSystemBuilder(morphlib.builder.BuilderBase):
if not os.path.exists(source_dir):
os.makedirs(source_dir)
morphlib.builder.extract_sources(
- self.app, self.repo_cache, s.repo, s.sha1, source_dir)
+ self.app, self.definitions_version, self.repo_cache,
+ s.repo, s.sha1, source_dir, s)
name = s.name
chunk_script = os.path.join(path, 'src', 'build-%s' % name)
@@ -256,6 +257,8 @@ class CrossBootstrapPlugin(cliapp.Plugin):
srcpool = build_command.create_source_pool(
root_repo, ref, [morph_name])
+ definitions_version = srcpool.definitions_version
+
# FIXME: this is a quick fix in order to get it working for
# Baserock 13 release, it is not a reasonable fix
def validate(self, root_artifact):
@@ -302,7 +305,8 @@ class CrossBootstrapPlugin(cliapp.Plugin):
system_artifact.source, build_env, use_chroot=False)
builder = BootstrapSystemBuilder(
self.app, staging_area, build_command.lac, build_command.rac,
- system_artifact.source, build_command.repo_cache, 1, False)
+ system_artifact.source, build_command.repo_cache, 1, False,
+ definitions_version)
builder.build_and_cache()
self.app.status(
diff --git a/morphlib/repocache.py b/morphlib/repocache.py
index f6978ec4..941d958a 100644
--- a/morphlib/repocache.py
+++ b/morphlib/repocache.py
@@ -369,7 +369,7 @@ class RepoCache(object):
return self._get_repo(repo_name)
def ensure_submodules(self, toplevel_repo,
- toplevel_ref): # pragma: no cover
+ toplevel_ref, submodules={}): # pragma: no cover
'''Ensure any submodules of a given repo are cached and up to date.'''
def submodules_for_repo(repo_path, ref):
@@ -377,7 +377,8 @@ class RepoCache(object):
submodules = morphlib.git.Submodules(repo_path, ref,
runcmd_cb=self.runcmd_cb)
submodules.load()
- return [(submod.url, submod.commit) for submod in submodules]
+ return [(submod.name, submod.url, submod.commit)
+ for submod in submodules]
except morphlib.git.NoModulesFileError:
return []
@@ -385,8 +386,10 @@ class RepoCache(object):
subs_to_process = submodules_for_repo(toplevel_repo.dirname,
toplevel_ref)
while subs_to_process:
- url, ref = subs_to_process.pop()
+ name, url, ref = subs_to_process.pop()
done.add((url, ref))
+ if name in submodules:
+ url = submodules[name]['url']
cached_repo = self.get_updated_repo(url, ref=ref)
diff --git a/morphlib/source.py b/morphlib/source.py
index 135c14cc..3f9d606c 100644
--- a/morphlib/source.py
+++ b/morphlib/source.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2012-2015 Codethink Limited
+# Copyright (C) 2012-2016 Codethink Limited
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
diff --git a/morphlib/sourceresolver.py b/morphlib/sourceresolver.py
index 5d04ece9..16a254f4 100644
--- a/morphlib/sourceresolver.py
+++ b/morphlib/sourceresolver.py
@@ -329,7 +329,7 @@ class SourceResolver(object):
# Morph code doesn't need to know about the predefined
# build instructions.
chunk_filename = c['name'] + '.morph'
- chunk_queue.add((c["name"], c['repo'], c['ref'],
+ chunk_queue.add((c['name'], c['repo'], c['ref'],
chunk_filename, c['build-system']))
return chunk_queue
@@ -376,7 +376,7 @@ class SourceResolver(object):
predefined_split_rules)
def traverse_morphs(self, definitions_repo, definitions_ref,
- system_filenames,
+ system_filenames, pool,
visit=lambda rn, rf, fn, arf, m: None,
definitions_original_ref=None):
@@ -403,6 +403,7 @@ class SourceResolver(object):
definitions_version = self._check_version_file(
definitions_checkout_dir)
+ pool.definitions_version = definitions_version
predefined_build_systems, predefined_split_rules = \
self._get_defaults(
@@ -428,6 +429,7 @@ class SourceResolver(object):
visit, predefined_build_systems,
predefined_split_rules)
+
class DuplicateChunkError(morphlib.Error):
def _make_msg(self, (name, sources)): # pragma: no cover
@@ -503,7 +505,7 @@ def create_source_pool(repo_cache, repo, ref, filenames,
resolver = SourceResolver(repo_cache, tree_cache_manager, status_cb)
resolver.traverse_morphs(repo, ref, filenames,
- visit=add_to_pool,
+ pool, visit=add_to_pool,
definitions_original_ref=original_ref)
# No two chunks may have the same name
diff --git a/yarns/building.yarn b/yarns/building.yarn
index 9284f7cf..1b981490 100644
--- a/yarns/building.yarn
+++ b/yarns/building.yarn
@@ -89,6 +89,14 @@ Morph Building Tests
AND there are 2 artifacts named core-runtime in the cache
FINALLY the git server is shut down
+ SCENARIO test recursive submodules overriding urls
+ GIVEN a git server
+ GIVEN a chunk with recursive submodules
+ WHEN the user clones definitions
+ WHEN the user attempts to build the system systems/test-system.morph in branch master
+ THEN morph succeeded
+ FINALLY the git server is shut down
+
System integrations
-------------------
diff --git a/yarns/implementations.yarn b/yarns/implementations.yarn
index 06380e7f..186fe316 100644
--- a/yarns/implementations.yarn
+++ b/yarns/implementations.yarn
@@ -45,6 +45,60 @@ locally, which we'll tell Morph to access using `file:` URLs. Specifically,
we'll create a repository to hold system and stratum morphologies, and
another to hold a chunk.
+ IMPLEMENTS GIVEN a chunk with recursive submodules
+ mkdir "$DATADIR/gits/grandchild-chunk"
+ cd "$DATADIR/gits/grandchild-chunk"
+ git init .
+ touch grandchild-file
+ git add .
+ git commit -m "Initial commit"
+
+ mkdir "$DATADIR/gits/child-chunk"
+ cd "$DATADIR/gits/child-chunk"
+ git init .
+ touch child-file
+ git add .
+ git commit -m "Initial commit"
+ git submodule add -b master file://$DATADIR/gits/grandchild-chunk
+ sed -i -e 's#file://$DATADIR/gits/granchild-chunk#fake/location#' .gitmodules
+ git add .
+ git commit -m "Initial submodule"
+
+ mkdir "$DATADIR/gits/chunk-with-submodules"
+ cd "$DATADIR/gits/chunk-with-submodules"
+ git init .
+ git add .
+ git commit --allow-empty -m "Initial commit"
+ git submodule add -b master file://$DATADIR/gits/child-chunk
+ sed -i -e 's#file://$DATADIR/gits/child-chunk#fake/location#' .gitmodules
+ git add .
+ git commit -m "Add submodule"
+
+ cd "$DATADIR/gits/definitions"
+ echo "version: 8" > VERSION
+ cat << EOF >> strata/core.morph
+ - name: chunk-with-submodules
+ morph: chunk-with-submodules.morph
+ repo: test:chunk-with-submodules
+ ref: master
+ submodules:
+ grandchild-chunk:
+ url: file://$DATADIR/gits/grandchild-chunk
+ child-chunk:
+ url: file://$DATADIR/gits/child-chunk
+ EOF
+
+ cat << EOF >> chunk-with-submodules.morph
+ name: chunk-with-submodules
+ kind: chunk
+ build-system: manual
+ build-commands:
+ - file exists child-chunk/child-file
+ - file exists child-chunk/grandchild-chunk/grandchild-file
+ EOF
+ git add .
+ git commit -m "Add moar stuff"
+
IMPLEMENTS GIVEN a git server
# Create a directory for all the git repositories.
@@ -216,7 +270,7 @@ another to hold a chunk.
mkdir "$DATADIR/gits/definitions"
cd "$DATADIR/gits/definitions"
git init .
- echo 'version: 7' > VERSION
+ echo 'version: 8' > VERSION
install -m644 -D /dev/stdin << EOF "DEFAULTS"
# This is a simplified version of the DEFAULTS file supplied with the