summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorTiago Gomes <tiago.gomes@codethink.co.uk>2015-11-20 13:57:31 (GMT)
committerPedro Alvarez <pedro.alvarez@codethink.co.uk>2016-03-25 21:02:33 (GMT)
commitddb8b27d5056074013a89d8ab277e39fdc1f069d (patch)
tree5ecf61560608dbea1ec1895e5cff2622959556b7
parent9404317020ff0455cbfd3ca7976d546af823759b (diff)
downloadmorph-ddb8b27d5056074013a89d8ab277e39fdc1f069d.tar.gz
WIP Add support for multiple sources per chunk
TODO: - Add API to the cache server to retrieve a submodule commit from a given path. - Fix cross-bootstrap command. Change-Id: I3475c2bcb648a272fee33bc878a521f79d4e6581
-rw-r--r--morphlib/artifactresolver.py1
-rw-r--r--morphlib/buildcommand.py24
-rw-r--r--morphlib/builder.py63
-rw-r--r--morphlib/builder_tests.py3
-rw-r--r--morphlib/cachekeycomputer.py5
-rw-r--r--morphlib/definitions_version.py2
-rw-r--r--morphlib/git.py47
-rw-r--r--morphlib/gitdir.py22
-rw-r--r--morphlib/gitdir_tests.py6
-rw-r--r--morphlib/morphloader.py34
-rw-r--r--morphlib/morphloader_tests.py59
-rw-r--r--morphlib/plugins/build_plugin.py1
-rw-r--r--morphlib/plugins/cross-bootstrap_plugin.py2
-rw-r--r--morphlib/source.py3
-rw-r--r--morphlib/sourceresolver.py95
-rw-r--r--yarns/building.yarn18
-rw-r--r--yarns/implementations.yarn75
17 files changed, 390 insertions, 70 deletions
diff --git a/morphlib/artifactresolver.py b/morphlib/artifactresolver.py
index f3936df..2b40752 100644
--- a/morphlib/artifactresolver.py
+++ b/morphlib/artifactresolver.py
@@ -189,6 +189,7 @@ class ArtifactResolver(object):
# Resolve now to avoid a search for the parent morphology later
chunk_source.build_mode = info['build-mode']
chunk_source.prefix = info['prefix']
+ chunk_source.extra_sources = info['extra-sources']
# Add these chunks to the processed artifacts, so other
# chunks may refer to them.
diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py
index e185a80..6edd8ce 100644
--- a/morphlib/buildcommand.py
+++ b/morphlib/buildcommand.py
@@ -95,6 +95,7 @@ class BuildCommand(object):
self.repo_cache, repo_name, ref, filenames,
original_ref=original_ref,
status_cb=self.app.status)
+ self.source_pool = srcpool
return srcpool
def validate_sources(self, srcpool):
@@ -388,10 +389,28 @@ class BuildCommand(object):
def fetch_sources(self, source):
'''Update the local git repository cache with the sources.'''
+ def fetch_extra_sources(repo_cache, parent_repo, parent_ref,
+ extra_sources):
+
+ for extra_source in extra_sources:
+ ref = extra_source.get('ref')
+ if not ref:
+ ref = parent_repo.get_submodule_commit(
+ parent_ref, extra_source['path'])
+ repo = repo_cache.get_updated_repo(extra_source['repo'],
+ ref=ref)
+ fetch_extra_sources(repo_cache, repo, ref,
+ extra_source.get('extra-sources', []))
+
repo_name = source.repo_name
source.repo = self.repo_cache.get_updated_repo(repo_name,
ref=source.sha1)
- self.repo_cache.ensure_submodules(source.repo, source.sha1)
+ if source.morphology['kind'] == 'chunk':
+ if self.source_pool.definitions_version >= 8:
+ fetch_extra_sources(self.repo_cache, source.repo, source.sha1,
+ source.extra_sources)
+ else:
+ self.repo_cache.ensure_submodules(source.repo, source.sha1)
def cache_artifacts_locally(self, artifacts):
'''Get artifacts missing from local cache from remote cache.'''
@@ -537,7 +556,8 @@ class BuildCommand(object):
name=source.name, sha1=source.sha1[:7])
builder = morphlib.builder.Builder(
self.app, staging_area, self.lac, self.rac, self.repo_cache,
- self.app.settings['max-jobs'], setup_mounts)
+ self.app.settings['max-jobs'], setup_mounts,
+ self.source_pool.definitions_version)
return builder.build_and_cache(source)
class InitiatorBuildCommand(BuildCommand):
diff --git a/morphlib/builder.py b/morphlib/builder.py
index c980a27..e2abee9 100644
--- a/morphlib/builder.py
+++ b/morphlib/builder.py
@@ -35,7 +35,8 @@ import morphlib.gitversion
SYSTEM_INTEGRATION_PATH = os.path.join('baserock', 'system-integration')
-def extract_sources(app, repo_cache, repo, sha1, srcdir): #pragma: no cover
+def extract_sources(app, definitions_version, repo_cache, repo, sha1,
+ destdir, source): #pragma: no cover
'''Get sources from git to a source directory, including submodules'''
def extract_repo(repo, sha1, destdir):
@@ -57,12 +58,48 @@ def extract_sources(app, repo_cache, repo, sha1, srcdir): #pragma: no cover
sub_dir = os.path.join(destdir, sub.path)
tuples.append((cached_repo, sub.commit, sub_dir))
return tuples
+ return []
- todo = [(repo, sha1, srcdir)]
- while todo:
- repo, sha1, srcdir = todo.pop()
- todo += extract_repo(repo, sha1, srcdir)
- set_mtime_recursively(srcdir)
+ def extract_repo_version_8(repo, ref, extra_sources,
+ rootdir, destdir):
+
+ app.status(msg='Extracting %(source)s into %(path)s',
+ source=repo.original_name,
+ path=destdir)
+ repo.checkout(ref, destdir)
+ morphlib.git.reset_workdir(app.runcmd, destdir)
+
+ for extra_source in extra_sources:
+ subrepo = repo_cache.get_repo(extra_source['repo'])
+ path = os.path.normpath(extra_source['path'])
+ checkout_dir = os.path.join(destdir, path)
+ if os.path.exists(checkout_dir):
+ if os.listdir(checkout_dir):
+ raise cliapp.AppException(
+ "Failed to clone '%s': the directory '%s' "
+ "is not empty" %
+ (subrepo.original_name, os.path.join(
+ rootdir, path)))
+ else:
+ os.makedirs(checkout_dir)
+ subref = extra_source.get('ref')
+ if not subref:
+ subref = repo.get_submodule_commit(ref, path)
+
+ extract_repo_version_8(subrepo, subref,
+ extra_source.get('extra-sources', []),
+ rootdir,
+ os.path.join(destdir, checkout_dir))
+ if definitions_version >= 8:
+ extract_repo_version_8(repo, sha1, source.extra_sources,
+ os.path.basename(destdir), destdir)
+ else:
+ todo = [(repo, sha1, destdir)]
+ while todo:
+ repo, sha1, destdir = todo.pop()
+ todo += extract_repo(repo, sha1, destdir)
+
+ set_mtime_recursively(destdir)
def set_mtime_recursively(root): # pragma: no cover
'''Set the mtime for every file in a directory tree to the same.
@@ -147,7 +184,7 @@ class BuilderBase(object):
def __init__(self, app, staging_area, local_artifact_cache,
remote_artifact_cache, source, repo_cache, max_jobs,
- setup_mounts):
+ setup_mounts, definitions_version):
self.app = app
self.staging_area = staging_area
self.local_artifact_cache = local_artifact_cache
@@ -157,6 +194,7 @@ class BuilderBase(object):
self.max_jobs = max_jobs
self.build_watch = morphlib.stopwatch.Stopwatch()
self.setup_mounts = setup_mounts
+ self.definitions_version = definitions_version
def save_build_times(self):
'''Write the times captured by the stopwatch'''
@@ -374,7 +412,6 @@ class ChunkBuilder(BuilderBase):
stderr=subprocess.STDOUT,
logfile=logfilepath,
ccache_dir=ccache_dir)
-
if stdout:
stdout.flush()
@@ -490,7 +527,8 @@ class ChunkBuilder(BuilderBase):
def get_sources(self, srcdir): # pragma: no cover
s = self.source
- extract_sources(self.app, self.repo_cache, s.repo, s.sha1, srcdir)
+ extract_sources(self.app, self.definitions_version, self.repo_cache,
+ s.repo, s.sha1, srcdir, s)
class StratumBuilder(BuilderBase):
@@ -725,7 +763,8 @@ class Builder(object): # pragma: no cover
}
def __init__(self, app, staging_area, local_artifact_cache,
- remote_artifact_cache, repo_cache, max_jobs, setup_mounts):
+ remote_artifact_cache, repo_cache, max_jobs, setup_mounts,
+ definitions_version):
self.app = app
self.staging_area = staging_area
self.local_artifact_cache = local_artifact_cache
@@ -733,6 +772,7 @@ class Builder(object): # pragma: no cover
self.repo_cache = repo_cache
self.max_jobs = max_jobs
self.setup_mounts = setup_mounts
+ self.definitions_version = definitions_version
def build_and_cache(self, source):
kind = source.morphology['kind']
@@ -740,7 +780,8 @@ class Builder(object): # pragma: no cover
self.local_artifact_cache,
self.remote_artifact_cache, source,
self.repo_cache, self.max_jobs,
- self.setup_mounts)
+ self.setup_mounts,
+ self.definitions_version)
self.app.status(msg='Builder.build: artifact %s with %s' %
(source.name, repr(o)),
chatty=True)
diff --git a/morphlib/builder_tests.py b/morphlib/builder_tests.py
index 54bc4a8..b70cbdd 100644
--- a/morphlib/builder_tests.py
+++ b/morphlib/builder_tests.py
@@ -156,7 +156,8 @@ class BuilderBaseTests(unittest.TestCase):
self.artifact,
self.repo_cache,
self.max_jobs,
- False)
+ False,
+ 7)
def test_runs_desired_command(self):
self.builder.runcmd(['foo', 'bar'])
diff --git a/morphlib/cachekeycomputer.py b/morphlib/cachekeycomputer.py
index 22f42aa..ff0d595 100644
--- a/morphlib/cachekeycomputer.py
+++ b/morphlib/cachekeycomputer.py
@@ -96,6 +96,11 @@ class CacheKeyComputer(object):
keys['split-rules'] = [(a, [rgx.pattern for rgx in r._regexes])
for (a, r) in source.split_rules]
+ # Avoid adding the 'extra-trees' key if there were no
+ # extra-sources, to avoid unnecessary rebuilds
+ if source.subtrees: # pragma no cover
+ keys['extra-trees'] = source.subtrees
+
# Include morphology contents, since it doesn't always come
# from the source tree
keys['devices'] = morphology.get('devices')
diff --git a/morphlib/definitions_version.py b/morphlib/definitions_version.py
index b531a02..7261475 100644
--- a/morphlib/definitions_version.py
+++ b/morphlib/definitions_version.py
@@ -24,7 +24,7 @@ import yaml
import morphlib
-SUPPORTED_VERSIONS = [7]
+SUPPORTED_VERSIONS = [7, 8]
class DefinitionsVersionError(cliapp.AppException):
diff --git a/morphlib/git.py b/morphlib/git.py
index cab551e..12c24b0 100644
--- a/morphlib/git.py
+++ b/morphlib/git.py
@@ -34,11 +34,17 @@ class NoModulesFileError(cliapp.AppException):
class Submodule(object):
- def __init__(self, name, url, path):
+ def __init__(self, name, url, sha1, path):
self.name = name
self.url = url
+ self.commit = sha1
self.path = path
+ def __str__(self):
+ return "{name}|{url}|{path}".format(name=self.name,
+ url=self.url,
+ path=self.path)
+
class InvalidSectionError(cliapp.AppException):
@@ -48,14 +54,6 @@ class InvalidSectionError(cliapp.AppException):
'title: [%s]' % (repo, ref, section))
-class MissingSubmoduleCommitError(cliapp.AppException):
-
- def __init__(self, repo, ref, submodule):
- Exception.__init__(self,
- '%s:%s:.gitmodules: No commit object found for '
- 'submodule "%s"' % (repo, ref, submodule))
-
-
class Submodules(object):
def __init__(self, repo, ref, runcmd_cb=cliapp.runcmd):
@@ -87,6 +85,7 @@ class Submodules(object):
raise NoModulesFileError(self.repo, self.ref)
def _validate_and_read_entries(self, parser):
+ gd = morphlib.gitdir.GitDirectory(self.repo)
for section in parser.sections():
# validate section name against the 'section "foo"' pattern
section_pattern = r'submodule "(.*)"'
@@ -97,33 +96,9 @@ class Submodules(object):
path = parser.get(section, 'path')
# create a submodule object
- submodule = Submodule(name, url, path)
- try:
- # list objects in the parent repo tree to find the commit
- # object that corresponds to the submodule
- commit = gitcmd(self.runcmd_cb, 'ls-tree', self.ref,
- submodule.path, cwd=self.repo)
-
- # read the commit hash from the output
- fields = commit.split()
- if len(fields) >= 2 and fields[1] == 'commit':
- submodule.commit = commit.split()[2]
-
- # fail if the commit hash is invalid
- if len(submodule.commit) != 40:
- raise MissingSubmoduleCommitError(self.repo,
- self.ref,
- submodule.name)
-
- # add a submodule object to the list
- self.submodules.append(submodule)
- else:
- logging.warning('Skipping submodule "%s" as %s:%s has '
- 'a non-commit object for it' %
- (submodule.name, self.repo, self.ref))
- except cliapp.AppException:
- raise MissingSubmoduleCommitError(self.repo, self.ref,
- submodule.name)
+ sha1 = gd.get_submodule_commit(self.ref, path)
+ submodule = Submodule(name, url, sha1, path)
+ self.submodules.append(submodule)
else:
raise InvalidSectionError(self.repo, self.ref, section)
diff --git a/morphlib/gitdir.py b/morphlib/gitdir.py
index ca4a4c7..24dd9ed 100644
--- a/morphlib/gitdir.py
+++ b/morphlib/gitdir.py
@@ -54,6 +54,14 @@ class ExpectedSha1Error(cliapp.AppException):
self, 'SHA1 expected, got %s' % ref)
+class MissingSubmoduleCommitError(cliapp.AppException):
+
+ def __init__(self, repo, ref, submodule):
+ cliapp.AppException.__init__(self, # pragma
+ '%s:%s:.gitmodules: No commit object found for '
+ 'submodule "%s"' % (repo, ref, submodule))
+
+
class RefChangeError(cliapp.AppException):
pass
@@ -822,6 +830,20 @@ class GitDirectory(object):
except Exception as e:
raise RefDeleteError(self, ref, old_sha1, e)
+ def get_submodule_commit(self, parent_ref,
+ submodule_path): # pragma: no cover
+ try:
+ lstree_output = morphlib.git.gitcmd(self._runcmd, 'ls-tree',
+ parent_ref, submodule_path)
+ except cliapp.AppException:
+ raise MissingSubmoduleCommitError(self.dirname, parent_ref,
+ submodule_path)
+ m = re.match("160000 commit (?P<sha1>\w{40})", lstree_output)
+ if not m:
+ raise MissingSubmoduleCommitError(self.dirname, parent_ref,
+ submodule_path)
+ return m.group('sha1')
+
def describe(self):
version = morphlib.git.gitcmd(self._runcmd, 'describe',
'--always', '--dirty=-unreproducible')
diff --git a/morphlib/gitdir_tests.py b/morphlib/gitdir_tests.py
index 4da98bb..d74b4a1 100644
--- a/morphlib/gitdir_tests.py
+++ b/morphlib/gitdir_tests.py
@@ -104,6 +104,12 @@ class GitDirectoryTests(unittest.TestCase):
gitdir = self.empty_git_directory()
self.assertIsInstance(gitdir.get_index(), morphlib.gitindex.GitIndex)
+ def test_non_existent_submodule_path_raises_error(self):
+ gitdir = self.empty_git_directory()
+ self.assertRaises(
+ morphlib.gitdir.MissingSubmoduleCommitError,
+ gitdir.get_submodule_commit, 'master', 'somepath')
+
class GitDirectoryAnchoredRefTests(unittest.TestCase):
diff --git a/morphlib/morphloader.py b/morphlib/morphloader.py
index 66f4763..0160e41 100644
--- a/morphlib/morphloader.py
+++ b/morphlib/morphloader.py
@@ -15,6 +15,7 @@
# =*= License: GPL-2 =*=
+import os
import collections
import warnings
import yaml
@@ -79,6 +80,13 @@ class InvalidTypeError(MorphologyValidationError):
(field, expected, actual, morphology_name))
+class InvalidPathError(MorphologyValidationError):
+
+ def __init__(self, path, spec, morph_filename):
+ self.msg = ("Invalid path '%s' in %s from morphology %s"
+ % (path, spec, morph_filename))
+
+
class UnknownArchitectureError(MorphologyValidationError):
def __init__(self, arch, morph_filename):
@@ -246,6 +254,7 @@ class MorphologyDumper(yaml.SafeDumper):
'build-mode',
'artifacts',
'max-jobs',
+ 'extra-sources',
'products',
'chunks',
'build-system',
@@ -357,6 +366,7 @@ class MorphologyLoader(object):
'strip-commands': None,
'post-strip-commands': None,
'devices': [],
+ 'extra-sources': [],
'products': [],
'max-jobs': None,
'build-system': 'manual',
@@ -561,6 +571,27 @@ class MorphologyLoader(object):
raise ChunkSpecNoBuildInstructionsError(
chunk_name, morph.filename)
+ def validate_extra_sources(extra_sources, morph_filename):
+ for extra_source in extra_sources:
+ validate_chunk_str_field('repo', extra_source,
+ morph.filename)
+ path = extra_source.get('path')
+ if not path:
+ raise MissingFieldError("'path' in %s" % extra_source,
+ morph.filename)
+ path = os.path.normpath(path)
+ if os.path.isabs(path) or path.startswith('..') or (
+ path == '.') or not path.strip():
+ raise InvalidPathError(extra_source['path'],
+ extra_source,
+ morph_filename)
+ if 'extra-sources' in extra_source:
+ validate_extra_sources(extra_source['extra-sources'],
+ morph.filename)
+
+ if 'extra-sources' in spec:
+ validate_extra_sources(spec['extra-sources'], morph.filename)
+
@classmethod
def _validate_chunk(cls, morphology):
errors = []
@@ -704,6 +735,9 @@ class MorphologyLoader(object):
if 'prefix' not in spec:
spec['prefix'] = \
self._static_defaults['chunk']['prefix']
+ if 'extra-sources' not in spec:
+ spec['extra-sources'] = \
+ self._static_defaults['chunk']['extra-sources']
def _set_chunk_defaults(self, morph):
if morph['max-jobs'] is not None:
diff --git a/morphlib/morphloader_tests.py b/morphlib/morphloader_tests.py
index db22264..0bf6961 100644
--- a/morphlib/morphloader_tests.py
+++ b/morphlib/morphloader_tests.py
@@ -244,6 +244,63 @@ build-system: manual
morphlib.morphloader.EmptyRefError):
self.loader.validate(m)
+ def test_fails_to_validate_stratum_with_a_missing_path(self):
+ m = morphlib.morphology.Morphology({
+ 'kind': 'stratum',
+ 'name': 'foo',
+ 'build-depends': [],
+ 'chunks': [
+ {
+ 'name': 'chunk',
+ 'repo': 'test:repo',
+ 'ref': 'master',
+ 'build-system': 'manual',
+ 'build-depends': [],
+ 'extra-sources':
+ [
+ {
+ 'repo': 'foo',
+ 'path': 'somepath',
+ 'extra-sources':
+ [
+ {
+ 'repo': 'bar',
+ 'ref': 'master'
+ }
+ ]
+ }
+ ]
+ }
+ ]
+ })
+ self.assertRaises(
+ morphlib.morphloader.MissingFieldError, self.loader.validate, m)
+
+ def test_fails_to_validate_stratum_with_invalid_path(self):
+ m = morphlib.morphology.Morphology({
+ 'kind': 'stratum',
+ 'name': 'foo',
+ 'build-depends': [],
+ 'chunks': [
+ {
+ 'name': 'chunk',
+ 'repo': 'test:repo',
+ 'ref': 'master',
+ 'build-system': 'manual',
+ 'build-depends': [],
+ 'extra-sources':
+ [
+ {
+ 'repo': 'foo',
+ 'path': '../foo'
+ }
+ ]
+ }
+ ]
+ })
+ self.assertRaises(
+ morphlib.morphloader.InvalidPathError, self.loader.validate, m)
+
def test_fails_to_validate_stratum_which_build_depends_on_self(self):
text = '''\
name: bad-stratum
@@ -567,6 +624,7 @@ build-system: manual
'pre-strip-commands': None,
'post-strip-commands': None,
+ 'extra-sources': [],
'products': [],
'system-integration': [],
'devices': [],
@@ -606,6 +664,7 @@ build-system: manual
"morph": "bar",
'build-mode': 'bootstrap',
'build-depends': [],
+ 'extra-sources': [],
'prefix': '/usr',
},
],
diff --git a/morphlib/plugins/build_plugin.py b/morphlib/plugins/build_plugin.py
index 0aab2d5..84d054d 100644
--- a/morphlib/plugins/build_plugin.py
+++ b/morphlib/plugins/build_plugin.py
@@ -226,6 +226,7 @@ class BuildPlugin(cliapp.Plugin):
'''
bc = morphlib.buildcommand.BuildCommand(self.app)
bc.validate_sources(source_pool)
+ bc.source_pool = source_pool
root = bc.resolve_artifacts(source_pool)
if not component_names:
component_names = [root.source.name]
diff --git a/morphlib/plugins/cross-bootstrap_plugin.py b/morphlib/plugins/cross-bootstrap_plugin.py
index c6ab801..981f6ed 100644
--- a/morphlib/plugins/cross-bootstrap_plugin.py
+++ b/morphlib/plugins/cross-bootstrap_plugin.py
@@ -302,7 +302,7 @@ class CrossBootstrapPlugin(cliapp.Plugin):
system_artifact.source, build_env, use_chroot=False)
builder = BootstrapSystemBuilder(
self.app, staging_area, build_command.lac, build_command.rac,
- system_artifact.source, build_command.repo_cache, 1, False)
+ system_artifact.source, build_command.repo_cache, 1, False, 8)
builder.build_and_cache()
self.app.status(
diff --git a/morphlib/source.py b/morphlib/source.py
index 135c14c..e7486b5 100644
--- a/morphlib/source.py
+++ b/morphlib/source.py
@@ -78,7 +78,7 @@ class Source(object):
def make_sources(reponame, ref, filename, absref, tree, morphology,
- default_split_rules={}):
+ default_split_rules={}, subtrees=[]):
kind = morphology['kind']
if kind in ('system', 'chunk'):
unifier = getattr(morphlib.artifactsplitrule,
@@ -92,6 +92,7 @@ def make_sources(reponame, ref, filename, absref, tree, morphology,
filename, split_rules)
source.artifacts = {name: morphlib.artifact.Artifact(source, name)
for name in split_rules.artifacts}
+ source.subtrees = subtrees
yield source
elif kind == 'stratum': # pragma: no cover
unifier = morphlib.artifactsplitrule.unify_stratum_matches
diff --git a/morphlib/sourceresolver.py b/morphlib/sourceresolver.py
index 5d04ece..a56b659 100644
--- a/morphlib/sourceresolver.py
+++ b/morphlib/sourceresolver.py
@@ -153,7 +153,7 @@ class SourceResolver(object):
The third layer of caching is a simple commit SHA1 -> tree SHA mapping. It
turns out that even if all repos are available locally, running
'git rev-parse' on hundreds of repos requires a lot of IO and can take
- several minutes. Likewise, on a slow network connection it is time
+ several /minutes. Likewise, on a slow network connection it is time
consuming to keep querying the remote repo cache. This third layer of
caching works around both of those issues.
@@ -259,8 +259,59 @@ class SourceResolver(object):
return morph
+ def _resolve_subtrees(self, parent_reponame, parent_ref, extra_sources,
+ resolved_trees):
+
+ subtrees = []
+
+ def resolve_extra_refs(reponame, ref, extra_sources, base_path):
+ for extra_source in extra_sources:
+ if (extra_source['repo'], extra_source['path']) in (
+ resolved_trees):
+ tree = resolved_trees[(extra_source['repo'],
+ extra_source['path'])]
+ else:
+ subref = extra_source.get('ref')
+ if not subref:
+ if self.lrc.has_repo(reponame):
+ repo = self.lrc.get_repo(reponame)
+ if self.update and (
+ repo.requires_update_for_ref(ref)):
+ self.status(msg='Updating cached git '
+ 'repository %(reponame)s for '
+ 'ref %(ref)s',
+ reponame=reponame, ref=ref)
+ repo.update()
+ else:
+ # TODO Add support to the cache server for
+ # retrieving the submodule commit from a path
+ self.status(msg='Updating cached git repository '
+ '%(reponame)s for ref %(ref)s',
+ reponame=reponame, ref=ref)
+ repo = self.lrc.get_updated_repo(reponame, ref)
+ subref = repo.get_submodule_commit(
+ ref, extra_source['path'])
+ _, tree = self._resolve_ref(resolved_trees,
+ extra_source['repo'], subref)
+ resolved_trees[(extra_source['repo'],
+ extra_source['path'])] = tree
+ path = os.path.normpath(os.path.join(base_path,
+ extra_source['path']))
+ subtrees.append({'path': path, 'tree': tree})
+ if 'extra-sources' in extra_source:
+ resolve_extra_refs(extra_source['repo'],
+ extra_source['ref'],
+ extra_source['extra-sources'],
+ path)
+
+ if extra_sources:
+ resolve_extra_refs(parent_reponame, parent_ref,
+ extra_sources, '.')
+ return subtrees
+
def _process_definitions_with_children(self,
resolved_morphologies,
+ resolved_trees,
definitions_checkout_dir,
definitions_repo,
definitions_ref,
@@ -274,7 +325,7 @@ class SourceResolver(object):
# We don't need system's filename, so use 'None'
definitions_queue = collections.deque((None, f)
for f in system_filenames)
- chunk_queue = set()
+ chunk_queue = []
def get_morphology(filename):
return self._get_morphology(resolved_morphologies,
@@ -291,7 +342,7 @@ class SourceResolver(object):
visit(definitions_repo, definitions_ref, filename,
definitions_absref, definitions_tree, morphology,
- predefined_split_rules)
+ predefined_split_rules, [])
if morphology['kind'] == 'cluster':
raise cliapp.AppException(
@@ -313,6 +364,9 @@ class SourceResolver(object):
sanitise_morphology_path(s['morph']))
for s in morphology['build-depends'])
for c in morphology['chunks']:
+ extra_sources = c.get('extra-sources')
+ subtrees = self._resolve_subtrees(c['repo'], c['ref'],
+ extra_sources, resolved_trees)
if 'morph' in c:
# Now, does this path actually exist?
path = c['morph']
@@ -322,15 +376,16 @@ class SourceResolver(object):
raise MorphologyReferenceNotFoundError(
path, filename)
- chunk_queue.add((c['name'], c['repo'], c['ref'],
- path, None))
+ chunk_queue.append((c['name'], c['repo'], c['ref'],
+ path, None, subtrees))
else:
# We invent a filename here, so that the rest of the
# Morph code doesn't need to know about the predefined
# build instructions.
chunk_filename = c['name'] + '.morph'
- chunk_queue.add((c["name"], c['repo'], c['ref'],
- chunk_filename, c['build-system']))
+ chunk_queue.append((c['name'], c['repo'], c['ref'],
+ chunk_filename, c['build-system'],
+ subtrees))
return chunk_queue
@@ -345,7 +400,8 @@ class SourceResolver(object):
def process_chunk(self, resolved_morphologies, resolved_trees,
definitions_checkout_dir, morph_loader, chunk_name,
chunk_repo, chunk_ref, filename, chunk_buildsystem,
- visit, predefined_build_systems, predefined_split_rules):
+ subtrees, visit, predefined_build_systems,
+ predefined_split_rules):
absref, tree = self._resolve_ref(resolved_trees, chunk_repo, chunk_ref)
if chunk_buildsystem is None:
@@ -373,10 +429,10 @@ class SourceResolver(object):
morph_loader, buildsystem, chunk_name)
visit(chunk_repo, chunk_ref, filename, absref, tree, morphology,
- predefined_split_rules)
+ predefined_split_rules, subtrees)
def traverse_morphs(self, definitions_repo, definitions_ref,
- system_filenames,
+ system_filenames, pool,
visit=lambda rn, rf, fn, arf, m: None,
definitions_original_ref=None):
@@ -403,6 +459,7 @@ class SourceResolver(object):
definitions_version = self._check_version_file(
definitions_checkout_dir)
+ pool.definitions_version = definitions_version
predefined_build_systems, predefined_split_rules = \
self._get_defaults(
@@ -415,19 +472,23 @@ class SourceResolver(object):
# will all live in the same Git repository, and will point to
# various chunk morphologies.
chunk_queue = self._process_definitions_with_children(
- resolved_morphologies, definitions_checkout_dir,
- definitions_repo, definitions_ref, definitions_absref,
+ resolved_morphologies, resolved_trees,
+ definitions_checkout_dir, definitions_repo,
+ definitions_ref, definitions_absref,
definitions_tree, morph_loader, system_filenames, visit,
predefined_split_rules)
# Now process all the chunks involved in the build.
- for name, repo, ref, filename, buildsystem in chunk_queue:
+ for name, repo, ref, filename, buildsystem, extra_sources in (
+ chunk_queue):
self.process_chunk(resolved_morphologies, resolved_trees,
definitions_checkout_dir, morph_loader,
name, repo, ref, filename, buildsystem,
- visit, predefined_build_systems,
+ extra_sources, visit,
+ predefined_build_systems,
predefined_split_rules)
+
class DuplicateChunkError(morphlib.Error):
def _make_msg(self, (name, sources)): # pragma: no cover
@@ -478,7 +539,7 @@ def create_source_pool(repo_cache, repo, ref, filenames,
pool = morphlib.sourcepool.SourcePool()
def add_to_pool(reponame, ref, filename, absref, tree, morphology,
- predefined_split_rules):
+ predefined_split_rules, subtrees):
# If there are duplicate chunks which have the same 'name' and the
# same build instructions, we might cause a stack overflow in
# cachekeycomputer.py when trying to hash the build graph. The
@@ -493,7 +554,7 @@ def create_source_pool(repo_cache, repo, ref, filenames,
sources = morphlib.source.make_sources(
reponame, ref, filename, absref, tree, morphology,
- predefined_split_rules)
+ predefined_split_rules, subtrees)
for source in sources:
pool.add(source)
@@ -503,7 +564,7 @@ def create_source_pool(repo_cache, repo, ref, filenames,
resolver = SourceResolver(repo_cache, tree_cache_manager, status_cb)
resolver.traverse_morphs(repo, ref, filenames,
- visit=add_to_pool,
+ pool, visit=add_to_pool,
definitions_original_ref=original_ref)
# No two chunks may have the same name
diff --git a/yarns/building.yarn b/yarns/building.yarn
index 9284f7c..3aea709 100644
--- a/yarns/building.yarn
+++ b/yarns/building.yarn
@@ -89,6 +89,24 @@ Morph Building Tests
AND there are 2 artifacts named core-runtime in the cache
FINALLY the git server is shut down
+ SCENARIO test recursive sources for a chunk
+ GIVEN a workspace
+ AND a git server
+ GIVEN a chunk with dependencies
+ WHEN the user checks out the system branch called master
+ WHEN the user attempts to build the system systems/test-system.morph in branch master
+ THEN morph succeeded
+ FINALLY the git server is shut down
+
+ SCENARIO test recursive sources with non-empty path
+ GIVEN a workspace
+ AND a git server
+ GIVEN a chunk with recursive sources with non-empty paths
+ WHEN the user checks out the system branch called master
+ WHEN the user attempts to build the system systems/test-system.morph in branch master
+ THEN morph failed
+ FINALLY the git server is shut down
+
System integrations
-------------------
diff --git a/yarns/implementations.yarn b/yarns/implementations.yarn
index 06380e7..8ed5a5e 100644
--- a/yarns/implementations.yarn
+++ b/yarns/implementations.yarn
@@ -45,6 +45,81 @@ locally, which we'll tell Morph to access using `file:` URLs. Specifically,
we'll create a repository to hold system and stratum morphologies, and
another to hold a chunk.
+ IMPLEMENTS GIVEN a chunk with recursive sources with non-empty paths
+ cd "$DATADIR/gits/morphs"
+ echo "version: 8" > VERSION
+ cat << EOF >> strata/core.morph
+ - name: invalid-chunk
+ repo: test:chunk-with-submodules
+ ref: master
+ extra-sources:
+ - repo: file://$DATADIR/gits/test-chunk
+ path: somepath
+ ref: master
+ - repo: file://$DATADIR/gits/child-chunk
+ path: somepath
+ EOF
+ git add .
+ git commit -m "More stuff"
+
+
+ IMPLEMENTS GIVEN a chunk with dependencies
+ mkdir "$DATADIR/gits/grandchild-chunk"
+ cd "$DATADIR/gits/grandchild-chunk"
+ git init .
+ touch grandchild-file
+ git add .
+ git commit -m "Initial commit"
+
+ mkdir "$DATADIR/gits/child-chunk"
+ cd "$DATADIR/gits/child-chunk"
+ git init .
+ touch child-file
+ git add .
+ git commit -m "Initial commit"
+ git submodule add -b master file://$DATADIR/gits/grandchild-chunk
+ git commit -m "Initial submodule"
+ git checkout -b new-work
+ git mv child-file child-file-renamed
+ git commit -m "Moar work"
+
+ mkdir "$DATADIR/gits/chunk-with-submodules"
+ cd "$DATADIR/gits/chunk-with-submodules"
+ git init .
+ git add .
+ git commit --allow-empty -m "Initial commit"
+ git submodule add -b master file://$DATADIR/gits/child-chunk
+ #( cd child-chunk && git checkout master)
+ git add .
+ git commit -m "Add submodule"
+
+ cd "$DATADIR/gits/morphs"
+ echo "version: 8" > VERSION
+ cat << EOF >> strata/core.morph
+ - name: chunk-with-submodules
+ morph: chunk-with-submodules.morph
+ repo: test:chunk-with-submodules
+ ref: master
+ extra-sources:
+ - repo: file://$DATADIR/gits/child-chunk
+ path: child-chunk
+ ref: new-work
+ extra-sources:
+ - repo: file://$DATADIR/gits/grandchild-chunk
+ path: grandchild-chunk
+ EOF
+
+ cat << EOF >> chunk-with-submodules.morph
+ name: chunk-with-submodules
+ kind: chunk
+ build-system: manual
+ build-commands:
+ - file exists child-chunk/child-file-renamed
+ - file exists child-chunk/grandchild-chunk/grandchild-file
+ EOF
+ git add .
+ git commit -m "Add moar stuff"
+
IMPLEMENTS GIVEN a git server
# Create a directory for all the git repositories.