summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--morphlib/artifactresolver.py1
-rw-r--r--morphlib/buildcommand.py22
-rw-r--r--morphlib/builder.py64
-rw-r--r--morphlib/builder_tests.py3
-rw-r--r--morphlib/cachedrepo.py4
-rw-r--r--morphlib/cachekeycomputer.py5
-rw-r--r--morphlib/definitions_version.py2
-rw-r--r--morphlib/git.py47
-rw-r--r--morphlib/gitdir.py22
-rw-r--r--morphlib/gitdir_tests.py6
-rw-r--r--morphlib/morphloader.py37
-rw-r--r--morphlib/morphloader_tests.py60
-rw-r--r--morphlib/plugins/build_plugin.py1
-rw-r--r--morphlib/plugins/cross-bootstrap_plugin.py2
-rw-r--r--morphlib/source.py3
-rw-r--r--morphlib/sourceresolver.py94
-rw-r--r--scripts/test-shell.c6
-rw-r--r--yarns/building.yarn18
-rw-r--r--yarns/implementations.yarn75
19 files changed, 401 insertions, 71 deletions
diff --git a/morphlib/artifactresolver.py b/morphlib/artifactresolver.py
index f3936df1..2b40752d 100644
--- a/morphlib/artifactresolver.py
+++ b/morphlib/artifactresolver.py
@@ -189,6 +189,7 @@ class ArtifactResolver(object):
# Resolve now to avoid a search for the parent morphology later
chunk_source.build_mode = info['build-mode']
chunk_source.prefix = info['prefix']
+ chunk_source.extra_sources = info['extra-sources']
# Add these chunks to the processed artifacts, so other
# chunks may refer to them.
diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py
index 222d229a..4918cec6 100644
--- a/morphlib/buildcommand.py
+++ b/morphlib/buildcommand.py
@@ -102,6 +102,7 @@ class BuildCommand(object):
original_ref=original_ref,
update_repos=not self.app.settings['no-git-update'],
status_cb=self.app.status)
+ self.source_pool = srcpool
return srcpool
def validate_sources(self, srcpool):
@@ -394,9 +395,25 @@ class BuildCommand(object):
def fetch_sources(self, source):
'''Update the local git repository cache with the sources.'''
+ def fetch_extra_sources(lrc, parent_repo, parent_ref, extra_sources):
+ for extra_source in extra_sources:
+ ref = extra_source.get('ref')
+ if not ref:
+ ref = parent_repo.get_submodule_commit(
+ parent_ref, extra_source['path'])
+ repo = self.lrc.get_updated_repo(extra_source['repo'],
+ ref=ref)
+ fetch_extra_sources(lrc, repo, ref,
+ extra_source.get('extra-sources', []))
+
repo_name = source.repo_name
source.repo = self.lrc.get_updated_repo(repo_name, ref=source.sha1)
- self.lrc.ensure_submodules(source.repo, source.sha1)
+ if source.morphology['kind'] == 'chunk':
+ if self.source_pool.definitions_version >= 8:
+ fetch_extra_sources(self.lrc, source.repo, source.sha1,
+ source.extra_sources)
+ else:
+ self.lrc.ensure_submodules(source.repo, source.sha1)
def cache_artifacts_locally(self, artifacts):
'''Get artifacts missing from local cache from remote cache.'''
@@ -542,7 +559,8 @@ class BuildCommand(object):
name=source.name, sha1=source.sha1[:7])
builder = morphlib.builder.Builder(
self.app, staging_area, self.lac, self.rac, self.lrc,
- self.app.settings['max-jobs'], setup_mounts)
+ self.app.settings['max-jobs'], setup_mounts,
+ self.source_pool.definitions_version)
return builder.build_and_cache(source)
class InitiatorBuildCommand(BuildCommand):
diff --git a/morphlib/builder.py b/morphlib/builder.py
index cfd5510c..44abaaa9 100644
--- a/morphlib/builder.py
+++ b/morphlib/builder.py
@@ -35,7 +35,8 @@ import morphlib.gitversion
SYSTEM_INTEGRATION_PATH = os.path.join('baserock', 'system-integration')
-def extract_sources(app, repo_cache, repo, sha1, srcdir): #pragma: no cover
+def extract_sources(app, definitions_version, repo_cache, repo, sha1,
+ destdir, source): #pragma: no cover
'''Get sources from git to a source directory, including submodules'''
def extract_repo(repo, sha1, destdir):
@@ -57,12 +58,48 @@ def extract_sources(app, repo_cache, repo, sha1, srcdir): #pragma: no cover
sub_dir = os.path.join(destdir, sub.path)
tuples.append((cached_repo, sub.commit, sub_dir))
return tuples
+ return []
- todo = [(repo, sha1, srcdir)]
- while todo:
- repo, sha1, srcdir = todo.pop()
- todo += extract_repo(repo, sha1, srcdir)
- set_mtime_recursively(srcdir)
+ def extract_repo_version_8(repo, ref, extra_sources,
+ rootdir, destdir):
+
+ app.status(msg='Extracting %(source)s into %(path)s',
+ source=repo.original_name,
+ path=destdir)
+ repo.checkout(ref, destdir)
+ morphlib.git.reset_workdir(app.runcmd, destdir)
+
+ for extra_source in extra_sources:
+ subrepo = repo_cache.get_repo(extra_source['repo'])
+ path = os.path.normpath(extra_source['path'])
+ checkout_dir = os.path.join(destdir, path)
+ if os.path.exists(checkout_dir):
+ if os.listdir(checkout_dir):
+ raise cliapp.AppException(
+ "Failed to clone '%s': the directory '%s' "
+ "is not empty" %
+ (subrepo.original_name, os.path.join(
+ rootdir, path)))
+ else:
+ os.makedirs(checkout_dir)
+ subref = extra_source.get('ref')
+ if not subref:
+ subref = repo.get_submodule_commit(ref, path)
+
+ extract_repo_version_8(subrepo, subref,
+ extra_source.get('extra-sources', []),
+ rootdir,
+ os.path.join(destdir, checkout_dir))
+ if definitions_version >= 8:
+ extract_repo_version_8(repo, sha1, source.extra_sources,
+ os.path.basename(destdir), destdir)
+ else:
+ todo = [(repo, sha1, destdir)]
+ while todo:
+ repo, sha1, destdir = todo.pop()
+ todo += extract_repo(repo, sha1, destdir)
+
+ set_mtime_recursively(destdir)
def set_mtime_recursively(root): # pragma: no cover
'''Set the mtime for every file in a directory tree to the same.
@@ -147,7 +184,7 @@ class BuilderBase(object):
def __init__(self, app, staging_area, local_artifact_cache,
remote_artifact_cache, source, repo_cache, max_jobs,
- setup_mounts):
+ setup_mounts, definitions_version):
self.app = app
self.staging_area = staging_area
self.local_artifact_cache = local_artifact_cache
@@ -157,6 +194,7 @@ class BuilderBase(object):
self.max_jobs = max_jobs
self.build_watch = morphlib.stopwatch.Stopwatch()
self.setup_mounts = setup_mounts
+ self.definitions_version = definitions_version
def save_build_times(self):
'''Write the times captured by the stopwatch'''
@@ -312,7 +350,6 @@ class ChunkBuilder(BuilderBase):
def run_commands(self, logfilepath, stdout=None): # pragma: no cover
m = self.source.morphology
bs = morphlib.buildsystem.lookup_build_system(m['build-system'])
-
relative_builddir = self.staging_area.relative_builddir()
relative_destdir = self.staging_area.relative_destdir()
ccache_dir = self.staging_area.ccache_dir()
@@ -375,7 +412,6 @@ class ChunkBuilder(BuilderBase):
stderr=subprocess.STDOUT,
logfile=logfilepath,
ccache_dir=ccache_dir)
-
if stdout:
stdout.flush()
@@ -491,7 +527,8 @@ class ChunkBuilder(BuilderBase):
def get_sources(self, srcdir): # pragma: no cover
s = self.source
- extract_sources(self.app, self.repo_cache, s.repo, s.sha1, srcdir)
+ extract_sources(self.app, self.definitions_version, self.repo_cache,
+ s.repo, s.sha1, srcdir, s)
class StratumBuilder(BuilderBase):
@@ -726,7 +763,8 @@ class Builder(object): # pragma: no cover
}
def __init__(self, app, staging_area, local_artifact_cache,
- remote_artifact_cache, repo_cache, max_jobs, setup_mounts):
+ remote_artifact_cache, repo_cache, max_jobs, setup_mounts,
+ definitions_version):
self.app = app
self.staging_area = staging_area
self.local_artifact_cache = local_artifact_cache
@@ -734,6 +772,7 @@ class Builder(object): # pragma: no cover
self.repo_cache = repo_cache
self.max_jobs = max_jobs
self.setup_mounts = setup_mounts
+ self.definitions_version = definitions_version
def build_and_cache(self, source):
kind = source.morphology['kind']
@@ -741,7 +780,8 @@ class Builder(object): # pragma: no cover
self.local_artifact_cache,
self.remote_artifact_cache, source,
self.repo_cache, self.max_jobs,
- self.setup_mounts)
+ self.setup_mounts,
+ self.definitions_version)
self.app.status(msg='Builder.build: artifact %s with %s' %
(source.name, repr(o)),
chatty=True)
diff --git a/morphlib/builder_tests.py b/morphlib/builder_tests.py
index a571e3d0..e9629e1e 100644
--- a/morphlib/builder_tests.py
+++ b/morphlib/builder_tests.py
@@ -162,7 +162,8 @@ class BuilderBaseTests(unittest.TestCase):
self.artifact,
self.repo_cache,
self.max_jobs,
- False)
+ False,
+ 7)
def test_runs_desired_command(self):
self.builder.runcmd(['foo', 'bar'])
diff --git a/morphlib/cachedrepo.py b/morphlib/cachedrepo.py
index 76cdaa86..168003b9 100644
--- a/morphlib/cachedrepo.py
+++ b/morphlib/cachedrepo.py
@@ -245,6 +245,10 @@ class CachedRepo(object):
else:
return False
+ def get_submodule_commit(self, parent_ref, submodule_path):
+ return self.gitdir.get_submodule_commit(parent_ref, # pragma: no cover
+ submodule_path)
+
def update(self):
'''Updates the cached repository using its origin remote.
diff --git a/morphlib/cachekeycomputer.py b/morphlib/cachekeycomputer.py
index 22f42aa0..ff0d595a 100644
--- a/morphlib/cachekeycomputer.py
+++ b/morphlib/cachekeycomputer.py
@@ -96,6 +96,11 @@ class CacheKeyComputer(object):
keys['split-rules'] = [(a, [rgx.pattern for rgx in r._regexes])
for (a, r) in source.split_rules]
+ # Avoid adding the 'extra-trees' key if there were no
+ # extra-sources, to avoid unnecessary rebuilds
+ if source.subtrees: # pragma no cover
+ keys['extra-trees'] = source.subtrees
+
# Include morphology contents, since it doesn't always come
# from the source tree
keys['devices'] = morphology.get('devices')
diff --git a/morphlib/definitions_version.py b/morphlib/definitions_version.py
index 2fb7785a..c39fe422 100644
--- a/morphlib/definitions_version.py
+++ b/morphlib/definitions_version.py
@@ -24,7 +24,7 @@ import yaml
import morphlib
-SUPPORTED_VERSIONS = [6, 7]
+SUPPORTED_VERSIONS = [6, 7, 8]
class DefinitionsVersionError(cliapp.AppException):
diff --git a/morphlib/git.py b/morphlib/git.py
index b6f54d02..b43b335e 100644
--- a/morphlib/git.py
+++ b/morphlib/git.py
@@ -34,11 +34,17 @@ class NoModulesFileError(cliapp.AppException):
class Submodule(object):
- def __init__(self, name, url, path):
+ def __init__(self, name, url, sha1, path):
self.name = name
self.url = url
+ self.commit = sha1
self.path = path
+ def __str__(self):
+ return "{name}|{url}|{path}".format(name=self.name,
+ url=self.url,
+ path=self.path)
+
class InvalidSectionError(cliapp.AppException):
@@ -48,14 +54,6 @@ class InvalidSectionError(cliapp.AppException):
'title: [%s]' % (repo, ref, section))
-class MissingSubmoduleCommitError(cliapp.AppException):
-
- def __init__(self, repo, ref, submodule):
- Exception.__init__(self,
- '%s:%s:.gitmodules: No commit object found for '
- 'submodule "%s"' % (repo, ref, submodule))
-
-
class Submodules(object):
def __init__(self, app, repo, ref):
@@ -86,6 +84,7 @@ class Submodules(object):
raise NoModulesFileError(self.repo, self.ref)
def _validate_and_read_entries(self, parser):
+ gd = morphlib.gitdir.GitDirectory(self.repo)
for section in parser.sections():
# validate section name against the 'section "foo"' pattern
section_pattern = r'submodule "(.*)"'
@@ -96,33 +95,9 @@ class Submodules(object):
path = parser.get(section, 'path')
# create a submodule object
- submodule = Submodule(name, url, path)
- try:
- # list objects in the parent repo tree to find the commit
- # object that corresponds to the submodule
- commit = gitcmd(self.app.runcmd, 'ls-tree', self.ref,
- submodule.path, cwd=self.repo)
-
- # read the commit hash from the output
- fields = commit.split()
- if len(fields) >= 2 and fields[1] == 'commit':
- submodule.commit = commit.split()[2]
-
- # fail if the commit hash is invalid
- if len(submodule.commit) != 40:
- raise MissingSubmoduleCommitError(self.repo,
- self.ref,
- submodule.name)
-
- # add a submodule object to the list
- self.submodules.append(submodule)
- else:
- logging.warning('Skipping submodule "%s" as %s:%s has '
- 'a non-commit object for it' %
- (submodule.name, self.repo, self.ref))
- except cliapp.AppException:
- raise MissingSubmoduleCommitError(self.repo, self.ref,
- submodule.name)
+ sha1 = gd.get_submodule_commit(self.ref, path)
+ submodule = Submodule(name, url, sha1, path)
+ self.submodules.append(submodule)
else:
raise InvalidSectionError(self.repo, self.ref, section)
diff --git a/morphlib/gitdir.py b/morphlib/gitdir.py
index d1770275..a4e51e30 100644
--- a/morphlib/gitdir.py
+++ b/morphlib/gitdir.py
@@ -53,6 +53,14 @@ class ExpectedSha1Error(cliapp.AppException):
self, 'SHA1 expected, got %s' % ref)
+class MissingSubmoduleCommitError(cliapp.AppException):
+
+ def __init__(self, repo, ref, submodule):
+ cliapp.AppException.__init__(self, # pragma
+ '%s:%s:.gitmodules: No commit object found for '
+ 'submodule "%s"' % (repo, ref, submodule))
+
+
class RefChangeError(cliapp.AppException):
pass
@@ -861,6 +869,20 @@ class GitDirectory(object):
except Exception as e:
raise RefDeleteError(self, ref, old_sha1, e)
+ def get_submodule_commit(self, parent_ref,
+ submodule_path): # pragma: no cover
+ try:
+ lstree_output = morphlib.git.gitcmd(self._runcmd, 'ls-tree',
+ parent_ref, submodule_path)
+ except cliapp.AppException:
+ raise MissingSubmoduleCommitError(self.dirname, parent_ref,
+ submodule_path)
+ m = re.match("160000 commit (?P<sha1>\w{40})", lstree_output)
+ if not m:
+ raise MissingSubmoduleCommitError(self.dirname, parent_ref,
+ submodule_path)
+ return m.group('sha1')
+
def describe(self):
version = morphlib.git.gitcmd(self._runcmd, 'describe',
'--always', '--dirty=-unreproducible')
diff --git a/morphlib/gitdir_tests.py b/morphlib/gitdir_tests.py
index 0ec7b0f1..e75ef483 100644
--- a/morphlib/gitdir_tests.py
+++ b/morphlib/gitdir_tests.py
@@ -102,6 +102,12 @@ class GitDirectoryTests(unittest.TestCase):
gitdir = self.empty_git_directory()
self.assertIsInstance(gitdir.get_index(), morphlib.gitindex.GitIndex)
+ def test_non_existent_submodule_path_raises_error(self):
+ gitdir = self.empty_git_directory()
+ self.assertRaises(
+ morphlib.gitdir.MissingSubmoduleCommitError,
+ gitdir.get_submodule_commit, 'master', 'somepath')
+
class GitDirectoryAnchoredRefTests(unittest.TestCase):
diff --git a/morphlib/morphloader.py b/morphlib/morphloader.py
index 7bd5c37a..79755b76 100644
--- a/morphlib/morphloader.py
+++ b/morphlib/morphloader.py
@@ -15,6 +15,7 @@
# =*= License: GPL-2 =*=
+import os
import collections
import warnings
import yaml
@@ -90,6 +91,13 @@ class InvalidTypeError(MorphologyValidationError):
(field, expected, actual, morphology_name))
+class InvalidPathError(MorphologyValidationError):
+
+ def __init__(self, path, spec, morph_filename):
+ self.msg = ("Invalid path '%s' in %s from morphology %s"
+ % (path, spec, morph_filename))
+
+
class UnknownArchitectureError(MorphologyValidationError):
def __init__(self, arch, morph_filename):
@@ -236,6 +244,7 @@ class MorphologyDumper(yaml.SafeDumper):
'build-mode',
'artifacts',
'max-jobs',
+ 'extra-sources',
'products',
'chunks',
'build-system',
@@ -347,6 +356,7 @@ class MorphologyLoader(object):
'strip-commands': None,
'post-strip-commands': None,
'devices': [],
+ 'extra-sources': [],
'products': [],
'max-jobs': None,
'build-system': 'manual',
@@ -554,6 +564,27 @@ class MorphologyLoader(object):
raise ChunkSpecNoBuildInstructionsError(
chunk_name, morph.filename)
+ def validate_extra_sources(extra_sources, morph_filename):
+ for extra_source in extra_sources:
+ validate_chunk_str_field('repo', extra_source,
+ morph.filename)
+ path = extra_source.get('path')
+ if not path:
+ raise MissingFieldError("'path' in %s" % extra_source,
+ morph.filename)
+ path = os.path.normpath(path)
+ if os.path.isabs(path) or path.startswith('..') or (
+ path == '.') or not path.strip():
+ raise InvalidPathError(extra_source['path'],
+ extra_source,
+ morph_filename)
+ if 'extra-sources' in extra_source:
+ validate_extra_sources(extra_source['extra-sources'],
+ morph.filename)
+
+ if 'extra-sources' in spec:
+ validate_extra_sources(spec['extra-sources'], morph.filename)
+
@classmethod
def _validate_chunk(cls, morphology):
errors = []
@@ -706,6 +737,9 @@ class MorphologyLoader(object):
if 'prefix' not in spec:
spec['prefix'] = \
self._static_defaults['chunk']['prefix']
+ if 'extra-sources' not in spec:
+ spec['extra-sources'] = \
+ self._static_defaults['chunk']['extra-sources']
def _unset_stratum_defaults(self, morph):
for spec in morph['chunks']:
@@ -717,6 +751,9 @@ class MorphologyLoader(object):
if 'prefix' in spec and spec['prefix'] == \
self._static_defaults['chunk']['prefix']:
del spec['prefix']
+ if 'extra-sources' in spec and spec['extra-sources'] == \
+ self._static_defaults['chunk']['extra-sources']:
+ del spec['extra-sources']
def _set_chunk_defaults(self, morph):
if morph['max-jobs'] is not None:
diff --git a/morphlib/morphloader_tests.py b/morphlib/morphloader_tests.py
index 8039dfc9..3f1f7cbe 100644
--- a/morphlib/morphloader_tests.py
+++ b/morphlib/morphloader_tests.py
@@ -278,6 +278,63 @@ build-system: manual
morphlib.morphloader.InvalidStringError):
self.loader.validate(m)
+ def test_fails_to_validate_stratum_with_a_missing_path(self):
+ m = morphlib.morphology.Morphology({
+ 'kind': 'stratum',
+ 'name': 'foo',
+ 'build-depends': [],
+ 'chunks': [
+ {
+ 'name': 'chunk',
+ 'repo': 'test:repo',
+ 'ref': 'master',
+ 'build-system': 'manual',
+ 'build-depends': [],
+ 'extra-sources':
+ [
+ {
+ 'repo': 'foo',
+ 'path': 'somepath',
+ 'extra-sources':
+ [
+ {
+ 'repo': 'bar',
+ 'ref': 'master'
+ }
+ ]
+ }
+ ]
+ }
+ ]
+ })
+ self.assertRaises(
+ morphlib.morphloader.MissingFieldError, self.loader.validate, m)
+
+ def test_fails_to_validate_stratum_with_invalid_path(self):
+ m = morphlib.morphology.Morphology({
+ 'kind': 'stratum',
+ 'name': 'foo',
+ 'build-depends': [],
+ 'chunks': [
+ {
+ 'name': 'chunk',
+ 'repo': 'test:repo',
+ 'ref': 'master',
+ 'build-system': 'manual',
+ 'build-depends': [],
+ 'extra-sources':
+ [
+ {
+ 'repo': 'foo',
+ 'path': '../foo'
+ }
+ ]
+ }
+ ]
+ })
+ self.assertRaises(
+ morphlib.morphloader.InvalidPathError, self.loader.validate, m)
+
def test_fails_to_validate_stratum_which_build_depends_on_self(self):
text = '''\
name: bad-stratum
@@ -601,6 +658,7 @@ build-system: manual
'pre-strip-commands': None,
'post-strip-commands': None,
+ 'extra-sources': [],
'products': [],
'system-integration': [],
'devices': [],
@@ -654,6 +712,7 @@ build-system: manual
"morph": "bar",
'build-mode': 'bootstrap',
'build-depends': [],
+ 'extra-sources': [],
'prefix': '/usr',
},
],
@@ -670,6 +729,7 @@ build-system: manual
"ref": "bar",
'build-mode': 'staging',
'build-depends': [],
+ 'extra-sources': [],
'prefix': '/usr',
},
],
diff --git a/morphlib/plugins/build_plugin.py b/morphlib/plugins/build_plugin.py
index 226a2c85..baa72756 100644
--- a/morphlib/plugins/build_plugin.py
+++ b/morphlib/plugins/build_plugin.py
@@ -310,6 +310,7 @@ class BuildPlugin(cliapp.Plugin):
'''
bc = morphlib.buildcommand.BuildCommand(self.app)
bc.validate_sources(source_pool)
+ bc.source_pool = source_pool
root = bc.resolve_artifacts(source_pool)
if not component_names:
component_names = [root.source.name]
diff --git a/morphlib/plugins/cross-bootstrap_plugin.py b/morphlib/plugins/cross-bootstrap_plugin.py
index 265b273b..c0885202 100644
--- a/morphlib/plugins/cross-bootstrap_plugin.py
+++ b/morphlib/plugins/cross-bootstrap_plugin.py
@@ -304,7 +304,7 @@ class CrossBootstrapPlugin(cliapp.Plugin):
system_artifact.source, build_env, use_chroot=False)
builder = BootstrapSystemBuilder(
self.app, staging_area, build_command.lac, build_command.rac,
- system_artifact.source, build_command.lrc, 1, False)
+ system_artifact.source, build_command.lrc, 1, False, 8)
builder.build_and_cache()
self.app.status(
diff --git a/morphlib/source.py b/morphlib/source.py
index 135c14cc..e7486b56 100644
--- a/morphlib/source.py
+++ b/morphlib/source.py
@@ -78,7 +78,7 @@ class Source(object):
def make_sources(reponame, ref, filename, absref, tree, morphology,
- default_split_rules={}):
+ default_split_rules={}, subtrees=[]):
kind = morphology['kind']
if kind in ('system', 'chunk'):
unifier = getattr(morphlib.artifactsplitrule,
@@ -92,6 +92,7 @@ def make_sources(reponame, ref, filename, absref, tree, morphology,
filename, split_rules)
source.artifacts = {name: morphlib.artifact.Artifact(source, name)
for name in split_rules.artifacts}
+ source.subtrees = subtrees
yield source
elif kind == 'stratum': # pragma: no cover
unifier = morphlib.artifactsplitrule.unify_stratum_matches
diff --git a/morphlib/sourceresolver.py b/morphlib/sourceresolver.py
index 108adc14..a3b1608f 100644
--- a/morphlib/sourceresolver.py
+++ b/morphlib/sourceresolver.py
@@ -145,7 +145,7 @@ class SourceResolver(object):
The third layer of caching is a simple commit SHA1 -> tree SHA mapping. It
turns out that even if all repos are available locally, running
'git rev-parse' on hundreds of repos requires a lot of IO and can take
- several minutes. Likewise, on a slow network connection it is time
+ several /minutes. Likewise, on a slow network connection it is time
consuming to keep querying the remote repo cache. This third layer of
caching works around both of those issues.
@@ -308,8 +308,59 @@ class SourceResolver(object):
return morph
+ def _resolve_subtrees(self, parent_reponame, parent_ref, extra_sources,
+ resolved_trees):
+
+ subtrees = []
+
+ def resolve_extra_refs(reponame, ref, extra_sources, base_path):
+ for extra_source in extra_sources:
+ if (extra_source['repo'], extra_source['path']) in (
+ resolved_trees):
+ tree = resolved_trees[(extra_source['repo'],
+ extra_source['path'])]
+ else:
+ subref = extra_source.get('ref')
+ if not subref:
+ if self.lrc.has_repo(reponame):
+ repo = self.lrc.get_repo(reponame)
+ if self.update and (
+ repo.requires_update_for_ref(ref)):
+ self.status(msg='Updating cached git '
+ 'repository %(reponame)s for '
+ 'ref %(ref)s',
+ reponame=reponame, ref=ref)
+ repo.update()
+ else:
+ # TODO Add support to the cache server for
+ # retrieving the submodule commit from a path
+ self.status(msg='Updating cached git repository '
+ '%(reponame)s for ref %(ref)s',
+ reponame=reponame, ref=ref)
+ repo = self.lrc.get_updated_repo(reponame, ref)
+ subref = repo.get_submodule_commit(
+ ref, extra_source['path'])
+ _, tree = self._resolve_ref(resolved_trees,
+ extra_source['repo'], subref)
+ resolved_trees[(extra_source['repo'],
+ extra_source['path'])] = tree
+ path = os.path.normpath(os.path.join(base_path,
+ extra_source['path']))
+ subtrees.append({'path': path, 'tree': tree})
+ if 'extra-sources' in extra_source:
+ resolve_extra_refs(extra_source['repo'],
+ extra_source['ref'],
+ extra_source['extra-sources'],
+ path)
+
+ if extra_sources:
+ resolve_extra_refs(parent_reponame, parent_ref,
+ extra_sources, '.')
+ return subtrees
+
def _process_definitions_with_children(self,
resolved_morphologies,
+ resolved_trees,
definitions_checkout_dir,
definitions_repo,
definitions_ref,
@@ -320,7 +371,7 @@ class SourceResolver(object):
visit,
predefined_split_rules):
definitions_queue = collections.deque(system_filenames)
- chunk_queue = set()
+ chunk_queue = []
def get_morphology(filename):
return self._get_morphology(resolved_morphologies,
@@ -336,7 +387,7 @@ class SourceResolver(object):
visit(definitions_repo, definitions_ref, filename,
definitions_absref, definitions_tree, morphology,
- predefined_split_rules)
+ predefined_split_rules, [])
if morphology['kind'] == 'cluster':
raise cliapp.AppException(
@@ -351,6 +402,9 @@ class SourceResolver(object):
sanitise_morphology_path(s['morph'])
for s in morphology['build-depends'])
for c in morphology['chunks']:
+ extra_sources = c.get('extra-sources')
+ subtrees = self._resolve_subtrees(c['repo'], c['ref'],
+ extra_sources, resolved_trees)
if 'morph' in c:
# Now, does this path actually exist?
path = c['morph']
@@ -360,14 +414,15 @@ class SourceResolver(object):
raise MorphologyReferenceNotFoundError(
path, filename)
- chunk_queue.add((c['repo'], c['ref'], path, None))
+ chunk_queue.append((c['repo'], c['ref'], path, None,
+ subtrees))
else:
# We invent a filename here, so that the rest of the
# Morph code doesn't need to know about the predefined
# build instructions.
chunk_name = c['name'] + '.morph'
- chunk_queue.add((c['repo'], c['ref'], chunk_name,
- c['build-system']))
+ chunk_queue.append((c['repo'], c['ref'], chunk_name,
+ c['build-system'], subtrees))
return chunk_queue
@@ -381,8 +436,8 @@ class SourceResolver(object):
def process_chunk(self, resolved_morphologies, resolved_trees,
definitions_checkout_dir, morph_loader, chunk_repo,
- chunk_ref, filename, chunk_buildsystem, visit,
- predefined_split_rules):
+ chunk_ref, filename, chunk_buildsystem, subtrees,
+ visit, predefined_split_rules):
absref, tree = self._resolve_ref(resolved_trees, chunk_repo, chunk_ref)
if chunk_buildsystem is None:
@@ -403,10 +458,10 @@ class SourceResolver(object):
morph_loader, buildsystem, filename)
visit(chunk_repo, chunk_ref, filename, absref, tree, morphology,
- predefined_split_rules)
+ predefined_split_rules, subtrees)
def traverse_morphs(self, definitions_repo, definitions_ref,
- system_filenames,
+ system_filenames, pool,
visit=lambda rn, rf, fn, arf, m: None,
definitions_original_ref=None):
@@ -433,6 +488,7 @@ class SourceResolver(object):
definitions_version = self._check_version_file(
definitions_checkout_dir)
+ pool.definitions_version = definitions_version
predefined_build_systems, predefined_split_rules = \
self._get_defaults(
@@ -445,18 +501,22 @@ class SourceResolver(object):
# will all live in the same Git repository, and will point to
# various chunk morphologies.
chunk_queue = self._process_definitions_with_children(
- resolved_morphologies, definitions_checkout_dir,
- definitions_repo, definitions_ref, definitions_absref,
+ resolved_morphologies, resolved_trees,
+ definitions_checkout_dir, definitions_repo,
+ definitions_ref, definitions_absref,
definitions_tree, morph_loader, system_filenames, visit,
predefined_split_rules)
# Now process all the chunks involved in the build.
- for repo, ref, filename, buildsystem in chunk_queue:
+ for repo, ref, filename, buildsystem, extra_sources in (
+ chunk_queue):
self.process_chunk(resolved_morphologies, resolved_trees,
definitions_checkout_dir, morph_loader,
- repo, ref, filename, buildsystem, visit,
+ repo, ref, filename, buildsystem,
+ extra_sources, visit,
predefined_split_rules)
+
class DuplicateChunkError(morphlib.Error):
def _make_msg(self, (name, sources)): # pragma: no cover
@@ -504,10 +564,10 @@ def create_source_pool(lrc, rrc, repo, ref, filenames, cachedir,
pool = morphlib.sourcepool.SourcePool()
def add_to_pool(reponame, ref, filename, absref, tree, morphology,
- predefined_split_rules):
+ predefined_split_rules, subtrees):
sources = morphlib.source.make_sources(
reponame, ref, filename, absref, tree, morphology,
- predefined_split_rules)
+ predefined_split_rules, subtrees)
for source in sources:
pool.add(source)
@@ -517,7 +577,7 @@ def create_source_pool(lrc, rrc, repo, ref, filenames, cachedir,
resolver = SourceResolver(lrc, rrc, tree_cache_manager, update_repos,
status_cb)
resolver.traverse_morphs(repo, ref, filenames,
- visit=add_to_pool,
+ pool, visit=add_to_pool,
definitions_original_ref=original_ref)
# No two chunks may have the same name
diff --git a/scripts/test-shell.c b/scripts/test-shell.c
index 963d128c..9dadedcb 100644
--- a/scripts/test-shell.c
+++ b/scripts/test-shell.c
@@ -163,6 +163,12 @@ int run_commands(FILE *cmdstream){
ret = 1;
break;
}
+ } else if (strstr(line, "file exists ") == line) {
+ char const *filename = line + sizeof("file exists ") -1;
+ fprintf(stderr, "FILENAME: %s\n", filename);
+ struct stat st;
+ int result = stat(filename, &st);
+ return result != 0;
} else if (strstr(line, "create file ") == line) {
char const *filename = line + sizeof("create file ") -1;
FILE *outfile = fopen(filename, "w");
diff --git a/yarns/building.yarn b/yarns/building.yarn
index 8a98e5d9..9470629e 100644
--- a/yarns/building.yarn
+++ b/yarns/building.yarn
@@ -37,6 +37,24 @@ Morph Building Tests
THEN morph succeeded
FINALLY the git server is shut down
+ SCENARIO test recursive sources for a chunk
+ GIVEN a workspace
+ AND a git server
+ GIVEN a chunk with dependencies
+ WHEN the user checks out the system branch called master
+ WHEN the user attempts to build the system systems/test-system.morph in branch master
+ THEN morph succeeded
+ FINALLY the git server is shut down
+
+ SCENARIO test recursive sources with non-empty path
+ GIVEN a workspace
+ AND a git server
+ GIVEN a chunk with recursive sources with non-empty paths
+ WHEN the user checks out the system branch called master
+ WHEN the user attempts to build the system systems/test-system.morph in branch master
+ THEN morph failed
+ FINALLY the git server is shut down
+
System integrations
-------------------
diff --git a/yarns/implementations.yarn b/yarns/implementations.yarn
index 9f23107b..afa5df9d 100644
--- a/yarns/implementations.yarn
+++ b/yarns/implementations.yarn
@@ -78,6 +78,81 @@ locally, which we'll tell Morph to access using `file:` URLs. Specifically,
we'll create a repository to hold system and stratum morphologies, and
another to hold a chunk.
+ IMPLEMENTS GIVEN a chunk with recursive sources with non-empty paths
+ cd "$DATADIR/gits/morphs"
+ echo "version: 8" > VERSION
+ cat << EOF >> strata/core.morph
+ - name: invalid-chunk
+ repo: test:chunk-with-submodules
+ ref: master
+ extra-sources:
+ - repo: file://$DATADIR/gits/test-chunk
+ path: somepath
+ ref: master
+ - repo: file://$DATADIR/gits/child-chunk
+ path: somepath
+ EOF
+ git add .
+ git commit -m "More stuff"
+
+
+ IMPLEMENTS GIVEN a chunk with dependencies
+ mkdir "$DATADIR/gits/grandchild-chunk"
+ cd "$DATADIR/gits/grandchild-chunk"
+ git init .
+ touch grandchild-file
+ git add .
+ git commit -m "Initial commit"
+
+ mkdir "$DATADIR/gits/child-chunk"
+ cd "$DATADIR/gits/child-chunk"
+ git init .
+ touch child-file
+ git add .
+ git commit -m "Initial commit"
+ git submodule add -b master file://$DATADIR/gits/grandchild-chunk
+ git commit -m "Initial submodule"
+ git checkout -b new-work
+ git mv child-file child-file-renamed
+ git commit -m "Moar work"
+
+ mkdir "$DATADIR/gits/chunk-with-submodules"
+ cd "$DATADIR/gits/chunk-with-submodules"
+ git init .
+ git add .
+ git commit --allow-empty -m "Initial commit"
+ git submodule add -b master file://$DATADIR/gits/child-chunk
+ #( cd child-chunk && git checkout master)
+ git add .
+ git commit -m "Add submodule"
+
+ cd "$DATADIR/gits/morphs"
+ echo "version: 8" > VERSION
+ cat << EOF >> strata/core.morph
+ - name: chunk-with-submodules
+ morph: chunk-with-submodules.morph
+ repo: test:chunk-with-submodules
+ ref: master
+ extra-sources:
+ - repo: file://$DATADIR/gits/child-chunk
+ path: child-chunk
+ ref: new-work
+ extra-sources:
+ - repo: file://$DATADIR/gits/grandchild-chunk
+ path: grandchild-chunk
+ EOF
+
+ cat << EOF >> chunk-with-submodules.morph
+ name: chunk-with-submodules
+ kind: chunk
+ build-system: manual
+ build-commands:
+ - file exists child-chunk/child-file-renamed
+ - file exists child-chunk/grandchild-chunk/grandchild-file
+ EOF
+ git add .
+ git commit -m "Add moar stuff"
+
IMPLEMENTS GIVEN a git server
# Create a directory for all the git repositories.