From ddb8b27d5056074013a89d8ab277e39fdc1f069d Mon Sep 17 00:00:00 2001 From: Tiago Gomes Date: Fri, 20 Nov 2015 13:57:31 +0000 Subject: WIP Add support for multiple sources per chunk TODO: - Add API to the cache server to retrieve a submodule commit from a given path. - Fix cross-bootstrap command. Change-Id: I3475c2bcb648a272fee33bc878a521f79d4e6581 --- morphlib/artifactresolver.py | 1 + morphlib/buildcommand.py | 24 +++++++- morphlib/builder.py | 63 ++++++++++++++++---- morphlib/builder_tests.py | 3 +- morphlib/cachekeycomputer.py | 5 ++ morphlib/definitions_version.py | 2 +- morphlib/git.py | 47 ++++----------- morphlib/gitdir.py | 22 +++++++ morphlib/gitdir_tests.py | 6 ++ morphlib/morphloader.py | 34 +++++++++++ morphlib/morphloader_tests.py | 59 +++++++++++++++++++ morphlib/plugins/build_plugin.py | 1 + morphlib/plugins/cross-bootstrap_plugin.py | 2 +- morphlib/source.py | 3 +- morphlib/sourceresolver.py | 95 ++++++++++++++++++++++++------ yarns/building.yarn | 18 ++++++ yarns/implementations.yarn | 75 +++++++++++++++++++++++ 17 files changed, 390 insertions(+), 70 deletions(-) diff --git a/morphlib/artifactresolver.py b/morphlib/artifactresolver.py index f3936df1..2b40752d 100644 --- a/morphlib/artifactresolver.py +++ b/morphlib/artifactresolver.py @@ -189,6 +189,7 @@ class ArtifactResolver(object): # Resolve now to avoid a search for the parent morphology later chunk_source.build_mode = info['build-mode'] chunk_source.prefix = info['prefix'] + chunk_source.extra_sources = info['extra-sources'] # Add these chunks to the processed artifacts, so other # chunks may refer to them. diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py index e185a808..6edd8ced 100644 --- a/morphlib/buildcommand.py +++ b/morphlib/buildcommand.py @@ -95,6 +95,7 @@ class BuildCommand(object): self.repo_cache, repo_name, ref, filenames, original_ref=original_ref, status_cb=self.app.status) + self.source_pool = srcpool return srcpool def validate_sources(self, srcpool): @@ -388,10 +389,28 @@ class BuildCommand(object): def fetch_sources(self, source): '''Update the local git repository cache with the sources.''' + def fetch_extra_sources(repo_cache, parent_repo, parent_ref, + extra_sources): + + for extra_source in extra_sources: + ref = extra_source.get('ref') + if not ref: + ref = parent_repo.get_submodule_commit( + parent_ref, extra_source['path']) + repo = repo_cache.get_updated_repo(extra_source['repo'], + ref=ref) + fetch_extra_sources(repo_cache, repo, ref, + extra_source.get('extra-sources', [])) + repo_name = source.repo_name source.repo = self.repo_cache.get_updated_repo(repo_name, ref=source.sha1) - self.repo_cache.ensure_submodules(source.repo, source.sha1) + if source.morphology['kind'] == 'chunk': + if self.source_pool.definitions_version >= 8: + fetch_extra_sources(self.repo_cache, source.repo, source.sha1, + source.extra_sources) + else: + self.repo_cache.ensure_submodules(source.repo, source.sha1) def cache_artifacts_locally(self, artifacts): '''Get artifacts missing from local cache from remote cache.''' @@ -537,7 +556,8 @@ class BuildCommand(object): name=source.name, sha1=source.sha1[:7]) builder = morphlib.builder.Builder( self.app, staging_area, self.lac, self.rac, self.repo_cache, - self.app.settings['max-jobs'], setup_mounts) + self.app.settings['max-jobs'], setup_mounts, + self.source_pool.definitions_version) return builder.build_and_cache(source) class InitiatorBuildCommand(BuildCommand): diff --git a/morphlib/builder.py b/morphlib/builder.py index c980a276..e2abee9e 100644 --- a/morphlib/builder.py +++ b/morphlib/builder.py @@ -35,7 +35,8 @@ import morphlib.gitversion SYSTEM_INTEGRATION_PATH = os.path.join('baserock', 'system-integration') -def extract_sources(app, repo_cache, repo, sha1, srcdir): #pragma: no cover +def extract_sources(app, definitions_version, repo_cache, repo, sha1, + destdir, source): #pragma: no cover '''Get sources from git to a source directory, including submodules''' def extract_repo(repo, sha1, destdir): @@ -57,12 +58,48 @@ def extract_sources(app, repo_cache, repo, sha1, srcdir): #pragma: no cover sub_dir = os.path.join(destdir, sub.path) tuples.append((cached_repo, sub.commit, sub_dir)) return tuples + return [] - todo = [(repo, sha1, srcdir)] - while todo: - repo, sha1, srcdir = todo.pop() - todo += extract_repo(repo, sha1, srcdir) - set_mtime_recursively(srcdir) + def extract_repo_version_8(repo, ref, extra_sources, + rootdir, destdir): + + app.status(msg='Extracting %(source)s into %(path)s', + source=repo.original_name, + path=destdir) + repo.checkout(ref, destdir) + morphlib.git.reset_workdir(app.runcmd, destdir) + + for extra_source in extra_sources: + subrepo = repo_cache.get_repo(extra_source['repo']) + path = os.path.normpath(extra_source['path']) + checkout_dir = os.path.join(destdir, path) + if os.path.exists(checkout_dir): + if os.listdir(checkout_dir): + raise cliapp.AppException( + "Failed to clone '%s': the directory '%s' " + "is not empty" % + (subrepo.original_name, os.path.join( + rootdir, path))) + else: + os.makedirs(checkout_dir) + subref = extra_source.get('ref') + if not subref: + subref = repo.get_submodule_commit(ref, path) + + extract_repo_version_8(subrepo, subref, + extra_source.get('extra-sources', []), + rootdir, + os.path.join(destdir, checkout_dir)) + if definitions_version >= 8: + extract_repo_version_8(repo, sha1, source.extra_sources, + os.path.basename(destdir), destdir) + else: + todo = [(repo, sha1, destdir)] + while todo: + repo, sha1, destdir = todo.pop() + todo += extract_repo(repo, sha1, destdir) + + set_mtime_recursively(destdir) def set_mtime_recursively(root): # pragma: no cover '''Set the mtime for every file in a directory tree to the same. @@ -147,7 +184,7 @@ class BuilderBase(object): def __init__(self, app, staging_area, local_artifact_cache, remote_artifact_cache, source, repo_cache, max_jobs, - setup_mounts): + setup_mounts, definitions_version): self.app = app self.staging_area = staging_area self.local_artifact_cache = local_artifact_cache @@ -157,6 +194,7 @@ class BuilderBase(object): self.max_jobs = max_jobs self.build_watch = morphlib.stopwatch.Stopwatch() self.setup_mounts = setup_mounts + self.definitions_version = definitions_version def save_build_times(self): '''Write the times captured by the stopwatch''' @@ -374,7 +412,6 @@ class ChunkBuilder(BuilderBase): stderr=subprocess.STDOUT, logfile=logfilepath, ccache_dir=ccache_dir) - if stdout: stdout.flush() @@ -490,7 +527,8 @@ class ChunkBuilder(BuilderBase): def get_sources(self, srcdir): # pragma: no cover s = self.source - extract_sources(self.app, self.repo_cache, s.repo, s.sha1, srcdir) + extract_sources(self.app, self.definitions_version, self.repo_cache, + s.repo, s.sha1, srcdir, s) class StratumBuilder(BuilderBase): @@ -725,7 +763,8 @@ class Builder(object): # pragma: no cover } def __init__(self, app, staging_area, local_artifact_cache, - remote_artifact_cache, repo_cache, max_jobs, setup_mounts): + remote_artifact_cache, repo_cache, max_jobs, setup_mounts, + definitions_version): self.app = app self.staging_area = staging_area self.local_artifact_cache = local_artifact_cache @@ -733,6 +772,7 @@ class Builder(object): # pragma: no cover self.repo_cache = repo_cache self.max_jobs = max_jobs self.setup_mounts = setup_mounts + self.definitions_version = definitions_version def build_and_cache(self, source): kind = source.morphology['kind'] @@ -740,7 +780,8 @@ class Builder(object): # pragma: no cover self.local_artifact_cache, self.remote_artifact_cache, source, self.repo_cache, self.max_jobs, - self.setup_mounts) + self.setup_mounts, + self.definitions_version) self.app.status(msg='Builder.build: artifact %s with %s' % (source.name, repr(o)), chatty=True) diff --git a/morphlib/builder_tests.py b/morphlib/builder_tests.py index 54bc4a8f..b70cbdd7 100644 --- a/morphlib/builder_tests.py +++ b/morphlib/builder_tests.py @@ -156,7 +156,8 @@ class BuilderBaseTests(unittest.TestCase): self.artifact, self.repo_cache, self.max_jobs, - False) + False, + 7) def test_runs_desired_command(self): self.builder.runcmd(['foo', 'bar']) diff --git a/morphlib/cachekeycomputer.py b/morphlib/cachekeycomputer.py index 22f42aa0..ff0d595a 100644 --- a/morphlib/cachekeycomputer.py +++ b/morphlib/cachekeycomputer.py @@ -96,6 +96,11 @@ class CacheKeyComputer(object): keys['split-rules'] = [(a, [rgx.pattern for rgx in r._regexes]) for (a, r) in source.split_rules] + # Avoid adding the 'extra-trees' key if there were no + # extra-sources, to avoid unnecessary rebuilds + if source.subtrees: # pragma no cover + keys['extra-trees'] = source.subtrees + # Include morphology contents, since it doesn't always come # from the source tree keys['devices'] = morphology.get('devices') diff --git a/morphlib/definitions_version.py b/morphlib/definitions_version.py index b531a021..7261475c 100644 --- a/morphlib/definitions_version.py +++ b/morphlib/definitions_version.py @@ -24,7 +24,7 @@ import yaml import morphlib -SUPPORTED_VERSIONS = [7] +SUPPORTED_VERSIONS = [7, 8] class DefinitionsVersionError(cliapp.AppException): diff --git a/morphlib/git.py b/morphlib/git.py index cab551ef..12c24b0e 100644 --- a/morphlib/git.py +++ b/morphlib/git.py @@ -34,11 +34,17 @@ class NoModulesFileError(cliapp.AppException): class Submodule(object): - def __init__(self, name, url, path): + def __init__(self, name, url, sha1, path): self.name = name self.url = url + self.commit = sha1 self.path = path + def __str__(self): + return "{name}|{url}|{path}".format(name=self.name, + url=self.url, + path=self.path) + class InvalidSectionError(cliapp.AppException): @@ -48,14 +54,6 @@ class InvalidSectionError(cliapp.AppException): 'title: [%s]' % (repo, ref, section)) -class MissingSubmoduleCommitError(cliapp.AppException): - - def __init__(self, repo, ref, submodule): - Exception.__init__(self, - '%s:%s:.gitmodules: No commit object found for ' - 'submodule "%s"' % (repo, ref, submodule)) - - class Submodules(object): def __init__(self, repo, ref, runcmd_cb=cliapp.runcmd): @@ -87,6 +85,7 @@ class Submodules(object): raise NoModulesFileError(self.repo, self.ref) def _validate_and_read_entries(self, parser): + gd = morphlib.gitdir.GitDirectory(self.repo) for section in parser.sections(): # validate section name against the 'section "foo"' pattern section_pattern = r'submodule "(.*)"' @@ -97,33 +96,9 @@ class Submodules(object): path = parser.get(section, 'path') # create a submodule object - submodule = Submodule(name, url, path) - try: - # list objects in the parent repo tree to find the commit - # object that corresponds to the submodule - commit = gitcmd(self.runcmd_cb, 'ls-tree', self.ref, - submodule.path, cwd=self.repo) - - # read the commit hash from the output - fields = commit.split() - if len(fields) >= 2 and fields[1] == 'commit': - submodule.commit = commit.split()[2] - - # fail if the commit hash is invalid - if len(submodule.commit) != 40: - raise MissingSubmoduleCommitError(self.repo, - self.ref, - submodule.name) - - # add a submodule object to the list - self.submodules.append(submodule) - else: - logging.warning('Skipping submodule "%s" as %s:%s has ' - 'a non-commit object for it' % - (submodule.name, self.repo, self.ref)) - except cliapp.AppException: - raise MissingSubmoduleCommitError(self.repo, self.ref, - submodule.name) + sha1 = gd.get_submodule_commit(self.ref, path) + submodule = Submodule(name, url, sha1, path) + self.submodules.append(submodule) else: raise InvalidSectionError(self.repo, self.ref, section) diff --git a/morphlib/gitdir.py b/morphlib/gitdir.py index ca4a4c76..24dd9ed7 100644 --- a/morphlib/gitdir.py +++ b/morphlib/gitdir.py @@ -54,6 +54,14 @@ class ExpectedSha1Error(cliapp.AppException): self, 'SHA1 expected, got %s' % ref) +class MissingSubmoduleCommitError(cliapp.AppException): + + def __init__(self, repo, ref, submodule): + cliapp.AppException.__init__(self, # pragma + '%s:%s:.gitmodules: No commit object found for ' + 'submodule "%s"' % (repo, ref, submodule)) + + class RefChangeError(cliapp.AppException): pass @@ -822,6 +830,20 @@ class GitDirectory(object): except Exception as e: raise RefDeleteError(self, ref, old_sha1, e) + def get_submodule_commit(self, parent_ref, + submodule_path): # pragma: no cover + try: + lstree_output = morphlib.git.gitcmd(self._runcmd, 'ls-tree', + parent_ref, submodule_path) + except cliapp.AppException: + raise MissingSubmoduleCommitError(self.dirname, parent_ref, + submodule_path) + m = re.match("160000 commit (?P\w{40})", lstree_output) + if not m: + raise MissingSubmoduleCommitError(self.dirname, parent_ref, + submodule_path) + return m.group('sha1') + def describe(self): version = morphlib.git.gitcmd(self._runcmd, 'describe', '--always', '--dirty=-unreproducible') diff --git a/morphlib/gitdir_tests.py b/morphlib/gitdir_tests.py index 4da98bbc..d74b4a13 100644 --- a/morphlib/gitdir_tests.py +++ b/morphlib/gitdir_tests.py @@ -104,6 +104,12 @@ class GitDirectoryTests(unittest.TestCase): gitdir = self.empty_git_directory() self.assertIsInstance(gitdir.get_index(), morphlib.gitindex.GitIndex) + def test_non_existent_submodule_path_raises_error(self): + gitdir = self.empty_git_directory() + self.assertRaises( + morphlib.gitdir.MissingSubmoduleCommitError, + gitdir.get_submodule_commit, 'master', 'somepath') + class GitDirectoryAnchoredRefTests(unittest.TestCase): diff --git a/morphlib/morphloader.py b/morphlib/morphloader.py index 66f4763a..0160e419 100644 --- a/morphlib/morphloader.py +++ b/morphlib/morphloader.py @@ -15,6 +15,7 @@ # =*= License: GPL-2 =*= +import os import collections import warnings import yaml @@ -79,6 +80,13 @@ class InvalidTypeError(MorphologyValidationError): (field, expected, actual, morphology_name)) +class InvalidPathError(MorphologyValidationError): + + def __init__(self, path, spec, morph_filename): + self.msg = ("Invalid path '%s' in %s from morphology %s" + % (path, spec, morph_filename)) + + class UnknownArchitectureError(MorphologyValidationError): def __init__(self, arch, morph_filename): @@ -246,6 +254,7 @@ class MorphologyDumper(yaml.SafeDumper): 'build-mode', 'artifacts', 'max-jobs', + 'extra-sources', 'products', 'chunks', 'build-system', @@ -357,6 +366,7 @@ class MorphologyLoader(object): 'strip-commands': None, 'post-strip-commands': None, 'devices': [], + 'extra-sources': [], 'products': [], 'max-jobs': None, 'build-system': 'manual', @@ -561,6 +571,27 @@ class MorphologyLoader(object): raise ChunkSpecNoBuildInstructionsError( chunk_name, morph.filename) + def validate_extra_sources(extra_sources, morph_filename): + for extra_source in extra_sources: + validate_chunk_str_field('repo', extra_source, + morph.filename) + path = extra_source.get('path') + if not path: + raise MissingFieldError("'path' in %s" % extra_source, + morph.filename) + path = os.path.normpath(path) + if os.path.isabs(path) or path.startswith('..') or ( + path == '.') or not path.strip(): + raise InvalidPathError(extra_source['path'], + extra_source, + morph_filename) + if 'extra-sources' in extra_source: + validate_extra_sources(extra_source['extra-sources'], + morph.filename) + + if 'extra-sources' in spec: + validate_extra_sources(spec['extra-sources'], morph.filename) + @classmethod def _validate_chunk(cls, morphology): errors = [] @@ -704,6 +735,9 @@ class MorphologyLoader(object): if 'prefix' not in spec: spec['prefix'] = \ self._static_defaults['chunk']['prefix'] + if 'extra-sources' not in spec: + spec['extra-sources'] = \ + self._static_defaults['chunk']['extra-sources'] def _set_chunk_defaults(self, morph): if morph['max-jobs'] is not None: diff --git a/morphlib/morphloader_tests.py b/morphlib/morphloader_tests.py index db22264f..0bf69612 100644 --- a/morphlib/morphloader_tests.py +++ b/morphlib/morphloader_tests.py @@ -244,6 +244,63 @@ build-system: manual morphlib.morphloader.EmptyRefError): self.loader.validate(m) + def test_fails_to_validate_stratum_with_a_missing_path(self): + m = morphlib.morphology.Morphology({ + 'kind': 'stratum', + 'name': 'foo', + 'build-depends': [], + 'chunks': [ + { + 'name': 'chunk', + 'repo': 'test:repo', + 'ref': 'master', + 'build-system': 'manual', + 'build-depends': [], + 'extra-sources': + [ + { + 'repo': 'foo', + 'path': 'somepath', + 'extra-sources': + [ + { + 'repo': 'bar', + 'ref': 'master' + } + ] + } + ] + } + ] + }) + self.assertRaises( + morphlib.morphloader.MissingFieldError, self.loader.validate, m) + + def test_fails_to_validate_stratum_with_invalid_path(self): + m = morphlib.morphology.Morphology({ + 'kind': 'stratum', + 'name': 'foo', + 'build-depends': [], + 'chunks': [ + { + 'name': 'chunk', + 'repo': 'test:repo', + 'ref': 'master', + 'build-system': 'manual', + 'build-depends': [], + 'extra-sources': + [ + { + 'repo': 'foo', + 'path': '../foo' + } + ] + } + ] + }) + self.assertRaises( + morphlib.morphloader.InvalidPathError, self.loader.validate, m) + def test_fails_to_validate_stratum_which_build_depends_on_self(self): text = '''\ name: bad-stratum @@ -567,6 +624,7 @@ build-system: manual 'pre-strip-commands': None, 'post-strip-commands': None, + 'extra-sources': [], 'products': [], 'system-integration': [], 'devices': [], @@ -606,6 +664,7 @@ build-system: manual "morph": "bar", 'build-mode': 'bootstrap', 'build-depends': [], + 'extra-sources': [], 'prefix': '/usr', }, ], diff --git a/morphlib/plugins/build_plugin.py b/morphlib/plugins/build_plugin.py index 0aab2d51..84d054d4 100644 --- a/morphlib/plugins/build_plugin.py +++ b/morphlib/plugins/build_plugin.py @@ -226,6 +226,7 @@ class BuildPlugin(cliapp.Plugin): ''' bc = morphlib.buildcommand.BuildCommand(self.app) bc.validate_sources(source_pool) + bc.source_pool = source_pool root = bc.resolve_artifacts(source_pool) if not component_names: component_names = [root.source.name] diff --git a/morphlib/plugins/cross-bootstrap_plugin.py b/morphlib/plugins/cross-bootstrap_plugin.py index c6ab8017..981f6ed6 100644 --- a/morphlib/plugins/cross-bootstrap_plugin.py +++ b/morphlib/plugins/cross-bootstrap_plugin.py @@ -302,7 +302,7 @@ class CrossBootstrapPlugin(cliapp.Plugin): system_artifact.source, build_env, use_chroot=False) builder = BootstrapSystemBuilder( self.app, staging_area, build_command.lac, build_command.rac, - system_artifact.source, build_command.repo_cache, 1, False) + system_artifact.source, build_command.repo_cache, 1, False, 8) builder.build_and_cache() self.app.status( diff --git a/morphlib/source.py b/morphlib/source.py index 135c14cc..e7486b56 100644 --- a/morphlib/source.py +++ b/morphlib/source.py @@ -78,7 +78,7 @@ class Source(object): def make_sources(reponame, ref, filename, absref, tree, morphology, - default_split_rules={}): + default_split_rules={}, subtrees=[]): kind = morphology['kind'] if kind in ('system', 'chunk'): unifier = getattr(morphlib.artifactsplitrule, @@ -92,6 +92,7 @@ def make_sources(reponame, ref, filename, absref, tree, morphology, filename, split_rules) source.artifacts = {name: morphlib.artifact.Artifact(source, name) for name in split_rules.artifacts} + source.subtrees = subtrees yield source elif kind == 'stratum': # pragma: no cover unifier = morphlib.artifactsplitrule.unify_stratum_matches diff --git a/morphlib/sourceresolver.py b/morphlib/sourceresolver.py index 5d04ece9..a56b6594 100644 --- a/morphlib/sourceresolver.py +++ b/morphlib/sourceresolver.py @@ -153,7 +153,7 @@ class SourceResolver(object): The third layer of caching is a simple commit SHA1 -> tree SHA mapping. It turns out that even if all repos are available locally, running 'git rev-parse' on hundreds of repos requires a lot of IO and can take - several minutes. Likewise, on a slow network connection it is time + several /minutes. Likewise, on a slow network connection it is time consuming to keep querying the remote repo cache. This third layer of caching works around both of those issues. @@ -259,8 +259,59 @@ class SourceResolver(object): return morph + def _resolve_subtrees(self, parent_reponame, parent_ref, extra_sources, + resolved_trees): + + subtrees = [] + + def resolve_extra_refs(reponame, ref, extra_sources, base_path): + for extra_source in extra_sources: + if (extra_source['repo'], extra_source['path']) in ( + resolved_trees): + tree = resolved_trees[(extra_source['repo'], + extra_source['path'])] + else: + subref = extra_source.get('ref') + if not subref: + if self.lrc.has_repo(reponame): + repo = self.lrc.get_repo(reponame) + if self.update and ( + repo.requires_update_for_ref(ref)): + self.status(msg='Updating cached git ' + 'repository %(reponame)s for ' + 'ref %(ref)s', + reponame=reponame, ref=ref) + repo.update() + else: + # TODO Add support to the cache server for + # retrieving the submodule commit from a path + self.status(msg='Updating cached git repository ' + '%(reponame)s for ref %(ref)s', + reponame=reponame, ref=ref) + repo = self.lrc.get_updated_repo(reponame, ref) + subref = repo.get_submodule_commit( + ref, extra_source['path']) + _, tree = self._resolve_ref(resolved_trees, + extra_source['repo'], subref) + resolved_trees[(extra_source['repo'], + extra_source['path'])] = tree + path = os.path.normpath(os.path.join(base_path, + extra_source['path'])) + subtrees.append({'path': path, 'tree': tree}) + if 'extra-sources' in extra_source: + resolve_extra_refs(extra_source['repo'], + extra_source['ref'], + extra_source['extra-sources'], + path) + + if extra_sources: + resolve_extra_refs(parent_reponame, parent_ref, + extra_sources, '.') + return subtrees + def _process_definitions_with_children(self, resolved_morphologies, + resolved_trees, definitions_checkout_dir, definitions_repo, definitions_ref, @@ -274,7 +325,7 @@ class SourceResolver(object): # We don't need system's filename, so use 'None' definitions_queue = collections.deque((None, f) for f in system_filenames) - chunk_queue = set() + chunk_queue = [] def get_morphology(filename): return self._get_morphology(resolved_morphologies, @@ -291,7 +342,7 @@ class SourceResolver(object): visit(definitions_repo, definitions_ref, filename, definitions_absref, definitions_tree, morphology, - predefined_split_rules) + predefined_split_rules, []) if morphology['kind'] == 'cluster': raise cliapp.AppException( @@ -313,6 +364,9 @@ class SourceResolver(object): sanitise_morphology_path(s['morph'])) for s in morphology['build-depends']) for c in morphology['chunks']: + extra_sources = c.get('extra-sources') + subtrees = self._resolve_subtrees(c['repo'], c['ref'], + extra_sources, resolved_trees) if 'morph' in c: # Now, does this path actually exist? path = c['morph'] @@ -322,15 +376,16 @@ class SourceResolver(object): raise MorphologyReferenceNotFoundError( path, filename) - chunk_queue.add((c['name'], c['repo'], c['ref'], - path, None)) + chunk_queue.append((c['name'], c['repo'], c['ref'], + path, None, subtrees)) else: # We invent a filename here, so that the rest of the # Morph code doesn't need to know about the predefined # build instructions. chunk_filename = c['name'] + '.morph' - chunk_queue.add((c["name"], c['repo'], c['ref'], - chunk_filename, c['build-system'])) + chunk_queue.append((c['name'], c['repo'], c['ref'], + chunk_filename, c['build-system'], + subtrees)) return chunk_queue @@ -345,7 +400,8 @@ class SourceResolver(object): def process_chunk(self, resolved_morphologies, resolved_trees, definitions_checkout_dir, morph_loader, chunk_name, chunk_repo, chunk_ref, filename, chunk_buildsystem, - visit, predefined_build_systems, predefined_split_rules): + subtrees, visit, predefined_build_systems, + predefined_split_rules): absref, tree = self._resolve_ref(resolved_trees, chunk_repo, chunk_ref) if chunk_buildsystem is None: @@ -373,10 +429,10 @@ class SourceResolver(object): morph_loader, buildsystem, chunk_name) visit(chunk_repo, chunk_ref, filename, absref, tree, morphology, - predefined_split_rules) + predefined_split_rules, subtrees) def traverse_morphs(self, definitions_repo, definitions_ref, - system_filenames, + system_filenames, pool, visit=lambda rn, rf, fn, arf, m: None, definitions_original_ref=None): @@ -403,6 +459,7 @@ class SourceResolver(object): definitions_version = self._check_version_file( definitions_checkout_dir) + pool.definitions_version = definitions_version predefined_build_systems, predefined_split_rules = \ self._get_defaults( @@ -415,19 +472,23 @@ class SourceResolver(object): # will all live in the same Git repository, and will point to # various chunk morphologies. chunk_queue = self._process_definitions_with_children( - resolved_morphologies, definitions_checkout_dir, - definitions_repo, definitions_ref, definitions_absref, + resolved_morphologies, resolved_trees, + definitions_checkout_dir, definitions_repo, + definitions_ref, definitions_absref, definitions_tree, morph_loader, system_filenames, visit, predefined_split_rules) # Now process all the chunks involved in the build. - for name, repo, ref, filename, buildsystem in chunk_queue: + for name, repo, ref, filename, buildsystem, extra_sources in ( + chunk_queue): self.process_chunk(resolved_morphologies, resolved_trees, definitions_checkout_dir, morph_loader, name, repo, ref, filename, buildsystem, - visit, predefined_build_systems, + extra_sources, visit, + predefined_build_systems, predefined_split_rules) + class DuplicateChunkError(morphlib.Error): def _make_msg(self, (name, sources)): # pragma: no cover @@ -478,7 +539,7 @@ def create_source_pool(repo_cache, repo, ref, filenames, pool = morphlib.sourcepool.SourcePool() def add_to_pool(reponame, ref, filename, absref, tree, morphology, - predefined_split_rules): + predefined_split_rules, subtrees): # If there are duplicate chunks which have the same 'name' and the # same build instructions, we might cause a stack overflow in # cachekeycomputer.py when trying to hash the build graph. The @@ -493,7 +554,7 @@ def create_source_pool(repo_cache, repo, ref, filenames, sources = morphlib.source.make_sources( reponame, ref, filename, absref, tree, morphology, - predefined_split_rules) + predefined_split_rules, subtrees) for source in sources: pool.add(source) @@ -503,7 +564,7 @@ def create_source_pool(repo_cache, repo, ref, filenames, resolver = SourceResolver(repo_cache, tree_cache_manager, status_cb) resolver.traverse_morphs(repo, ref, filenames, - visit=add_to_pool, + pool, visit=add_to_pool, definitions_original_ref=original_ref) # No two chunks may have the same name diff --git a/yarns/building.yarn b/yarns/building.yarn index 9284f7cf..3aea709f 100644 --- a/yarns/building.yarn +++ b/yarns/building.yarn @@ -89,6 +89,24 @@ Morph Building Tests AND there are 2 artifacts named core-runtime in the cache FINALLY the git server is shut down + SCENARIO test recursive sources for a chunk + GIVEN a workspace + AND a git server + GIVEN a chunk with dependencies + WHEN the user checks out the system branch called master + WHEN the user attempts to build the system systems/test-system.morph in branch master + THEN morph succeeded + FINALLY the git server is shut down + + SCENARIO test recursive sources with non-empty path + GIVEN a workspace + AND a git server + GIVEN a chunk with recursive sources with non-empty paths + WHEN the user checks out the system branch called master + WHEN the user attempts to build the system systems/test-system.morph in branch master + THEN morph failed + FINALLY the git server is shut down + System integrations ------------------- diff --git a/yarns/implementations.yarn b/yarns/implementations.yarn index 06380e7f..8ed5a5e9 100644 --- a/yarns/implementations.yarn +++ b/yarns/implementations.yarn @@ -45,6 +45,81 @@ locally, which we'll tell Morph to access using `file:` URLs. Specifically, we'll create a repository to hold system and stratum morphologies, and another to hold a chunk. + IMPLEMENTS GIVEN a chunk with recursive sources with non-empty paths + cd "$DATADIR/gits/morphs" + echo "version: 8" > VERSION + cat << EOF >> strata/core.morph + - name: invalid-chunk + repo: test:chunk-with-submodules + ref: master + extra-sources: + - repo: file://$DATADIR/gits/test-chunk + path: somepath + ref: master + - repo: file://$DATADIR/gits/child-chunk + path: somepath + EOF + git add . + git commit -m "More stuff" + + + IMPLEMENTS GIVEN a chunk with dependencies + mkdir "$DATADIR/gits/grandchild-chunk" + cd "$DATADIR/gits/grandchild-chunk" + git init . + touch grandchild-file + git add . + git commit -m "Initial commit" + + mkdir "$DATADIR/gits/child-chunk" + cd "$DATADIR/gits/child-chunk" + git init . + touch child-file + git add . + git commit -m "Initial commit" + git submodule add -b master file://$DATADIR/gits/grandchild-chunk + git commit -m "Initial submodule" + git checkout -b new-work + git mv child-file child-file-renamed + git commit -m "Moar work" + + mkdir "$DATADIR/gits/chunk-with-submodules" + cd "$DATADIR/gits/chunk-with-submodules" + git init . + git add . + git commit --allow-empty -m "Initial commit" + git submodule add -b master file://$DATADIR/gits/child-chunk + #( cd child-chunk && git checkout master) + git add . + git commit -m "Add submodule" + + cd "$DATADIR/gits/morphs" + echo "version: 8" > VERSION + cat << EOF >> strata/core.morph + - name: chunk-with-submodules + morph: chunk-with-submodules.morph + repo: test:chunk-with-submodules + ref: master + extra-sources: + - repo: file://$DATADIR/gits/child-chunk + path: child-chunk + ref: new-work + extra-sources: + - repo: file://$DATADIR/gits/grandchild-chunk + path: grandchild-chunk + EOF + + cat << EOF >> chunk-with-submodules.morph + name: chunk-with-submodules + kind: chunk + build-system: manual + build-commands: + - file exists child-chunk/child-file-renamed + - file exists child-chunk/grandchild-chunk/grandchild-file + EOF + git add . + git commit -m "Add moar stuff" + IMPLEMENTS GIVEN a git server # Create a directory for all the git repositories. -- cgit v1.2.1