From 45e39e81698df91251c7fac5a642e211e98d834b Mon Sep 17 00:00:00 2001 From: Pedro Alvarez Date: Sat, 26 Mar 2016 18:59:01 +0000 Subject: Add support for definitions version 8 This code is a rework from changes done by: - Tiago Gomes https://storyboard.baserock.org/#!/story/86 Change-Id: I3475c2bcb648a272fee33bc878a521f79d4e6581 --- morphlib/artifactresolver.py | 3 +- morphlib/buildcommand.py | 11 +++- morphlib/builder.py | 42 +++++++++---- morphlib/builder_tests.py | 3 +- morphlib/definitions_version.py | 4 +- morphlib/git.py | 47 ++++---------- morphlib/gitdir.py | 22 +++++++ morphlib/gitdir_tests.py | 6 ++ morphlib/morphloader.py | 76 ++++++++++++++--------- morphlib/morphloader_tests.py | 98 +++++++++++++++++++++++++++--- morphlib/plugins/build_plugin.py | 3 +- morphlib/plugins/cross-bootstrap_plugin.py | 8 ++- morphlib/repocache.py | 9 ++- morphlib/source.py | 2 +- morphlib/sourceresolver.py | 8 ++- yarns/building.yarn | 8 +++ yarns/implementations.yarn | 56 ++++++++++++++++- 17 files changed, 302 insertions(+), 104 deletions(-) diff --git a/morphlib/artifactresolver.py b/morphlib/artifactresolver.py index f3936df1..c1418924 100644 --- a/morphlib/artifactresolver.py +++ b/morphlib/artifactresolver.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2015 Codethink Limited +# Copyright (C) 2012-2016 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -189,6 +189,7 @@ class ArtifactResolver(object): # Resolve now to avoid a search for the parent morphology later chunk_source.build_mode = info['build-mode'] chunk_source.prefix = info['prefix'] + chunk_source.submodules = info['submodules'] # Add these chunks to the processed artifacts, so other # chunks may refer to them. diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py index e185a808..346fdc8a 100644 --- a/morphlib/buildcommand.py +++ b/morphlib/buildcommand.py @@ -95,6 +95,7 @@ class BuildCommand(object): self.repo_cache, repo_name, ref, filenames, original_ref=original_ref, status_cb=self.app.status) + self.source_pool = srcpool return srcpool def validate_sources(self, srcpool): @@ -391,7 +392,12 @@ class BuildCommand(object): repo_name = source.repo_name source.repo = self.repo_cache.get_updated_repo(repo_name, ref=source.sha1) - self.repo_cache.ensure_submodules(source.repo, source.sha1) + if source.morphology['kind'] == 'chunk': + if self.source_pool.definitions_version >= 8: + self.repo_cache.ensure_submodules( + source.repo, source.sha1, source.submodules) + else: + self.repo_cache.ensure_submodules(source.repo, source.sha1) def cache_artifacts_locally(self, artifacts): '''Get artifacts missing from local cache from remote cache.''' @@ -537,7 +543,8 @@ class BuildCommand(object): name=source.name, sha1=source.sha1[:7]) builder = morphlib.builder.Builder( self.app, staging_area, self.lac, self.rac, self.repo_cache, - self.app.settings['max-jobs'], setup_mounts) + self.app.settings['max-jobs'], setup_mounts, + self.source_pool.definitions_version) return builder.build_and_cache(source) class InitiatorBuildCommand(BuildCommand): diff --git a/morphlib/builder.py b/morphlib/builder.py index c980a276..166b790b 100644 --- a/morphlib/builder.py +++ b/morphlib/builder.py @@ -35,10 +35,11 @@ import morphlib.gitversion SYSTEM_INTEGRATION_PATH = os.path.join('baserock', 'system-integration') -def extract_sources(app, repo_cache, repo, sha1, srcdir): #pragma: no cover +def extract_sources(app, definitions_version, repo_cache, repo, sha1, + destdir, source): #pragma: no cover '''Get sources from git to a source directory, including submodules''' - def extract_repo(repo, sha1, destdir): + def extract_repo(repo, sha1, destdir, submodules_map=None): app.status(msg='Extracting %(source)s into %(target)s', source=repo.original_name, target=destdir) @@ -53,16 +54,27 @@ def extract_sources(app, repo_cache, repo, sha1, srcdir): #pragma: no cover else: tuples = [] for sub in submodules: - cached_repo = repo_cache.get_updated_repo(sub.url, sub.commit) + if submodules_map and sub.name in submodules_map: + url = submodules_map[sub.name]['url'] + else: + url = sub.url + cached_repo = repo_cache.get_updated_repo(url, sub.commit) sub_dir = os.path.join(destdir, sub.path) tuples.append((cached_repo, sub.commit, sub_dir)) return tuples - todo = [(repo, sha1, srcdir)] - while todo: - repo, sha1, srcdir = todo.pop() - todo += extract_repo(repo, sha1, srcdir) - set_mtime_recursively(srcdir) + if definitions_version >= 8: + todo = [(repo, sha1, destdir)] + while todo: + repo, sha1, destdir = todo.pop() + todo += extract_repo(repo, sha1, destdir, source.submodules) + else: + todo = [(repo, sha1, destdir)] + while todo: + repo, sha1, destdir = todo.pop() + todo += extract_repo(repo, sha1, destdir) + + set_mtime_recursively(destdir) def set_mtime_recursively(root): # pragma: no cover '''Set the mtime for every file in a directory tree to the same. @@ -147,7 +159,7 @@ class BuilderBase(object): def __init__(self, app, staging_area, local_artifact_cache, remote_artifact_cache, source, repo_cache, max_jobs, - setup_mounts): + setup_mounts, definitions_version): self.app = app self.staging_area = staging_area self.local_artifact_cache = local_artifact_cache @@ -157,6 +169,7 @@ class BuilderBase(object): self.max_jobs = max_jobs self.build_watch = morphlib.stopwatch.Stopwatch() self.setup_mounts = setup_mounts + self.definitions_version = definitions_version def save_build_times(self): '''Write the times captured by the stopwatch''' @@ -374,7 +387,6 @@ class ChunkBuilder(BuilderBase): stderr=subprocess.STDOUT, logfile=logfilepath, ccache_dir=ccache_dir) - if stdout: stdout.flush() @@ -490,7 +502,8 @@ class ChunkBuilder(BuilderBase): def get_sources(self, srcdir): # pragma: no cover s = self.source - extract_sources(self.app, self.repo_cache, s.repo, s.sha1, srcdir) + extract_sources(self.app, self.definitions_version, self.repo_cache, + s.repo, s.sha1, srcdir, s) class StratumBuilder(BuilderBase): @@ -725,7 +738,8 @@ class Builder(object): # pragma: no cover } def __init__(self, app, staging_area, local_artifact_cache, - remote_artifact_cache, repo_cache, max_jobs, setup_mounts): + remote_artifact_cache, repo_cache, max_jobs, setup_mounts, + definitions_version): self.app = app self.staging_area = staging_area self.local_artifact_cache = local_artifact_cache @@ -733,6 +747,7 @@ class Builder(object): # pragma: no cover self.repo_cache = repo_cache self.max_jobs = max_jobs self.setup_mounts = setup_mounts + self.definitions_version = definitions_version def build_and_cache(self, source): kind = source.morphology['kind'] @@ -740,7 +755,8 @@ class Builder(object): # pragma: no cover self.local_artifact_cache, self.remote_artifact_cache, source, self.repo_cache, self.max_jobs, - self.setup_mounts) + self.setup_mounts, + self.definitions_version) self.app.status(msg='Builder.build: artifact %s with %s' % (source.name, repr(o)), chatty=True) diff --git a/morphlib/builder_tests.py b/morphlib/builder_tests.py index 54bc4a8f..6d88a472 100644 --- a/morphlib/builder_tests.py +++ b/morphlib/builder_tests.py @@ -156,7 +156,8 @@ class BuilderBaseTests(unittest.TestCase): self.artifact, self.repo_cache, self.max_jobs, - False) + False, + 8) def test_runs_desired_command(self): self.builder.runcmd(['foo', 'bar']) diff --git a/morphlib/definitions_version.py b/morphlib/definitions_version.py index b531a021..e887ddd8 100644 --- a/morphlib/definitions_version.py +++ b/morphlib/definitions_version.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015 Codethink Limited +# Copyright (C) 2015-2016 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -24,7 +24,7 @@ import yaml import morphlib -SUPPORTED_VERSIONS = [7] +SUPPORTED_VERSIONS = [7, 8] class DefinitionsVersionError(cliapp.AppException): diff --git a/morphlib/git.py b/morphlib/git.py index cab551ef..12c24b0e 100644 --- a/morphlib/git.py +++ b/morphlib/git.py @@ -34,11 +34,17 @@ class NoModulesFileError(cliapp.AppException): class Submodule(object): - def __init__(self, name, url, path): + def __init__(self, name, url, sha1, path): self.name = name self.url = url + self.commit = sha1 self.path = path + def __str__(self): + return "{name}|{url}|{path}".format(name=self.name, + url=self.url, + path=self.path) + class InvalidSectionError(cliapp.AppException): @@ -48,14 +54,6 @@ class InvalidSectionError(cliapp.AppException): 'title: [%s]' % (repo, ref, section)) -class MissingSubmoduleCommitError(cliapp.AppException): - - def __init__(self, repo, ref, submodule): - Exception.__init__(self, - '%s:%s:.gitmodules: No commit object found for ' - 'submodule "%s"' % (repo, ref, submodule)) - - class Submodules(object): def __init__(self, repo, ref, runcmd_cb=cliapp.runcmd): @@ -87,6 +85,7 @@ class Submodules(object): raise NoModulesFileError(self.repo, self.ref) def _validate_and_read_entries(self, parser): + gd = morphlib.gitdir.GitDirectory(self.repo) for section in parser.sections(): # validate section name against the 'section "foo"' pattern section_pattern = r'submodule "(.*)"' @@ -97,33 +96,9 @@ class Submodules(object): path = parser.get(section, 'path') # create a submodule object - submodule = Submodule(name, url, path) - try: - # list objects in the parent repo tree to find the commit - # object that corresponds to the submodule - commit = gitcmd(self.runcmd_cb, 'ls-tree', self.ref, - submodule.path, cwd=self.repo) - - # read the commit hash from the output - fields = commit.split() - if len(fields) >= 2 and fields[1] == 'commit': - submodule.commit = commit.split()[2] - - # fail if the commit hash is invalid - if len(submodule.commit) != 40: - raise MissingSubmoduleCommitError(self.repo, - self.ref, - submodule.name) - - # add a submodule object to the list - self.submodules.append(submodule) - else: - logging.warning('Skipping submodule "%s" as %s:%s has ' - 'a non-commit object for it' % - (submodule.name, self.repo, self.ref)) - except cliapp.AppException: - raise MissingSubmoduleCommitError(self.repo, self.ref, - submodule.name) + sha1 = gd.get_submodule_commit(self.ref, path) + submodule = Submodule(name, url, sha1, path) + self.submodules.append(submodule) else: raise InvalidSectionError(self.repo, self.ref, section) diff --git a/morphlib/gitdir.py b/morphlib/gitdir.py index ca4a4c76..24dd9ed7 100644 --- a/morphlib/gitdir.py +++ b/morphlib/gitdir.py @@ -54,6 +54,14 @@ class ExpectedSha1Error(cliapp.AppException): self, 'SHA1 expected, got %s' % ref) +class MissingSubmoduleCommitError(cliapp.AppException): + + def __init__(self, repo, ref, submodule): + cliapp.AppException.__init__(self, # pragma + '%s:%s:.gitmodules: No commit object found for ' + 'submodule "%s"' % (repo, ref, submodule)) + + class RefChangeError(cliapp.AppException): pass @@ -822,6 +830,20 @@ class GitDirectory(object): except Exception as e: raise RefDeleteError(self, ref, old_sha1, e) + def get_submodule_commit(self, parent_ref, + submodule_path): # pragma: no cover + try: + lstree_output = morphlib.git.gitcmd(self._runcmd, 'ls-tree', + parent_ref, submodule_path) + except cliapp.AppException: + raise MissingSubmoduleCommitError(self.dirname, parent_ref, + submodule_path) + m = re.match("160000 commit (?P\w{40})", lstree_output) + if not m: + raise MissingSubmoduleCommitError(self.dirname, parent_ref, + submodule_path) + return m.group('sha1') + def describe(self): version = morphlib.git.gitcmd(self._runcmd, 'describe', '--always', '--dirty=-unreproducible') diff --git a/morphlib/gitdir_tests.py b/morphlib/gitdir_tests.py index 4da98bbc..d74b4a13 100644 --- a/morphlib/gitdir_tests.py +++ b/morphlib/gitdir_tests.py @@ -104,6 +104,12 @@ class GitDirectoryTests(unittest.TestCase): gitdir = self.empty_git_directory() self.assertIsInstance(gitdir.get_index(), morphlib.gitindex.GitIndex) + def test_non_existent_submodule_path_raises_error(self): + gitdir = self.empty_git_directory() + self.assertRaises( + morphlib.gitdir.MissingSubmoduleCommitError, + gitdir.get_submodule_commit, 'master', 'somepath') + class GitDirectoryAnchoredRefTests(unittest.TestCase): diff --git a/morphlib/morphloader.py b/morphlib/morphloader.py index 66f4763a..31123e95 100644 --- a/morphlib/morphloader.py +++ b/morphlib/morphloader.py @@ -15,6 +15,7 @@ # =*= License: GPL-2 =*= +import os import collections import warnings import yaml @@ -34,8 +35,10 @@ class MorphologyNotYamlError(MorphologySyntaxError): class NotADictionaryError(MorphologySyntaxError): - def __init__(self, morph_filename): + def __init__(self, morph_filename, errmsg=None): self.msg = 'Not a dictionary: morphology %s' % morph_filename + if errmsg: + self.msg += "\n%s" % (errmsg) class MorphologyValidationError(morphlib.Error): @@ -58,6 +61,17 @@ class MissingFieldError(MorphologyValidationError): 'Missing field %s from morphology %s' % (field, morphology_name)) +class InvalidStringError(MorphologyValidationError): + + def __init__(self, field, spec, morph_filename): + self.field = field + self.spec = spec + self.morph_filename = morph_filename + MorphologyValidationError.__init__( + self, "Field '%(field)s' must be a non-empty string in %(spec)s"\ + " for morphology %(morph_filename)s" % locals()) + + class InvalidFieldError(MorphologyValidationError): def __init__(self, field, morphology_name): @@ -119,27 +133,6 @@ class DuplicateChunkError(MorphologyValidationError): 'in stratum %(stratum_name)s' % locals()) -class EmptyRefError(MorphologyValidationError): - - def __init__(self, ref_location, morph_filename): - self.ref_location = ref_location - self.morph_filename = morph_filename - MorphologyValidationError.__init__( - self, 'Empty ref found for %(ref_location)s '\ - 'in %(morph_filename)s' % locals()) - - -class ChunkSpecRefNotStringError(MorphologyValidationError): - - def __init__(self, ref_value, chunk_name, stratum_name): - self.ref_value = ref_value - self.chunk_name = chunk_name - self.stratum_name = stratum_name - MorphologyValidationError.__init__( - self, 'Ref %(ref_value)s for %(chunk_name)s '\ - 'in stratum %(stratum_name)s is not a string' % locals()) - - class ChunkSpecConflictingFieldsError(MorphologyValidationError): def __init__(self, fields, chunk_name, stratum_name): @@ -246,6 +239,7 @@ class MorphologyDumper(yaml.SafeDumper): 'build-mode', 'artifacts', 'max-jobs', + 'submodules', 'products', 'chunks', 'build-system', @@ -357,6 +351,7 @@ class MorphologyLoader(object): 'strip-commands': None, 'post-strip-commands': None, 'devices': [], + 'submodules': {}, 'products': [], 'max-jobs': None, 'build-system': 'manual', @@ -537,14 +532,20 @@ class MorphologyLoader(object): for spec in morph['chunks']: chunk_name = spec['name'] - # All chunk refs must be strings. - if 'ref' in spec: - ref = spec['ref'] - if ref == None: - raise EmptyRefError(spec['name'], morph.filename) - elif not isinstance(ref, basestring): - raise ChunkSpecRefNotStringError( - ref, spec['name'], morph.filename) + # All chunks repos and refs must be strings + + def validate_chunk_str_field(field, spec, morph_filename): + if field not in spec: + raise MissingFieldError('%s in %s' % (field, spec), + morph.filename) + val = spec[field] + if not val or not isinstance(val, basestring) or ( + not val.strip()): + raise InvalidStringError( + field, spec, morph_filename) + + validate_chunk_str_field('repo', spec, morph.filename) + validate_chunk_str_field('ref', spec, morph.filename) # The build-depends field must be a list. if 'build-depends' in spec: @@ -561,6 +562,18 @@ class MorphologyLoader(object): raise ChunkSpecNoBuildInstructionsError( chunk_name, morph.filename) + def validate_submodules(submodules, morph_filename): + for sub_name in submodules: + validate_chunk_str_field('url', submodules[sub_name], + morph_filename) + + if 'submodules' in spec: + if not isinstance(spec['submodules'], dict): + raise NotADictionaryError( + morph.filename, "The 'submodules' in chunk '%s' have " + "to be a dict" % (chunk_name)) + validate_submodules(spec['submodules'], morph.filename) + @classmethod def _validate_chunk(cls, morphology): errors = [] @@ -704,6 +717,9 @@ class MorphologyLoader(object): if 'prefix' not in spec: spec['prefix'] = \ self._static_defaults['chunk']['prefix'] + if 'submodules' not in spec: + spec['submodules'] = \ + self._static_defaults['chunk']['submodules'] def _set_chunk_defaults(self, morph): if morph['max-jobs'] is not None: diff --git a/morphlib/morphloader_tests.py b/morphlib/morphloader_tests.py index db22264f..b5d3f568 100644 --- a/morphlib/morphloader_tests.py +++ b/morphlib/morphloader_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2015 Codethink Limited +# Copyright (C) 2013-2016 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -208,7 +208,42 @@ build-system: manual self.assertRaises( morphlib.morphloader.InvalidFieldError, self.loader.validate, m) - def test_validate_requires_chunk_refs_in_stratum_to_be_strings(self): + def test_validate_requires_chunk_repos_to_exist(self): + m = morphlib.morphology.Morphology({ + 'kind': 'stratum', + 'name': 'foo', + 'build-depends': [], + 'chunks': [ + { + 'name': 'chunk', + 'ref': 'master', + 'build-depends': [] + } + ] + }) + with self.assertRaises( + morphlib.morphloader.MissingFieldError): + self.loader.validate(m) + + def test_validate_requires_chunk_repos_in_stratum_to_be_strings(self): + m = morphlib.morphology.Morphology({ + 'kind': 'stratum', + 'name': 'foo', + 'build-depends': [], + 'chunks': [ + { + 'name': 'chunk', + 'repo': 1, + 'ref': 'master', + 'build-depends': [] + } + ] + }) + with self.assertRaises( + morphlib.morphloader.InvalidStringError): + self.loader.validate(m) + + def test_validate_requires_chunk_refs_to_exist(self): m = morphlib.morphology.Morphology({ 'kind': 'stratum', 'name': 'foo', @@ -217,33 +252,78 @@ build-system: manual { 'name': 'chunk', 'repo': 'test:repo', - 'ref': 1, 'build-depends': [] } ] }) with self.assertRaises( - morphlib.morphloader.ChunkSpecRefNotStringError): + morphlib.morphloader.MissingFieldError): self.loader.validate(m) - def test_fails_to_validate_stratum_with_empty_refs_for_a_chunk(self): + def test_validate_requires_chunk_refs_in_stratum_to_be_strings(self): m = morphlib.morphology.Morphology({ 'kind': 'stratum', 'name': 'foo', 'build-depends': [], - 'chunks' : [ + 'chunks': [ { 'name': 'chunk', 'repo': 'test:repo', - 'ref': None, + 'ref': 1, 'build-depends': [] } ] }) with self.assertRaises( - morphlib.morphloader.EmptyRefError): + morphlib.morphloader.InvalidStringError): self.loader.validate(m) + def test_fails_to_validate_stratum_with_a_missing_url(self): + m = morphlib.morphology.Morphology({ + 'kind': 'stratum', + 'name': 'foo', + 'build-depends': [], + 'chunks': [ + { + 'name': 'chunk', + 'repo': 'test:repo', + 'ref': 'master', + 'build-system': 'manual', + 'build-depends': [], + 'submodules': + { + 'foolib': {} + } + } + ] + }) + self.assertRaises( + morphlib.morphloader.MissingFieldError, self.loader.validate, m) + + def test_fails_to_validate_stratum_with_no_dict_submodules(self): + m = morphlib.morphology.Morphology({ + 'kind': 'stratum', + 'name': 'foo', + 'build-depends': [], + 'chunks': [ + { + 'name': 'chunk', + 'repo': 'test:repo', + 'ref': 'master', + 'build-system': 'manual', + 'build-depends': [], + 'submodules': + [ + { + 'foolib': {} + } + ] + } + ] + }) + self.assertRaises( + morphlib.morphloader.NotADictionaryError, self.loader.validate, m) + def test_fails_to_validate_stratum_which_build_depends_on_self(self): text = '''\ name: bad-stratum @@ -567,6 +647,7 @@ build-system: manual 'pre-strip-commands': None, 'post-strip-commands': None, + 'submodules': {}, 'products': [], 'system-integration': [], 'devices': [], @@ -606,6 +687,7 @@ build-system: manual "morph": "bar", 'build-mode': 'bootstrap', 'build-depends': [], + 'submodules': {}, 'prefix': '/usr', }, ], diff --git a/morphlib/plugins/build_plugin.py b/morphlib/plugins/build_plugin.py index 0aab2d51..be6d9602 100644 --- a/morphlib/plugins/build_plugin.py +++ b/morphlib/plugins/build_plugin.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2015 Codethink Limited +# Copyright (C) 2012-2016 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -226,6 +226,7 @@ class BuildPlugin(cliapp.Plugin): ''' bc = morphlib.buildcommand.BuildCommand(self.app) bc.validate_sources(source_pool) + bc.source_pool = source_pool root = bc.resolve_artifacts(source_pool) if not component_names: component_names = [root.source.name] diff --git a/morphlib/plugins/cross-bootstrap_plugin.py b/morphlib/plugins/cross-bootstrap_plugin.py index c6ab8017..daed260e 100644 --- a/morphlib/plugins/cross-bootstrap_plugin.py +++ b/morphlib/plugins/cross-bootstrap_plugin.py @@ -104,7 +104,8 @@ class BootstrapSystemBuilder(morphlib.builder.BuilderBase): if not os.path.exists(source_dir): os.makedirs(source_dir) morphlib.builder.extract_sources( - self.app, self.repo_cache, s.repo, s.sha1, source_dir) + self.app, self.definitions_version, self.repo_cache, + s.repo, s.sha1, source_dir, s) name = s.name chunk_script = os.path.join(path, 'src', 'build-%s' % name) @@ -256,6 +257,8 @@ class CrossBootstrapPlugin(cliapp.Plugin): srcpool = build_command.create_source_pool( root_repo, ref, [morph_name]) + definitions_version = srcpool.definitions_version + # FIXME: this is a quick fix in order to get it working for # Baserock 13 release, it is not a reasonable fix def validate(self, root_artifact): @@ -302,7 +305,8 @@ class CrossBootstrapPlugin(cliapp.Plugin): system_artifact.source, build_env, use_chroot=False) builder = BootstrapSystemBuilder( self.app, staging_area, build_command.lac, build_command.rac, - system_artifact.source, build_command.repo_cache, 1, False) + system_artifact.source, build_command.repo_cache, 1, False, + definitions_version) builder.build_and_cache() self.app.status( diff --git a/morphlib/repocache.py b/morphlib/repocache.py index f6978ec4..941d958a 100644 --- a/morphlib/repocache.py +++ b/morphlib/repocache.py @@ -369,7 +369,7 @@ class RepoCache(object): return self._get_repo(repo_name) def ensure_submodules(self, toplevel_repo, - toplevel_ref): # pragma: no cover + toplevel_ref, submodules={}): # pragma: no cover '''Ensure any submodules of a given repo are cached and up to date.''' def submodules_for_repo(repo_path, ref): @@ -377,7 +377,8 @@ class RepoCache(object): submodules = morphlib.git.Submodules(repo_path, ref, runcmd_cb=self.runcmd_cb) submodules.load() - return [(submod.url, submod.commit) for submod in submodules] + return [(submod.name, submod.url, submod.commit) + for submod in submodules] except morphlib.git.NoModulesFileError: return [] @@ -385,8 +386,10 @@ class RepoCache(object): subs_to_process = submodules_for_repo(toplevel_repo.dirname, toplevel_ref) while subs_to_process: - url, ref = subs_to_process.pop() + name, url, ref = subs_to_process.pop() done.add((url, ref)) + if name in submodules: + url = submodules[name]['url'] cached_repo = self.get_updated_repo(url, ref=ref) diff --git a/morphlib/source.py b/morphlib/source.py index 135c14cc..3f9d606c 100644 --- a/morphlib/source.py +++ b/morphlib/source.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2015 Codethink Limited +# Copyright (C) 2012-2016 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/morphlib/sourceresolver.py b/morphlib/sourceresolver.py index 5d04ece9..16a254f4 100644 --- a/morphlib/sourceresolver.py +++ b/morphlib/sourceresolver.py @@ -329,7 +329,7 @@ class SourceResolver(object): # Morph code doesn't need to know about the predefined # build instructions. chunk_filename = c['name'] + '.morph' - chunk_queue.add((c["name"], c['repo'], c['ref'], + chunk_queue.add((c['name'], c['repo'], c['ref'], chunk_filename, c['build-system'])) return chunk_queue @@ -376,7 +376,7 @@ class SourceResolver(object): predefined_split_rules) def traverse_morphs(self, definitions_repo, definitions_ref, - system_filenames, + system_filenames, pool, visit=lambda rn, rf, fn, arf, m: None, definitions_original_ref=None): @@ -403,6 +403,7 @@ class SourceResolver(object): definitions_version = self._check_version_file( definitions_checkout_dir) + pool.definitions_version = definitions_version predefined_build_systems, predefined_split_rules = \ self._get_defaults( @@ -428,6 +429,7 @@ class SourceResolver(object): visit, predefined_build_systems, predefined_split_rules) + class DuplicateChunkError(morphlib.Error): def _make_msg(self, (name, sources)): # pragma: no cover @@ -503,7 +505,7 @@ def create_source_pool(repo_cache, repo, ref, filenames, resolver = SourceResolver(repo_cache, tree_cache_manager, status_cb) resolver.traverse_morphs(repo, ref, filenames, - visit=add_to_pool, + pool, visit=add_to_pool, definitions_original_ref=original_ref) # No two chunks may have the same name diff --git a/yarns/building.yarn b/yarns/building.yarn index 9284f7cf..1b981490 100644 --- a/yarns/building.yarn +++ b/yarns/building.yarn @@ -89,6 +89,14 @@ Morph Building Tests AND there are 2 artifacts named core-runtime in the cache FINALLY the git server is shut down + SCENARIO test recursive submodules overriding urls + GIVEN a git server + GIVEN a chunk with recursive submodules + WHEN the user clones definitions + WHEN the user attempts to build the system systems/test-system.morph in branch master + THEN morph succeeded + FINALLY the git server is shut down + System integrations ------------------- diff --git a/yarns/implementations.yarn b/yarns/implementations.yarn index 06380e7f..186fe316 100644 --- a/yarns/implementations.yarn +++ b/yarns/implementations.yarn @@ -45,6 +45,60 @@ locally, which we'll tell Morph to access using `file:` URLs. Specifically, we'll create a repository to hold system and stratum morphologies, and another to hold a chunk. + IMPLEMENTS GIVEN a chunk with recursive submodules + mkdir "$DATADIR/gits/grandchild-chunk" + cd "$DATADIR/gits/grandchild-chunk" + git init . + touch grandchild-file + git add . + git commit -m "Initial commit" + + mkdir "$DATADIR/gits/child-chunk" + cd "$DATADIR/gits/child-chunk" + git init . + touch child-file + git add . + git commit -m "Initial commit" + git submodule add -b master file://$DATADIR/gits/grandchild-chunk + sed -i -e 's#file://$DATADIR/gits/granchild-chunk#fake/location#' .gitmodules + git add . + git commit -m "Initial submodule" + + mkdir "$DATADIR/gits/chunk-with-submodules" + cd "$DATADIR/gits/chunk-with-submodules" + git init . + git add . + git commit --allow-empty -m "Initial commit" + git submodule add -b master file://$DATADIR/gits/child-chunk + sed -i -e 's#file://$DATADIR/gits/child-chunk#fake/location#' .gitmodules + git add . + git commit -m "Add submodule" + + cd "$DATADIR/gits/definitions" + echo "version: 8" > VERSION + cat << EOF >> strata/core.morph + - name: chunk-with-submodules + morph: chunk-with-submodules.morph + repo: test:chunk-with-submodules + ref: master + submodules: + grandchild-chunk: + url: file://$DATADIR/gits/grandchild-chunk + child-chunk: + url: file://$DATADIR/gits/child-chunk + EOF + + cat << EOF >> chunk-with-submodules.morph + name: chunk-with-submodules + kind: chunk + build-system: manual + build-commands: + - file exists child-chunk/child-file + - file exists child-chunk/grandchild-chunk/grandchild-file + EOF + git add . + git commit -m "Add moar stuff" + IMPLEMENTS GIVEN a git server # Create a directory for all the git repositories. @@ -216,7 +270,7 @@ another to hold a chunk. mkdir "$DATADIR/gits/definitions" cd "$DATADIR/gits/definitions" git init . - echo 'version: 7' > VERSION + echo 'version: 8' > VERSION install -m644 -D /dev/stdin << EOF "DEFAULTS" # This is a simplified version of the DEFAULTS file supplied with the -- cgit v1.2.1