From 81ebe71089d802061c2c3cb03bfd548388d04cb8 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Thu, 23 Jul 2015 17:48:03 +0100 Subject: Remove support for Baserock definitions format versions 3, 4 and 5 Change-Id: Iad95af65bd5c528d2e72f5b2ffa80a01152f50ff --- morphlib/artifactresolver_tests.py | 2 + morphlib/buildsystem.py | 86 +-------- morphlib/buildsystem_tests.py | 98 ---------- morphlib/cachekeycomputer_tests.py | 3 + morphlib/definitions_version.py | 9 +- morphlib/morphloader.py | 31 ++-- morphlib/morphloader_tests.py | 4 +- morphlib/sourceresolver.py | 308 ++++++-------------------------- morphlib/sourceresolver_tests.py | 355 ------------------------------------- 9 files changed, 84 insertions(+), 812 deletions(-) delete mode 100644 morphlib/sourceresolver_tests.py (limited to 'morphlib') diff --git a/morphlib/artifactresolver_tests.py b/morphlib/artifactresolver_tests.py index 141ff948..20617c65 100644 --- a/morphlib/artifactresolver_tests.py +++ b/morphlib/artifactresolver_tests.py @@ -292,11 +292,13 @@ class ArtifactResolverTests(unittest.TestCase): - name: chunk1 repo: repo ref: original/ref + build-system: manual build-depends: - chunk2 - name: chunk2 repo: repo ref: original/ref + build-system: manual build-depends: [] ''') sources = morphlib.source.make_sources('repo', 'original/ref', diff --git a/morphlib/buildsystem.py b/morphlib/buildsystem.py index 4655f2ee..5096a7c4 100644 --- a/morphlib/buildsystem.py +++ b/morphlib/buildsystem.py @@ -44,15 +44,11 @@ _STRIP_COMMAND = r'''find "$DESTDIR" -type f \ class BuildSystem(object): - '''An abstraction of an upstream build system. + '''Predefined commands for common build systems. - Some build systems are well known: autotools, for example. - Others are purely manual: there's a set of commands to run that - are specific for that project, and (almost) no other project uses them. - The Linux kernel would be an example of that. - - This class provides an abstraction for these, including a method - to autodetect well known build systems. + Some build systems are well known: autotools, for example. We provide + pre-defined build commands for these so that they don't need to be copied + and pasted many times in the build instructions. ''' @@ -86,15 +82,6 @@ class BuildSystem(object): 'build-system': self.name, }) - def used_by_project(self, file_list): - '''Does a project use this build system? - - ``exists`` is a function that returns a boolean telling if a - filename, relative to the project source directory, exists or not. - - ''' - raise NotImplementedError() # pragma: no cover - class ManualBuildSystem(BuildSystem): @@ -102,9 +89,6 @@ class ManualBuildSystem(BuildSystem): name = 'manual' - def used_by_project(self, file_list): - return False - class DummyBuildSystem(BuildSystem): @@ -120,9 +104,6 @@ class DummyBuildSystem(BuildSystem): self.install_commands = ['echo dummy install'] self.strip_commands = ['echo dummy strip'] - def used_by_project(self, file_list): - return False - class AutotoolsBuildSystem(BuildSystem): @@ -149,18 +130,6 @@ class AutotoolsBuildSystem(BuildSystem): ] self.strip_commands = [_STRIP_COMMAND] - def used_by_project(self, file_list): - indicators = [ - 'autogen', - 'autogen.sh', - 'configure', - 'configure.ac', - 'configure.in', - 'configure.in.in', - ] - - return any(x in file_list for x in indicators) - class PythonDistutilsBuildSystem(BuildSystem): @@ -182,13 +151,6 @@ class PythonDistutilsBuildSystem(BuildSystem): ] self.strip_commands = [_STRIP_COMMAND] - def used_by_project(self, file_list): - indicators = [ - 'setup.py', - ] - - return any(x in file_list for x in indicators) - class ExtUtilsMakeMakerBuildSystem(BuildSystem): @@ -228,12 +190,6 @@ class ExtUtilsMakeMakerBuildSystem(BuildSystem): ] self.strip_commands = [_STRIP_COMMAND] - def used_by_project(self, file_list): - indicators = [ - 'Makefile.PL', - ] - - return any(x in file_list for x in indicators) class ModuleBuildBuildSystem(BuildSystem): @@ -265,13 +221,6 @@ class ModuleBuildBuildSystem(BuildSystem): './Build install' ] - def used_by_project(self, file_list): - indicators = [ - 'Build.PL' - ] - - return any(x in file_list for x in indicators) - class CMakeBuildSystem(BuildSystem): @@ -294,12 +243,6 @@ class CMakeBuildSystem(BuildSystem): ] self.strip_commands = [_STRIP_COMMAND] - def used_by_project(self, file_list): - indicators = [ - 'CMakeLists.txt', - ] - - return any(x in file_list for x in indicators) class QMakeBuildSystem(BuildSystem): @@ -322,14 +265,6 @@ class QMakeBuildSystem(BuildSystem): ] self.strip_commands = [_STRIP_COMMAND] - def used_by_project(self, file_list): - indicator = '.pro' - - for x in file_list: - if x.endswith(indicator): - return True - - return False build_systems = [ ManualBuildSystem(), @@ -343,19 +278,6 @@ build_systems = [ ] -def detect_build_system(file_list): - '''Automatically detect the build system, if possible. - - If the build system cannot be detected automatically, return None. - For ``exists`` see the ``BuildSystem.exists`` method. - - ''' - for bs in build_systems: - if bs.used_by_project(file_list): - return bs - return None - - def lookup_build_system(name): '''Return build system that corresponds to the name. diff --git a/morphlib/buildsystem_tests.py b/morphlib/buildsystem_tests.py index 80898ebd..b49d30ae 100644 --- a/morphlib/buildsystem_tests.py +++ b/morphlib/buildsystem_tests.py @@ -21,16 +21,6 @@ import unittest import morphlib -def touch(pathname): - with open(pathname, 'w'): - pass - -manual_project = [] -autotools_project = ['configure.in'] -qmake_project = ['foo.pro'] -cmake_project = ['CMakeLists.txt'] - - class BuildSystemTests(unittest.TestCase): def setUp(self): @@ -54,94 +44,6 @@ class BuildSystemTests(unittest.TestCase): self.assertTrue(morph.__class__.__name__ == 'Morphology') -class ManualBuildSystemTests(unittest.TestCase): - - def setUp(self): - self.bs = morphlib.buildsystem.ManualBuildSystem() - - def test_does_not_autodetect_empty(self): - self.assertFalse(self.bs.used_by_project(manual_project)) - - def test_does_not_autodetect_autotools(self): - self.assertFalse(self.bs.used_by_project(autotools_project)) - - def test_does_not_autodetect_qmake(self): - self.assertFalse(self.bs.used_by_project(qmake_project)) - - def test_does_not_autodetect_cmake(self): - self.assertFalse(self.bs.used_by_project(cmake_project)) - - -class DummyBuildSystemTests(unittest.TestCase): - - def setUp(self): - self.bs = morphlib.buildsystem.DummyBuildSystem() - - def test_does_not_autodetect_empty(self): - self.assertFalse(self.bs.used_by_project(manual_project)) - - def test_does_not_autodetect_autotools(self): - self.assertFalse(self.bs.used_by_project(autotools_project)) - - def test_does_not_autodetect_cmake(self): - self.assertFalse(self.bs.used_by_project(cmake_project)) - - def test_does_not_autodetect_qmake(self): - self.assertFalse(self.bs.used_by_project(qmake_project)) - - -class AutotoolsBuildSystemTests(unittest.TestCase): - - def setUp(self): - self.bs = morphlib.buildsystem.AutotoolsBuildSystem() - - def test_does_not_autodetect_empty(self): - self.assertFalse(self.bs.used_by_project(manual_project)) - - def test_autodetects_autotools(self): - self.assertTrue(self.bs.used_by_project(autotools_project)) - -class CMakeBuildSystemTests(unittest.TestCase): - - def setUp(self): - self.bs = morphlib.buildsystem.CMakeBuildSystem() - - def test_does_not_autodetect_empty(self): - self.assertFalse(self.bs.used_by_project(manual_project)) - - def test_autodetects_cmake(self): - self.assertTrue(self.bs.used_by_project(cmake_project)) - -class QMakeBuildSystemTests(unittest.TestCase): - - def setUp(self): - self.bs = morphlib.buildsystem.QMakeBuildSystem() - - def test_does_not_autodetect_empty(self): - self.assertFalse(self.bs.used_by_project(manual_project)) - - def test_autodetects_qmake(self): - self.assertTrue(self.bs.used_by_project(qmake_project)) - -class DetectBuildSystemTests(unittest.TestCase): - - def test_does_not_autodetect_manual(self): - bs = morphlib.buildsystem.detect_build_system(manual_project) - self.assertEqual(bs, None) - - def test_autodetects_autotools(self): - bs = morphlib.buildsystem.detect_build_system(autotools_project) - self.assertEqual(type(bs), morphlib.buildsystem.AutotoolsBuildSystem) - - def test_autodetects_cmake(self): - bs = morphlib.buildsystem.detect_build_system(cmake_project) - self.assertEqual(type(bs), morphlib.buildsystem.CMakeBuildSystem) - - def test_autodetects_qmake(self): - bs = morphlib.buildsystem.detect_build_system(qmake_project) - self.assertEqual(type(bs), morphlib.buildsystem.QMakeBuildSystem) - - class LookupBuildSystemTests(unittest.TestCase): def lookup(self, name): diff --git a/morphlib/cachekeycomputer_tests.py b/morphlib/cachekeycomputer_tests.py index aa217dfc..fbf680f0 100644 --- a/morphlib/cachekeycomputer_tests.py +++ b/morphlib/cachekeycomputer_tests.py @@ -55,6 +55,7 @@ class CacheKeyComputerTests(unittest.TestCase): build-depends: [] chunks: - name: chunk + morph: chunk.morph repo: repo ref: original/ref build-depends: [] @@ -65,10 +66,12 @@ class CacheKeyComputerTests(unittest.TestCase): build-depends: [] chunks: - name: chunk2 + morph: chunk2.morph repo: repo ref: original/ref build-depends: [] - name: chunk3 + morph: chunk3.morph repo: repo ref: original/ref build-depends: [] diff --git a/morphlib/definitions_version.py b/morphlib/definitions_version.py index 44bed178..3ed5d19a 100644 --- a/morphlib/definitions_version.py +++ b/morphlib/definitions_version.py @@ -24,7 +24,7 @@ import yaml import morphlib -SUPPORTED_VERSIONS = [3, 4, 5, 6] +SUPPORTED_VERSIONS = [6] class DefinitionsVersionError(cliapp.AppException): @@ -38,8 +38,9 @@ class UnknownVersionError(DefinitionsVersionError): # pragma: no cover class InvalidVersionFileError(DefinitionsVersionError): # pragma: no cover - def __init__(self): - DefinitionsVersionError.__init__(self, "invalid VERSION file") + def __init__(self, text): + DefinitionsVersionError.__init__( + self, "invalid VERSION file: '%s'" % text) def parse_version_file(version_text): @@ -72,7 +73,7 @@ def check_version_file(version_text): # pragma: no cover version = morphlib.definitions_version.parse_version_file(version_text) if version == None: - raise InvalidVersionFileError() + raise InvalidVersionFileError(version_text) if version not in SUPPORTED_VERSIONS: raise UnknownVersionError(version) diff --git a/morphlib/morphloader.py b/morphlib/morphloader.py index 479bc8fb..f85c5d4d 100644 --- a/morphlib/morphloader.py +++ b/morphlib/morphloader.py @@ -16,8 +16,6 @@ import collections -import copy -import logging import warnings import yaml @@ -378,6 +376,9 @@ class MorphologyLoader(object): 'pre-install-commands': None, 'install-commands': None, 'post-install-commands': None, + 'pre-strip-commands': None, + 'strip-commands': None, + 'post-strip-commands': None, 'devices': [], 'products': [], 'max-jobs': None, @@ -402,16 +403,8 @@ class MorphologyLoader(object): }, } - def __init__(self, definitions_version=0, + def __init__(self, lookup_build_system=morphlib.buildsystem.lookup_build_system): - if definitions_version >= 5: # pragma: no cover - self._static_defaults = copy.deepcopy(self._static_defaults) - self._static_defaults['chunk'].update({ - 'pre-strip-commands': None, - 'strip-commands': None, - 'post-strip-commands': None}) - - self._definitions_version = definitions_version self._lookup_build_system = lookup_build_system def parse_morphology_text(self, text, morph_filename): @@ -609,15 +602,13 @@ class MorphologyLoader(object): '%s.build-depends' % chunk_name, list, type(spec['build-depends']), morph['name']) - if self._definitions_version >= 6: - # Either 'morph' or 'build-system' must be specified. - if 'morph' in spec and 'build-system' in spec: - raise ChunkSpecConflictingFieldsError( - ['morph', 'build-system'], chunk_name, morph.filename) - if 'morph' not in spec and 'build-system' not in spec: - raise ChunkSpecNoBuildInstructionsError( - chunk_name, morph.filename) - + # Either 'morph' or 'build-system' must be specified. + if 'morph' in spec and 'build-system' in spec: + raise ChunkSpecConflictingFieldsError( + ['morph', 'build-system'], chunk_name, morph.filename) + if 'morph' not in spec and 'build-system' not in spec: + raise ChunkSpecNoBuildInstructionsError( + chunk_name, morph.filename) @classmethod def _validate_chunk(cls, morphology): diff --git a/morphlib/morphloader_tests.py b/morphlib/morphloader_tests.py index 6cb93094..6117573e 100644 --- a/morphlib/morphloader_tests.py +++ b/morphlib/morphloader_tests.py @@ -49,8 +49,7 @@ def stratum_template(name): class MorphologyLoaderTests(unittest.TestCase): def setUp(self): - self.loader = morphlib.morphloader.MorphologyLoader( - definitions_version=6) + self.loader = morphlib.morphloader.MorphologyLoader() self.tempdir = tempfile.mkdtemp() self.filename = os.path.join(self.tempdir, 'foo.morph') @@ -970,7 +969,6 @@ build-system: dummy def test_smoketest_strip_commands(self): dummy_buildsystem = morphlib.buildsystem.DummyBuildSystem() loader = morphlib.morphloader.MorphologyLoader( - definitions_version=5, lookup_build_system=lambda x: dummy_buildsystem) m = morphlib.morphology.Morphology( {'name': 'test', 'kind': 'chunk', 'build-system': 'dummy'}) diff --git a/morphlib/sourceresolver.py b/morphlib/sourceresolver.py index cbab0f7f..0b32598f 100644 --- a/morphlib/sourceresolver.py +++ b/morphlib/sourceresolver.py @@ -19,8 +19,6 @@ import cPickle import logging import os import pylru -import shutil -import tempfile import yaml import cliapp @@ -31,11 +29,9 @@ from morphlib.util import sanitise_morphology_path tree_cache_size = 10000 tree_cache_filename = 'trees.cache.pickle' -buildsystem_cache_size = 10000 -buildsystem_cache_filename = 'detected-chunk-buildsystems.cache.pickle' -class PickleCacheManager(object): # pragma: no cover +class PickleCacheManager(object): '''Cache manager for PyLRU that reads and writes to Pickle files. The 'pickle' format is less than ideal in many ways and is actually @@ -97,13 +93,13 @@ class SourceResolverError(cliapp.AppException): pass -class MorphologyNotFoundError(SourceResolverError): # pragma: no cover +class MorphologyNotFoundError(SourceResolverError): def __init__(self, filename): SourceResolverError.__init__( self, "Couldn't find morphology: %s" % filename) -class MorphologyReferenceNotFoundError(SourceResolverError): # pragma: no cover +class MorphologyReferenceNotFoundError(SourceResolverError): def __init__(self, filename, reference_file): SourceResolverError.__init__(self, "Couldn't find morphology: %s " @@ -115,7 +111,7 @@ class MorphologyReferenceNotFoundError(SourceResolverError): # pragma: no cover # InvalidRefError in the definitions.git repo. Currently a separate exception # type seems the easiest way to do that, but adding enough detail to the # gitdir.InvalidRefError class may make this class redundant in future. -class InvalidDefinitionsRefError(SourceResolverError): # pragma: no cover +class InvalidDefinitionsRefError(SourceResolverError): def __init__(self, repo_url, ref): self.repo_url = repo_url self.ref = ref @@ -165,17 +161,16 @@ class SourceResolver(object): ''' def __init__(self, local_repo_cache, remote_repo_cache, - tree_cache_manager, buildsystem_cache_manager, update_repos, + tree_cache_manager, update_repos, status_cb=None): self.lrc = local_repo_cache self.rrc = remote_repo_cache self.tree_cache_manager = tree_cache_manager - self.buildsystem_cache_manager = buildsystem_cache_manager self.update = update_repos self.status = status_cb - def _resolve_ref(self, resolved_trees, reponame, ref): # pragma: no cover + def _resolve_ref(self, resolved_trees, reponame, ref): '''Resolves commit and tree sha1s of the ref in a repo and returns it. If update is True then this has the side-effect of updating or cloning @@ -230,7 +225,7 @@ class SourceResolver(object): return absref, tree def _get_file_contents_from_definitions(self, definitions_checkout_dir, - filename): # pragma: no cover + filename): fp = os.path.join(definitions_checkout_dir, filename) if os.path.exists(fp): with open(fp) as f: @@ -239,34 +234,9 @@ class SourceResolver(object): logging.debug("Didn't find %s in definitions", filename) return None - def _get_file_contents_from_repo(self, reponame, - sha1, filename): # pragma: no cover - if self.lrc.has_repo(reponame): - self.status(msg="Looking for %(reponame)s:%(filename)s in the " - "local repo cache.", - reponame=reponame, filename=filename, chatty=True) - try: - repo = self.lrc.get_repo(reponame) - text = repo.read_file(filename, sha1) - except IOError: - text = None - elif self.rrc is not None: - self.status(msg="Looking for %(reponame)s:%(filename)s in the " - "remote repo cache.", - reponame=reponame, filename=filename, chatty=True) - try: - text = self.rrc.cat_file(reponame, sha1, filename) - except morphlib.remoterepocache.CatFileError: - text = None - else: # pragma: no cover - repo = self.lrc.get_updated_repo(reponame, sha1) - text = repo.read_file(filename, sha1) - - return text - def _get_file_contents(self, definitions_checkout_dir, definitions_repo, definitions_absref, reponame, sha1, - filename): # pragma: no cover + filename): '''Read the file at the specified location. Returns None if the file does not exist in the specified commit. @@ -275,7 +245,7 @@ class SourceResolver(object): text = None if reponame == definitions_repo and \ - sha1 == definitions_absref: # pragma: no cover + sha1 == definitions_absref: text = self._get_file_contents_from_definitions( definitions_checkout_dir, filename) else: @@ -283,31 +253,28 @@ class SourceResolver(object): return text - def _check_version_file(self, definitions_checkout_dir): # pragma: no cover + def _check_version_file(self, definitions_checkout_dir): version_text = self._get_file_contents_from_definitions( definitions_checkout_dir, 'VERSION') return morphlib.definitions_version.check_version_file(version_text) def _get_morphology(self, resolved_morphologies, definitions_checkout_dir, - definitions_repo, definitions_absref, morph_loader, - reponame, sha1, filename): # pragma: no cover + morph_loader, filename): '''Read the morphology at the specified location. Returns None if the file does not exist in the specified commit. ''' - key = (reponame, sha1, filename) - if key in resolved_morphologies: - return resolved_morphologies[key] + if filename in resolved_morphologies: + return resolved_morphologies[filename] - text = self._get_file_contents(definitions_checkout_dir, - definitions_repo, definitions_absref, - reponame, sha1, filename) + text = self._get_file_contents_from_definitions( + definitions_checkout_dir, filename) morph = morph_loader.load_from_string(text, filename) if morph is not None: - resolved_morphologies[key] = morph + resolved_morphologies[filename] = morph return morph @@ -318,24 +285,20 @@ class SourceResolver(object): definitions_ref, definitions_absref, definitions_tree, - definitions_version, morph_loader, system_filenames, - visit): # pragma: no cover + visit): definitions_queue = collections.deque(system_filenames) chunk_queue = set() - def get_morphology(repo, sha1, filename): + def get_morphology(filename): return self._get_morphology(resolved_morphologies, - definitions_checkout_dir, - definitions_repo, definitions_absref, - morph_loader, repo, sha1, filename) - + definitions_checkout_dir, morph_loader, + filename) while definitions_queue: filename = definitions_queue.popleft() - morphology = get_morphology(definitions_repo, definitions_absref, - filename) + morphology = get_morphology(filename) if morphology is None: raise MorphologyNotFoundError(filename) @@ -356,214 +319,68 @@ class SourceResolver(object): sanitise_morphology_path(s['morph']) for s in morphology['build-depends']) for c in morphology['chunks']: - # This field is only valid in strata from definitions - # version 6 onwards. Validation is done in morphloader.py. - buildsystem = c.get('build-system') - if 'morph' not in c: - # Autodetect a path if one is not given. This is to - # support the deprecated approach of putting the chunk - # .morph file in the toplevel directory of the chunk - # repo, instead of putting it in the definitions.git - # repo. - # - # All users should be specifying a full path to the - # chunk morph file, using the 'morph' field, and this - # code path should be removed. - path = sanitise_morphology_path( - c.get('morph', c['name'])) - - chunk_queue.add((c['repo'], c['ref'], path, - buildsystem)) - else: + if 'morph' in c: # Now, does this path actually exist? path = c['morph'] - morphology = get_morphology(definitions_repo, - definitions_absref, - path) + morphology = get_morphology(path) if morphology is None: raise MorphologyReferenceNotFoundError( path, filename) - chunk_queue.add((c['repo'], c['ref'], path, - buildsystem)) + chunk_queue.add((c['repo'], c['ref'], path, None)) + else: + # We invent a filename here, so that the rest of the + # Morph code doesn't need to know about the predefined + # build instructions. + chunk_name = c['name'] + '.morph' + chunk_queue.add((c['repo'], c['ref'], chunk_name, + c['build-system'])) return chunk_queue - @staticmethod - def _create_morphology_for_build_system(morph_loader, buildsystem, - morph_name): # pragma: no cover + def _create_morphology_for_build_system(self, morph_loader, buildsystem, + morph_name): morph = buildsystem.get_morphology(morph_name) morph_loader.validate(morph) morph_loader.set_commands(morph) morph_loader.set_defaults(morph) return morph - @classmethod - def _generate_morph_and_cache_buildsystem(cls, resolved_morphologies, - resolved_buildsystems, - morph_loader, - definition_key, chunk_key, - buildsystem, - morph_name): # pragma: no cover - logging.debug('Caching build system for chunk with key %s', chunk_key) - - resolved_buildsystems[chunk_key] = buildsystem.name - - morphology = cls._create_morphology_for_build_system( - morph_loader, buildsystem, morph_name) - resolved_morphologies[definition_key] = morphology - return morphology - - def _detect_build_system(self, reponame, sha1, expected_filename): - '''Attempt to detect buildsystem of the given commit. - - Returns None if no known build system was detected. - - ''' - self.status(msg="File %s doesn't exist: attempting to infer " - "chunk morph from repo's build system" % - expected_filename, chatty=True) - - file_list = None - - if self.lrc.has_repo(reponame): - repo = self.lrc.get_repo(reponame) - try: - file_list = repo.list_files(ref=sha1, recurse=False) - except morphlib.gitdir.InvalidRefError: # pragma: no cover - pass - elif self.rrc is not None: - try: - # This may or may not succeed; if the is repo not - # hosted on the same Git server as the cache server then - # it'll definitely fail. - file_list = self.rrc.ls_tree(reponame, sha1) - except morphlib.remoterepocache.LsTreeError: - pass - - if not file_list: - repo = self.lrc.get_updated_repo(reponame, sha1) - file_list = repo.list_files(ref=sha1, recurse=False) - - buildsystem = morphlib.buildsystem.detect_build_system(file_list) - - if buildsystem is None: - # It might surprise you to discover that if we can't autodetect a - # build system, we raise MorphologyNotFoundError. Users are - # required to provide a morphology for any chunk where Morph can't - # infer the build instructions automatically, so this is the right - # error. - raise MorphologyNotFoundError(expected_filename) - - return buildsystem - def process_chunk(self, resolved_morphologies, resolved_trees, - resolved_buildsystems, definitions_checkout_dir, - definitions_repo, definitions_absref, - definitions_version, morph_loader, chunk_repo, chunk_ref, - filename, chunk_buildsystem, visit): # pragma: no cover - absref = None - tree = None - chunk_key = None - buildsystem = None - - morph_name = os.path.splitext(os.path.basename(filename))[0] - - def get_morphology(repo, sha1, filename): - return self._get_morphology( - resolved_morphologies, definitions_checkout_dir, - definitions_repo, definitions_absref, morph_loader, - repo, sha1, filename) - - # Get morphology from definitions repo - definition_key = (definitions_repo, definitions_absref, filename) - morphology = get_morphology(*definition_key) - - if morphology: - absref, tree = self._resolve_ref(resolved_trees, chunk_repo, - chunk_ref) - visit(chunk_repo, chunk_ref, filename, absref, tree, morphology) - return - + definitions_checkout_dir, morph_loader, chunk_repo, + chunk_ref, filename, chunk_buildsystem, + visit): absref, tree = self._resolve_ref(resolved_trees, chunk_repo, chunk_ref) - chunk_key = (chunk_repo, absref, filename) - - def generate_morph_and_cache_buildsystem(buildsystem): - return self._generate_morph_and_cache_buildsystem( - resolved_morphologies, resolved_buildsystems, morph_loader, - definition_key, chunk_key, buildsystem, morph_name) - - if definitions_version >= 6: - # All build-system information is specified in the definitions from - # version 6 onwards. Either 'morph' or 'build-system' should be - # specified for each chunk. - if chunk_buildsystem is None: - # The validation done in 'morphloader' should mean that this - # never happens. - raise SourceResolverError( - 'Please specify either "build-system" or "morph" for %s.' % - chunk_key) + + if chunk_buildsystem is None: + # Build instructions defined in a chunk .morph file. An error is + # already raised in _process_definitions_with_children() if the + # 'morph' field points to a file that doesn't exist. + morphology = self._get_morphology(resolved_morphologies, + definitions_checkout_dir, + morph_loader, filename) + else: + # Chunk uses one of the predefined build systems. In this case + # 'filename' will be faked (name of chunk + '.morph'). buildsystem = morphlib.buildsystem.lookup_build_system( chunk_buildsystem) - if definition_key in resolved_morphologies: - morphology = resolved_morphologies[definition_key] - else: - morphology = generate_morph_and_cache_buildsystem(buildsystem) - - elif chunk_key in resolved_buildsystems: - logging.debug('Build system for %s is cached', str(chunk_key)) - self.status(msg='Build system for %(chunk)s is cached', - chunk=str(chunk_key), - chatty=True) - buildsystem_name = resolved_buildsystems[chunk_key] - buildsystem = morphlib.buildsystem.lookup_build_system( - buildsystem_name) - - # If the build system for this chunk is cached then: - # * the chunk does not have a chunk morph - # (so we don't need to look for one) - # - # * a suitable (generated) morphology may already be cached. - # - # If the morphology is not already cached we can generate it - # from the build-system and cache it. - if definition_key in resolved_morphologies: - morphology = resolved_morphologies[definition_key] - else: - morphology = generate_morph_and_cache_buildsystem(buildsystem) - else: - logging.debug('Build system for %s is NOT cached', str(chunk_key)) - # build-system not cached, look for morphology in chunk repo - # this can be slow (we may need to clone the repo from a remote) - morphology = get_morphology(*chunk_key) - - if morphology != None: - resolved_morphologies[definition_key] = morphology - else: - # This chunk doesn't have a chunk morph - buildsystem = self._detect_build_system(*chunk_key) - - if buildsystem is None: - raise MorphologyNotFoundError(filename) - else: - morphology = generate_morph_and_cache_buildsystem( - buildsystem) + morphology = self._create_morphology_for_build_system( + morph_loader, buildsystem, filename) visit(chunk_repo, chunk_ref, filename, absref, tree, morphology) def traverse_morphs(self, definitions_repo, definitions_ref, system_filenames, visit=lambda rn, rf, fn, arf, m: None, - definitions_original_ref=None): # pragma: no cover + definitions_original_ref=None): resolved_morphologies = {} with morphlib.util.temp_dir() as definitions_checkout_dir, \ - self.tree_cache_manager.open() as resolved_trees, \ - self.buildsystem_cache_manager.open() as resolved_buildsystems: + self.tree_cache_manager.open() as resolved_trees: # Resolve the repo, ref pair for definitions repo, cache result try: @@ -582,9 +399,8 @@ class SourceResolver(object): definitions_absref, definitions_checkout_dir) definitions_version = self._check_version_file( - definitions_checkout_dir) - morph_loader = morphlib.morphloader.MorphologyLoader( - definitions_version=definitions_version) + definitions_checkout_dir) + morph_loader = morphlib.morphloader.MorphologyLoader() # First, process the system and its stratum morphologies. These # will all live in the same Git repository, and will point to @@ -592,22 +408,19 @@ class SourceResolver(object): chunk_queue = self._process_definitions_with_children( resolved_morphologies, definitions_checkout_dir, definitions_repo, definitions_ref, definitions_absref, - definitions_tree, definitions_version, morph_loader, + definitions_tree, morph_loader, system_filenames, visit) # Now process all the chunks involved in the build. for repo, ref, filename, buildsystem in chunk_queue: self.process_chunk(resolved_morphologies, resolved_trees, - resolved_buildsystems, - definitions_checkout_dir, - definitions_repo, definitions_absref, - definitions_version, morph_loader, repo, - ref, filename, buildsystem, visit) + definitions_checkout_dir, morph_loader, + repo, ref, filename, buildsystem, visit) def create_source_pool(lrc, rrc, repo, ref, filenames, cachedir, original_ref=None, update_repos=True, - status_cb=None): # pragma: no cover + status_cb=None): '''Find all the sources involved in building a given system. Given a system morphology, this function will traverse the tree of stratum @@ -634,12 +447,7 @@ def create_source_pool(lrc, rrc, repo, ref, filenames, cachedir, tree_cache_manager = PickleCacheManager( os.path.join(cachedir, tree_cache_filename), tree_cache_size) - buildsystem_cache_manager = PickleCacheManager( - os.path.join(cachedir, buildsystem_cache_filename), - buildsystem_cache_size) - - resolver = SourceResolver(lrc, rrc, tree_cache_manager, - buildsystem_cache_manager, update_repos, + resolver = SourceResolver(lrc, rrc, tree_cache_manager, update_repos, status_cb) resolver.traverse_morphs(repo, ref, filenames, visit=add_to_pool, diff --git a/morphlib/sourceresolver_tests.py b/morphlib/sourceresolver_tests.py deleted file mode 100644 index 5985579c..00000000 --- a/morphlib/sourceresolver_tests.py +++ /dev/null @@ -1,355 +0,0 @@ -# Copyright (C) 2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see . - - -import os -import shutil -import tempfile -import unittest - -import morphlib -from morphlib.sourceresolver import (SourceResolver, - PickleCacheManager, - MorphologyNotFoundError) -from morphlib.remoterepocache import CatFileError, LsTreeError - - -class FakeRemoteRepoCache(object): - - def cat_file(self, reponame, sha1, filename): - if filename.endswith('.morph'): - return '''{ - "name": "%s", - "kind": "chunk", - "build-system": "dummy" - }''' % filename[:-len('.morph')] - return 'text' - - def ls_tree(self, reponame, sha1): - return [] - - -class FakeLocalRepo(object): - - morphologies = { - 'chunk.morph': ''' - name: chunk - kind: chunk - build-system: dummy - ''', - 'chunk-split.morph': ''' - name: chunk-split - kind: chunk - build-system: dummy - products: - - artifact: chunk-split-runtime - include: [] - - artifact: chunk-split-devel - include: [] - ''', - 'stratum.morph': ''' - name: stratum - kind: stratum - chunks: - - name: chunk - repo: test:repo - ref: sha1 - build-mode: bootstrap - build-depends: [] - ''', - 'stratum-no-bdeps-no-bootstrap.morph': ''' - name: stratum-no-bdeps-no-bootstrap - kind: stratum - chunks: - - name: chunk - repo: test:repo - ref: sha1 - build-depends: [] - ''', - 'stratum-bdeps-no-bootstrap.morph': ''' - name: stratum-bdeps-no-bootstrap - kind: stratum - build-depends: - - morph: stratum - chunks: - - name: chunk - repo: test:repo - ref: sha1 - build-depends: [] - ''', - 'stratum-empty.morph': ''' - name: stratum-empty - kind: stratum - ''', - 'system.morph': ''' - name: system - kind: system - arch: %(arch)s - strata: - - morph: stratum - ''', - 'parse-error.morph': ''' name''', - 'name-mismatch.morph': ''' - name: fred - kind: stratum - ''', - } - - def __init__(self): - self.arch = 'x86_64' - - def read_file(self, filename, ref): - if filename in self.morphologies: - values = { - 'arch': self.arch, - } - return self.morphologies[filename] % values - elif filename.endswith('.morph'): - return ''' - name: %s - kind: chunk - build-system: dummy''' % filename[:-len('.morph')] - return 'text' - - def list_files(self, ref, recurse): - return self.morphologies.keys() - - def update(self): - pass - - -class FakeLocalRepoCache(object): - - def __init__(self, lr): - self.lr = lr - - def has_repo(self, reponame): - return True - - def get_repo(self, reponame): - return self.lr - - def cache_repo(self, reponame): - return self.lr - - def get_updated_repo(self, reponame, ref=None): - return self.lr - - -class SourceResolverTests(unittest.TestCase): - - def setUp(self): - # create temp "definitions" repo - # set self.sr._definitions_repo to that - # trick it into presenting temp repo using FakeLocalRepoCache - # magic - self.lr = FakeLocalRepo() - self.lrc = FakeLocalRepoCache(self.lr) - self.rrc = FakeRemoteRepoCache() - - self.cachedir = tempfile.mkdtemp() - buildsystem_cache_file = os.path.join(self.cachedir, - 'detected-chunk-buildsystems.cache.pickle') - buildsystem_cache_manager = PickleCacheManager( - buildsystem_cache_file, 1000) - - tree_cache_file = os.path.join(self.cachedir, 'trees.cache.pickle') - tree_cache_manager = PickleCacheManager(tree_cache_file, 1000) - - def status(msg='', **kwargs): - pass - - self.sr = SourceResolver(self.lrc, self.rrc, tree_cache_manager, - buildsystem_cache_manager, True, status) - self.lsr = SourceResolver(self.lrc, None, tree_cache_manager, - buildsystem_cache_manager, True, status) - - def tearDown(self): - shutil.rmtree(self.cachedir) - - def nolocalfile(self, *args): - raise IOError('File not found') - - def noremotefile(self, *args): - raise CatFileError('reponame', 'ref', 'filename') - - def noremoterepo(self, *args): - raise LsTreeError('reponame', 'ref') - - def localmorph(self, *args): - return ['chunk.morph'] - - def nolocalmorph(self, *args): - if args[0].endswith('.morph'): - raise IOError('File not found') - return 'text' - - def autotoolsbuildsystem(self, *args, **kwargs): - return ['configure.in'] - - def emptytree(self, *args, **kwargs): - return [] - - def remotemorph(self, *args, **kwargs): - return ['remote-chunk.morph'] - - def noremotemorph(self, *args): - if args[-1].endswith('.morph'): - raise CatFileError('reponame', 'ref', 'filename') - return 'text' - - def doesnothaverepo(self, reponame): - return False - - def test_gets_morph_from_local_repo(self): - self.lr.list_files = self.localmorph - morph = self.sr._get_morphology( - {}, None, None, None, - morphlib.morphloader.MorphologyLoader(), 'reponame', - 'sha1', 'chunk.morph') - self.assertEqual('chunk', morph['name']) - - def test_gets_morph_from_cache(self): - self.lr.list_files = self.localmorph - morph_from_repo = self.sr._get_morphology( - {}, None, None, None, - morphlib.morphloader.MorphologyLoader(), 'reponame', - 'sha1', 'chunk.morph') - morph_from_cache = self.sr._get_morphology( - {}, None, None, None, - morphlib.morphloader.MorphologyLoader(), 'reponame', - 'sha1', 'chunk.morph') - self.assertEqual(morph_from_repo, morph_from_cache) - - def test_gets_morph_from_remote_repo(self): - self.rrc.ls_tree = self.remotemorph - self.lrc.has_repo = self.doesnothaverepo - morph = self.sr._get_morphology( - {}, None, None, None, - morphlib.morphloader.MorphologyLoader(), 'reponame', - 'sha1', 'remote-chunk.morph') - self.assertEqual('remote-chunk', morph['name']) - - def test_autodetects_local_morphology(self): - self.lr.read_file = self.nolocalmorph - self.lr.list_files = self.autotoolsbuildsystem - bs = self.sr._detect_build_system('reponame', 'sha1', - 'assumed-local.morph') - self.assertEqual('autotools', bs.name) - - def test_cache_repo_if_not_in_either_cache(self): - self.lrc.has_repo = self.doesnothaverepo - self.lr.read_file = self.nolocalmorph - self.lr.list_files = self.autotoolsbuildsystem - self.rrc.ls_tree = self.noremoterepo - bs = self.sr._detect_build_system('reponame', 'sha1', - 'assumed-local.morph') - self.assertEqual('autotools', bs.name) - - def test_autodetects_remote_morphology(self): - self.lrc.has_repo = self.doesnothaverepo - self.rrc.cat_file = self.noremotemorph - self.rrc.ls_tree = self.autotoolsbuildsystem - bs = self.sr._detect_build_system('reponame', 'sha1', - 'assumed-remote.morph') - self.assertEqual('autotools', bs.name) - - def test_returns_none_when_no_local_morph(self): - self.lr.read_file = self.nolocalfile - morph = self.sr._get_morphology( - {}, None, None, None, - morphlib.morphloader.MorphologyLoader(), 'reponame', - 'sha1', 'unreached.morph') - self.assertEqual(morph, None) - - def test_raises_error_when_repo_does_not_exist(self): - self.lrc.has_repo = self.doesnothaverepo - self.assertRaises(MorphologyNotFoundError, - self.lsr._detect_build_system, - 'reponame', 'sha1', 'non-existent.morph') - - def test_raises_error_when_failed_to_detect_build_system(self): - self.lr.read_file = self.nolocalfile - self.lr.list_files = self.emptytree - self.assertRaises(MorphologyNotFoundError, - self.sr._detect_build_system, - 'reponame', 'sha1', 'undetected.morph') - - def test_raises_error_when_name_mismatches(self): - self.assertRaises(morphlib.Error, self.sr._get_morphology, {}, - None, None, None, morphlib.morphloader.MorphologyLoader(), - 'reponame', 'sha1', 'name-mismatch.morph') - - def test_looks_locally_with_no_remote(self): - self.lr.list_files = self.localmorph - morph = self.lsr._get_morphology( - {}, None, None, None, - morphlib.morphloader.MorphologyLoader(), 'reponame', - 'sha1', 'chunk.morph') - self.assertEqual('chunk', morph['name']) - - def test_autodetects_locally_with_no_remote(self): - self.lr.read_file = self.nolocalmorph - self.lr.list_files = self.autotoolsbuildsystem - bs = self.sr._detect_build_system('reponame', 'sha1', - 'assumed-local.morph') - self.assertEqual('autotools', bs.name) - - def test_succeeds_when_local_not_cached_and_no_remote(self): - self.lrc.has_repo = self.doesnothaverepo - self.lr.list_files = self.localmorph - morph = self.sr._get_morphology( - {}, None, None, None, - morphlib.morphloader.MorphologyLoader(), 'reponame', - 'sha1', 'chunk.morph') - self.assertEqual('chunk', morph['name']) - - def test_arch_is_validated(self): - self.lr.arch = 'unknown' - self.assertRaises(morphlib.Error, self.sr._get_morphology, {}, - None, None, None, morphlib.morphloader.MorphologyLoader(), - 'reponame', 'sha1', 'system.morph') - - def test_arch_arm_defaults_to_le(self): - self.lr.arch = 'armv7' - morph = self.sr._get_morphology( - {}, None, None, None, - morphlib.morphloader.MorphologyLoader(), 'reponame', - 'sha1', 'system.morph') - self.assertEqual(morph['arch'], 'armv7l') - - def test_fails_on_parse_error(self): - self.assertRaises(morphlib.Error, self.sr._get_morphology, {}, - None, None, None, morphlib.morphloader.MorphologyLoader(), - 'reponame', 'sha1', 'parse-error.morph') - - def test_fails_on_no_bdeps_or_bootstrap(self): - self.assertRaises( - morphlib.morphloader.NoStratumBuildDependenciesError, - self.sr._get_morphology, {}, None, None, None, - morphlib.morphloader.MorphologyLoader(), 'reponame', 'sha1', - 'stratum-no-bdeps-no-bootstrap.morph') - - def test_succeeds_on_bdeps_no_bootstrap(self): - self.sr._get_morphology({}, None, None, None, - morphlib.morphloader.MorphologyLoader(), 'reponame', 'sha1', - 'stratum-bdeps-no-bootstrap.morph') - - def test_fails_on_empty_stratum(self): - self.assertRaises( - morphlib.morphloader.EmptyStratumError, - self.sr._get_morphology, {}, None, None, None, - morphlib.morphloader.MorphologyLoader(), 'reponame', 'sha1', - 'stratum-empty.morph') - -- cgit v1.2.1