From ba574f4d1ae390d4f94dc62817f68fe390b38778 Mon Sep 17 00:00:00 2001 From: Adam Coldrick Date: Wed, 21 Jan 2015 16:00:32 +0000 Subject: Add tests for sourceresolver This only adds tests for the bits which were moved from morphologyfactory into sourceresolver, namely detection of build systems and the '_get_morphology()' function. These are just the morphologyfactory tests reworked slightly to work properly with the modified API. --- morphlib/sourceresolver.py | 20 +-- morphlib/sourceresolver_tests.py | 331 +++++++++++++++++++++++++++++++++++++++ without-test-modules | 4 - 3 files changed, 341 insertions(+), 14 deletions(-) create mode 100644 morphlib/sourceresolver_tests.py diff --git a/morphlib/sourceresolver.py b/morphlib/sourceresolver.py index b8af8aee..29069d7d 100644 --- a/morphlib/sourceresolver.py +++ b/morphlib/sourceresolver.py @@ -32,7 +32,7 @@ buildsystem_cache_size = 10000 buildsystem_cache_filename = 'detected-chunk-buildsystems.cache.pickle' -class PickleCacheManager(object): +class PickleCacheManager(object): # pragma: no cover '''Cache manager for PyLRU that reads and writes to Pickle files. The 'pickle' format is less than ideal in many ways and is actually @@ -84,7 +84,7 @@ class SourceResolverError(cliapp.AppException): pass -class MorphologyNotFoundError(SourceResolverError): +class MorphologyNotFoundError(SourceResolverError): # pragma: no cover def __init__(self, filename): SourceResolverError.__init__( self, "Couldn't find morphology: %s" % filename) @@ -155,7 +155,7 @@ class SourceResolver(object): self._definitions_checkout_dir = None - def _resolve_ref(self, reponame, ref): + def _resolve_ref(self, reponame, ref): # pragma: no cover '''Resolves commit and tree sha1s of the ref in a repo and returns it. If update is True then this has the side-effect of updating or cloning @@ -225,7 +225,7 @@ class SourceResolver(object): return self._resolved_morphologies[key] if reponame == self._definitions_repo and \ - sha1 == self._definitions_absref: + sha1 == self._definitions_absref: # pragma: no cover defs_filename = os.path.join(self._definitions_checkout_dir, filename) else: @@ -233,7 +233,7 @@ class SourceResolver(object): loader = morphlib.morphloader.MorphologyLoader() - if defs_filename and os.path.exists(defs_filename): + if defs_filename and os.path.exists(defs_filename): # pragma: no cover morph = loader.load_from_file(defs_filename) elif self.lrc.has_repo(reponame): self.status(msg="Looking for %(reponame)s:%(filename)s in the " @@ -304,7 +304,7 @@ class SourceResolver(object): return buildsystem.name def _create_morphology_for_build_system(self, buildsystem_name, - morph_name): + morph_name): # pragma: no cover bs = morphlib.buildsystem.lookup_build_system(buildsystem_name) loader = morphlib.morphloader.MorphologyLoader() morph = bs.get_morphology(morph_name) @@ -318,7 +318,7 @@ class SourceResolver(object): definitions_ref, definitions_absref, definitions_tree, - visit): + visit): # pragma: no cover definitions_queue = collections.deque(system_filenames) chunk_in_definitions_repo_queue = set() chunk_in_source_repo_queue = set() @@ -360,7 +360,7 @@ class SourceResolver(object): return chunk_in_definitions_repo_queue, chunk_in_source_repo_queue def process_chunk(self, definition_repo, definition_ref, chunk_repo, - chunk_ref, filename, visit): + chunk_ref, filename, visit): # pragma: no cover definition_key = (definition_repo, definition_ref, filename) chunk_key = (chunk_repo, chunk_ref, filename) @@ -396,7 +396,7 @@ class SourceResolver(object): def traverse_morphs(self, definitions_repo, definitions_ref, system_filenames, visit=lambda rn, rf, fn, arf, m: None, - definitions_original_ref=None): + definitions_original_ref=None): # pragma: no cover self._resolved_trees = self.tree_cache_manager.load_cache() self._resolved_buildsystems = \ self.buildsystem_cache_manager.load_cache() @@ -453,7 +453,7 @@ class SourceResolver(object): def create_source_pool(lrc, rrc, repo, ref, filename, cachedir, original_ref=None, update_repos=True, - status_cb=None): + status_cb=None): # pragma: no cover '''Find all the sources involved in building a given system. Given a system morphology, this function will traverse the tree of stratum diff --git a/morphlib/sourceresolver_tests.py b/morphlib/sourceresolver_tests.py new file mode 100644 index 00000000..2410218a --- /dev/null +++ b/morphlib/sourceresolver_tests.py @@ -0,0 +1,331 @@ +# Copyright (C) 2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import os +import shutil +import tempfile +import unittest + +import morphlib +from morphlib.sourceresolver import (SourceResolver, + PickleCacheManager, + MorphologyNotFoundError, + NotcachedError) +from morphlib.remoterepocache import CatFileError + + +class FakeRemoteRepoCache(object): + + def cat_file(self, reponame, sha1, filename): + if filename.endswith('.morph'): + return '''{ + "name": "%s", + "kind": "chunk", + "build-system": "dummy" + }''' % filename[:-len('.morph')] + return 'text' + + def ls_tree(self, reponame, sha1): + return [] + + +class FakeLocalRepo(object): + + morphologies = { + 'chunk.morph': ''' + name: chunk + kind: chunk + build-system: dummy + ''', + 'chunk-split.morph': ''' + name: chunk-split + kind: chunk + build-system: dummy + products: + - artifact: chunk-split-runtime + include: [] + - artifact: chunk-split-devel + include: [] + ''', + 'stratum.morph': ''' + name: stratum + kind: stratum + chunks: + - name: chunk + repo: test:repo + ref: sha1 + build-mode: bootstrap + build-depends: [] + ''', + 'stratum-no-chunk-bdeps.morph': ''' + name: stratum-no-chunk-bdeps + kind: stratum + chunks: + - name: chunk + repo: test:repo + ref: sha1 + build-mode: bootstrap + ''', + 'stratum-no-bdeps-no-bootstrap.morph': ''' + name: stratum-no-bdeps-no-bootstrap + kind: stratum + chunks: + - name: chunk + repo: test:repo + ref: sha1 + build-depends: [] + ''', + 'stratum-bdeps-no-bootstrap.morph': ''' + name: stratum-bdeps-no-bootstrap + kind: stratum + build-depends: + - morph: stratum + chunks: + - name: chunk + repo: test:repo + ref: sha1 + build-depends: [] + ''', + 'stratum-empty.morph': ''' + name: stratum-empty + kind: stratum + ''', + 'system.morph': ''' + name: system + kind: system + arch: %(arch)s + strata: + - morph: stratum + ''', + 'parse-error.morph': ''' name''', + 'name-mismatch.morph': ''' + name: fred + kind: stratum + ''', + } + + def __init__(self): + self.arch = 'x86_64' + + def read_file(self, filename, ref): + if filename in self.morphologies: + values = { + 'arch': self.arch, + } + return self.morphologies[filename] % values + elif filename.endswith('.morph'): + return '''name: %s + kind: chunk + build-system: dummy''' % filename[:-len('.morph')] + return 'text' + + def list_files(self, ref, recurse): + return self.morphologies.keys() + + +class FakeLocalRepoCache(object): + + def __init__(self, lr): + self.lr = lr + + def has_repo(self, reponame): + return True + + def get_repo(self, reponame): + return self.lr + + +class SourceResolverTests(unittest.TestCase): + + def setUp(self): + # create temp "definitions" repo + # set self.sr._definitions_repo to that + # trick it into presenting temp repo using FakeLocalRepoCache + # magic + self.lr = FakeLocalRepo() + self.lrc = FakeLocalRepoCache(self.lr) + self.rrc = FakeRemoteRepoCache() + + self.cachedir = tempfile.mkdtemp() + buildsystem_cache_file = os.path.join(self.cachedir, + 'detected-chunk-buildsystems.cache.pickle') + buildsystem_cache_manager = PickleCacheManager( + buildsystem_cache_file, 1000) + + tree_cache_file = os.path.join(self.cachedir, 'trees.cache.pickle') + tree_cache_manager = PickleCacheManager(tree_cache_file, 1000) + + def status(msg='', **kwargs): + pass + + self.sr = SourceResolver(self.lrc, self.rrc, tree_cache_manager, + buildsystem_cache_manager, True, status) + self.lsr = SourceResolver(self.lrc, None, tree_cache_manager, + buildsystem_cache_manager, True, status) + + self.sr._definitions_repo = None + self.lsr._definitions_repo = None + + def tearDown(self): + shutil.rmtree(self.cachedir) + + def nolocalfile(self, *args): + raise IOError('File not found') + + def noremotefile(self, *args): + raise CatFileError('reponame', 'ref', 'filename') + + def localmorph(self, *args): + return ['chunk.morph'] + + def nolocalmorph(self, *args): + if args[0].endswith('.morph'): + raise IOError('File not found') + return 'text' + + def autotoolsbuildsystem(self, *args, **kwargs): + return ['configure.in'] + + def emptytree(self, *args, **kwargs): + return [] + + def remotemorph(self, *args, **kwargs): + return ['remote-chunk.morph'] + + def noremotemorph(self, *args): + if args[-1].endswith('.morph'): + raise CatFileError('reponame', 'ref', 'filename') + return 'text' + + def doesnothaverepo(self, reponame): + return False + + def test_gets_morph_from_local_repo(self): + self.lr.list_files = self.localmorph + morph = self.sr._get_morphology('reponame', 'sha1', + 'chunk.morph') + self.assertEqual('chunk', morph['name']) + + def test_gets_morph_from_cache(self): + self.lr.list_files = self.localmorph + morph_from_repo = self.sr._get_morphology('reponame', 'sha1', + 'chunk.morph') + morph_from_cache = self.sr._get_morphology('reponame', 'sha1', + 'chunk.morph') + self.assertEqual(morph_from_repo, morph_from_cache) + + def test_gets_morph_from_remote_repo(self): + self.rrc.ls_tree = self.remotemorph + self.lrc.has_repo = self.doesnothaverepo + morph = self.sr._get_morphology('reponame', 'sha1', + 'remote-chunk.morph') + self.assertEqual('remote-chunk', morph['name']) + + def test_autodetects_local_morphology(self): + self.lr.read_file = self.nolocalmorph + self.lr.list_files = self.autotoolsbuildsystem + name = self.sr._detect_build_system('reponame', 'sha1', + 'assumed-local.morph') + self.assertEqual('autotools', name) + + def test_autodetects_remote_morphology(self): + self.lrc.has_repo = self.doesnothaverepo + self.rrc.cat_file = self.noremotemorph + self.rrc.ls_tree = self.autotoolsbuildsystem + name = self.sr._detect_build_system('reponame', 'sha1', + 'assumed-remote.morph') + self.assertEqual('autotools', name) + + def test_returns_none_when_no_local_morph(self): + self.lr.read_file = self.nolocalfile + morph = self.sr._get_morphology('reponame', 'sha1', 'unreached.morph') + self.assertEqual(morph, None) + + def test_returns_none_when_fails_no_remote_morph(self): + self.lrc.has_repo = self.doesnothaverepo + self.rrc.cat_file = self.noremotefile + morph = self.sr._get_morphology('reponame', 'sha1', 'unreached.morph') + self.assertEqual(morph, None) + + def test_raises_error_when_repo_does_not_exist(self): + self.lrc.has_repo = self.doesnothaverepo + self.assertRaises(NotcachedError, self.lsr._detect_build_system, + 'reponame', 'sha1', 'non-existent.morph') + + def test_raises_error_when_failed_to_detect_build_system(self): + self.lr.read_file = self.nolocalfile + self.lr.list_files = self.emptytree + self.assertRaises(MorphologyNotFoundError, + self.sr._detect_build_system, + 'reponame', 'sha1', 'undetected.morph') + + def test_raises_error_when_name_mismatches(self): + self.assertRaises(morphlib.Error, self.sr._get_morphology, + 'reponame', 'sha1', 'name-mismatch.morph') + + def test_looks_locally_with_no_remote(self): + self.lr.list_files = self.localmorph + morph = self.lsr._get_morphology('reponame', 'sha1', + 'chunk.morph') + self.assertEqual('chunk', morph['name']) + + def test_autodetects_locally_with_no_remote(self): + self.lr.read_file = self.nolocalmorph + self.lr.list_files = self.autotoolsbuildsystem + name = self.sr._detect_build_system('reponame', 'sha1', + 'assumed-local.morph') + self.assertEqual('autotools', name) + + def test_fails_when_local_not_cached_and_no_remote(self): + self.lrc.has_repo = self.doesnothaverepo + self.assertRaises(NotcachedError, self.lsr._get_morphology, + 'reponame', 'sha1', 'unreached.morph') + + def test_arch_is_validated(self): + self.lr.arch = 'unknown' + self.assertRaises(morphlib.Error, self.sr._get_morphology, + 'reponame', 'sha1', 'system.morph') + + def test_arch_arm_defaults_to_le(self): + self.lr.arch = 'armv7' + morph = self.sr._get_morphology('reponame', 'sha1', 'system.morph') + self.assertEqual(morph['arch'], 'armv7l') + + def test_fails_on_parse_error(self): + self.assertRaises(morphlib.Error, self.sr._get_morphology, + 'reponame', 'sha1', 'parse-error.morph') + + def test_fails_on_no_chunk_bdeps(self): + self.assertRaises(morphlib.morphloader.NoBuildDependenciesError, + self.sr._get_morphology, 'reponame', 'sha1', + 'stratum-no-chunk-bdeps.morph') + + def test_fails_on_no_bdeps_or_bootstrap(self): + self.assertRaises( + morphlib.morphloader.NoStratumBuildDependenciesError, + self.sr._get_morphology, 'reponame', 'sha1', + 'stratum-no-bdeps-no-bootstrap.morph') + + def test_succeeds_on_bdeps_no_bootstrap(self): + self.sr._get_morphology( + 'reponame', 'sha1', + 'stratum-bdeps-no-bootstrap.morph') + + def test_fails_on_empty_stratum(self): + self.assertRaises( + morphlib.morphloader.EmptyStratumError, + self.sr._get_morphology, 'reponame', 'sha1', 'stratum-empty.morph') + diff --git a/without-test-modules b/without-test-modules index 530deb4f..55e5291d 100644 --- a/without-test-modules +++ b/without-test-modules @@ -52,7 +52,3 @@ distbuild/timer_event_source.py distbuild/worker_build_scheduler.py # Not unit tested, since it needs a full system branch morphlib/buildbranch.py - -# Requires rather a lot of fake data in order to be unit tested; better to -# leave it to the functional tests. -morphlib/sourceresolver.py -- cgit v1.2.1