From 25b98cf567ba8faf1f717677cb1f9da2292c22dc Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Tue, 25 Nov 2014 21:40:06 +0000 Subject: Move MorphologyFactory into SourceResolver There's no need for this stuff to be in a separate class. This allows integrating it with the caching in the SourceResolver class. --- morphlib/__init__.py | 1 - morphlib/morphologyfactory.py | 90 ------------ morphlib/morphologyfactory_tests.py | 281 ------------------------------------ morphlib/sourceresolver.py | 78 ++++++++-- 4 files changed, 69 insertions(+), 381 deletions(-) delete mode 100644 morphlib/morphologyfactory.py delete mode 100644 morphlib/morphologyfactory_tests.py diff --git a/morphlib/__init__.py b/morphlib/__init__.py index a10ebe7b..d54340df 100644 --- a/morphlib/__init__.py +++ b/morphlib/__init__.py @@ -68,7 +68,6 @@ import gitindex import localartifactcache import localrepocache import mountableimage -import morphologyfactory import morphologyfinder import morphology import morphloader diff --git a/morphlib/morphologyfactory.py b/morphlib/morphologyfactory.py deleted file mode 100644 index a3ac2749..00000000 --- a/morphlib/morphologyfactory.py +++ /dev/null @@ -1,90 +0,0 @@ -# Copyright (C) 2012-2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -import os - -import morphlib -import cliapp - - -class MorphologyFactoryError(cliapp.AppException): - pass - - -class MorphologyNotFoundError(MorphologyFactoryError): - def __init__(self, filename): - MorphologyFactoryError.__init__( - self, "Couldn't find morphology: %s" % filename) - - -class NotcachedError(MorphologyFactoryError): - def __init__(self, repo_name): - MorphologyFactoryError.__init__( - self, "Repository %s is not cached locally and there is no " - "remote cache specified" % repo_name) - - -class MorphologyFactory(object): - - '''A way of creating morphologies which will provide a default''' - - def __init__(self, local_repo_cache, remote_repo_cache=None, - status_cb=None): - self._lrc = local_repo_cache - self._rrc = remote_repo_cache - - null_status_function = lambda **kwargs: None - self.status = status_cb or null_status_function - - def get_morphology(self, reponame, sha1, filename): - morph_name = os.path.splitext(os.path.basename(filename))[0] - loader = morphlib.morphloader.MorphologyLoader() - if self._lrc.has_repo(reponame): - self.status(msg="Looking for %s in local repo cache" % filename, - chatty=True) - try: - repo = self._lrc.get_repo(reponame) - text = repo.read_file(filename, sha1) - morph = loader.load_from_string(text) - except IOError: - morph = None - file_list = repo.list_files(ref=sha1, recurse=False) - elif self._rrc is not None: - self.status(msg="Retrieving %(reponame)s %(sha1)s %(filename)s" - " from the remote git cache.", - reponame=reponame, sha1=sha1, filename=filename, - chatty=True) - try: - text = self._rrc.cat_file(reponame, sha1, filename) - morph = loader.load_from_string(text) - except morphlib.remoterepocache.CatFileError: - morph = None - file_list = self._rrc.ls_tree(reponame, sha1) - else: - raise NotcachedError(reponame) - - if morph is None: - self.status(msg="File %s doesn't exist: attempting to infer " - "chunk morph from repo's build system" - % filename, chatty=True) - bs = morphlib.buildsystem.detect_build_system(file_list) - if bs is None: - raise MorphologyNotFoundError(filename) - morph = bs.get_morphology(morph_name) - loader.validate(morph) - loader.set_commands(morph) - loader.set_defaults(morph) - return morph diff --git a/morphlib/morphologyfactory_tests.py b/morphlib/morphologyfactory_tests.py deleted file mode 100644 index 5222ca6d..00000000 --- a/morphlib/morphologyfactory_tests.py +++ /dev/null @@ -1,281 +0,0 @@ -# Copyright (C) 2012-2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -import unittest - -import morphlib -from morphlib.morphologyfactory import (MorphologyFactory, - MorphologyNotFoundError, - NotcachedError) -from morphlib.remoterepocache import CatFileError - - -class FakeRemoteRepoCache(object): - - def cat_file(self, reponame, sha1, filename): - if filename.endswith('.morph'): - return '''{ - "name": "%s", - "kind": "chunk", - "build-system": "dummy" - }''' % filename[:-len('.morph')] - return 'text' - - def ls_tree(self, reponame, sha1): - return [] - - -class FakeLocalRepo(object): - - morphologies = { - 'chunk.morph': ''' - name: chunk - kind: chunk - build-system: dummy - ''', - 'chunk-split.morph': ''' - name: chunk-split - kind: chunk - build-system: dummy - products: - - artifact: chunk-split-runtime - include: [] - - artifact: chunk-split-devel - include: [] - ''', - 'stratum.morph': ''' - name: stratum - kind: stratum - chunks: - - name: chunk - repo: test:repo - ref: sha1 - build-mode: bootstrap - build-depends: [] - ''', - 'stratum-no-chunk-bdeps.morph': ''' - name: stratum-no-chunk-bdeps - kind: stratum - chunks: - - name: chunk - repo: test:repo - ref: sha1 - build-mode: bootstrap - ''', - 'stratum-no-bdeps-no-bootstrap.morph': ''' - name: stratum-no-bdeps-no-bootstrap - kind: stratum - chunks: - - name: chunk - repo: test:repo - ref: sha1 - build-depends: [] - ''', - 'stratum-bdeps-no-bootstrap.morph': ''' - name: stratum-bdeps-no-bootstrap - kind: stratum - build-depends: - - morph: stratum - chunks: - - name: chunk - repo: test:repo - ref: sha1 - build-depends: [] - ''', - 'stratum-empty.morph': ''' - name: stratum-empty - kind: stratum - ''', - 'system.morph': ''' - name: system - kind: system - arch: %(arch)s - strata: - - morph: stratum - ''', - 'parse-error.morph': ''' name''', - 'name-mismatch.morph': ''' - name: fred - kind: stratum - ''', - } - - def __init__(self): - self.arch = 'x86_64' - - def read_file(self, filename, ref): - if filename in self.morphologies: - values = { - 'arch': self.arch, - } - return self.morphologies[filename] % values - elif filename.endswith('.morph'): - return '''{ - "name": "%s", - "kind": "chunk", - "build-system": "dummy" - }''' % filename[:-len('.morph')] - return 'text' - - def list_files(self, ref, recurse): - return self.morphologies.keys() - - -class FakeLocalRepoCache(object): - - def __init__(self, lr): - self.lr = lr - - def has_repo(self, reponame): - return True - - def get_repo(self, reponame): - return self.lr - - -class MorphologyFactoryTests(unittest.TestCase): - - def setUp(self): - self.lr = FakeLocalRepo() - self.lrc = FakeLocalRepoCache(self.lr) - self.rrc = FakeRemoteRepoCache() - self.mf = MorphologyFactory(self.lrc, self.rrc) - self.lmf = MorphologyFactory(self.lrc, None) - - def nolocalfile(self, *args): - raise IOError('File not found') - - def noremotefile(self, *args): - raise CatFileError('reponame', 'ref', 'filename') - - def localmorph(self, *args): - return ['chunk.morph'] - - def nolocalmorph(self, *args): - if args[0].endswith('.morph'): - raise IOError('File not found') - return 'text' - - def autotoolsbuildsystem(self, *args, **kwargs): - return ['configure.in'] - - def remotemorph(self, *args, **kwargs): - return ['remote-chunk.morph'] - - def noremotemorph(self, *args): - if args[-1].endswith('.morph'): - raise CatFileError('reponame', 'ref', 'filename') - return 'text' - - def doesnothaverepo(self, reponame): - return False - - def test_gets_morph_from_local_repo(self): - self.lr.list_files = self.localmorph - morph = self.mf.get_morphology('reponame', 'sha1', - 'chunk.morph') - self.assertEqual('chunk', morph['name']) - - def test_gets_morph_from_remote_repo(self): - self.rrc.ls_tree = self.remotemorph - self.lrc.has_repo = self.doesnothaverepo - morph = self.mf.get_morphology('reponame', 'sha1', - 'remote-chunk.morph') - self.assertEqual('remote-chunk', morph['name']) - - def test_autodetects_local_morphology(self): - self.lr.read_file = self.nolocalmorph - self.lr.list_files = self.autotoolsbuildsystem - morph = self.mf.get_morphology('reponame', 'sha1', - 'assumed-local.morph') - self.assertEqual('assumed-local', morph['name']) - - def test_autodetects_remote_morphology(self): - self.lrc.has_repo = self.doesnothaverepo - self.rrc.cat_file = self.noremotemorph - self.rrc.ls_tree = self.autotoolsbuildsystem - morph = self.mf.get_morphology('reponame', 'sha1', - 'assumed-remote.morph') - self.assertEqual('assumed-remote', morph['name']) - - def test_raises_error_when_no_local_morph(self): - self.lr.read_file = self.nolocalfile - self.assertRaises(MorphologyNotFoundError, self.mf.get_morphology, - 'reponame', 'sha1', 'unreached.morph') - - def test_raises_error_when_fails_no_remote_morph(self): - self.lrc.has_repo = self.doesnothaverepo - self.rrc.cat_file = self.noremotefile - self.assertRaises(MorphologyNotFoundError, self.mf.get_morphology, - 'reponame', 'sha1', 'unreached.morph') - - def test_raises_error_when_name_mismatches(self): - self.assertRaises(morphlib.Error, self.mf.get_morphology, - 'reponame', 'sha1', 'name-mismatch.morph') - - def test_looks_locally_with_no_remote(self): - self.lr.list_files = self.localmorph - morph = self.lmf.get_morphology('reponame', 'sha1', - 'chunk.morph') - self.assertEqual('chunk', morph['name']) - - def test_autodetects_locally_with_no_remote(self): - self.lr.read_file = self.nolocalmorph - self.lr.list_files = self.autotoolsbuildsystem - morph = self.mf.get_morphology('reponame', 'sha1', - 'assumed-local.morph') - self.assertEqual('assumed-local', morph['name']) - - def test_fails_when_local_not_cached_and_no_remote(self): - self.lrc.has_repo = self.doesnothaverepo - self.assertRaises(NotcachedError, self.lmf.get_morphology, - 'reponame', 'sha1', 'unreached.morph') - - def test_arch_is_validated(self): - self.lr.arch = 'unknown' - self.assertRaises(morphlib.Error, self.mf.get_morphology, - 'reponame', 'sha1', 'system.morph') - - def test_arch_arm_defaults_to_le(self): - self.lr.arch = 'armv7' - morph = self.mf.get_morphology('reponame', 'sha1', 'system.morph') - self.assertEqual(morph['arch'], 'armv7l') - - def test_fails_on_parse_error(self): - self.assertRaises(morphlib.Error, self.mf.get_morphology, - 'reponame', 'sha1', 'parse-error.morph') - - def test_fails_on_no_chunk_bdeps(self): - self.assertRaises(morphlib.morphloader.NoBuildDependenciesError, - self.mf.get_morphology, 'reponame', 'sha1', - 'stratum-no-chunk-bdeps.morph') - - def test_fails_on_no_bdeps_or_bootstrap(self): - self.assertRaises( - morphlib.morphloader.NoStratumBuildDependenciesError, - self.mf.get_morphology, 'reponame', 'sha1', - 'stratum-no-bdeps-no-bootstrap.morph') - - def test_succeeds_on_bdeps_no_bootstrap(self): - self.mf.get_morphology( - 'reponame', 'sha1', - 'stratum-bdeps-no-bootstrap.morph') - - def test_fails_on_empty_stratum(self): - self.assertRaises( - morphlib.morphloader.EmptyStratumError, - self.mf.get_morphology, 'reponame', 'sha1', 'stratum-empty.morph') - diff --git a/morphlib/sourceresolver.py b/morphlib/sourceresolver.py index 4bc55bf2..d9ffd049 100644 --- a/morphlib/sourceresolver.py +++ b/morphlib/sourceresolver.py @@ -22,6 +22,23 @@ import logging import morphlib +class SourceResolverError(cliapp.AppException): + pass + + +class MorphologyNotFoundError(SourceResolverError): + def __init__(self, filename): + SourceResolverError.__init__( + self, "Couldn't find morphology: %s" % filename) + + +class NotcachedError(SourceResolverError): + def __init__(self, repo_name): + SourceResolverError.__init__( + self, "Repository %s is not cached locally and there is no " + "remote cache specified" % repo_name) + + class SourceResolver(object): '''Provides a way of resolving the set of sources for a given system. @@ -55,6 +72,8 @@ class SourceResolver(object): self.status = status_cb + self._resolved_morphologies = {} + def resolve_ref(self, reponame, ref): '''Resolves commit and tree sha1s of the ref in a repo and returns it. @@ -96,12 +115,57 @@ class SourceResolver(object): tree = repo.resolve_ref_to_tree(absref) return absref, tree + def _get_morphology(self, reponame, sha1, filename): + '''Read the morphology at the specified location.''' + key = (reponame, sha1, filename) + if key in self._resolved_morphologies: + return self._resolved_morphologies[key] + + morph_name = os.path.splitext(os.path.basename(filename))[0] + loader = morphlib.morphloader.MorphologyLoader() + if self._lrc.has_repo(reponame): + self.status(msg="Looking for %s in local repo cache" % filename, + chatty=True) + try: + repo = self._lrc.get_repo(reponame) + text = repo.read_file(filename, sha1) + morph = loader.load_from_string(text) + except IOError: + morph = None + file_list = repo.list_files(ref=sha1, recurse=False) + elif self._rrc is not None: + self.status(msg="Retrieving %(reponame)s %(sha1)s %(filename)s" + " from the remote git cache.", + reponame=reponame, sha1=sha1, filename=filename, + chatty=True) + try: + text = self._rrc.cat_file(reponame, sha1, filename) + morph = loader.load_from_string(text) + except morphlib.remoterepocache.CatFileError: + morph = None + file_list = self._rrc.ls_tree(reponame, sha1) + else: + raise NotcachedError(reponame) + + if morph is None: + self.status(msg="File %s doesn't exist: attempting to infer " + "chunk morph from repo's build system" + % filename, chatty=True) + bs = morphlib.buildsystem.detect_build_system(file_list) + if bs is None: + raise MorphologyNotFoundError(filename) + morph = bs.get_morphology(morph_name) + loader.validate(morph) + loader.set_commands(morph) + loader.set_defaults(morph) + + self._resolved_morphologies[morph] = morph + return morph + def traverse_morphs(self, definitions_repo, definitions_ref, system_filenames, visit=lambda rn, rf, fn, arf, m: None, definitions_original_ref=None): - morph_factory = morphlib.morphologyfactory.MorphologyFactory( - self.lrc, self.rrc, self.status) definitions_queue = collections.deque(system_filenames) chunk_in_definitions_repo_queue = [] chunk_in_source_repo_queue = [] @@ -124,10 +188,8 @@ class SourceResolver(object): while definitions_queue: filename = definitions_queue.popleft() - key = (definitions_repo, definitions_absref, filename) - if not key in resolved_morphologies: - resolved_morphologies[key] = morph_factory.get_morphology(*key) - morphology = resolved_morphologies[key] + morphology = self._get_morphology( + definitions_repo, definitions_absref, filename) visit(definitions_repo, definitions_ref, filename, definitions_absref, definitions_tree, morphology) @@ -166,9 +228,7 @@ class SourceResolver(object): absref = resolved_commits[repo, ref] tree = resolved_trees[repo, absref] key = (definitions_repo, definitions_absref, filename) - if not key in resolved_morphologies: - resolved_morphologies[key] = morph_factory.get_morphology(*key) - morphology = resolved_morphologies[key] + morphology = self._get_morphology(*key) visit(repo, ref, filename, absref, tree, morphology) for repo, ref, filename in chunk_in_definitions_repo_queue: -- cgit v1.2.1