summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorSam Thursfield <sam.thursfield@codethink.co.uk>2014-11-25 21:40:06 +0000
committerAdam Coldrick <adam.coldrick@codethink.co.uk>2015-01-23 09:39:08 +0000
commit25b98cf567ba8faf1f717677cb1f9da2292c22dc (patch)
treedf02a05db32b19a5865bf50c5b7d798fb9defb47
parent18662acf24d1db465d9383559348d98749a33606 (diff)
downloadmorph-25b98cf567ba8faf1f717677cb1f9da2292c22dc.tar.gz
Move MorphologyFactory into SourceResolver
There's no need for this stuff to be in a separate class. This allows integrating it with the caching in the SourceResolver class.
-rw-r--r--morphlib/__init__.py1
-rw-r--r--morphlib/morphologyfactory.py90
-rw-r--r--morphlib/morphologyfactory_tests.py281
-rw-r--r--morphlib/sourceresolver.py78
4 files changed, 69 insertions, 381 deletions
diff --git a/morphlib/__init__.py b/morphlib/__init__.py
index a10ebe7b..d54340df 100644
--- a/morphlib/__init__.py
+++ b/morphlib/__init__.py
@@ -68,7 +68,6 @@ import gitindex
import localartifactcache
import localrepocache
import mountableimage
-import morphologyfactory
import morphologyfinder
import morphology
import morphloader
diff --git a/morphlib/morphologyfactory.py b/morphlib/morphologyfactory.py
deleted file mode 100644
index a3ac2749..00000000
--- a/morphlib/morphologyfactory.py
+++ /dev/null
@@ -1,90 +0,0 @@
-# Copyright (C) 2012-2014 Codethink Limited
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; version 2 of the License.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-
-import os
-
-import morphlib
-import cliapp
-
-
-class MorphologyFactoryError(cliapp.AppException):
- pass
-
-
-class MorphologyNotFoundError(MorphologyFactoryError):
- def __init__(self, filename):
- MorphologyFactoryError.__init__(
- self, "Couldn't find morphology: %s" % filename)
-
-
-class NotcachedError(MorphologyFactoryError):
- def __init__(self, repo_name):
- MorphologyFactoryError.__init__(
- self, "Repository %s is not cached locally and there is no "
- "remote cache specified" % repo_name)
-
-
-class MorphologyFactory(object):
-
- '''A way of creating morphologies which will provide a default'''
-
- def __init__(self, local_repo_cache, remote_repo_cache=None,
- status_cb=None):
- self._lrc = local_repo_cache
- self._rrc = remote_repo_cache
-
- null_status_function = lambda **kwargs: None
- self.status = status_cb or null_status_function
-
- def get_morphology(self, reponame, sha1, filename):
- morph_name = os.path.splitext(os.path.basename(filename))[0]
- loader = morphlib.morphloader.MorphologyLoader()
- if self._lrc.has_repo(reponame):
- self.status(msg="Looking for %s in local repo cache" % filename,
- chatty=True)
- try:
- repo = self._lrc.get_repo(reponame)
- text = repo.read_file(filename, sha1)
- morph = loader.load_from_string(text)
- except IOError:
- morph = None
- file_list = repo.list_files(ref=sha1, recurse=False)
- elif self._rrc is not None:
- self.status(msg="Retrieving %(reponame)s %(sha1)s %(filename)s"
- " from the remote git cache.",
- reponame=reponame, sha1=sha1, filename=filename,
- chatty=True)
- try:
- text = self._rrc.cat_file(reponame, sha1, filename)
- morph = loader.load_from_string(text)
- except morphlib.remoterepocache.CatFileError:
- morph = None
- file_list = self._rrc.ls_tree(reponame, sha1)
- else:
- raise NotcachedError(reponame)
-
- if morph is None:
- self.status(msg="File %s doesn't exist: attempting to infer "
- "chunk morph from repo's build system"
- % filename, chatty=True)
- bs = morphlib.buildsystem.detect_build_system(file_list)
- if bs is None:
- raise MorphologyNotFoundError(filename)
- morph = bs.get_morphology(morph_name)
- loader.validate(morph)
- loader.set_commands(morph)
- loader.set_defaults(morph)
- return morph
diff --git a/morphlib/morphologyfactory_tests.py b/morphlib/morphologyfactory_tests.py
deleted file mode 100644
index 5222ca6d..00000000
--- a/morphlib/morphologyfactory_tests.py
+++ /dev/null
@@ -1,281 +0,0 @@
-# Copyright (C) 2012-2014 Codethink Limited
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; version 2 of the License.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-
-import unittest
-
-import morphlib
-from morphlib.morphologyfactory import (MorphologyFactory,
- MorphologyNotFoundError,
- NotcachedError)
-from morphlib.remoterepocache import CatFileError
-
-
-class FakeRemoteRepoCache(object):
-
- def cat_file(self, reponame, sha1, filename):
- if filename.endswith('.morph'):
- return '''{
- "name": "%s",
- "kind": "chunk",
- "build-system": "dummy"
- }''' % filename[:-len('.morph')]
- return 'text'
-
- def ls_tree(self, reponame, sha1):
- return []
-
-
-class FakeLocalRepo(object):
-
- morphologies = {
- 'chunk.morph': '''
- name: chunk
- kind: chunk
- build-system: dummy
- ''',
- 'chunk-split.morph': '''
- name: chunk-split
- kind: chunk
- build-system: dummy
- products:
- - artifact: chunk-split-runtime
- include: []
- - artifact: chunk-split-devel
- include: []
- ''',
- 'stratum.morph': '''
- name: stratum
- kind: stratum
- chunks:
- - name: chunk
- repo: test:repo
- ref: sha1
- build-mode: bootstrap
- build-depends: []
- ''',
- 'stratum-no-chunk-bdeps.morph': '''
- name: stratum-no-chunk-bdeps
- kind: stratum
- chunks:
- - name: chunk
- repo: test:repo
- ref: sha1
- build-mode: bootstrap
- ''',
- 'stratum-no-bdeps-no-bootstrap.morph': '''
- name: stratum-no-bdeps-no-bootstrap
- kind: stratum
- chunks:
- - name: chunk
- repo: test:repo
- ref: sha1
- build-depends: []
- ''',
- 'stratum-bdeps-no-bootstrap.morph': '''
- name: stratum-bdeps-no-bootstrap
- kind: stratum
- build-depends:
- - morph: stratum
- chunks:
- - name: chunk
- repo: test:repo
- ref: sha1
- build-depends: []
- ''',
- 'stratum-empty.morph': '''
- name: stratum-empty
- kind: stratum
- ''',
- 'system.morph': '''
- name: system
- kind: system
- arch: %(arch)s
- strata:
- - morph: stratum
- ''',
- 'parse-error.morph': ''' name''',
- 'name-mismatch.morph': '''
- name: fred
- kind: stratum
- ''',
- }
-
- def __init__(self):
- self.arch = 'x86_64'
-
- def read_file(self, filename, ref):
- if filename in self.morphologies:
- values = {
- 'arch': self.arch,
- }
- return self.morphologies[filename] % values
- elif filename.endswith('.morph'):
- return '''{
- "name": "%s",
- "kind": "chunk",
- "build-system": "dummy"
- }''' % filename[:-len('.morph')]
- return 'text'
-
- def list_files(self, ref, recurse):
- return self.morphologies.keys()
-
-
-class FakeLocalRepoCache(object):
-
- def __init__(self, lr):
- self.lr = lr
-
- def has_repo(self, reponame):
- return True
-
- def get_repo(self, reponame):
- return self.lr
-
-
-class MorphologyFactoryTests(unittest.TestCase):
-
- def setUp(self):
- self.lr = FakeLocalRepo()
- self.lrc = FakeLocalRepoCache(self.lr)
- self.rrc = FakeRemoteRepoCache()
- self.mf = MorphologyFactory(self.lrc, self.rrc)
- self.lmf = MorphologyFactory(self.lrc, None)
-
- def nolocalfile(self, *args):
- raise IOError('File not found')
-
- def noremotefile(self, *args):
- raise CatFileError('reponame', 'ref', 'filename')
-
- def localmorph(self, *args):
- return ['chunk.morph']
-
- def nolocalmorph(self, *args):
- if args[0].endswith('.morph'):
- raise IOError('File not found')
- return 'text'
-
- def autotoolsbuildsystem(self, *args, **kwargs):
- return ['configure.in']
-
- def remotemorph(self, *args, **kwargs):
- return ['remote-chunk.morph']
-
- def noremotemorph(self, *args):
- if args[-1].endswith('.morph'):
- raise CatFileError('reponame', 'ref', 'filename')
- return 'text'
-
- def doesnothaverepo(self, reponame):
- return False
-
- def test_gets_morph_from_local_repo(self):
- self.lr.list_files = self.localmorph
- morph = self.mf.get_morphology('reponame', 'sha1',
- 'chunk.morph')
- self.assertEqual('chunk', morph['name'])
-
- def test_gets_morph_from_remote_repo(self):
- self.rrc.ls_tree = self.remotemorph
- self.lrc.has_repo = self.doesnothaverepo
- morph = self.mf.get_morphology('reponame', 'sha1',
- 'remote-chunk.morph')
- self.assertEqual('remote-chunk', morph['name'])
-
- def test_autodetects_local_morphology(self):
- self.lr.read_file = self.nolocalmorph
- self.lr.list_files = self.autotoolsbuildsystem
- morph = self.mf.get_morphology('reponame', 'sha1',
- 'assumed-local.morph')
- self.assertEqual('assumed-local', morph['name'])
-
- def test_autodetects_remote_morphology(self):
- self.lrc.has_repo = self.doesnothaverepo
- self.rrc.cat_file = self.noremotemorph
- self.rrc.ls_tree = self.autotoolsbuildsystem
- morph = self.mf.get_morphology('reponame', 'sha1',
- 'assumed-remote.morph')
- self.assertEqual('assumed-remote', morph['name'])
-
- def test_raises_error_when_no_local_morph(self):
- self.lr.read_file = self.nolocalfile
- self.assertRaises(MorphologyNotFoundError, self.mf.get_morphology,
- 'reponame', 'sha1', 'unreached.morph')
-
- def test_raises_error_when_fails_no_remote_morph(self):
- self.lrc.has_repo = self.doesnothaverepo
- self.rrc.cat_file = self.noremotefile
- self.assertRaises(MorphologyNotFoundError, self.mf.get_morphology,
- 'reponame', 'sha1', 'unreached.morph')
-
- def test_raises_error_when_name_mismatches(self):
- self.assertRaises(morphlib.Error, self.mf.get_morphology,
- 'reponame', 'sha1', 'name-mismatch.morph')
-
- def test_looks_locally_with_no_remote(self):
- self.lr.list_files = self.localmorph
- morph = self.lmf.get_morphology('reponame', 'sha1',
- 'chunk.morph')
- self.assertEqual('chunk', morph['name'])
-
- def test_autodetects_locally_with_no_remote(self):
- self.lr.read_file = self.nolocalmorph
- self.lr.list_files = self.autotoolsbuildsystem
- morph = self.mf.get_morphology('reponame', 'sha1',
- 'assumed-local.morph')
- self.assertEqual('assumed-local', morph['name'])
-
- def test_fails_when_local_not_cached_and_no_remote(self):
- self.lrc.has_repo = self.doesnothaverepo
- self.assertRaises(NotcachedError, self.lmf.get_morphology,
- 'reponame', 'sha1', 'unreached.morph')
-
- def test_arch_is_validated(self):
- self.lr.arch = 'unknown'
- self.assertRaises(morphlib.Error, self.mf.get_morphology,
- 'reponame', 'sha1', 'system.morph')
-
- def test_arch_arm_defaults_to_le(self):
- self.lr.arch = 'armv7'
- morph = self.mf.get_morphology('reponame', 'sha1', 'system.morph')
- self.assertEqual(morph['arch'], 'armv7l')
-
- def test_fails_on_parse_error(self):
- self.assertRaises(morphlib.Error, self.mf.get_morphology,
- 'reponame', 'sha1', 'parse-error.morph')
-
- def test_fails_on_no_chunk_bdeps(self):
- self.assertRaises(morphlib.morphloader.NoBuildDependenciesError,
- self.mf.get_morphology, 'reponame', 'sha1',
- 'stratum-no-chunk-bdeps.morph')
-
- def test_fails_on_no_bdeps_or_bootstrap(self):
- self.assertRaises(
- morphlib.morphloader.NoStratumBuildDependenciesError,
- self.mf.get_morphology, 'reponame', 'sha1',
- 'stratum-no-bdeps-no-bootstrap.morph')
-
- def test_succeeds_on_bdeps_no_bootstrap(self):
- self.mf.get_morphology(
- 'reponame', 'sha1',
- 'stratum-bdeps-no-bootstrap.morph')
-
- def test_fails_on_empty_stratum(self):
- self.assertRaises(
- morphlib.morphloader.EmptyStratumError,
- self.mf.get_morphology, 'reponame', 'sha1', 'stratum-empty.morph')
-
diff --git a/morphlib/sourceresolver.py b/morphlib/sourceresolver.py
index 4bc55bf2..d9ffd049 100644
--- a/morphlib/sourceresolver.py
+++ b/morphlib/sourceresolver.py
@@ -22,6 +22,23 @@ import logging
import morphlib
+class SourceResolverError(cliapp.AppException):
+ pass
+
+
+class MorphologyNotFoundError(SourceResolverError):
+ def __init__(self, filename):
+ SourceResolverError.__init__(
+ self, "Couldn't find morphology: %s" % filename)
+
+
+class NotcachedError(SourceResolverError):
+ def __init__(self, repo_name):
+ SourceResolverError.__init__(
+ self, "Repository %s is not cached locally and there is no "
+ "remote cache specified" % repo_name)
+
+
class SourceResolver(object):
'''Provides a way of resolving the set of sources for a given system.
@@ -55,6 +72,8 @@ class SourceResolver(object):
self.status = status_cb
+ self._resolved_morphologies = {}
+
def resolve_ref(self, reponame, ref):
'''Resolves commit and tree sha1s of the ref in a repo and returns it.
@@ -96,12 +115,57 @@ class SourceResolver(object):
tree = repo.resolve_ref_to_tree(absref)
return absref, tree
+ def _get_morphology(self, reponame, sha1, filename):
+ '''Read the morphology at the specified location.'''
+ key = (reponame, sha1, filename)
+ if key in self._resolved_morphologies:
+ return self._resolved_morphologies[key]
+
+ morph_name = os.path.splitext(os.path.basename(filename))[0]
+ loader = morphlib.morphloader.MorphologyLoader()
+ if self._lrc.has_repo(reponame):
+ self.status(msg="Looking for %s in local repo cache" % filename,
+ chatty=True)
+ try:
+ repo = self._lrc.get_repo(reponame)
+ text = repo.read_file(filename, sha1)
+ morph = loader.load_from_string(text)
+ except IOError:
+ morph = None
+ file_list = repo.list_files(ref=sha1, recurse=False)
+ elif self._rrc is not None:
+ self.status(msg="Retrieving %(reponame)s %(sha1)s %(filename)s"
+ " from the remote git cache.",
+ reponame=reponame, sha1=sha1, filename=filename,
+ chatty=True)
+ try:
+ text = self._rrc.cat_file(reponame, sha1, filename)
+ morph = loader.load_from_string(text)
+ except morphlib.remoterepocache.CatFileError:
+ morph = None
+ file_list = self._rrc.ls_tree(reponame, sha1)
+ else:
+ raise NotcachedError(reponame)
+
+ if morph is None:
+ self.status(msg="File %s doesn't exist: attempting to infer "
+ "chunk morph from repo's build system"
+ % filename, chatty=True)
+ bs = morphlib.buildsystem.detect_build_system(file_list)
+ if bs is None:
+ raise MorphologyNotFoundError(filename)
+ morph = bs.get_morphology(morph_name)
+ loader.validate(morph)
+ loader.set_commands(morph)
+ loader.set_defaults(morph)
+
+ self._resolved_morphologies[morph] = morph
+ return morph
+
def traverse_morphs(self, definitions_repo, definitions_ref,
system_filenames,
visit=lambda rn, rf, fn, arf, m: None,
definitions_original_ref=None):
- morph_factory = morphlib.morphologyfactory.MorphologyFactory(
- self.lrc, self.rrc, self.status)
definitions_queue = collections.deque(system_filenames)
chunk_in_definitions_repo_queue = []
chunk_in_source_repo_queue = []
@@ -124,10 +188,8 @@ class SourceResolver(object):
while definitions_queue:
filename = definitions_queue.popleft()
- key = (definitions_repo, definitions_absref, filename)
- if not key in resolved_morphologies:
- resolved_morphologies[key] = morph_factory.get_morphology(*key)
- morphology = resolved_morphologies[key]
+ morphology = self._get_morphology(
+ definitions_repo, definitions_absref, filename)
visit(definitions_repo, definitions_ref, filename,
definitions_absref, definitions_tree, morphology)
@@ -166,9 +228,7 @@ class SourceResolver(object):
absref = resolved_commits[repo, ref]
tree = resolved_trees[repo, absref]
key = (definitions_repo, definitions_absref, filename)
- if not key in resolved_morphologies:
- resolved_morphologies[key] = morph_factory.get_morphology(*key)
- morphology = resolved_morphologies[key]
+ morphology = self._get_morphology(*key)
visit(repo, ref, filename, absref, tree, morphology)
for repo, ref, filename in chunk_in_definitions_repo_queue: